diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index e3d01d1539..d7a332ca5c 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -767,3 +767,8 @@ jobs: go test -race -v -run '^(TestExplicitLocalRepositoryWindowsPathSemantics|TestListDirectory.*|TestDriveRootsFromMask|TestSplitAbsForRoot)$' ./internal/task/service + + # Pure GOOS-branch logic (no syscalls), but it must actually observe + # Windows to prove the unavailable branch, not just compile under it. + - name: Test Windows agent-survival flag availability + run: go test -race -v -run '^TestAgentSurvivalAvailabilityMatchesPlatformScope$' ./internal/runtimeflags diff --git a/apps/backend/cmd/agentctl/diagnostic_log.go b/apps/backend/cmd/agentctl/diagnostic_log.go new file mode 100644 index 0000000000..a6370edcc9 --- /dev/null +++ b/apps/backend/cmd/agentctl/diagnostic_log.go @@ -0,0 +1,54 @@ +package main + +import ( + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/common/logger" +) + +// F71 bound values for agentctl's own detached diagnostic log +// (AC-EXECUTORS-SURVIVAL-001.3): fixed implementation constants, not +// operator settings -- the sink adds no configuration key and no +// environment variable of its own. Chosen to mirror the backend's own +// logger's order of magnitude while staying a visibly distinct budget. +const ( + diagnosticLogMaxSizeMB = 16 + diagnosticLogMaxBackups = 10 + diagnosticLogMaxAgeDays = 14 +) + +// diagnosticLoggingConfig builds the file-backed, bounded logging +// configuration for agentctl's own diagnostic output while it may be +// running detached from any backend. Not yet wired into runMain's logger +// construction: switching away from stdout is a decision that depends on +// whether the agent-survival capability is engaged for this launch, which +// Layer 5.9 threads into agentctl's config (kill-path #6, "inherited +// stdout", is only a problem once that capability is active). This function +// exists so 5.9 has a ready, already-tested LoggingConfig to switch to +// instead of hand-assembling one at the call site. +func diagnosticLoggingConfig(level, format, diagnosticLogPath string) logger.LoggingConfig { + return logger.LoggingConfig{ + Level: level, + Format: format, + OutputPath: diagnosticLogPath, + MaxSizeMB: diagnosticLogMaxSizeMB, + MaxBackups: diagnosticLogMaxBackups, + MaxAgeDays: diagnosticLogMaxAgeDays, + } +} + +// resolveRunLoggingConfig is Layer 5.9's kill-path #6 gate ("inherited +// stdout", design 01's kill paths): stdout is only a liveness hazard once +// the agent-survival capability is actually engaged for this launch, so +// every other case -- capability disabled, or a diagnostic log path that +// failed to resolve for some reason -- keeps today's stdout logging +// unchanged rather than risk an unwritable sink. +func resolveRunLoggingConfig(cfg *config.Config) logger.LoggingConfig { + if cfg.AgentSurvivalEnabled && cfg.DiagnosticLogPath != "" { + return diagnosticLoggingConfig(cfg.LogLevel, cfg.LogFormat, cfg.DiagnosticLogPath) + } + return logger.LoggingConfig{ + Level: cfg.LogLevel, + Format: cfg.LogFormat, + OutputPath: "stdout", + } +} diff --git a/apps/backend/cmd/agentctl/diagnostic_log_test.go b/apps/backend/cmd/agentctl/diagnostic_log_test.go new file mode 100644 index 0000000000..53b9f61a2b --- /dev/null +++ b/apps/backend/cmd/agentctl/diagnostic_log_test.go @@ -0,0 +1,101 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/common/logger" +) + +// TestDiagnosticLoggingConfigAppliesTheF71Bounds pins the fixed bound values +// Build picked for AC-EXECUTORS-SURVIVAL-001.3 (MaxSizeMB=16, MaxBackups=10, +// MaxAgeDays=14) and that the level/format/path passed through are used +// verbatim, not silently overridden. +func TestDiagnosticLoggingConfigAppliesTheF71Bounds(t *testing.T) { + got := diagnosticLoggingConfig("debug", "json", "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log") + + want := logger.LoggingConfig{ + Level: "debug", + Format: "json", + OutputPath: "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", + MaxSizeMB: 16, + MaxBackups: 10, + MaxAgeDays: 14, + } + if got != want { + t.Fatalf("diagnosticLoggingConfig() = %+v, want %+v", got, want) + } +} + +// TestDiagnosticLoggingConfigWritesToADurableFile proves the wire contract +// end to end: constructing a real *logger.Logger from diagnosticLoggingConfig +// and writing a line actually lands in the named file, rather than a +// hand-fabricated fixture asserting only on the struct fields. +func TestDiagnosticLoggingConfigWritesToADurableFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "logs", "agentctl-diagnostic.log") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll: %v", err) + } + + log, err := logger.NewLogger(diagnosticLoggingConfig("info", "json", path)) + if err != nil { + t.Fatalf("NewLogger: %v", err) + } + log.Info("agentctl running detached") + if err := log.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + contents, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile(%q): %v", path, err) + } + if !strings.Contains(string(contents), "agentctl running detached") { + t.Fatalf("diagnostic log file contents = %q, want it to contain the logged message", contents) + } +} + +// TestResolveRunLoggingConfigStaysOnStdoutUnlessSurvivalIsEngaged pins the +// Layer 5.9 gate: kill-path #6 ("inherited stdout", see design 01's kill +// paths) is only a problem once the capability is actually active for this +// launch, so every other case -- disabled, or a diagnostic log path that +// somehow failed to resolve -- must keep today's stdout behavior byte-for- +// byte rather than silently switching sinks. +func TestResolveRunLoggingConfigStaysOnStdoutUnlessSurvivalIsEngaged(t *testing.T) { + cases := []struct { + name string + agentSurvivalEnabled bool + diagnosticLogPath string + wantStdout bool + }{ + {"disabled with a resolved path", false, "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", true}, + {"enabled but path unresolved", true, "", true}, + {"enabled with a resolved path", true, "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{ + LogLevel: "debug", + LogFormat: "json", + AgentSurvivalEnabled: tc.agentSurvivalEnabled, + DiagnosticLogPath: tc.diagnosticLogPath, + } + got := resolveRunLoggingConfig(cfg) + if tc.wantStdout { + want := logger.LoggingConfig{Level: "debug", Format: "json", OutputPath: "stdout"} + if got != want { + t.Fatalf("resolveRunLoggingConfig() = %+v, want %+v", got, want) + } + return + } + want := diagnosticLoggingConfig("debug", "json", tc.diagnosticLogPath) + if got != want { + t.Fatalf("resolveRunLoggingConfig() = %+v, want %+v", got, want) + } + }) + } +} diff --git a/apps/backend/cmd/agentctl/main.go b/apps/backend/cmd/agentctl/main.go index 10daca35d9..dad382ce76 100644 --- a/apps/backend/cmd/agentctl/main.go +++ b/apps/backend/cmd/agentctl/main.go @@ -109,12 +109,10 @@ func runMain() int { cfg.Port = *portFlag } + applySIGPIPEDisposition(cfg, signal.Ignore) + // Initialize logger - log, err := logger.NewLogger(logger.LoggingConfig{ - Level: cfg.LogLevel, - Format: cfg.LogFormat, - OutputPath: "stdout", - }) + log, err := logger.NewLogger(resolveRunLoggingConfig(cfg)) if err != nil { fmt.Fprintf(os.Stderr, "failed to initialize logger: %v\n", err) return 1 @@ -206,6 +204,13 @@ func run(cfg *config.Config, log *logger.Logger) { // Create instance manager instMgr := instance.NewManager(cfg, log) + // Declared here and assigned below, before any instance can actually be + // created: the server factory closure reads controlServer.CredentialSource() + // lazily (at CreateInstance time, not at SetServerFactory time), so the + // forward reference is safe as long as controlServer is assigned before + // the first instance is created. + var controlServer *api.ControlServer + // Set the server factory to create API servers for each instance instMgr.SetServerFactory(func(instCfg *config.InstanceConfig, procMgr *process.Manager, instLog *logger.Logger) http.Handler { // Create MCP backend client for bidirectional communication through agent stream @@ -225,11 +230,15 @@ func run(cfg *config.Config, log *logger.Logger) { instLog.Info("MCP server enabled (channel-based)", zap.String("session_id", instCfg.SessionID)) - return api.NewServer(instCfg, procMgr, mcpSrv, mcpBackendClient, instLog).Router() + srv := api.NewServer(instCfg, procMgr, mcpSrv, mcpBackendClient, instLog) + srv.SetCredentialSource(controlServer.CredentialSource()) + return srv.Router() }) // Create control server - controlServer := api.NewControlServer(cfg, instMgr, log) + controlServer = api.NewControlServer(cfg, instMgr, log) + stopUnownedReaper := startUnownedReaperIfEnabled(cfg, controlServer) + defer stopUnownedReaper() // Start HTTP server. When no auth token is configured (auth disabled), // ListenHost binds to loopback only so the unauthenticated command/shell/ @@ -261,8 +270,9 @@ func run(cfg *config.Config, log *logger.Logger) { // Returns a channel that closes when the parent dies. parentDied := monitorParentLiveness(log) - // Wait for shutdown signal (OS signal or parent death) - waitForShutdown(log, parentDied, func(ctx context.Context) { + // Wait for shutdown signal (OS signal, parent death, or an authenticated + // ownership-shutdown operation from an adopting/owning backend). + waitForShutdown(log, parentDied, controlServer.ShutdownRequested(), func(ctx context.Context) { // Flush pending traces before stopping instances if err := shared.ShutdownTracing(ctx); err != nil { log.Error("error shutting down tracing", zap.Error(err)) @@ -280,23 +290,24 @@ func run(cfg *config.Config, log *logger.Logger) { }) } -// waitForShutdown waits for a shutdown trigger (OS signal or parent death) and -// runs the cleanup function. parentDied may be nil when no parent monitor is active. -func waitForShutdown(log *logger.Logger, parentDied <-chan struct{}, cleanup func(ctx context.Context)) { +// waitForShutdown waits for a shutdown trigger (OS signal, parent death, or +// an ownership-shutdown operation) and runs the cleanup function. parentDied +// may be nil when no parent monitor is active. ownershipShutdown may be nil +// in tests that don't exercise a control server. +func waitForShutdown(log *logger.Logger, parentDied <-chan struct{}, ownershipShutdown <-chan struct{}, cleanup func(ctx context.Context)) { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) - if parentDied == nil { - // No parent monitor — wait for OS signal only. - sig := <-sigCh + // A nil channel (no parent monitor, no control server) blocks forever in + // a select, so that case is simply never chosen -- no separate branching + // needed for the "not active" cases. + select { + case sig := <-sigCh: log.Info("received signal", zap.String("signal", sig.String())) - } else { - select { - case sig := <-sigCh: - log.Info("received signal", zap.String("signal", sig.String())) - case <-parentDied: - log.Warn("parent process died, initiating shutdown") - } + case <-parentDied: + log.Warn("parent process died, initiating shutdown") + case <-ownershipShutdown: + log.Warn("ownership-shutdown operation invoked, initiating shutdown") } log.Info("shutting down agentctl...") diff --git a/apps/backend/cmd/agentctl/sigpipe.go b/apps/backend/cmd/agentctl/sigpipe.go new file mode 100644 index 0000000000..cec41bcb48 --- /dev/null +++ b/apps/backend/cmd/agentctl/sigpipe.go @@ -0,0 +1,36 @@ +package main + +import ( + "os" + "syscall" + + "github.com/kandev/kandev/internal/agentctl/server/config" +) + +// applySIGPIPEDisposition decides whether this process keeps running after +// the backend that spawned it goes away. +// +// The launcher always pipes this process's stdout/stderr back to the +// spawning backend for logging (internal/agent/runtime/agentctl/launcher's +// pipeOutput). When that backend exits, the kernel closes its end of those +// pipes, and Go's default SIGPIPE disposition kills this process the next +// time it writes to fd 1 or 2, which every subsequent log line does. An +// inherited pipe is therefore one of the ways a departing backend takes its +// agentctl with it. +// +// Which behavior is correct depends on the capability. With agent survival +// enabled the process is meant to outlive a graceful restart, so the write +// must degrade to an ordinary error, which the logger already tolerates. +// With the capability disabled nothing may outlive the backend, so this kill +// path stays intact alongside the others. +// +// Called after configuration is resolved: a process that dies before then +// dies on the default disposition, and the short-lived CLI and +// credential-helper invocations dispatched before configuration keep the +// ordinary disposition a command-line tool should have. +func applySIGPIPEDisposition(cfg *config.Config, ignore func(...os.Signal)) { + if !cfg.AgentSurvivalEnabled { + return + } + ignore(syscall.SIGPIPE) +} diff --git a/apps/backend/cmd/agentctl/sigpipe_test.go b/apps/backend/cmd/agentctl/sigpipe_test.go new file mode 100644 index 0000000000..20e35f4baf --- /dev/null +++ b/apps/backend/cmd/agentctl/sigpipe_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "os" + "syscall" + "testing" + + "github.com/kandev/kandev/internal/agentctl/server/config" +) + +// TestSIGPIPEDispositionFollowsTheSurvivalCapability pins that ignoring +// SIGPIPE is part of the survival capability rather than an unconditional +// property of the binary. With the capability off, an inherited stdout whose +// reader has gone away must still be able to end this process, because that +// is one of the paths by which a departing backend takes its agentctl with +// it; removing it for every launch would leave a detached agentctl behind on +// installations that never enabled survival at all. +func TestSIGPIPEDispositionFollowsTheSurvivalCapability(t *testing.T) { + t.Run("disabled leaves the default disposition intact", func(t *testing.T) { + var ignored []os.Signal + applySIGPIPEDisposition(&config.Config{AgentSurvivalEnabled: false}, func(s ...os.Signal) { + ignored = append(ignored, s...) + }) + if len(ignored) != 0 { + t.Fatalf("signals ignored = %v, want none with the capability disabled", ignored) + } + }) + + t.Run("enabled ignores SIGPIPE", func(t *testing.T) { + var ignored []os.Signal + applySIGPIPEDisposition(&config.Config{AgentSurvivalEnabled: true}, func(s ...os.Signal) { + ignored = append(ignored, s...) + }) + if len(ignored) != 1 || ignored[0] != syscall.SIGPIPE { + t.Fatalf("signals ignored = %v, want exactly [SIGPIPE]", ignored) + } + }) +} diff --git a/apps/backend/cmd/agentctl/unowned_reaper_gate.go b/apps/backend/cmd/agentctl/unowned_reaper_gate.go new file mode 100644 index 0000000000..9dde682b2a --- /dev/null +++ b/apps/backend/cmd/agentctl/unowned_reaper_gate.go @@ -0,0 +1,24 @@ +package main + +import "github.com/kandev/kandev/internal/agentctl/server/config" + +// unownedReaper is satisfied by *api.ControlServer, narrowed here so the +// gate decision below is testable without a real control server. +type unownedReaper interface { + StartUnownedReaper() + StopUnownedReaper() +} + +// startUnownedReaperIfEnabled starts the unowned-period reaper only when the +// agent-survival capability is engaged for this launch, per +// StartUnownedReaper's own caller contract: starting it unconditionally +// would self-terminate a healthy attached instance once the period elapses, +// since nothing renews ownership without that capability's backend-side +// claim loop. StopUnownedReaper is always safe to call even when Start was +// never called, so the returned stop func needs no second branch. +func startUnownedReaperIfEnabled(cfg *config.Config, reaper unownedReaper) (stop func()) { + if cfg.AgentSurvivalEnabled { + reaper.StartUnownedReaper() + } + return reaper.StopUnownedReaper +} diff --git a/apps/backend/cmd/agentctl/unowned_reaper_gate_test.go b/apps/backend/cmd/agentctl/unowned_reaper_gate_test.go new file mode 100644 index 0000000000..c19189ac1d --- /dev/null +++ b/apps/backend/cmd/agentctl/unowned_reaper_gate_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "testing" + + "github.com/kandev/kandev/internal/agentctl/server/config" +) + +// fakeUnownedReaper is a test double for unownedReaper, letting the gate +// decision be asserted without exercising the real reaper goroutine's +// lifecycle (already covered directly by the api package's own tests). +type fakeUnownedReaper struct { + started bool + stopped bool +} + +func (f *fakeUnownedReaper) StartUnownedReaper() { f.started = true } +func (f *fakeUnownedReaper) StopUnownedReaper() { f.stopped = true } + +// TestStartUnownedReaperIfEnabled pins Layer 5.9's gate on +// StartUnownedReaper's own caller contract: unconditionally starting it on +// every agentctl launch would self-terminate a healthy attached instance +// once the period elapses, since nothing renews ownership without the +// agent-survival capability and its backend-side claim loop. +func TestStartUnownedReaperIfEnabled(t *testing.T) { + t.Run("enabled starts the reaper", func(t *testing.T) { + reaper := &fakeUnownedReaper{} + stop := startUnownedReaperIfEnabled(&config.Config{AgentSurvivalEnabled: true}, reaper) + if !reaper.started { + t.Fatal("StartUnownedReaper was not called with the capability enabled") + } + stop() + if !reaper.stopped { + t.Fatal("the returned stop function did not call StopUnownedReaper") + } + }) + + t.Run("disabled never starts the reaper but stop stays safe", func(t *testing.T) { + reaper := &fakeUnownedReaper{} + stop := startUnownedReaperIfEnabled(&config.Config{AgentSurvivalEnabled: false}, reaper) + if reaper.started { + t.Fatal("StartUnownedReaper was called with the capability disabled") + } + stop() + if !reaper.stopped { + t.Fatal("the returned stop function did not call StopUnownedReaper even when disabled") + } + }) +} diff --git a/apps/backend/internal/agent/runtime/agentctl/control.go b/apps/backend/internal/agent/runtime/agentctl/control.go index 1a9d8e18db..b3c8f6960b 100644 --- a/apps/backend/internal/agent/runtime/agentctl/control.go +++ b/apps/backend/internal/agent/runtime/agentctl/control.go @@ -6,11 +6,13 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "net/http" "sync" "time" + "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/common/subproc" mcpprofile "github.com/kandev/kandev/internal/mcp/profile" @@ -107,6 +109,18 @@ type InstanceInfo struct { AgentCommand string `json:"agent_command"` Env map[string]string `json:"env,omitempty"` CreatedAt time.Time `json:"created_at"` + // SessionID is the task session ID this instance was created for, if any. + SessionID string `json:"session_id,omitempty"` + // TaskID is the task ID this instance was created for, if any. + TaskID string `json:"task_id,omitempty"` + // WorkspaceSourceRoots is the instance's live, current source-root + // allowlist, read back rather than pushed (AC-EXECUTORS-SURVIVAL-002.14's + // "workspace source roots" reconstruction row). + WorkspaceSourceRoots []string `json:"workspace_source_roots,omitempty"` + // ProviderSessionID is the live agent CLI's own session identity, read + // back from the adopted instance (AC-EXECUTORS-SURVIVAL-002.14's + // "provider session identity" reconstruction row). + ProviderSessionID string `json:"provider_session_id,omitempty"` } // ControlClientOption configures optional ControlClient settings. @@ -350,6 +364,245 @@ func (c *ControlClient) GetInstance(ctx context.Context, instanceID string) (*In return &info, nil } +// IdentityInfo is the response from GET /identity: installation identity and +// the capability set this control server advertises. +type IdentityInfo struct { + ServerIdentity string `json:"server_identity"` + Capabilities []string `json:"capabilities"` + // UnownedPeriodMS is the reporting server's own resolved unowned period + // in milliseconds (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2/.7) -- the value + // that server itself enforces, which an adopting backend's local config + // can disagree with across a restart. Use this, not local config, to + // compute an adopted server's ownership-renewal cadence. + UnownedPeriodMS int64 `json:"unowned_period_ms"` +} + +// ServerDetails carries the control-server values an adopting backend +// records but that are withheld from the unauthenticated identity endpoint. +type ServerDetails struct { + HomeDir string `json:"home_dir"` + DiagnosticLogPath string `json:"diagnostic_log_path"` +} + +// GetIdentity fetches the control server's identity and capability set. It +// deliberately does not require a valid auth token to succeed server-side +// (see agentctl's identity handler): identity retrieval decides adoption +// compatibility, so it cannot itself be gated behind the answer. +func (c *ControlClient) GetIdentity(ctx context.Context) (*IdentityInfo, error) { + req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/identity", nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get identity: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get identity: status %d", resp.StatusCode) + } + + var info IdentityInfo + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &info, nil +} + +// ErrOwnershipCredentialSuperseded indicates the control server rejected an +// ownership-claim request because the presented credential is no longer +// current (HTTP 401) -- distinct from a transient failure (network error, +// 5xx). This is not the same backend's credential being stale from a +// renewal it initiated: a rotation this backend performs always leaves it +// holding the replacement, so a rejection here means some other party +// rotated the credential out from under it, most likely a second backend +// that adopted the server. Callers must not retry on this error +// (AC-EXECUTORS-CONTROL-OWNERSHIP-002.3). +var ErrOwnershipCredentialSuperseded = errors.New("ownership claim rejected: credential superseded") + +// ClaimOwnership establishes or renews this backend's ownership of the +// control server. It carries no instance identity: a server with zero +// instances is still owned. Refused (non-nil error) once the server's +// unowned-shutdown one-way door has fired. +func (c *ControlClient) ClaimOwnership(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/api/v1/ownership/claim", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to claim ownership: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusUnauthorized { + return ErrOwnershipCredentialSuperseded + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to claim ownership: status %d", resp.StatusCode) + } + return nil +} + +// CredentialRotationResult is the response from the rotate half of the +// two-phase credential rotation: a rotation identifier from the control +// server's strictly increasing sequence, and the replacement credential to +// authenticate with from now on. +type CredentialRotationResult struct { + RotationID int64 `json:"rotation_id"` + Credential string `json:"credential"` +} + +// ProveOwnership asks the control server to demonstrate it already holds +// the credential this backend has stored for it, over a challenge generated +// for this attempt. It carries no auth token, because it is what +// establishes that the server is worth sending one to: the answer is a +// keyed digest the server can only produce from the credential, and it +// never carries the credential itself. +func (c *ControlClient) ProveOwnership(ctx context.Context, challenge string) ([]string, error) { + payload, err := json.Marshal(map[string]string{"challenge": challenge}) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/ownership/prove", bytes.NewReader(payload)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to request ownership proof: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to request ownership proof: status %d", resp.StatusCode) + } + + var decoded struct { + Proofs []string `json:"proofs"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return decoded.Proofs, nil +} + +// GetServerDetails reads the control-server values withheld from the +// unauthenticated identity endpoint. Requires a valid credential. +func (c *ControlClient) GetServerDetails(ctx context.Context) (*ServerDetails, error) { + req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/v1/ownership/details", nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get server details: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get server details: status %d", resp.StatusCode) + } + + var details ServerDetails + if err := json.NewDecoder(resp.Body).Decode(&details); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &details, nil +} + +// RotateCredential presents this client's current auth token (via the +// normal Authorization header) and asks the control server to replace it. +// The response's replacement credential authenticates every operation +// immediately; the presented (now superseded) credential remains valid for +// a further rotation attempt or the ownership-shutdown operation only, +// until ConfirmCredentialRotation is called. Idempotent under retry: a +// retry presenting the same (superseded) credential while unconfirmed +// returns the same rotation again rather than allocating a new one. +func (c *ControlClient) RotateCredential(ctx context.Context) (*CredentialRotationResult, error) { + req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/api/v1/ownership/rotate", nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to rotate credential: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to rotate credential: status %d", resp.StatusCode) + } + + var result CredentialRotationResult + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &result, nil +} + +// ConfirmCredentialRotation names rotationID as durably stored (both the +// replacement credential and the control-server record's reference to it). +// Call this only after both writes have durably completed; sending it +// earlier and then failing can strand the server unadoptable until its +// unowned period elapses (AC-EXECUTORS-CONTROL-OWNERSHIP-002.7). +func (c *ControlClient) ConfirmCredentialRotation(ctx context.Context, rotationID int64) error { + body, err := json.Marshal(struct { + RotationID int64 `json:"rotation_id"` + }{RotationID: rotationID}) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/api/v1/ownership/confirm", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to confirm credential rotation: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to confirm credential rotation: status %d", resp.StatusCode) + } + return nil +} + +// ShutdownControlServer invokes the ownership-shutdown operation: the +// control server stops every instance it supervises, together with their +// agent subprocesses, and exits. It requires no prior adoption or rotation +// and is authenticated by any credential in the acceptable set, including a +// superseded one that has not yet been confirmed away. +func (c *ControlClient) ShutdownControlServer(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/api/v1/ownership/shutdown", nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to shut down control server: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to shut down control server: status %d", resp.StatusCode) + } + return nil +} + // ListInstances lists all running agent instances. func (c *ControlClient) ListInstances(ctx context.Context) ([]*InstanceInfo, error) { req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/v1/instances", nil) @@ -375,3 +628,79 @@ func (c *ControlClient) ListInstances(ctx context.Context) ([]*InstanceInfo, err } return result.Instances, nil } + +// TurnOutcome is a retained terminal turn outcome for one instance +// (AC-EXECUTORS-SURVIVAL-004.1): the turn identifier the control server +// assigned, and the terminal event itself. +type TurnOutcome struct { + TurnID int64 `json:"turn_id"` + Event streams.AgentEvent `json:"event"` +} + +// GetTurnOutcome retrieves the named instance's retained last terminal turn +// outcome, if any. Returns (nil, nil) when the instance exists but nothing +// is retained -- the AC-EXECUTORS-SURVIVAL-004.5 case that publishes the +// session as running. The read is repeatable: it never discards what it +// returns (AC-EXECUTORS-SURVIVAL-004.6); only AckTurnOutcome does. +func (c *ControlClient) GetTurnOutcome(ctx context.Context, instanceID string) (*TurnOutcome, error) { + req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/v1/instances/"+instanceID+"/turn-outcome", nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to get turn outcome: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("instance %q not found", instanceID) + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get turn outcome: status %d", resp.StatusCode) + } + + var body struct { + Retained bool `json:"retained"` + TurnID int64 `json:"turn_id"` + Event streams.AgentEvent `json:"event"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + if !body.Retained { + return nil, nil + } + return &TurnOutcome{TurnID: body.TurnID, Event: body.Event}, nil +} + +// AckTurnOutcome acknowledges the named instance's retained outcome by turn +// identifier, discarding it. Naming an identifier the control server no +// longer holds, or never held, is accepted and changes nothing +// (AC-EXECUTORS-SURVIVAL-004.6), so a retried acknowledgement is always safe. +func (c *ControlClient) AckTurnOutcome(ctx context.Context, instanceID string, turnID int64) error { + body, err := json.Marshal(struct { + TurnID int64 `json:"turn_id"` + }{TurnID: turnID}) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, "POST", c.baseURL+"/api/v1/instances/"+instanceID+"/turn-outcome/ack", bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to ack turn outcome: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to ack turn outcome: status %d", resp.StatusCode) + } + return nil +} diff --git a/apps/backend/internal/agent/runtime/agentctl/control_test.go b/apps/backend/internal/agent/runtime/agentctl/control_test.go index 416cff6a04..77c58440b7 100644 --- a/apps/backend/internal/agent/runtime/agentctl/control_test.go +++ b/apps/backend/internal/agent/runtime/agentctl/control_test.go @@ -607,34 +607,11 @@ func TestGetInstance_FailureModes(t *testing.T) { } } -func TestListInstances_UnwrapsInstancesEnvelope(t *testing.T) { - srv, got := captureServer(t, jsonResponder(http.StatusOK, `{ - "instances":[ - {"id":"a","port":1,"status":"running","workspace_path":"/w/a","agent_command":"cmd-a"}, - {"id":"b","port":2,"status":"stopped","workspace_path":"/w/b","agent_command":"cmd-b"} - ] - }`)) - - instances, err := newTestControlClient(t, srv).ListInstances(context.Background()) - if err != nil { - t.Fatalf("ListInstances: %v", err) - } - - if got.Method != http.MethodGet || got.Path != "/api/v1/instances" { - t.Errorf("request = %s %s, want GET /api/v1/instances", got.Method, got.Path) - } - if len(instances) != 2 { - t.Fatalf("instances = %d, want 2", len(instances)) - } - if instances[0].ID != "a" || instances[0].Port != 1 || - instances[0].Status != "running" || instances[0].WorkspacePath != "/w/a" || - instances[0].AgentCommand != "cmd-a" { - t.Errorf("instances[0] = %+v", instances[0]) - } - if instances[1].ID != "b" || instances[1].Port != 2 || instances[1].Status != "stopped" { - t.Errorf("instances[1] = %+v", instances[1]) - } -} +// The envelope-decode + request method/path contract this used to pin with a +// hand-written response fixture is now covered end to end (real handler, real +// client) by TestListInstancesReturnsEnvelopeWithSessionAndTaskID in +// internal/agentctl/server/api — a fixture here could assert an envelope +// shape the real handler had stopped producing without ever failing. func TestListInstances_FailureModes(t *testing.T) { tests := []struct { @@ -661,3 +638,52 @@ func TestListInstances_FailureModes(t *testing.T) { }) } } + +// TestClaimOwnership_UnauthorizedReturnsSupersededSentinel pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.3: a 401 response must be distinguished +// from any other failure via ErrOwnershipCredentialSuperseded, so a caller +// like OwnershipRenewer can tell "credential is no longer current" apart +// from a transient failure it should retry. +func TestClaimOwnership_UnauthorizedReturnsSupersededSentinel(t *testing.T) { + srv := httptest.NewServer(jsonResponder(http.StatusUnauthorized, `{"error":"invalid auth token"}`)) + t.Cleanup(srv.Close) + + err := newTestControlClient(t, srv).ClaimOwnership(context.Background()) + if !errors.Is(err, ErrOwnershipCredentialSuperseded) { + t.Fatalf("error = %v, want errors.Is(err, ErrOwnershipCredentialSuperseded)", err) + } +} + +func TestClaimOwnership_OtherFailureModesDoNotReturnSupersededSentinel(t *testing.T) { + tests := []struct { + name string + status int + wantErr string + }{ + {"internal server error", http.StatusInternalServerError, "failed to claim ownership: status 500"}, + {"not found", http.StatusNotFound, "failed to claim ownership: status 404"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + srv := httptest.NewServer(jsonResponder(tc.status, `{}`)) + t.Cleanup(srv.Close) + + err := newTestControlClient(t, srv).ClaimOwnership(context.Background()) + if errors.Is(err, ErrOwnershipCredentialSuperseded) { + t.Fatalf("error = %v, want NOT errors.Is(err, ErrOwnershipCredentialSuperseded)", err) + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %v, want %q", err, tc.wantErr) + } + }) + } +} + +func TestClaimOwnership_SuccessReturnsNilError(t *testing.T) { + srv := httptest.NewServer(jsonResponder(http.StatusOK, `{}`)) + t.Cleanup(srv.Close) + + if err := newTestControlClient(t, srv).ClaimOwnership(context.Background()); err != nil { + t.Fatalf("ClaimOwnership() error = %v, want nil", err) + } +} diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go index 06400a12b9..a1eb072093 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher.go @@ -22,6 +22,7 @@ import ( agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" commonconfig "github.com/kandev/kandev/internal/common/config" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/common/netprobe" "go.uber.org/zap" ) @@ -141,6 +142,16 @@ func agentctlBinaryName() string { return "agentctl" } +// FindAgentctlBinary resolves the local agentctl binary path using the same +// search order New uses for an unspecified BinaryPath. Exported so a caller +// that adopts an already-running control server -- and therefore never +// calls New -- can still resolve the local binary artifact for callers that +// need it (e.g. uploading it into a remote/container executor), independent +// of whether this launch adopted or spawned. +func FindAgentctlBinary() string { + return findAgentctlBinary() +} + // findAgentctlBinary attempts to locate the agentctl binary. func findAgentctlBinary() string { name := agentctlBinaryName() @@ -245,11 +256,22 @@ func (l *Launcher) buildAndStartProcess(nonce string) error { overrides = append(overrides, commonconfig.InternalAgentctlStartupConfigEnv+"="+encoded) } l.cmd.Env = environmentWithOverrides(os.Environ(), overrides...) - l.cmd.SysProcAttr = buildSysProcAttr() + l.cmd.SysProcAttr = buildSysProcAttr(l.startupConfig.AgentSurvivalEnabled) - pipeWrite, err := setupLivenessPipe(l.cmd) - if err != nil { - return err + // Kill-path #1 (design 01's kill-paths list, "parent-liveness pipe"): + // skipped entirely when the capability is engaged for this launch, for + // the same AC-EXECUTORS-SURVIVAL-001.2 reason buildSysProcAttr above + // omits Pdeathsig -- both fire on any parent death, including SIGKILL, + // independent of anything Stop() does. + var pipeWrite *os.File + if l.startupConfig.AgentSurvivalEnabled { + clearInheritedLivenessPipeEnv(l.cmd) + } else { + var err error + pipeWrite, err = setupLivenessPipe(l.cmd) + if err != nil { + return err + } } stdout, err := l.cmd.StdoutPipe() @@ -322,8 +344,20 @@ func (l *Launcher) performHandshake(ctx context.Context, nonce string) (string, return token, nil } -// Stop gracefully shuts down the agentctl subprocess. +// Stop gracefully shuts down the agentctl subprocess. Kill-path #5 (design +// 01's kill-paths list, "registered cleanup"): when the agent-survival +// capability is engaged for this launch, Stop is the sole thing the +// backend's registered shutdown cleanup calls (see provider.go), and +// AC-EXECUTORS-SURVIVAL-001.1 requires that a graceful backend shutdown +// "shall not issue a stop" to the control server or its instances -- so +// Stop becomes a no-op rather than closing the (already-unarmed) pipe and +// sending SIGTERM. func (l *Launcher) Stop(ctx context.Context) error { + if l.startupConfig.AgentSurvivalEnabled { + l.logger.Info("agent-survival capability engaged; leaving agentctl running instead of stopping it") + return nil + } + l.mu.Lock() if l.cmd == nil || l.cmd.Process == nil { @@ -393,14 +427,20 @@ func (l *Launcher) closeParentPipeLocked() { } } -// checkPortAvailable verifies the given port is not in use. -// It checks by attempting a wildcard bind (matching what agentctl does with ":port"). +// checkPortAvailable reports whether the given port is free, using the same +// probe contract the process launcher uses (internal/common/netprobe): a +// dual-stack loopback connect must find nothing listening AND a fresh +// loopback bind must succeed. +// +// A bind alone is not enough. A surviving agentctl holds the wildcard +// address, and on macOS/BSD a bind against an active wildcard listener can +// still succeed, which would report the occupied control port as free and +// send this launch to a second server on a port the record does not name. func checkPortAvailable(port int) error { - ln, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) - if err != nil { - return err + if !netprobe.PortAvailable(port) { + return fmt.Errorf("port %d is already in use", port) } - return ln.Close() + return nil } // findFreePort asks the OS for an available port by binding to :0. diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_unix.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_unix.go index a536a2942f..ea5bdf8d31 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_unix.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_unix.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "os/exec" + "strings" ) // setupLivenessPipe creates a pipe and configures the command to pass the @@ -42,3 +43,22 @@ func closeChildPipeEnd(cmd *exec.Cmd) { _ = cmd.ExtraFiles[0].Close() } } + +// clearInheritedLivenessPipeEnv removes any KANDEV_PARENT_PIPE_FD the +// launcher process itself inherited from its own ambient environment (for +// example when it is itself running under a survivable agentctl launch). +// Used when the pipe is deliberately not armed for this launch (kill-path +// #1 disabled, AC-EXECUTORS-SURVIVAL-001.2): without this, a stale inherited +// value would point the child at an FD 3 that ExtraFiles never set up, +// which monitorParentLiveness would misinterpret rather than correctly +// treating as absent. +func clearInheritedLivenessPipeEnv(cmd *exec.Cmd) { + filtered := cmd.Env[:0] + for _, entry := range cmd.Env { + if strings.HasPrefix(entry, "KANDEV_PARENT_PIPE_FD=") { + continue + } + filtered = append(filtered, entry) + } + cmd.Env = filtered +} diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_windows.go b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_windows.go index 5da371c503..87a84504e4 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_windows.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/launcher_pipe_windows.go @@ -20,3 +20,7 @@ func closePipeOnStartFailure(_ *os.File, _ *exec.Cmd) {} // closeChildPipeEnd is a no-op on Windows (no pipe to close). func closeChildPipeEnd(_ *exec.Cmd) {} + +// clearInheritedLivenessPipeEnv is a no-op on Windows: setupLivenessPipe +// never sets KANDEV_PARENT_PIPE_FD here, so there is nothing to strip. +func clearInheritedLivenessPipeEnv(_ *exec.Cmd) {} diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/platform_unix_test.go b/apps/backend/internal/agent/runtime/agentctl/launcher/platform_unix_test.go index c78b0ee7c0..7cd81ea6e2 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/platform_unix_test.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/platform_unix_test.go @@ -49,7 +49,7 @@ func TestLauncherForceKillSignalsProcessGroup(t *testing.T) { childPIDFile := filepath.Join(tmp, "child.pid") cmd := exec.Command("sh", "-c", `sleep 30 & echo $! > "$CHILD_PID_FILE"; wait`) cmd.Env = append(os.Environ(), "CHILD_PID_FILE="+childPIDFile) - cmd.SysProcAttr = buildSysProcAttr() + cmd.SysProcAttr = buildSysProcAttr(false) if err := cmd.Start(); err != nil { t.Fatalf("start command: %v", err) } diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/survival_kill_path_test.go b/apps/backend/internal/agent/runtime/agentctl/launcher/survival_kill_path_test.go new file mode 100644 index 0000000000..3ede4e5f31 --- /dev/null +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/survival_kill_path_test.go @@ -0,0 +1,180 @@ +//go:build !windows + +package launcher + +import ( + "context" + "os/exec" + "strings" + "syscall" + "testing" + "time" + + commonconfig "github.com/kandev/kandev/internal/common/config" +) + +// TestBuildAndStartProcessSkipsParentLivenessPipeWhenSurvivalEnabled pins +// kill-path #1 from design 01's kill-paths list ("parent-liveness pipe"): +// when the agent-survival capability is engaged for this launch, agentctl +// must not depend on any backend shutdown step to keep running +// (AC-EXECUTORS-SURVIVAL-001.2), so the launcher must not arm the pipe that +// would otherwise make agentctl self-terminate the instant the write-end +// closes -- including on a backend SIGKILL, which never gives Stop() a +// chance to run. +func TestBuildAndStartProcessSkipsParentLivenessPipeWhenSurvivalEnabled(t *testing.T) { + l := newSurvivalKillPathTestLauncher(t, true) + if err := l.buildAndStartProcess("test-nonce"); err != nil { + t.Fatalf("buildAndStartProcess: %v", err) + } + t.Cleanup(func() { _ = l.cmd.Process.Kill() }) + <-l.exited // "true" exits immediately; wait so ExtraFiles/env are stable to inspect + + if len(l.cmd.ExtraFiles) != 0 { + t.Fatalf("ExtraFiles = %v, want none: the parent-liveness pipe must not be armed", l.cmd.ExtraFiles) + } + if l.parentPipe != nil { + t.Fatal("parentPipe was set even though the capability is engaged for this launch") + } + for _, env := range l.cmd.Env { + if strings.HasPrefix(env, "KANDEV_PARENT_PIPE_FD=") { + t.Fatalf("KANDEV_PARENT_PIPE_FD was set in the child environment: %s", env) + } + } +} + +// TestBuildAndStartProcessStripsInheritedLivenessPipeEnvWhenSurvivalEnabled +// covers a real hazard this session's own sandbox surfaced: the launcher +// process can itself be running under an ambient KANDEV_PARENT_PIPE_FD (for +// example a survivable agentctl launch nested inside another one). Without +// stripping it, that stale value would leak into the child's environment +// even though ExtraFiles was never set up to back FD 3, and +// monitorParentLiveness would misread it as a live pipe rather than absent. +func TestBuildAndStartProcessStripsInheritedLivenessPipeEnvWhenSurvivalEnabled(t *testing.T) { + t.Setenv("KANDEV_PARENT_PIPE_FD", "3") + + l := newSurvivalKillPathTestLauncher(t, true) + if err := l.buildAndStartProcess("test-nonce"); err != nil { + t.Fatalf("buildAndStartProcess: %v", err) + } + t.Cleanup(func() { _ = l.cmd.Process.Kill() }) + <-l.exited + + for _, env := range l.cmd.Env { + if strings.HasPrefix(env, "KANDEV_PARENT_PIPE_FD=") { + t.Fatalf("inherited KANDEV_PARENT_PIPE_FD leaked into the child environment: %s", env) + } + } +} + +// TestBuildAndStartProcessArmsParentLivenessPipeWhenSurvivalDisabled is the +// regression companion: today's behavior (capability off, or an unmanaged +// launch) must be unchanged -- the pipe is armed exactly as before. +func TestBuildAndStartProcessArmsParentLivenessPipeWhenSurvivalDisabled(t *testing.T) { + l := newSurvivalKillPathTestLauncher(t, false) + if err := l.buildAndStartProcess("test-nonce"); err != nil { + t.Fatalf("buildAndStartProcess: %v", err) + } + t.Cleanup(func() { _ = l.cmd.Process.Kill() }) + <-l.exited + + if len(l.cmd.ExtraFiles) != 1 { + t.Fatalf("ExtraFiles = %v, want exactly 1 (the pipe read-end)", l.cmd.ExtraFiles) + } + if l.parentPipe == nil { + t.Fatal("parentPipe was not set with the capability disabled") + } + _ = l.parentPipe.Close() +} + +// TestLauncherStopIsANoOpWhenSurvivalEnabled pins kill-path #5 ("registered +// cleanup"): AC-EXECUTORS-SURVIVAL-001.1 requires that a graceful backend +// shutdown "shall not issue a stop" to the control server or its instances +// when the capability is enabled. Stop() is the only thing the backend's +// registered cleanup calls (see provider.go), so making Stop() itself a +// no-op closes that path without needing a second "should I actually stop" +// branch at the one call site. +func TestLauncherStopIsANoOpWhenSurvivalEnabled(t *testing.T) { + cmd := exec.Command("sleep", "30") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatalf("start long-lived test process: %v", err) + } + t.Cleanup(func() { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }) + + l := &Launcher{ + cmd: cmd, + exited: make(chan struct{}), // never closed: nothing reaps this process in the test + logger: newLauncherTestLogger(t), + startupConfig: commonconfig.AgentctlStartupConfig{Configured: true, AgentSurvivalEnabled: true}, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := l.Stop(ctx); err != nil { + t.Fatalf("Stop() returned an error for a no-op stop: %v", err) + } + + if err := cmd.Process.Signal(syscall.Signal(0)); err != nil { + t.Fatalf("process is no longer alive after a supposedly no-op Stop(): %v", err) + } +} + +// TestLauncherStopTerminatesTheServerWhenSurvivalDisabled is the closed side +// of the same kill path. With the capability off, the control server is the +// backend's to own for the length of the backend's life, so the registered +// cleanup's Stop() must actually terminate it rather than leave an orphan +// running with no backend attached to it. +func TestLauncherStopTerminatesTheServerWhenSurvivalDisabled(t *testing.T) { + cmd := exec.Command("sleep", "30") + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + if err := cmd.Start(); err != nil { + t.Fatalf("start long-lived test process: %v", err) + } + t.Cleanup(func() { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }) + + exited := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(exited) + }() + + l := &Launcher{ + cmd: cmd, + exited: exited, + logger: newLauncherTestLogger(t), + startupConfig: commonconfig.AgentctlStartupConfig{Configured: true, AgentSurvivalEnabled: false}, + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := l.Stop(ctx); err != nil { + t.Fatalf("Stop() = %v, want nil", err) + } + + select { + case <-exited: + case <-time.After(5 * time.Second): + t.Fatal("process is still running after Stop() with the capability disabled") + } +} + +func newSurvivalKillPathTestLauncher(t *testing.T, survivalEnabled bool) *Launcher { + t.Helper() + binary, err := exec.LookPath("true") + if err != nil { + t.Skipf("no 'true' binary on PATH: %v", err) + } + return &Launcher{ + binaryPath: binary, + host: "localhost", + port: 0, + exited: make(chan struct{}), + logger: newLauncherTestLogger(t), + startupConfig: commonconfig.AgentctlStartupConfig{ + Configured: true, + IdleReaperInterval: time.Minute, + NotificationQueueCapacity: 4096, + AgentSurvivalEnabled: survivalEnabled, + }, + } +} diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_default.go b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_default.go index 16c9316fa6..9e5317072a 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_default.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_default.go @@ -4,7 +4,10 @@ package launcher import "syscall" -func buildSysProcAttr() *syscall.SysProcAttr { +// buildSysProcAttr configures the child's process attributes. survivalEnabled +// is unused on this platform: syscall.SysProcAttr here carries no Pdeathsig +// field (that kill path is Linux-only), so there is nothing to gate. +func buildSysProcAttr(_ bool) *syscall.SysProcAttr { return &syscall.SysProcAttr{ // Keep standalone agentctl out of the terminal foreground process // group so Ctrl+C is sequenced by the backend shutdown path. diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux.go b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux.go index b634567127..e8f87d7ef3 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux.go @@ -4,11 +4,19 @@ package launcher import "syscall" -func buildSysProcAttr() *syscall.SysProcAttr { - return &syscall.SysProcAttr{ - // Keep standalone agentctl out of the terminal foreground process - // group so Ctrl+C is sequenced by the backend shutdown path. - Setpgid: true, - Pdeathsig: syscall.SIGTERM, +// buildSysProcAttr configures the child's process attributes. Pdeathsig is +// kill-path #2 of design 01's kill-paths list: it fires from the kernel on +// ANY parent exit, including SIGKILL, so it must be omitted entirely when +// the agent-survival capability is engaged for this launch +// (AC-EXECUTORS-SURVIVAL-001.2 forbids depending on any backend shutdown +// step). Setpgid stays set regardless -- it isolates agentctl from terminal +// Ctrl+C, unrelated to survival. +func buildSysProcAttr(survivalEnabled bool) *syscall.SysProcAttr { + attr := &syscall.SysProcAttr{ + Setpgid: true, } + if !survivalEnabled { + attr.Pdeathsig = syscall.SIGTERM + } + return attr } diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux_test.go b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux_test.go new file mode 100644 index 0000000000..614aa2f3fc --- /dev/null +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_linux_test.go @@ -0,0 +1,33 @@ +//go:build linux + +package launcher + +import ( + "syscall" + "testing" +) + +// TestBuildSysProcAttrOmitsPdeathsigWhenSurvivalEnabled pins kill-path #2 +// from design 01's kill-paths list ("Pdeathsig"): AC-EXECUTORS-SURVIVAL-001.2 +// requires agentctl to keep running when the backend exits without a +// graceful shutdown, including SIGKILL, "without depending on any backend +// shutdown step" -- Pdeathsig fires from the kernel on ANY parent exit, so it +// must not be armed at all when the capability is engaged for this launch. +func TestBuildSysProcAttrOmitsPdeathsigWhenSurvivalEnabled(t *testing.T) { + attr := buildSysProcAttr(true) + if attr.Pdeathsig != 0 { + t.Fatalf("Pdeathsig = %v, want unset (0) when the capability is engaged", attr.Pdeathsig) + } + if !attr.Setpgid { + t.Error("Setpgid must stay true regardless of the capability: it isolates agentctl from terminal Ctrl+C, unrelated to survival") + } +} + +// TestBuildSysProcAttrSetsPdeathsigWhenSurvivalDisabled is the regression +// companion: today's behavior is unchanged when the capability is off. +func TestBuildSysProcAttrSetsPdeathsigWhenSurvivalDisabled(t *testing.T) { + attr := buildSysProcAttr(false) + if attr.Pdeathsig != syscall.SIGTERM { + t.Fatalf("Pdeathsig = %v, want SIGTERM when the capability is disabled", attr.Pdeathsig) + } +} diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_other.go b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_other.go index b0dee6b390..103eee33da 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_other.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_other.go @@ -4,6 +4,8 @@ package launcher import "syscall" -func buildSysProcAttr() *syscall.SysProcAttr { +// buildSysProcAttr configures the child's process attributes. survivalEnabled +// is unused on this platform: there is no parent-death primitive here to gate. +func buildSysProcAttr(_ bool) *syscall.SysProcAttr { return &syscall.SysProcAttr{} } diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_test.go b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_test.go index 990d5851b3..05ce1a8867 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_test.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_test.go @@ -7,7 +7,7 @@ import ( ) func TestBuildSysProcAttr_IsolatesAgentctlFromTerminalInterrupt(t *testing.T) { - attr := buildSysProcAttr() + attr := buildSysProcAttr(false) if !attr.Setpgid { t.Error("Setpgid must be true: standalone agentctl should not receive terminal Ctrl+C directly") } diff --git a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_windows.go b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_windows.go index 18f3c2ebf4..a9d51c51e4 100644 --- a/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_windows.go +++ b/apps/backend/internal/agent/runtime/agentctl/launcher/sysprocattr_windows.go @@ -4,7 +4,11 @@ package launcher import "syscall" -func buildSysProcAttr() *syscall.SysProcAttr { +// buildSysProcAttr configures the child's process attributes. survivalEnabled +// is unused on this platform: the capability is unavailable on Windows (see +// agentSurvivalAvailability in runtimeflags/registry.go), and the job-object +// kill path (design 01's kill-path #3) stays unconditional regardless. +func buildSysProcAttr(_ bool) *syscall.SysProcAttr { // CREATE_NEW_PROCESS_GROUP so Ctrl+C doesn't propagate directly return &syscall.SysProcAttr{ CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP, diff --git a/apps/backend/internal/agent/runtime/control_server.go b/apps/backend/internal/agent/runtime/control_server.go new file mode 100644 index 0000000000..ff89b13d99 --- /dev/null +++ b/apps/backend/internal/agent/runtime/control_server.go @@ -0,0 +1,118 @@ +package runtime + +import ( + "context" + "time" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/secrets" +) + +// Standalone control-server ownership, exposed through the agent-runtime seam. +// +// The adoption handshake, its durable record, and the ownership renewer all +// live in runtime/lifecycle, but the caller that drives them at startup is the +// backend application package, which must not import lifecycle directly (see +// apps/backend/CLAUDE.md, "only internal/agent/runtime/ ... may import +// runtime/lifecycle"). These aliases and wrappers are that seam, in the same +// shape as SSHTaskDirReclaimer above: higher-level packages depend on this +// contract rather than on the lifecycle implementation. + +// InheritedRecordScope reports what this launch found at the recorded control +// endpoint, which decides how an inherited executors_running row may be judged. +type InheritedRecordScope = lifecycle.InheritedRecordScope + +const ( + // InheritedRecordScopeNoServer means nothing answered, so no server holds + // the inherited rows and the process-identifier probe applies. + InheritedRecordScopeNoServer = lifecycle.InheritedRecordScopeNoServer + // InheritedRecordScopeForeignServer means a server answered but was not + // adopted, so its instances may still be alive on a server this backend + // does not drive. + InheritedRecordScopeForeignServer = lifecycle.InheritedRecordScopeForeignServer + // InheritedRecordScopeAdopted means this backend adopted the recorded + // server, so its enumeration is authoritative for inherited rows. + InheritedRecordScopeAdopted = lifecycle.InheritedRecordScopeAdopted +) + +// AdoptionRecordStore is the narrow persistence contract the adoption +// orchestration reads and writes the installation-scoped record through. +type AdoptionRecordStore = lifecycle.AdoptionRecordStore + +// AdoptionControlClient is the ownership surface adoption drives on a live +// control server. +type AdoptionControlClient = lifecycle.AdoptionControlClient + +// AdoptionControlClientFactory builds a client targeting a recorded endpoint. +type AdoptionControlClientFactory = lifecycle.AdoptionControlClientFactory + +// AdoptionOutcome is the result of attempting to adopt a detached server. +type AdoptionOutcome = lifecycle.AdoptionOutcome + +// OwnershipRenewer keeps an adopted server's ownership lease renewed for as +// long as this backend drives it. +type OwnershipRenewer = lifecycle.OwnershipRenewer + +// OwnershipClaimer is the single ownership-claim operation the renewer issues. +type OwnershipClaimer = lifecycle.OwnershipClaimer + +// RequiredSurvivalCapabilities is the capability set this backend requires of +// a control server before adopting it. +var RequiredSurvivalCapabilities = lifecycle.RequiredSurvivalCapabilities + +// AttemptAdoptControlServer runs the adoption handshake against the recorded +// control endpoint and reports whether this backend may drive that server. +func AttemptAdoptControlServer( + ctx context.Context, + store AdoptionRecordStore, + secretStore secrets.SecretStore, + newClient AdoptionControlClientFactory, + homeDir string, + requiredCapabilities []string, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, + log *logger.Logger, +) AdoptionOutcome { + return lifecycle.AttemptAdoptControlServer(ctx, store, secretStore, newClient, homeDir, + requiredCapabilities, recoveryReadTimeout, recoveryReadRetries, log) +} + +// ReclaimUnneededControlServer stops and clears a recorded server this launch +// will not adopt because the capability is disabled. +func ReclaimUnneededControlServer( + ctx context.Context, + store AdoptionRecordStore, + secretStore secrets.SecretStore, + newClient AdoptionControlClientFactory, + homeDir string, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, + log *logger.Logger, +) { + lifecycle.ReclaimUnneededControlServer(ctx, store, secretStore, newClient, homeDir, + recoveryReadTimeout, recoveryReadRetries, log) +} + +// RecordFreshControlServer persists the record and credential for a control +// server this backend just spawned, so a later launch can adopt it. +func RecordFreshControlServer( + ctx context.Context, + store AdoptionRecordStore, + secretStore secrets.SecretStore, + client AdoptionControlClient, + endpoint string, + credential string, +) error { + return lifecycle.RecordFreshControlServer(ctx, store, secretStore, client, endpoint, credential) +} + +// NewOwnershipRenewer builds the renewer that holds an adopted server's +// ownership lease for this backend's lifetime. +func NewOwnershipRenewer( + claimer OwnershipClaimer, + interval time.Duration, + log *logger.Logger, +) *OwnershipRenewer { + return lifecycle.NewOwnershipRenewer(claimer, interval, log) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup.go b/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup.go index d493d34894..81feeddd47 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup.go +++ b/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup.go @@ -2,8 +2,23 @@ package lifecycle import ( commonconfig "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/task/models" ) +// agentctlStartupConfigForExecutor keeps the host-only survival reaper out of +// managed remote/container environments. The backend owns the renewal loop for +// its host control server; a remote agentctl has no such loop and would reap +// its own instances after the unowned timeout. +func agentctlStartupConfigForExecutor( + startup commonconfig.AgentctlStartupConfig, + executorType string, +) commonconfig.AgentctlStartupConfig { + if models.IsRemoteExecutorType(models.ExecutorType(executorType)) { + startup.AgentSurvivalEnabled = false + } + return startup +} + func validateAgentctlStartupConfig(startup commonconfig.AgentctlStartupConfig) error { if !startup.Configured { return nil diff --git a/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup_test.go b/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup_test.go index f5f5790a79..0afdae5f16 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/agentctl_startup_test.go @@ -7,6 +7,7 @@ import ( "github.com/kandev/kandev/internal/agent/agents" commonconfig "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/task/models" ) func testAgentctlStartupConfig() commonconfig.AgentctlStartupConfig { @@ -19,6 +20,33 @@ func testAgentctlStartupConfig() commonconfig.AgentctlStartupConfig { } } +func TestAgentctlStartupConfigDisablesSurvivalOutsideHostExecutors(t *testing.T) { + for _, executorType := range []models.ExecutorType{ + models.ExecutorTypeLocalDocker, + models.ExecutorTypeRemoteDocker, + models.ExecutorTypeSprites, + models.ExecutorTypeSSH, + models.ExecutorTypeKubernetes, + models.ExecutorTypeMockRemote, + } { + startup := testAgentctlStartupConfig() + startup.AgentSurvivalEnabled = true + got := agentctlStartupConfigForExecutor(startup, string(executorType)) + if got.AgentSurvivalEnabled { + t.Fatalf("executor %q retained host survival reaper", executorType) + } + } + + for _, executorType := range []string{string(models.ExecutorTypeLocal), string(models.ExecutorTypeWorktree), legacyExecutorTypeLocalPC} { + startup := testAgentctlStartupConfig() + startup.AgentSurvivalEnabled = true + got := agentctlStartupConfigForExecutor(startup, executorType) + if !got.AgentSurvivalEnabled { + t.Fatalf("host executor %q lost survival reaper", executorType) + } + } +} + func TestAgentctlStartupContractReachesContainerSpriteAndSSHEnvironments(t *testing.T) { startup := testAgentctlStartupConfig() encoded, err := commonconfig.EncodeAgentctlStartupConfig(startup) diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption.go new file mode 100644 index 0000000000..c505996f65 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption.go @@ -0,0 +1,561 @@ +package lifecycle + +import ( + "context" + "fmt" + "strings" + "time" + + "go.uber.org/zap" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" +) + +// RequiredSurvivalCapabilities is the capability set this backend requires +// of a control server before it will adopt it (design 01, "Capability +// compatibility": "The backend requires a set and adopts only when its +// required set is a subset"). Must stay a subset of +// api.SurvivalCapabilities -- kept as an independent literal here rather +// than imported, since agentctl/server/api is not a package the backend +// package layer depends on. +var RequiredSurvivalCapabilities = []string{"agent-survival.v1"} + +// AdoptionRecordStore is the narrow persistence interface the adoption +// orchestration reads and writes the single installation-scoped +// control-server record through, satisfied by *sqlite.Repository. +type AdoptionRecordStore interface { + GetControlServerRecord(ctx context.Context) (*models.ControlServerRecord, error) + UpsertControlServerRecord(ctx context.Context, record *models.ControlServerRecord) error +} + +// staleExecutionRepairStore is the optional persistence capability +// AC-EXECUTORS-CONTROL-OWNERSHIP-004.3 and AC-EXECUTORS-SURVIVAL-005.5's +// immediate-repair step need: list this installation's live standalone +// recovery-inventory records and repair each in place. Satisfied by +// *sqlite.Repository alongside AdoptionRecordStore; type-asserted rather +// than folded into AdoptionRecordStore itself so RecordFreshControlServer's +// store parameter and every existing AdoptionRecordStore test double stay +// unaffected, matching this package's turnOutcomeApplier/recoveryStopper +// optional-capability convention. +type staleExecutionRepairStore interface { + ListExecutorsRunningLiveStandalone(ctx context.Context) ([]*models.ExecutorRunning, error) + RepairExecutorRunningDead(ctx context.Context, sessionID string) error +} + +// repairLiveStandaloneRecordsAfterOwnServerStopped implements +// AC-EXECUTORS-CONTROL-OWNERSHIP-004.3's "repair their recovery-inventory +// records itself... performed immediately" for a control server this +// backend has just proved is its own and has itself successfully stopped: +// unlike the deferred unowned-shutdown repair of AC-EXECUTORS-CONTROL- +// OWNERSHIP-003.5 (no backend attached at kill time), a backend is attached +// here and can write the durable store right away. Since exactly one +// control server is ever recorded per installation, stopping it makes every +// currently-live standalone recovery-inventory record definitively dead -- +// there is no live enumeration to correlate against, unlike an ordinary +// recovery pass. Repair preserves resume token and worktree identity +// (RepairExecutorRunningDead), never deletes. Best-effort: a listing or +// per-record repair failure is logged and does not block the caller, since +// the stopped server is gone regardless and a later reconciliation pass +// will eventually repair any row this call missed. +func repairLiveStandaloneRecordsAfterOwnServerStopped(ctx context.Context, store staleExecutionRepairStore, log *logger.Logger) { + records, err := store.ListExecutorsRunningLiveStandalone(ctx) + if err != nil { + log.Warn("failed to list live standalone recovery-inventory records for immediate repair after stopping this installation's own control server", zap.Error(err)) + return + } + for _, record := range records { + if record == nil || record.SessionID == "" { + continue + } + if err := store.RepairExecutorRunningDead(ctx, record.SessionID); err != nil { + log.Warn("failed to repair recovery-inventory record after stopping this installation's own control server", + zap.String("session_id", record.SessionID), zap.Error(err)) + } + } +} + +// shutdownControlServerWithRetry issues the ownership-shutdown operation +// within the same bounded per-attempt timeout and retry count as +// AC-EXECUTORS-SURVIVAL-002.13 (AC-EXECUTORS-CONTROL-OWNERSHIP-004.7), using +// the operator-configured override when the caller supplies one (a zero +// timeout or negative retry count falls back to the AC's own two-second/ +// two-retry default, mirroring StandaloneExecutor.stopWithRetry). A +// dial-level connection-refused error counts as success without consuming a +// retry: AC-004.7's explicit "already stopping or already absent" carve-out, +// reached when the target has already exited on its own (e.g. via +// runUnownedReaper) before this call was ever issued. +func shutdownControlServerWithRetry(ctx context.Context, client AdoptionControlClient, timeout time.Duration, retries int) error { + if timeout <= 0 { + timeout = defaultRecoveryReadTimeout + } + if retries < 0 { + retries = defaultRecoveryReadRetries + } + + var lastErr error + for attempt := 0; attempt <= retries; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + lastErr = client.ShutdownControlServer(attemptCtx) + cancel() + if lastErr == nil || isAlreadyAbsentShutdownError(lastErr) { + return nil + } + } + return lastErr +} + +// isAlreadyAbsentShutdownError reports whether err is the dial-level +// connection-refused shape produced when the target control server has +// already exited before this call reached it. +func isAlreadyAbsentShutdownError(err error) bool { + return err != nil && strings.Contains(err.Error(), "connection refused") +} + +// withAdoptionRetry executes fn up to retries+1 times, each attempt bounded +// by its own per-attempt timeout derived from ctx rather than ctx's own +// (possibly absent) deadline -- the same bounded-per-attempt shape as +// shutdownControlServerWithRetry and StandaloneExecutor.listInstancesWithRetry +// (AC-EXECUTORS-SURVIVAL-002.13). A zero timeout or negative retry count +// falls back to the AC's own two-second/two-retry default. +func withAdoptionRetry[T any](ctx context.Context, timeout time.Duration, retries int, fn func(context.Context) (T, error)) (T, error) { + if timeout <= 0 { + timeout = defaultRecoveryReadTimeout + } + if retries < 0 { + retries = defaultRecoveryReadRetries + } + + var lastErr error + var zero T + for attempt := 0; attempt <= retries; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + result, err := fn(attemptCtx) + cancel() + if err == nil { + return result, nil + } + lastErr = err + } + return zero, lastErr +} + +// AdoptionControlClient is the subset of agentctl.ControlClient's ownership +// surface the adoption orchestration drives, narrowed so the decision below +// is testable without a real HTTP round trip. +type AdoptionControlClient interface { + SetAuthToken(token string) + GetIdentity(ctx context.Context) (*agentctl.IdentityInfo, error) + ProveOwnership(ctx context.Context, challenge string) ([]string, error) + GetServerDetails(ctx context.Context) (*agentctl.ServerDetails, error) + RotateCredential(ctx context.Context) (*agentctl.CredentialRotationResult, error) + ConfirmCredentialRotation(ctx context.Context, rotationID int64) error + ShutdownControlServer(ctx context.Context) error +} + +// AdoptionControlClientFactory builds a client targeting the recorded +// control endpoint. Injected so tests never need a real listener. +type AdoptionControlClientFactory func(endpoint string) (AdoptionControlClient, error) + +// AdoptionReason is a fixed, stable adoption-outcome label for observability +// (design 02: "adoption attempted, adopted, and refused with a reason drawn +// from a fixed set"). AC-EXECUTORS-CONTROL-OWNERSHIP-001.8's "no recorded +// endpoint, or nothing answers, or already shutting down" bucket is +// deliberately NOT a refusal reason (nothing about it was refused), so it +// gets its own value outside the AC-001.6 refusal-reason precedence list. +// +// AC-001.6's own refusal-reason list also names "no answer" as one of its six +// enumerated reasons, ahead of identity mismatch in its precedence order -- +// which reads as a refusal, in tension with AC-001.8 treating the identical +// case ("an endpoint that nothing answers") as no refusal at all, and with +// AC-EXECUTORS-CONTROL-OWNERSHIP-003.9 explicitly mapping its own +// shutting-down outcome onto AC-001.8's bucket rather than AC-001.6's. Two +// ACs describing this case as "not a refusal" against one phrase in a third +// naming it as one: AC-001.8 (reinforced by 003.9) is treated as the +// authoritative behavior, and AC-001.6's "no answer" is read as the same +// no_server bucket by another name, not a distinct recorded refusal reason. +type AdoptionReason string + +const ( + // AdoptionReasonNoServer covers "no recorded control endpoint", "the + // recorded endpoint answers nothing", and "the recorded endpoint reports + // it has already begun an unowned shutdown" alike (AC-001.8): none of + // these is a refusal, so none carries a refusal reason, and all three + // result in the same action, spawn fresh and report nothing recovered. + AdoptionReasonNoServer AdoptionReason = "no_server" + // AdoptionReasonIdentityMismatch covers a server that advertises no + // identity or no capability set, one whose identity differs from the + // recorded one, and one that cannot prove it holds the recorded + // credential (AC-001.4, AC-001.5) -- refused, left running untouched, + // never contacted again by this adoption attempt, and never sent the + // credential. + AdoptionReasonIdentityMismatch AdoptionReason = "identity_mismatch" + // AdoptionReasonCredentialUnavailable is the stored credential could not + // be retrieved (AC-001.9): refused before any authentication was + // attempted, left running untouched, its own unowned shutdown reclaims it. + AdoptionReasonCredentialUnavailable AdoptionReason = "credential_unavailable" + // AdoptionReasonAuthenticationFailed is an attempted rotation the server + // rejected -- including a rotate refused because the server has already + // begun an unowned shutdown after this attempt's identity read succeeded, + // which is intentionally not distinguished from a wrong credential here: + // both refuse without issuing a stop and take the identical spawn-fresh + // path, and only the logged reason label would differ. + AdoptionReasonAuthenticationFailed AdoptionReason = "authentication_failed" + // AdoptionReasonCapabilityIncompatible is an authenticated, identified, + // own server missing a required capability (AC-004.3/004.4): refused + // AND stopped, because only an own authenticated server may ever be + // stopped. + AdoptionReasonCapabilityIncompatible AdoptionReason = "capability_incompatible" + // AdoptionReasonCredentialRotationFailed is a rotation the server + // accepted whose durable storage (secret write or record write) did not + // complete (AC-002.5): adoption is incomplete, nothing further is issued + // to that server, and it is left exactly as it was left by the + // two-phase window -- still holding both the superseded and the new + // credential, unconfirmed. + AdoptionReasonCredentialRotationFailed AdoptionReason = "credential_rotation_failed" +) + +// AdoptionOutcome is the result of attempting to adopt a previously-detached +// control server. Endpoint and Credential are populated only when Adopted. +// ContactedAt is the instant this attempt first reached the recorded control +// endpoint (the GetIdentity call) -- the AC-EXECUTORS-SURVIVAL-003.7 recovery +// deadline's clock start. It is left zero when no live server was ever +// reached (no record, or the client/transport itself could not be built), +// since nothing recovery-relevant is running in that case. +type AdoptionOutcome struct { + Adopted bool + Reason AdoptionReason + Endpoint string + // Credential is the single credential that authenticates both + // control-plane and per-instance operations and streams (design 01 + // "Single driver", AC-EXECUTORS-CONTROL-OWNERSHIP-002.6): use it for + // every client built against this control server or any instance it + // supervises. + Credential string + ContactedAt time.Time + // UnownedPeriod is the adopted server's own resolved unowned period, read + // back from its /identity response (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2/ + // .7). Only this value is safe to compute an ownership-renewal cadence + // from: this backend's local config can disagree with what the adopted + // process actually enforces across a restart that changed + // agentctl.unownedPeriod. Zero when the server answered without the + // field at all (a legacy pre-upgrade server) -- callers apply their own + // floor in that case, never a locally-resolved value. + UnownedPeriod time.Duration +} + +// AttemptAdoptControlServer implements startup steps 4 through 6 of design +// 02's "Startup" control flow: read the recorded control endpoint and +// identity, evaluate the gates in the required order -- home match, then +// authentication, then capability subset (AC-EXECUTORS-CONTROL-OWNERSHIP-001.3) +// -- and on a match rotate the ownership credential and persist the new +// record. It never spawns a process: the caller decides what to do with a +// refusal, which is always "spawn a fresh server on a free port and record +// it" regardless of which reason produced the refusal. +func AttemptAdoptControlServer( + ctx context.Context, + store AdoptionRecordStore, + secretStore secrets.SecretStore, + newClient AdoptionControlClientFactory, + homeDir string, + requiredCapabilities []string, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, + log *logger.Logger, +) AdoptionOutcome { + record, err := store.GetControlServerRecord(ctx) + if err != nil { + return AdoptionOutcome{Reason: AdoptionReasonNoServer} + } + + client, err := newClient(record.Endpoint) + if err != nil { + return AdoptionOutcome{Reason: AdoptionReasonNoServer} + } + + // The AC-EXECUTORS-SURVIVAL-003.7 recovery deadline's clock starts here, + // at the first real contact with the recorded control endpoint -- + // regardless of which gate below ultimately refuses or accepts adoption. + contactedAt := time.Now() + identity, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, client.GetIdentity) + if err != nil { + return AdoptionOutcome{Reason: AdoptionReasonNoServer} + } + + if !identityMatchesRecordedServer(identity, record) || len(identity.Capabilities) == 0 { + return AdoptionOutcome{Reason: AdoptionReasonIdentityMismatch, ContactedAt: contactedAt} + } + + credential, err := revealControlServerCredential(ctx, secretStore, record.CredentialSecretID) + if err != nil { + return AdoptionOutcome{Reason: AdoptionReasonCredentialUnavailable, ContactedAt: contactedAt} + } + + if !controlServerHoldsCredential(ctx, client, credential, homeDir, recoveryReadTimeout, recoveryReadRetries) { + return AdoptionOutcome{Reason: AdoptionReasonIdentityMismatch, ContactedAt: contactedAt} + } + + client.SetAuthToken(credential) + rotated, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, client.RotateCredential) + if err != nil { + return AdoptionOutcome{Reason: AdoptionReasonAuthenticationFailed, ContactedAt: contactedAt} + } + + if !capabilitiesSatisfy(requiredCapabilities, identity.Capabilities) { + client.SetAuthToken(rotated.Credential) + return stopIncompatibleControlServer(ctx, client, store, record, recoveryReadTimeout, recoveryReadRetries, contactedAt, log) + } + + client.SetAuthToken(rotated.Credential) + return finalizeControlServerAdoption(ctx, finalizeAdoptionArgs{ + store: store, + secretStore: secretStore, + client: client, + record: record, + identity: identity, + rotated: rotated, + contactedAt: contactedAt, + recoveryReadTimeout: recoveryReadTimeout, + recoveryReadRetries: recoveryReadRetries, + log: log, + }) +} + +// finalizeAdoptionArgs carries the state the two durable writes and the +// confirmation call need, so the adoption gates above stay readable as a +// sequence of refusals. +type finalizeAdoptionArgs struct { + store AdoptionRecordStore + secretStore secrets.SecretStore + client AdoptionControlClient + record *models.ControlServerRecord + identity *agentctl.IdentityInfo + rotated *agentctl.CredentialRotationResult + contactedAt time.Time + recoveryReadTimeout time.Duration + recoveryReadRetries int + log *logger.Logger +} + +// finalizeControlServerAdoption performs the two durable writes adoption +// depends on -- the rotated credential and the updated record -- and only +// then tells the server the rotation is confirmed. Either write failing +// leaves adoption incomplete, because confirming a credential this backend +// did not durably record would tell the server to drop the one credential +// still recoverable from durable state. +func finalizeControlServerAdoption(ctx context.Context, a finalizeAdoptionArgs) AdoptionOutcome { + secretID, err := storeControlServerCredential(ctx, a.secretStore, a.record.CredentialSecretID, a.rotated.Credential) + if err != nil { + return AdoptionOutcome{Reason: AdoptionReasonCredentialRotationFailed, ContactedAt: a.contactedAt} + } + + updated := &models.ControlServerRecord{ + Endpoint: a.record.Endpoint, + ServerIdentity: a.identity.ServerIdentity, + CredentialSecretID: secretID, + Capabilities: a.identity.Capabilities, + DiagnosticLogPath: recordedDiagnosticLogPath(ctx, a.client, a.recoveryReadTimeout, a.recoveryReadRetries), + CreatedAt: a.record.CreatedAt, + } + if err := a.store.UpsertControlServerRecord(ctx, updated); err != nil { + return AdoptionOutcome{Reason: AdoptionReasonCredentialRotationFailed, ContactedAt: a.contactedAt} + } + + // Sent only after both durable writes above completed + // (AC-EXECUTORS-CONTROL-OWNERSHIP-002.7). A failure here is not treated + // as fatal to adoption: the durable state already names the rotated + // credential as current, so a future retry or the server's own + // idempotent replay (AC-002.10) recovers it, and the superseded + // credential merely stays adoption-only-acceptable a little longer than + // necessary in the meantime. + if _, err := withAdoptionRetry(ctx, a.recoveryReadTimeout, a.recoveryReadRetries, func(c context.Context) (struct{}, error) { + return struct{}{}, a.client.ConfirmCredentialRotation(c, a.rotated.RotationID) + }); err != nil { + a.log.Warn("control server credential rotation stored durably but confirmation call failed", + zap.String("endpoint", a.record.Endpoint), zap.Error(err)) + } + + return AdoptionOutcome{ + Adopted: true, + Endpoint: a.record.Endpoint, + Credential: a.rotated.Credential, + ContactedAt: a.contactedAt, + UnownedPeriod: time.Duration(a.identity.UnownedPeriodMS) * time.Millisecond, + } +} + +// RecordFreshControlServer durably records a freshly spawned (not adopted) +// control server as the new installation-scoped record. Per the failure +// table's "Own server started after a refused or failed adoption" row, the +// single control-server record is always rewritten to name the new server: +// the record cannot keep pointing at a server that is about to reap itself +// through its own unowned shutdown. credential is the server's initial +// ownership credential (the bootstrap auth token minted at launch). A +// prior record's CredentialSecretID, when one is readable, is reused so a +// spawn-after-refused-adoption does not leave an orphaned secret row behind. +func RecordFreshControlServer( + ctx context.Context, + store AdoptionRecordStore, + secretStore secrets.SecretStore, + client AdoptionControlClient, + endpoint string, + credential string, +) error { + identity, err := client.GetIdentity(ctx) + if err != nil { + return fmt.Errorf("read freshly spawned control server identity: %w", err) + } + + var priorSecretID string + if prior, err := store.GetControlServerRecord(ctx); err == nil { + priorSecretID = prior.CredentialSecretID + } + + secretID, err := storeControlServerCredential(ctx, secretStore, priorSecretID, credential) + if err != nil { + return fmt.Errorf("store freshly spawned control server credential: %w", err) + } + + record := &models.ControlServerRecord{ + Endpoint: endpoint, + ServerIdentity: identity.ServerIdentity, + CredentialSecretID: secretID, + Capabilities: identity.Capabilities, + DiagnosticLogPath: recordedDiagnosticLogPath(ctx, client, defaultRecoveryReadTimeout, defaultRecoveryReadRetries), + } + return store.UpsertControlServerRecord(ctx, record) +} + +// ReclaimUnneededControlServer implements AC-EXECUTORS-SURVIVAL-005.5: when +// the agent-survival capability is disabled, a detached control server left +// running by an earlier survival-enabled launch is stopped rather than left +// running unowned -- but only when this backend can prove that server is +// its own by the same identity+credential test AttemptAdoptControlServer +// applies to adoption. A server this backend cannot prove is its own is +// left untouched, and so is one that answers nothing (nothing left to +// reclaim) or has no recorded endpoint at all. +// +// This never adopts: it does not rotate the credential, so a backend +// holding only a superseded credential can still perform it, which +// AC-EXECUTORS-SURVIVAL-005.1 forbids adoption itself from doing. The stop +// is issued as the ownership-shutdown operation of +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.9, which requires no prior adoption or +// rotation for exactly this reason. Every failure mode here is logged and +// non-fatal to startup: the caller always falls through to spawning a fresh +// server regardless of what this reclaim attempt did. +func ReclaimUnneededControlServer( + ctx context.Context, + store AdoptionRecordStore, + secretStore secrets.SecretStore, + newClient AdoptionControlClientFactory, + homeDir string, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, + log *logger.Logger, +) { + record, err := store.GetControlServerRecord(ctx) + if err != nil { + return + } + + client, err := newClient(record.Endpoint) + if err != nil { + return + } + + identity, err := client.GetIdentity(ctx) + if err != nil { + return + } + if !identityMatchesRecordedServer(identity, record) { + return + } + + credential, err := revealControlServerCredential(ctx, secretStore, record.CredentialSecretID) + if err != nil { + log.Warn("cannot reclaim detached control server: stored credential unavailable", + zap.String("endpoint", record.Endpoint), zap.Error(err)) + return + } + + if !controlServerHoldsCredential(ctx, client, credential, homeDir, recoveryReadTimeout, recoveryReadRetries) { + log.Warn("cannot reclaim detached control server: it could not prove it holds this installation's credential", + zap.String("endpoint", record.Endpoint)) + return + } + + client.SetAuthToken(credential) + if err := shutdownControlServerWithRetry(ctx, client, recoveryReadTimeout, recoveryReadRetries); err != nil { + log.Warn("failed to stop detached control server left by an earlier survival-enabled launch after exhausting retries; leaving it to its own unowned shutdown", + zap.String("endpoint", record.Endpoint), zap.Error(err)) + return + } + + log.Info("stopped a detached control server left by an earlier survival-enabled launch now that the capability is disabled", + zap.String("endpoint", record.Endpoint)) + if repairStore, ok := store.(staleExecutionRepairStore); ok { + repairLiveStandaloneRecordsAfterOwnServerStopped(ctx, repairStore, log) + } +} + +// stopIncompatibleControlServer stops an authenticated, identified, own +// server that is missing a required capability and, on success, repairs its +// live standalone records immediately (AC-EXECUTORS-CONTROL-OWNERSHIP-004.3). +// Always returns the AdoptionReasonCapabilityIncompatible refusal regardless +// of whether the stop itself succeeded. +func stopIncompatibleControlServer( + ctx context.Context, + client AdoptionControlClient, + store AdoptionRecordStore, + record *models.ControlServerRecord, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, + contactedAt time.Time, + log *logger.Logger, +) AdoptionOutcome { + if stopErr := shutdownControlServerWithRetry(ctx, client, recoveryReadTimeout, recoveryReadRetries); stopErr != nil { + // AC-EXECUTORS-CONTROL-OWNERSHIP-004.7: retries exhausted -- record no + // repair, report no recovered instances (already true, Adopted stays + // false below), and leave the server to its own unowned shutdown; a + // later backend repairs through the deferred + // AC-EXECUTORS-CONTROL-OWNERSHIP-003.5 path once it stops itself. + log.Warn("failed to stop incompatible control server after exhausting retries; leaving it to its own unowned shutdown", + zap.String("endpoint", record.Endpoint), zap.Error(stopErr)) + } else if repairStore, ok := store.(staleExecutionRepairStore); ok { + // AC-EXECUTORS-CONTROL-OWNERSHIP-004.3: the stop succeeded and a + // backend is attached, so repair every live standalone record + // immediately rather than deferring to the next start. + repairLiveStandaloneRecordsAfterOwnServerStopped(ctx, repairStore, log) + } + return AdoptionOutcome{Reason: AdoptionReasonCapabilityIncompatible, ContactedAt: contactedAt} +} + +// identityMatchesRecordedServer reports whether a live server's per-launch +// identity nonce is the exact one the control-server record carries, which is +// what proves the live server IS the recorded process rather than merely +// another launch. Installation identity (the home directory) is a separate +// question, proved over the authenticated ownership-details exchange rather +// than here, because /identity deliberately discloses no filesystem path +// (AC-EXECUTORS-CONTROL-OWNERSHIP-001.11). An identity either side cannot +// produce leaves the match unprovable and is refused rather than assumed. +func identityMatchesRecordedServer(identity *agentctl.IdentityInfo, record *models.ControlServerRecord) bool { + if record.ServerIdentity == "" || identity.ServerIdentity != record.ServerIdentity { + return false + } + return true +} + +func capabilitiesSatisfy(required, advertised []string) bool { + have := make(map[string]struct{}, len(advertised)) + for _, c := range advertised { + have[c] = struct{}{} + } + for _, r := range required { + if _, ok := have[r]; !ok { + return false + } + } + return true +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_identity_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_identity_test.go new file mode 100644 index 0000000000..7cee6ce80e --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_identity_test.go @@ -0,0 +1,43 @@ +package lifecycle + +import ( + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/task/models" +) + +// TestIdentityMatchesRecordedServerRequiresRecordedIdentity pins the closed +// side of the identity proof. A record with no recorded server identity cannot +// prove which process it describes, and an installation-wide value such as the +// home directory proves nothing beyond a shared installation, which every +// control server on the machine satisfies. An unprovable identity is refused +// rather than adopted. +func TestIdentityMatchesRecordedServerRequiresRecordedIdentity(t *testing.T) { + identity := &agentctl.IdentityInfo{ServerIdentity: "live-nonce"} + + if identityMatchesRecordedServer(identity, &models.ControlServerRecord{}) { + t.Fatal("a record carrying no server identity was treated as proven") + } + + matching := &models.ControlServerRecord{ServerIdentity: "live-nonce"} + if !identityMatchesRecordedServer(identity, matching) { + t.Fatal("a record whose server identity matches the live one was refused") + } + + other := &models.ControlServerRecord{ServerIdentity: "some-other-nonce"} + if identityMatchesRecordedServer(identity, other) { + t.Fatal("a record whose server identity differs from the live one was accepted") + } +} + +// TestIdentityMatchesRecordedServerRequiresAdvertisedIdentity covers the other +// half: a live server that advertises no identity is equally unprovable, so it +// cannot satisfy a record that names one. +func TestIdentityMatchesRecordedServerRequiresAdvertisedIdentity(t *testing.T) { + record := &models.ControlServerRecord{ServerIdentity: "recorded-nonce"} + + if identityMatchesRecordedServer(&agentctl.IdentityInfo{}, record) { + t.Fatal("a server advertising no identity was treated as proven") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_precedence_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_precedence_test.go new file mode 100644 index 0000000000..939fb7199e --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_precedence_test.go @@ -0,0 +1,61 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" +) + +// TestAttemptAdoptControlServerIdentityMismatchTakesPrecedenceOverCredentialUnavailable +// pins AC-001.6's refusal-reason precedence order: identity mismatch is +// evaluated before credential unavailability, so when both conditions hold -- +// here, a mismatched ServerIdentity AND a CredentialSecretID the secret store +// has no row for -- the recorded reason is identity_mismatch, not +// credential_unavailable, because the credential-reveal gate is never +// reached. +func TestAttemptAdoptControlServerIdentityMismatchTakesPrecedenceOverCredentialUnavailable(t *testing.T) { + record := validRecord() + record.ServerIdentity = "recorded-identity" + record.CredentialSecretID = "does-not-exist" + identity := validIdentity() + identity.ServerIdentity = "a-different-live-identity" + client := &fakeAdoptionControlClient{identity: identity} + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonIdentityMismatch { + t.Fatalf("outcome = %+v, want unadopted identity_mismatch even though credential_unavailable also applies", outcome) + } + if len(client.presentedTokens) != 0 { + t.Fatal("RotateCredential's SetAuthToken was reached despite an identity mismatch") + } +} + +// TestAttemptAdoptControlServerCredentialUnavailableTakesPrecedenceOverAuthenticationFailed +// pins AC-001.6's refusal-reason precedence order: credential unavailability +// is evaluated before authentication failure, so when both conditions hold -- +// here, a CredentialSecretID the secret store has no row for AND a client +// that would refuse any rotation attempt -- the recorded reason is +// credential_unavailable, not authentication_failed, because RotateCredential +// is never called. +func TestAttemptAdoptControlServerCredentialUnavailableTakesPrecedenceOverAuthenticationFailed(t *testing.T) { + record := validRecord() + record.CredentialSecretID = "does-not-exist" + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + rotateErr: errors.New("401 invalid auth token"), + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCredentialUnavailable { + t.Fatalf("outcome = %+v, want unadopted credential_unavailable even though authentication_failed also applies", outcome) + } + if len(client.presentedTokens) != 0 { + t.Fatal("RotateCredential's SetAuthToken was reached despite no credential being available") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_store_failure_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_store_failure_test.go new file mode 100644 index 0000000000..8cd31df25b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_store_failure_test.go @@ -0,0 +1,68 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" +) + +// TestAdoptionTreatsAnUnreadableRecordAsNoServer pins the durable-read +// failure branch. A store that cannot answer is not evidence that a control +// server is absent, but it is also not a refusal: there is nothing to refuse +// yet. Adoption takes the same spawn-fresh path as a genuinely empty record, +// and in particular never contacts an endpoint it could not read. +func TestAdoptionTreatsAnUnreadableRecordAsNoServer(t *testing.T) { + store := &fakeAdoptionRecordStore{getErr: errors.New("database is locked")} + factory := func(string) (AdoptionControlClient, error) { + t.Fatal("a client was built for a record that could not be read") + return nil, nil + } + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted { + t.Fatalf("outcome = %+v, want unadopted", outcome) + } + if outcome.Reason != AdoptionReasonNoServer { + t.Fatalf("Reason = %q, want %q", outcome.Reason, AdoptionReasonNoServer) + } +} + +// TestAdoptionIsIncompleteWhenTheRecordWriteFails is the sibling of the +// secret-write failure. The rotation succeeded on the server, so the server +// now holds a credential this backend could not durably record. Confirming +// it would tell the server to drop the superseded credential, which is the +// only one still recoverable from durable state, stranding the server +// permanently. Adoption is therefore incomplete and confirm is never sent. +func TestAdoptionIsIncompleteWhenTheRecordWriteFails(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "current-token", + rotateResult: &agentctl.CredentialRotationResult{RotationID: 9, Credential: "rotated-token"}, + } + store, factory := adoptionFixture(t, record, client) + store.putErr = errors.New("database is locked") + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCredentialRotationFailed { + t.Fatalf("outcome = %+v, want unadopted credential_rotation_failed", outcome) + } + if client.confirmCalled { + t.Fatal("ConfirmCredentialRotation was sent despite the record write failing") + } + if len(store.upserts) != 0 { + t.Fatalf("upserts = %+v, want none recorded when the write failed", store.upserts) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_test.go new file mode 100644 index 0000000000..dd6eb45dcd --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_adoption_test.go @@ -0,0 +1,952 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + "time" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/common/ownershipproof" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" +) + +// failingUpdateSecretStore fails Update for one specific secret ID, so a +// test can force the durable-storage failure branch of adoption's rotation +// finalize step without the fallback-to-create path in +// storeControlServerCredential silently absorbing it. +type failingUpdateSecretStore struct { + *inMemorySecretStore + failUpdateFor string +} + +func (s *failingUpdateSecretStore) Update(ctx context.Context, id string, req *secrets.UpdateSecretRequest) error { + if id == s.failUpdateFor { + return errors.New("injected secret update failure") + } + return s.inMemorySecretStore.Update(ctx, id, req) +} + +func newAdoptionTestLogger(t *testing.T) *logger.Logger { + t.Helper() + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatalf("new logger: %v", err) + } + return log +} + +// fakeAdoptionRecordStore is an in-memory AdoptionRecordStore double that +// also implements the optional staleExecutionRepairStore capability, so +// tests can observe the immediate-repair step of AC-EXECUTORS-CONTROL- +// OWNERSHIP-004.3/AC-EXECUTORS-SURVIVAL-005.5. +type fakeAdoptionRecordStore struct { + record *models.ControlServerRecord + getErr error + upserts []*models.ControlServerRecord + putErr error + + liveStandaloneRecords []*models.ExecutorRunning + listLiveErr error + repairedSessionIDs []string + repairErr error +} + +func (f *fakeAdoptionRecordStore) ListExecutorsRunningLiveStandalone(context.Context) ([]*models.ExecutorRunning, error) { + if f.listLiveErr != nil { + return nil, f.listLiveErr + } + return f.liveStandaloneRecords, nil +} + +func (f *fakeAdoptionRecordStore) RepairExecutorRunningDead(_ context.Context, sessionID string) error { + if f.repairErr != nil { + return f.repairErr + } + f.repairedSessionIDs = append(f.repairedSessionIDs, sessionID) + return nil +} + +func (f *fakeAdoptionRecordStore) GetControlServerRecord(context.Context) (*models.ControlServerRecord, error) { + if f.getErr != nil { + return nil, f.getErr + } + if f.record == nil { + return nil, models.ErrControlServerRecordNotFound + } + return f.record, nil +} + +func (f *fakeAdoptionRecordStore) UpsertControlServerRecord(_ context.Context, record *models.ControlServerRecord) error { + if f.putErr != nil { + return f.putErr + } + f.upserts = append(f.upserts, record) + return nil +} + +// fakeAdoptionControlClient is an in-memory AdoptionControlClient double. +// +// heldCredential and boundHome are what this double's server actually +// possesses: it answers an ownership challenge by deriving from them, so a +// double whose held credential differs from the one the backend reveals, or +// whose bound home differs from the backend's, cannot produce an acceptable +// proof -- which is exactly the position a foreign control server is in. +// boundHome defaults to testHomeDir, so a double only has to name it when +// the point of the test is that the two installations differ. +type fakeAdoptionControlClient struct { + identity *agentctl.IdentityInfo + identityErr error + heldCredential string + boundHome string + proveErr error + challenges []string + rotateResult *agentctl.CredentialRotationResult + rotateErr error + confirmErr error + shutdownErr error + shutdownCalled bool + confirmedID int64 + confirmCalled bool + presentedTokens []string +} + +func (f *fakeAdoptionControlClient) SetAuthToken(token string) { + f.presentedTokens = append(f.presentedTokens, token) +} + +func (f *fakeAdoptionControlClient) GetIdentity(context.Context) (*agentctl.IdentityInfo, error) { + return f.identity, f.identityErr +} + +func (f *fakeAdoptionControlClient) ProveOwnership(_ context.Context, challenge string) ([]string, error) { + f.challenges = append(f.challenges, challenge) + if f.proveErr != nil { + return nil, f.proveErr + } + home := f.boundHome + if home == "" { + home = testHomeDir + } + return []string{ownershipproof.Derive(f.heldCredential, challenge, home)}, nil +} + +func (f *fakeAdoptionControlClient) GetServerDetails(context.Context) (*agentctl.ServerDetails, error) { + return &agentctl.ServerDetails{ + HomeDir: testHomeDir, + DiagnosticLogPath: testDiagnosticLogPath, + }, nil +} + +func (f *fakeAdoptionControlClient) RotateCredential(context.Context) (*agentctl.CredentialRotationResult, error) { + return f.rotateResult, f.rotateErr +} + +func (f *fakeAdoptionControlClient) ConfirmCredentialRotation(_ context.Context, rotationID int64) error { + f.confirmCalled = true + f.confirmedID = rotationID + return f.confirmErr +} + +func (f *fakeAdoptionControlClient) ShutdownControlServer(context.Context) error { + f.shutdownCalled = true + return f.shutdownErr +} + +const ( + testHomeDir = "/home/kandev" + testDiagnosticLogPath = "/home/kandev/logs/agentctl-diagnostic.log" +) + +func validRecord() *models.ControlServerRecord { + return &models.ControlServerRecord{ + Endpoint: "127.0.0.1:9999", + ServerIdentity: "matching-identity", + CredentialSecretID: "secret-1", + Capabilities: []string{"agent-survival.v1"}, + DiagnosticLogPath: testDiagnosticLogPath, + } +} + +func validIdentity() *agentctl.IdentityInfo { + return &agentctl.IdentityInfo{ + ServerIdentity: "matching-identity", + Capabilities: []string{"agent-survival.v1"}, + UnownedPeriodMS: 300000, + } +} + +func adoptionFixture(t *testing.T, record *models.ControlServerRecord, client *fakeAdoptionControlClient) (*fakeAdoptionRecordStore, func(string) (AdoptionControlClient, error)) { + t.Helper() + store := &fakeAdoptionRecordStore{record: record} + factory := func(string) (AdoptionControlClient, error) { return client, nil } + return store, factory +} + +// TestAttemptAdoptControlServerNoRecordSpawnsFresh pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-001.8's "no recorded control endpoint" +// branch: nothing to adopt, no refusal reason, caller spawns fresh. +func TestAttemptAdoptControlServerNoRecordSpawnsFresh(t *testing.T) { + store := &fakeAdoptionRecordStore{} + factory := func(string) (AdoptionControlClient, error) { + t.Fatal("client factory must not be called with no record") + return nil, nil + } + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted { + t.Fatal("Adopted = true, want false") + } + if outcome.Reason != AdoptionReasonNoServer { + t.Fatalf("Reason = %q, want %q", outcome.Reason, AdoptionReasonNoServer) + } +} + +// TestAttemptAdoptControlServerUnreachableSpawnsFresh pins the "recorded +// endpoint answers nothing" branch of AC-001.8, folded into the same +// no-refusal reason as no-record. +func TestAttemptAdoptControlServerUnreachableSpawnsFresh(t *testing.T) { + client := &fakeAdoptionControlClient{identityErr: errors.New("connection refused")} + store, factory := adoptionFixture(t, validRecord(), client) + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonNoServer { + t.Fatalf("outcome = %+v, want unadopted no_server", outcome) + } +} + +// blockingIdentityControlClient's GetIdentity blocks until its call's own +// context is done, for proving a stalled control server cannot hold +// AttemptAdoptControlServer open past its configured per-attempt timeout. +type blockingIdentityControlClient struct { + fakeAdoptionControlClient + identityCalls int +} + +func (c *blockingIdentityControlClient) GetIdentity(ctx context.Context) (*agentctl.IdentityInfo, error) { + c.identityCalls++ + <-ctx.Done() + return nil, ctx.Err() +} + +// TestAttemptAdoptControlServerGetIdentityBoundedByConfiguredTimeout pins +// Review round 5 finding 2: a stalled control server's GetIdentity call must +// not hold AttemptAdoptControlServer open past the configured +// recoveryReadTimeout -- it must not fall back to the HTTP client's own much +// larger internal timeout (or block forever on an undeadlined context). +func TestAttemptAdoptControlServerGetIdentityBoundedByConfiguredTimeout(t *testing.T) { + client := &blockingIdentityControlClient{} + store, _ := adoptionFixture(t, validRecord(), &client.fakeAdoptionControlClient) + factory := func(string) (AdoptionControlClient, error) { return client, nil } + + start := time.Now() + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 20*time.Millisecond, 0, newAdoptionTestLogger(t)) + elapsed := time.Since(start) + + if outcome.Adopted || outcome.Reason != AdoptionReasonNoServer { + t.Fatalf("outcome = %+v, want unadopted no_server (a timed-out identity read never answered)", outcome) + } + if elapsed > 500*time.Millisecond { + t.Fatalf("AttemptAdoptControlServer took %v, want bounded by the configured 20ms per-attempt timeout, not an unbounded block", elapsed) + } + if client.identityCalls == 0 { + t.Fatal("expected GetIdentity to have been attempted at least once") + } +} + +// TestAttemptAdoptControlServerHomeMismatchRefusesWithoutStop pins AC-001.4: +// a server belonging to a different installation is refused, and -- +// critically -- ShutdownControlServer is never called, because a process +// this installation cannot prove is its own is never stopped. The mismatch +// is not something the server reports about itself; it falls out of the +// proof, which binds the home directory, so a server bound to another home +// cannot answer this backend's challenge. +func TestAttemptAdoptControlServerHomeMismatchRefusesWithoutStop(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "own-token", + boundHome: "/home/someone-else", + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonIdentityMismatch { + t.Fatalf("outcome = %+v, want unadopted identity_mismatch", outcome) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called for a home-mismatched server") + } + if len(client.presentedTokens) != 0 { + t.Fatalf("credential was sent to a server bound to another home: %v", client.presentedTokens) + } +} + +// TestAttemptAdoptControlServerNoCapabilitiesAdvertisedIsIdentityMismatch +// pins AC-001.5: a server that advertises no capability set at all is +// treated as AC-001.4, not as a capability-incompatible server -- so it is +// never stopped either. +func TestAttemptAdoptControlServerNoCapabilitiesAdvertisedIsIdentityMismatch(t *testing.T) { + identity := validIdentity() + identity.Capabilities = nil + client := &fakeAdoptionControlClient{identity: identity} + store, factory := adoptionFixture(t, validRecord(), client) + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Reason != AdoptionReasonIdentityMismatch { + t.Fatalf("Reason = %q, want %q", outcome.Reason, AdoptionReasonIdentityMismatch) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called for a server advertising no capabilities") + } +} + +// TestAttemptAdoptControlServerCredentialUnavailableRefusesWithoutStop pins +// AC-001.9: the stored credential cannot be retrieved (here, a stale +// reference the fake secret store has no row for), so no authentication is +// even attempted, the server is left running, and RotateCredential is never +// called. +func TestAttemptAdoptControlServerCredentialUnavailableRefusesWithoutStop(t *testing.T) { + client := &fakeAdoptionControlClient{identity: validIdentity()} + record := validRecord() + record.CredentialSecretID = "does-not-exist" + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCredentialUnavailable { + t.Fatalf("outcome = %+v, want unadopted credential_unavailable", outcome) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a credential-unavailable refusal") + } + if len(client.presentedTokens) != 0 { + t.Fatal("RotateCredential's SetAuthToken was reached despite no credential being available") + } +} + +// TestAttemptAdoptControlServerAuthenticationFailureRefusesWithoutStop pins +// AC-001.4's authentication-failure branch: rotate is attempted and refused +// by the server, so this backend treats it as foreign and never stops it. +func TestAttemptAdoptControlServerAuthenticationFailureRefusesWithoutStop(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "stale-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + heldCredential: "stale-token", + identity: validIdentity(), + rotateErr: errors.New("401 invalid auth token"), + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonAuthenticationFailed { + t.Fatalf("outcome = %+v, want unadopted authentication_failed", outcome) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on an authentication-failure refusal") + } +} + +// TestAttemptAdoptControlServerIncompatibleCapabilityStopsSurvivor pins +// AC-004.3/004.4: an own, authenticated server missing a required +// capability is refused AND stopped -- the one refusal reason that DOES +// issue a stop, because only an own authenticated server may ever be +// stopped. +func TestAttemptAdoptControlServerIncompatibleCapabilityStopsSurvivor(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + identity := validIdentity() + identity.Capabilities = []string{"some-other-capability"} + client := &fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: identity, + rotateResult: &agentctl.CredentialRotationResult{RotationID: 7, Credential: "rotated-token"}, + } + store, factory := adoptionFixture(t, record, client) + store.liveStandaloneRecords = []*models.ExecutorRunning{ + {SessionID: "session-1"}, {SessionID: "session-2"}, + } + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCapabilityIncompatible { + t.Fatalf("outcome = %+v, want unadopted capability_incompatible", outcome) + } + if !client.shutdownCalled { + t.Fatal("ShutdownControlServer was not called for an incompatible own server") + } + if len(store.upserts) != 0 { + t.Fatal("control server record was rewritten by the refused adoption itself; that is the caller's job after spawning fresh") + } + // AC-EXECUTORS-CONTROL-OWNERSHIP-004.3 (Review round 1, finding 4): a + // successful stop of a confirmed-own server immediately repairs every + // live standalone recovery-inventory record, since exactly one control + // server exists per installation and this one just proved it was that + // server. + if len(store.repairedSessionIDs) != 2 { + t.Fatalf("repaired sessions = %v, want session-1 and session-2 repaired immediately after the successful stop", store.repairedSessionIDs) + } +} + +// TestAttemptAdoptControlServerIncompatibleCapabilityRetriesStopBeforeGivingUp +// pins AC-EXECUTORS-CONTROL-OWNERSHIP-004.7: the stop this branch issues is +// retried within a bounded budget, not a single unretried call, and no +// record is repaired when every retry fails. The injected failure is a +// genuinely retryable server error, not a connection-refused/dial failure -- +// AC-004.7's "already absent counts as success" carve-out means that shape +// resolves on the first attempt instead of exhausting retries (see +// TestAttemptAdoptControlServerIncompatibleCapabilityConnectionRefusedCountsAsSuccess). +func TestAttemptAdoptControlServerIncompatibleCapabilityRetriesStopBeforeGivingUp(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + identity := validIdentity() + identity.Capabilities = []string{"some-other-capability"} + client := &countingShutdownControlClient{ + fakeAdoptionControlClient: fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: identity, + rotateResult: &agentctl.CredentialRotationResult{RotationID: 7, Credential: "rotated-token"}, + shutdownErr: errors.New("500 internal server error"), + }, + } + store, _ := adoptionFixture(t, record, &client.fakeAdoptionControlClient) + factory := func(string) (AdoptionControlClient, error) { return client, nil } + store.liveStandaloneRecords = []*models.ExecutorRunning{{SessionID: "session-1"}} + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCapabilityIncompatible { + t.Fatalf("outcome = %+v, want unadopted capability_incompatible", outcome) + } + if client.shutdownAttempts < 2 { + t.Fatalf("shutdown attempts = %d, want more than one (bounded retry, not a single unretried call)", client.shutdownAttempts) + } + if len(store.repairedSessionIDs) != 0 { + t.Fatalf("repaired sessions = %v, want none: AC-004.7 forbids repair when the stop's retries are exhausted", store.repairedSessionIDs) + } +} + +// countingShutdownControlClient wraps fakeAdoptionControlClient to count +// ShutdownControlServer attempts, for asserting a retry budget was actually +// exercised rather than a single call. +type countingShutdownControlClient struct { + fakeAdoptionControlClient + shutdownAttempts int +} + +func (c *countingShutdownControlClient) ShutdownControlServer(ctx context.Context) error { + c.shutdownAttempts++ + return c.fakeAdoptionControlClient.ShutdownControlServer(ctx) +} + +// TestAttemptAdoptControlServerIncompatibleCapabilityConnectionRefusedCountsAsSuccess +// pins AC-EXECUTORS-CONTROL-OWNERSHIP-004.7's "a control server reporting +// that it is already stopping or already absent shall count as success" +// carve-out: a dial-level connection-refused error (the shape produced when +// the target already exited on its own, e.g. via its own unowned reaper) +// must not be retried to exhaustion and reported as a genuine failure -- the +// immediate-repair step must still run. +func TestAttemptAdoptControlServerIncompatibleCapabilityConnectionRefusedCountsAsSuccess(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + identity := validIdentity() + identity.Capabilities = []string{"some-other-capability"} + client := &countingShutdownControlClient{ + fakeAdoptionControlClient: fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: identity, + rotateResult: &agentctl.CredentialRotationResult{RotationID: 7, Credential: "rotated-token"}, + shutdownErr: errors.New("dial tcp 127.0.0.1:9999: connect: connection refused"), + }, + } + store, _ := adoptionFixture(t, record, &client.fakeAdoptionControlClient) + factory := func(string) (AdoptionControlClient, error) { return client, nil } + store.liveStandaloneRecords = []*models.ExecutorRunning{{SessionID: "session-1"}} + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCapabilityIncompatible { + t.Fatalf("outcome = %+v, want unadopted capability_incompatible", outcome) + } + if client.shutdownAttempts != 1 { + t.Fatalf("shutdown attempts = %d, want exactly 1: a connection-refused error must not be retried", client.shutdownAttempts) + } + if len(store.repairedSessionIDs) != 1 || store.repairedSessionIDs[0] != "session-1" { + t.Fatalf("repaired sessions = %v, want session-1 repaired: an already-absent server counts as a successful stop", store.repairedSessionIDs) + } +} + +// TestAttemptAdoptControlServerSucceedsRotatesAndPersists pins the happy +// path: home matches, rotation succeeds, capability subset is satisfied, +// the new credential is durably stored under the SAME secret ID, the record +// is rewritten with the freshly observed identity fields, and confirm is +// sent last. +func TestAttemptAdoptControlServerSucceedsRotatesAndPersists(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + identity := validIdentity() + client := &fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: identity, + rotateResult: &agentctl.CredentialRotationResult{RotationID: 3, Credential: "rotated-token"}, + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if !outcome.Adopted { + t.Fatalf("outcome = %+v, want Adopted", outcome) + } + if outcome.Endpoint != record.Endpoint { + t.Fatalf("Endpoint = %q, want %q", outcome.Endpoint, record.Endpoint) + } + if outcome.Credential != "rotated-token" { + t.Fatalf("Credential = %q, want rotated-token", outcome.Credential) + } + // Review round 3, finding 5: the adopted server's own reported unowned + // period must travel back on the outcome, not be left for the caller to + // recompute from its own (potentially stale) local config. + if outcome.UnownedPeriod != 300*time.Second { + t.Fatalf("UnownedPeriod = %v, want 300s (identity.UnownedPeriodMS read back verbatim)", outcome.UnownedPeriod) + } + if !client.confirmCalled || client.confirmedID != 3 { + t.Fatalf("confirm not called with rotation id 3: called=%v id=%d", client.confirmCalled, client.confirmedID) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a successful adoption") + } + + got, err := secretStore.Reveal(context.Background(), secretID) + if err != nil { + t.Fatalf("Reveal: %v", err) + } + if got != "rotated-token" { + t.Fatalf("stored credential = %q, want rotated-token", got) + } + + if len(store.upserts) != 1 { + t.Fatalf("upserts = %d, want 1", len(store.upserts)) + } + written := store.upserts[0] + if written.ServerIdentity != identity.ServerIdentity { + t.Fatalf("ServerIdentity = %q, want %q", written.ServerIdentity, identity.ServerIdentity) + } + if written.CredentialSecretID != secretID { + t.Fatalf("CredentialSecretID = %q, want unchanged %q (rotation reuses the same secret)", written.CredentialSecretID, secretID) + } + if written.Endpoint != record.Endpoint { + t.Fatalf("Endpoint = %q, want unchanged %q", written.Endpoint, record.Endpoint) + } + // The remaining two fields decide whether a LATER launch can adopt this + // server at all: capabilities gate compatibility, and CreatedAt must + // survive rather than being reset to now by every adoption, which would + // make the record look freshly created forever. + if len(written.Capabilities) != len(identity.Capabilities) { + t.Fatalf("Capabilities = %v, want the live server's %v", written.Capabilities, identity.Capabilities) + } + for i, capability := range identity.Capabilities { + if written.Capabilities[i] != capability { + t.Fatalf("Capabilities = %v, want the live server's %v", written.Capabilities, identity.Capabilities) + } + } + if !written.CreatedAt.Equal(record.CreatedAt) { + t.Fatalf("CreatedAt = %v, want the recorded %v carried through", written.CreatedAt, record.CreatedAt) + } + if written.DiagnosticLogPath != testDiagnosticLogPath { + t.Fatalf("DiagnosticLogPath = %q, want the live server's %q", written.DiagnosticLogPath, testDiagnosticLogPath) + } +} + +// TestAttemptAdoptControlServerServerIdentityMismatchRefusesWithoutStop pins +// AC-CONTROL-OWNERSHIP-001.2: a live server whose home directory matches but +// whose per-launch ServerIdentity does not match the previously recorded +// value is a different process sharing the same home directory, not the +// server this record describes, and must be refused exactly like a +// home-directory mismatch -- never stopped, since this backend cannot prove +// it is its own. +func TestAttemptAdoptControlServerServerIdentityMismatchRefusesWithoutStop(t *testing.T) { + record := validRecord() + record.ServerIdentity = "recorded-identity" + identity := validIdentity() + identity.ServerIdentity = "a-different-live-identity" + client := &fakeAdoptionControlClient{identity: identity} + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonIdentityMismatch { + t.Fatalf("outcome = %+v, want unadopted identity_mismatch", outcome) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called for a ServerIdentity-mismatched server") + } +} + +// TestAttemptAdoptControlServerEmptyRecordedServerIdentityRefuses pins the +// fail-closed side of the identity proof end to end: a record that carries no +// server identity cannot prove which process it describes, and the home +// directory it shares with every control server on the installation is not a +// substitute. The server is left unadopted rather than claimed on an +// unprovable match. +func TestAttemptAdoptControlServerEmptyRecordedServerIdentityRefuses(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.ServerIdentity = "" + record.CredentialSecretID = secretID + identity := validIdentity() + identity.ServerIdentity = "whatever-this-server-reports" + client := &fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: identity, + rotateResult: &agentctl.CredentialRotationResult{RotationID: 1, Credential: "rotated-token"}, + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonIdentityMismatch { + t.Fatalf("outcome = %+v, want unadopted identity_mismatch for an unset recorded ServerIdentity", outcome) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called for a server whose identity could not be proven") + } +} + +// TestAttemptAdoptControlServerUnownedPeriodFallsBackToFloorWhenUnreported +// pins the legacy/pre-upgrade shape: a server that answers /identity without +// an unowned_period_ms field (zero value) reports a zero UnownedPeriod on the +// outcome, rather than a value this package fabricates -- the caller +// (backendapp.resolveAdoptedRenewalPeriod) owns the floor fallback. +func TestAttemptAdoptControlServerUnownedPeriodFallsBackToFloorWhenUnreported(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + identity := validIdentity() + identity.UnownedPeriodMS = 0 + client := &fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: identity, + rotateResult: &agentctl.CredentialRotationResult{RotationID: 1, Credential: "rotated-token"}, + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if !outcome.Adopted { + t.Fatalf("outcome = %+v, want Adopted", outcome) + } + if outcome.UnownedPeriod != 0 { + t.Fatalf("UnownedPeriod = %v, want 0 (unreported), so the caller applies its own floor", outcome.UnownedPeriod) + } +} + +// TestAttemptAdoptControlServerCredentialKeepsRotatingAcrossAdoptions pins a +// chain of two or more consecutive restarts -- this feature's own core +// scenario: each adoption's rotated credential is the one the NEXT adoption +// must present and receive a further rotation for, under the SAME secret +// slot throughout, since design 01 "Single driver" uses exactly one +// credential for both control-plane and per-instance operations and there +// is nothing else to keep synchronized across the chain. +func TestAttemptAdoptControlServerCredentialKeepsRotatingAcrossAdoptions(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "bootstrap-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + heldCredential: "bootstrap-token", + identity: validIdentity(), + rotateResult: &agentctl.CredentialRotationResult{RotationID: 1, Credential: "rotated-token-1"}, + } + store, factory := adoptionFixture(t, record, client) + + first := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + if !first.Adopted { + t.Fatalf("first adoption = %+v, want Adopted", first) + } + if first.Credential != "rotated-token-1" { + t.Fatalf("first Credential = %q, want rotated-token-1", first.Credential) + } + if len(store.upserts) != 1 { + t.Fatalf("upserts = %d, want 1 after the first adoption", len(store.upserts)) + } + if store.upserts[0].CredentialSecretID != secretID { + t.Fatalf("CredentialSecretID = %q, want unchanged %q across the chain", store.upserts[0].CredentialSecretID, secretID) + } + + // Simulate the next restart reading back the record the first call wrote. + // The server now holds the credential the first adoption rotated it to, + // so that is what it can prove possession of. + store.record = store.upserts[0] + client.heldCredential = "rotated-token-1" + client.rotateResult = &agentctl.CredentialRotationResult{RotationID: 2, Credential: "rotated-token-2"} + + second := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + if !second.Adopted { + t.Fatalf("second adoption = %+v, want Adopted", second) + } + if second.Credential != "rotated-token-2" { + t.Fatalf("second Credential = %q, want rotated-token-2", second.Credential) + } +} + +// TestAttemptAdoptControlServerConfirmFailureStillReportsAdopted pins the +// narrow edge this Build turn documents explicitly: both durable writes +// (secret + record) already completed by the time confirm is attempted, so +// a confirm-call failure does not undo local durable state and adoption is +// still reported successful. The credential and record already name the +// rotated value regardless of whether the server ever hears about it. +func TestAttemptAdoptControlServerConfirmFailureStillReportsAdopted(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: validIdentity(), + rotateResult: &agentctl.CredentialRotationResult{RotationID: 9, Credential: "rotated-token"}, + confirmErr: errors.New("connection reset"), + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if !outcome.Adopted { + t.Fatalf("outcome = %+v, want Adopted despite confirm failure", outcome) + } +} + +// TestAttemptAdoptControlServerRotationStorageFailureIsIncomplete pins +// AC-002.5: when the durable secret write fails, adoption is incomplete, +// confirm must never be sent (it would tell the server to drop a credential +// this backend never durably recorded), and the record must not be rewritten. +func TestAttemptAdoptControlServerRotationStorageFailureIsIncomplete(t *testing.T) { + secretStore := &failingUpdateSecretStore{inMemorySecretStore: newInMemorySecretStore()} + secretID, err := storeControlServerCredential(context.Background(), secretStore.inMemorySecretStore, "", "current-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + secretStore.failUpdateFor = secretID + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + heldCredential: "current-token", + identity: validIdentity(), + rotateResult: &agentctl.CredentialRotationResult{RotationID: 5, Credential: "rotated-token"}, + } + store, factory := adoptionFixture(t, record, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted || outcome.Reason != AdoptionReasonCredentialRotationFailed { + t.Fatalf("outcome = %+v, want unadopted credential_rotation_failed", outcome) + } + if client.confirmCalled { + t.Fatal("ConfirmCredentialRotation was called despite the durable secret write failing") + } + if len(store.upserts) != 0 { + t.Fatal("control server record was rewritten despite the durable secret write failing") + } +} + +// TestRecordFreshControlServerReusesPriorSecretID pins the "own server +// started after a refused or failed adoption" failure-table row: the record +// is rewritten to name the new server, and the existing credential secret +// row is reused in place (updated to the new server's bootstrap token) +// rather than leaving the old row orphaned. +func TestRecordFreshControlServerReusesPriorSecretID(t *testing.T) { + secretStore := newInMemorySecretStore() + priorSecretID, err := storeControlServerCredential(context.Background(), secretStore, "", "old-servers-token") + if err != nil { + t.Fatalf("seed prior credential: %v", err) + } + store := &fakeAdoptionRecordStore{record: &models.ControlServerRecord{ + Endpoint: "127.0.0.1:8888", + CredentialSecretID: priorSecretID, + }} + client := &fakeAdoptionControlClient{heldCredential: "old-servers-token", identity: validIdentity()} + + err = RecordFreshControlServer(context.Background(), store, secretStore, client, "127.0.0.1:9001", "new-servers-token") + if err != nil { + t.Fatalf("RecordFreshControlServer: %v", err) + } + + if len(store.upserts) != 1 { + t.Fatalf("upserts = %d, want 1", len(store.upserts)) + } + written := store.upserts[0] + if written.Endpoint != "127.0.0.1:9001" { + t.Fatalf("Endpoint = %q, want the new server's endpoint", written.Endpoint) + } + if written.CredentialSecretID != priorSecretID { + t.Fatalf("CredentialSecretID = %q, want reused %q", written.CredentialSecretID, priorSecretID) + } + + got, err := secretStore.Reveal(context.Background(), priorSecretID) + if err != nil { + t.Fatalf("Reveal: %v", err) + } + if got != "new-servers-token" { + t.Fatalf("stored credential = %q, want new-servers-token", got) + } +} + +// TestRecordFreshControlServerCreatesSecretWhenNoPriorRecord pins the +// first-ever-launch case: no prior record exists at all, so a brand new +// secret is created rather than a reuse being attempted. +func TestRecordFreshControlServerCreatesSecretWhenNoPriorRecord(t *testing.T) { + secretStore := newInMemorySecretStore() + store := &fakeAdoptionRecordStore{} + client := &fakeAdoptionControlClient{identity: validIdentity()} + + err := RecordFreshControlServer(context.Background(), store, secretStore, client, "127.0.0.1:9001", "fresh-token") + if err != nil { + t.Fatalf("RecordFreshControlServer: %v", err) + } + + if len(store.upserts) != 1 { + t.Fatalf("upserts = %d, want 1", len(store.upserts)) + } + written := store.upserts[0] + if written.CredentialSecretID == "" { + t.Fatalf("written = %+v, want a populated credential secret ID", written) + } + got, err := secretStore.Reveal(context.Background(), written.CredentialSecretID) + if err != nil { + t.Fatalf("Reveal: %v", err) + } + if got != "fresh-token" { + t.Fatalf("stored credential = %q, want fresh-token", got) + } +} + +// TestRecordFreshControlServerPropagatesIdentityFailure pins that a failure +// to read the freshly spawned server's own identity aborts before any +// durable write, rather than writing a record with blank fields. +func TestRecordFreshControlServerPropagatesIdentityFailure(t *testing.T) { + secretStore := newInMemorySecretStore() + store := &fakeAdoptionRecordStore{} + client := &fakeAdoptionControlClient{identityErr: errors.New("connection refused")} + + err := RecordFreshControlServer(context.Background(), store, secretStore, client, "127.0.0.1:9001", "fresh-token") + if err == nil { + t.Fatal("want error, got nil") + } + if len(store.upserts) != 0 { + t.Fatal("record was written despite the identity read failing") + } +} + +// --- ReclaimUnneededControlServer (Review round 1, finding 1: AC-EXECUTORS-SURVIVAL-005.5) --- + +// TestReclaimUnneededControlServerNoRecordDoesNothing pins that a disabled +// launch with no recorded control endpoint contacts nothing. +// TestReclaimUnneededControlServerNothingAnswersDoesNothing pins that an +// endpoint that answers nothing is left alone -- nothing to reclaim. The +// credential store is seeded exactly as +// TestReclaimUnneededControlServerStopsOwnServerAndRepairsRecords is, so a +// real credential is available: if the GetIdentity-error check under test +// were ever removed, the function would proceed to a genuine shutdown +// attempt and this assertion would actually catch it, rather than passing +// only because credential-unavailable happened to short-circuit first. +// TestReclaimUnneededControlServerIdentityMismatchLeavesServerUntouched pins +// that a server belonging to another installation is never stopped -- the +// same ownership gate adoption applies, on the path that runs with the +// capability off. The credential store is seeded exactly as +// TestReclaimUnneededControlServerStopsOwnServerAndRepairsRecords is, so a +// real credential is available: if the ownership check under test were ever +// removed, the function would proceed to a genuine shutdown attempt and this +// assertion would actually catch it, rather than passing only because +// credential-unavailable happened to short-circuit first. +// TestReclaimUnneededControlServerServerIdentityMismatchLeavesServerUntouched +// pins the same ServerIdentity gate AttemptAdoptControlServer applies +// (Review round 3, finding 2) for the reclaim path too: a home-directory +// match alone is not enough to prove this is the exact recorded server. +// TestReclaimUnneededControlServerCredentialUnavailableLeavesServerUntouched +// pins AC-EXECUTORS-SURVIVAL-005.5's refusal-before-any-stop-attempt when +// the stored credential cannot be read. +// TestReclaimUnneededControlServerStopsOwnServerAndRepairsRecords pins the +// success path: a proven-own server is stopped via the ownership-shutdown +// operation without ever rotating the credential (adoption is forbidden +// while the capability is disabled, AC-EXECUTORS-SURVIVAL-005.1), and every +// live standalone recovery-inventory record is repaired immediately. +// TestReclaimUnneededControlServerRetriesStopAndSkipsRepairOnFailure pins +// the shared AC-EXECUTORS-CONTROL-OWNERSHIP-004.7 retry contract applied to +// this stop too: bounded retries, and no repair when they're exhausted. The +// injected failure is a genuinely retryable server error, not a +// connection-refused/dial failure -- see +// TestReclaimUnneededControlServerConnectionRefusedCountsAsSuccess for that +// carve-out. +// TestReclaimUnneededControlServerConnectionRefusedCountsAsSuccess pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-004.7's "already absent counts as success" +// carve-out for this stop too: a dial-level connection-refused error must +// not be retried to exhaustion, and the immediate-repair step still runs. diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_credential.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_credential.go new file mode 100644 index 0000000000..581e6aad08 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_credential.go @@ -0,0 +1,128 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/kandev/kandev/internal/secrets" +) + +// controlServerCredentialSecretName is the fixed secret name for the +// installation's single standalone control-server ownership credential. It +// is never looked up by name -- only by the ID recorded in +// models.ControlServerRecord.CredentialSecretID -- so a fixed, non-unique +// name is safe. +const controlServerCredentialSecretName = "agent-survival-control-server-credential" + +const controlServerCredentialSecretIDPrefix = "kandev-runtime:control-server:" + +// storeControlServerCredential durably stores the control server's ownership +// credential and returns the secret ID that +// models.ControlServerRecord.CredentialSecretID should reference. Storing +// the token itself in the record would put a bearer credential for a +// service that executes commands in a user's worktree into the database in +// clear (design 01, "Ownership identity and credential") -- the record +// holds only this reference. +// +// existingSecretID, when non-empty, updates that secret in place so an +// ordinary credential rotation reuses the same reference and the record's +// CredentialSecretID never has to change. When the referenced secret is +// absent (e.g. deleted out of band), this falls back to creating a new one +// rather than failing: refusing to store a rotated credential would leave +// the backend unable to ever adopt this server again. When the referenced +// secret is present but its current value cannot be read (a transient +// failure, not absence), this also falls back to creating a new one rather +// than overwriting it: see tryReuseCredentialSecret. +func storeControlServerCredential(ctx context.Context, store secrets.SecretStore, existingSecretID, token string) (string, error) { + if store == nil { + return "", errors.New("secret store is unavailable") + } + if token == "" { + return "", errors.New("control server credential is required") + } + + // Older installations used a generated, user-visible secret ID. Do not + // update that row during rotation because it would expose the control-server + // bearer credential through the user secret APIs. New and migrated rows use + // the internal runtime prefix and remain hidden by UserVisibleStore. + if isControlServerCredentialSecretID(existingSecretID) { + reused, err := tryReuseCredentialSecret(ctx, store, existingSecretID, token) + if err != nil { + return "", err + } + if reused { + return existingSecretID, nil + } + } + + secret := &secrets.SecretWithValue{ + Secret: secrets.Secret{ + ID: controlServerCredentialSecretIDPrefix + uuid.NewString(), + Name: controlServerCredentialSecretName, + }, + Value: token, + } + if err := store.Create(ctx, secret); err != nil { + return "", err + } + return secret.ID, nil +} + +func isControlServerCredentialSecretID(id string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(id)), controlServerCredentialSecretIDPrefix) +} + +// tryReuseCredentialSecret overwrites existingSecretID in place with token +// and reports whether it did. +// +// It first reveals the secret's current value to prove the row is genuinely +// present and readable before overwriting it: Update alone cannot tell +// "absent" apart from "present but transiently unreadable" -- it blindly +// overwrites either way, since it never re-decrypts the prior value. +// Overwriting a slot whose value cannot currently be read would destroy +// whatever credential is there over what may be only a transient failure, +// which AC-EXECUTORS-CONTROL-OWNERSHIP-001.9 forbids ("shall not delete the +// stored credential" on a transient read error, "because the failure is on +// this side and destroying a secret over a transient read error is +// unrecoverable"). +// +// A genuinely absent secret, or one that fails to reveal for any other +// reason, both report (false, nil) rather than an error: the caller falls +// back to creating a fresh secret instead of failing the whole operation, so +// a transient blip reading the OLD credential's row never blocks durably +// recording a new one (leaving one orphaned row behind is an acceptable cost +// next to destroying a credential that might still be needed). Only a +// failure of the Update call itself -- the row having just been proven +// present and readable, then failing to write -- is surfaced as an error. +func tryReuseCredentialSecret(ctx context.Context, store secrets.SecretStore, existingSecretID, token string) (bool, error) { + if _, err := revealGlobalSecret(ctx, store, existingSecretID); err != nil { + return false, nil + } + if err := store.Update(ctx, existingSecretID, &secrets.UpdateSecretRequest{Value: &token}); err != nil { + if errors.Is(err, secrets.ErrNotFound) { + return false, nil + } + return false, err + } + return true, nil +} + +// revealControlServerCredential retrieves the decrypted ownership credential +// referenced by a control-server record. An empty reference is rejected +// before any store contact, distinct from a not-found lookup, so a caller +// can tell "record has no credential" apart from "stored reference is +// stale" -- design 02's failure table treats these differently +// (credential-unavailable vs authentication failure). +func revealControlServerCredential(ctx context.Context, store secrets.SecretStore, secretID string) (string, error) { + if secretID == "" { + return "", errors.New("control server credential reference is empty") + } + value, err := revealGlobalSecret(ctx, store, secretID) + if err != nil { + return "", fmt.Errorf("reveal control server credential: %w", err) + } + return value, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_credential_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_credential_test.go new file mode 100644 index 0000000000..ed5a1cf725 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_credential_test.go @@ -0,0 +1,188 @@ +package lifecycle + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/kandev/kandev/internal/secrets" +) + +// TestStoreControlServerCredentialCreatesWhenNoExistingSecretID pins the +// bootstrap path: the control-server record starts with no +// CredentialSecretID, so storing the token received from the bootstrap +// handshake must create a new secret and return its ID for the caller to +// persist onto the record. +func TestStoreControlServerCredentialCreatesWhenNoExistingSecretID(t *testing.T) { + store := newInMemorySecretStore() + + secretID, err := storeControlServerCredential(context.Background(), store, "", "the-bearer-token") + if err != nil { + t.Fatalf("storeControlServerCredential: %v", err) + } + if secretID == "" { + t.Fatal("secretID is empty, want a generated ID") + } + if !strings.HasPrefix(secretID, controlServerCredentialSecretIDPrefix) { + t.Fatalf("secretID = %q, want an internal runtime secret ID", secretID) + } + + got, err := store.Reveal(context.Background(), secretID) + if err != nil { + t.Fatalf("Reveal: %v", err) + } + if got != "the-bearer-token" { + t.Fatalf("revealed value = %q, want %q", got, "the-bearer-token") + } +} + +// TestStoreControlServerCredentialUpdatesInPlaceWhenSecretIDGiven pins +// rotation: a credential rotation replaces the token value under the SAME +// secret ID, so ControlServerRecord.CredentialSecretID never has to change +// on an ordinary rotation. +func TestStoreControlServerCredentialUpdatesInPlaceWhenSecretIDGiven(t *testing.T) { + store := newInMemorySecretStore() + existingID, err := storeControlServerCredential(context.Background(), store, "", "original-token") + if err != nil { + t.Fatalf("storeControlServerCredential(initial): %v", err) + } + + gotID, err := storeControlServerCredential(context.Background(), store, existingID, "rotated-token") + if err != nil { + t.Fatalf("storeControlServerCredential(rotation): %v", err) + } + if gotID != existingID { + t.Fatalf("gotID = %q, want unchanged %q", gotID, existingID) + } + + got, err := store.Reveal(context.Background(), existingID) + if err != nil { + t.Fatalf("Reveal: %v", err) + } + if got != "rotated-token" { + t.Fatalf("revealed value = %q, want %q", got, "rotated-token") + } +} + +// TestStoreControlServerCredentialFallsBackToCreateWhenSecretIDMissing pins +// the defensive case: the record names a secret ID the store no longer has +// (e.g. deleted out of band). Rotation must still durably store the new +// credential rather than fail, because the alternative -- refusing rotation +// -- would leave the backend unable to ever adopt this server again. +func TestStoreControlServerCredentialFallsBackToCreateWhenSecretIDMissing(t *testing.T) { + store := newInMemorySecretStore() + + secretID, err := storeControlServerCredential(context.Background(), store, "does-not-exist", "fresh-token") + if err != nil { + t.Fatalf("storeControlServerCredential: %v", err) + } + if secretID == "" || secretID == "does-not-exist" { + t.Fatalf("secretID = %q, want a freshly generated ID", secretID) + } + + got, err := store.Reveal(context.Background(), secretID) + if err != nil { + t.Fatalf("Reveal: %v", err) + } + if got != "fresh-token" { + t.Fatalf("revealed value = %q, want %q", got, "fresh-token") + } +} + +// TestStoreControlServerCredentialDoesNotOverwriteUnreadableExistingSecret +// pins AC-EXECUTORS-CONTROL-OWNERSHIP-001.9's "shall not delete the stored +// credential" for a transient read error: when the existing secret is +// present but a Reveal on it currently fails (not absent -- a real row a +// later attempt might still recover), storing a new credential must not +// blindly overwrite that row via Update, since Update never re-decrypts the +// prior value and would silently destroy it. It must fall back to a fresh +// secret instead, and the old row's value must remain exactly as it was. +func TestStoreControlServerCredentialDoesNotOverwriteUnreadableExistingSecret(t *testing.T) { + store := newInMemorySecretStore() + existingID, err := storeControlServerCredential(context.Background(), store, "", "still-needed-token") + if err != nil { + t.Fatalf("storeControlServerCredential(initial): %v", err) + } + + store.revealErr = errors.New("transient read failure") + + gotID, err := storeControlServerCredential(context.Background(), store, existingID, "unrelated-fresh-token") + if err != nil { + t.Fatalf("storeControlServerCredential: %v", err) + } + if gotID == existingID { + t.Fatalf("gotID = %q, want a freshly allocated ID distinct from the unreadable existing one", gotID) + } + + store.revealErr = nil + got, err := store.Reveal(context.Background(), existingID) + if err != nil { + t.Fatalf("Reveal(existingID) after fallback: %v", err) + } + if got != "still-needed-token" { + t.Fatalf("existing secret value = %q, want untouched %q", got, "still-needed-token") + } + + gotFresh, err := store.Reveal(context.Background(), gotID) + if err != nil { + t.Fatalf("Reveal(gotID): %v", err) + } + if gotFresh != "unrelated-fresh-token" { + t.Fatalf("fresh secret value = %q, want %q", gotFresh, "unrelated-fresh-token") + } +} + +// TestStoreControlServerCredentialRejectsEmptyToken pins that an empty +// bearer token is never durably stored -- storing a blank credential would +// silently make adoption impossible to distinguish from "not yet bootstrapped". +func TestStoreControlServerCredentialRejectsEmptyToken(t *testing.T) { + store := newInMemorySecretStore() + + if _, err := storeControlServerCredential(context.Background(), store, "", ""); err == nil { + t.Fatal("want error for empty token, got nil") + } +} + +// TestRevealControlServerCredentialDelegatesToStore pins the read side: the +// helper returns the decrypted token for a known reference. +func TestRevealControlServerCredentialDelegatesToStore(t *testing.T) { + store := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), store, "", "the-bearer-token") + if err != nil { + t.Fatalf("storeControlServerCredential: %v", err) + } + + got, err := revealControlServerCredential(context.Background(), store, secretID) + if err != nil { + t.Fatalf("revealControlServerCredential: %v", err) + } + if got != "the-bearer-token" { + t.Fatalf("got = %q, want %q", got, "the-bearer-token") + } +} + +// TestRevealControlServerCredentialRejectsEmptyReference pins the +// credential-unavailable case distinct from authentication failure (design +// 02's failure table): a control-server record with no credential reference +// at all must fail before ever contacting the secret store. +func TestRevealControlServerCredentialRejectsEmptyReference(t *testing.T) { + store := newInMemorySecretStore() + + if _, err := revealControlServerCredential(context.Background(), store, ""); err == nil { + t.Fatal("want error for empty secret reference, got nil") + } +} + +// TestRevealControlServerCredentialPropagatesNotFound pins that a stale +// reference (secret deleted out of band) surfaces as secrets.ErrNotFound so +// the caller can map it to the distinct credential-unavailable adoption +// refusal reason rather than a generic failure. +func TestRevealControlServerCredentialPropagatesNotFound(t *testing.T) { + store := newInMemorySecretStore() + + _, err := revealControlServerCredential(context.Background(), store, "does-not-exist") + if !errors.Is(err, secrets.ErrNotFound) { + t.Fatalf("err = %v, want wrapped %v", err, secrets.ErrNotFound) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_ownership_proof.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_ownership_proof.go new file mode 100644 index 0000000000..6cae8c5a72 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_ownership_proof.go @@ -0,0 +1,75 @@ +package lifecycle + +import ( + "context" + "crypto/rand" + "encoding/hex" + "time" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/ownershipproof" +) + +// newAdoptionChallenge returns a fresh challenge for one adoption attempt. +// A proof is only ever accepted for the challenge generated for that +// attempt, so this must never be derived from anything an observer could +// predict or replay. +func newAdoptionChallenge() (string, error) { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +// controlServerHoldsCredential reports whether the process answering the +// recorded endpoint can demonstrate it already holds credential. Until it +// does, this backend has established only that something answers and echoes +// the values the identity endpoint hands out unauthenticated; sending the +// credential to it, or storing a replacement it invents, would hand an +// installation-wide secret to whatever took the port. +// +// A failure to answer is not a proof: the caller refuses adoption. +func controlServerHoldsCredential( + ctx context.Context, + client AdoptionControlClient, + credential string, + binding string, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, +) bool { + challenge, err := newAdoptionChallenge() + if err != nil { + return false + } + + proofs, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, + func(ctx context.Context) ([]string, error) { + return client.ProveOwnership(ctx, challenge) + }) + if err != nil { + return false + } + + return ownershipproof.Matches(credential, challenge, binding, proofs) +} + +// recordedDiagnosticLogPath reads the adopted server's diagnostic log +// location, which AC-EXECUTORS-CONTROL-OWNERSHIP-001.1 requires the record +// to name. It is retrieved after authentication because the endpoint that +// decides whether to authenticate discloses no filesystem path. +func recordedDiagnosticLogPath( + ctx context.Context, + client AdoptionControlClient, + recoveryReadTimeout time.Duration, + recoveryReadRetries int, +) string { + details, err := withAdoptionRetry(ctx, recoveryReadTimeout, recoveryReadRetries, + func(ctx context.Context) (*agentctl.ServerDetails, error) { + return client.GetServerDetails(ctx) + }) + if err != nil || details == nil { + return "" + } + return details.DiagnosticLogPath +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_ownership_proof_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_ownership_proof_test.go new file mode 100644 index 0000000000..71a0085b24 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_ownership_proof_test.go @@ -0,0 +1,163 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" +) + +// seedProofFixture builds an adoption fixture whose durable record names a +// credential the secret store really holds, so every refusal these tests +// observe is the ownership gate refusing and not an earlier gate short- +// circuiting on a missing record, identity, or secret. +func seedProofFixture(t *testing.T, client *fakeAdoptionControlClient) (*fakeAdoptionRecordStore, AdoptionControlClientFactory, *inMemorySecretStore) { + t.Helper() + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "backend-credential") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + store, factory := adoptionFixture(t, record, client) + return store, factory, secretStore +} + +// TestAdoptionNeverSendsTheCredentialToAServerThatCannotProvePossession is +// the reason this gate exists. Everything a control server needs in order to +// be adopted up to this point -- the endpoint, the advertised identity, the +// capability set -- is either public or attacker-chosen, so a process that +// binds the recorded port and echoes the record's values satisfies all of +// it. Only possession of the credential distinguishes the real server, and +// the credential itself must never be the thing that demonstrates it: if a +// backend sends it first and reads the response, it has already handed a +// long-lived secret to whatever answered. +func TestAdoptionNeverSendsTheCredentialToAServerThatCannotProvePossession(t *testing.T) { + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "some-other-credential", + rotateResult: &agentctl.CredentialRotationResult{RotationID: 1, Credential: "rotated"}, + } + store, factory, secretStore := seedProofFixture(t, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted { + t.Fatalf("outcome = %+v, want unadopted", outcome) + } + if outcome.Reason != AdoptionReasonIdentityMismatch { + t.Fatalf("Reason = %q, want %q", outcome.Reason, AdoptionReasonIdentityMismatch) + } + if len(client.presentedTokens) != 0 { + t.Fatalf("credential was transmitted to an unproven server: %v", client.presentedTokens) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a server this backend cannot prove is its own") + } + if len(store.upserts) != 0 { + t.Fatalf("the record was rewritten for an unproven server: %+v", store.upserts) + } +} + +// TestReclaimNeverSendsTheCredentialToAServerThatCannotProvePossession +// covers the same gate on the path that runs with the capability OFF. That +// path reaches a foreign server through the identical sequence and then +// stops it, so leaving it ungated would keep the vulnerability reachable in +// every shipped profile. +func TestReclaimNeverSendsTheCredentialToAServerThatCannotProvePossession(t *testing.T) { + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "some-other-credential", + } + store, factory, secretStore := seedProofFixture(t, client) + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if len(client.presentedTokens) != 0 { + t.Fatalf("credential was transmitted to an unproven server: %v", client.presentedTokens) + } + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a server this backend cannot prove is its own") + } +} + +// TestAdoptionRefusesAServerThatWillNotAnswerTheChallenge pins the fail- +// closed direction: a server that errors on the proof request, or answers +// with nothing, has not proved possession, so it is refused exactly like one +// that answers wrongly. +func TestAdoptionRefusesAServerThatWillNotAnswerTheChallenge(t *testing.T) { + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "backend-credential", + proveErr: errors.New("404 page not found"), + } + store, factory, secretStore := seedProofFixture(t, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if outcome.Adopted { + t.Fatalf("outcome = %+v, want unadopted", outcome) + } + if len(client.presentedTokens) != 0 { + t.Fatalf("credential was transmitted despite no proof: %v", client.presentedTokens) + } +} + +// TestEachAdoptionAttemptIssuesAFreshChallenge pins that the challenge is +// per-attempt. A fixed challenge would let an attacker who once observed a +// valid answer replay it forever without ever holding the credential, which +// is the entire property the proof buys. +func TestEachAdoptionAttemptIssuesAFreshChallenge(t *testing.T) { + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "backend-credential", + rotateResult: &agentctl.CredentialRotationResult{RotationID: 1, Credential: "backend-credential"}, + } + store, factory, secretStore := seedProofFixture(t, client) + + for range 3 { + AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + } + + if len(client.challenges) != 3 { + t.Fatalf("challenges issued = %d, want 3 (one per attempt)", len(client.challenges)) + } + seen := make(map[string]bool, len(client.challenges)) + for _, challenge := range client.challenges { + if challenge == "" { + t.Fatal("an empty challenge was issued, which proves nothing") + } + if seen[challenge] { + t.Fatalf("challenge %q was reused across attempts", challenge) + } + seen[challenge] = true + } +} + +// TestAdoptionAcceptsAServerThatProvesPossession is the open side: the same +// gate that refuses everything above must let the real server through, or it +// would be indistinguishable from disabling adoption outright. +func TestAdoptionAcceptsAServerThatProvesPossession(t *testing.T) { + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "backend-credential", + rotateResult: &agentctl.CredentialRotationResult{RotationID: 4, Credential: "rotated-credential"}, + } + store, factory, secretStore := seedProofFixture(t, client) + + outcome := AttemptAdoptControlServer(context.Background(), store, secretStore, factory, + testHomeDir, RequiredSurvivalCapabilities, 0, -1, newAdoptionTestLogger(t)) + + if !outcome.Adopted { + t.Fatalf("outcome = %+v, want adopted", outcome) + } + if len(client.presentedTokens) == 0 || client.presentedTokens[0] != "backend-credential" { + t.Fatalf("presented tokens = %v, want the recorded credential first", client.presentedTokens) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/control_server_reclaim_test.go b/apps/backend/internal/agent/runtime/lifecycle/control_server_reclaim_test.go new file mode 100644 index 0000000000..7dc57b193b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/control_server_reclaim_test.go @@ -0,0 +1,187 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +func TestReclaimUnneededControlServerNoRecordDoesNothing(t *testing.T) { + store := &fakeAdoptionRecordStore{} + factory := func(string) (AdoptionControlClient, error) { + t.Fatal("client factory must not be called with no record") + return nil, nil + } + + ReclaimUnneededControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) +} + +func TestReclaimUnneededControlServerNothingAnswersDoesNothing(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{heldCredential: "own-token", identityErr: errors.New("connection refused")} + store, factory := adoptionFixture(t, record, client) + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a server that answered no identity") + } +} + +func TestReclaimUnneededControlServerIdentityMismatchLeavesServerUntouched(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{ + identity: validIdentity(), + heldCredential: "own-token", + boundHome: "/home/someone-else", + } + store, factory := adoptionFixture(t, record, client) + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a server belonging to another installation") + } + if len(client.presentedTokens) != 0 { + t.Fatalf("credential was sent to a server that could not prove possession: %v", client.presentedTokens) + } +} + +func TestReclaimUnneededControlServerServerIdentityMismatchLeavesServerUntouched(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + record.ServerIdentity = "recorded-identity" + identity := validIdentity() + identity.ServerIdentity = "a-different-live-identity" + client := &fakeAdoptionControlClient{heldCredential: "own-token", identity: identity} + store, factory := adoptionFixture(t, record, client) + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called on a ServerIdentity-mismatched server") + } +} + +func TestReclaimUnneededControlServerCredentialUnavailableLeavesServerUntouched(t *testing.T) { + record := validRecord() + record.CredentialSecretID = "does-not-exist" + client := &fakeAdoptionControlClient{identity: validIdentity()} + store, factory := adoptionFixture(t, record, client) + + ReclaimUnneededControlServer(context.Background(), store, newInMemorySecretStore(), factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if client.shutdownCalled { + t.Fatal("ShutdownControlServer was called despite the credential being unavailable") + } +} + +func TestReclaimUnneededControlServerStopsOwnServerAndRepairsRecords(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &fakeAdoptionControlClient{heldCredential: "own-token", identity: validIdentity()} + store, factory := adoptionFixture(t, record, client) + store.liveStandaloneRecords = []*models.ExecutorRunning{{SessionID: "session-1"}} + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if !client.shutdownCalled { + t.Fatal("ShutdownControlServer was not called for a proven-own detached server") + } + if client.rotateResult != nil || len(client.presentedTokens) == 0 || client.presentedTokens[0] != "own-token" { + t.Fatalf("presented tokens = %v, want the stored own-token used directly, never rotated", client.presentedTokens) + } + if len(store.repairedSessionIDs) != 1 || store.repairedSessionIDs[0] != "session-1" { + t.Fatalf("repaired sessions = %v, want session-1 repaired immediately after the successful stop", store.repairedSessionIDs) + } +} + +func TestReclaimUnneededControlServerRetriesStopAndSkipsRepairOnFailure(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &countingShutdownControlClient{ + fakeAdoptionControlClient: fakeAdoptionControlClient{ + heldCredential: "own-token", + identity: validIdentity(), + shutdownErr: errors.New("500 internal server error"), + }, + } + store, _ := adoptionFixture(t, record, &client.fakeAdoptionControlClient) + factory := func(string) (AdoptionControlClient, error) { return client, nil } + store.liveStandaloneRecords = []*models.ExecutorRunning{{SessionID: "session-1"}} + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if client.shutdownAttempts < 2 { + t.Fatalf("shutdown attempts = %d, want more than one (bounded retry)", client.shutdownAttempts) + } + if len(store.repairedSessionIDs) != 0 { + t.Fatalf("repaired sessions = %v, want none when every stop retry fails", store.repairedSessionIDs) + } +} + +func TestReclaimUnneededControlServerConnectionRefusedCountsAsSuccess(t *testing.T) { + secretStore := newInMemorySecretStore() + secretID, err := storeControlServerCredential(context.Background(), secretStore, "", "own-token") + if err != nil { + t.Fatalf("seed credential: %v", err) + } + record := validRecord() + record.CredentialSecretID = secretID + client := &countingShutdownControlClient{ + fakeAdoptionControlClient: fakeAdoptionControlClient{ + heldCredential: "own-token", + identity: validIdentity(), + shutdownErr: errors.New("dial tcp 127.0.0.1:9999: connect: connection refused"), + }, + } + store, _ := adoptionFixture(t, record, &client.fakeAdoptionControlClient) + factory := func(string) (AdoptionControlClient, error) { return client, nil } + store.liveStandaloneRecords = []*models.ExecutorRunning{{SessionID: "session-1"}} + + ReclaimUnneededControlServer(context.Background(), store, secretStore, factory, + testHomeDir, 0, -1, newAdoptionTestLogger(t)) + + if client.shutdownAttempts != 1 { + t.Fatalf("shutdown attempts = %d, want exactly 1: a connection-refused error must not be retried", client.shutdownAttempts) + } + if len(store.repairedSessionIDs) != 1 || store.repairedSessionIDs[0] != "session-1" { + t.Fatalf("repaired sessions = %v, want session-1 repaired: an already-absent server counts as a successful stop", store.repairedSessionIDs) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/execution_store.go b/apps/backend/internal/agent/runtime/lifecycle/execution_store.go index d0288fc694..da61b5e152 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/execution_store.go +++ b/apps/backend/internal/agent/runtime/lifecycle/execution_store.go @@ -323,6 +323,10 @@ func (s *ExecutionStore) BeginPrompt(executionID string) (uint64, error) { } func beginExecutionPrompt(execution *AgentExecution) uint64 { + // A prompt is about to be dispatched through this object, so a + // recovered-but-not-yet-adopted generation must not later clobber it with + // a stale pre-restart completion (see recoveredPromptGenerationPending). + execution.recoveredPromptGenerationPending.Store(false) execution.promptGeneration++ execution.promptCompletionGeneration = 0 // The new generation is not dispatched until its triggerPrompt succeeds; a diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go b/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go index 0345ced837..8f7b096ff5 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go @@ -147,8 +147,11 @@ type ExecutorBackend interface { StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error // RecoverInstances discovers and recovers instances that were running before a restart. + // records is the live standalone recovery-inventory read at startup step 3 + // (Manager.ListLiveStandaloneExecutorsRunning); only the standalone runtime + // acts on it today, every other implementation ignores it. // Returns recovered instances that can be re-tracked by the manager. - RecoverInstances(ctx context.Context) ([]*ExecutorInstance, error) + RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) // GetInteractiveRunner returns the interactive runner for passthrough mode. // May return nil if the runtime doesn't support passthrough mode. @@ -256,7 +259,12 @@ const ( MetadataKeyModelOverride = "model_override" // Office metadata keys - MetadataKeySkillManifestJSON = "skill_manifest_json" + MetadataKeySkillManifestJSON = "skill_manifest_json" + // MetadataKeyOfficeAgentProfileID persists AgentExecution.OfficeAgentProfileID + // (AC-EXECUTORS-SURVIVAL-002.14's "Office profile identity" reconstruction + // row): a new key in the existing metadata column rather than a schema + // change. Empty for every non-Office launch, which is legitimate, not + // missing. MetadataKeyOfficeAgentProfileID = "office_agent_profile_id" // SSH runtime metadata keys (per-session, except SSHWorkdirRoot which is per-profile). @@ -661,6 +669,34 @@ type ExecutorInstance struct { StopReason string AgentStopFailed bool + // Env is the adopted instance's own runtime environment, read back rather + // than pushed (AC-EXECUTORS-SURVIVAL-002.14's "runtime environment" row: + // deliberately memory-only, so it is only ever populated by a recovery + // path -- StandaloneExecutor.RecoverInstances -- reading it back from the + // live instance; a fresh launch's environment flows through + // ExecutorCreateRequest.Env instead, never through this field. + Env map[string]string + + // WorkspaceSourceRoots is the adopted instance's own live source-root + // allowlist, read back rather than pushed (AC-EXECUTORS-SURVIVAL-002.14's + // "workspace source roots" row). Same recovery-only shape as Env above. + WorkspaceSourceRoots []string + + // ProviderSessionID is the adopted instance's own live agent-CLI session + // identity, read back rather than pushed (AC-EXECUTORS-SURVIVAL-002.14's + // "provider session identity" row). Same recovery-only shape as Env above. + ProviderSessionID string + + // AgentProfileID is a recovery-only carrier for + // AC-EXECUTORS-SURVIVAL-002.14's "agent profile identity" row: the + // recovery-inventory record's execution-profile column, read by + // StandaloneExecutor.buildRecoveredInstances so the recovery loop can set + // AgentExecution.AgentProfileID before re-deriving agent identity, without + // waiting on persistExecutorRunningResult's separate DB-backed self-heal. + // A fresh launch's profile flows through ExecutorCreateRequest.AgentProfileID + // instead, never through this field. + AgentProfileID string + // AuthToken is the agentctl auth token retrieved via handshake. // Populated by authenticated container/remote executors for encrypted storage in SecretStore. // Empty for standalone (launcher-owned token wired via cfg.Agent.StandaloneAuthToken) diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_docker.go b/apps/backend/internal/agent/runtime/lifecycle/executor_docker.go index 5a2cf44b4c..9bdd40d8e9 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_docker.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_docker.go @@ -771,7 +771,7 @@ func dockerCleanupContext(ctx context.Context, agentStopFailed bool) (context.Co return ctx, func() {} } -func (r *DockerExecutor) RecoverInstances(_ context.Context) ([]*ExecutorInstance, error) { +func (r *DockerExecutor) RecoverInstances(_ context.Context, _ []*models.ExecutorRunning) ([]*ExecutorInstance, error) { // No-op: Docker client is initialized lazily on first use. // If no session has used Docker yet, there's nothing to recover. // Running containers from a previous backend process will be detected diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_docker_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_docker_test.go index 7cbe8fa270..7fa1a916a7 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_docker_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_docker_test.go @@ -109,7 +109,7 @@ func TestDockerExecutor_RecoverInstances(t *testing.T) { log := newTestDockerLogger() exec := NewDockerExecutor(config.DockerConfig{}, "", log) - instances, err := exec.RecoverInstances(context.Background()) + instances, err := exec.RecoverInstances(context.Background(), nil) if err != nil { t.Errorf("expected nil error, got: %v", err) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go index 643973036a..234a5dcb04 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go @@ -20,6 +20,7 @@ import ( agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agentctl/server/process" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" ) const ( @@ -612,7 +613,7 @@ func kubernetesProfileConfigFromMetadata(metadata map[string]interface{}) (kubee return kubeexecutor.ParseProfileConfig(values) } -func (r *KubernetesExecutor) RecoverInstances(context.Context) ([]*ExecutorInstance, error) { +func (r *KubernetesExecutor) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*ExecutorInstance, error) { return nil, nil } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_registry.go b/apps/backend/internal/agent/runtime/lifecycle/executor_registry.go index 78cc2bb758..b4481dbbea 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_registry.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_registry.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/agent/executor" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" ) // ErrExecutorNotFound is returned when a runtime doesn't exist in the registry. @@ -116,10 +117,13 @@ func (r *ExecutorRegistry) CloseAll() { } } -// RecoverAll recovers instances from all registered runtimes. +// RecoverAll recovers instances from all registered runtimes. records is the +// live standalone recovery-inventory read at startup step 3 +// (AC-EXECUTORS-SURVIVAL-002.8); it is passed to every registered runtime +// unchanged, and only the standalone runtime acts on it today. // Returns all recovered instances and any error encountered. // If multiple runtimes fail, only the last error is returned. -func (r *ExecutorRegistry) RecoverAll(ctx context.Context) ([]*ExecutorInstance, error) { +func (r *ExecutorRegistry) RecoverAll(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) { r.mu.RLock() backends := make(map[executor.Name]ExecutorBackend, len(r.backends)) for name, rt := range r.backends { @@ -131,7 +135,7 @@ func (r *ExecutorRegistry) RecoverAll(ctx context.Context) ([]*ExecutorInstance, var lastErr error for name, rt := range backends { - instances, err := rt.RecoverInstances(ctx) + instances, err := rt.RecoverInstances(ctx, records) if err != nil { r.logger.Error("failed to recover instances from runtime", zap.String("runtime", string(name)), diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_registry_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_registry_test.go index e362eed719..f449897de2 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_registry_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_registry_test.go @@ -9,6 +9,7 @@ import ( "github.com/kandev/kandev/internal/agent/executor" "github.com/kandev/kandev/internal/agentctl/server/process" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" ) // MockExecutor implements Runtime interface for testing @@ -17,6 +18,13 @@ type MockExecutor struct { healthCheckErr error recoverInstances []*ExecutorInstance recoverErr error + // recoverRecords captures the records RecoverInstances was called with, + // so tests can assert RecoverAll passes them through unchanged. + recoverRecords []*models.ExecutorRunning + // stopInstanceCalls records every instance StopInstance was called with, + // so tests can assert a refused recovered instance was actually stopped. + stopInstanceCalls []*ExecutorInstance + stopInstanceErr error } func (m *MockExecutor) Name() executor.Name { return m.name } @@ -27,9 +35,11 @@ func (m *MockExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateRe return nil, nil } func (m *MockExecutor) StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error { - return nil + m.stopInstanceCalls = append(m.stopInstanceCalls, instance) + return m.stopInstanceErr } -func (m *MockExecutor) RecoverInstances(ctx context.Context) ([]*ExecutorInstance, error) { +func (m *MockExecutor) RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) { + m.recoverRecords = records return m.recoverInstances, m.recoverErr } func (m *MockExecutor) GetInteractiveRunner() *process.InteractiveRunner { @@ -224,7 +234,7 @@ func TestExecutorRegistry_RecoverAll(t *testing.T) { registry.Register(&MockExecutor{name: executor.NameDocker, recoverInstances: nil}) registry.Register(&MockExecutor{name: executor.NameStandalone, recoverInstances: []*ExecutorInstance{}}) - instances, err := registry.RecoverAll(ctx) + instances, err := registry.RecoverAll(ctx, nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -248,7 +258,7 @@ func TestExecutorRegistry_RecoverAll(t *testing.T) { registry.Register(&MockExecutor{name: executor.NameDocker, recoverInstances: dockerInstances}) registry.Register(&MockExecutor{name: executor.NameStandalone, recoverInstances: standaloneInstances}) - instances, err := registry.RecoverAll(ctx) + instances, err := registry.RecoverAll(ctx, nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -269,7 +279,7 @@ func TestExecutorRegistry_RecoverAll(t *testing.T) { registry.Register(&MockExecutor{name: executor.NameDocker, recoverErr: recoverErr}) registry.Register(&MockExecutor{name: executor.NameStandalone, recoverInstances: standaloneInstances}) - instances, err := registry.RecoverAll(ctx) + instances, err := registry.RecoverAll(ctx, nil) // Should still get instances from successful runtimes // Note: map iteration order is not guaranteed, so we check that at least standalone instances are recovered if len(instances) == 0 && err == nil { @@ -283,7 +293,7 @@ func TestExecutorRegistry_RecoverAll(t *testing.T) { log := newTestRegistryLogger() registry := NewExecutorRegistry(log) - instances, err := registry.RecoverAll(ctx) + instances, err := registry.RecoverAll(ctx, nil) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -291,6 +301,33 @@ func TestExecutorRegistry_RecoverAll(t *testing.T) { t.Errorf("expected no instances, got %d", len(instances)) } }) + + // TestExecutorRegistry_RecoverAllPassesRecordsThroughUnchanged pins that + // the live-standalone recovery-inventory records read at startup step 3 + // reach every registered runtime's RecoverInstances unchanged, even one + // that ultimately ignores them (design 03's "producer receives records + // as a parameter"; five of six executors are exactly such an ignorer). + t.Run("records pass through unchanged to every runtime", func(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + + docker := &MockExecutor{name: executor.NameDocker} + standalone := &MockExecutor{name: executor.NameStandalone} + registry.Register(docker) + registry.Register(standalone) + + records := []*models.ExecutorRunning{{ID: "row-1", SessionID: "session-1"}} + if _, err := registry.RecoverAll(ctx, records); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(docker.recoverRecords) != 1 || docker.recoverRecords[0] != records[0] { + t.Errorf("docker executor recoverRecords = %v, want the same records slice", docker.recoverRecords) + } + if len(standalone.recoverRecords) != 1 || standalone.recoverRecords[0] != records[0] { + t.Errorf("standalone executor recoverRecords = %v, want the same records slice", standalone.recoverRecords) + } + }) } func TestExecutorRegistry_ThreadSafety(t *testing.T) { diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_remote_docker.go b/apps/backend/internal/agent/runtime/lifecycle/executor_remote_docker.go index e879fd39b4..2f339b95eb 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_remote_docker.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_remote_docker.go @@ -9,6 +9,7 @@ import ( "github.com/kandev/kandev/internal/agent/executor" "github.com/kandev/kandev/internal/agentctl/server/process" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" ) // RemoteDockerExecutor implements Runtime for remote Docker-based agent execution. @@ -57,7 +58,7 @@ func (r *RemoteDockerExecutor) StopInstance(ctx context.Context, instance *Execu return fmt.Errorf("remote_docker runtime is not yet implemented") } -func (r *RemoteDockerExecutor) RecoverInstances(ctx context.Context) ([]*ExecutorInstance, error) { +func (r *RemoteDockerExecutor) RecoverInstances(ctx context.Context, _ []*models.ExecutorRunning) ([]*ExecutorInstance, error) { // Remote docker instances are not recovered on restart. // The containers on remote hosts are ephemeral. return nil, nil diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_running_live_standalone_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_running_live_standalone_test.go new file mode 100644 index 0000000000..3e2a863a94 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_running_live_standalone_test.go @@ -0,0 +1,56 @@ +package lifecycle + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// listingWriter is a fake ExecutorRunningWriter that also implements the +// optional executorRunningLister capability, so a test can pin +// Manager.ListLiveStandaloneExecutorsRunning's delegation without a real DB. +type listingWriter struct { + invariantWriter + rows []*models.ExecutorRunning + err error +} + +func (w *listingWriter) ListExecutorsRunningLiveStandalone(context.Context) ([]*models.ExecutorRunning, error) { + return w.rows, w.err +} + +// TestListLiveStandaloneExecutorsRunningDelegatesToWriter pins that the +// manager forwards to the writer's optional lister capability when present. +// This inventory is read at startup step 3, before any control-server +// contact, to build the recovery guard/correlation set (discovery H). +func TestListLiveStandaloneExecutorsRunningDelegatesToWriter(t *testing.T) { + mgr := newTestManager(t) + want := []*models.ExecutorRunning{{SessionID: "session-1"}} + mgr.SetExecutorRunningWriter(&listingWriter{rows: want}) + + got, err := mgr.ListLiveStandaloneExecutorsRunning(context.Background()) + if err != nil { + t.Fatalf("ListLiveStandaloneExecutorsRunning: %v", err) + } + if len(got) != 1 || got[0].SessionID != "session-1" { + t.Fatalf("got = %#v, want %#v", got, want) + } +} + +// TestListLiveStandaloneExecutorsRunningWithoutListerReturnsEmpty pins the +// best-effort fallback: a writer that doesn't support listing (e.g. a test +// double) yields no candidates rather than an error, matching the rest of +// this file's optional-interface pattern. +func TestListLiveStandaloneExecutorsRunningWithoutListerReturnsEmpty(t *testing.T) { + mgr := newTestManager(t) + mgr.SetExecutorRunningWriter(&invariantWriter{}) + + got, err := mgr.ListLiveStandaloneExecutorsRunning(context.Background()) + if err != nil { + t.Fatalf("ListLiveStandaloneExecutorsRunning: %v", err) + } + if len(got) != 0 { + t.Fatalf("got = %#v, want empty", got) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_sprites.go b/apps/backend/internal/agent/runtime/lifecycle/executor_sprites.go index 32a706700d..ba9ac4e948 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_sprites.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_sprites.go @@ -587,7 +587,7 @@ func (r *SpritesExecutor) buildInstanceResult( } } -func (r *SpritesExecutor) RecoverInstances(_ context.Context) ([]*ExecutorInstance, error) { +func (r *SpritesExecutor) RecoverInstances(_ context.Context, _ []*models.ExecutorRunning) ([]*ExecutorInstance, error) { return nil, nil } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go index 9f6fe53286..5493fe6e0e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go @@ -86,7 +86,7 @@ func TestSpritesExecutorStaticSurface(t *testing.T) { if exec.GetInteractiveRunner() != nil { t.Fatal("sprites have no host-side interactive runner") } - instances, err := exec.RecoverInstances(context.Background()) + instances, err := exec.RecoverInstances(context.Background(), nil) if err != nil || instances != nil { t.Fatalf("RecoverInstances() = %v, %v", instances, err) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go index 7114b0afe2..e5b316d066 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh.go @@ -912,7 +912,7 @@ func instanceID(instance *ExecutorInstance) string { // Recovery semantics are documented in the spec; persisted metadata keys // (ssh_host / ssh_user / ssh_remote_agentctl_port / etc.) are honored by // ResumeRemoteInstance below. -func (r *SSHExecutor) RecoverInstances(_ context.Context) ([]*ExecutorInstance, error) { +func (r *SSHExecutor) RecoverInstances(_ context.Context, _ []*models.ExecutorRunning) ([]*ExecutorInstance, error) { return nil, nil } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go index ee6633e67e..307097e7f7 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go @@ -76,7 +76,7 @@ func TestSSHExecutorStaticSurface(t *testing.T) { if exec.GetInteractiveRunner() != nil { t.Fatal("SSH has no host-side interactive runner") } - instances, err := exec.RecoverInstances(context.Background()) + instances, err := exec.RecoverInstances(context.Background(), nil) if err != nil || instances != nil { t.Fatalf("RecoverInstances() = %v, %v; want nil, nil", instances, err) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone.go b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone.go index fee46b5c06..eb387a52af 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone.go @@ -3,6 +3,7 @@ package lifecycle import ( "context" "fmt" + "sync" "time" "go.uber.org/zap" @@ -13,17 +14,49 @@ import ( "github.com/kandev/kandev/internal/agentctl/server/process" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/common/subproc" + "github.com/kandev/kandev/internal/task/models" ) +// defaultRecoveryReadTimeout and defaultRecoveryReadRetries are the +// AC-EXECUTORS-SURVIVAL-002.13 defaults (two seconds, two retries), used +// whenever SetRecoveryRetryConfig has not installed a configured value. +const ( + defaultRecoveryReadTimeout = 2 * time.Second + defaultRecoveryReadRetries = 2 +) + +// UnstoppableSessionRecorder is the recovery-guard sink StandaloneExecutor +// drives for a session's not-re-tracked outcome during recovery: retains a +// session's guard for the rest of this backend's lifetime when +// AC-EXECUTORS-SURVIVAL-002.16 applies (at least one live instance could not +// be stopped despite AC-EXECUTORS-SURVIVAL-002.15's bounded retry), marks a +// session whose stop is still resolving when the AC-EXECUTORS-SURVIVAL-003.7 +// recovery deadline elapses mid-retry, and releases that guard once the +// resolution is known. Satisfied structurally by *lifecycle.RecoveryGuard +// (see Manager.RecoveryGuard). +type UnstoppableSessionRecorder interface { + RetainAsUnstoppable(sessionID string) + // MarkStopInFlight records that sessionID's guard must survive + // ReleaseAllExceptRetained until its still-resolving stop is known, + // rather than being released at the deadline bound. + MarkStopInFlight(sessionID string) + // Release drops sessionID's guard once its resolution is known. A no-op + // for a session RetainAsUnstoppable has already claimed. + Release(sessionID string) +} + // StandaloneExecutor implements Runtime for standalone agentctl execution. // In this mode, a single agentctl control server manages multiple agent instances. type StandaloneExecutor struct { - ctl *agentctl.ControlClient - host string - port int - authToken string // per-launch auth token from launcher - logger *logger.Logger - interactiveRunner *process.InteractiveRunner + ctl *agentctl.ControlClient + host string + port int + authToken string // per-launch auth token from launcher + logger *logger.Logger + interactiveRunner *process.InteractiveRunner + recoveryReadTimeout time.Duration + recoveryReadRetries int + unstoppableRecorder UnstoppableSessionRecorder } // NewStandaloneExecutor creates a new standalone runtime. @@ -33,6 +66,10 @@ func NewStandaloneExecutor(ctl *agentctl.ControlClient, host string, port int, l host: host, port: port, logger: log.WithFields(zap.String("runtime", "standalone")), + // -1 is the "unset" sentinel: zero is itself a valid configured + // retry count (AC-EXECUTORS-SURVIVAL-002.13 allows zero retries), so + // the zero value of an unset int field can't be used to mean unset. + recoveryReadRetries: -1, } } @@ -41,6 +78,78 @@ func (r *StandaloneExecutor) SetAuthToken(token string) { r.authToken = token } +// SetRecoveryRetryConfig installs the bounded per-attempt timeout and retry +// count AC-EXECUTORS-SURVIVAL-002.13/002.15 require for recovery reads and +// stops. A zero timeout or negative retry count falls back to the two-second/ +// two-retry default the AC itself specifies. +func (r *StandaloneExecutor) SetRecoveryRetryConfig(timeout time.Duration, retries int) { + r.recoveryReadTimeout = timeout + r.recoveryReadRetries = retries +} + +// SetUnstoppableSessionRecorder installs the AC-EXECUTORS-SURVIVAL-002.16 sink. +// Unset means an unstoppable instance is only logged, never retained -- +// acceptable for callers that haven't wired a recovery guard. +func (r *StandaloneExecutor) SetUnstoppableSessionRecorder(recorder UnstoppableSessionRecorder) { + r.unstoppableRecorder = recorder +} + +// stopWithRetry stops instanceID within the bounded per-attempt timeout and +// retry count of AC-EXECUTORS-SURVIVAL-002.13/002.15, with no delay between +// attempts. DeleteInstance itself already treats an already-absent instance +// (404) as success. +func (r *StandaloneExecutor) stopWithRetry(ctx context.Context, instanceID string) error { + timeout := r.recoveryReadTimeout + if timeout <= 0 { + timeout = defaultRecoveryReadTimeout + } + retries := r.recoveryReadRetries + if retries < 0 { + retries = defaultRecoveryReadRetries + } + + var lastErr error + for attempt := 0; attempt <= retries; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + lastErr = r.ctl.DeleteInstance(attemptCtx, instanceID) + cancel() + if lastErr == nil { + return nil + } + } + return lastErr +} + +// listInstancesWithRetry enumerates the adopted control server's live +// instances within the same bounded per-attempt timeout and retry count as +// stopWithRetry (AC-EXECUTORS-SURVIVAL-002.13): this enumeration is the sole +// read source for two AC-EXECUTORS-SURVIVAL-002.3 reconstruction-table +// values -- workspace source roots and provider session identity -- both +// declared to come from the adopted instance, so a transient failure here +// must be retried before an instance's reconstruction is given up on. +func (r *StandaloneExecutor) listInstancesWithRetry(ctx context.Context) ([]*agentctl.InstanceInfo, error) { + timeout := r.recoveryReadTimeout + if timeout <= 0 { + timeout = defaultRecoveryReadTimeout + } + retries := r.recoveryReadRetries + if retries < 0 { + retries = defaultRecoveryReadRetries + } + + var lastErr error + for attempt := 0; attempt <= retries; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + instances, err := r.ctl.ListInstances(attemptCtx) + cancel() + if err == nil { + return instances, nil + } + lastErr = err + } + return nil, lastErr +} + func (r *StandaloneExecutor) Name() executor.Name { return executor.NameStandalone } @@ -217,10 +326,320 @@ func (r *StandaloneExecutor) StopInstance(ctx context.Context, instance *Executo return nil } -func (r *StandaloneExecutor) RecoverInstances(ctx context.Context) ([]*ExecutorInstance, error) { - // Standalone instances are not persisted - they are transient processes - // managed by agentctl. Session resume will restart them as needed. - return nil, nil +// RecoverInstances enumerates the adopted control server's live instances and +// correlates them to the live standalone recovery-inventory records read at +// startup step 3 (AC-EXECUTORS-SURVIVAL-002.1/002.8), applying the +// AC-EXECUTORS-SURVIVAL-002.10 duplicate tiebreak via CorrelateRecoveryInstances. +// Every losing duplicate, ambiguous-session instance, and record-less orphan +// (AC-EXECUTORS-SURVIVAL-002.6) is stopped within the bounded retry budget of +// AC-EXECUTORS-SURVIVAL-002.13/002.15 (SetRecoveryRetryConfig); every winner +// is returned for the caller to re-track. When a losing duplicate's stop +// exhausts its retries, that session's winner is also stopped and dropped +// from the result rather than re-tracked (AC-EXECUTORS-SURVIVAL-002.15); if +// the winner's own stop then also fails, the session is reported to the +// installed UnstoppableSessionRecorder (AC-EXECUTORS-SURVIVAL-002.16). +// +// Every stop runs in its own goroutine against context.Background(), never +// ctx: if ctx carries a deadline (AC-EXECUTORS-SURVIVAL-003.7) and it elapses +// while stops are still outstanding, this stops WAITING for them -- treating +// every session whose losing duplicate hasn't yet resolved as not re-tracked +// right now -- but never cancels work already dispatched. Outstanding stops, +// and any winner-stop or unstoppable report they trigger, keep running and +// recording their outcome in the background after this call returns. +// +// Enumeration itself is retried within the AC-EXECUTORS-SURVIVAL-002.13 +// bounded budget (listInstancesWithRetry) before being treated as failed -- +// workspace source roots and provider session identity are read back from +// this same response, so a transient enumeration failure must not skip the +// retry that AC applies to every other adopted-instance read. When the +// adopted server still cannot be enumerated after retries are exhausted, +// this reports nothing recovered, stops nothing, and leaves every record to +// the existing stale-execution repair path (AC-EXECUTORS-SURVIVAL-002.12) +// rather than treating an enumeration failure as though every instance were +// an orphan. +func (r *StandaloneExecutor) RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) { + instances, err := r.listInstancesWithRetry(ctx) + if err != nil { + r.logger.Warn("failed to enumerate standalone instances for recovery; leaving every record to the existing repair path", + zap.Error(err)) + return nil, nil + } + + correlation := CorrelateRecoveryInstances(records, instances) + + // winnersBySession is a stable snapshot of every session's recovered + // winner, independent of correlation.Winners (which this call mutates + // and returns): AC-EXECUTORS-SURVIVAL-003.7 requires an in-flight loser + // stop, once dispatched, to still be able to find and stop its session's + // winner even after that session has already been dropped from the + // returned map at the deadline. + winnersBySession := make(map[string]*agentctl.InstanceInfo, len(correlation.Winners)) + for sessionID, winner := range correlation.Winners { + winnersBySession[sessionID] = winner + } + + pending := pendingLoserCounts(correlation.ToStop, correlation.Winners) + results := r.dispatchRecoveryStops(correlation.ToStop) + tracker := &jointFailureTracker{exec: r, winners: winnersBySession} + r.collectRecoveryStops(ctx, results, correlation.Winners, pending, tracker) + + return r.buildRecoveredInstances(correlation.Winners, indexRecordsBySession(records)), nil +} + +// recoveryStopOutcome is one instance's bounded stop attempt result. +type recoveryStopOutcome struct { + inst *agentctl.InstanceInfo + err error +} + +// dispatchRecoveryStops starts one goroutine per not-re-tracked instance, +// each stopping it within the bounded retry budget against +// context.Background() -- never the caller's ctx, so an elapsed recovery +// deadline can never abort a stop already in flight +// (AC-EXECUTORS-SURVIVAL-003.7). The returned channel is closed once every +// goroutine has reported. +func (r *StandaloneExecutor) dispatchRecoveryStops(toStop []*agentctl.InstanceInfo) <-chan recoveryStopOutcome { + results := make(chan recoveryStopOutcome, len(toStop)) + var wg sync.WaitGroup + for _, inst := range toStop { + wg.Add(1) + go func(inst *agentctl.InstanceInfo) { + defer wg.Done() + results <- recoveryStopOutcome{inst: inst, err: r.stopWithRetry(context.Background(), inst.ID)} + }(inst) + } + go func() { + wg.Wait() + close(results) + }() + return results +} + +// pendingLoserCounts counts, per session with a winner, how many of its +// losing duplicates have not yet reported a stop outcome. +func pendingLoserCounts(toStop []*agentctl.InstanceInfo, winners map[string]*agentctl.InstanceInfo) map[string]int { + pending := make(map[string]int, len(winners)) + for _, inst := range toStop { + if _, hasWinner := winners[inst.SessionID]; hasWinner { + pending[inst.SessionID]++ + } + } + return pending +} + +// collectRecoveryStops waits for dispatchRecoveryStops's results, applying +// each as it arrives, until either every result has landed or ctx's deadline +// (if any) elapses first. On deadline elapse it synchronously drops every +// session with an outstanding loser stop from winners -- AC-EXECUTORS-SURVIVAL-003.7's +// "treated as not-re-tracked" -- then keeps draining the remaining results in +// the background so their outcomes (including a later AC-EXECUTORS-SURVIVAL-002.16 +// report) still get recorded. +// +// winners is only ever mutated from this call's own goroutine, and only +// before this function returns: the caller iterates it immediately after +// (buildRecoveredInstances), and the background drain goroutine started at +// the deadline branch never touches it, to avoid a concurrent map +// read/write with that iteration. +func (r *StandaloneExecutor) collectRecoveryStops( + ctx context.Context, + results <-chan recoveryStopOutcome, + winners map[string]*agentctl.InstanceInfo, + pending map[string]int, + tracker *jointFailureTracker, +) { + deadlineCh, stopTimer := recoveryDeadlineChannel(ctx) + defer stopTimer() + + for { + select { + case res, ok := <-results: + if !ok { + return + } + r.handleRecoveryStopResult(res, pending, tracker) + if res.err != nil { + delete(winners, res.inst.SessionID) + } + case <-deadlineCh: + for sessionID, n := range pending { + if n > 0 { + delete(winners, sessionID) + // AC-EXECUTORS-SURVIVAL-003.7: this session's stop is still + // retrying past the deadline -- its guard must survive + // ReleaseAllExceptRetained until drainRecoveryStopsAfterDeadline + // below learns the resolution and releases (or retains) it. + if r.unstoppableRecorder != nil { + r.unstoppableRecorder.MarkStopInFlight(sessionID) + } + } + } + go r.drainRecoveryStopsAfterDeadline(results, pending, tracker) + return + } + } +} + +// drainRecoveryStopsAfterDeadline finishes applying every recovery stop +// outcome still outstanding once collectRecoveryStops has already returned +// past the AC-EXECUTORS-SURVIVAL-003.7 deadline. Each affected session's +// MarkStopInFlight guard is released once every one of its pending stops has +// resolved: either directly here (every loser stopped, nothing left to +// jointly stop) or by stopWinnerAfterLoserFailure's own terminal Release/ +// RetainAsUnstoppable call -- so the "retain until that stop resolves" +// exception never outlives the actual resolution. Release is safe to call a +// second time here even after stopWinnerAfterLoserFailure already resolved +// the guard: it no-ops for an already-released or already-retained session. +func (r *StandaloneExecutor) drainRecoveryStopsAfterDeadline( + results <-chan recoveryStopOutcome, + pending map[string]int, + tracker *jointFailureTracker, +) { + for res := range results { + r.handleRecoveryStopResult(res, pending, tracker) + if r.unstoppableRecorder != nil && pending[res.inst.SessionID] == 0 { + r.unstoppableRecorder.Release(res.inst.SessionID) + } + } +} + +// recoveryDeadlineChannel returns a channel that fires at ctx's deadline, or +// a nil channel (which never fires) when ctx carries none. +func recoveryDeadlineChannel(ctx context.Context) (<-chan time.Time, func()) { + deadline, ok := ctx.Deadline() + if !ok { + return nil, func() {} + } + timer := time.NewTimer(time.Until(deadline)) + return timer.C, func() { timer.Stop() } +} + +// handleRecoveryStopResult applies one stop outcome: on success it only +// clears the session's pending-loser count; on failure it also logs and, for +// a losing duplicate whose session has a winner, triggers the joint failure +// handling (AC-EXECUTORS-SURVIVAL-002.15). +func (r *StandaloneExecutor) handleRecoveryStopResult(res recoveryStopOutcome, pending map[string]int, tracker *jointFailureTracker) { + sessionID := res.inst.SessionID + if _, tracked := pending[sessionID]; tracked { + pending[sessionID]-- + } + if res.err == nil { + return + } + r.logger.Warn("failed to stop a not-re-tracked standalone instance during recovery after exhausting retries", + zap.String("instance_id", res.inst.ID), zap.String("session_id", sessionID), zap.Error(res.err)) + tracker.onLoserStopFailed(sessionID) +} + +// jointFailureTracker ensures AC-EXECUTORS-SURVIVAL-002.15/002.16's joint +// winner-stop-and-possible-unstoppable-report handling runs at most once per +// session, whether triggered synchronously (before the recovery deadline) or +// from the background drain after it (AC-EXECUTORS-SURVIVAL-003.7). +type jointFailureTracker struct { + exec *StandaloneExecutor + winners map[string]*agentctl.InstanceInfo + handled sync.Map // sessionID -> *sync.Once +} + +func (t *jointFailureTracker) onLoserStopFailed(sessionID string) { + winner, ok := t.winners[sessionID] + if !ok { + return // orphan, or a session with no winner: nothing to jointly affect + } + once, _ := t.handled.LoadOrStore(sessionID, &sync.Once{}) + once.(*sync.Once).Do(func() { + // The loser has already proved that this session cannot be made + // single-owner. Retain the guard before stopping the winner so a + // successful winner stop cannot release the protection for a still-live + // loser. + if t.exec.unstoppableRecorder != nil { + t.exec.unstoppableRecorder.RetainAsUnstoppable(sessionID) + } + t.exec.stopWinnerAfterLoserFailure(sessionID, winner) + }) +} + +// stopWinnerAfterLoserFailure stops a session's winning instance on the same +// bounded retry terms after one of its losing duplicates could not be +// stopped. The caller retains the session guard before entering this method, +// and that retention survives either winner-stop result (AC-EXECUTORS- +// SURVIVAL-002.16). Always runs to completion once started, even past the +// recovery deadline (AC-EXECUTORS-SURVIVAL-003.7). +func (r *StandaloneExecutor) stopWinnerAfterLoserFailure(sessionID string, winner *agentctl.InstanceInfo) { + if err := r.stopWithRetry(context.Background(), winner.ID); err != nil { + r.logger.Warn("failed to stop the winning instance after its losing duplicate could not be stopped; retaining session as unstoppable", + zap.String("instance_id", winner.ID), zap.String("session_id", sessionID), zap.Error(err)) + return + } + r.logger.Warn("stopped the winning instance because its losing duplicate could not be stopped; session left not re-tracked", + zap.String("instance_id", winner.ID), zap.String("session_id", sessionID)) +} + +// indexRecordsBySession maps every named recovery-inventory record by +// session ID for use while building recovered executions. +func indexRecordsBySession(records []*models.ExecutorRunning) map[string]*models.ExecutorRunning { + recordBySession := make(map[string]*models.ExecutorRunning, len(records)) + for _, rec := range records { + if rec != nil && rec.SessionID != "" { + recordBySession[rec.SessionID] = rec + } + } + return recordBySession +} + +// buildRecoveredInstances converts every still-winning correlated instance +// into an ExecutorInstance for the caller to re-track. +func (r *StandaloneExecutor) buildRecoveredInstances( + winners map[string]*agentctl.InstanceInfo, + recordBySession map[string]*models.ExecutorRunning, +) []*ExecutorInstance { + recovered := make([]*ExecutorInstance, 0, len(winners)) + for sessionID, inst := range winners { + record := recordBySession[sessionID] + + client := agentctl.NewClient(r.host, inst.Port, r.logger, + agentctl.WithExecutionID(inst.ID), + agentctl.WithSessionID(sessionID), + agentctl.WithAuthToken(r.authToken)) + + // AC-EXECUTORS-SURVIVAL-002.14: task identity and workspace path's + // declared source is the recovery-inventory record, never the + // adopted instance -- an instance must not be able to influence what + // the backend believes either belongs to. A record with no TaskID + // (or, defensively, no record at all -- correlation should never + // produce a winner without one) is left empty here rather than + // falling back to the instance's self-reported value; the caller + // routes an empty TaskID to the AC-EXECUTORS-SURVIVAL-002.4 refusal + // path instead of tracking a session it cannot safely operate. + var taskID, workspacePath string + var metadata map[string]interface{} + var agentProfileID string + if record != nil { + taskID = record.TaskID + workspacePath = record.WorktreePath + metadata = record.Metadata + // AC-EXECUTORS-SURVIVAL-002.14: agent profile identity's declared + // source is the recovery-inventory record's execution-profile + // column -- never the adopted instance. + agentProfileID = record.ExecutionProfileID + } + + recovered = append(recovered, &ExecutorInstance{ + InstanceID: inst.ID, + TaskID: taskID, + SessionID: sessionID, + AgentProfileID: agentProfileID, + RuntimeName: r.Name(), + Client: client, + StandaloneInstanceID: inst.ID, + StandalonePort: inst.Port, + WorkspacePath: workspacePath, + Metadata: metadata, + Env: inst.Env, + WorkspaceSourceRoots: inst.WorkspaceSourceRoots, + ProviderSessionID: inst.ProviderSessionID, + }) + } + return recovered } // SetInteractiveRunner sets the interactive runner for passthrough mode. diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go index 7a0e9d4509..be46aa9477 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go @@ -3,6 +3,7 @@ package lifecycle import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "net/url" @@ -16,6 +17,8 @@ import ( "github.com/kandev/kandev/internal/agent/executor" agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/task/models" ) // standaloneControlServer is an in-process stand-in for the host agentctl @@ -28,11 +31,64 @@ type standaloneControlServer struct { deleted []string createStatus int server *httptest.Server + // listInstances is what GET /api/v1/instances returns; nil (vs. an empty + // non-nil slice) tells the fake to answer with a listing failure instead, + // for exercising AC-EXECUTORS-SURVIVAL-002.12. + listInstances []*agentctlclient.InstanceInfo + listInstancesErr bool + // listInstancesFailures is the number of remaining GET + // /api/v1/instances attempts to fail (500) before it starts answering + // with listInstances -- lets a test pin the exact bounded-retry attempt + // count (AC-EXECUTORS-SURVIVAL-002.13) for the enumeration read itself. + listInstancesFailures int + listInstancesAttempts int + // deleteFailures, keyed by instance ID, is the number of remaining DELETE + // attempts to fail (500) for that instance before it starts succeeding -- + // lets a test pin the exact bounded-retry attempt count + // (AC-EXECUTORS-SURVIVAL-002.15). A count so high it never reaches zero + // simulates a stop that never succeeds. + deleteFailures map[string]int + deleteAttempts map[string]int + // deleteDelay, keyed by instance ID, makes that instance's DELETE + // response take at least this long -- used to prove a recovery deadline + // (AC-EXECUTORS-SURVIVAL-003.7) can elapse while a stop is genuinely + // still in flight, without the fixture holding s.mu for the whole sleep + // (which would otherwise serialize every other concurrent stop request). + deleteDelay map[string]time.Duration + + // turnOutcomes, keyed by instance ID, is what GET + // .../turn-outcome answers with; an absent key answers "nothing + // retained" (retained: false), matching AC-EXECUTORS-SURVIVAL-004.5's + // no-outcome case. + turnOutcomes map[string]agentctlclient.TurnOutcome + // turnOutcomeFailures, keyed by instance ID, is the number of remaining + // GET .../turn-outcome attempts to fail (500) before it starts + // answering normally -- pins the bounded-retry attempt count + // (AC-EXECUTORS-SURVIVAL-004.5/002.13). + turnOutcomeFailures map[string]int + turnOutcomeAttempts map[string]int + // ackedTurnOutcomes records every accepted POST .../turn-outcome/ack + // call, in order. + ackedTurnOutcomes []ackedTurnOutcome +} + +type ackedTurnOutcome struct { + instanceID string + turnID int64 } func newStandaloneControlServer(t *testing.T, healthy bool) *standaloneControlServer { t.Helper() - s := &standaloneControlServer{healthy: healthy, createStatus: http.StatusOK} + s := &standaloneControlServer{ + healthy: healthy, + createStatus: http.StatusOK, + deleteFailures: make(map[string]int), + deleteAttempts: make(map[string]int), + deleteDelay: make(map[string]time.Duration), + turnOutcomes: make(map[string]agentctlclient.TurnOutcome), + turnOutcomeFailures: make(map[string]int), + turnOutcomeAttempts: make(map[string]int), + } s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { s.mu.Lock() defer s.mu.Unlock() @@ -55,7 +111,61 @@ func newStandaloneControlServer(t *testing.T, healthy bool) *standaloneControlSe } _ = json.NewEncoder(w).Encode(agentctlclient.CreateInstanceResponse{ID: "std-1", Port: 45678}) case strings.HasPrefix(r.URL.Path, "/api/v1/instances/") && r.Method == http.MethodDelete: - s.deleted = append(s.deleted, strings.TrimPrefix(r.URL.Path, "/api/v1/instances/")) + id := strings.TrimPrefix(r.URL.Path, "/api/v1/instances/") + s.deleteAttempts[id]++ + delay := s.deleteDelay[id] + fail := s.deleteFailures[id] > 0 + if fail { + s.deleteFailures[id]-- + } + if delay > 0 { + s.mu.Unlock() + time.Sleep(delay) + s.mu.Lock() + } + if fail { + w.WriteHeader(http.StatusInternalServerError) + return + } + s.deleted = append(s.deleted, id) + case r.URL.Path == "/api/v1/instances" && r.Method == http.MethodGet: + s.listInstancesAttempts++ + if s.listInstancesErr { + w.WriteHeader(http.StatusInternalServerError) + return + } + if s.listInstancesFailures > 0 { + s.listInstancesFailures-- + w.WriteHeader(http.StatusInternalServerError) + return + } + _ = json.NewEncoder(w).Encode(struct { + Instances []*agentctlclient.InstanceInfo `json:"instances"` + }{Instances: s.listInstances}) + case strings.HasPrefix(r.URL.Path, "/api/v1/instances/") && strings.HasSuffix(r.URL.Path, "/turn-outcome") && + r.Method == http.MethodGet: + id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/instances/"), "/turn-outcome") + s.turnOutcomeAttempts[id]++ + if s.turnOutcomeFailures[id] > 0 { + s.turnOutcomeFailures[id]-- + w.WriteHeader(http.StatusInternalServerError) + return + } + outcome, retained := s.turnOutcomes[id] + _ = json.NewEncoder(w).Encode(struct { + Retained bool `json:"retained"` + TurnID int64 `json:"turn_id"` + Event streams.AgentEvent `json:"event"` + }{Retained: retained, TurnID: outcome.TurnID, Event: outcome.Event}) + case strings.HasPrefix(r.URL.Path, "/api/v1/instances/") && strings.HasSuffix(r.URL.Path, "/turn-outcome/ack") && + r.Method == http.MethodPost: + id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/instances/"), "/turn-outcome/ack") + var body struct { + TurnID int64 `json:"turn_id"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + s.ackedTurnOutcomes = append(s.ackedTurnOutcomes, ackedTurnOutcome{instanceID: id, turnID: body.TurnID}) + w.WriteHeader(http.StatusOK) default: w.WriteHeader(http.StatusNotFound) } @@ -99,9 +209,9 @@ func TestStandaloneExecutorStaticSurface(t *testing.T) { if exec.IsAlwaysResumable() { t.Fatal("standalone instances are transient and not always resumable") } - instances, err := exec.RecoverInstances(context.Background()) - if err != nil || instances != nil { - t.Fatalf("RecoverInstances() = %v, %v; want nil, nil", instances, err) + instances, err := exec.RecoverInstances(context.Background(), nil) + if err != nil || len(instances) != 0 { + t.Fatalf("RecoverInstances() = %v, %v; want none recovered, no error", instances, err) } runner := &process.InteractiveRunner{} @@ -285,6 +395,527 @@ func TestStandaloneExecutorStopInstance(t *testing.T) { }) } +// TestStandaloneExecutorRecoverInstancesReTracksWinnerAndStopsOrphan pins +// AC-EXECUTORS-SURVIVAL-002.1/002.2/002.6: a live instance whose session +// matches a recovery-inventory record is re-tracked and returned; a live +// instance with no matching record is stopped as an orphan. +func TestStandaloneExecutorRecoverInstancesReTracksWinnerAndStopsOrphan(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + { + ID: "instance-1", Port: 5001, SessionID: "session-1", TaskID: "task-1", WorkspacePath: "/ws/1", + Env: map[string]string{"KANDEV_RUN_ID": "run-42", "PATH": "/usr/bin"}, + WorkspaceSourceRoots: []string{"/ws/1", "/ws/1-sibling"}, + ProviderSessionID: "provider-session-9", + }, + {ID: "orphan-instance", Port: 5002, SessionID: "session-orphan"}, + } + exec := control.executor(t) + exec.SetAuthToken("survival-token") + + records := []*models.ExecutorRunning{ + {SessionID: "session-1", TaskID: "task-1", AgentExecutionID: "instance-1", WorktreePath: "/ws/1-from-record", Metadata: map[string]interface{}{"k": "v"}}, + } + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 1 { + t.Fatalf("recovered = %+v, want exactly one winner", recovered) + } + got := recovered[0] + if got.SessionID != "session-1" || got.TaskID != "task-1" || got.InstanceID != "instance-1" { + t.Fatalf("recovered instance identity = %+v", got) + } + // AC-EXECUTORS-SURVIVAL-002.14 (Review round 3, finding 3): workspace + // path's declared source is the recovery-inventory record, never the + // adopted instance's own report (which the fixture deliberately sets to + // a different value, "/ws/1", to prove it is ignored). + if got.WorkspacePath != "/ws/1-from-record" { + t.Fatalf("WorkspacePath = %q, want the record's WorktreePath, not the instance's self-reported path", got.WorkspacePath) + } + if got.Metadata["k"] != "v" { + t.Fatalf("Metadata = %+v, want the record's persisted metadata", got.Metadata) + } + if got.Env["KANDEV_RUN_ID"] != "run-42" || got.Env["PATH"] != "/usr/bin" { + t.Fatalf("Env = %+v, want the adopted instance's own runtime environment read back (AC-EXECUTORS-SURVIVAL-002.14)", got.Env) + } + if len(got.WorkspaceSourceRoots) != 2 || got.WorkspaceSourceRoots[0] != "/ws/1" || got.WorkspaceSourceRoots[1] != "/ws/1-sibling" { + t.Fatalf("WorkspaceSourceRoots = %+v, want the adopted instance's own live allowlist read back (AC-EXECUTORS-SURVIVAL-002.14)", got.WorkspaceSourceRoots) + } + if got.ProviderSessionID != "provider-session-9" { + t.Fatalf("ProviderSessionID = %q, want the adopted instance's own live provider session read back (AC-EXECUTORS-SURVIVAL-002.14)", got.ProviderSessionID) + } + if got.Client == nil { + t.Fatal("recovered instance must carry a usable agentctl client") + } + + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 1 || control.deleted[0] != "orphan-instance" { + t.Fatalf("deleted = %v, want the orphan instance stopped", control.deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesStopsDuplicateLoser pins +// AC-EXECUTORS-SURVIVAL-002.10: of two live instances for one session, the +// one matching the record's agent execution identifier is re-tracked and the +// other is stopped. +func TestStandaloneExecutorRecoverInstancesStopsDuplicateLoser(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "winner", Port: 5001, SessionID: "session-1"}, + {ID: "loser", Port: 5002, SessionID: "session-1"}, + } + exec := control.executor(t) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "winner"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 1 || recovered[0].InstanceID != "winner" { + t.Fatalf("recovered = %+v, want only the winner", recovered) + } + + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 1 || control.deleted[0] != "loser" { + t.Fatalf("deleted = %v, want the loser stopped", control.deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesRecordWithNoInstanceIsUntouched pins +// AC-EXECUTORS-SURVIVAL-002.7: a record with no live instance is left alone, +// not stopped and not reported recovered. +func TestStandaloneExecutorRecoverInstancesRecordWithNoInstanceIsUntouched(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = nil + exec := control.executor(t) + + records := []*models.ExecutorRunning{{SessionID: "session-cold", AgentExecutionID: "instance-cold"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 0 { + t.Fatalf("deleted = %v, want none: a record with no instance is left to the existing repair path", control.deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesEnumerationFailureRecoversNothing +// pins AC-EXECUTORS-SURVIVAL-002.12: when the adopted server cannot be +// enumerated, recovery reports nothing recovered, stops nothing, and does +// not error out (a hard error here would fail backend startup outright). +func TestStandaloneExecutorRecoverInstancesEnumerationFailureRecoversNothing(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstancesErr = true + exec := control.executor(t) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "instance-1"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v, want a logged-and-swallowed enumeration failure", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 0 { + t.Fatalf("deleted = %v, want none: an unenumerable server must not have instances stopped blind", control.deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesRetriesTransientEnumerationFailure +// pins AC-EXECUTORS-SURVIVAL-002.13's bounded read retry applied to the +// enumeration call itself: workspace source roots and provider session +// identity are read back only from this response, so a transient failure +// must be retried, not treated as a hard enumeration failure on the first +// error. +func TestStandaloneExecutorRecoverInstancesRetriesTransientEnumerationFailure(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "instance-1", Port: 5001, SessionID: "session-1"}, + } + control.listInstancesFailures = 1 // fails once, succeeds on the retry + exec := control.executor(t) + exec.SetRecoveryRetryConfig(50*time.Millisecond, 1) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "instance-1"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 1 { + t.Fatalf("recovered = %+v, want the one instance recovered after the retry succeeds", recovered) + } + + control.mu.Lock() + defer control.mu.Unlock() + if control.listInstancesAttempts != 2 { + t.Fatalf("listInstances attempts = %d, want exactly 2 (1 failure plus 1 retry)", control.listInstancesAttempts) + } +} + +// TestStandaloneExecutorRecoverInstancesExhaustingEnumerationRetryRecoversNothing +// pins the exhausted-budget half of AC-EXECUTORS-SURVIVAL-002.13 for the +// enumeration read: once every attempt fails, recovery falls back to +// AC-EXECUTORS-SURVIVAL-002.12's existing repair path rather than retrying +// indefinitely. +func TestStandaloneExecutorRecoverInstancesExhaustingEnumerationRetryRecoversNothing(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "instance-1", Port: 5001, SessionID: "session-1"}, + } + control.listInstancesFailures = 999 // never succeeds within the retry budget + exec := control.executor(t) + exec.SetRecoveryRetryConfig(20*time.Millisecond, 1) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "instance-1"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v, want a logged-and-swallowed exhausted-retry enumeration failure", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + + control.mu.Lock() + defer control.mu.Unlock() + if control.listInstancesAttempts != 2 { + t.Fatalf("listInstances attempts = %d, want exactly 2 (1 retry configured)", control.listInstancesAttempts) + } +} + +// fakeUnstoppableRecorder captures RetainAsUnstoppable/MarkStopInFlight/ +// Release calls without needing a real RecoveryGuard. +type fakeUnstoppableRecorder struct { + mu sync.Mutex + retained []string + markedInFlight []string + released []string +} + +func (f *fakeUnstoppableRecorder) RetainAsUnstoppable(sessionID string) { + f.mu.Lock() + defer f.mu.Unlock() + f.retained = append(f.retained, sessionID) +} + +func (f *fakeUnstoppableRecorder) MarkStopInFlight(sessionID string) { + f.mu.Lock() + defer f.mu.Unlock() + f.markedInFlight = append(f.markedInFlight, sessionID) +} + +func (f *fakeUnstoppableRecorder) Release(sessionID string) { + f.mu.Lock() + defer f.mu.Unlock() + f.released = append(f.released, sessionID) +} + +// TestStandaloneExecutorRecoverInstancesRetriesTransientStopFailure pins +// AC-EXECUTORS-SURVIVAL-002.15: a stop that fails once and then succeeds is +// retried within the bounded count rather than being treated as a permanent +// failure. +func TestStandaloneExecutorRecoverInstancesRetriesTransientStopFailure(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "orphan-instance", Port: 5002, SessionID: "session-orphan"}, + } + control.deleteFailures["orphan-instance"] = 1 + exec := control.executor(t) + + recovered, err := exec.RecoverInstances(context.Background(), nil) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 1 || control.deleted[0] != "orphan-instance" { + t.Fatalf("deleted = %v, want the orphan eventually stopped after a retry", control.deleted) + } + if control.deleteAttempts["orphan-instance"] != 2 { + t.Fatalf("delete attempts = %d, want exactly 2 (one failure plus one retry)", control.deleteAttempts["orphan-instance"]) + } +} + +// TestStandaloneExecutorRecoverInstancesExhaustingLoserStopDropsAndStopsWinner +// pins AC-EXECUTORS-SURVIVAL-002.15: when a losing duplicate's stop exhausts +// its retries, the winning instance for that session is also not re-tracked +// and is itself stopped on the same terms. +func TestStandaloneExecutorRecoverInstancesExhaustingLoserStopDropsAndStopsWinner(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "winner", Port: 5001, SessionID: "session-1"}, + {ID: "loser", Port: 5002, SessionID: "session-1"}, + } + control.deleteFailures["loser"] = 999 // never succeeds within the retry budget + exec := control.executor(t) + exec.SetRecoveryRetryConfig(50*time.Millisecond, 1) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "winner"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none: the session must not be re-tracked once its loser can't be stopped", recovered) + } + + control.mu.Lock() + defer control.mu.Unlock() + if control.deleteAttempts["loser"] != 2 { + t.Fatalf("loser delete attempts = %d, want exactly 2 (1 retry configured)", control.deleteAttempts["loser"]) + } + if len(control.deleted) != 1 || control.deleted[0] != "winner" { + t.Fatalf("deleted = %v, want the winner also stopped since the session won't be re-tracked", control.deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesLoserFailureWinnerSuccessRetainsGuard +// pins the ownership handoff when only the losing duplicate cannot be +// stopped. A successful winner stop must not release the session guard while +// the loser is still alive. +func TestStandaloneExecutorRecoverInstancesLoserFailureWinnerSuccessRetainsGuard(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "winner", Port: 5001, SessionID: "session-1"}, + {ID: "loser", Port: 5002, SessionID: "session-1"}, + } + control.deleteFailures["loser"] = 999 + exec := control.executor(t) + exec.SetRecoveryRetryConfig(50*time.Millisecond, 0) + guard := NewRecoveryGuard() + guard.AcquireOrObserve("session-1") + exec.SetUnstoppableSessionRecorder(guard) + + recovered, err := exec.RecoverInstances(context.Background(), []*models.ExecutorRunning{{ + SessionID: "session-1", AgentExecutionID: "winner", + }}) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + if err := guard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("guard after winner stop success = %v, want ErrSessionUnstoppableAgent", err) + } + + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 1 || control.deleted[0] != "winner" { + t.Fatalf("deleted = %v, want only winner; the loser stop failed", control.deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesJointStopFailureRetainsSessionUnstoppable +// pins AC-EXECUTORS-SURVIVAL-002.16: when both the losing duplicate's stop +// and the winner's own stop exhaust their retries, the session is recorded +// as unstoppable rather than silently left not-re-tracked. +func TestStandaloneExecutorRecoverInstancesJointStopFailureRetainsSessionUnstoppable(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "winner", Port: 5001, SessionID: "session-1"}, + {ID: "loser", Port: 5002, SessionID: "session-1"}, + } + control.deleteFailures["loser"] = 999 + control.deleteFailures["winner"] = 999 + exec := control.executor(t) + exec.SetRecoveryRetryConfig(50*time.Millisecond, 0) + recorder := &fakeUnstoppableRecorder{} + exec.SetUnstoppableSessionRecorder(recorder) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "winner"}} + + recovered, err := exec.RecoverInstances(context.Background(), records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + + recorder.mu.Lock() + defer recorder.mu.Unlock() + if len(recorder.retained) != 1 || recorder.retained[0] != "session-1" { + t.Fatalf("retained = %v, want session-1 retained as unstoppable", recorder.retained) + } +} + +// TestStandaloneExecutorRecoverInstancesDeadlineDropsPendingSessionAndFinishesInBackground +// pins AC-EXECUTORS-SURVIVAL-003.7: once ctx's deadline elapses, a session +// whose losing duplicate hasn't finished stopping yet is treated as +// not-re-tracked immediately -- RecoverInstances returns without waiting for +// it -- but the in-flight stop, and the consequent joint winner-stop and +// AC-EXECUTORS-SURVIVAL-002.16 unstoppable report it triggers, are not +// aborted: they run to completion in the background and still get recorded. +func TestStandaloneExecutorRecoverInstancesDeadlineDropsPendingSessionAndFinishesInBackground(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "winner", Port: 5001, SessionID: "session-1"}, + {ID: "loser", Port: 5002, SessionID: "session-1"}, + } + control.deleteFailures["loser"] = 999 // never succeeds within its retry budget + control.deleteFailures["winner"] = 999 // never succeeds either, once tried + control.deleteDelay["loser"] = 150 * time.Millisecond + exec := control.executor(t) + exec.SetRecoveryRetryConfig(20*time.Millisecond, 0) + recorder := &fakeUnstoppableRecorder{} + exec.SetUnstoppableSessionRecorder(recorder) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "winner"}} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + start := time.Now() + recovered, err := exec.RecoverInstances(ctx, records) + elapsed := time.Since(start) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none: the session's loser stop was still unresolved at the deadline", recovered) + } + if elapsed > 100*time.Millisecond { + t.Fatalf("RecoverInstances took %s, want it to return promptly at the deadline instead of waiting for the in-flight stop", elapsed) + } + + pollDeadline := time.Now().Add(2 * time.Second) + for { + recorder.mu.Lock() + retained := len(recorder.retained) + recorder.mu.Unlock() + if retained == 1 { + break + } + if time.Now().After(pollDeadline) { + t.Fatal("timed out waiting for the background drain to finish joint-failure handling") + } + time.Sleep(5 * time.Millisecond) + } + + recorder.mu.Lock() + defer recorder.mu.Unlock() + if recorder.retained[0] != "session-1" { + t.Fatalf("retained = %v, want session-1", recorder.retained) + } +} + +// TestStandaloneExecutorRecoverInstancesDeadlineMarksGuardInFlightAndReleasesOnResolution +// pins Review round 2 finding 3 (AC-EXECUTORS-SURVIVAL-002.8/003.7): a +// session dropped at the recovery deadline because its loser's stop is still +// in flight must have its guard marked stop-in-flight, so a Start() pass's +// ReleaseAllExceptRetained leaves it held rather than releasing it while a +// launch racing the still-resolving stop could still recreate two live +// agents for the session -- and the guard must actually be released once the +// background drain learns the stop resolved, not left held forever. Uses the +// real RecoveryGuard rather than a fake so both state transitions are pinned +// against the type Manager.Start actually wires in. +func TestStandaloneExecutorRecoverInstancesDeadlineMarksGuardInFlightAndReleasesOnResolution(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "winner", Port: 5001, SessionID: "session-1"}, + {ID: "loser", Port: 5002, SessionID: "session-1"}, + } + // Long enough per-attempt timeout to tolerate the delay below, so the + // loser's stop eventually succeeds instead of failing on a client-side + // timeout -- this test is about the success/resolution path, not the + // joint-failure path TestStandaloneExecutorRecoverInstancesJointStopFailureRetainsSessionUnstoppable + // already covers. + control.deleteDelay["loser"] = 30 * time.Millisecond + exec := control.executor(t) + exec.SetRecoveryRetryConfig(200*time.Millisecond, 0) + guard := NewRecoveryGuard() + guard.AcquireOrObserve("session-1") + exec.SetUnstoppableSessionRecorder(guard) + + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "winner"}} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + recovered, err := exec.RecoverInstances(ctx, records) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none: the loser stop was still unresolved at the deadline", recovered) + } + + // Simulate the bulk release a Start() pass performs once every + // reconstructed session's outcome is published: it must not release + // this session's guard yet, since its stop is still resolving. + guard.ReleaseAllExceptRetained() + if err := guard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("guard state right after the deadline = %v, want still held (ErrSessionRecoveryGuarded)", err) + } + + pollDeadline := time.Now().Add(2 * time.Second) + for guard.CheckLaunchAllowed("session-1") != nil { + if time.Now().After(pollDeadline) { + t.Fatal("timed out waiting for the background drain to release the guard") + } + time.Sleep(5 * time.Millisecond) + } + + control.mu.Lock() + deleted := append([]string(nil), control.deleted...) + control.mu.Unlock() + if len(deleted) != 1 || deleted[0] != "loser" { + t.Fatalf("deleted = %v, want only the loser stopped: the winner must be untouched when the loser eventually succeeds", deleted) + } +} + +// TestStandaloneExecutorRecoverInstancesOrphanStopFailureIsRecordedNotJoint +// pins that an orphan's (AC-EXECUTORS-SURVIVAL-002.6) exhausted stop retry +// has no session to affect: it is only logged, never routed through the +// unstoppable-session recorder. +func TestStandaloneExecutorRecoverInstancesOrphanStopFailureIsRecordedNotJoint(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "orphan-instance", Port: 5002, SessionID: "session-orphan"}, + } + control.deleteFailures["orphan-instance"] = 999 + exec := control.executor(t) + exec.SetRecoveryRetryConfig(50*time.Millisecond, 0) + recorder := &fakeUnstoppableRecorder{} + exec.SetUnstoppableSessionRecorder(recorder) + + recovered, err := exec.RecoverInstances(context.Background(), nil) + if err != nil { + t.Fatalf("RecoverInstances: %v", err) + } + if len(recovered) != 0 { + t.Fatalf("recovered = %+v, want none", recovered) + } + + recorder.mu.Lock() + defer recorder.mu.Unlock() + if len(recorder.retained) != 0 { + t.Fatalf("retained = %v, want none: an orphan has no session to retain", recorder.retained) + } +} + func TestBuildStandaloneCreateInstanceRequestMapsEveryField(t *testing.T) { autoApprove := true req := &ExecutorCreateRequest{ diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_turn_outcome.go b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_turn_outcome.go new file mode 100644 index 0000000000..c60f3a24fd --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_turn_outcome.go @@ -0,0 +1,54 @@ +package lifecycle + +import ( + "context" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" +) + +// fetchTurnOutcomeWithRetry retrieves instanceID's retained terminal turn +// outcome within the same bounded per-attempt timeout and retry count as +// listInstancesWithRetry/stopWithRetry (AC-EXECUTORS-SURVIVAL-004.5 requires +// the same bound as AC-EXECUTORS-SURVIVAL-002.13). Returns (nil, nil) when +// the read succeeded and nothing was retained -- ControlClient.GetTurnOutcome +// already distinguishes that from a failure, and this method preserves the +// distinction rather than treating it as an error to retry. +func (r *StandaloneExecutor) fetchTurnOutcomeWithRetry(ctx context.Context, instanceID string) (*agentctl.TurnOutcome, error) { + timeout := r.recoveryReadTimeout + if timeout <= 0 { + timeout = defaultRecoveryReadTimeout + } + retries := r.recoveryReadRetries + if retries < 0 { + retries = defaultRecoveryReadRetries + } + + var lastErr error + for attempt := 0; attempt <= retries; attempt++ { + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + outcome, err := r.ctl.GetTurnOutcome(attemptCtx, instanceID) + cancel() + if err == nil { + return outcome, nil + } + lastErr = err + } + return nil, lastErr +} + +// ackTurnOutcome acknowledges instanceID's retained outcome by turn +// identifier, discarding it (AC-EXECUTORS-SURVIVAL-004.6). Bounded by a +// single attempt at the same per-attempt timeout as the read: the +// acknowledgement is idempotent and safe to retry (naming an identifier the +// control server no longer holds changes nothing), so a failed ack here is +// left for the next backend's re-application of the same outcome rather than +// retried in a loop that would delay this recovery pass. +func (r *StandaloneExecutor) ackTurnOutcome(ctx context.Context, instanceID string, turnID int64) error { + timeout := r.recoveryReadTimeout + if timeout <= 0 { + timeout = defaultRecoveryReadTimeout + } + attemptCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + return r.ctl.AckTurnOutcome(attemptCtx, instanceID, turnID) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_turn_outcome_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_turn_outcome_test.go new file mode 100644 index 0000000000..4a7c47bf2a --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_turn_outcome_test.go @@ -0,0 +1,116 @@ +package lifecycle + +import ( + "context" + "testing" + + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/types/streams" +) + +// TestFetchTurnOutcomeWithRetryReturnsNilWhenNothingRetained pins +// AC-EXECUTORS-SURVIVAL-004.5's "nothing retained" case: a successful read +// with no outcome returns (nil, nil), never an error. +func TestFetchTurnOutcomeWithRetryReturnsNilWhenNothingRetained(t *testing.T) { + control := newStandaloneControlServer(t, true) + exec := control.executor(t) + + outcome, err := exec.fetchTurnOutcomeWithRetry(context.Background(), "inst-1") + if err != nil { + t.Fatalf("fetchTurnOutcomeWithRetry() error = %v, want nil", err) + } + if outcome != nil { + t.Fatalf("fetchTurnOutcomeWithRetry() = %+v, want nil", outcome) + } + if control.turnOutcomeAttempts["inst-1"] != 1 { + t.Fatalf("attempts = %d, want 1 (no retry needed on success)", control.turnOutcomeAttempts["inst-1"]) + } +} + +// TestFetchTurnOutcomeWithRetryReturnsRetainedOutcome pins the "outcome +// present" case, including that the turn identifier and event round-trip +// intact. +func TestFetchTurnOutcomeWithRetryReturnsRetainedOutcome(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.turnOutcomes["inst-1"] = agentctlclient.TurnOutcome{ + TurnID: 42, + Event: streams.AgentEvent{Type: streams.EventTypeComplete, Text: "done"}, + } + exec := control.executor(t) + + outcome, err := exec.fetchTurnOutcomeWithRetry(context.Background(), "inst-1") + if err != nil { + t.Fatalf("fetchTurnOutcomeWithRetry() error = %v", err) + } + if outcome == nil { + t.Fatal("fetchTurnOutcomeWithRetry() = nil, want the retained outcome") + } + if outcome.TurnID != 42 || outcome.Event.Text != "done" { + t.Fatalf("outcome = %+v, want TurnID 42 and Event.Text %q", outcome, "done") + } +} + +// TestFetchTurnOutcomeWithRetryRetriesTransientFailure pins +// AC-EXECUTORS-SURVIVAL-004.5's bound: the read is retried the configured +// number of times before giving up. +func TestFetchTurnOutcomeWithRetryRetriesTransientFailure(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.turnOutcomeFailures["inst-1"] = 1 + control.turnOutcomes["inst-1"] = agentctlclient.TurnOutcome{ + TurnID: 7, + Event: streams.AgentEvent{Type: streams.EventTypeError}, + } + exec := control.executor(t) + exec.SetRecoveryRetryConfig(0, 2) + + outcome, err := exec.fetchTurnOutcomeWithRetry(context.Background(), "inst-1") + if err != nil { + t.Fatalf("fetchTurnOutcomeWithRetry() error = %v, want success after retry", err) + } + if outcome == nil || outcome.TurnID != 7 { + t.Fatalf("outcome = %+v, want TurnID 7", outcome) + } + if control.turnOutcomeAttempts["inst-1"] != 2 { + t.Fatalf("attempts = %d, want 2 (one failure, one success)", control.turnOutcomeAttempts["inst-1"]) + } +} + +// TestFetchTurnOutcomeWithRetryExhaustsConfiguredRetries pins the exact +// attempt count when every attempt fails: AC-EXECUTORS-SURVIVAL-004.5's +// read-failed case, which the recovery loop must treat as "not-re-tracked," +// never as "nothing retained." +func TestFetchTurnOutcomeWithRetryExhaustsConfiguredRetries(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.turnOutcomeFailures["inst-1"] = 100 + exec := control.executor(t) + exec.SetRecoveryRetryConfig(0, 2) + + outcome, err := exec.fetchTurnOutcomeWithRetry(context.Background(), "inst-1") + if err == nil { + t.Fatal("fetchTurnOutcomeWithRetry() error = nil, want an error after exhausting retries") + } + if outcome != nil { + t.Fatalf("outcome = %+v, want nil on failure", outcome) + } + if control.turnOutcomeAttempts["inst-1"] != 3 { + t.Fatalf("attempts = %d, want 3 (1 initial + 2 retries)", control.turnOutcomeAttempts["inst-1"]) + } +} + +// TestAckTurnOutcomeSendsInstanceAndTurnID pins the wire shape of the +// acknowledgement call. +func TestAckTurnOutcomeSendsInstanceAndTurnID(t *testing.T) { + control := newStandaloneControlServer(t, true) + exec := control.executor(t) + + if err := exec.ackTurnOutcome(context.Background(), "inst-1", 42); err != nil { + t.Fatalf("ackTurnOutcome() error = %v", err) + } + if len(control.ackedTurnOutcomes) != 1 { + t.Fatalf("acked calls = %d, want 1", len(control.ackedTurnOutcomes)) + } + got := control.ackedTurnOutcomes[0] + if got.instanceID != "inst-1" || got.turnID != 42 { + t.Fatalf("acked = %+v, want {inst-1 42}", got) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/liveness_inherited_scope.go b/apps/backend/internal/agent/runtime/lifecycle/liveness_inherited_scope.go new file mode 100644 index 0000000000..41da7b79b3 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/liveness_inherited_scope.go @@ -0,0 +1,44 @@ +package lifecycle + +// InheritedRecordScope names what this launch found at the recorded control +// endpoint. It governs only records this backend did NOT create during this +// process lifetime -- the ones inherited from an earlier launch -- because a +// record this backend created is judged against whichever server it is using, +// adopted or freshly started. +// +// The distinction matters because a fresh server's enumeration is not +// evidence about an instance it never launched. Judging an inherited record +// against it would report a still-running agent as gone. +type InheritedRecordScope int + +const ( + // InheritedRecordScopeNoServer is nothing answered at the recorded + // endpoint, or no endpoint was recorded. No control server holds any + // inherited instance, so the process-identifier probe applies and + // genuinely dead records are still repaired. This is the zero value: + // a launch that never attempted adoption is in exactly this position. + InheritedRecordScopeNoServer InheritedRecordScope = iota + + // InheritedRecordScopeForeignServer is a server answered at the recorded + // endpoint and was left running, which is every path where adoption did + // not complete against a reachable server. A surviving instance runs on + // a server this backend does not drive, so its absence from the one this + // backend does drive proves nothing and every inherited record is + // unknown. Classifying it dead here would repair or prune a record whose + // agent is still running. + InheritedRecordScopeForeignServer + + // InheritedRecordScopeAdopted is this backend adopted the recorded + // server, so its enumeration is authoritative for inherited records. + InheritedRecordScopeAdopted +) + +// SetInheritedRecordScope installs what this launch found at the recorded +// control endpoint, so liveness classification of records inherited from an +// earlier launch can tell "no server holds these" from "a server this +// backend does not drive might". Set once during startup wiring, before +// Start runs; the zero value (no server) is correct for a launch that never +// attempted adoption. +func (m *Manager) SetInheritedRecordScope(scope InheritedRecordScope) { + m.inheritedRecordScope = scope +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/liveness_inherited_scope_test.go b/apps/backend/internal/agent/runtime/lifecycle/liveness_inherited_scope_test.go new file mode 100644 index 0000000000..0cfce1e161 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/liveness_inherited_scope_test.go @@ -0,0 +1,82 @@ +package lifecycle + +import ( + "testing" + + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +// TestInheritedRecordIsProbedWhenNothingAnsweredAtTheRecordedEndpoint is the +// case that made every inherited record permanently unrepairable. On a start +// where nothing answered at the recorded endpoint, no control server holds +// those instances at all, so the process-identifier probe is the right +// authority and a genuinely dead record must still be repaired. Judging the +// record against the enumeration of a server this launch just started +// answers Unknown forever, and Unknown means "leave it alone". +func TestInheritedRecordIsProbedWhenNothingAnsweredAtTheRecordedEndpoint(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetInheritedRecordScope(InheritedRecordScopeNoServer) + row := &models.ExecutorRunning{ + SessionID: "session-inherited-noserver", + Runtime: agentruntime.RuntimeStandalone, + LocalPID: spawnAndReapPID(t), + } + // Reachable and empty: this is the freshly started server's enumeration, + // which never launched the inherited instance and is therefore not + // evidence about it either way. + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{}} + + if got := mgr.classifyStandaloneLiveness(row, scope); got != models.ProcessLivenessDead { + t.Fatalf("classifyStandaloneLiveness = %v, want Dead via the process-identifier probe", got) + } +} + +// TestInheritedRecordIsUnknownWhenAForeignServerAnswered is the opposite +// direction and the reason the probe cannot simply always apply. A server +// answered at the recorded endpoint and was left running, so the instance +// may still be alive on it. Probing a recycled or reused local PID there +// could report the record dead and repair a record whose agent is still +// working. +func TestInheritedRecordIsUnknownWhenAForeignServerAnswered(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetInheritedRecordScope(InheritedRecordScopeForeignServer) + row := &models.ExecutorRunning{ + SessionID: "session-inherited-foreign", + Runtime: agentruntime.RuntimeStandalone, + LocalPID: spawnAndReapPID(t), + } + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{}} + + if got := mgr.classifyStandaloneLiveness(row, scope); got != models.ProcessLivenessUnknown { + t.Fatalf("classifyStandaloneLiveness = %v, want Unknown while a server this backend did not adopt is running", got) + } +} + +// TestRecordCreatedThisLifetimeIgnoresTheInheritedScope pins the split the +// rule turns on. A record this backend created is judged against whichever +// server it is using, adopted or freshly started, because that server is the +// one that launched the instance. Only records inherited from an earlier +// launch depend on what answered at the recorded endpoint. +func TestRecordCreatedThisLifetimeIgnoresTheInheritedScope(t *testing.T) { + for _, scopeKind := range []InheritedRecordScope{ + InheritedRecordScopeNoServer, + InheritedRecordScopeForeignServer, + InheritedRecordScopeAdopted, + } { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetInheritedRecordScope(scopeKind) + mgr.markSessionCreatedThisLifetime("session-own") + + alive := &models.ExecutorRunning{SessionID: "session-own", Runtime: agentruntime.RuntimeStandalone} + present := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{"session-own": {}}} + if got := mgr.classifyStandaloneLiveness(alive, present); got != models.ProcessLivenessAlive { + t.Fatalf("scope %v: present own record = %v, want Alive", scopeKind, got) + } + + absent := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{}} + if got := mgr.classifyStandaloneLiveness(alive, absent); got != models.ProcessLivenessDead { + t.Fatalf("scope %v: absent own record = %v, want Dead", scopeKind, got) + } + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/liveness_standalone.go b/apps/backend/internal/agent/runtime/lifecycle/liveness_standalone.go new file mode 100644 index 0000000000..3165198da8 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/liveness_standalone.go @@ -0,0 +1,174 @@ +package lifecycle + +import ( + "context" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/task/models" +) + +// standaloneLivenessScope is a single adopted-server enumeration snapshot +// (design 02 "Persistence": "liveness for a standalone record is judged by +// the presence of that record's instance in the adopted server's +// enumeration"). Reachable is false when the enumeration attempt itself +// failed -- nothing answered -- as distinct from a successful enumeration +// that simply found no matching instance. Produced by +// StandaloneExecutor.newLivenessScope and Manager.NewStandaloneLivenessScope; +// consumed only by classifyStandaloneLiveness. Never memoized across calls: +// a caller that wants one enumeration reused across many rows (a +// reconciliation pass) must build it once and thread it through itself; any +// other caller must build a fresh one for each classification (design 02 +// "two kinds of caller, and only one of them is a pass"). +type standaloneLivenessScope struct { + reachable bool + liveBySession map[string]struct{} +} + +// newLivenessScope takes one enumeration of the adopted server's live +// instances, keyed by session ID for O(1) row lookups. Reuses the same +// bounded per-attempt timeout/retry shape as listInstancesWithRetry's other +// callers (AC-EXECUTORS-SURVIVAL-002.13). +func (r *StandaloneExecutor) newLivenessScope(ctx context.Context) *standaloneLivenessScope { + instances, err := r.listInstancesWithRetry(ctx) + if err != nil { + return &standaloneLivenessScope{reachable: false} + } + live := make(map[string]struct{}, len(instances)) + for _, inst := range instances { + if inst.SessionID != "" { + live[inst.SessionID] = struct{}{} + } + } + return &standaloneLivenessScope{reachable: true, liveBySession: live} +} + +// NewStandaloneLivenessScope takes one adopted-server enumeration for reuse +// across every row of a single reconciliation pass. Returns nil when no +// standalone backend is registered; classifyStandaloneLiveness treats a nil +// scope identically to an unreachable one (falls back to the +// process-identifier probe). +func (m *Manager) NewStandaloneLivenessScope(ctx context.Context) interface{} { + backend, err := m.executorRegistry.GetBackend(executor.NameStandalone) + if err != nil { + return (*standaloneLivenessScope)(nil) + } + standalone, ok := backend.(*StandaloneExecutor) + if !ok { + return (*standaloneLivenessScope)(nil) + } + return standalone.newLivenessScope(ctx) +} + +// classifyStandaloneLiveness implements design 02's "Persistence" liveness +// rule for a single row, given a (possibly nil/unreachable) enumeration +// scope: +// +// - non-standalone runtime: unchanged, delegates to RowProcessLiveness +// (Unknown -- never probed by a local process check). +// - agent survival capability disabled: unchanged, delegates to +// RowProcessLiveness -- an enumeration scope is never consulted when the +// capability that makes a standalone control server outlive this +// backend isn't itself enabled. +// - scope nil or unreachable ("nothing answered"): presence is +// undeterminable, so Unknown -- except where no control server answered +// at the recorded endpoint at all, which falls back to today's +// process-identifier probe so a genuinely dead row is still repaired on +// the common case of a first start with no survivor. +// - present in the enumeration, but this session's stop was still in +// flight when the enumeration was taken: Unknown, not Alive -- the +// snapshot predates the stop's actual completion. +// - present, no stop in flight: Alive. +// - absent, and this backend created the row during its own process +// lifetime: Dead -- this exact server is the only authority for it, so +// "not present" is determinate. +// - absent, and the row was inherited from an earlier launch while this +// backend adopted the recorded server: Unknown. +// +// A record inherited from an earlier launch is not judged against the +// enumeration at all unless this backend adopted the recorded server, since +// a server this backend started never launched that instance. What happens +// instead turns on what answered at the recorded endpoint: nothing answered +// means no server holds it, so the process-identifier probe applies and a +// genuinely dead record is still repaired; a server that answered and was +// left running means the instance may still be alive on a server this +// backend does not drive, so the record is Unknown and is left alone. +func (m *Manager) classifyStandaloneLiveness(row *models.ExecutorRunning, scope *standaloneLivenessScope) models.ProcessLiveness { + if row == nil { + return models.ProcessLivenessUnknown + } + if !isLocalRuntime(row.Runtime) { + return RowProcessLiveness(row) + } + if !m.agentSurvivalEnabled { + return RowProcessLiveness(row) + } + if scope == nil || !scope.reachable { + // AC-EXECUTORS-SURVIVAL-003.6: presence could not be determined, so + // this is unknown rather than live or dead. The process-identifier + // probe cannot stand in for it here: a standalone row carries the + // SHARED control server's identifier, so with a server still running + // it answers live for an instance that is already gone -- reporting a + // record live solely because the shared control server is running, + // which that criterion forbids, and leaving the row to escape repair + // indefinitely. The probe is only evidence where no control server + // holds these instances at all, which is exactly the case the + // criterion names: nothing answered at the recorded endpoint, or no + // endpoint was recorded. + if m.inheritedRecordScope == InheritedRecordScopeNoServer { + return RowProcessLiveness(row) + } + return models.ProcessLivenessUnknown + } + sessionID := row.SessionID + createdHere := m.wasCreatedThisLifetime(sessionID) + if !createdHere && m.inheritedRecordScope != InheritedRecordScopeAdopted { + // An inherited record is judged only against a server this backend + // adopted. A freshly started server never launched this instance, so + // its enumeration is not evidence either way and is not consulted. + if m.inheritedRecordScope == InheritedRecordScopeNoServer { + return RowProcessLiveness(row) + } + return models.ProcessLivenessUnknown + } + if _, present := scope.liveBySession[sessionID]; present { + if m.recoveryGuard.IsStopInFlight(sessionID) { + return models.ProcessLivenessUnknown + } + return models.ProcessLivenessAlive + } + if createdHere { + return models.ProcessLivenessDead + } + return models.ProcessLivenessUnknown +} + +// RowLivenessScoped classifies row's liveness using scope (from +// NewStandaloneLivenessScope), reused across every row of one reconciliation +// pass. scope must be the interface{} value NewStandaloneLivenessScope +// returned (or nil); any other type is treated as unreachable. +func (m *Manager) RowLivenessScoped(row *models.ExecutorRunning, scope interface{}) models.ProcessLiveness { + s, _ := scope.(*standaloneLivenessScope) + return m.classifyStandaloneLiveness(row, s) +} + +// RowLiveness classifies row's liveness for a caller outside a reconciliation +// pass (e.g. the single-session idle reclaim path): it always takes its own +// fresh enumeration rather than reading one a pass cached (design 02 "two +// kinds of caller"), since such a caller can fire at any moment and a cached +// answer would age without bound. +// +// AC-EXECUTORS-SURVIVAL-003.6: such a caller firing before Start's recovery +// pass has finished for this process's lifetime must answer Unknown +// immediately -- neither enumerate nor wait -- since re-tracking has not yet +// reached an outcome for every record and a live enumeration taken now could +// race work recovery itself has not finished doing. +func (m *Manager) RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness { + if row == nil || !isLocalRuntime(row.Runtime) { + return RowProcessLiveness(row) + } + if !m.recoveryComplete.Load() { + return models.ProcessLivenessUnknown + } + scope, _ := m.NewStandaloneLivenessScope(context.Background()).(*standaloneLivenessScope) + return m.classifyStandaloneLiveness(row, scope) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/liveness_standalone_test.go b/apps/backend/internal/agent/runtime/lifecycle/liveness_standalone_test.go new file mode 100644 index 0000000000..8b24137b60 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/liveness_standalone_test.go @@ -0,0 +1,264 @@ +package lifecycle + +import ( + "context" + "os" + "testing" + "time" + + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +// newLivenessTestManager builds a Manager whose registered standalone +// backend talks to a real fake control server, so enumeration-based liveness +// classification exercises the actual HTTP round trip. The capability is +// enabled by default since these tests exercise the enumeration-scope +// classification path; a test of the disabled path overrides it explicitly. +func newLivenessTestManager(t *testing.T, control *standaloneControlServer) *Manager { + t.Helper() + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + execRegistry.Register(control.executor(t)) + mgr := NewManager( + newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, + ExecutorFallbackWarn, "", log, + ) + mgr.SetAgentSurvivalEnabled(true) + cleanupManagerStopCh(t, mgr) + return mgr +} + +func TestStandaloneExecutorNewLivenessScopeReachablePopulatesSessions(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "instance-1", SessionID: "session-1"}, + {ID: "instance-2", SessionID: "session-2"}, + } + exec := control.executor(t) + + scope := exec.newLivenessScope(context.Background()) + if !scope.reachable { + t.Fatal("expected a successful enumeration to report reachable") + } + if _, ok := scope.liveBySession["session-1"]; !ok { + t.Fatal("expected session-1 to be present in the enumeration") + } + if _, ok := scope.liveBySession["session-2"]; !ok { + t.Fatal("expected session-2 to be present in the enumeration") + } + if len(scope.liveBySession) != 2 { + t.Fatalf("liveBySession = %+v, want exactly 2 entries", scope.liveBySession) + } +} + +func TestStandaloneExecutorNewLivenessScopeUnreachableWhenEnumerationFails(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstancesErr = true + exec := control.executor(t) + exec.SetRecoveryRetryConfig(10*time.Millisecond, 0) + + scope := exec.newLivenessScope(context.Background()) + if scope.reachable { + t.Fatal("expected an enumeration failure to report unreachable, not reachable-with-nothing") + } +} + +func TestManagerClassifyStandaloneLivenessNonStandaloneRuntimeUnchanged(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + row := &models.ExecutorRunning{SessionID: "session-1", Runtime: agentruntime.RuntimeSSH} + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{"session-1": {}}} + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessUnknown { + t.Fatalf("classifyStandaloneLiveness for a non-standalone row = %v, want Unknown regardless of scope", got) + } +} + +func TestManagerClassifyStandaloneLivenessNothingAnsweredFallsBackToProcessProbe(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + aliveRow := &models.ExecutorRunning{ + SessionID: "session-alive", Runtime: agentruntime.RuntimeStandalone, LocalPID: os.Getpid(), + } + deadRow := &models.ExecutorRunning{ + SessionID: "session-dead", Runtime: agentruntime.RuntimeStandalone, LocalPID: spawnAndReapPID(t), + } + + for _, scope := range []*standaloneLivenessScope{nil, {reachable: false}} { + if got := mgr.classifyStandaloneLiveness(aliveRow, scope); got != models.ProcessLivenessAlive { + t.Fatalf("scope=%+v: classifyStandaloneLiveness(aliveRow) = %v, want Alive via process-identifier fallback", scope, got) + } + if got := mgr.classifyStandaloneLiveness(deadRow, scope); got != models.ProcessLivenessDead { + t.Fatalf("scope=%+v: classifyStandaloneLiveness(deadRow) = %v, want Dead via process-identifier fallback", scope, got) + } + } +} + +func TestManagerClassifyStandaloneLivenessPresentNoStopInFlightIsAlive(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetInheritedRecordScope(InheritedRecordScopeAdopted) + row := &models.ExecutorRunning{SessionID: "session-present", Runtime: agentruntime.RuntimeStandalone} + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{"session-present": {}}} + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessAlive { + t.Fatalf("classifyStandaloneLiveness = %v, want Alive for a present instance with no stop in flight", got) + } +} + +func TestManagerClassifyStandaloneLivenessPresentWithStopInFlightIsUnknown(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetInheritedRecordScope(InheritedRecordScopeAdopted) + row := &models.ExecutorRunning{SessionID: "session-stopping", Runtime: agentruntime.RuntimeStandalone} + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{"session-stopping": {}}} + mgr.recoveryGuard.AcquireOrObserve("session-stopping") + mgr.recoveryGuard.MarkStopInFlight("session-stopping") + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessUnknown { + t.Fatalf("classifyStandaloneLiveness = %v, want Unknown when the row's stop is still in flight", got) + } +} + +func TestManagerClassifyStandaloneLivenessAbsentOwnRecordIsDead(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + row := &models.ExecutorRunning{SessionID: "session-own", Runtime: agentruntime.RuntimeStandalone} + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{}} + mgr.markSessionCreatedThisLifetime("session-own") + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessDead { + t.Fatalf("classifyStandaloneLiveness = %v, want Dead for an own record absent from the enumeration", got) + } +} + +// TestManagerClassifyStandaloneLivenessCapabilityDisabledFallsBackToProcessProbe +// pins Review round 5 finding 3: when agentSurvivalEnabled is false, an +// enumeration scope must never be consulted -- classification always falls +// back to the process-identifier probe, exactly as it did before the +// standalone control server's enumeration-based liveness existed. Without +// this gate, an inherited row absent from a reachable enumeration would +// classify Unknown even though its process is provably dead. +func TestManagerClassifyStandaloneLivenessCapabilityDisabledFallsBackToProcessProbe(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetAgentSurvivalEnabled(false) + row := &models.ExecutorRunning{ + SessionID: "session-inherited-disabled", Runtime: agentruntime.RuntimeStandalone, LocalPID: spawnAndReapPID(t), + } + // Reachable and would report the row absent -- if consulted, this would + // wrongly yield Unknown (inherited-absent) instead of Dead. + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{}} + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessDead { + t.Fatalf("classifyStandaloneLiveness with capability disabled = %v, want Dead via process-identifier fallback", got) + } +} + +func TestManagerClassifyStandaloneLivenessAbsentInheritedRecordIsUnknown(t *testing.T) { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + // This backend adopted the recorded server, so its enumeration is + // authoritative for a record an earlier launch left behind. + mgr.SetInheritedRecordScope(InheritedRecordScopeAdopted) + row := &models.ExecutorRunning{SessionID: "session-inherited", Runtime: agentruntime.RuntimeStandalone} + scope := &standaloneLivenessScope{reachable: true, liveBySession: map[string]struct{}{}} + // Deliberately NOT marked via markSessionCreatedThisLifetime: this is an + // inherited row this backend never created. + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessUnknown { + t.Fatalf("classifyStandaloneLiveness = %v, want Unknown for an inherited record absent from the enumeration", got) + } +} + +func TestManagerRowLivenessTakesFreshEnumerationEachCall(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{{ID: "instance-1", SessionID: "session-1"}} + mgr := newLivenessTestManager(t, control) + // A caller outside Start's recovery pass only enumerates once recovery + // has completed for this process's lifetime (AC-EXECUTORS-SURVIVAL-003.6). + mgr.recoveryComplete.Store(true) + row := &models.ExecutorRunning{SessionID: "session-1", Runtime: agentruntime.RuntimeStandalone} + + mgr.RowLiveness(row) + mgr.RowLiveness(row) + + control.mu.Lock() + defer control.mu.Unlock() + if control.listInstancesAttempts != 2 { + t.Fatalf("listInstances attempts = %d, want exactly 2 (one fresh enumeration per out-of-pass call, never cached)", control.listInstancesAttempts) + } +} + +// TestManagerRowLivenessReturnsUnknownBeforeRecoveryCompletes pins Review +// round 1 finding 8 (AC-EXECUTORS-SURVIVAL-003.6): a caller outside a +// reconciliation pass (e.g. idle reclaim) firing before Start's recovery +// pass has finished for this process's lifetime must get Unknown immediately +// -- neither enumerate nor wait -- rather than racing a live enumeration +// against work recovery itself has not finished doing. +func TestManagerRowLivenessReturnsUnknownBeforeRecoveryCompletes(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{{ID: "instance-1", SessionID: "session-1"}} + mgr := newLivenessTestManager(t, control) + row := &models.ExecutorRunning{SessionID: "session-1", Runtime: agentruntime.RuntimeStandalone} + + got := mgr.RowLiveness(row) + if got != models.ProcessLivenessUnknown { + t.Fatalf("RowLiveness before recovery completes = %v, want Unknown", got) + } + + control.mu.Lock() + defer control.mu.Unlock() + if control.listInstancesAttempts != 0 { + t.Fatalf("listInstances attempts = %d, want 0 (must not enumerate before recovery completes)", control.listInstancesAttempts) + } +} + +func TestManagerRowLivenessScopedReusesSuppliedScope(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{{ID: "instance-1", SessionID: "session-1"}} + mgr := newLivenessTestManager(t, control) + rowA := &models.ExecutorRunning{SessionID: "session-1", Runtime: agentruntime.RuntimeStandalone} + rowB := &models.ExecutorRunning{SessionID: "session-2", Runtime: agentruntime.RuntimeStandalone} + + scope := mgr.NewStandaloneLivenessScope(context.Background()) + mgr.RowLivenessScoped(rowA, scope) + mgr.RowLivenessScoped(rowB, scope) + + control.mu.Lock() + defer control.mu.Unlock() + if control.listInstancesAttempts != 1 { + t.Fatalf("listInstances attempts = %d, want exactly 1 -- one enumeration reused across every row of the pass", control.listInstancesAttempts) + } +} + +// TestManagerClassifyStandaloneLivenessEnumerationFailedAgainstAServerIsUnknown +// pins the half of AC-EXECUTORS-SURVIVAL-003.6 that separates "nothing +// answered at the recorded endpoint" from "a server answered but could not be +// enumerated". Both reach classification as an unreachable scope, but only the +// former licenses the process-identifier probe: a standalone row carries the +// SHARED control server's identifier, so with a server still running the probe +// answers Alive for an instance that may already be gone -- reporting a record +// live solely because the shared control server is running, which that +// criterion forbids, and leaving the row to escape repair indefinitely. +func TestManagerClassifyStandaloneLivenessEnumerationFailedAgainstAServerIsUnknown(t *testing.T) { + // LocalPID is this process, which is certainly alive, so the + // process-identifier probe would answer Alive if it were consulted. + row := &models.ExecutorRunning{ + SessionID: "session-unknown", Runtime: agentruntime.RuntimeStandalone, LocalPID: os.Getpid(), + } + + for _, inherited := range []InheritedRecordScope{InheritedRecordScopeAdopted, InheritedRecordScopeForeignServer} { + for _, scope := range []*standaloneLivenessScope{nil, {reachable: false}} { + mgr := newLivenessTestManager(t, newStandaloneControlServer(t, true)) + mgr.SetInheritedRecordScope(inherited) + + got := mgr.classifyStandaloneLiveness(row, scope) + if got != models.ProcessLivenessUnknown { + t.Fatalf("inherited=%v scope=%+v: classifyStandaloneLiveness = %v, want Unknown when a server answered but its enumeration did not", + inherited, scope, got) + } + } + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager.go b/apps/backend/internal/agent/runtime/lifecycle/manager.go index 18277ee111..e0aeb6f295 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager.go @@ -89,6 +89,82 @@ type Manager struct { // permissions, so they get different checks. sessionExecCheck func(ctx context.Context, sessionID string) error + // recoveryGuard is the in-memory per-session acquire-or-observe map of + // AC-EXECUTORS-SURVIVAL-002.8, taken over the live standalone + // recovery-inventory records at startup step 3 before any control server + // is contacted, and consulted by Launch to refuse a concurrent request + // for a session that might still be re-tracking. Always non-nil. + recoveryGuard *RecoveryGuard + + // retrackedSessionsMu guards retrackedSessions. + retrackedSessionsMu sync.RWMutex + + // retrackedSessions is the set of session IDs this backend successfully + // re-tracked during its most recent Start() recovery pass + // (AC-EXECUTORS-SURVIVAL-003.1). Reset at the start of each Start() call, + // then populated once per recovered execution, right after it lands in + // executionStore -- never for an instance this pass explicitly refused + // to re-track (unreconstructable agent identity, unretrievable turn + // status). Queried by the orchestrator's startup reconciliation via + // WasSessionRetracked so a session whose agent actually survived the + // restart skips "backend died mid-turn" cleanup + // (AC-EXECUTORS-SURVIVAL-003.2/.3/.4/.5). Always non-nil after + // construction. + retrackedSessions map[string]struct{} + + // standaloneOwnSessionsMu guards standaloneOwnSessions. + standaloneOwnSessionsMu sync.RWMutex + + // standaloneOwnSessions is the set of session IDs whose executors_running + // row this backend itself created via Launch during this process's + // lifetime -- never populated by the recovery path (which adds directly + // to executionStore and marks retrackedSessions instead). Consulted by + // the standalone liveness classifier's own-record-vs-inherited-record + // carve-out (design 02 "Persistence": "A server this backend STARTED is + // not a server it adopted" -- a row this backend created is checkable + // against whichever control server this backend is currently using, + // adopted or freshly started, because that server is the only authority + // that can answer for it; an inherited row that server has never heard of + // classifies unknown instead of dead). Unlike retrackedSessions, this is + // NEVER reset by Start() -- it must accumulate for the whole process + // lifetime, not just one recovery pass. Always non-nil after + // construction. + standaloneOwnSessions map[string]struct{} + + // passthroughLookup reads a session's durable passthrough mode + // (TaskSession.IsPassthrough) for AC-EXECUTORS-SURVIVAL-005.3's startup + // guard exclusion. Nil = every live standalone session is guarded (the + // safe default: guarding a passthrough session by mistake only costs one + // refused launch, never excluding a real candidate). See + // SetPassthroughLookup. + passthroughLookup PassthroughLookup + + // recoveryDeadline and recoveryDeadlineStart implement the + // AC-EXECUTORS-SURVIVAL-003.7 single bound covering this startup pass's + // adoption+enumeration+reconstruction work. recoveryDeadlineStart is the + // instant this launch first contacted a recorded control endpoint (zero + // when it never did -- Start then falls back to its own invocation + // time); recoveryDeadline is the configured duration from that instant. + // A zero recoveryDeadline falls back to the AC's own 30s default. See + // SetRecoveryDeadline / SetRecoveryDeadlineStart. + recoveryDeadline time.Duration + recoveryDeadlineStart time.Time + + // agentSurvivalEnabled mirrors the features.agentSurvival runtime flag + // (config.Config.Features.AgentSurvival). When true and a stop's reason is + // StopReasonBackendShutdown, StopAgentWithReason takes the survivable-detach + // branch instead of the terminating one (design 01/02 kill-path #4). Set + // once during startup wiring via SetAgentSurvivalEnabled; false (today's + // unconditional terminating stop) is the correct zero value. + agentSurvivalEnabled bool + + // inheritedRecordScope records what this launch found at the recorded + // control endpoint, so classifyStandaloneLiveness can judge records + // inherited from an earlier launch correctly. Set once during startup + // wiring via SetInheritedRecordScope; the zero value (no server + // answered) is correct for a launch that never attempted adoption. + inheritedRecordScope InheritedRecordScope + // environmentAccessCheck is the environment-keyed sibling of // sessionAccessCheck, used by the terminal environment-shell route which // resolves executions by environment ID. Nil = no scoping. @@ -130,6 +206,15 @@ type Manager struct { // confusing errors against children already being stopped. shuttingDown atomic.Bool + // recoveryComplete is flipped true once Start's synchronous recovery + // pass (adoption, enumeration, and per-instance reconstruction) has + // finished for this process's lifetime. AC-EXECUTORS-SURVIVAL-003.6 + // requires a caller outside that pass (e.g. RowLiveness, the idle + // reclaim path) to answer Unknown without enumerating while recovery is + // still in flight, rather than racing a live enumeration against work + // recovery itself has not finished doing. + recoveryComplete atomic.Bool + // pollAggregator routes hub session-mode events to agentctl. See // manager_subscription.go. pollAggregator *workspacePollAggregator @@ -320,6 +405,9 @@ func NewManager( stopCh: stopCh, skillDeployer: NoopSkillDeployer(), remediateNpxCache: routingerr.RemediateNpxCache, + recoveryGuard: NewRecoveryGuard(), + retrackedSessions: make(map[string]struct{}), + standaloneOwnSessions: make(map[string]struct{}), } // Initialize stream manager with callbacks that delegate to manager methods // mcpHandler will be set later via SetMCPHandler. @@ -486,6 +574,123 @@ func (m *Manager) CheckSessionExecAccess(ctx context.Context, sessionID string) return m.sessionExecCheck(ctx, sessionID) } +// SetPassthroughLookup installs the durable passthrough-mode read used at +// startup step 3 to exclude passthrough sessions from the recovery guard +// (AC-EXECUTORS-SURVIVAL-005.3). Must read TaskSession.IsPassthrough from the +// durable store, never from in-memory execution state -- that state is empty +// at this point in startup, which would silently guard every passthrough +// session instead of excluding it. Set once during startup wiring, before +// Start runs. +func (m *Manager) SetPassthroughLookup(lookup PassthroughLookup) { + m.passthroughLookup = lookup +} + +// SetRecoveryGuard installs a pre-populated recovery guard, replacing the +// empty one NewManager created. Backend startup composition uses this to +// hand Start the same guard TakeStartupRecoveryGuards already took sessions +// on before any control server was contacted (AC-EXECUTORS-SURVIVAL-002.8) -- +// something that must happen ahead of an adoption attempt this Manager +// doesn't yet exist to perform itself. Start's own guard-taking is still +// safe to run afterward: AcquireOrObserve treats an already-guarded session +// as observed, not re-acquired. A nil guard is ignored so a caller with +// nothing pre-taken leaves the Manager's own default in place. +func (m *Manager) SetRecoveryGuard(guard *RecoveryGuard) { + if guard == nil { + return + } + m.recoveryGuard = guard +} + +// SetRecoveryDeadline installs the configured AC-EXECUTORS-SURVIVAL-003.7 +// recovery deadline duration. Zero (unset) falls back to the AC's own 30s +// default at Start. +func (m *Manager) SetRecoveryDeadline(d time.Duration) { + m.recoveryDeadline = d +} + +// SetRecoveryDeadlineStart installs the AC-EXECUTORS-SURVIVAL-003.7 recovery +// deadline's clock start: the instant this launch first contacted a recorded +// control endpoint. A zero value (no adoption attempted, or none recorded) +// falls back to Start's own invocation time. Set once during startup wiring, +// before Start runs. +func (m *Manager) SetRecoveryDeadlineStart(t time.Time) { + m.recoveryDeadlineStart = t +} + +// SetAgentSurvivalEnabled installs the features.agentSurvival capability +// state that StopAgentWithReason reads to decide between a survivable detach +// and today's terminating stop on backend shutdown. Set once during startup +// wiring, before Start runs. +func (m *Manager) SetAgentSurvivalEnabled(enabled bool) { + m.agentSurvivalEnabled = enabled +} + +// RecoveryGuard exposes the in-memory recovery guard so it can be wired into +// an ExecutorBackend that supports AC-EXECUTORS-SURVIVAL-002.16 (currently +// StandaloneExecutor, via SetUnstoppableSessionRecorder). Always non-nil. +func (m *Manager) RecoveryGuard() *RecoveryGuard { + return m.recoveryGuard +} + +// WasSessionRetracked reports whether sessionID was successfully re-tracked +// during this backend's most recent Start() recovery pass +// (AC-EXECUTORS-SURVIVAL-003.1). Safe for concurrent use. +func (m *Manager) WasSessionRetracked(sessionID string) bool { + if sessionID == "" { + return false + } + m.retrackedSessionsMu.RLock() + defer m.retrackedSessionsMu.RUnlock() + _, ok := m.retrackedSessions[sessionID] + return ok +} + +// markSessionRetracked records sessionID as re-tracked for the current +// Start() pass. See retrackedSessions. +func (m *Manager) markSessionRetracked(sessionID string) { + if sessionID == "" { + return + } + m.retrackedSessionsMu.Lock() + m.retrackedSessions[sessionID] = struct{}{} + m.retrackedSessionsMu.Unlock() +} + +// resetRetrackedSessions clears the re-tracked set at the start of a new +// Start() pass. +func (m *Manager) resetRetrackedSessions() { + m.retrackedSessionsMu.Lock() + m.retrackedSessions = make(map[string]struct{}) + m.retrackedSessionsMu.Unlock() +} + +// wasCreatedThisLifetime reports whether sessionID's executors_running row +// was created by THIS backend process via Launch, as opposed to inherited +// from an earlier launch and re-tracked at startup. See +// standaloneOwnSessions. Safe for concurrent use. +func (m *Manager) wasCreatedThisLifetime(sessionID string) bool { + if sessionID == "" { + return false + } + m.standaloneOwnSessionsMu.RLock() + defer m.standaloneOwnSessionsMu.RUnlock() + _, ok := m.standaloneOwnSessions[sessionID] + return ok +} + +// markSessionCreatedThisLifetime records sessionID as created by this +// backend process. See standaloneOwnSessions. Never cleared: unlike +// retrackedSessions this must survive for the whole process lifetime, not +// just one Start() pass. +func (m *Manager) markSessionCreatedThisLifetime(sessionID string) { + if sessionID == "" { + return + } + m.standaloneOwnSessionsMu.Lock() + m.standaloneOwnSessions[sessionID] = struct{}{} + m.standaloneOwnSessionsMu.Unlock() +} + // SetAttachmentReader wires the backend attachment reader used by prompt // dispatch. Claimed descriptors are streamed into the active agentctl // session immediately before an ACP prompt is sent. diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_events.go b/apps/backend/internal/agent/runtime/lifecycle/manager_events.go index 31c965b24a..7096205329 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_events.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_events.go @@ -278,7 +278,13 @@ func (m *Manager) claimPromptCompletion( claim.locked = true claimed := false err := m.executionStore.WithLock(execution.ID, func(current *AgentExecution) { - if current != execution || current.promptGeneration != event.PromptGeneration { + if current != execution { + return + } + if current.recoveredPromptGenerationPending.CompareAndSwap(true, false) { + current.promptGeneration = event.PromptGeneration + } + if current.promptGeneration != event.PromptGeneration { return } if current.promptCompletionGeneration == event.PromptGeneration { @@ -853,6 +859,15 @@ func (m *Manager) handleAgentEventWithAttempt( attemptID string, ) { event.AttemptID = attemptID + // A terminal event that was already applied from a retained turn outcome + // can be redelivered when the live stream attaches. Drop that exact event + // instead of applying the completion a second time. + if execution.isRecoveryDuplicateEvent(&event) { + m.logger.Debug("dropping live event: already applied via retained turn outcome", + zap.String("execution_id", execution.ID), + zap.Int64("control_turn_id", event.ControlTurnID)) + return + } if m.handleMCPAttachmentEvent(execution, &event, attemptID) { return } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go b/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go index e5efa774c4..7b9865ea73 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go @@ -1013,7 +1013,7 @@ func (m *Manager) prepareExecutionCreateRequest( PreviousExecutionID: info.AgentExecutionID, AuthToken: authToken, BootstrapNonce: m.revealRuntimeSecret(ctx, info.Metadata, MetadataKeyBootstrapNonceSecret), - AgentctlStartupConfig: m.agentctlStartupConfig, + AgentctlStartupConfig: agentctlStartupConfigForExecutor(m.agentctlStartupConfig, info.ExecutorType), RemoteContributions: remoteContributions, ContributionDestinations: contributionDestinations, ManagedRuntimeVersion: managedRuntimeVersion, diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go index 90951d4b65..4aee691234 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go @@ -970,6 +970,23 @@ func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, r // a potentially live runtime as idle. RemoveExecution releases the lease on // the successful path. + // AC-EXECUTORS-SURVIVAL kill-path #4 (design 01 "Kill paths that must + // change together", design 02 "Shutdown"): a graceful backend shutdown + // must not terminate the instance when the capability is enabled. Every + // other stop reason, including force, keeps the terminating path below. + // The capability covers only the standalone (worktree/local) runtime -- + // every other runtime's StopAllAgents call must still terminate normally + // even while the capability is globally enabled for the installation. + // A passthrough session is excluded too (AC-EXECUTORS-SURVIVAL-005.3, + // design 02 "Passthrough scope"): its agent runs on a terminal this + // backend process owns, so it dies with the backend regardless, and + // detaching would leave an executors_running row claiming a live agent + // with no agent.stopped published. See isPassthroughExecution. + if m.agentSurvivalEnabled && reason == StopReasonBackendShutdown && + execution.RuntimeName == executor.NameStandalone && !isPassthroughExecution(execution) { + return m.detachAgentExecution(executionID, execution) + } + m.logger.Info("stopping agent", zap.String("execution_id", executionID), zap.String("reason", reason), @@ -1025,6 +1042,52 @@ func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, r return nil } +// detachAgentExecution implements the AC-EXECUTORS-SURVIVAL survivable-detach +// branch of StopAgentWithReason (design 01/02 kill-path #4): when the +// agent-survival capability is enabled and the stop reason is graceful +// backend shutdown, this backend releases its own hold on the execution -- +// its agentctl stream subscription and in-memory tracking -- without +// stopping the agentctl-side instance or calling the runtime backend's +// StopInstance. The instance is left running, unowned by this backend, so +// the adoption path (Layer 6.1) or a fresh recovery pass on the next startup +// can find and re-track it. +// +// Deliberately does NOT: call client.Stop, call stopAgentViaBackend, set +// execution.Status to stopped, or publish events.AgentStopped. The execution +// is not stopped -- publishing that event would run the orchestrator's +// handleAgentStopped, which persists a terminal task-session state for a +// session whose agent is still running, corrupting the very state recovery +// depends on. The executors_running row is intentionally left untouched so +// AC-EXECUTORS-SURVIVAL-002's recovery-inventory read still finds it live. +// +// Also deliberately does NOT call client.Close(): closing the client tears +// down the agentctl updates stream, which makes any in-flight SendPrompt's +// stream-reader goroutine observe a disconnect and report it through the +// same error path a genuine agent crash uses (handleInitialPromptFailure / +// handleErrorEvent) -- persisting the executors_running row as failed out +// from under the very read this function's contract promises to leave +// untouched, and racing RemoveExecution below to do it. Simply dropping the +// local reference lets this backend's process exit (moments later, in the +// same shutdown) reclaim the socket without disturbing the still-running +// agentctl instance or its live stream. +func (m *Manager) detachAgentExecution(executionID string, execution *AgentExecution) error { + execution.agentctlLifecycleMu.Lock() + execution.detachAgentctlClient() + execution.agentctlLifecycleMu.Unlock() + + execution.EndSessionSpan() + m.RemoveExecution(executionID) + m.clearRemoteStatus(execution.SessionID) + + m.logger.Info("detached agent execution for survivable backend shutdown; instance left running", + zap.String("execution_id", executionID), + zap.String("session_id", execution.SessionID), + zap.String("task_id", execution.TaskID), + zap.Stringer("runtime", execution.RuntimeName)) + + return nil +} + func (m *Manager) stopExecutionAgentctl( ctx context.Context, executionID string, diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go index 3b0e4a408f..300d21e44b 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go @@ -1053,7 +1053,7 @@ func (m *Manager) launchBuildExecutorRequest(ctx context.Context, executionID st McpProfile: reqWithWorktree.McpProfile, AuthToken: launchAuthToken, BootstrapNonce: m.revealRuntimeSecret(ctx, metadata, MetadataKeyBootstrapNonceSecret), - AgentctlStartupConfig: m.agentctlStartupConfig, + AgentctlStartupConfig: agentctlStartupConfigForExecutor(m.agentctlStartupConfig, reqWithWorktree.ExecutorType), OnProgress: onProgress, RemoteContributions: remoteContributions, ContributionDestinations: contributionDestinations, @@ -1342,6 +1342,15 @@ func (m *Manager) publishLaunchPrepareCompleted(req *LaunchRequest, result *EnvP // If req.SessionID is empty (quick chat / pre-session contexts), no // deduplication key exists and we fall through to direct execution. func (m *Manager) Launch(ctx context.Context, req *LaunchRequest) (*AgentExecution, error) { + // AC-EXECUTORS-SURVIVAL-002.8/002.16: refuse rather than queue or block a + // launch for a session whose recovery guard is currently held. Checked + // first, before any activity lease or singleflight coalescing, so a + // guarded session can never partially acquire launch-path state. + if req.SessionID != "" { + if err := m.recoveryGuard.CheckLaunchAllowed(req.SessionID); err != nil { + return nil, err + } + } if req.SessionID == "" { activityLease, err := m.acquireActivity(ctx, activity.KindExecutionStarting) if err != nil { @@ -1807,6 +1816,12 @@ func (m *Manager) registerAndPublishExecution( } return fmt.Errorf("failed to register execution: %w", addErr) } + // This execution is durably in the store as of the Add above, and it got + // there via Launch -- never via the recovery path, which adds directly to + // executionStore and marks retrackedSessions instead -- so sessionID's row + // is "created this lifetime" from this point on (see + // standaloneOwnSessions). + m.markSessionCreatedThisLifetime(sessionID) isKubernetes := execution.RuntimeName == agentruntime.RuntimeKubernetes var createdRuntimeSecrets map[string]bool if isKubernetes { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go index f9899c45f4..5d3da0b07f 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go @@ -12,6 +12,7 @@ import ( "github.com/kandev/kandev/internal/agent/executor" "github.com/kandev/kandev/internal/agentctl/tracing" agentctltypes "github.com/kandev/kandev/internal/agentctl/types" + "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" ) @@ -43,17 +44,103 @@ func (m *Manager) Start(ctx context.Context) error { } } - // Try to recover executions from all runtimes - recovered, err := m.executorRegistry.RecoverAll(ctx) - if err != nil { - m.logger.Warn("failed to recover executions from some runtimes", zap.Error(err)) + // AC-EXECUTORS-SURVIVAL-003.1: start this pass's re-tracked-session set + // empty so a stale entry from an earlier Start() call (tests, or a + // hypothetical re-Start) never leaks into this pass's outcome. + m.resetRetrackedSessions() + + // Read the live standalone recovery-inventory records (startup step 3, + // AC-EXECUTORS-SURVIVAL-002.8) before recovery contacts any control + // server, and hand them to every runtime's RecoverInstances unchanged. + records, listErr := m.ListLiveStandaloneExecutorsRunning(ctx) + if listErr != nil { + // A failed read leaves the record set unknown, which is not the same + // thing as empty. AC-EXECUTORS-SURVIVAL-002.6 stops a live instance + // because it is known to have no record; passing an unknown set on as + // an empty one would make every live instance look record-less and + // send all of them down that orphan-stop path, so a transient database + // error during startup would kill every agent that just survived the + // restart. Recover nothing instead, on the same terms + // AC-EXECUTORS-SURVIVAL-002.12 sets for the mirror failure -- the + // adopted server that cannot be enumerated: report no recovered + // instances, stop no instance, and leave every record to the existing + // stale-execution repair path. + m.logger.Error("skipping recovery: live standalone recovery-inventory records could not be read, so no live instance can be correlated to a session", + zap.Error(listErr)) + records = nil } + + // Take a recovery guard for every named session except a confirmed + // passthrough one, BEFORE any control server is contacted, so a + // concurrent launch request cannot race re-tracking + // (AC-EXECUTORS-SURVIVAL-002.8, AC-EXECUTORS-SURVIVAL-005.3). No lookup + // configured is treated the same as a failed read: every session is + // guarded rather than excluded. + passthroughLookup := m.passthroughLookup + if passthroughLookup == nil { + passthroughLookup = func(context.Context, string) (bool, bool) { return false, false } + } + guardedSessions := SessionsToGuard(ctx, sessionIDsFromExecutorRunning(records), passthroughLookup) + for _, sessionID := range guardedSessions { + m.recoveryGuard.AcquireOrObserve(sessionID) + } + + // AC-EXECUTORS-SURVIVAL-005.3: the same classification also decides what + // re-tracking may reach. A confirmed-passthrough session owns a real + // agentctl instance that does outlive this backend, so without this the + // enumeration would correlate it to its record and re-track a session + // whose PTY agent died with the backend -- reporting it as having + // survived, which the AC forbids. Only standalone records reach here + // (ListLiveStandaloneExecutorsRunning), and StandaloneExecutor is the + // only backend that reads them, so this narrows nothing else. + records = recoverableRecords(records, guardedSessions) + + // AC-EXECUTORS-SURVIVAL-003.7: bound this pass's + // adoption+enumeration+reconstruction work with a single deadline, clocked + // from this launch's first control-server contact (or, absent one, from + // right now). recoveryCtx now covers not just RecoverAll (the producer) + // but every per-instance reconstruction step in the consumer loop below, + // per design 02's "Startup" steps 4-7 sharing one deadline. A backend + // implementation that honors ctx.Deadline() (see + // StandaloneExecutor.RecoverInstances) may still let work already in + // flight at the deadline finish in the background rather than aborting + // it -- cancelling recoveryCtx once this pass is over only stops the + // deadline timer itself from leaking, it never reaches that background + // work, which deliberately runs against context.Background() instead. + recoveryCtx, cancelRecovery := context.WithDeadline(ctx, m.recoveryDeadlineDeadline()) + var recovered []*ExecutorInstance + if listErr == nil { + var err error + recovered, err = m.executorRegistry.RecoverAll(recoveryCtx, records) + if err != nil { + m.logger.Warn("failed to recover executions from some runtimes", zap.Error(err)) + } + } + + var stopWG sync.WaitGroup if len(recovered) > 0 { for _, ri := range recovered { + if recoveryCtx.Err() != nil { + // AC-EXECUTORS-SURVIVAL-003.7: the deadline elapsed before this + // record could be reconstructed -- treat it as not re-tracked + // and stop it (against context.Background(), matching every + // other recovery-time stop: an elapsed deadline must not abort + // a stop already dispatched). + m.logger.Warn("recovery deadline elapsed before this record could be reconstructed; treating as not re-tracked", + zap.String("instance_id", ri.InstanceID), + zap.String("session_id", ri.SessionID)) + m.dispatchUnreconstructableStop(&stopWG, ri) + continue + } execution := &AgentExecution{ - ID: ri.InstanceID, - TaskID: ri.TaskID, - SessionID: ri.SessionID, + ID: ri.InstanceID, + TaskID: ri.TaskID, + SessionID: ri.SessionID, + // AC-EXECUTORS-SURVIVAL-002.14: agent profile identity's + // declared source is the recovery-inventory record's + // execution-profile column, carried onto ri by + // buildRecoveredInstances -- never the adopted instance. + AgentProfileID: ri.AgentProfileID, ContainerID: ri.ContainerID, ContainerIP: ri.ContainerIP, WorkspacePath: ri.WorkspacePath, @@ -65,6 +152,92 @@ func (m *Manager) Start(ctx context.Context) error { standaloneInstanceID: ri.StandaloneInstanceID, standalonePort: ri.StandalonePort, promptDoneCh: make(chan PromptCompletionSignal, 1), + // AC-EXECUTORS-SURVIVAL-002.14: run identity is re-derived from + // the runtime environment, which is itself read back from the + // adopted instance rather than the database (both deliberately + // memory-only, per design 03's "runtime environment" / + // "run identity" rows). + RunID: ri.Env["KANDEV_RUN_ID"], + // AC-EXECUTORS-SURVIVAL-002.14: workspace source roots are read + // back from the adopted instance, never pushed -- agentctl owns + // this allowlist and a rebind may have changed it since launch. + WorkspaceSourceRoots: ri.WorkspaceSourceRoots, + // AC-EXECUTORS-SURVIVAL-002.14: provider session identity comes + // from the adopted instance, which holds the live provider + // session -- never from a durable/database value, which could be + // stale relative to what the instance actually resumed. + ACPSessionID: ri.ProviderSessionID, + } + // AC-EXECUTORS-SURVIVAL-002.14's declared source for task + // identity is the recovery-inventory record alone, never the + // adopted instance (buildRecoveredInstances no longer falls + // back to the instance's self-reported task ID). An empty + // value here means the record itself is authoritatively + // missing it: refuse to re-track, per AC-EXECUTORS-SURVIVAL-002.4, + // rather than trusting the instance's own claim about which + // task it belongs to. + if execution.TaskID == "" { + m.logger.Error("refusing to re-track recovered execution: task identity was not present in the recovery-inventory record", + zap.String("instance_id", execution.ID), + zap.String("session_id", execution.SessionID)) + m.dispatchUnreconstructableStop(&stopWG, ri) + continue + } + // A recovered instance is resuming a live provider session (its + // ACPSessionID above comes straight from the adopted instance), so + // ACP session setup is already complete -- unlike a fresh launch, + // there is no InitializeAndPrompt call on this path to mark it. + // Leaving this false blocks SetSessionMode/SetSessionModel/ + // SetSessionConfigOption ("ACP session is not ready") on every + // re-tracked execution until its next full relaunch. + execution.setSessionInitialized(true) + execution.setRuntimeEnvironment(ri.Env) + if err := m.hydrateRecoveredTaskEnvironmentID(recoveryCtx, execution); err != nil { + m.logger.Error("refusing to re-track recovered execution: task environment identity could not be reconstructed", + zap.String("instance_id", execution.ID), + zap.String("session_id", execution.SessionID), + zap.Error(err)) + m.dispatchUnreconstructableStop(&stopWG, ri) + continue + } + // AC-EXECUTORS-SURVIVAL-002.14: Office profile identity is a new key + // in the same persisted metadata this record already carries, not a + // new source -- an empty value here is a legitimate non-Office + // launch, not a missing reconstruction. + execution.OfficeAgentProfileID = getMetadataString(ri.Metadata, MetadataKeyOfficeAgentProfileID) + // AC-EXECUTORS-SURVIVAL-002.14: agent identity, agent and + // continuation commands and their arguments, and the history + // setting are re-derived from the restored agent profile and the + // agent-type registry, never read from the instance. When the + // re-derivation cannot answer (AC-EXECUTORS-SURVIVAL-002.4: the + // declared source answered and the value is not present -- an + // unresolvable profile or an agent type no longer in the + // registry), this instance is authoritatively missing a required + // value: refuse to re-track it and leave it to the stop path + // AC-EXECUTORS-SURVIVAL-002.6 defines, rather than tracking a + // session the backend cannot safely operate. + if err := m.reDeriveRecoveredAgentIdentity(recoveryCtx, execution); err != nil { + m.logger.Error("refusing to re-track recovered execution: agent identity could not be reconstructed", + zap.String("instance_id", execution.ID), + zap.String("session_id", execution.SessionID), + zap.String("agent_profile_id", execution.AgentProfileID), + zap.Error(err)) + m.dispatchUnreconstructableStop(&stopWG, ri) + continue + } + // AC-EXECUTORS-SURVIVAL-004.2/004.5: retrieve this instance's + // retained turn status before this session's state is published. + // A read failure (retries exhausted) must not publish the session + // as running -- it is authoritatively unknown, not running -- so + // it takes the same not-re-tracked stop path as an + // unreconstructable agent identity above. + turnOutcome, turnOutcomeResult := m.retrieveRecoveredTurnOutcome(recoveryCtx, ri) + if turnOutcomeResult == recoveredTurnOutcomeReadFailed { + m.logger.Error("refusing to re-track recovered execution: turn status could not be retrieved", + zap.String("instance_id", execution.ID), + zap.String("session_id", execution.SessionID)) + m.dispatchUnreconstructableStop(&stopWG, ri) + continue } // Create trace span for the recovered session _, recoverySpan := tracing.TraceSessionRecovered( @@ -96,12 +269,17 @@ func (m *Manager) Start(ctx context.Context) error { continue } m.setRuntimeInterest(execution.SessionID, true) + // AC-EXECUTORS-SURVIVAL-003.1: this execution is durably in the + // store as of the Add above, so its session is re-tracked from + // this point on regardless of which branch below applies the + // retained turn outcome or publishes running. + m.markSessionRetracked(execution.SessionID) // Reconcile the persistence row to match the recovered in-memory ID. // If executors_running.agent_execution_id had drifted (e.g. from a // prior bug or manual edit), the recovered runtime instance is the // truth — overwrite the row to match. No-op if already in sync. - m.persistExecutorRunning(ctx, execution) + m.persistExecutorRunning(recoveryCtx, execution) // Re-seed the base-branch map before reconnecting. A surviving // agentctl instance never passes through waitForAgentctlReady, so @@ -109,11 +287,32 @@ func (m *Manager) Start(ctx context.Context) error { // if it was created by a path that could not supply one — and // their diff stats stay pinned to an integration-branch fallback. if client, releaseClient := execution.AcquireAgentCtlClient(); client != nil { - m.pushTaskBaseBranches(ctx, execution.TaskID, execution.ID, client) - m.pushTaskComparisonTargets(ctx, execution.TaskID, execution.ID, client) + m.pushTaskBaseBranches(recoveryCtx, execution.TaskID, execution.ID, client) + m.pushTaskComparisonTargets(recoveryCtx, execution.TaskID, execution.ID, client) releaseClient() } + // AC-EXECUTORS-SURVIVAL-004.2: apply the retained outcome, or + // publish running when nothing was retained, before streams + // reconnect below -- reconnecting is what would eventually + // deliver the same terminal event live (AC-EXECUTORS-SURVIVAL- + // 004.3/004.4), so this ordering makes the recovery-time + // application the one that wins in the common case. + if turnOutcomeResult == recoveredTurnOutcomeApplied { + m.applyRecoveredTurnOutcome(recoveryCtx, execution, ri, turnOutcome) + } else { + m.publishRecoveredExecutionRunning(recoveryCtx, execution) + } + + // AC-EXECUTORS-SURVIVAL-003.7/002.8: release this session's guard + // now that its outcome has actually been published, rather than + // waiting for every session in this pass to finish (design 02 + // step 7's "releasing each session's guard as its outcome is + // published"). ReleaseAllExceptRetained below still covers any + // session a later layer marked retained-unstoppable or + // stop-in-flight, and any session this loop never reached. + m.recoveryGuard.Release(execution.SessionID) + // Reconnect to workspace streams (shell, git, file changes) in background // This is needed so shell.input, git status, etc. work after backend restart go m.streamManager.ReconnectAll(execution) @@ -122,6 +321,32 @@ func (m *Manager) Start(ctx context.Context) error { } m.logger.Info("recovered executions", zap.Int("count", len(recovered))) } + cancelRecovery() + + // AC-EXECUTORS-SURVIVAL-002.11: every dispatchUnreconstructableStop call + // above ran on its own goroutine so one record's bounded-retry stop + // latency never starves another record's reconstruction of this pass's + // shared deadline budget. Wait for all of them to resolve (each one + // settles its own session's guard via RetainAsUnstoppable or Release + // before returning) so Start remains synchronous overall and + // ReleaseAllExceptRetained below never races a still-resolving stop. + stopWG.Wait() + + // Recovery is synchronous above: by this point every guarded session's + // outcome (re-tracked or not) is already decided, so every guard still + // held for this pass is released now. Most were already released + // individually inside the loop as each session's outcome was published; + // this covers a session whose record was skipped by the deadline check, + // a duplicate-execution anomaly, or anything a later layer marks + // retained-unstoppable or stop-in-flight (AC-EXECUTORS-SURVIVAL-002.16, + // AC-EXECUTORS-SURVIVAL-003.7), which ReleaseAllExceptRetained leaves held. + m.recoveryGuard.ReleaseAllExceptRetained() + + // AC-EXECUTORS-SURVIVAL-003.6: this pass's recovery work (adoption, + // enumeration, and every recovered instance's reconstruction above) is + // synchronous and now complete, so a caller outside a reconciliation + // pass may safely enumerate live standalone instances from this point on. + m.recoveryComplete.Store(true) // Start remote status polling loop for runtimes exposing remote status. m.wg.Add(1) @@ -152,6 +377,188 @@ func (m *Manager) Start(ctx context.Context) error { return nil } +// hydrateRecoveredTaskEnvironmentID fills in a recovered execution's +// task-environment identity (AC-EXECUTORS-SURVIVAL-002.14): the declared +// source is the durable store, via the session's own task-environment +// reference, not the adopted instance -- ExecutorInstance carries no such +// field at all. Task-environment identity is one of AC-EXECUTORS-SURVIVAL-002.3's +// required reconstructed values, so unlike an earlier version of this method +// (which logged and ignored both failure shapes below), neither is silently +// swallowed: a lookup failure is reported to the caller as an +// AC-EXECUTORS-SURVIVAL-002.13 failed read, and a durable store that answers +// successfully with no value at all is the AC-EXECUTORS-SURVIVAL-002.4 +// authoritatively-absent case. A nil provider is a capability this +// deployment never wired at all -- every production backend wires one via +// SetWorkspaceInfoProvider, only bare-bones test doubles don't -- so it is +// treated like any other absent optional recovery capability: skip, not +// refuse. +func (m *Manager) hydrateRecoveredTaskEnvironmentID(ctx context.Context, execution *AgentExecution) error { + if m.workspaceInfoProvider == nil { + return nil + } + info, err := m.workspaceInfoProvider.GetWorkspaceInfoForSession(ctx, execution.TaskID, execution.SessionID) + if err != nil { + return fmt.Errorf("task environment identity read failed: %w", err) + } + if info == nil || info.TaskEnvironmentID == "" { + return errors.New("task environment identity: durable store answered with no value") + } + execution.TaskEnvironmentID = info.TaskEnvironmentID + return nil +} + +// reDeriveRecoveredAgentIdentity implements AC-EXECUTORS-SURVIVAL-002.14's +// last reconstruction-table row: agent identity, agent and continuation +// commands and their arguments, and the history setting are re-derived from +// the restored agent profile and the agent-type registry -- the same +// computation an ordinary launch performs, never read from the adopted +// instance, so a compromised or stale instance cannot tell the backend which +// command to run for the session's next turn. +// +// Re-derivation reflects the profile and registry as they stand at recovery, +// not as they stood before the restart (AC-EXECUTORS-SURVIVAL-002.3): a +// profile edited or an agent type removed during the outage takes effect +// exactly as it would on the session's next ordinary turn. +// +// getAgentConfigForExecution and buildFreshAgentCommand are the same +// primitives an in-session context reset uses (manager_interaction.go's +// prepareAgentRestart), reused as-is: both already consume nothing but +// execution fields already restored by this point (AgentProfileID, +// RuntimeName, metadata), with no launch-request-only dependency. +func (m *Manager) reDeriveRecoveredAgentIdentity(ctx context.Context, execution *AgentExecution) error { + agentConfig, err := m.getAgentConfigForExecution(execution) + if err != nil { + return fmt.Errorf("agent identity: %w", err) + } + execution.AgentID = agentConfig.ID() + if rt := agentConfig.Runtime(); rt != nil { + execution.historyEnabled = rt.SessionConfig.HistoryContextInjection || + rt.SessionConfig.NewSessionOnWorkspaceRebind + } + commands, err := m.buildFreshAgentCommand(ctx, execution, agentConfig) + if err != nil { + return fmt.Errorf("agent and continuation commands: %w", err) + } + execution.AgentCommand = commands.initial + execution.ContinueCommand = commands.continue_ + execution.AgentArgs = commands.args + execution.ContinueArgs = commands.continueArgs + return nil +} + +// recoveryStopper is the optional capability satisfied by *StandaloneExecutor +// that stops a recovered instance within the bounded per-attempt timeout and +// retry count AC-EXECUTORS-SURVIVAL-002.15 requires for every recovery-time +// stop -- the same budget stopWithRetry already applies to the +// AC-EXECUTORS-SURVIVAL-002.6/002.10 loser/orphan stop paths in +// StandaloneExecutor.RecoverInstances. A backend that doesn't implement it +// falls back to a single StopInstance call: AC-EXECUTORS-SURVIVAL-002.15's +// retry contract covers the WORKTREE and LOCAL executor types only, both +// standalone-runtime. +type recoveryStopper interface { + stopWithRetry(ctx context.Context, instanceID string) error +} + +// dispatchUnreconstructableStop starts stopUnreconstructableRecoveredInstance +// on its own goroutine and registers it on wg, so one record's bounded-retry +// stop latency cannot block the caller's loop from reconstructing every +// other record within the same AC-EXECUTORS-SURVIVAL-003.7 shared deadline +// (AC-EXECUTORS-SURVIVAL-002.11: "no such instance's outcome shall depend on +// another's"). MarkStopInFlight runs synchronously first, before the loop +// can reach ReleaseAllExceptRetained, so this session's guard survives the +// bulk release until stopUnreconstructableRecoveredInstance resolves it +// (Release on success, RetainAsUnstoppable on failure). +func (m *Manager) dispatchUnreconstructableStop(wg *sync.WaitGroup, ri *ExecutorInstance) { + m.recoveryGuard.MarkStopInFlight(ri.SessionID) + wg.Add(1) + go func() { + defer wg.Done() + m.stopUnreconstructableRecoveredInstance(context.Background(), ri) + }() +} + +// stopUnreconstructableRecoveredInstance implements the +// AC-EXECUTORS-SURVIVAL-002.4 refusal path's "leave the instance to the stop +// path defined by AC-EXECUTORS-SURVIVAL-002.6" for an instance whose agent +// identity could not be re-derived: it was never added to the execution +// store, so RemoveExecution's teardown does not apply -- stop the live +// instance directly through its owning runtime backend. Run against +// context.Background(), matching every sibling AC-002.6/002.10 recovery stop +// (executor_standalone.go's dispatchRecoveryStops): an elapsed recovery +// deadline (AC-EXECUTORS-SURVIVAL-003.7) must not abort a stop already in +// flight. Every return path resolves this session's recovery guard +// (AC-EXECUTORS-SURVIVAL-002.16): a live instance this call could not stop +// (no backend for its runtime, or the stop itself failed) retains the guard +// for the rest of this backend's lifetime rather than letting it fall +// through to a bulk release that would let a fresh launch race the still-live +// instance; the instance is left to the existing stale-execution repair path +// like any other recovery-time stop failure that isn't part of the joint +// winner/loser accounting. +func (m *Manager) stopUnreconstructableRecoveredInstance(ctx context.Context, ri *ExecutorInstance) { + backend, err := m.executorRegistry.GetBackend(ri.RuntimeName) + if err != nil { + m.logger.Warn("cannot stop unreconstructable recovered instance: no backend for its runtime", + zap.String("instance_id", ri.InstanceID), + zap.String("session_id", ri.SessionID), + zap.String("runtime", string(ri.RuntimeName)), + zap.Error(err)) + m.recoveryGuard.RetainAsUnstoppable(ri.SessionID) + return + } + if stopper, ok := backend.(recoveryStopper); ok { + if err := stopper.stopWithRetry(context.Background(), ri.StandaloneInstanceID); err != nil { + m.logger.Warn("failed to stop unreconstructable recovered instance after exhausting retries", + zap.String("instance_id", ri.InstanceID), + zap.String("session_id", ri.SessionID), + zap.Error(err)) + m.recoveryGuard.RetainAsUnstoppable(ri.SessionID) + return + } + m.recoveryGuard.Release(ri.SessionID) + return + } + if err := backend.StopInstance(ctx, ri, true); err != nil { + m.logger.Warn("failed to stop unreconstructable recovered instance", + zap.String("instance_id", ri.InstanceID), + zap.String("session_id", ri.SessionID), + zap.Error(err)) + m.recoveryGuard.RetainAsUnstoppable(ri.SessionID) + return + } + m.recoveryGuard.Release(ri.SessionID) +} + +// defaultRecoveryDeadline is the AC-EXECUTORS-SURVIVAL-003.7 fallback used +// when no recovery deadline was configured (recoveryDeadline left zero). +const defaultRecoveryDeadline = 30 * time.Second + +// recoveryDeadlineDeadline resolves the wall-clock instant this startup +// pass's recovery work must stop admitting new work by, per +// AC-EXECUTORS-SURVIVAL-003.7. +func (m *Manager) recoveryDeadlineDeadline() time.Time { + start := m.recoveryDeadlineStart + if start.IsZero() { + start = time.Now() + } + period := m.recoveryDeadline + if period <= 0 { + period = defaultRecoveryDeadline + } + return start.Add(period) +} + +// sessionIDsFromExecutorRunning extracts the distinct, non-empty session +// identities named by a set of recovery-inventory records. +func sessionIDsFromExecutorRunning(records []*models.ExecutorRunning) []string { + sessionIDs := make([]string, 0, len(records)) + for _, rec := range records { + if rec != nil && rec.SessionID != "" { + sessionIDs = append(sessionIDs, rec.SessionID) + } + } + return sessionIDs +} + // GetRecoveredExecutions returns a snapshot of all currently tracked executions // This can be used by the orchestrator to sync with the database func (m *Manager) GetRecoveredExecutions() []RecoveredExecution { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_recovery_deadline_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_recovery_deadline_test.go new file mode 100644 index 0000000000..e619d6cd58 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_recovery_deadline_test.go @@ -0,0 +1,167 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/task/models" +) + +// deadlineRecoveringExecutor is a minimal ExecutorBackend that returns a +// fixed set of recovered instances from RecoverInstances (unlike +// guardObservingExecutor, which always returns none) and also implements +// the optional recoveryStopper capability, so a test can observe whether a +// recovered instance's not-re-tracked stop used the bounded-retry path. +type deadlineRecoveringExecutor struct { + name executor.Name + recovered []*ExecutorInstance + stopWithRetryCalls []string +} + +func (e *deadlineRecoveringExecutor) Name() executor.Name { return e.name } +func (e *deadlineRecoveringExecutor) HealthCheck(context.Context) error { return nil } +func (e *deadlineRecoveringExecutor) CreateInstance(context.Context, *ExecutorCreateRequest) (*ExecutorInstance, error) { + return nil, nil +} +func (e *deadlineRecoveringExecutor) StopInstance(context.Context, *ExecutorInstance, bool) error { + return nil +} +func (e *deadlineRecoveringExecutor) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*ExecutorInstance, error) { + return e.recovered, nil +} +func (e *deadlineRecoveringExecutor) GetInteractiveRunner() *process.InteractiveRunner { return nil } +func (e *deadlineRecoveringExecutor) RequiresCloneURL() bool { return false } +func (e *deadlineRecoveringExecutor) ShouldApplyPreferredShell() bool { return false } +func (e *deadlineRecoveringExecutor) IsAlwaysResumable() bool { return false } +func (e *deadlineRecoveringExecutor) stopWithRetry(_ context.Context, instanceID string) error { + e.stopWithRetryCalls = append(e.stopWithRetryCalls, instanceID) + return nil +} + +// TestManagerStartTreatsRecordsAsNotRetrackedOnceRecoveryDeadlineElapses +// pins Review round 1 finding 2 (AC-EXECUTORS-SURVIVAL-003.7): once the +// recovery deadline has elapsed, a record the consumer loop has not yet +// reconstructed must be treated as not re-tracked -- stopped via the bounded +// retry path and never added to the execution store or published running -- +// rather than reconstructed anyway on the ambient unbounded context. +func TestManagerStartTreatsRecordsAsNotRetrackedOnceRecoveryDeadlineElapses(t *testing.T) { + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + backend := &deadlineRecoveringExecutor{ + name: executor.NameStandalone, + recovered: []*ExecutorInstance{ + { + InstanceID: "exec-1", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: "profile-1", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-1", + }, + }, + } + execRegistry.Register(backend) + + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{{SessionID: "session-1"}}}) + // The recovery deadline already elapsed before Start() is even called: + // its clock started an hour ago and the configured bound is far shorter. + mgr.SetRecoveryDeadlineStart(time.Now().Add(-time.Hour)) + mgr.SetRecoveryDeadline(time.Millisecond) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + if _, exists := mgr.executionStore.GetBySessionID("session-1"); exists { + t.Fatal("session-1 was added to the execution store despite the recovery deadline having elapsed") + } + if len(backend.stopWithRetryCalls) != 1 || backend.stopWithRetryCalls[0] != "standalone-1" { + t.Fatalf("stopWithRetry calls = %v, want exactly one call for standalone-1", backend.stopWithRetryCalls) + } + // The guard must not be left held forever: ReleaseAllExceptRetained + // still runs after the loop even though this session was never + // individually released inside it. + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); err != nil { + t.Fatalf("expected session-1's guard released once Start finished, got %v", err) + } +} + +// guardObservingProfileResolver wraps a ProfileResolver and records the +// recovery guard's state for one session at the moment ResolveProfile is +// called -- a point squarely inside the consumer loop's per-instance +// reconstruction (reDeriveRecoveredAgentIdentity), well before that +// session's outcome is added to the execution store or published. +type guardObservingProfileResolver struct { + mgr *Manager + sessionID string + guardErrDuringResolve error + observed bool +} + +func (r *guardObservingProfileResolver) ResolveProfile(ctx context.Context, profileID string) (*AgentProfileInfo, error) { + r.guardErrDuringResolve = r.mgr.recoveryGuard.CheckLaunchAllowed(r.sessionID) + r.observed = true + return &AgentProfileInfo{ + ProfileID: profileID, + ProfileName: "Test Profile", + AgentID: "augment-agent", + AgentName: "auggie", + Model: "claude-sonnet-4-20250514", + }, nil +} + +// TestManagerStartHoldsGuardThroughReconstructionAndReleasesAfterPublish +// pins Review round 1 finding 3: a recovered session's guard must still be +// held while that session's instance is being reconstructed (identity +// re-derivation, turn-outcome read) and is released only once its outcome +// is actually published -- not released in bulk immediately after RecoverAll +// returns, before any reconstruction for it has even started. Under the +// pre-fix code the bulk release ran before this loop began at all, so the +// guard-observing hook below would have found it already released. +func TestManagerStartHoldsGuardThroughReconstructionAndReleasesAfterPublish(t *testing.T) { + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + backend := &deadlineRecoveringExecutor{ + name: executor.NameStandalone, + recovered: []*ExecutorInstance{ + { + InstanceID: "exec-1", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: "profile-1", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-1", + }, + }, + } + execRegistry.Register(backend) + + resolver := &guardObservingProfileResolver{sessionID: "session-1"} + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, resolver, nil, ExecutorFallbackWarn, "", log) + resolver.mgr = mgr + cleanupManagerStopCh(t, mgr) + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{{SessionID: "session-1"}}}) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + if !resolver.observed { + t.Fatal("ResolveProfile was never called; the guard-observing hook did not fire") + } + if !errors.Is(resolver.guardErrDuringResolve, ErrSessionRecoveryGuarded) { + t.Fatalf("guard state mid-reconstruction = %v, want ErrSessionRecoveryGuarded (guard must still be held)", resolver.guardErrDuringResolve) + } + if _, exists := mgr.executionStore.GetBySessionID("session-1"); !exists { + t.Fatal("session-1 was not re-tracked despite recovering well within the deadline") + } + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); err != nil { + t.Fatalf("expected session-1's guard released once its outcome was published, got %v", err) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go index f863884dde..733c20f278 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go @@ -13,6 +13,7 @@ import ( "github.com/kandev/kandev/internal/agent/executor" agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" ) @@ -335,7 +336,7 @@ func TestManagerStopAllAgentsPassesBackendShutdownReason(t *testing.T) { t.Fatalf("StopInstance reason = %q, want %q", mock.stopReason, StopReasonBackendShutdown) } } -func (m *mockStopTracker) RecoverInstances(ctx context.Context) ([]*ExecutorInstance, error) { +func (m *mockStopTracker) RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) { return nil, nil } func (m *mockStopTracker) GetInteractiveRunner() *process.InteractiveRunner { @@ -400,9 +401,10 @@ func TestManager_StartSeedsRecoveredExecution(t *testing.T) { execRegistry.Register(&MockExecutor{ name: executor.NameStandalone, recoverInstances: []*ExecutorInstance{{ - InstanceID: "exec-recovered", - TaskID: "task-recovered", - Client: client, + InstanceID: "exec-recovered", + TaskID: "task-recovered", + AgentProfileID: "profile-1", + Client: client, }}, }) mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) @@ -424,3 +426,39 @@ func TestManager_StartSeedsRecoveredExecution(t *testing.T) { t.Fatal("timed out waiting for recovered execution base-branch seed") } } + +// TestManagerStartRefusesRecoveredExecutionWithEmptyTaskID pins Review round +// 3, finding 3: a recovered instance whose TaskID is empty (the +// recovery-inventory record's declared source answered with nothing -- +// executor_standalone.go's buildRecoveredInstances no longer falls back to +// the instance's own claim) is authoritatively missing a required value +// (AC-EXECUTORS-SURVIVAL-002.4): it must never be published as a tracked +// execution, and is instead stopped through the AC-EXECUTORS-SURVIVAL-002.6 +// stop path. +func TestManagerStartRefusesRecoveredExecutionWithEmptyTaskID(t *testing.T) { + log := newTestRegistryLogger() + execRegistry := NewExecutorRegistry(log) + mockExec := &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-no-task", + TaskID: "", + SessionID: "session-no-task", + RuntimeName: executor.NameStandalone, + }}, + } + execRegistry.Register(mockExec) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + t.Cleanup(func() { _ = mgr.Stop() }) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + if _, ok := mgr.GetExecution("exec-no-task"); ok { + t.Fatal("an execution with an empty TaskID must not be tracked") + } + if len(mockExec.stopInstanceCalls) != 1 || mockExec.stopInstanceCalls[0].InstanceID != "exec-no-task" { + t.Fatalf("stopInstanceCalls = %+v, want the unreconstructable instance stopped", mockExec.stopInstanceCalls) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_liveness_ownership_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_liveness_ownership_test.go new file mode 100644 index 0000000000..e5408ca6f4 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_liveness_ownership_test.go @@ -0,0 +1,59 @@ +package lifecycle + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestManagerLaunchMarksSessionCreatedThisLifetime pins the own-record side +// of design 02's "own-record-created-this-lifetime vs inherited-record" +// carve-out (Layer 6.7): a session launched fresh through Manager.Launch is +// "created this lifetime", never a session Manager has simply heard of +// through some other path. +func TestManagerLaunchMarksSessionCreatedThisLifetime(t *testing.T) { + mgr, _ := newEnvironmentExecutionTestManager(t, nil) + + if mgr.wasCreatedThisLifetime("session-launched") { + t.Fatal("expected session to NOT be marked before Launch is called") + } + + _, err := mgr.Launch(context.Background(), &LaunchRequest{ + TaskID: "task-launched", + SessionID: "session-launched", + AgentProfileID: "profile-launched", + ExecutorType: string(models.ExecutorTypeLocal), + IsEphemeral: true, + }) + if err != nil { + t.Fatalf("Manager.Launch: %v", err) + } + + if !mgr.wasCreatedThisLifetime("session-launched") { + t.Fatal("expected session to be marked created-this-lifetime after a successful Launch") + } + if mgr.wasCreatedThisLifetime("some-other-session") { + t.Fatal("expected an unrelated session to remain unmarked") + } + if mgr.wasCreatedThisLifetime("") { + t.Fatal("expected an empty session ID to never be marked") + } +} + +// TestManagerMarkSessionCreatedThisLifetimeNeverReset pins that, unlike +// retrackedSessions, standaloneOwnSessions must survive across a Start() +// call -- it needs to accumulate for the whole process lifetime, not just +// one recovery pass. +func TestManagerMarkSessionCreatedThisLifetimeNeverReset(t *testing.T) { + mgr, _ := newEnvironmentExecutionTestManager(t, nil) + mgr.markSessionCreatedThisLifetime("session-own") + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Manager.Start: %v", err) + } + + if !mgr.wasCreatedThisLifetime("session-own") { + t.Fatal("expected standaloneOwnSessions to survive a Start() call, unlike retrackedSessions") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough_scope_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough_scope_test.go new file mode 100644 index 0000000000..2d117771e1 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough_scope_test.go @@ -0,0 +1,145 @@ +package lifecycle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/task/models" +) + +// TestStopAgentWithReasonTerminatesPassthroughOnBackendShutdownWhenSurvivalEnabled +// pins AC-EXECUTORS-SURVIVAL-005.3's "never detached" clause. A passthrough +// session's agent runs on a terminal owned by this backend process, so it dies +// with the backend whatever the control server does. StopAllAgents calls +// StopAgentWithReason with StopReasonBackendShutdown for every tracked +// execution, including passthrough ones on the standalone runtime, so without +// the exclusion such a session takes the survivable-detach branch: no +// StopInstance, no agent.stopped, and an executors_running row left claiming a +// live agent that no later pass repairs (AC-EXECUTORS-SURVIVAL-004.5). +func TestStopAgentWithReasonTerminatesPassthroughOnBackendShutdownWhenSurvivalEnabled(t *testing.T) { + mgr, mockExecutor := newDetachTestManager(t, true) + execution := newDetachTestExecution("exec-passthrough", "session-passthrough") + execution.IsPassthrough = true + require.NoError(t, mgr.executionStore.Add(execution)) + + err := mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonBackendShutdown, false) + require.NoError(t, err) + + require.Len(t, mockExecutor.stopInstanceCalls, 1, + "a passthrough session must behave as if the capability were disabled: the runtime instance is still stopped") + + eventBus := mgr.eventBus.(*MockEventBus) + require.True(t, hasAgentStoppedEvent(eventBus.PublishedEvents), + "a passthrough session must still publish agent.stopped: its PTY agent is genuinely gone") +} + +// TestStopAgentWithReasonTerminatesPassthroughIdentifiedByProcessID covers the +// second half of the in-memory passthrough idiom this package already uses +// (IsAgentReadyForPrompt, RecoverAgentPromptStream): an execution whose +// terminal process exists is passthrough even if the flag was not carried. +func TestStopAgentWithReasonTerminatesPassthroughIdentifiedByProcessID(t *testing.T) { + mgr, mockExecutor := newDetachTestManager(t, true) + execution := newDetachTestExecution("exec-passthrough-pid", "session-passthrough-pid") + execution.PassthroughProcessID = "pty-1" + require.NoError(t, mgr.executionStore.Add(execution)) + + err := mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonBackendShutdown, false) + require.NoError(t, err) + + require.Len(t, mockExecutor.stopInstanceCalls, 1, + "an execution carrying a terminal process is passthrough and must take the terminating path") + eventBus := mgr.eventBus.(*MockEventBus) + require.True(t, hasAgentStoppedEvent(eventBus.PublishedEvents), "a terminating stop must publish agent.stopped") +} + +// TestManagerStartExcludesPassthroughRecordFromRecovery pins +// AC-EXECUTORS-SURVIVAL-005.3's "never re-tracked" clause at the seam that +// decides it. A passthrough session owns a real agentctl instance -- created +// by CreateInstance like any other execution, or promoted from a +// workspace-only one -- so that instance really does outlive this backend and +// would correlate to the session's recovery-inventory record. Handing the +// record to RecoverInstances is therefore all it takes to re-track a session +// whose agent is gone and report it as having survived. +func TestManagerStartExcludesPassthroughRecordFromRecovery(t *testing.T) { + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + + var recovered []*models.ExecutorRunning + mock := &guardObservingExecutor{ + name: executor.NameStandalone, + onRecoverInstances: func(_ context.Context, records []*models.ExecutorRunning) { + recovered = records + }, + } + execRegistry.Register(mock) + + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{ + {SessionID: "session-normal"}, + {SessionID: "session-passthrough"}, + {SessionID: "session-unreadable"}, + }}) + mgr.SetPassthroughLookup(func(_ context.Context, sessionID string) (bool, bool) { + switch sessionID { + case "session-passthrough": + return true, true + case "session-unreadable": + return false, false + default: + return false, true + } + }) + + require.NoError(t, mgr.Start(context.Background())) + + sessions := make([]string, 0, len(recovered)) + for _, record := range recovered { + sessions = append(sessions, record.SessionID) + } + require.NotContains(t, sessions, "session-passthrough", + "a confirmed-passthrough record must never reach re-tracking") + require.Contains(t, sessions, "session-normal", "an ordinary standalone record must still be recovered") + require.Contains(t, sessions, "session-unreadable", + "a session whose passthrough mode could not be read is guarded rather than excluded, so it stays recoverable") +} + +// TestRecoverableRecordsKeepsEverythingButConfirmedPassthrough pins the +// filter's exact scope: it removes only records the pass classified as +// passthrough. A record with no session identity is not something the pass +// could classify either way, and correlation already skips it, so dropping it +// here would narrow recovery for a reason the AC does not give. +func TestRecoverableRecordsKeepsEverythingButConfirmedPassthrough(t *testing.T) { + records := []*models.ExecutorRunning{ + {SessionID: "guarded-a"}, + {SessionID: "passthrough"}, + {SessionID: ""}, + nil, + {SessionID: "guarded-b"}, + } + + got := recoverableRecords(records, []string{"guarded-a", "guarded-b"}) + + sessions := make([]string, 0, len(got)) + for _, record := range got { + require.NotNil(t, record) + sessions = append(sessions, record.SessionID) + } + require.Equal(t, []string{"guarded-a", "", "guarded-b"}, sessions) +} + +// TestRecoverableRecordsKeepsEveryRecordWhenNothingIsExcluded guards the +// no-passthrough case, which is every ordinary installation: the filter must +// be a pass-through when SessionsToGuard excluded nothing. +func TestRecoverableRecordsKeepsEveryRecordWhenNothingIsExcluded(t *testing.T) { + records := []*models.ExecutorRunning{{SessionID: "a"}, {SessionID: "b"}} + got := recoverableRecords(records, []string{"a", "b"}) + require.Len(t, got, 2) + require.Equal(t, records[0], got[0]) + require.Equal(t, records[1], got[1]) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_duplicate_session_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_duplicate_session_test.go new file mode 100644 index 0000000000..a16e2e82e2 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_duplicate_session_test.go @@ -0,0 +1,91 @@ +package lifecycle + +import ( + "context" + "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/common/logger" +) + +// TestManagerStartSkipsDuplicateSessionAcrossRuntimesDuringRecovery pins +// AC-EXECUTORS-SURVIVAL-002.9: if two different runtime backends each report +// a live, otherwise-reconstructable instance for the very same session ID in +// one Manager.Start recovery pass (the "should not happen... DB consistency +// issue" case manager_lifecycle.go's executionStore.Add call already guards), +// exactly one of them lands in the store -- never two, and never a panic or +// corrupted index. CorrelateRecoveryInstances' per-runtime Winners map cannot +// itself produce this (it is keyed by session ID within one runtime), but +// RecoverAll concatenates every registered runtime's winners without any +// cross-runtime session dedup, so this collision is reachable in practice +// when the same session is (incorrectly) live on two runtimes at once. +func TestManagerStartSkipsDuplicateSessionAcrossRuntimesDuringRecovery(t *testing.T) { + core, logs := observer.New(zapcore.ErrorLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatal(err) + } + + registry := NewExecutorRegistry(log) + standaloneBackend := &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-standalone", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "std-inst-1", + }}, + } + dockerBackend := &MockExecutor{ + name: executor.NameDocker, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-docker", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameDocker, + }}, + } + registry.Register(standaloneBackend) + registry.Register(dockerBackend) + + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v", err) + } + + execution, ok := mgr.GetExecutionBySessionID("session-1") + if !ok { + t.Fatal("expected exactly one of the two colliding instances to be tracked for session-1") + } + if execution.ID != "exec-standalone" && execution.ID != "exec-docker" { + t.Fatalf("tracked execution id = %q, want one of exec-standalone/exec-docker", execution.ID) + } + + // The loser must never be independently reachable: it was rejected by + // ExecutionStore.Add before insertion, not raced into the map afterward. + loserID := "exec-docker" + if execution.ID == loserID { + loserID = "exec-standalone" + } + if _, ok := mgr.GetExecution(loserID); ok { + t.Fatalf("loser execution %q was unexpectedly tracked alongside the winner", loserID) + } + + entries := logs.FilterMessage("skipping duplicate execution during recovery").All() + if len(entries) != 1 { + t.Fatalf("duplicate-skip log entries = %d, want exactly 1", len(entries)) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_guard_wiring_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_guard_wiring_test.go new file mode 100644 index 0000000000..6454e21f99 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_guard_wiring_test.go @@ -0,0 +1,183 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/task/models" +) + +// guardObservingExecutor is a minimal ExecutorBackend whose RecoverInstances +// calls back into the test so it can observe recovery-guard state from +// inside the same synchronous call Manager.Start makes, before +// ReleaseAllExceptRetained runs. +type guardObservingExecutor struct { + name executor.Name + onRecoverInstances func(ctx context.Context, records []*models.ExecutorRunning) + stopInstanceCalls int +} + +func (e *guardObservingExecutor) Name() executor.Name { return e.name } +func (e *guardObservingExecutor) HealthCheck(context.Context) error { return nil } +func (e *guardObservingExecutor) CreateInstance(context.Context, *ExecutorCreateRequest) (*ExecutorInstance, error) { + return nil, nil +} +func (e *guardObservingExecutor) StopInstance(context.Context, *ExecutorInstance, bool) error { + e.stopInstanceCalls++ + return nil +} +func (e *guardObservingExecutor) RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) { + if e.onRecoverInstances != nil { + e.onRecoverInstances(ctx, records) + } + return nil, nil +} +func (e *guardObservingExecutor) GetInteractiveRunner() *process.InteractiveRunner { return nil } +func (e *guardObservingExecutor) RequiresCloneURL() bool { return false } +func (e *guardObservingExecutor) ShouldApplyPreferredShell() bool { return false } +func (e *guardObservingExecutor) IsAlwaysResumable() bool { return false } + +// TestManagerStartGuardsLiveStandaloneSessionsExceptPassthroughAndReleasesAfterRecovery +// pins AC-EXECUTORS-SURVIVAL-002.8/005.3: every session named by a live +// standalone recovery-inventory record is guarded before any runtime's +// RecoverInstances is called, a confirmed-passthrough session is excluded, +// and every non-retained guard is released once recovery completes. +func TestManagerStartGuardsLiveStandaloneSessionsExceptPassthroughAndReleasesAfterRecovery(t *testing.T) { + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + + var mgr *Manager + var guardedErr, passthroughErr error + mock := &guardObservingExecutor{ + name: executor.NameStandalone, + onRecoverInstances: func(context.Context, []*models.ExecutorRunning) { + guardedErr = mgr.recoveryGuard.CheckLaunchAllowed("session-guarded") + passthroughErr = mgr.recoveryGuard.CheckLaunchAllowed("session-passthrough") + }, + } + execRegistry.Register(mock) + + mgr = NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{ + {SessionID: "session-guarded"}, + {SessionID: "session-passthrough"}, + }}) + mgr.SetPassthroughLookup(func(_ context.Context, sessionID string) (bool, bool) { + return sessionID == "session-passthrough", true + }) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + if guardedErr == nil { + t.Fatal("expected session-guarded to be guarded during recovery") + } + if !errors.Is(guardedErr, ErrSessionRecoveryGuarded) { + t.Fatalf("session-guarded guard check = %v, want ErrSessionRecoveryGuarded", guardedErr) + } + if passthroughErr != nil { + t.Fatalf("expected passthrough session to be excluded from guarding, got %v", passthroughErr) + } + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-guarded"); err != nil { + t.Fatalf("expected guard released once recovery finished, got %v", err) + } +} + +// TestManagerStartGuardsSessionWhenPassthroughLookupUnconfigured pins the +// documented default: with no SetPassthroughLookup call, every live +// standalone session is guarded rather than excluded. +func TestManagerStartGuardsSessionWhenPassthroughLookupUnconfigured(t *testing.T) { + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + + var mgr *Manager + var guardedErr error + mock := &guardObservingExecutor{ + name: executor.NameStandalone, + onRecoverInstances: func(context.Context, []*models.ExecutorRunning) { + guardedErr = mgr.recoveryGuard.CheckLaunchAllowed("session-no-lookup") + }, + } + execRegistry.Register(mock) + + mgr = NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{ + {SessionID: "session-no-lookup"}, + }}) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + if !errors.Is(guardedErr, ErrSessionRecoveryGuarded) { + t.Fatalf("session-no-lookup guard check = %v, want ErrSessionRecoveryGuarded", guardedErr) + } +} + +// TestManagerLaunchRefusedForGuardedSession pins that Launch checks the +// recovery guard before doing anything else (AC-EXECUTORS-SURVIVAL-002.8): +// a session whose guard is currently held is refused immediately with the +// retryable sentinel, not queued behind recovery. +func TestManagerLaunchRefusedForGuardedSession(t *testing.T) { + mgr := newTestManager(t) + mgr.recoveryGuard.AcquireOrObserve("session-guarded") + + execution, err := mgr.Launch(context.Background(), &LaunchRequest{ + TaskID: "task-guarded", + SessionID: "session-guarded", + }) + if execution != nil { + t.Fatalf("expected nil execution, got %+v", execution) + } + if !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("Launch error = %v, want ErrSessionRecoveryGuarded", err) + } +} + +// TestManagerLaunchRefusedForRetainedUnstoppableSession pins the +// AC-EXECUTORS-SURVIVAL-002.16 non-retryable refusal. +func TestManagerLaunchRefusedForRetainedUnstoppableSession(t *testing.T) { + mgr := newTestManager(t) + mgr.recoveryGuard.RetainAsUnstoppable("session-unstoppable") + + execution, err := mgr.Launch(context.Background(), &LaunchRequest{ + TaskID: "task-unstoppable", + SessionID: "session-unstoppable", + }) + if execution != nil { + t.Fatalf("expected nil execution, got %+v", execution) + } + if !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("Launch error = %v, want ErrSessionUnstoppableAgent", err) + } +} + +// TestManagerLaunchProceedsForUnguardedSession pins that the new guard check +// does not interfere with an ordinary launch for a session that was never +// guarded. +func TestManagerLaunchProceedsForUnguardedSession(t *testing.T) { + mgr, backend := newEnvironmentExecutionTestManager(t, nil) + + _, err := mgr.Launch(context.Background(), &LaunchRequest{ + TaskID: "task-unguarded", + SessionID: "session-unguarded", + AgentProfileID: "profile-unguarded", + ExecutorType: string(models.ExecutorTypeLocal), + IsEphemeral: true, + }) + if err != nil { + t.Fatalf("Manager.Launch: %v", err) + } + if backend.lastRequest == nil { + t.Fatal("expected executor CreateInstance to be called for an unguarded session") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_inventory_unknown_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_inventory_unknown_test.go new file mode 100644 index 0000000000..bb34db2531 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_inventory_unknown_test.go @@ -0,0 +1,103 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + "time" + + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/task/models" +) + +// TestManagerStartUnreadableRecoveryInventoryStopsNothing pins that a failed +// read of the live standalone recovery-inventory records is treated as an +// UNKNOWN record set, never an empty one. +// +// AC-EXECUTORS-SURVIVAL-002.6 stops a live instance because it is known to +// have no record. Correlation cannot tell an empty inventory from an unread +// one, so handing it the empty slice a failed read returns makes every live +// instance look record-less and sends all of them down that orphan-stop path: +// one transient database error during startup would stop every agent that had +// just survived the restart. The outcome required here is the one +// AC-EXECUTORS-SURVIVAL-002.12 sets for the mirror failure -- recover +// nothing, stop nothing, leave every record to the existing repair path. +func TestManagerStartUnreadableRecoveryInventoryStopsNothing(t *testing.T) { + control := newStandaloneControlServer(t, true) + // Two live instances that WOULD each be stopped as a record-less orphan + // if an unreadable inventory were passed on as an empty one. + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "instance-1", SessionID: "session-1"}, + {ID: "instance-2", SessionID: "session-2"}, + } + + mgr := newLivenessTestManager(t, control) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + mgr.SetExecutorRunningWriter(&listingWriter{err: errors.New("database unavailable")}) + mgr.SetRecoveryDeadline(5 * time.Second) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v, want an unreadable inventory to be survivable at startup", err) + } + + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 0 { + t.Fatalf("deleted = %v, want none: an unreadable recovery inventory must not make live instances look record-less", control.deleted) + } +} + +// TestManagerStartUnreadableRecoveryInventoryRetracksNothing pins the other +// half of the same outcome: nothing is re-tracked either. Re-tracking off an +// unknown inventory would publish sessions whose identity could not be +// correlated to a record. +func TestManagerStartUnreadableRecoveryInventoryRetracksNothing(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "instance-1", SessionID: "session-1"}, + } + + mgr := newLivenessTestManager(t, control) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + mgr.SetExecutorRunningWriter(&listingWriter{err: errors.New("database unavailable")}) + mgr.SetRecoveryDeadline(5 * time.Second) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v", err) + } + + if _, ok := mgr.GetExecutionBySessionID("session-1"); ok { + t.Fatal("session-1 was re-tracked from an unreadable recovery inventory; nothing may be correlated to an unknown record set") + } +} + +// TestManagerStartEmptyRecoveryInventoryStillStopsOrphans pins the boundary +// the fix above must not blur: a SUCCESSFUL read that returns no records is +// authoritative, and a live instance really is a record-less orphan then, so +// AC-EXECUTORS-SURVIVAL-002.6 still stops it. Without this, treating an +// unreadable inventory as unknown could be over-applied to an empty one and +// leave genuine orphans running. +func TestManagerStartEmptyRecoveryInventoryStillStopsOrphans(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.listInstances = []*agentctlclient.InstanceInfo{ + {ID: "instance-1", SessionID: "session-1"}, + } + + mgr := newLivenessTestManager(t, control) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{}}) + mgr.SetRecoveryDeadline(5 * time.Second) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v", err) + } + + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 1 || control.deleted[0] != "instance-1" { + t.Fatalf("deleted = %v, want [instance-1]: an authoritative empty inventory still makes a live instance an orphan", control.deleted) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_stop_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_stop_test.go new file mode 100644 index 0000000000..a731e9767d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_stop_test.go @@ -0,0 +1,352 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" +) + +// recoveryStoppingExecutor is a minimal ExecutorBackend that also implements +// the optional recoveryStopper capability, so a test can observe which path +// stopUnreconstructableRecoveredInstance took. +type recoveryStoppingExecutor struct { + name executor.Name + stopInstanceCalls int + stopWithRetryCalls []string + stopWithRetryErr error +} + +func (e *recoveryStoppingExecutor) Name() executor.Name { return e.name } +func (e *recoveryStoppingExecutor) HealthCheck(context.Context) error { return nil } +func (e *recoveryStoppingExecutor) CreateInstance(context.Context, *ExecutorCreateRequest) (*ExecutorInstance, error) { + return nil, nil +} +func (e *recoveryStoppingExecutor) StopInstance(context.Context, *ExecutorInstance, bool) error { + e.stopInstanceCalls++ + return nil +} +func (e *recoveryStoppingExecutor) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*ExecutorInstance, error) { + return nil, nil +} +func (e *recoveryStoppingExecutor) GetInteractiveRunner() *process.InteractiveRunner { return nil } +func (e *recoveryStoppingExecutor) RequiresCloneURL() bool { return false } +func (e *recoveryStoppingExecutor) ShouldApplyPreferredShell() bool { return false } +func (e *recoveryStoppingExecutor) IsAlwaysResumable() bool { return false } +func (e *recoveryStoppingExecutor) stopWithRetry(_ context.Context, instanceID string) error { + e.stopWithRetryCalls = append(e.stopWithRetryCalls, instanceID) + return e.stopWithRetryErr +} + +func newRecoveryStopTestManager(t *testing.T, backend ExecutorBackend) *Manager { + t.Helper() + log := newTestLogger() + execRegistry := NewExecutorRegistry(log) + execRegistry.Register(backend) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + return mgr +} + +// TestStopUnreconstructableRecoveredInstanceUsesBoundedRetryWhenSupported +// pins Review round 1 finding 6 (AC-EXECUTORS-SURVIVAL-002.15): a standalone +// backend's bounded-retry stop path must be used for an unreconstructable +// recovered instance, matching the AC-002.6/002.10 loser/orphan stop paths +// in the same package -- not a single unretried StopInstance call. +func TestStopUnreconstructableRecoveredInstanceUsesBoundedRetryWhenSupported(t *testing.T) { + backend := &recoveryStoppingExecutor{name: executor.NameStandalone} + mgr := newRecoveryStopTestManager(t, backend) + + ri := &ExecutorInstance{ + InstanceID: "exec-1", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-1", + } + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if len(backend.stopWithRetryCalls) != 1 || backend.stopWithRetryCalls[0] != "standalone-1" { + t.Fatalf("stopWithRetry calls = %v, want exactly one call for standalone-1", backend.stopWithRetryCalls) + } + if backend.stopInstanceCalls != 0 { + t.Fatalf("StopInstance calls = %d, want 0 (bounded-retry path should be used instead)", backend.stopInstanceCalls) + } +} + +// TestStopUnreconstructableRecoveredInstanceLogsRetryExhaustion pins that a +// bounded-retry failure is logged rather than silently dropped +// (AC-EXECUTORS-SURVIVAL-002.4/002.15): the warning must actually be emitted, +// carrying the instance identity and the underlying error, not just leave the +// call completing without panicking and without falling back to StopInstance. +func TestStopUnreconstructableRecoveredInstanceLogsRetryExhaustion(t *testing.T) { + retryErr := errors.New("boom") + backend := &recoveryStoppingExecutor{name: executor.NameStandalone, stopWithRetryErr: retryErr} + core, logs := observer.New(zapcore.WarnLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatal(err) + } + execRegistry := NewExecutorRegistry(log) + execRegistry.Register(backend) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + + ri := &ExecutorInstance{ + InstanceID: "exec-1", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-1", + } + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if len(backend.stopWithRetryCalls) != 1 { + t.Fatalf("stopWithRetry calls = %d, want 1", len(backend.stopWithRetryCalls)) + } + if backend.stopInstanceCalls != 0 { + t.Fatalf("StopInstance calls = %d, want 0 even when the retry path fails", backend.stopInstanceCalls) + } + + entries := logs.FilterMessage("failed to stop unreconstructable recovered instance after exhausting retries").All() + if len(entries) != 1 { + t.Fatalf("warn entries = %d, want 1", len(entries)) + } + fields := entries[0].ContextMap() + if got := fields["instance_id"]; got != "exec-1" { + t.Fatalf("instance_id = %v, want exec-1", got) + } + if got, ok := fields["error"].(string); !ok || got != retryErr.Error() { + t.Fatalf("error = %v, want %q", fields["error"], retryErr.Error()) + } +} + +// TestStopUnreconstructableRecoveredInstanceFallsBackWithoutRetrySupport +// pins that a backend without the recoveryStopper capability (Docker, +// Sprites, SSH, Kubernetes) keeps using the plain StopInstance call -- +// AC-EXECUTORS-SURVIVAL-002.15's retry contract is standalone-only. +func TestStopUnreconstructableRecoveredInstanceFallsBackWithoutRetrySupport(t *testing.T) { + mock := &guardObservingExecutor{name: executor.NameDocker} + mgr := newRecoveryStopTestManager(t, mock) + + ri := &ExecutorInstance{InstanceID: "exec-1", RuntimeName: executor.NameDocker} + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if mock.stopInstanceCalls != 1 { + t.Fatalf("StopInstance calls = %d, want 1 via the plain fallback path", mock.stopInstanceCalls) + } +} + +// TestStopUnreconstructableRecoveredInstanceRetainsGuardOnBoundedRetryFailure +// pins AC-EXECUTORS-SURVIVAL-002.16: when the bounded-retry stop path +// exhausts its retries, the session's recovery guard must be retained for +// the rest of this backend's lifetime -- not left for ReleaseAllExceptRetained +// to release, which would let a fresh launch race the still-live instance. +func TestStopUnreconstructableRecoveredInstanceRetainsGuardOnBoundedRetryFailure(t *testing.T) { + backend := &recoveryStoppingExecutor{name: executor.NameStandalone, stopWithRetryErr: errors.New("boom")} + mgr := newRecoveryStopTestManager(t, backend) + mgr.recoveryGuard.AcquireOrObserve("session-1") + + ri := &ExecutorInstance{ + InstanceID: "exec-1", + SessionID: "session-1", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-1", + } + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("CheckLaunchAllowed = %v, want ErrSessionUnstoppableAgent", err) + } + mgr.recoveryGuard.ReleaseAllExceptRetained() + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("guard must survive ReleaseAllExceptRetained, got %v", err) + } +} + +// TestStopUnreconstructableRecoveredInstanceReleasesGuardOnBoundedRetrySuccess +// pins the success counterpart: once the bounded-retry stop actually +// succeeds, the session's guard must not be left retained -- a session whose +// live instance really did stop is not the AC-EXECUTORS-SURVIVAL-002.16 case. +func TestStopUnreconstructableRecoveredInstanceReleasesGuardOnBoundedRetrySuccess(t *testing.T) { + backend := &recoveryStoppingExecutor{name: executor.NameStandalone} + mgr := newRecoveryStopTestManager(t, backend) + mgr.recoveryGuard.AcquireOrObserve("session-1") + + ri := &ExecutorInstance{ + InstanceID: "exec-1", + SessionID: "session-1", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-1", + } + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); err != nil { + t.Fatalf("CheckLaunchAllowed = %v, want nil (guard released on stop success)", err) + } +} + +// TestStopUnreconstructableRecoveredInstanceRetainsGuardWhenPlainStopInstanceFails +// pins AC-EXECUTORS-SURVIVAL-002.16 on the non-standalone fallback path too: +// a Docker/Sprites/SSH/Kubernetes instance that could not be stopped via the +// plain StopInstance call must retain its guard exactly like the +// bounded-retry path does. +func TestStopUnreconstructableRecoveredInstanceRetainsGuardWhenPlainStopInstanceFails(t *testing.T) { + mock := &MockExecutor{name: executor.NameDocker, stopInstanceErr: errors.New("boom")} + mgr := newRecoveryStopTestManager(t, mock) + mgr.recoveryGuard.AcquireOrObserve("session-1") + + ri := &ExecutorInstance{InstanceID: "exec-1", SessionID: "session-1", RuntimeName: executor.NameDocker} + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("CheckLaunchAllowed = %v, want ErrSessionUnstoppableAgent", err) + } +} + +// TestStopUnreconstructableRecoveredInstanceRetainsGuardWhenNoBackendForRuntime +// pins AC-EXECUTORS-SURVIVAL-002.16 on the defensive no-backend branch: an +// instance whose runtime has no registered backend at all was never even +// attempted to stop, so it must be treated the same as a failed stop -- +// retained, not silently left for a bulk release to clear. +func TestStopUnreconstructableRecoveredInstanceRetainsGuardWhenNoBackendForRuntime(t *testing.T) { + backend := &recoveryStoppingExecutor{name: executor.NameStandalone} + mgr := newRecoveryStopTestManager(t, backend) + mgr.recoveryGuard.AcquireOrObserve("session-1") + + ri := &ExecutorInstance{InstanceID: "exec-1", SessionID: "session-1", RuntimeName: executor.NameDocker} + mgr.stopUnreconstructableRecoveredInstance(context.Background(), ri) + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("CheckLaunchAllowed = %v, want ErrSessionUnstoppableAgent", err) + } +} + +// blockingUnreconstructableStopper is a minimal ExecutorBackend + recoveryStopper +// whose stopWithRetry blocks one named instance until the test releases it, +// so a Manager.Start test can observe whether other recovered records are +// reconstructed while that stop is still in flight. +type blockingUnreconstructableStopper struct { + name executor.Name + recovered []*ExecutorInstance + blockedInst string + started chan struct{} + unblock chan struct{} +} + +func (e *blockingUnreconstructableStopper) Name() executor.Name { return e.name } +func (e *blockingUnreconstructableStopper) HealthCheck(context.Context) error { return nil } +func (e *blockingUnreconstructableStopper) CreateInstance(context.Context, *ExecutorCreateRequest) (*ExecutorInstance, error) { + return nil, nil +} +func (e *blockingUnreconstructableStopper) StopInstance(context.Context, *ExecutorInstance, bool) error { + return nil +} +func (e *blockingUnreconstructableStopper) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*ExecutorInstance, error) { + return e.recovered, nil +} +func (e *blockingUnreconstructableStopper) GetInteractiveRunner() *process.InteractiveRunner { + return nil +} +func (e *blockingUnreconstructableStopper) RequiresCloneURL() bool { return false } +func (e *blockingUnreconstructableStopper) ShouldApplyPreferredShell() bool { return false } +func (e *blockingUnreconstructableStopper) IsAlwaysResumable() bool { return false } +func (e *blockingUnreconstructableStopper) stopWithRetry(_ context.Context, instanceID string) error { + if instanceID == e.blockedInst { + close(e.started) + <-e.unblock + } + return nil +} + +// TestManagerStartDoesNotBlockOtherRecordsOnOneUnreconstructableStop pins +// AC-EXECUTORS-SURVIVAL-002.11 ("no such instance's outcome shall depend on +// another's"): one recovered record whose not-re-tracked stop is still +// retrying must not prevent a different record in the same pass from being +// fully reconstructed and published. Manager.Start dispatches +// stopUnreconstructableRecoveredInstance on its own goroutine +// (dispatchUnreconstructableStop) rather than blocking the shared consumer +// loop inline. +func TestManagerStartDoesNotBlockOtherRecordsOnOneUnreconstructableStop(t *testing.T) { + log := newTestRegistryLogger() + execRegistry := NewExecutorRegistry(log) + backend := &blockingUnreconstructableStopper{ + name: executor.NameStandalone, + recovered: []*ExecutorInstance{ + { + InstanceID: "exec-blocked", + TaskID: "", // empty TaskID triggers the AC-002.4 refusal path + SessionID: "session-blocked", + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-blocked", + }, + { + InstanceID: "exec-fine", + TaskID: "task-fine", + SessionID: "session-fine", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "standalone-fine", + }, + }, + blockedInst: "standalone-blocked", + started: make(chan struct{}), + unblock: make(chan struct{}), + } + execRegistry.Register(backend) + + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, nil, nil, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + registerRecoveryTestAgentProfile(t, mgr) + mgr.SetExecutorRunningWriter(&listingWriter{rows: []*models.ExecutorRunning{ + {SessionID: "session-blocked"}, + {SessionID: "session-fine"}, + }}) + + done := make(chan error, 1) + go func() { done <- mgr.Start(context.Background()) }() + + select { + case <-backend.started: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the blocked instance's stop to begin") + } + + // session-blocked's stop stays in flight (backend.unblock is not closed + // yet) for this entire poll window: on the pre-fix synchronous dispatch, + // session-fine could never reach the store until after that stop + // resolved, so this loop would exhaust its deadline and fail. It only + // succeeds here because dispatchUnreconstructableStop runs the stop on + // its own goroutine, letting the shared consumer loop reconstruct + // session-fine independently. + deadline := time.Now().Add(2 * time.Second) + for { + if _, ok := mgr.executionStore.GetBySessionID("session-fine"); ok { + break + } + if time.Now().After(deadline) { + t.Fatal("session-fine was not re-tracked while session-blocked's stop was still in flight") + } + time.Sleep(5 * time.Millisecond) + } + + close(backend.unblock) + + select { + case err := <-done: + if err != nil { + t.Fatalf("Start: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for Start to finish after unblocking the stop") + } + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-blocked"); err != nil { + t.Fatalf("expected session-blocked's guard released once its stop succeeded, got %v", err) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_task_environment_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_task_environment_test.go new file mode 100644 index 0000000000..f0635486b4 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_task_environment_test.go @@ -0,0 +1,133 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/agent/executor" +) + +// fakeTaskEnvironmentProvider is a minimal WorkspaceInfoProvider double +// scoped to hydrateRecoveredTaskEnvironmentID's own tests. +type fakeTaskEnvironmentProvider struct { + info *WorkspaceInfo + err error +} + +func (p *fakeTaskEnvironmentProvider) GetWorkspaceInfoForSession(context.Context, string, string) (*WorkspaceInfo, error) { + return p.info, p.err +} + +func (p *fakeTaskEnvironmentProvider) GetWorkspaceInfoForEnvironment(context.Context, string) (*WorkspaceInfo, error) { + return p.info, p.err +} + +// TestHydrateRecoveredTaskEnvironmentIDNilProviderIsANoOp pins that an +// unwired provider (every production backend wires one; only bare-bones test +// doubles don't) is treated as a capability this deployment never had, not a +// failed read -- same convention as every other optional recovery capability. +func TestHydrateRecoveredTaskEnvironmentIDNilProviderIsANoOp(t *testing.T) { + mgr := newTestManager(t) + execution := &AgentExecution{TaskID: "task-1", SessionID: "session-1"} + + if err := mgr.hydrateRecoveredTaskEnvironmentID(context.Background(), execution); err != nil { + t.Fatalf("hydrateRecoveredTaskEnvironmentID: %v, want nil with no provider configured", err) + } + if execution.TaskEnvironmentID != "" { + t.Fatalf("TaskEnvironmentID = %q, want empty", execution.TaskEnvironmentID) + } +} + +// TestHydrateRecoveredTaskEnvironmentIDReturnsErrorOnLookupFailure pins +// Review round 3, finding 4: a lookup failure must propagate as an error so +// the caller can refuse to re-track the instance (AC-EXECUTORS-SURVIVAL-002.13), +// rather than being logged and silently ignored as an earlier version of this +// method did. +func TestHydrateRecoveredTaskEnvironmentIDReturnsErrorOnLookupFailure(t *testing.T) { + mgr := newTestManager(t) + mgr.SetWorkspaceInfoProvider(&fakeTaskEnvironmentProvider{err: errors.New("db unavailable")}) + execution := &AgentExecution{TaskID: "task-1", SessionID: "session-1"} + + if err := mgr.hydrateRecoveredTaskEnvironmentID(context.Background(), execution); err == nil { + t.Fatal("want an error when the lookup itself fails") + } +} + +// TestHydrateRecoveredTaskEnvironmentIDReturnsErrorWhenAnsweredEmpty pins the +// AC-EXECUTORS-SURVIVAL-002.4 half of finding 4: the durable store answering +// successfully with no task-environment identity at all is authoritatively +// absent, not "nothing to hydrate." +func TestHydrateRecoveredTaskEnvironmentIDReturnsErrorWhenAnsweredEmpty(t *testing.T) { + mgr := newTestManager(t) + mgr.SetWorkspaceInfoProvider(&fakeTaskEnvironmentProvider{info: &WorkspaceInfo{TaskEnvironmentID: ""}}) + execution := &AgentExecution{TaskID: "task-1", SessionID: "session-1"} + + if err := mgr.hydrateRecoveredTaskEnvironmentID(context.Background(), execution); err == nil { + t.Fatal("want an error when the durable store answers with no task-environment identity") + } +} + +// TestHydrateRecoveredTaskEnvironmentIDReturnsErrorWhenInfoIsNil covers the +// same authoritatively-absent case for a nil *WorkspaceInfo, not just an +// empty field on a non-nil one. +func TestHydrateRecoveredTaskEnvironmentIDReturnsErrorWhenInfoIsNil(t *testing.T) { + mgr := newTestManager(t) + mgr.SetWorkspaceInfoProvider(&fakeTaskEnvironmentProvider{info: nil, err: nil}) + execution := &AgentExecution{TaskID: "task-1", SessionID: "session-1"} + + if err := mgr.hydrateRecoveredTaskEnvironmentID(context.Background(), execution); err == nil { + t.Fatal("want an error when the durable store answers with a nil info and no error") + } +} + +// TestHydrateRecoveredTaskEnvironmentIDSetsValueOnSuccess pins the happy +// path still works after the fix: a populated answer sets the field and +// returns no error. +func TestHydrateRecoveredTaskEnvironmentIDSetsValueOnSuccess(t *testing.T) { + mgr := newTestManager(t) + mgr.SetWorkspaceInfoProvider(&fakeTaskEnvironmentProvider{info: &WorkspaceInfo{TaskEnvironmentID: "env-1"}}) + execution := &AgentExecution{TaskID: "task-1", SessionID: "session-1"} + + if err := mgr.hydrateRecoveredTaskEnvironmentID(context.Background(), execution); err != nil { + t.Fatalf("hydrateRecoveredTaskEnvironmentID: %v, want nil", err) + } + if execution.TaskEnvironmentID != "env-1" { + t.Fatalf("TaskEnvironmentID = %q, want env-1", execution.TaskEnvironmentID) + } +} + +// TestManagerStartRefusesRecoveredExecutionWhenTaskEnvironmentIdentityCannotBeReconstructed +// pins the full Manager.Start wiring: a recovered instance whose +// task-environment identity cannot be reconstructed (durable lookup fails) +// must never be published as tracked, and is instead routed to the +// AC-EXECUTORS-SURVIVAL-002.6 stop path, exactly like a failed agent-identity +// or turn-outcome reconstruction already is. +func TestManagerStartRefusesRecoveredExecutionWhenTaskEnvironmentIdentityCannotBeReconstructed(t *testing.T) { + log := newTestRegistryLogger() + execRegistry := NewExecutorRegistry(log) + mockExec := &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-bad-env", + TaskID: "task-1", + SessionID: "session-bad-env", + RuntimeName: executor.NameStandalone, + }}, + } + execRegistry.Register(mockExec) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log) + t.Cleanup(func() { _ = mgr.Stop() }) + mgr.SetWorkspaceInfoProvider(&fakeTaskEnvironmentProvider{err: errors.New("db unavailable")}) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start: %v", err) + } + + if _, ok := mgr.GetExecution("exec-bad-env"); ok { + t.Fatal("an execution whose task-environment identity could not be reconstructed must not be tracked") + } + if len(mockExec.stopInstanceCalls) != 1 || mockExec.stopInstanceCalls[0].InstanceID != "exec-bad-env" { + t.Fatalf("stopInstanceCalls = %+v, want the unreconstructable instance stopped", mockExec.stopInstanceCalls) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_turn_outcome.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_turn_outcome.go new file mode 100644 index 0000000000..d178a9bd56 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_turn_outcome.go @@ -0,0 +1,126 @@ +package lifecycle + +import ( + "context" + + "go.uber.org/zap" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/events" +) + +// turnOutcomeApplier is the optional capability satisfied by +// *StandaloneExecutor: only standalone-backed instances have agentctl turn- +// outcome retention (AC-EXECUTORS-SURVIVAL-004) -- this capability covers the +// WORKTREE and LOCAL executor types only, both standalone-runtime. A backend +// that doesn't implement it (Docker, Sprites, SSH, Kubernetes) is out of +// scope, and recovery treats that exactly like "nothing retained." +type turnOutcomeApplier interface { + fetchTurnOutcomeWithRetry(ctx context.Context, instanceID string) (*agentctl.TurnOutcome, error) + ackTurnOutcome(ctx context.Context, instanceID string, turnID int64) error +} + +// recoveredTurnOutcomeResult is the outcome of retrieveRecoveredTurnOutcome: +// exactly one of the three cases AC-EXECUTORS-SURVIVAL-004.5 distinguishes. +type recoveredTurnOutcomeResult int + +const ( + // recoveredTurnOutcomeNone: the read succeeded and nothing was retained, + // or the backend does not support turn-outcome retention at all. Publish + // the session as running. + recoveredTurnOutcomeNone recoveredTurnOutcomeResult = iota + // recoveredTurnOutcomeApplied: an outcome was retrieved and must be applied. + recoveredTurnOutcomeApplied + // recoveredTurnOutcomeReadFailed: the read failed and its retries are + // exhausted. AC-EXECUTORS-SURVIVAL-004.5 forbids publishing running here; + // the instance goes to the not-re-tracked path instead. + recoveredTurnOutcomeReadFailed +) + +// retrieveRecoveredTurnOutcome implements AC-EXECUTORS-SURVIVAL-004.2's read +// step for one recovered instance, bounded by the same per-read timeout and +// retry count as AC-EXECUTORS-SURVIVAL-002.13 (AC-EXECUTORS-SURVIVAL-004.5). +func (m *Manager) retrieveRecoveredTurnOutcome(ctx context.Context, ri *ExecutorInstance) (*agentctl.TurnOutcome, recoveredTurnOutcomeResult) { + backend, err := m.executorRegistry.GetBackend(ri.RuntimeName) + if err != nil { + return nil, recoveredTurnOutcomeNone + } + applier, ok := backend.(turnOutcomeApplier) + if !ok { + return nil, recoveredTurnOutcomeNone + } + outcome, err := applier.fetchTurnOutcomeWithRetry(ctx, ri.StandaloneInstanceID) + if err != nil { + m.logger.Warn("failed to retrieve recovered instance's turn outcome after exhausting retries", + zap.String("instance_id", ri.InstanceID), + zap.String("session_id", ri.SessionID), + zap.Error(err)) + return nil, recoveredTurnOutcomeReadFailed + } + if outcome == nil { + return nil, recoveredTurnOutcomeNone + } + return outcome, recoveredTurnOutcomeApplied +} + +// applyRecoveredTurnOutcome implements AC-EXECUTORS-SURVIVAL-004.2's apply +// step. It seeds the execution's prompt-completion generation guard from the +// retained event's own PromptGeneration -- recreated from zero by recovery, +// per design 03's "Turn outcome across the detached gap" -- so the existing +// completion-claim path in handleAgentEvent (via claimPromptCompletion) +// processes this event exactly as it would have had a backend been attached +// when the turn ended, publishing whatever state that produces. It records +// the applied control-server turn identifier before processing so a live +// redelivery of the same event (agentctl's parked send finally unblocked by +// this stream reconnecting) is recognized and dropped +// (AC-EXECUTORS-SURVIVAL-004.4, see AgentExecution.isRecoveryDuplicateEvent). +// +// The acknowledgement is sent only after handleAgentEvent returns +// (AC-EXECUTORS-SURVIVAL-004.6): on the default in-memory event bus, Publish +// delivers to subscribers synchronously before returning, so the +// orchestrator's session-state and task-state writes are both complete by +// then; on a NATS-backed bus this ordering is best-effort, the same +// documented precedent internal/orchestrator/event_handlers_streaming.go's +// markReadyTurn comment relies on for durable-TurnID-based ordering. A failed +// or best-effort-late acknowledgement is not retried here: the outcome stays +// retained and a later backend re-applies it idempotently +// (AC-EXECUTORS-SURVIVAL-004.6's "applying to a turn that already reached +// that outcome... changes nothing"). +func (m *Manager) applyRecoveredTurnOutcome( + ctx context.Context, execution *AgentExecution, ri *ExecutorInstance, outcome *agentctl.TurnOutcome, +) { + _ = m.executionStore.WithLock(execution.ID, func(current *AgentExecution) { + current.promptGeneration = outcome.Event.PromptGeneration + }) + execution.markRecoveryTurnOutcomeApplied(outcome.TurnID) + + m.handleAgentEvent(execution, outcome.Event) + + backend, err := m.executorRegistry.GetBackend(ri.RuntimeName) + if err != nil { + return + } + applier, ok := backend.(turnOutcomeApplier) + if !ok { + return + } + if err := applier.ackTurnOutcome(ctx, ri.StandaloneInstanceID, outcome.TurnID); err != nil { + m.logger.Warn("failed to acknowledge applied turn outcome; a later backend will re-apply it idempotently", + zap.String("instance_id", ri.InstanceID), + zap.Int64("turn_id", outcome.TurnID), + zap.Error(err)) + } +} + +// publishRecoveredExecutionRunning implements AC-EXECUTORS-SURVIVAL-004.5's +// "nothing retained" case: publish the session as running exactly as an +// ordinary launch's first activity would. +func (m *Manager) publishRecoveredExecutionRunning(ctx context.Context, execution *AgentExecution) { + // Nothing was retained, so any turn still in flight across the restart + // carries a PromptGeneration this freshly reconstructed execution object + // has never seen (its own counter was recreated from zero). Let the first + // live completion adopt it instead of being rejected as superseded -- + // see recoveredPromptGenerationPending. + execution.recoveredPromptGenerationPending.Store(true) + m.eventPublisher.PublishAgentEvent(ctx, events.AgentRunning, execution) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_turn_outcome_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_turn_outcome_test.go new file mode 100644 index 0000000000..a9c7d86c36 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_recovery_turn_outcome_test.go @@ -0,0 +1,388 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/agent/executor" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" +) + +// fakeTurnOutcomeBackend embeds *MockExecutor so it satisfies ExecutorBackend +// via the existing mock, and additionally implements the unexported +// turnOutcomeApplier capability so tests can drive +// Manager.retrieveRecoveredTurnOutcome/applyRecoveredTurnOutcome without a +// real agentctl control server. +type fakeTurnOutcomeBackend struct { + *MockExecutor + outcome *agentctl.TurnOutcome + fetchErr error + ackErr error + + // eventBus, when set by the test, lets ackTurnOutcome record how many + // events had already been published at ack time -- a monotonic counter + // proving AC-EXECUTORS-SURVIVAL-004.6's apply-before-ack ordering rather + // than merely that both happened. + eventBus *MockEventBus + + fetchCalls []string + ackedCalls []ackedTurnOutcome + publishedCountsAtAck []int +} + +func (f *fakeTurnOutcomeBackend) fetchTurnOutcomeWithRetry(_ context.Context, instanceID string) (*agentctl.TurnOutcome, error) { + f.fetchCalls = append(f.fetchCalls, instanceID) + return f.outcome, f.fetchErr +} + +func (f *fakeTurnOutcomeBackend) ackTurnOutcome(_ context.Context, instanceID string, turnID int64) error { + f.ackedCalls = append(f.ackedCalls, ackedTurnOutcome{instanceID: instanceID, turnID: turnID}) + if f.eventBus != nil { + f.publishedCountsAtAck = append(f.publishedCountsAtAck, len(f.eventBus.PublishedEvents)) + } + return f.ackErr +} + +func newTurnOutcomeTestManager(t *testing.T, backend *fakeTurnOutcomeBackend) (*Manager, *MockEventBus) { + t.Helper() + log := newTestLogger() + registry := NewExecutorRegistry(log) + registry.Register(backend) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + return mgr, eventBus +} + +func hasEventType(published []*bus.Event, eventType string) bool { + for _, evt := range published { + if evt.Type == eventType { + return true + } + } + return false +} + +// TestRetrieveRecoveredTurnOutcomeCases pins AC-EXECUTORS-SURVIVAL-004.5's +// three-way result, plus the out-of-scope backend case (a runtime that never +// implements turn-outcome retention, e.g. Docker/Sprites/SSH/Kubernetes, +// which this capability does not cover at all). +func TestRetrieveRecoveredTurnOutcomeCases(t *testing.T) { + t.Run("backend does not implement turnOutcomeApplier", func(t *testing.T) { + mgr, _ := newTurnOutcomeTestManagerPlain(t) + outcome, result := mgr.retrieveRecoveredTurnOutcome(context.Background(), &ExecutorInstance{RuntimeName: executor.NameStandalone}) + if result != recoveredTurnOutcomeNone || outcome != nil { + t.Fatalf("result = %v, outcome = %v, want (recoveredTurnOutcomeNone, nil)", result, outcome) + } + }) + + t.Run("no backend for runtime", func(t *testing.T) { + mgr, _ := newTurnOutcomeTestManagerPlain(t) + outcome, result := mgr.retrieveRecoveredTurnOutcome(context.Background(), &ExecutorInstance{RuntimeName: "unknown"}) + if result != recoveredTurnOutcomeNone || outcome != nil { + t.Fatalf("result = %v, outcome = %v, want (recoveredTurnOutcomeNone, nil)", result, outcome) + } + }) + + t.Run("nothing retained", func(t *testing.T) { + backend := &fakeTurnOutcomeBackend{MockExecutor: &MockExecutor{name: executor.NameStandalone}} + mgr, _ := newTurnOutcomeTestManager(t, backend) + outcome, result := mgr.retrieveRecoveredTurnOutcome(context.Background(), &ExecutorInstance{ + RuntimeName: executor.NameStandalone, StandaloneInstanceID: "inst-1", + }) + if result != recoveredTurnOutcomeNone || outcome != nil { + t.Fatalf("result = %v, outcome = %v, want (recoveredTurnOutcomeNone, nil)", result, outcome) + } + if len(backend.fetchCalls) != 1 || backend.fetchCalls[0] != "inst-1" { + t.Fatalf("fetchCalls = %v, want [inst-1]", backend.fetchCalls) + } + }) + + t.Run("outcome retained", func(t *testing.T) { + want := &agentctl.TurnOutcome{TurnID: 9, Event: streams.AgentEvent{Type: streams.EventTypeComplete}} + backend := &fakeTurnOutcomeBackend{MockExecutor: &MockExecutor{name: executor.NameStandalone}, outcome: want} + mgr, _ := newTurnOutcomeTestManager(t, backend) + outcome, result := mgr.retrieveRecoveredTurnOutcome(context.Background(), &ExecutorInstance{ + RuntimeName: executor.NameStandalone, StandaloneInstanceID: "inst-1", + }) + if result != recoveredTurnOutcomeApplied || outcome != want { + t.Fatalf("result = %v, outcome = %v, want (recoveredTurnOutcomeApplied, %v)", result, outcome, want) + } + }) + + t.Run("read failed", func(t *testing.T) { + backend := &fakeTurnOutcomeBackend{ + MockExecutor: &MockExecutor{name: executor.NameStandalone}, + fetchErr: errors.New("boom"), + } + mgr, _ := newTurnOutcomeTestManager(t, backend) + outcome, result := mgr.retrieveRecoveredTurnOutcome(context.Background(), &ExecutorInstance{ + RuntimeName: executor.NameStandalone, StandaloneInstanceID: "inst-1", + }) + if result != recoveredTurnOutcomeReadFailed || outcome != nil { + t.Fatalf("result = %v, outcome = %v, want (recoveredTurnOutcomeReadFailed, nil)", result, outcome) + } + }) +} + +func newTurnOutcomeTestManagerPlain(t *testing.T) (*Manager, *MockEventBus) { + t.Helper() + log := newTestLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{name: executor.NameStandalone}) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + return mgr, eventBus +} + +// TestApplyRecoveredTurnOutcomeAppliesAndAcks pins +// AC-EXECUTORS-SURVIVAL-004.2/004.6: applying a retained outcome publishes +// the state that outcome produces (here, agent.ready from a successful +// completion), seeds the generation guard from the retained event's own +// PromptGeneration, and only then acknowledges it. +func TestApplyRecoveredTurnOutcomeAppliesAndAcks(t *testing.T) { + outcome := &agentctl.TurnOutcome{ + TurnID: 9, + Event: streams.AgentEvent{ + Type: streams.EventTypeComplete, + PromptGeneration: 5, + }, + } + backend := &fakeTurnOutcomeBackend{MockExecutor: &MockExecutor{name: executor.NameStandalone}, outcome: outcome} + mgr, eventBus := newTurnOutcomeTestManager(t, backend) + backend.eventBus = eventBus + execution := createTestExecution("exec-1", "task-1", "session-1") + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatalf("add execution: %v", err) + } + ri := &ExecutorInstance{RuntimeName: executor.NameStandalone, StandaloneInstanceID: "inst-1"} + + mgr.applyRecoveredTurnOutcome(context.Background(), execution, ri, outcome) + + if !hasEventType(eventBus.PublishedEvents, events.AgentReady) { + t.Fatal("expected agent.ready to be published for a successful completion outcome") + } + if len(backend.ackedCalls) != 1 || backend.ackedCalls[0] != (ackedTurnOutcome{instanceID: "inst-1", turnID: 9}) { + t.Fatalf("ackedCalls = %v, want [{inst-1 9}]", backend.ackedCalls) + } + if got := execution.promptGenerationSnapshot(); got != 5 { + t.Fatalf("promptGeneration = %d, want 5 (seeded from the retained event)", got) + } + if !execution.isRecoveryDuplicateEvent(&streams.AgentEvent{ControlTurnID: 9}) { + t.Fatal("expected the applied turn ID (9) to be recorded so a live redelivery is recognized") + } + // AC-EXECUTORS-SURVIVAL-004.6: the outcome must be applied (here, the + // agent.ready publish above) before it is acknowledged -- not merely + // that both eventually happen. The published-event count observed at + // ack time is a monotonic witness of that ordering: it can only be + // nonzero if the apply's publish already ran. + if len(backend.publishedCountsAtAck) != 1 { + t.Fatalf("publishedCountsAtAck = %v, want exactly 1 ack observed", backend.publishedCountsAtAck) + } + if backend.publishedCountsAtAck[0] < 1 { + t.Fatalf("published event count at ack time = %d, want >= 1 (apply's publish must precede ack)", + backend.publishedCountsAtAck[0]) + } +} + +// TestApplyRecoveredTurnOutcomeDedupesLiveRedelivery pins +// AC-EXECUTORS-SURVIVAL-004.4: once an outcome is applied, a subsequent live +// event carrying the same ControlTurnID (agentctl's parked send, finally +// unblocked by the stream reconnecting) must not be re-applied -- no second +// agent.ready. +func TestApplyRecoveredTurnOutcomeDedupesLiveRedelivery(t *testing.T) { + outcome := &agentctl.TurnOutcome{ + TurnID: 9, + Event: streams.AgentEvent{ + Type: streams.EventTypeComplete, + PromptGeneration: 5, + }, + } + backend := &fakeTurnOutcomeBackend{MockExecutor: &MockExecutor{name: executor.NameStandalone}, outcome: outcome} + mgr, eventBus := newTurnOutcomeTestManager(t, backend) + execution := createTestExecution("exec-1", "task-1", "session-1") + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatalf("add execution: %v", err) + } + ri := &ExecutorInstance{RuntimeName: executor.NameStandalone, StandaloneInstanceID: "inst-1"} + mgr.applyRecoveredTurnOutcome(context.Background(), execution, ri, outcome) + readyCountAfterApply := countEventType(eventBus.PublishedEvents, events.AgentReady) + + // The live redelivery: the same event, stamped with the control-server + // turn identifier, as it would arrive once the stream reconnects. + redelivered := outcome.Event + redelivered.ControlTurnID = outcome.TurnID + mgr.handleAgentEvent(execution, redelivered) + + if got := countEventType(eventBus.PublishedEvents, events.AgentReady); got != readyCountAfterApply { + t.Fatalf("agent.ready published %d times after redelivery, want %d (redelivery must be a no-op)", + got, readyCountAfterApply) + } +} + +func countEventType(published []*bus.Event, eventType string) int { + count := 0 + for _, evt := range published { + if evt.Type == eventType { + count++ + } + } + return count +} + +// TestPublishRecoveredExecutionRunningPublishesAgentRunning pins +// AC-EXECUTORS-SURVIVAL-004.5's "nothing retained" case. +func TestPublishRecoveredExecutionRunningPublishesAgentRunning(t *testing.T) { + mgr, eventBus := newTurnOutcomeTestManagerPlain(t) + execution := createTestExecution("exec-1", "task-1", "session-1") + + mgr.publishRecoveredExecutionRunning(context.Background(), execution) + + if !hasEventType(eventBus.PublishedEvents, events.AgentRunning) { + t.Fatal("expected agent.running to be published") + } +} + +// newRecoveryTurnOutcomeExecutorInstance builds the ExecutorInstance a +// Manager.Start recovery pass would hand to the turn-outcome step, wired with +// a resolvable agent profile so it clears the earlier +// reDeriveRecoveredAgentIdentity gate. +func newRecoveryTurnOutcomeExecutorInstance() *ExecutorInstance { + return &ExecutorInstance{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + StandaloneInstanceID: "std-inst-1", + } +} + +// TestManagerStartAppliesRetainedTurnOutcome pins the full Manager.Start +// wiring for AC-EXECUTORS-SURVIVAL-004.2: a retained outcome is retrieved, +// applied (publishing whatever state it produces), and acknowledged. +func TestManagerStartAppliesRetainedTurnOutcome(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &fakeTurnOutcomeBackend{ + MockExecutor: &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{newRecoveryTurnOutcomeExecutorInstance()}, + }, + outcome: &agentctl.TurnOutcome{ + TurnID: 3, + Event: streams.AgentEvent{Type: streams.EventTypeComplete, PromptGeneration: 11}, + }, + } + registry.Register(backend) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v", err) + } + + execution, ok := mgr.GetExecutionBySessionID("session-1") + if !ok { + t.Fatal("expected the recovered session to be tracked") + } + // The final published state is agent.ready, matching "whatever state + // that outcome produces" (AC-EXECUTORS-SURVIVAL-004.2). An agent.running + // may also be published ahead of it: this execution's firstActivityOnce + // has never fired, so claimPromptCompletion treats this first-ever + // completion exactly as it would for a live first turn, publishing + // Running before Ready -- that is ordinary completion semantics, not the + // recovery-specific "nothing retained" case this test is not exercising. + if !hasEventType(eventBus.PublishedEvents, events.AgentReady) { + t.Fatal("expected agent.ready to be published for the retained completion outcome") + } + if got := execution.promptGenerationSnapshot(); got != 11 { + t.Fatalf("promptGeneration = %d, want 11", got) + } + if len(backend.ackedCalls) != 1 || backend.ackedCalls[0] != (ackedTurnOutcome{instanceID: "std-inst-1", turnID: 3}) { + t.Fatalf("ackedCalls = %v, want [{std-inst-1 3}]", backend.ackedCalls) + } +} + +// TestManagerStartPublishesRunningWhenNoTurnOutcomeRetained pins +// AC-EXECUTORS-SURVIVAL-004.5's "nothing retained" case through the full +// Manager.Start wiring. +func TestManagerStartPublishesRunningWhenNoTurnOutcomeRetained(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &fakeTurnOutcomeBackend{ + MockExecutor: &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{newRecoveryTurnOutcomeExecutorInstance()}, + }, + } + registry.Register(backend) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v", err) + } + + if _, ok := mgr.GetExecutionBySessionID("session-1"); !ok { + t.Fatal("expected the recovered session to be tracked") + } + if !hasEventType(eventBus.PublishedEvents, events.AgentRunning) { + t.Fatal("expected agent.running to be published when nothing was retained") + } + if len(backend.ackedCalls) != 0 { + t.Fatalf("ackedCalls = %v, want none: nothing was retained, so nothing to acknowledge", backend.ackedCalls) + } +} + +// TestManagerStartRefusesAndStopsRecoveredExecutionWhenTurnOutcomeCannotBeRetrieved +// pins AC-EXECUTORS-SURVIVAL-004.5's read-failed case: the instance must not +// be re-tracked or published as running -- it takes the same not-re-tracked +// stop path as an unreconstructable agent identity. +func TestManagerStartRefusesAndStopsRecoveredExecutionWhenTurnOutcomeCannotBeRetrieved(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &fakeTurnOutcomeBackend{ + MockExecutor: &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{newRecoveryTurnOutcomeExecutorInstance()}, + }, + fetchErr: errors.New("control server unreachable"), + } + registry.Register(backend) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + if err := mgr.Start(context.Background()); err != nil { + t.Fatalf("Start() error = %v", err) + } + + if _, ok := mgr.GetExecutionBySessionID("session-1"); ok { + t.Fatal("expected the session NOT to be tracked when its turn outcome could not be retrieved") + } + if hasEventType(eventBus.PublishedEvents, events.AgentRunning) { + t.Fatal("expected agent.running NOT to be published: a read failure is not 'nothing retained'") + } + if len(backend.stopInstanceCalls) != 1 || backend.stopInstanceCalls[0].InstanceID != "exec-recovered" { + t.Fatalf("stopInstanceCalls = %v, want exactly one call for exec-recovered", backend.stopInstanceCalls) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_startup_recovery_guard_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_startup_recovery_guard_test.go new file mode 100644 index 0000000000..1c5d02c0da --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_startup_recovery_guard_test.go @@ -0,0 +1,119 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +func alwaysNotPassthrough(context.Context, string) (bool, bool) { return false, true } + +// TestTakeStartupRecoveryGuardsGuardsLiveNonPassthroughSessions pins Review +// round 3, finding 1 / AC-EXECUTORS-SURVIVAL-002.8: the returned guard must +// already hold every live standalone recovery-inventory session before +// backend startup composition contacts any control server (adoption +// included), not just by the time Manager.Start gets around to it. +func TestTakeStartupRecoveryGuardsGuardsLiveNonPassthroughSessions(t *testing.T) { + writer := &listingWriter{rows: []*models.ExecutorRunning{ + {SessionID: "session-1"}, {SessionID: "session-2"}, + }} + + guard := TakeStartupRecoveryGuards(context.Background(), writer, alwaysNotPassthrough, newTestLogger()) + + if err := guard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("session-1 CheckLaunchAllowed = %v, want ErrSessionRecoveryGuarded", err) + } + if err := guard.CheckLaunchAllowed("session-2"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("session-2 CheckLaunchAllowed = %v, want ErrSessionRecoveryGuarded", err) + } +} + +// TestTakeStartupRecoveryGuardsExcludesConfirmedPassthroughSessions pins +// AC-EXECUTORS-SURVIVAL-005.3's exclusion still applying at this earlier +// guard-taking point. +func TestTakeStartupRecoveryGuardsExcludesConfirmedPassthroughSessions(t *testing.T) { + writer := &listingWriter{rows: []*models.ExecutorRunning{{SessionID: "session-passthrough"}}} + isPassthrough := func(context.Context, string) (bool, bool) { return true, true } + + guard := TakeStartupRecoveryGuards(context.Background(), writer, isPassthrough, newTestLogger()) + + if err := guard.CheckLaunchAllowed("session-passthrough"); err != nil { + t.Fatalf("CheckLaunchAllowed = %v, want nil (excluded)", err) + } +} + +// TestTakeStartupRecoveryGuardsGuardsWhenPassthroughLookupFails pins the +// fail-closed default: a lookup that could not read the durable record +// guards the session rather than excluding it. +func TestTakeStartupRecoveryGuardsGuardsWhenPassthroughLookupFails(t *testing.T) { + writer := &listingWriter{rows: []*models.ExecutorRunning{{SessionID: "session-unknown"}}} + failedLookup := func(context.Context, string) (bool, bool) { return false, false } + + guard := TakeStartupRecoveryGuards(context.Background(), writer, failedLookup, newTestLogger()) + + if err := guard.CheckLaunchAllowed("session-unknown"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("CheckLaunchAllowed = %v, want ErrSessionRecoveryGuarded", err) + } +} + +// TestTakeStartupRecoveryGuardsReturnsEmptyGuardOnListError pins the +// best-effort fallback already established for +// ListLiveStandaloneExecutorsRunning: a failed inventory read yields an +// empty (but usable) guard rather than a nil one or a panic. +func TestTakeStartupRecoveryGuardsReturnsEmptyGuardOnListError(t *testing.T) { + writer := &listingWriter{err: errors.New("db unavailable")} + + guard := TakeStartupRecoveryGuards(context.Background(), writer, alwaysNotPassthrough, newTestLogger()) + + if guard == nil { + t.Fatal("guard = nil, want a usable empty guard") + } + if err := guard.CheckLaunchAllowed("session-1"); err != nil { + t.Fatalf("CheckLaunchAllowed = %v, want nil (nothing to guard)", err) + } +} + +// TestTakeStartupRecoveryGuardsAcceptsNilPassthroughLookup pins the same +// nil-lookup fallback Start applies: no lookup configured guards every +// session, same as a failed read. +func TestTakeStartupRecoveryGuardsAcceptsNilPassthroughLookup(t *testing.T) { + writer := &listingWriter{rows: []*models.ExecutorRunning{{SessionID: "session-1"}}} + + guard := TakeStartupRecoveryGuards(context.Background(), writer, nil, newTestLogger()) + + if err := guard.CheckLaunchAllowed("session-1"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("CheckLaunchAllowed = %v, want ErrSessionRecoveryGuarded", err) + } +} + +// TestSetRecoveryGuardReplacesDefaultGuard pins that a guard populated +// before Start (by TakeStartupRecoveryGuards) is the one Start's own +// (idempotent) guard-taking and the launch-refusal path both observe -- +// not a fresh, still-empty guard the Manager created for itself. +func TestSetRecoveryGuardReplacesDefaultGuard(t *testing.T) { + mgr := newTestManager(t) + pre := NewRecoveryGuard() + pre.AcquireOrObserve("session-pretaken") + + mgr.SetRecoveryGuard(pre) + + if err := mgr.recoveryGuard.CheckLaunchAllowed("session-pretaken"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("CheckLaunchAllowed = %v, want ErrSessionRecoveryGuarded", err) + } +} + +// TestSetRecoveryGuardIgnoresNil pins that a nil guard leaves the manager's +// existing guard (its own default, or one installed earlier) in place +// instead of leaving the manager without a guard at all. +func TestSetRecoveryGuardIgnoresNil(t *testing.T) { + mgr := newTestManager(t) + existing := mgr.recoveryGuard + + mgr.SetRecoveryGuard(nil) + + if mgr.recoveryGuard != existing { + t.Fatal("SetRecoveryGuard(nil) replaced the manager's existing guard") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_stop_detach_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_stop_detach_test.go new file mode 100644 index 0000000000..6e24b4153b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_stop_detach_test.go @@ -0,0 +1,157 @@ +package lifecycle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/agent/executor" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" +) + +// newDetachTestManager builds a Manager wired with a MockExecutor registered +// under executor.NameStandalone, so tests can assert whether +// StopAgentWithReason's survivable-detach branch (AC-EXECUTORS-SURVIVAL kill +// path #4) actually reaches StopInstance. +func newDetachTestManager(t *testing.T, agentSurvivalEnabled bool) (*Manager, *MockExecutor) { + t.Helper() + log := newTestLogger() + mockExecutor := &MockExecutor{name: executor.NameStandalone} + registry := NewExecutorRegistry(log) + registry.Register(mockExecutor) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + mgr.SetAgentSurvivalEnabled(agentSurvivalEnabled) + return mgr, mockExecutor +} + +func newDetachTestExecution(id, sessionID string) *AgentExecution { + return &AgentExecution{ + ID: id, + SessionID: sessionID, + TaskID: "task-" + id, + RuntimeName: executor.NameStandalone, + agentctl: agentctl.NewClient("127.0.0.1", 12345, newTestLogger()), + } +} + +// TestStopAgentWithReasonDetachesOnBackendShutdownWhenSurvivalEnabled pins +// design 01/02's kill-path #4 fix: a graceful backend shutdown with the +// capability enabled must not reach the executor's StopInstance or publish +// events.AgentStopped (which would drive the orchestrator to persist a +// terminal task-session state for a session whose agent is still running) -- +// it only releases this backend's own tracking and stream subscription. +func TestStopAgentWithReasonDetachesOnBackendShutdownWhenSurvivalEnabled(t *testing.T) { + mgr, mockExecutor := newDetachTestManager(t, true) + execution := newDetachTestExecution("exec-detach", "session-detach") + require.NoError(t, mgr.executionStore.Add(execution)) + + err := mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonBackendShutdown, false) + require.NoError(t, err) + + _, exists := mgr.executionStore.Get(execution.ID) + require.False(t, exists, "detach must remove this backend's own tracking") + require.Empty(t, mockExecutor.stopInstanceCalls, "detach must never call StopInstance; the instance is left running") + + client, release := execution.AcquireAgentCtlClient() + release() + require.Nil(t, client, "detach must release this backend's agentctl stream subscription") + + eventBus := mgr.eventBus.(*MockEventBus) + for _, evt := range eventBus.PublishedEvents { + require.NotEqual(t, events.AgentStopped, evt.Type, + "detach must not publish agent.stopped: it would drive the orchestrator to persist a terminal state for a still-running agent") + } +} + +// TestStopAgentWithReasonTerminatesForNonShutdownReasonEvenWhenSurvivalEnabled +// confirms the detach branch is scoped exactly to StopReasonBackendShutdown: +// every other stop reason -- user stops, rollback cleanup, task deletion -- +// keeps today's terminating semantics even when the capability is enabled. +func TestStopAgentWithReasonTerminatesForNonShutdownReasonEvenWhenSurvivalEnabled(t *testing.T) { + mgr, mockExecutor := newDetachTestManager(t, true) + execution := newDetachTestExecution("exec-terminate", "session-terminate") + require.NoError(t, mgr.executionStore.Add(execution)) + + err := mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonTaskDeleted, false) + require.NoError(t, err) + + require.Len(t, mockExecutor.stopInstanceCalls, 1, "a non-shutdown reason must still stop the runtime instance") + + eventBus := mgr.eventBus.(*MockEventBus) + require.True(t, hasAgentStoppedEvent(eventBus.PublishedEvents), "a terminating stop must still publish agent.stopped") +} + +// TestStopAgentWithReasonTerminatesOnBackendShutdownWhenSurvivalDisabled +// confirms the zero value (capability disabled) preserves today's behavior: +// backend shutdown terminates every agent exactly as it did before this +// capability existed. +func TestStopAgentWithReasonTerminatesOnBackendShutdownWhenSurvivalDisabled(t *testing.T) { + mgr, mockExecutor := newDetachTestManager(t, false) + execution := newDetachTestExecution("exec-disabled", "session-disabled") + require.NoError(t, mgr.executionStore.Add(execution)) + + err := mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonBackendShutdown, false) + require.NoError(t, err) + + require.Len(t, mockExecutor.stopInstanceCalls, 1, "capability disabled must keep the terminating stop on backend shutdown") + + eventBus := mgr.eventBus.(*MockEventBus) + require.True(t, hasAgentStoppedEvent(eventBus.PublishedEvents), "a terminating stop must still publish agent.stopped") +} + +func hasAgentStoppedEvent(published []*bus.Event) bool { + for _, evt := range published { + if evt.Type == events.AgentStopped { + return true + } + } + return false +} + +// TestStopAgentWithReasonTerminatesNonStandaloneRuntimeEvenWhenSurvivalEnabled +// pins the scope boundary the capability is documented to have: the +// survivable-detach branch exists only for the standalone (worktree/local) +// runtime this feature covers. `StopAllAgents` calls StopAgentWithReason with +// StopReasonBackendShutdown for every tracked execution regardless of +// runtime, so a Docker/SSH/Sprites/Kubernetes/remote-docker execution must +// still reach the terminating path -- including the real StopInstance call +// and the agent.stopped publish -- even while the capability is globally +// enabled for the installation's standalone sessions. +func TestStopAgentWithReasonTerminatesNonStandaloneRuntimeEvenWhenSurvivalEnabled(t *testing.T) { + log := newTestLogger() + mockExecutor := &MockExecutor{name: executor.NameDocker} + registry := NewExecutorRegistry(log) + registry.Register(mockExecutor) + eventBus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), eventBus, registry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + mgr.SetAgentSurvivalEnabled(true) + + execution := &AgentExecution{ + ID: "exec-docker", + SessionID: "session-docker", + TaskID: "task-exec-docker", + RuntimeName: executor.NameDocker, + } + require.NoError(t, mgr.executionStore.Add(execution)) + + err := mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonBackendShutdown, false) + require.NoError(t, err) + + require.Len(t, mockExecutor.stopInstanceCalls, 1, + "a non-standalone runtime must still be stopped on backend shutdown even when the survival capability is enabled") + + eventBus2 := mgr.eventBus.(*MockEventBus) + require.True(t, hasAgentStoppedEvent(eventBus2.PublishedEvents), + "a non-standalone runtime's terminating stop must still publish agent.stopped") + + _, exists := mgr.executionStore.Get(execution.ID) + require.False(t, exists, "a terminated (non-detached) execution must be removed from tracking") +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_stop_worktree_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_stop_worktree_test.go new file mode 100644 index 0000000000..e1bd1a2615 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_stop_worktree_test.go @@ -0,0 +1,76 @@ +package lifecycle + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// seedWorktree builds a directory standing in for a session's worktree, with +// one tracked file and one gitignored file, and returns the path plus a +// closure that fails the test if anything under it changed. +func seedWorktree(t *testing.T) (string, func()) { + t.Helper() + root := filepath.Join(t.TempDir(), "worktree") + require.NoError(t, os.MkdirAll(filepath.Join(root, "src"), 0o755)) + + files := map[string]string{ + filepath.Join(root, "src", "main.go"): "package main\n\nfunc main() {}\n", + filepath.Join(root, ".env.local"): "SECRET=value\n", + } + for path, contents := range files { + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + } + + return root, func() { + t.Helper() + info, err := os.Stat(root) + require.NoErrorf(t, err, "the worktree directory is gone after the stop") + require.True(t, info.IsDir(), "the worktree path is no longer a directory") + for path, want := range files { + got, err := os.ReadFile(path) //nolint:gosec // test-owned path under t.TempDir() + require.NoErrorf(t, err, "worktree file %s is gone after the stop", path) + require.Equalf(t, want, string(got), "worktree file %s was modified by the stop", path) + } + } +} + +// TestBackendShutdownLeavesTheWorktreeUntouchedOnBothStopPaths pins +// AC-EXECUTORS-SURVIVAL-001.7 across the branch this capability introduced. +// A worktree holds the user's uncommitted work, so neither outcome of a +// backend shutdown may touch it: not the survivable detach that leaves the +// agent running, and not the terminating stop the same shutdown takes when +// the capability is off. Only an explicit task teardown removes a worktree, +// and a shutdown is not one. +func TestBackendShutdownLeavesTheWorktreeUntouchedOnBothStopPaths(t *testing.T) { + for _, tc := range []struct { + name string + survivalEnabled bool + wantStopInstance bool + }{ + {name: "survivable detach", survivalEnabled: true, wantStopInstance: false}, + {name: "terminating stop", survivalEnabled: false, wantStopInstance: true}, + } { + t.Run(tc.name, func(t *testing.T) { + mgr, mockExecutor := newDetachTestManager(t, tc.survivalEnabled) + worktree, assertUnchanged := seedWorktree(t) + + execution := newDetachTestExecution("exec-worktree", "session-worktree") + execution.WorkspacePath = worktree + require.NoError(t, mgr.executionStore.Add(execution)) + + require.NoError(t, mgr.StopAgentWithReason(t.Context(), execution.ID, StopReasonBackendShutdown, false)) + + // Guards the premise: if the branch under test stopped being + // reached, the worktree assertion below would pass vacuously. + if tc.wantStopInstance { + require.NotEmpty(t, mockExecutor.stopInstanceCalls, "expected the terminating path") + } else { + require.Empty(t, mockExecutor.stopInstanceCalls, "expected the detach path") + } + assertUnchanged() + }) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan_guard_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan_guard_test.go index 111b82e7f3..aece982bd9 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan_guard_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan_guard_test.go @@ -3,6 +3,7 @@ package lifecycle import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "sync" @@ -10,9 +11,27 @@ import ( "github.com/stretchr/testify/require" + "github.com/kandev/kandev/internal/agent/agents" "github.com/kandev/kandev/internal/agent/executor" ) +// registerRecoveryTestAgentProfile wires a resolvable agent profile onto mgr +// so a recovered execution's agent identity re-derivation +// (AC-EXECUTORS-SURVIVAL-002.14) succeeds. Tests that assert a recovered +// session lands in the store, but don't care about command-building details, +// use this instead of repeating the wiring; the matching fixture's +// AgentProfileID must be recoveryTestAgentProfileID. +const recoveryTestAgentProfileID = "profile-1" + +func registerRecoveryTestAgentProfile(t *testing.T, mgr *Manager) { + t.Helper() + require.NoError(t, mgr.registry.Register(&cliFlagTestAgent{testAgent: testAgent{id: "recovery-test-agent"}})) + mgr.profileResolver = &restartProfileResolver{profile: &AgentProfileInfo{ + ProfileID: recoveryTestAgentProfileID, + AgentName: "recovery-test-agent", + }} +} + type recordedRescan struct { WorkDir string `json:"work_dir"` SourceRoots []string `json:"workspace_source_roots"` @@ -125,19 +144,21 @@ func TestManagerStartRecoversInstancesIntoStore(t *testing.T) { registry.Register(&MockExecutor{ name: executor.NameStandalone, recoverInstances: []*ExecutorInstance{{ - InstanceID: "exec-recovered", - TaskID: "task-1", - SessionID: "session-1", - ContainerID: "container-1", - ContainerIP: "10.0.0.2", - WorkspacePath: "/workspace", - RuntimeName: executor.NameStandalone, + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + ContainerID: "container-1", + ContainerIP: "10.0.0.2", + WorkspacePath: "/workspace", + RuntimeName: executor.NameStandalone, }}, }) mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, ExecutorFallbackWarn, "", log) cleanupManagerStopCh(t, mgr) t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) require.NoError(t, mgr.Start(context.Background())) @@ -148,6 +169,330 @@ func TestManagerStartRecoversInstancesIntoStore(t *testing.T) { require.Equal(t, "/workspace", execution.WorkspacePath) } +// TestManagerStartRecoveredExecutionCarriesEnvAndRunID pins +// AC-EXECUTORS-SURVIVAL-002.14's "runtime environment" and "run identity" +// reconstruction rows: both are sourced from the adopted instance's own +// environment (never the database), with run identity re-derived from its +// KANDEV_RUN_ID key. +func TestManagerStartRecoveredExecutionCarriesEnvAndRunID(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + Env: map[string]string{"KANDEV_RUN_ID": "run-42", "PATH": "/usr/bin"}, + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Equal(t, "run-42", execution.RunID) + require.Equal(t, map[string]string{"KANDEV_RUN_ID": "run-42", "PATH": "/usr/bin"}, execution.RuntimeEnvironment()) +} + +// TestManagerStartRecoveredExecutionCarriesTaskEnvironmentID pins +// AC-EXECUTORS-SURVIVAL-002.14's "task-environment identity" reconstruction +// row: sourced from the durable store via the session's own task-environment +// reference (WorkspaceInfoProvider), never re-derived from the adopted +// instance, which carries no such identity at all. +func TestManagerStartRecoveredExecutionCarriesTaskEnvironmentID(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + mgr.SetWorkspaceInfoProvider(&mockWorkspaceInfoProvider{infos: map[string]*WorkspaceInfo{ + "session-1": {TaskEnvironmentID: "env-9"}, + }}) + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Equal(t, "env-9", execution.TaskEnvironmentID) +} + +// TestManagerStartRecoveredExecutionRefusesOnWorkspaceInfoLookupFailure pins +// Review round 3, finding 4: task-environment identity is one of +// AC-EXECUTORS-SURVIVAL-002.3's required reconstructed values, so a +// durable-store read failure during recovery must not be tolerated as a soft +// failure -- AC-EXECUTORS-SURVIVAL-002.13 forbids publishing the session as +// re-tracked when a required read fails, and this instance is instead routed +// to the AC-EXECUTORS-SURVIVAL-002.6 stop path. (An earlier version of this +// test pinned the opposite, buggy behavior.) +func TestManagerStartRecoveredExecutionRefusesOnWorkspaceInfoLookupFailure(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + mockExec := &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + }}, + } + registry.Register(mockExec) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + mgr.SetWorkspaceInfoProvider(&mockWorkspaceInfoProvider{err: errors.New("db down")}) + + require.NoError(t, mgr.Start(context.Background())) + + _, ok := mgr.GetExecutionBySessionID("session-1") + require.False(t, ok, "recovery must refuse to re-track when task-environment identity cannot be reconstructed") + require.Len(t, mockExec.stopInstanceCalls, 1) + require.Equal(t, "exec-recovered", mockExec.stopInstanceCalls[0].InstanceID) +} + +// TestManagerStartRecoveredExecutionCarriesOfficeAgentProfileID pins +// AC-EXECUTORS-SURVIVAL-002.14's "Office profile identity" reconstruction +// row: sourced from the recovery-inventory record's own persisted metadata +// (a new key in the existing column), not re-derived or read from the +// instance. +func TestManagerStartRecoveredExecutionCarriesOfficeAgentProfileID(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + Metadata: map[string]interface{}{ + MetadataKeyOfficeAgentProfileID: "office-profile-9", + }, + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Equal(t, "office-profile-9", execution.OfficeAgentProfileID) +} + +// TestManagerStartRecoveredExecutionLeavesOfficeAgentProfileIDEmptyForNonOffice +// pins that an absent metadata key (the common, non-Office case) is a +// legitimately empty value, not an error. +func TestManagerStartRecoveredExecutionLeavesOfficeAgentProfileIDEmptyForNonOffice(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Empty(t, execution.OfficeAgentProfileID) +} + +// TestManagerStartRecoveredExecutionCarriesWorkspaceSourceRoots pins +// AC-EXECUTORS-SURVIVAL-002.14's "workspace source roots" reconstruction +// row: sourced from the adopted instance, read back rather than pushed. +func TestManagerStartRecoveredExecutionCarriesWorkspaceSourceRoots(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + WorkspaceSourceRoots: []string{"/ws/task-1/backend", "/ws/task-1/frontend"}, + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Equal(t, []string{"/ws/task-1/backend", "/ws/task-1/frontend"}, execution.WorkspaceSourceRoots) +} + +// TestManagerStartRecoveredExecutionCarriesProviderSessionID pins +// AC-EXECUTORS-SURVIVAL-002.14's "provider session identity" reconstruction +// row: sourced from the adopted instance, which holds the live provider +// session, never from a durable/database value. +func TestManagerStartRecoveredExecutionCarriesProviderSessionID(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: recoveryTestAgentProfileID, + RuntimeName: executor.NameStandalone, + ProviderSessionID: "provider-session-9", + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + registerRecoveryTestAgentProfile(t, mgr) + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Equal(t, "provider-session-9", execution.ACPSessionID) +} + +// agentIdentityHistoryTestAgent is a resolvable agent whose runtime config +// opts in to history-context injection, so +// TestManagerStartRecoveredExecutionCarriesAgentIdentityCommandsAndHistorySetting +// can assert historyEnabled alongside the built commands. +type agentIdentityHistoryTestAgent struct { + testAgent +} + +func (a *agentIdentityHistoryTestAgent) BuildCommand(_ agents.CommandOptions) agents.Command { + return agents.Cmd("test-agent-cli", "--acp").Build() +} + +// TestManagerStartRecoveredExecutionCarriesAgentIdentityCommandsAndHistorySetting +// pins AC-EXECUTORS-SURVIVAL-002.14's last reconstruction-table row: agent +// identity, the agent and continuation commands and their arguments, and the +// history setting are re-derived from the restored agent profile and the +// agent-type registry -- the same computation an ordinary launch performs -- +// never read from the adopted instance. +func TestManagerStartRecoveredExecutionCarriesAgentIdentityCommandsAndHistorySetting(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-recovered", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: "profile-history", + RuntimeName: executor.NameStandalone, + }}, + }) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + require.NoError(t, mgr.registry.Register(&agentIdentityHistoryTestAgent{ + testAgent: testAgent{ + id: "history-agent", + runtimeConfig: &agents.RuntimeConfig{ + Cmd: agents.NewCommand("test-agent-cli"), + SessionConfig: agents.SessionConfig{HistoryContextInjection: true}, + }, + }, + })) + mgr.profileResolver = &restartProfileResolver{profile: &AgentProfileInfo{ + ProfileID: "profile-history", + AgentName: "history-agent", + }} + + require.NoError(t, mgr.Start(context.Background())) + + execution, ok := mgr.GetExecutionBySessionID("session-1") + require.True(t, ok) + require.Equal(t, "history-agent", execution.AgentID) + require.Equal(t, "test-agent-cli --acp", execution.AgentCommand) + require.Equal(t, []string{"test-agent-cli", "--acp"}, execution.AgentArgs) + require.True(t, execution.historyEnabled, "history-context-injection agent must have historyEnabled restored") +} + +// TestManagerStartRefusesAndStopsRecoveredExecutionWhenAgentIdentityCannotBeReconstructed +// pins AC-EXECUTORS-SURVIVAL-002.4's refusal path for +// AC-EXECUTORS-SURVIVAL-002.14's re-derivation row: when the restored +// AgentProfileID no longer resolves to a registered agent type (declared +// source answered, value not present), the instance is not re-tracked and is +// left to the stop path AC-EXECUTORS-SURVIVAL-002.6 defines. +func TestManagerStartRefusesAndStopsRecoveredExecutionWhenAgentIdentityCannotBeReconstructed(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + mockExecutor := &MockExecutor{ + name: executor.NameStandalone, + recoverInstances: []*ExecutorInstance{{ + InstanceID: "exec-unreconstructable", + TaskID: "task-1", + SessionID: "session-1", + AgentProfileID: "profile-deleted", + RuntimeName: executor.NameStandalone, + }}, + } + registry.Register(mockExecutor) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, + ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + t.Cleanup(func() { _ = mgr.Stop() }) + // No agent registered under "history-agent"/whatever this profile names, + // and the profile resolver reports genuine failure -- either way, + // getAgentConfigForExecution cannot answer. + mgr.profileResolver = &restartProfileResolver{err: errors.New("profile deleted during outage")} + + require.NoError(t, mgr.Start(context.Background())) + + _, ok := mgr.GetExecutionBySessionID("session-1") + require.False(t, ok, "an execution whose agent identity cannot be reconstructed must not be re-tracked") + require.Len(t, mockExecutor.stopInstanceCalls, 1) + require.Equal(t, "exec-unreconstructable", mockExecutor.stopInstanceCalls[0].InstanceID) +} + func TestManagerStartWithoutRegistryIsNoOp(t *testing.T) { mgr := newTestManager(t) diff --git a/apps/backend/internal/agent/runtime/lifecycle/ownership_renewal.go b/apps/backend/internal/agent/runtime/lifecycle/ownership_renewal.go new file mode 100644 index 0000000000..8f21e0361a --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/ownership_renewal.go @@ -0,0 +1,131 @@ +package lifecycle + +import ( + "context" + "errors" + "sync" + "time" + + "go.uber.org/zap" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/logger" +) + +// OwnershipClaimer is the narrow surface the renewal loop drives, satisfied +// by *agentctl.ControlClient. Narrowed to one method so tests can drive the +// loop without a real HTTP round trip. +type OwnershipClaimer interface { + ClaimOwnership(ctx context.Context) error +} + +// OwnershipRenewer periodically issues the ownership-claim operation +// (AC-EXECUTORS-CONTROL-OWNERSHIP-003.8) so the adopted or freshly spawned +// control server never begins an unowned shutdown while this backend is +// running (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2). It does not claim +// immediately on Start: the bootstrap handshake (fresh spawn) or a +// successful credential rotation (adoption) already counted as the first +// renewal at the moment this loop is started, so an immediate claim here +// would be a redundant fourth renewing operation not listed by AC-003.8. +// +// Follows the goroutine-ownership convention (explicit Start/Stop, +// WaitGroup-registered, idempotent both ends, selects on a stop signal +// rather than sleeping) established by internal/integrations/healthpoll. +type OwnershipRenewer struct { + claimer OwnershipClaimer + interval time.Duration + logger *logger.Logger + + mu sync.Mutex + cancel context.CancelFunc + wg sync.WaitGroup + started bool +} + +// NewOwnershipRenewer builds a renewer that issues ClaimOwnership on the +// given interval, which callers must derive via +// ownershipperiod.RenewalInterval so it stays strictly inside a third of +// the resolved unowned period. +func NewOwnershipRenewer(claimer OwnershipClaimer, interval time.Duration, log *logger.Logger) *OwnershipRenewer { + return &OwnershipRenewer{claimer: claimer, interval: interval, logger: log} +} + +// Start launches the background loop. Calling Start more than once without +// Stop is a no-op. +func (r *OwnershipRenewer) Start(ctx context.Context) { + r.mu.Lock() + defer r.mu.Unlock() + if r.started || r.claimer == nil || r.interval <= 0 { + return + } + r.started = true + ctx, r.cancel = context.WithCancel(ctx) + r.wg.Add(1) + go r.loop(ctx) +} + +// Stop cancels the loop and waits for it to drain. Safe to call even if +// Start was never called, and safe to call more than once. +func (r *OwnershipRenewer) Stop() { + r.mu.Lock() + if !r.started { + r.mu.Unlock() + return + } + cancel := r.cancel + r.mu.Unlock() + if cancel != nil { + cancel() + } + r.wg.Wait() + r.mu.Lock() + r.started = false + r.mu.Unlock() +} + +func (r *OwnershipRenewer) loop(ctx context.Context) { + defer r.wg.Done() + ticker := time.NewTicker(r.interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if err := r.claimer.ClaimOwnership(ctx); err != nil { + if errors.Is(err, agentctl.ErrOwnershipCredentialSuperseded) { + // AC-EXECUTORS-CONTROL-OWNERSHIP-002.3: a superseded + // credential means some other party rotated it out from + // under this backend (most likely a second backend that + // adopted the server). Retrying would keep presenting a + // credential that can never succeed again, and this loop + // must not mint or fetch a replacement on its own -- only + // a fresh adoption may do that. Stop instead of looping + // back to the ticker. + r.logger.Error("ownership renewal claim rejected: credential superseded; stopping renewal loop without retry", + zap.Error(err)) + r.stopSelf() + return + } + r.logger.Warn("ownership renewal claim failed; will retry on the next interval", + zap.Error(err)) + } + } + } +} + +// stopSelf marks the renewer stopped from within its own loop goroutine. +// Unlike Stop, it must not call wg.Wait(): the loop goroutine invoking this +// is exactly what wg is waiting on, so waiting here would deadlock. A +// concurrent explicit Stop() call remains safe: it will see r.started +// already false and return immediately, or it will call the (idempotent) +// cancel and then wg.Wait(), which unblocks as soon as this goroutine +// returns. +func (r *OwnershipRenewer) stopSelf() { + r.mu.Lock() + defer r.mu.Unlock() + if r.cancel != nil { + r.cancel() + } + r.started = false +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/ownership_renewal_test.go b/apps/backend/internal/agent/runtime/lifecycle/ownership_renewal_test.go new file mode 100644 index 0000000000..6589af72f4 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/ownership_renewal_test.go @@ -0,0 +1,228 @@ +package lifecycle + +import ( + "context" + "sync" + "testing" + "testing/synctest" + "time" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/logger" +) + +// countingClaimer records every ClaimOwnership call, optionally failing the +// first N calls to exercise the retry-on-next-interval path, or returning a +// specific error (nextErr, consumed once) to exercise a distinguished +// rejection such as a superseded credential. +type countingClaimer struct { + mu sync.Mutex + calls int + failNext int + nextErr error +} + +func (c *countingClaimer) ClaimOwnership(_ context.Context) error { + c.mu.Lock() + defer c.mu.Unlock() + c.calls++ + if c.nextErr != nil { + err := c.nextErr + c.nextErr = nil + return err + } + if c.failNext > 0 { + c.failNext-- + return context.DeadlineExceeded + } + return nil +} + +func (c *countingClaimer) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.calls +} + +func TestOwnershipRenewerStartStopIdempotent(t *testing.T) { + r := NewOwnershipRenewer(&countingClaimer{}, time.Second, logger.Default()) + + r.Start(context.Background()) + t.Cleanup(r.Stop) + r.Start(context.Background()) // second start is a no-op + r.Stop() + r.Stop() // second stop is a no-op +} + +func TestOwnershipRenewerStopSafeWithoutStart(t *testing.T) { + r := NewOwnershipRenewer(&countingClaimer{}, time.Second, logger.Default()) + r.Stop() +} + +// TestOwnershipRenewerDoesNotClaimImmediately pins that Start does not issue +// an immediate claim: the caller's rotation/handshake that just happened is +// already the first renewal per AC-EXECUTORS-CONTROL-OWNERSHIP-003.8, so an +// immediate claim here would be a redundant fourth renewing operation. +func TestOwnershipRenewerDoesNotClaimImmediately(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + t.Cleanup(r.Stop) + + synctest.Wait() + if got := claimer.callCount(); got != 0 { + t.Fatalf("callCount immediately after Start = %d, want 0", got) + } + }) +} + +// TestOwnershipRenewerClaimsOnInterval pins that the loop issues one claim +// per elapsed interval, on the interval computed by the caller. +func TestOwnershipRenewerClaimsOnInterval(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + t.Cleanup(r.Stop) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + if got := claimer.callCount(); got != 1 { + t.Fatalf("callCount after one interval = %d, want 1", got) + } + + time.Sleep(time.Minute) + synctest.Wait() + if got := claimer.callCount(); got != 2 { + t.Fatalf("callCount after two intervals = %d, want 2", got) + } + }) +} + +// TestOwnershipRenewerRetriesAfterFailure pins that a failed claim does not +// stop the loop: the next interval issues another attempt. +func TestOwnershipRenewerRetriesAfterFailure(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{failNext: 1} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + t.Cleanup(r.Stop) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + if got := claimer.callCount(); got != 1 { + t.Fatalf("callCount after failed interval = %d, want 1", got) + } + + time.Sleep(time.Minute) + synctest.Wait() + if got := claimer.callCount(); got != 2 { + t.Fatalf("callCount after retry interval = %d, want 2", got) + } + }) +} + +// TestOwnershipRenewerStopsWithoutRetryOnSupersededCredential pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.3: a superseded-credential rejection +// (distinct from a transient failure) must not be retried on the next +// interval -- unlike TestOwnershipRenewerRetriesAfterFailure, the loop ends +// instead of ticking again. +func TestOwnershipRenewerStopsWithoutRetryOnSupersededCredential(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{nextErr: agentctl.ErrOwnershipCredentialSuperseded} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + t.Cleanup(r.Stop) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + if got := claimer.callCount(); got != 1 { + t.Fatalf("callCount after superseded rejection = %d, want 1", got) + } + + time.Sleep(3 * time.Minute) + synctest.Wait() + if got := claimer.callCount(); got != 1 { + t.Fatalf("callCount after further intervals = %d, want 1 (no retry)", got) + } + }) +} + +// TestOwnershipRenewerSupersededCredentialAllowsRestart pins that the loop +// self-stopping on a superseded credential leaves the renewer in a state a +// later Start (after the manager re-adopts and obtains a current +// credential) can reuse -- AC-002.3's "shall not acquire a fresh credential +// without adopting again" is the caller's job, but this renewer must not be +// left stuck "started" forever, which would make a legitimate restart a +// silent no-op. +func TestOwnershipRenewerSupersededCredentialAllowsRestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{nextErr: agentctl.ErrOwnershipCredentialSuperseded} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + if got := claimer.callCount(); got != 1 { + t.Fatalf("callCount after superseded rejection = %d, want 1", got) + } + + // Simulate re-adoption obtaining a fresh credential and restarting + // the renewer against it. + r.Start(context.Background()) + t.Cleanup(r.Stop) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + if got := claimer.callCount(); got != 2 { + t.Fatalf("callCount after restart interval = %d, want 2 (restart took effect)", got) + } + }) +} + +// TestOwnershipRenewerExplicitStopAfterSupersededCredentialIsSafe pins that +// an explicit Stop() call racing with (or arriving after) the loop's own +// self-stop neither panics nor hangs. +func TestOwnershipRenewerExplicitStopAfterSupersededCredentialIsSafe(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{nextErr: agentctl.ErrOwnershipCredentialSuperseded} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + + r.Stop() + r.Stop() // still safe a second time + }) +} + +// TestOwnershipRenewerStopEndsLoop pins that Stop halts further claims. +func TestOwnershipRenewerStopEndsLoop(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + claimer := &countingClaimer{} + r := NewOwnershipRenewer(claimer, time.Minute, logger.Default()) + r.Start(context.Background()) + + time.Sleep(time.Minute + time.Second) + synctest.Wait() + r.Stop() + + time.Sleep(5 * time.Minute) + synctest.Wait() + if got := claimer.callCount(); got != 1 { + t.Fatalf("callCount after Stop = %d, want 1 (no further claims)", got) + } + }) +} + +func TestOwnershipRenewerNoOpWithoutClaimerOrInterval(t *testing.T) { + r := NewOwnershipRenewer(nil, time.Minute, logger.Default()) + r.Start(context.Background()) + r.Stop() + + r2 := NewOwnershipRenewer(&countingClaimer{}, 0, logger.Default()) + r2.Start(context.Background()) + r2.Stop() +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/passthrough_scope.go b/apps/backend/internal/agent/runtime/lifecycle/passthrough_scope.go new file mode 100644 index 0000000000..ee167a40ed --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/passthrough_scope.go @@ -0,0 +1,66 @@ +package lifecycle + +import "github.com/kandev/kandev/internal/task/models" + +// AC-EXECUTORS-SURVIVAL-005.3 (design 02 "Passthrough scope"): a passthrough +// session shall behave as if the capability were disabled for that session. +// The agent runs on a terminal owned by *this* backend process, so it dies +// with the backend no matter what the control server does -- there is nothing +// to survive. The two obligations that follow are implemented here and used +// by the two paths that would otherwise treat such a session as survivable: +// +// - never detached: StopAgentWithReason must take the terminating path for +// a passthrough execution even on a graceful backend shutdown, so the +// runtime instance is stopped, agent.stopped is published, and the +// executors_running row stops claiming a live agent. Detaching instead +// leaves a record asserting a running agent that no later pass repairs +// (AC-EXECUTORS-SURVIVAL-004.5). +// - never re-tracked: a passthrough session still owns a real agentctl +// instance (it is created by CreateInstance like any other execution, or +// promoted from a workspace-only one), so that instance genuinely does +// survive on the detached control server and would correlate to the +// session's recovery-inventory record. Re-tracking it would publish the +// session as running while its PTY agent is gone. + +// isPassthroughExecution reports whether a tracked execution is a passthrough +// (PTY/TUI) one. Mirrors the existing in-memory idiom used elsewhere in this +// package (IsAgentReadyForPrompt, RecoverAgentPromptStream): the flag is set +// at launch and at workspace-execution promotion, and PassthroughProcessID is +// set once the terminal process exists, so either being present is the signal. +// In-memory state is authoritative here because the caller holds a live, +// tracked execution -- unlike startup, where nothing is tracked yet and the +// durable session record is the only source (SessionsToGuard). +func isPassthroughExecution(execution *AgentExecution) bool { + return execution != nil && (execution.IsPassthrough || execution.PassthroughProcessID != "") +} + +// recoverableRecords drops the confirmed-passthrough records from a recovery +// pass's inventory so re-tracking never reaches them. guarded is the set +// SessionsToGuard already computed for this pass -- "every named session minus +// the confirmed-passthrough ones" -- so a session whose passthrough mode could +// not be read stays both guarded and recoverable, which is the fail-safe +// direction AC-EXECUTORS-SURVIVAL-005.3 requires ("when that mode cannot be +// read for a record, the session shall be guarded rather than excluded"). +// +// A record with no session identity is kept rather than dropped: it is not a +// session this pass could have classified either way, and correlation already +// skips it. The only records this removes are the ones confirmed passthrough. +func recoverableRecords(records []*models.ExecutorRunning, guarded []string) []*models.ExecutorRunning { + guardedSet := make(map[string]struct{}, len(guarded)) + for _, sessionID := range guarded { + guardedSet[sessionID] = struct{}{} + } + recoverable := make([]*models.ExecutorRunning, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + if record.SessionID != "" { + if _, ok := guardedSet[record.SessionID]; !ok { + continue + } + } + recoverable = append(recoverable, record) + } + return recoverable +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/persistence.go b/apps/backend/internal/agent/runtime/lifecycle/persistence.go index 2121fb6e14..21b78e5154 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/persistence.go +++ b/apps/backend/internal/agent/runtime/lifecycle/persistence.go @@ -73,6 +73,12 @@ func buildRunningFromExecution(execution *AgentExecution, prior *models.Executor } metadata := execution.MetadataSnapshot() + if execution.OfficeAgentProfileID != "" { + if metadata == nil { + metadata = make(map[string]interface{}) + } + metadata[MetadataKeyOfficeAgentProfileID] = execution.OfficeAgentProfileID + } running := &models.ExecutorRunning{ ID: execution.SessionID, SessionID: execution.SessionID, @@ -456,6 +462,28 @@ type executorRunningReader interface { GetExecutorRunningBySessionID(ctx context.Context, sessionID string) (*models.ExecutorRunning, error) } +// executorRunningLister is the optional read-side used to enumerate the +// startup recovery inventory: every live (non-terminal) executors_running row +// on the standalone control server (worktree/local executors). A writer that +// doesn't implement it has no recovery candidates to offer. +type executorRunningLister interface { + ListExecutorsRunningLiveStandalone(ctx context.Context) ([]*models.ExecutorRunning, error) +} + +// ListLiveStandaloneExecutorsRunning returns the startup recovery inventory: +// every live standalone executors_running row, read at startup step 3 before +// any control-server contact, so the recovery guard can be taken against it +// (discovery H). Best-effort: a writer that doesn't support listing yields no +// candidates rather than an error, matching this file's other optional +// capabilities. +func (m *Manager) ListLiveStandaloneExecutorsRunning(ctx context.Context) ([]*models.ExecutorRunning, error) { + lister, ok := m.runningWriter.(executorRunningLister) + if !ok { + return nil, nil + } + return lister.ListExecutorsRunningLiveStandalone(ctx) +} + type executorRunningCASWriter interface { RepairExecutorRunningDeadIfCurrent( ctx context.Context, diff --git a/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go b/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go index a0eba1acdf..14a3a9ec82 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go @@ -104,6 +104,27 @@ func TestBuildRunningFromExecutionPreservesPriorExecutorIdentity(t *testing.T) { } } +func TestBuildRunningFromExecutionPersistsOfficeAgentProfileID(t *testing.T) { + running := buildRunningFromExecution(&AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + OfficeAgentProfileID: "office-profile-9", + }, nil) + + if got := running.Metadata[MetadataKeyOfficeAgentProfileID]; got != "office-profile-9" { + t.Fatalf("Metadata[%q] = %v, want office-profile-9", MetadataKeyOfficeAgentProfileID, got) + } +} + +func TestBuildRunningFromExecutionOmitsOfficeAgentProfileIDForNonOfficeLaunch(t *testing.T) { + running := buildRunningFromExecution(&AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + }, nil) + + if _, ok := running.Metadata[MetadataKeyOfficeAgentProfileID]; ok { + t.Fatal("Metadata should not carry an office_agent_profile_id key for a non-Office launch") + } +} + func TestBuildRunningFromExecutionBindsResumeTokenToExecutionProfile(t *testing.T) { prior := &models.ExecutorRunning{ ExecutionProfileID: "codex-profile", diff --git a/apps/backend/internal/agent/runtime/lifecycle/recovery_correlation.go b/apps/backend/internal/agent/runtime/lifecycle/recovery_correlation.go new file mode 100644 index 0000000000..e0004298fc --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/recovery_correlation.go @@ -0,0 +1,103 @@ +package lifecycle + +import ( + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/task/models" +) + +// RecoveryCorrelationResult is the outcome of correlating adopted-server +// instances to recovery-inventory records by session identity +// (AC-EXECUTORS-SURVIVAL-002.1), including the AC-EXECUTORS-SURVIVAL-002.10 +// duplicate tiebreak. +type RecoveryCorrelationResult struct { + // Winners maps a session identity to the single live instance to + // re-track for it. + Winners map[string]*agentctl.InstanceInfo + // ToStop lists every instance that must be stopped: duplicate losers, + // ambiguous-session instances (AC-EXECUTORS-SURVIVAL-002.10), and + // orphans with no matching record (AC-EXECUTORS-SURVIVAL-002.6). + ToStop []*agentctl.InstanceInfo +} + +// CorrelateRecoveryInstances joins enumerated instances to recovery-inventory +// records by the session identity the control server itself reports +// (AC-EXECUTORS-SURVIVAL-002.1): +// +// - Exactly one live instance for a session with a record re-tracks that +// instance whether or not its instance identifier equals the record's +// agent execution identifier (AC-EXECUTORS-SURVIVAL-002.2). +// - More than one live instance for the same session re-tracks only the +// one whose instance identifier equals that session's record's agent +// execution identifier; no match or more than one match re-tracks none +// of them and stops every one (AC-EXECUTORS-SURVIVAL-002.10). +// - A live instance whose session has no record is stopped, because it has +// no session identity to attach it to (AC-EXECUTORS-SURVIVAL-002.6). +// - A record with no live instance is left alone entirely: it is not an +// input this function stops or wins, it is left to the existing +// stale-execution repair path (AC-EXECUTORS-SURVIVAL-002.7). +// +// Records are keyed by session identity, so this function has no dependency +// on iteration or arrival order between sessions (AC-EXECUTORS-SURVIVAL-002.11). +func CorrelateRecoveryInstances( + records []*models.ExecutorRunning, + instances []*agentctl.InstanceInfo, +) *RecoveryCorrelationResult { + recordBySession := make(map[string]*models.ExecutorRunning, len(records)) + for _, rec := range records { + if rec == nil || rec.SessionID == "" { + continue + } + recordBySession[rec.SessionID] = rec + } + + instancesBySession := make(map[string][]*agentctl.InstanceInfo) + for _, inst := range instances { + if inst == nil { + continue + } + instancesBySession[inst.SessionID] = append(instancesBySession[inst.SessionID], inst) + } + + result := &RecoveryCorrelationResult{Winners: make(map[string]*agentctl.InstanceInfo)} + for sessionID, group := range instancesBySession { + record, hasRecord := recordBySession[sessionID] + if !hasRecord { + result.ToStop = append(result.ToStop, group...) + continue + } + if len(group) == 1 { + result.Winners[sessionID] = group[0] + continue + } + result.resolveDuplicates(sessionID, record, group) + } + + return result +} + +// resolveDuplicates applies the AC-EXECUTORS-SURVIVAL-002.10 tiebreak for a +// session with more than one live instance. +func (r *RecoveryCorrelationResult) resolveDuplicates( + sessionID string, + record *models.ExecutorRunning, + group []*agentctl.InstanceInfo, +) { + var matches []*agentctl.InstanceInfo + for _, inst := range group { + if inst.ID == record.AgentExecutionID { + matches = append(matches, inst) + } + } + if len(matches) != 1 { + // No match or an ambiguous match: re-track none, stop every candidate. + r.ToStop = append(r.ToStop, group...) + return + } + winner := matches[0] + r.Winners[sessionID] = winner + for _, inst := range group { + if inst != winner { + r.ToStop = append(r.ToStop, inst) + } + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/recovery_correlation_test.go b/apps/backend/internal/agent/runtime/lifecycle/recovery_correlation_test.go new file mode 100644 index 0000000000..97feb3750b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/recovery_correlation_test.go @@ -0,0 +1,139 @@ +package lifecycle + +import ( + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/task/models" +) + +func TestCorrelateRecoveryInstancesSingleInstanceReTracksRegardlessOfIDMatch(t *testing.T) { + records := []*models.ExecutorRunning{ + {SessionID: "session-1", AgentExecutionID: "some-other-id"}, + } + instance := &agentctl.InstanceInfo{ID: "instance-1", SessionID: "session-1"} + + result := CorrelateRecoveryInstances(records, []*agentctl.InstanceInfo{instance}) + + if got := result.Winners["session-1"]; got != instance { + t.Fatalf("Winners[session-1] = %v, want the sole live instance despite ID mismatch", got) + } + if len(result.ToStop) != 0 { + t.Fatalf("ToStop = %v, want empty", result.ToStop) + } +} + +func TestCorrelateRecoveryInstancesOrphanInstanceIsStopped(t *testing.T) { + instance := &agentctl.InstanceInfo{ID: "instance-1", SessionID: "session-no-record"} + + result := CorrelateRecoveryInstances(nil, []*agentctl.InstanceInfo{instance}) + + if len(result.Winners) != 0 { + t.Fatalf("Winners = %v, want empty", result.Winners) + } + if len(result.ToStop) != 1 || result.ToStop[0] != instance { + t.Fatalf("ToStop = %v, want [instance]", result.ToStop) + } +} + +func TestCorrelateRecoveryInstancesRecordWithNoInstanceIsLeftAlone(t *testing.T) { + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "instance-1"}} + + result := CorrelateRecoveryInstances(records, nil) + + if len(result.Winners) != 0 { + t.Fatalf("Winners = %v, want empty", result.Winners) + } + if len(result.ToStop) != 0 { + t.Fatalf("ToStop = %v, want empty (left to the existing stale-execution repair path)", result.ToStop) + } +} + +// TestCorrelateRecoveryInstancesDuplicateTiebreakExactMatchWins pins +// AC-EXECUTORS-SURVIVAL-002.10: with two live instances for one session, the +// one whose ID matches the record's agent execution identifier wins and the +// other is stopped. +func TestCorrelateRecoveryInstancesDuplicateTiebreakExactMatchWins(t *testing.T) { + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "winner"}} + winner := &agentctl.InstanceInfo{ID: "winner", SessionID: "session-1"} + loser := &agentctl.InstanceInfo{ID: "loser", SessionID: "session-1"} + + result := CorrelateRecoveryInstances(records, []*agentctl.InstanceInfo{loser, winner}) + + if got := result.Winners["session-1"]; got != winner { + t.Fatalf("Winners[session-1] = %v, want winner", got) + } + if len(result.ToStop) != 1 || result.ToStop[0] != loser { + t.Fatalf("ToStop = %v, want [loser]", result.ToStop) + } +} + +// TestCorrelateRecoveryInstancesDuplicateNoMatchStopsAll pins the "no match" +// branch of AC-EXECUTORS-SURVIVAL-002.10: neither instance's ID equals the +// record's agent execution identifier, so none is re-tracked and both stop. +func TestCorrelateRecoveryInstancesDuplicateNoMatchStopsAll(t *testing.T) { + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "neither"}} + instanceA := &agentctl.InstanceInfo{ID: "a", SessionID: "session-1"} + instanceB := &agentctl.InstanceInfo{ID: "b", SessionID: "session-1"} + + result := CorrelateRecoveryInstances(records, []*agentctl.InstanceInfo{instanceA, instanceB}) + + if len(result.Winners) != 0 { + t.Fatalf("Winners = %v, want empty", result.Winners) + } + if len(result.ToStop) != 2 { + t.Fatalf("ToStop = %v, want both instances stopped", result.ToStop) + } +} + +// TestCorrelateRecoveryInstancesDuplicateAmbiguousMatchStopsAll pins the +// "more than one match" branch of AC-EXECUTORS-SURVIVAL-002.10: attributing +// the transcript to the wrong instance is worse than a cold resume, so an +// ambiguous match re-tracks nothing. +func TestCorrelateRecoveryInstancesDuplicateAmbiguousMatchStopsAll(t *testing.T) { + records := []*models.ExecutorRunning{{SessionID: "session-1", AgentExecutionID: "dup"}} + instanceA := &agentctl.InstanceInfo{ID: "dup", SessionID: "session-1"} + instanceB := &agentctl.InstanceInfo{ID: "dup", SessionID: "session-1"} + + result := CorrelateRecoveryInstances(records, []*agentctl.InstanceInfo{instanceA, instanceB}) + + if len(result.Winners) != 0 { + t.Fatalf("Winners = %v, want empty (ambiguous match)", result.Winners) + } + if len(result.ToStop) != 2 { + t.Fatalf("ToStop = %v, want both instances stopped", result.ToStop) + } +} + +// TestCorrelateRecoveryInstancesMultipleSessionsAreIndependent pins +// AC-EXECUTORS-SURVIVAL-002.11: session-1's single-instance re-track and +// session-2's ambiguous-duplicate stop-all are decided independently, and +// neither's outcome depends on the other's. +func TestCorrelateRecoveryInstancesMultipleSessionsAreIndependent(t *testing.T) { + records := []*models.ExecutorRunning{ + {SessionID: "session-1", AgentExecutionID: "s1-instance"}, + {SessionID: "session-2", AgentExecutionID: "neither-s2-candidate"}, + } + s1Instance := &agentctl.InstanceInfo{ID: "s1-instance", SessionID: "session-1"} + s2A := &agentctl.InstanceInfo{ID: "s2-a", SessionID: "session-2"} + s2B := &agentctl.InstanceInfo{ID: "s2-b", SessionID: "session-2"} + + result := CorrelateRecoveryInstances(records, []*agentctl.InstanceInfo{s1Instance, s2A, s2B}) + + if got := result.Winners["session-1"]; got != s1Instance { + t.Fatalf("Winners[session-1] = %v, want s1Instance", got) + } + if _, ok := result.Winners["session-2"]; ok { + t.Fatalf("Winners[session-2] should not exist: neither of session-2's two instances matched") + } + if len(result.ToStop) != 2 { + t.Fatalf("ToStop = %v, want both session-2 instances stopped, session-1 untouched", result.ToStop) + } +} + +func TestCorrelateRecoveryInstancesEmptyInputsReturnEmptyResult(t *testing.T) { + result := CorrelateRecoveryInstances(nil, nil) + if len(result.Winners) != 0 || len(result.ToStop) != 0 { + t.Fatalf("CorrelateRecoveryInstances(nil, nil) = %+v, want empty result", result) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/recovery_guard.go b/apps/backend/internal/agent/runtime/lifecycle/recovery_guard.go new file mode 100644 index 0000000000..c9e2a3f61d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/recovery_guard.go @@ -0,0 +1,230 @@ +package lifecycle + +import ( + "context" + "errors" + "sync" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" +) + +// ErrSessionRecoveryGuarded is returned when a launch is requested for a +// session whose recovery guard is currently held (AC-EXECUTORS-SURVIVAL-002.8): +// re-tracking for that session may still be in flight, so a concurrent launch +// is refused rather than queued or blocked. Retryable: the guard is released +// either when that session's recovery outcome is published, or (absent the +// AC-EXECUTORS-SURVIVAL-002.16 exception) when the recovery deadline elapses. +var ErrSessionRecoveryGuarded = errors.New("session is guarded pending recovery") + +// ErrSessionUnstoppableAgent is returned when a launch is requested for a +// session whose recovery guard was retained under +// AC-EXECUTORS-SURVIVAL-002.16: at least one live instance for that session +// could not be stopped, so nothing in this backend's remaining lifetime will +// resolve the condition. Distinct from ErrSessionRecoveryGuarded: this is NOT +// retryable within the backend's lifetime -- only a backend restart clears +// it, since guards do not survive one and the next start re-evaluates the +// instance. +var ErrSessionUnstoppableAgent = errors.New("session has an unstoppable agent from a prior launch") + +// RecoveryGuard is the in-memory, per-session acquire-or-observe map of +// AC-EXECUTORS-SURVIVAL-002.8. It exists for exactly one backend process +// lifetime and needs no durable representation. A guard is normally released +// when that session's recovery outcome is published, or in bulk when the +// recovery deadline (AC-EXECUTORS-SURVIVAL-003.7) elapses -- except a session +// retained as unstoppable (AC-EXECUTORS-SURVIVAL-002.16) or one whose stop is +// still in flight at the deadline, both of which ReleaseAllExceptRetained +// leaves held. +// +// Safe for concurrent use. Does not itself decide which sessions to guard at +// startup (see SessionsToGuard) or when the recovery deadline elapses (that +// clock starts at first control-server contact, AC-EXECUTORS-SURVIVAL-003.7, +// and is owned by the caller driving adoption/re-tracking). +type RecoveryGuard struct { + mu sync.Mutex + state map[string]guardState +} + +type guardState int + +const ( + guardHeld guardState = iota + guardStopInFlight + guardRetainedUnstoppable +) + +// NewRecoveryGuard returns an empty guard map. +func NewRecoveryGuard() *RecoveryGuard { + return &RecoveryGuard{state: make(map[string]guardState)} +} + +// AcquireOrObserve takes a guard for sessionID if it is not already guarded. +// Returns true when this call newly acquired the guard (the caller may begin +// re-tracking it); false when the session was already guarded (the caller +// must not begin re-tracking it -- AC-EXECUTORS-SURVIVAL-002.8's "a session +// already guarded ... is observed, not guarded twice"). +// +// Callers that also need to check the in-memory execution store for an +// already-tracked session must fold that check into the same critical +// section as their own call to this method to preserve the AC's atomicity +// requirement; RecoveryGuard alone cannot enforce that across packages. +func (g *RecoveryGuard) AcquireOrObserve(sessionID string) bool { + g.mu.Lock() + defer g.mu.Unlock() + if _, held := g.state[sessionID]; held { + return false + } + g.state[sessionID] = guardHeld + return true +} + +// Release drops a session's guard, normally called once that session's +// recovery outcome has been published. A no-op for a session that isn't +// guarded, or one retained under RetainAsUnstoppable (only a backend restart +// clears that case, per AC-EXECUTORS-SURVIVAL-002.16). +func (g *RecoveryGuard) Release(sessionID string) { + g.mu.Lock() + defer g.mu.Unlock() + if g.state[sessionID] == guardRetainedUnstoppable { + return + } + delete(g.state, sessionID) +} + +// MarkStopInFlight records that a stop for this session's losing/orphaned +// instance is still retrying when the recovery deadline elapses +// (AC-EXECUTORS-SURVIVAL-003.7): ReleaseAllExceptRetained must leave it held +// until the stop resolves, rather than releasing it at the bound. A no-op +// for a session that isn't guarded. +func (g *RecoveryGuard) MarkStopInFlight(sessionID string) { + g.mu.Lock() + defer g.mu.Unlock() + if _, held := g.state[sessionID]; !held { + return + } + g.state[sessionID] = guardStopInFlight +} + +// IsStopInFlight reports whether sessionID's guard is currently in the +// guardStopInFlight state (see MarkStopInFlight): its losing/orphaned +// instance's stop was still retrying when the recovery deadline elapsed and +// has not yet resolved. Consulted by standalone liveness classification +// (design 02 "Persistence": "the snapshot must post-date recovery's own +// stops") so a row whose stop is still in flight when an adopted-server +// enumeration is taken classifies unknown rather than alive, since the +// enumeration is a snapshot that predates the stop's actual completion. +func (g *RecoveryGuard) IsStopInFlight(sessionID string) bool { + g.mu.Lock() + defer g.mu.Unlock() + return g.state[sessionID] == guardStopInFlight +} + +// RetainAsUnstoppable marks sessionID's guard to survive +// ReleaseAllExceptRetained and outlive recovery entirely +// (AC-EXECUTORS-SURVIVAL-002.16): at least one live instance for it could +// not be stopped, so only a backend restart may clear the guard. Idempotent. +func (g *RecoveryGuard) RetainAsUnstoppable(sessionID string) { + g.mu.Lock() + defer g.mu.Unlock() + g.state[sessionID] = guardRetainedUnstoppable +} + +// ReleaseAllExceptRetained releases every guard currently held, except a +// session marked in-flight via MarkStopInFlight (until its stop resolves and +// the caller explicitly calls Release or RetainAsUnstoppable) or retained via +// RetainAsUnstoppable. Called once the recovery deadline +// (AC-EXECUTORS-SURVIVAL-003.7) elapses, so "no guard outlives recovery" +// holds except for those two documented exceptions. +func (g *RecoveryGuard) ReleaseAllExceptRetained() { + g.mu.Lock() + defer g.mu.Unlock() + for sessionID, state := range g.state { + if state == guardHeld { + delete(g.state, sessionID) + } + } +} + +// CheckLaunchAllowed reports whether a launch may proceed for sessionID, +// returning the distinct refusal errors AC-EXECUTORS-SURVIVAL-002.8 (retryable) +// and AC-EXECUTORS-SURVIVAL-002.16 (not retryable) require, or nil when the +// session carries no guard. +func (g *RecoveryGuard) CheckLaunchAllowed(sessionID string) error { + g.mu.Lock() + defer g.mu.Unlock() + state, held := g.state[sessionID] + if !held { + return nil + } + if state == guardRetainedUnstoppable { + return ErrSessionUnstoppableAgent + } + return ErrSessionRecoveryGuarded +} + +// PassthroughLookup reads whether sessionID is a passthrough session from +// its durable record. ok is false when the read failed (not when the +// session is simply not passthrough) -- AC-EXECUTORS-SURVIVAL-005.3 requires +// treating a failed read as "guard it" rather than "exclude it." +type PassthroughLookup func(ctx context.Context, sessionID string) (isPassthrough bool, ok bool) + +// SessionsToGuard applies AC-EXECUTORS-SURVIVAL-005.3's exclusion to the +// live standalone recovery-inventory session identities read at startup step +// 3 (2.1's ListLiveStandaloneExecutorsRunning), returning the subset +// AC-EXECUTORS-SURVIVAL-002.8 requires a guard taken for: every session +// except a confirmed passthrough one. A session whose passthrough mode +// couldn't be read is guarded, not excluded -- failing to guard a +// re-tracking candidate risks two live agents for one session, while a +// wrongly guarded passthrough session only costs one refused launch until +// the deadline elapses. +func SessionsToGuard(ctx context.Context, liveSessionIDs []string, isPassthrough PassthroughLookup) []string { + guarded := make([]string, 0, len(liveSessionIDs)) + for _, sessionID := range liveSessionIDs { + passthrough, ok := isPassthrough(ctx, sessionID) + if ok && passthrough { + continue + } + guarded = append(guarded, sessionID) + } + return guarded +} + +// TakeStartupRecoveryGuards reads the live standalone recovery-inventory +// records and takes a guard for every session AC-EXECUTORS-SURVIVAL-002.8 +// requires one for, exactly as Manager.Start does, but callable before a +// Manager exists at all. Backend startup composition calls this ahead of +// its own control-server adoption attempt (design 02 "Startup" step 3 must +// run before step 4): by the time Start would otherwise take these same +// guards, that adoption contact has already happened. The returned guard is +// meant to be installed via Manager.SetRecoveryGuard before Start runs. +// +// A nil passthroughLookup, or one that fails a given session's read, is +// treated as "guard it" rather than "exclude it" -- the same fail-closed +// default Start applies. A failed inventory read yields an empty guard +// rather than an error, matching ListLiveStandaloneExecutorsRunning's own +// best-effort contract. +func TakeStartupRecoveryGuards( + ctx context.Context, + runningWriter ExecutorRunningWriter, + passthroughLookup PassthroughLookup, + log *logger.Logger, +) *RecoveryGuard { + guard := NewRecoveryGuard() + lister, ok := runningWriter.(executorRunningLister) + if !ok { + return guard + } + records, err := lister.ListExecutorsRunningLiveStandalone(ctx) + if err != nil { + log.Warn("failed to read live standalone recovery-inventory records", zap.Error(err)) + return guard + } + if passthroughLookup == nil { + passthroughLookup = func(context.Context, string) (bool, bool) { return false, false } + } + for _, sessionID := range SessionsToGuard(ctx, sessionIDsFromExecutorRunning(records), passthroughLookup) { + guard.AcquireOrObserve(sessionID) + } + return guard +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/recovery_guard_test.go b/apps/backend/internal/agent/runtime/lifecycle/recovery_guard_test.go new file mode 100644 index 0000000000..537992e09c --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/recovery_guard_test.go @@ -0,0 +1,220 @@ +package lifecycle + +import ( + "context" + "errors" + "sync" + "testing" +) + +func TestRecoveryGuardAcquireOrObserveIsAtomicPerSession(t *testing.T) { + g := NewRecoveryGuard() + + if acquired := g.AcquireOrObserve("session-1"); !acquired { + t.Fatal("first AcquireOrObserve = false, want true (newly acquired)") + } + if acquired := g.AcquireOrObserve("session-1"); acquired { + t.Fatal("second AcquireOrObserve on the same session = true, want false (observed, not guarded twice)") + } + if acquired := g.AcquireOrObserve("session-2"); !acquired { + t.Fatal("AcquireOrObserve on a different session = false, want true") + } +} + +func TestRecoveryGuardReleaseAllowsReacquisition(t *testing.T) { + g := NewRecoveryGuard() + g.AcquireOrObserve("session-1") + g.Release("session-1") + + if acquired := g.AcquireOrObserve("session-1"); !acquired { + t.Fatal("AcquireOrObserve after Release = false, want true") + } +} + +func TestRecoveryGuardReleaseOfUnguardedSessionIsSafe(t *testing.T) { + g := NewRecoveryGuard() + g.Release("never-guarded") // must not panic +} + +func TestRecoveryGuardCheckLaunchAllowedForUnguardedSession(t *testing.T) { + g := NewRecoveryGuard() + if err := g.CheckLaunchAllowed("session-1"); err != nil { + t.Fatalf("CheckLaunchAllowed for an unguarded session = %v, want nil", err) + } +} + +// TestRecoveryGuardCheckLaunchAllowedRefusesWithRetryableReason pins +// AC-EXECUTORS-SURVIVAL-002.8: a guarded session's launch is refused with +// the distinct retryable recovery reason, not queued or blocked. +func TestRecoveryGuardCheckLaunchAllowedRefusesWithRetryableReason(t *testing.T) { + g := NewRecoveryGuard() + g.AcquireOrObserve("session-1") + + err := g.CheckLaunchAllowed("session-1") + if !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("CheckLaunchAllowed for a guarded session = %v, want ErrSessionRecoveryGuarded", err) + } +} + +// TestRecoveryGuardRetainAsUnstoppableRefusesNonRetryably pins +// AC-EXECUTORS-SURVIVAL-002.16: a session with an unstoppable instance is +// refused with a distinct, non-retryable reason, and stays that way even +// across a Release call (only a backend restart clears it). +func TestRecoveryGuardRetainAsUnstoppableRefusesNonRetryably(t *testing.T) { + g := NewRecoveryGuard() + g.AcquireOrObserve("session-1") + g.RetainAsUnstoppable("session-1") + + err := g.CheckLaunchAllowed("session-1") + if !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("CheckLaunchAllowed for a retained session = %v, want ErrSessionUnstoppableAgent", err) + } + + g.Release("session-1") + err = g.CheckLaunchAllowed("session-1") + if !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("CheckLaunchAllowed after Release on a retained session = %v, want ErrSessionUnstoppableAgent (Release must not clear retention)", err) + } +} + +// TestRecoveryGuardReleaseAllExceptRetainedHonoursBothExceptions pins +// AC-EXECUTORS-SURVIVAL-003.7: the deadline-triggered mass release must +// leave a retained-unstoppable session AND a stop-in-flight session held, +// releasing only an ordinary guarded session. +func TestRecoveryGuardReleaseAllExceptRetainedHonoursBothExceptions(t *testing.T) { + g := NewRecoveryGuard() + g.AcquireOrObserve("ordinary") + g.AcquireOrObserve("retained") + g.RetainAsUnstoppable("retained") + g.AcquireOrObserve("in-flight") + g.MarkStopInFlight("in-flight") + + g.ReleaseAllExceptRetained() + + if err := g.CheckLaunchAllowed("ordinary"); err != nil { + t.Fatalf("ordinary session after deadline release = %v, want nil (released)", err) + } + if err := g.CheckLaunchAllowed("retained"); !errors.Is(err, ErrSessionUnstoppableAgent) { + t.Fatalf("retained session after deadline release = %v, want ErrSessionUnstoppableAgent (must stay held)", err) + } + if err := g.CheckLaunchAllowed("in-flight"); !errors.Is(err, ErrSessionRecoveryGuarded) { + t.Fatalf("in-flight session after deadline release = %v, want ErrSessionRecoveryGuarded (must stay held)", err) + } +} + +// TestRecoveryGuardStopInFlightThenReleaseClearsIt pins that once an +// in-flight stop resolves successfully, the caller's own Release call (not +// ReleaseAllExceptRetained, which already ran) is what actually frees it. +func TestRecoveryGuardStopInFlightThenReleaseClearsIt(t *testing.T) { + g := NewRecoveryGuard() + g.AcquireOrObserve("session-1") + g.MarkStopInFlight("session-1") + g.ReleaseAllExceptRetained() // must not release it + + g.Release("session-1") // the stop resolved; caller releases explicitly + + if err := g.CheckLaunchAllowed("session-1"); err != nil { + t.Fatalf("CheckLaunchAllowed after explicit Release = %v, want nil", err) + } +} + +func TestRecoveryGuardMarkStopInFlightOnUnguardedSessionIsSafe(t *testing.T) { + g := NewRecoveryGuard() + g.MarkStopInFlight("never-guarded") // must not panic or guard it + if err := g.CheckLaunchAllowed("never-guarded"); err != nil { + t.Fatalf("CheckLaunchAllowed for a never-guarded session = %v, want nil", err) + } +} + +func TestRecoveryGuardIsStopInFlight(t *testing.T) { + g := NewRecoveryGuard() + + if g.IsStopInFlight("session-1") { + t.Fatal("expected an unguarded session to not report stop-in-flight") + } + + g.AcquireOrObserve("session-1") + if g.IsStopInFlight("session-1") { + t.Fatal("expected a plainly held guard to not report stop-in-flight") + } + + g.MarkStopInFlight("session-1") + if !g.IsStopInFlight("session-1") { + t.Fatal("expected session-1 to report stop-in-flight after MarkStopInFlight") + } + + g.Release("session-1") + if g.IsStopInFlight("session-1") { + t.Fatal("expected stop-in-flight to clear once the guard is released") + } +} + +func TestRecoveryGuardConcurrentAcquireOrObserveIsRace_Free(t *testing.T) { + g := NewRecoveryGuard() + var wg sync.WaitGroup + var acquiredCount int + var mu sync.Mutex + + for range 50 { + wg.Add(1) + go func() { + defer wg.Done() + if g.AcquireOrObserve("shared-session") { + mu.Lock() + acquiredCount++ + mu.Unlock() + } + }() + } + wg.Wait() + + if acquiredCount != 1 { + t.Fatalf("acquiredCount = %d, want exactly 1 across 50 concurrent callers", acquiredCount) + } +} + +func TestSessionsToGuardExcludesConfirmedPassthrough(t *testing.T) { + lookup := func(_ context.Context, sessionID string) (bool, bool) { + return sessionID == "passthrough-session", true + } + + got := SessionsToGuard(context.Background(), []string{"normal-session", "passthrough-session"}, lookup) + + if len(got) != 1 || got[0] != "normal-session" { + t.Fatalf("SessionsToGuard = %v, want [normal-session]", got) + } +} + +// TestSessionsToGuardIncludesOnLookupFailure pins AC-EXECUTORS-SURVIVAL-005.3: +// when passthrough mode can't be read for a record, the session is guarded +// rather than excluded. +func TestSessionsToGuardIncludesOnLookupFailure(t *testing.T) { + lookup := func(_ context.Context, _ string) (bool, bool) { + return false, false // read failed + } + + got := SessionsToGuard(context.Background(), []string{"unreadable-session"}, lookup) + + if len(got) != 1 || got[0] != "unreadable-session" { + t.Fatalf("SessionsToGuard on lookup failure = %v, want [unreadable-session] (guarded, not excluded)", got) + } +} + +func TestSessionsToGuardIncludesOrdinarySessions(t *testing.T) { + lookup := func(_ context.Context, _ string) (bool, bool) { + return false, true + } + + got := SessionsToGuard(context.Background(), []string{"session-1", "session-2"}, lookup) + + if len(got) != 2 { + t.Fatalf("SessionsToGuard = %v, want both sessions included", got) + } +} + +func TestSessionsToGuardEmptyInputReturnsEmpty(t *testing.T) { + got := SessionsToGuard(context.Background(), nil, func(context.Context, string) (bool, bool) { return false, true }) + if len(got) != 0 { + t.Fatalf("SessionsToGuard(nil) = %v, want empty", got) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/types.go b/apps/backend/internal/agent/runtime/lifecycle/types.go index 8afa905abb..8d32d76b7a 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/types.go +++ b/apps/backend/internal/agent/runtime/lifecycle/types.go @@ -99,6 +99,37 @@ type AgentExecution struct { promptTurnID string promptLifecycleMu sync.Mutex + // recoveryAppliedControlTurnID is the control-server-assigned turn + // identifier (streams.AgentEvent.ControlTurnID) of a retained turn + // outcome this execution already applied during re-tracking + // (AC-EXECUTORS-SURVIVAL-004.2). A later live-delivered event carrying + // the same identifier is the redelivery design 03 describes -- the + // backend re-attaching finally unblocks agentctl's parked send of the + // exact same event the recovery loop already fetched and applied via + // GetTurnOutcome -- and must be a no-op (AC-EXECUTORS-SURVIVAL-004.4). + // Zero means "nothing applied via recovery," matching ControlTurnID's + // own "not retained" zero value, so the check in handleAgentEvent is a + // single comparison with no separate presence flag needed. atomic + // because it is set once during recovery (before streams reconnect) but + // read from the stream-processing goroutine handleAgentEvent runs on. + recoveryAppliedControlTurnID atomic.Int64 + + // recoveredPromptGenerationPending is true when this execution was + // reconstructed by recovery with a turn believed still in flight (nothing + // retained per AC-EXECUTORS-SURVIVAL-004.2's outcome peek): + // promptGeneration was recreated from zero along with the rest of this + // execution object, so it cannot match the nonzero PromptGeneration the + // live completion for that pre-restart turn will carry. claimPromptCompletion + // adopts the event's own PromptGeneration as this execution's generation + // the first time this flag is observed true, mirroring what + // applyRecoveredTurnOutcome already does explicitly for a turn that + // completed while the backend was detached -- there is nothing to + // supersede yet, since no prompt has ever been dispatched through this + // object. beginExecutionPrompt clears it the moment a prompt IS + // dispatched through this object, so a late, genuinely-stale completion + // from before that dispatch cannot clobber the newly assigned generation. + recoveredPromptGenerationPending atomic.Bool + // PrepareResult carries the environment preparation result back to the caller // so it can be persisted synchronously before UpdateTaskSession clobbers metadata. PrepareResult *EnvPrepareResult `json:"-"` @@ -392,6 +423,31 @@ func (e *AgentExecution) promptGenerationSnapshot() uint64 { return e.promptGeneration } +// markRecoveryTurnOutcomeApplied records turnID as the control-server- +// assigned identifier of a retained turn outcome just applied during +// re-tracking (AC-EXECUTORS-SURVIVAL-004.2). A zero turnID is a no-op: it +// means nothing was retained, and zero is also ControlTurnID's own +// "not retained" value, so recording it would make every ordinary live +// event with an unset ControlTurnID look like a match. +func (e *AgentExecution) markRecoveryTurnOutcomeApplied(turnID int64) { + if turnID == 0 { + return + } + e.recoveryAppliedControlTurnID.Store(turnID) +} + +// isRecoveryDuplicateEvent reports whether event is a live redelivery of a +// turn outcome this execution already applied from a retained read +// (AC-EXECUTORS-SURVIVAL-004.4). An event with no ControlTurnID stamp (the +// overwhelming majority -- only a terminal event that was ever retained +// while detached carries one) never matches. +func (e *AgentExecution) isRecoveryDuplicateEvent(event *agentctl.AgentEvent) bool { + if event == nil || event.ControlTurnID == 0 { + return false + } + return e.recoveryAppliedControlTurnID.Load() == event.ControlTurnID +} + func (e *AgentExecution) armPromptActivity() { e.lastActivityAtMu.Lock() e.lastActivityAt = time.Now() @@ -1342,6 +1398,14 @@ type WorkspaceInfoProvider interface { GetWorkspaceInfoForEnvironment(ctx context.Context, taskEnvironmentID string) (*WorkspaceInfo, error) } +// PassthroughSessionProvider reads a task session's durable passthrough-mode +// snapshot (TaskSession.IsPassthrough) so backend startup composition can +// build the PassthroughLookup used by SetPassthroughLookup +// (AC-EXECUTORS-SURVIVAL-005.3). +type PassthroughSessionProvider interface { + GetTaskSession(ctx context.Context, sessionID string) (*models.TaskSession, error) +} + // RecoveredExecution contains info about an execution recovered from a runtime. type RecoveredExecution struct { ExecutionID string diff --git a/apps/backend/internal/agent/runtime/lifecycle/types_test.go b/apps/backend/internal/agent/runtime/lifecycle/types_test.go new file mode 100644 index 0000000000..5d61fbe9a1 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/types_test.go @@ -0,0 +1,60 @@ +package lifecycle + +import ( + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" +) + +// TestIsRecoveryDuplicateEvent pins AC-EXECUTORS-SURVIVAL-004.4's decision +// rule directly on AgentExecution, independent of the Manager-level +// apply/dedup flow covered elsewhere: only an event whose ControlTurnID +// matches the identifier this execution recorded as applied is a duplicate. +func TestIsRecoveryDuplicateEvent(t *testing.T) { + t.Run("no outcome applied yet", func(t *testing.T) { + var e AgentExecution + if e.isRecoveryDuplicateEvent(&agentctl.AgentEvent{ControlTurnID: 9}) { + t.Fatal("expected no match before markRecoveryTurnOutcomeApplied was ever called") + } + }) + + t.Run("nil event", func(t *testing.T) { + var e AgentExecution + e.markRecoveryTurnOutcomeApplied(9) + if e.isRecoveryDuplicateEvent(nil) { + t.Fatal("expected a nil event to never match") + } + }) + + t.Run("event with no ControlTurnID stamp", func(t *testing.T) { + var e AgentExecution + e.markRecoveryTurnOutcomeApplied(9) + if e.isRecoveryDuplicateEvent(&agentctl.AgentEvent{}) { + t.Fatal("expected an unstamped event (ControlTurnID zero) to never match") + } + }) + + t.Run("mismatched turn ID", func(t *testing.T) { + var e AgentExecution + e.markRecoveryTurnOutcomeApplied(9) + if e.isRecoveryDuplicateEvent(&agentctl.AgentEvent{ControlTurnID: 10}) { + t.Fatal("expected a different turn's event to never match") + } + }) + + t.Run("matching turn ID", func(t *testing.T) { + var e AgentExecution + e.markRecoveryTurnOutcomeApplied(9) + if !e.isRecoveryDuplicateEvent(&agentctl.AgentEvent{ControlTurnID: 9}) { + t.Fatal("expected the same turn's redelivered event to match") + } + }) + + t.Run("zero turnID is a no-op", func(t *testing.T) { + var e AgentExecution + e.markRecoveryTurnOutcomeApplied(0) + if e.isRecoveryDuplicateEvent(&agentctl.AgentEvent{ControlTurnID: 0}) { + t.Fatal("expected marking zero to never make an unstamped event match") + } + }) +} diff --git a/apps/backend/internal/agentctl/AGENTS.md b/apps/backend/internal/agentctl/AGENTS.md index 466888392d..81320e275a 100644 --- a/apps/backend/internal/agentctl/AGENTS.md +++ b/apps/backend/internal/agentctl/AGENTS.md @@ -15,6 +15,28 @@ agentctl exposes these route groups (see `server/api/`): - `/workspace/*` - File operations, search, tree - `/vscode/*` - VS Code integration proxy +The **control server** (`server/api/control_server.go`) is a separate router from +the per-instance ones above, and it owns the agent-survival surface: +- `/identity` - opaque per-launch server identity, advertised capability set, + and this server's own resolved unowned period. Unauthenticated, because it is + what decides whether to authenticate. It therefore carries no filesystem path. +- `/ownership/prove` - challenge/response proving this process holds the + credential the caller's record names. Also unauthenticated, for the same + reason, and safe because a proof is a keyed digest over a caller-chosen + challenge and cannot be inverted to the credential. Derivation is shared with + the backend through `internal/common/ownershipproof`. +- `/api/v1/ownership/claim`, `/rotate`, `/confirm`, `/shutdown`, `/details` - + the authenticated ownership operations (`claim` is the renewal an owning + backend repeats on a cadence derived from the unowned period), plus the + filesystem paths kept off `/identity`. + +Auth on that router is `controlCredentialAuth`, which has three tiers rather +than on/off. `/health`, `/auth/handshake`, `/identity`, and `/ownership/prove` +are exempt entirely. The paths in `adoptionOnlyPaths` (`rotate`, `shutdown`) +additionally accept the **superseded** credential until the rotation that +replaced it is confirmed, so a backend that crashed mid-rotation can still +adopt. Every other authenticated route accepts only the current credential. + ## Multi-repository Git and review payloads Preserve the repository-scoped fields `repository_name`, `base_ref`, and diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_session.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_session.go index c7c3d28714..c7bede7159 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_session.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_session.go @@ -746,17 +746,16 @@ func (a *Adapter) emitSetModelEvent( SessionModels: convertSessionModels(cachedModels), ConfigOptions: outConfig, } - sent := a.sendUpdateLocked(event) - closed := a.closed a.mu.Unlock() a.logger.Info("emitting session_models convergence event after SetModel", zap.String("session_id", sessionID), zap.String("model_id", modelID), ) - if !sent && !closed { - a.logger.Warn("updates channel full, dropping event", zap.String("type", event.Type)) - } + // COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6): sendUpdate blocks rather + // than drops on a full updatesCh and must run without a.mu held -- see + // emitDialectContextWindow's doc comment for the deadlock this avoids. + a.sendUpdate(event) } // currentModelFromConfig returns the CurrentValue of the model-shaped @@ -1142,12 +1141,11 @@ func (a *Adapter) emitAuthoritativeConfigOptions( } delete(a.contextSamples, sessionID) } - sent := a.sendUpdateLocked(event) - closed := a.closed a.mu.Unlock() - if !sent && !closed { - a.logger.Warn("updates channel full, dropping event", zap.String("type", event.Type)) - } + // COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6): sendUpdate blocks rather + // than drops on a full updatesCh and must run without a.mu held -- see + // emitDialectContextWindow's doc comment for the deadlock this avoids. + a.sendUpdate(event) } // emitSetConfigOptionEvent emits a session_models convergence event after a @@ -1213,8 +1211,6 @@ func (a *Adapter) emitSetConfigOptionEvent( SessionModels: convertSessionModels(cachedModels), ConfigOptions: outConfig, } - sent := a.sendUpdateLocked(event) - closed := a.closed a.mu.Unlock() a.logger.Info("emitting session_models convergence event after SetConfigOption", @@ -1222,9 +1218,10 @@ func (a *Adapter) emitSetConfigOptionEvent( zap.String("config_id", configID), zap.String("value", value), ) - if !sent && !closed { - a.logger.Warn("updates channel full, dropping event", zap.String("type", event.Type)) - } + // COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6): sendUpdate blocks rather + // than drops on a full updatesCh and must run without a.mu held -- see + // emitDialectContextWindow's doc comment for the deadlock this avoids. + a.sendUpdate(event) } // isModelConfigID reports whether configID identifies the model-shaped diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_updates.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_updates.go index 4c50cbcfd3..c4e1040b00 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_updates.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_updates.go @@ -290,10 +290,20 @@ func (a *Adapter) observeCodexProviderEvidence(promptGeneration uint64, event *A return capacity && (systemError || wasSystemError) } -// emitDialectContextWindow derives and enqueues an agent-specific context -// sample while holding the same lock that protects the active session/model. -// This keeps a model switch from interleaving between size selection and event -// delivery. +// emitDialectContextWindow derives an agent-specific context sample while +// holding the same lock that protects the active session/model -- this keeps +// a model switch from interleaving between size selection and event +// delivery -- then delivers it after releasing that lock. +// +// This is a COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6): delivery uses the +// existing sendUpdate helper, which blocks rather than drops when updatesCh is +// full and is cancelable via lifetimeCtx/closedCh on adapter shutdown -- it +// was already built for exactly this purpose (see its doc comment) and is +// already used by sibling sites in this package (e.g. SetMode), so no new +// primitive is needed here. sendUpdate must not be called while a.mu is held: +// Close() acquires a.mu as its first act before cancelling lifetimeCtx, so +// parking under the lock (the old sendUpdateLocked behavior this replaces) +// would deadlock Close against this very send. func (a *Adapter) emitDialectContextWindow(sessionID string, meta map[string]any) *AgentEvent { a.mu.Lock() if a.closed || sessionID != a.sessionID { @@ -314,15 +324,9 @@ func (a *Adapter) emitDialectContextWindow(sessionID string, meta map[string]any ContextWindowRemaining: remaining, ContextEfficiency: float64(sample.used) / float64(sample.size) * 100, } - sent := a.sendUpdateLocked(event) - if sent { - a.contextSamples[sessionID] = sample - } + a.contextSamples[sessionID] = sample a.mu.Unlock() - if !sent { - a.logger.Warn("updates channel full, dropping event", zap.String("type", event.Type)) - return nil - } + a.sendUpdate(event) return &event } diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/convergence_event_blocking_test.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/convergence_event_blocking_test.go new file mode 100644 index 0000000000..123aae6a09 --- /dev/null +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/convergence_event_blocking_test.go @@ -0,0 +1,125 @@ +package acp + +import ( + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/types/streams" +) + +// TestConvergenceEventEmittersParkOnFullChannelThenDeliver pins +// AC-EXECUTORS-SURVIVAL-001.5/.6 for the three session_models convergence +// emitters in adapter_session.go (emitSetModelEvent, emitSetConfigOptionEvent, +// emitAuthoritativeConfigOptions): a full updatesCh must park the call rather +// than silently dropping the convergence event, and release once a consumer +// drains a slot. All three used to call sendUpdateLocked (drop-on-full) while +// holding a.mu; they now build the event under a.mu, release the lock, and +// deliver via the existing sendUpdate helper (see emitDialectContextWindow's +// doc comment for why the lock must be released first). +func TestConvergenceEventEmittersParkOnFullChannelThenDeliver(t *testing.T) { + cachedModels := []modelInfo{{ModelId: "gpt-5", Name: "GPT-5"}} + cachedConfig := []streams.ConfigOption{ + {Type: "select", ID: "model", Category: "model", CurrentValue: "gpt-5"}, + {Type: "select", ID: "reasoning_effort", CurrentValue: "low"}, + } + + tests := []struct { + name string + emit func(a *Adapter) + }{ + { + name: "emitSetModelEvent", + emit: func(a *Adapter) { + a.emitSetModelEvent("sess-1", "gpt-5.6", cachedModels, cachedConfig) + }, + }, + { + name: "emitSetConfigOptionEvent", + emit: func(a *Adapter) { + a.emitSetConfigOptionEvent("sess-1", "reasoning_effort", "high", cachedModels, cachedConfig) + }, + }, + { + name: "emitAuthoritativeConfigOptions", + emit: func(a *Adapter) { + a.emitAuthoritativeConfigOptions("sess-1", "reasoning_effort", nil, cachedModels) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a := newTestAdapter() + t.Cleanup(func() { _ = a.Close() }) + a.sessionID = "sess-1" + a.updatesCh = make(chan AgentEvent, 1) + a.updatesCh <- AgentEvent{Type: streams.EventTypeMessageChunk} // fill the buffer + + done := make(chan struct{}) + go func() { + tt.emit(a) + close(done) + }() + + select { + case <-done: + t.Fatal("emitter returned before the channel had room -- it must park, not drop") + case <-time.After(50 * time.Millisecond): + } + + <-a.updatesCh // drain the pre-filled slot, freeing room for the parked send + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("emitter did not return promptly after the channel had room") + } + + select { + case event := <-a.updatesCh: + if event.Type != streams.EventTypeSessionModels { + t.Fatalf("event type = %q, want session_models", event.Type) + } + default: + t.Fatal("the parked convergence event was never delivered") + } + }) + } +} + +// TestConvergenceEventEmittersReleaseOnAdapterClose pins the pause-not-a-hang +// half: a parked convergence emitter must release once the adapter closes, +// without ever delivering the event. +func TestConvergenceEventEmittersReleaseOnAdapterClose(t *testing.T) { + cachedModels := []modelInfo{{ModelId: "gpt-5", Name: "GPT-5"}} + cachedConfig := []streams.ConfigOption{ + {Type: "select", ID: "model", Category: "model", CurrentValue: "gpt-5"}, + } + + a := newTestAdapter() + a.sessionID = "sess-1" + a.updatesCh = make(chan AgentEvent, 1) + a.updatesCh <- AgentEvent{Type: streams.EventTypeMessageChunk} + + done := make(chan struct{}) + go func() { + a.emitSetModelEvent("sess-1", "gpt-5.6", cachedModels, cachedConfig) + close(done) + }() + + select { + case <-done: + t.Fatal("emitter returned before the channel had room or the adapter closed") + case <-time.After(50 * time.Millisecond): + } + + if err := a.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("emitter did not return promptly after Close") + } +} diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_grok_test.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_grok_test.go index b8b946a81c..8d2eeae76f 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_grok_test.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/dialect_grok_test.go @@ -519,7 +519,49 @@ func TestGrokUserMessageEchoIsSuppressedWithoutDroppingContext(t *testing.T) { } } -func TestGrokContextRetriesAfterFullUpdateChannel(t *testing.T) { +// TestGrokContextParksOnFullUpdateChannelThenDelivers pins +// AC-EXECUTORS-SURVIVAL-001.5/.6 for this COVERED site: a full updatesCh +// must park emitDialectContextWindow rather than dropping the sample, and +// deliver it once a consumer drains a slot -- replacing the old drop-and- +// retry-on-next-notification behavior. +func TestGrokContextParksOnFullUpdateChannelThenDelivers(t *testing.T) { + models := []modelInfo{{ + ModelId: "grok-current", + Meta: map[string]any{"totalContextTokens": float64(500_000)}, + }} + a := grokAdapterWithModels(t, models, []streams.ConfigOption{{ + Type: "select", ID: configOptionIDModel, Category: configOptionIDModel, CurrentValue: "grok-current", + }}) + t.Cleanup(func() { _ = a.Close() }) + a.updatesCh = make(chan AgentEvent, 1) + a.updatesCh <- AgentEvent{Type: streams.EventTypeMessageChunk} + meta := map[string]any{"totalTokens": float64(42_000)} + + done := make(chan *AgentEvent, 1) + go func() { done <- a.emitDialectContextWindow("sess-grok", meta) }() + + select { + case <-done: + t.Fatal("emitDialectContextWindow returned before the channel had room -- it must park, not drop") + case <-time.After(50 * time.Millisecond): + } + + <-a.updatesCh // drain the pre-filled slot, freeing room for the parked send + + select { + case event := <-done: + if event == nil || event.Type != streams.EventTypeContextWindow { + t.Fatalf("emitDialectContextWindow() = %#v after room freed up, want a context_window event", event) + } + case <-time.After(time.Second): + t.Fatal("emitDialectContextWindow did not return promptly after the channel had room") + } +} + +// TestGrokContextParkReleasesOnAdapterClose pins the pause-not-a-hang half: +// a parked emitDialectContextWindow call must release once the adapter +// closes, without ever delivering the event. +func TestGrokContextParkReleasesOnAdapterClose(t *testing.T) { models := []modelInfo{{ ModelId: "grok-current", Meta: map[string]any{"totalContextTokens": float64(500_000)}, @@ -531,16 +573,23 @@ func TestGrokContextRetriesAfterFullUpdateChannel(t *testing.T) { a.updatesCh <- AgentEvent{Type: streams.EventTypeMessageChunk} meta := map[string]any{"totalTokens": float64(42_000)} - if event := a.emitDialectContextWindow("sess-grok", meta); event != nil { - t.Fatalf("emitDialectContextWindow() = %#v with full channel, want nil", event) - } - if _, cached := a.contextSamples["sess-grok"]; cached { - t.Fatal("dropped context sample must not be cached") - } - <-a.updatesCh + done := make(chan *AgentEvent, 1) + go func() { done <- a.emitDialectContextWindow("sess-grok", meta) }() - if event := a.emitDialectContextWindow("sess-grok", meta); event == nil { - t.Fatal("same context sample must be retried after channel capacity returns") + select { + case <-done: + t.Fatal("emitDialectContextWindow returned before the channel had room or the adapter closed") + case <-time.After(50 * time.Millisecond): + } + + if err := a.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("emitDialectContextWindow did not return promptly after Close") } } diff --git a/apps/backend/internal/agentctl/server/api/agent.go b/apps/backend/internal/agentctl/server/api/agent.go index c93f3dfe45..ad9e8fcfc3 100644 --- a/apps/backend/internal/agentctl/server/api/agent.go +++ b/apps/backend/internal/agentctl/server/api/agent.go @@ -154,9 +154,33 @@ func (s *Server) handleAgentStreamWS(c *gin.Context) { s.logger.Info("agent stream WebSocket connected") streamID := uuid.NewString() + // This is the agentctl-local "instance is attached" signal + // (AC-EXECUTORS-SURVIVAL-001.5/.6): the permission-request notification + // site reads it to decide whether to auto-cancel on a five-second + // timeout (attached) or park (detached) when its channel is full. + s.procMgr.MarkAttached() + defer s.procMgr.MarkDetached() + ctx, cancel := context.WithCancel(c.Request.Context()) defer cancel() + // AC-EXECUTORS-CONTROL-OWNERSHIP-002.2: terminate this stream if the + // control server's credential rotates while it is open, so a prior + // holder cannot keep consuming an instance's events past the moment its + // credential is superseded. The channel comes from instanceAuth's + // context value, captured atomically with the request's own accept + // check -- not a fresh Invalidated() call here, which would be a second, + // independent lock acquisition racing a concurrent rotation. + if invalidated := credentialInvalidatedFromContext(c); invalidated != nil { + go func() { + select { + case <-invalidated: + cancel() + case <-ctx.Done(): + } + }() + } + // Use a mutex for writing to the WebSocket var writeMu sync.Mutex writeMessage := func(data []byte) error { @@ -658,6 +682,11 @@ func (s *Server) handleWSPrompt(ctx context.Context, msg *ws.Message) *ws.Messag return resp } + // The retained slot belongs to the previous terminal turn. Clear it before + // accepting this prompt so recovery cannot replay an old completion as the + // result of the new turn. + s.procMgr.ClearTurnOutcome(req.PromptGeneration) + // Cancel any pending permissions so the agent isn't blocked waiting for // the user to approve a previous tool call while processing the new prompt. s.procMgr.CancelPendingPermissions() diff --git a/apps/backend/internal/agentctl/server/api/auth.go b/apps/backend/internal/agentctl/server/api/auth.go index eb4982b749..7d59c20828 100644 --- a/apps/backend/internal/agentctl/server/api/auth.go +++ b/apps/backend/internal/agentctl/server/api/auth.go @@ -8,6 +8,34 @@ import ( "github.com/gin-gonic/gin" ) +// Shared auth-rejection messages, reused by bearerTokenAuth, instanceAuth, +// and controlCredentialAuth (credential_rotation.go) so the three auth +// checks -- static token, per-instance dynamic credential, and control-plane +// rotating credential -- report identical wording (goconst: 3+ occurrences). +const ( + errMissingAuthHeader = "missing or invalid Authorization header" + errInvalidAuthToken = "invalid auth token" +) + +// credentialInvalidatedContextKey is the gin context key instanceAuth uses to +// hand a stream handler the invalidation channel for the exact credential +// generation that authenticated the request. A stream handler must read it +// from here rather than calling credentialSource.Invalidated() itself: that +// would be a second, independent lock acquisition, and a rotation landing +// between the two could authenticate the request against the generation it +// just superseded while handing the handler a channel for the generation +// that replaced it -- a channel that never closes for the rotation that +// actually invalidated the accepted credential (AC-EXECUTORS-CONTROL- +// OWNERSHIP-002.2). +const credentialInvalidatedContextKey = "credentialInvalidated" + +// afterInstanceAuthAccepted is invoked, if non-nil, immediately after +// instanceAuth accepts a credentialSource-backed request and captures its +// invalidation channel, before the request is allowed to proceed. It exists +// solely so a test can deterministically land a concurrent rotation in this +// exact window; always nil in production. +var afterInstanceAuthAccepted func() + // bearerTokenAuth returns a gin middleware that validates a Bearer token // on every request except the exempted paths (e.g., /health). // If expectedToken is empty, authentication is disabled (no-op middleware). @@ -29,16 +57,12 @@ func bearerTokenAuth(expectedToken string, exemptPaths ...string) gin.HandlerFun token := extractBearerToken(c.Request) if token == "" { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "missing or invalid Authorization header", - }) + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errMissingAuthHeader}) return } if !tokenEqual(token, expectedToken) { - c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ - "error": "invalid auth token", - }) + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errInvalidAuthToken}) return } @@ -46,6 +70,82 @@ func bearerTokenAuth(expectedToken string, exemptPaths ...string) gin.HandlerFun } } +// instanceAuth returns a gin middleware validating a Bearer token on every +// request except the exempted paths, exactly like bearerTokenAuth, but reads +// its accepted token dynamically on each request rather than capturing one +// at registration time. +// +// When s.credentialSource is set (via SetCredentialSource), it authenticates +// against the control server's current highest-numbered rotation instead of +// staticToken -- the single-credential model of design 01 "Single driver" +// (AC-EXECUTORS-CONTROL-OWNERSHIP-002.6). s.credentialSource is read fresh +// per request rather than baked into a closure at Use()-time because +// SetCredentialSource is only ever called after NewServer returns, once the +// caller has a control server to wire it to. +// +// When credentialSource is nil, behavior is identical to +// bearerTokenAuth(staticToken, exemptPaths...), including "empty token +// disables auth" -- every existing test constructing a Server directly is +// unaffected. +func (s *Server) instanceAuth(staticToken string, exemptPaths ...string) gin.HandlerFunc { + exempt := make(map[string]bool, len(exemptPaths)) + for _, p := range exemptPaths { + exempt[p] = true + } + + return func(c *gin.Context) { + if exempt[c.Request.URL.Path] { + c.Next() + return + } + + if s.credentialSource == nil && staticToken == "" { + c.Next() + return + } + + token := extractBearerToken(c.Request) + if token == "" { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errMissingAuthHeader}) + return + } + + if s.credentialSource != nil { + accepted, invalidated := s.credentialSource.AcceptsFullWithInvalidation(token) + if !accepted { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errInvalidAuthToken}) + return + } + if afterInstanceAuthAccepted != nil { + afterInstanceAuthAccepted() + } + c.Set(credentialInvalidatedContextKey, invalidated) + c.Next() + return + } + + if !tokenEqual(token, staticToken) { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errInvalidAuthToken}) + return + } + c.Next() + } +} + +// credentialInvalidatedFromContext reads the invalidation channel instanceAuth +// captured atomically with its accept check (see credentialInvalidatedContextKey). +// Returns nil when no credentialSource is wired -- the caller's watcher +// goroutine is then correctly skipped, matching legacy no-control-server +// behavior. +func credentialInvalidatedFromContext(c *gin.Context) <-chan struct{} { + v, ok := c.Get(credentialInvalidatedContextKey) + if !ok { + return nil + } + ch, _ := v.(<-chan struct{}) + return ch +} + // extractBearerToken extracts the token from the Authorization header. func extractBearerToken(r *http.Request) string { const prefix = "Bearer " diff --git a/apps/backend/internal/agentctl/server/api/control_instance_list_test.go b/apps/backend/internal/agentctl/server/api/control_instance_list_test.go new file mode 100644 index 0000000000..9ab8670f9f --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/control_instance_list_test.go @@ -0,0 +1,122 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/pkg/agent" +) + +// TestListInstancesReturnsEnvelopeWithSessionAndTaskID drives GET +// /api/v1/instances through the real handler and the real agentctl client +// (agentctl.ControlClient), rather than a hand-written response fixture. +// agentctl.ControlClient.ListInstances has always decoded a +// {"instances": [...]} envelope, but handleListInstances wrote the bare +// []*InstanceInfo array straight from instance.Manager.ListInstances — a +// mismatch a fixture-based client test can never see, since it fabricates +// the "correct" response by hand instead of asking the server for one. It +// also pins that CreateInstance's SessionID/TaskID survive into the listed +// InstanceInfo, needed to correlate a recovered instance back to its owning +// session on a backend restart (discoveries G, J). +func TestListInstancesReturnsEnvelopeWithSessionAndTaskID(t *testing.T) { + log := logger.Default() + mgr := instance.NewManager(&config.Config{ + Ports: config.PortConfig{Base: 0, Max: 0}, + Defaults: config.InstanceDefaults{Protocol: agent.ProtocolACP}, + }, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + mgr.SetServerFactory(func(*config.InstanceConfig, *process.Manager, *logger.Logger) http.Handler { + return http.NotFoundHandler() + }) + + created, err := mgr.CreateInstance(t.Context(), &instance.CreateRequest{ + WorkspacePath: t.TempDir(), + SessionID: "session-123", + TaskID: "task-456", + }) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + t.Cleanup(func() { _ = mgr.StopInstance(t.Context(), created.ID) }) + + cs := NewControlServer(&config.Config{}, mgr, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) + + instances, err := client.ListInstances(t.Context()) + if err != nil { + t.Fatalf("ListInstances: %v (handler response did not decode as the {\"instances\":[...]} envelope the client expects)", err) + } + if len(instances) != 1 { + t.Fatalf("instances = %d, want 1", len(instances)) + } + got := instances[0] + if got.ID != created.ID { + t.Errorf("ID = %q, want %q", got.ID, created.ID) + } + if got.SessionID != "session-123" { + t.Errorf("SessionID = %q, want %q", got.SessionID, "session-123") + } + if got.TaskID != "task-456" { + t.Errorf("TaskID = %q, want %q", got.TaskID, "task-456") + } +} + +// TestListInstancesReturnsLiveWorkspaceSourceRoots drives GET +// /api/v1/instances through the real handler and the real agentctl client to +// pin AC-EXECUTORS-SURVIVAL-002.14's "workspace source roots" reconstruction +// row: the backend must read this back from the adopted instance rather than +// push its own, so the wire contract has to expose whatever the instance is +// currently enforcing, not just what it was created with. +func TestListInstancesReturnsLiveWorkspaceSourceRoots(t *testing.T) { + log := logger.Default() + mgr := instance.NewManager(&config.Config{ + Ports: config.PortConfig{Base: 0, Max: 0}, + Defaults: config.InstanceDefaults{Protocol: agent.ProtocolACP}, + }, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + mgr.SetServerFactory(func(*config.InstanceConfig, *process.Manager, *logger.Logger) http.Handler { + return http.NotFoundHandler() + }) + + workspacePath, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatalf("EvalSymlinks: %v", err) + } + created, err := mgr.CreateInstance(t.Context(), &instance.CreateRequest{ + WorkspacePath: workspacePath, + SessionID: "session-123", + WorkspaceSourceRoots: []string{workspacePath}, + }) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + t.Cleanup(func() { _ = mgr.StopInstance(t.Context(), created.ID) }) + + cs := NewControlServer(&config.Config{}, mgr, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) + + instances, err := client.ListInstances(t.Context()) + if err != nil { + t.Fatalf("ListInstances: %v", err) + } + if len(instances) != 1 { + t.Fatalf("instances = %d, want 1", len(instances)) + } + got := instances[0].WorkspaceSourceRoots + if len(got) != 1 || got[0] != workspacePath { + t.Errorf("WorkspaceSourceRoots = %v, want [%q]", got, workspacePath) + } +} diff --git a/apps/backend/internal/agentctl/server/api/control_server.go b/apps/backend/internal/agentctl/server/api/control_server.go index 7e75c72ed5..0c76d51bd8 100644 --- a/apps/backend/internal/agentctl/server/api/control_server.go +++ b/apps/backend/internal/agentctl/server/api/control_server.go @@ -4,6 +4,7 @@ package api import ( "context" "net/http" + "sync" "time" "github.com/gin-gonic/gin" @@ -12,6 +13,7 @@ import ( commonconfig "github.com/kandev/kandev/internal/common/config" "github.com/kandev/kandev/internal/common/httpmw" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/common/ownershipperiod" "github.com/kandev/kandev/internal/common/subproc" "go.uber.org/zap" ) @@ -19,26 +21,68 @@ import ( // ControlServer provides instance management endpoints on the control port. // It exposes the same API regardless of deployment context (Docker or host). // Each instance runs its own HTTP server with agent-specific endpoints. +// adoptionOnlyPaths lists control-server routes where a superseded +// credential remains valid on the adoption-only terms of +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.7/-002.9, until confirmed. Every other +// authenticated route accepts only the current highest-numbered rotation's +// credential. +var adoptionOnlyPaths = map[string]bool{ + "/api/v1/ownership/rotate": true, + "/api/v1/ownership/shutdown": true, +} + type ControlServer struct { - cfg *config.Config - instMgr *instance.Manager - logger *logger.Logger - router *gin.Engine + cfg *config.Config + instMgr *instance.Manager + logger *logger.Logger + router *gin.Engine + ownership *ownershipState + credentials *credentialState + shutdownRequested chan struct{} + shutdownOnce sync.Once + unownedPeriod time.Duration + reaperStop chan struct{} + reaperStopOnce sync.Once + reaperWG sync.WaitGroup + + // decideUnownedShutdown is the reaper's shutdown decision, defaulted to + // ownership.TryBeginShutdownIfUnownedFor. Tests override it to pin the + // exact call the reaper makes rather than only observing its side + // effects, so reverting to a stale two-call sequence (check then latch, + // instead of the atomic check-and-latch) fails a test even though it + // would still pass every other reaper assertion. + decideUnownedShutdown func(time.Duration) bool } // NewControlServer creates a new ControlServer for instance management. func NewControlServer(cfg *config.Config, instMgr *instance.Manager, log *logger.Logger) *ControlServer { gin.SetMode(gin.ReleaseMode) - cs := &ControlServer{ - cfg: cfg, - instMgr: instMgr, - logger: log.WithFields(zap.String("component", "control-server")), - router: gin.New(), + csLogger := log.WithFields(zap.String("component", "control-server")) + unownedPeriod, adjustments := ownershipperiod.Resolve(cfg.UnownedPeriod, cfg.IdleTimeout) + for _, reason := range adjustments { + csLogger.Warn(reason, + zap.Duration("configured_unowned_period", cfg.UnownedPeriod), + zap.Duration("idle_timeout", cfg.IdleTimeout), + zap.Duration("effective_unowned_period", unownedPeriod)) } + cs := &ControlServer{ + cfg: cfg, + instMgr: instMgr, + logger: csLogger, + router: gin.New(), + ownership: newOwnershipState(), + credentials: newCredentialState(cfg.AuthToken), + shutdownRequested: make(chan struct{}), + unownedPeriod: unownedPeriod, + reaperStop: make(chan struct{}), + } + + cs.decideUnownedShutdown = cs.ownership.TryBeginShutdownIfUnownedFor + cs.router.Use(httpmw.RequestLogger(cs.logger, "agentctl-control")) - cs.router.Use(bearerTokenAuth(cfg.AuthToken, "/health", "/auth/handshake")) + cs.router.Use(controlCredentialAuth(cs.credentials, adoptionOnlyPaths, "/health", "/auth/handshake", "/identity", "/ownership/prove")) cs.setupRoutes() return cs @@ -49,6 +93,30 @@ func (m *ControlServer) Router() http.Handler { return m.router } +// CredentialSource exposes this control server's single rotating credential +// so a per-instance Server can authenticate its own requests and streams +// against it instead of a static per-instance token (design 01 "Single +// driver", AC-EXECUTORS-CONTROL-OWNERSHIP-002.6). Wire it via +// Server.SetCredentialSource before an instance starts accepting requests. +func (m *ControlServer) CredentialSource() InstanceCredentialSource { + return m.credentials +} + +// ShutdownRequested returns a channel that closes exactly once the +// ownership-shutdown operation (AC-EXECUTORS-CONTROL-OWNERSHIP-002.9) has +// been invoked. The run loop (cmd/agentctl/main.go) selects on this +// alongside its OS-signal and parent-death triggers and runs the same +// stop-every-instance-and-exit sequence. +func (m *ControlServer) ShutdownRequested() <-chan struct{} { + return m.shutdownRequested +} + +// requestShutdown signals ShutdownRequested exactly once. Safe to call more +// than once (a repeated ownership-shutdown call is a no-op beyond the first). +func (m *ControlServer) requestShutdown() { + m.shutdownOnce.Do(func() { close(m.shutdownRequested) }) +} + func (m *ControlServer) setupRoutes() { // Health check m.router.GET("/health", m.handleHealth) @@ -56,13 +124,27 @@ func (m *ControlServer) setupRoutes() { // Bootstrap handshake — nonce-authenticated, returns the self-generated auth token m.router.POST("/auth/handshake", m.handleHandshake) + // Identity/capability — unauthenticated, decides adoption compatibility + m.router.GET("/identity", m.handleIdentity) + + // Ownership proof — unauthenticated, establishes to an adopting backend + // that this process holds the credential before that backend sends it + m.router.POST("/ownership/prove", m.handleOwnershipProve) + // Instance management API - same endpoints regardless of mode api := m.router.Group("/api/v1") api.POST("/instances", m.handleCreateInstance) api.GET("/instances", m.handleListInstances) api.GET("/instances/:id", m.handleGetInstance) api.DELETE("/instances/:id", m.handleDeleteInstance) + api.GET("/instances/:id/turn-outcome", m.handleGetTurnOutcome) + api.POST("/instances/:id/turn-outcome/ack", m.handleAckTurnOutcome) api.GET("/debug/subprocess-admission", m.handleSubprocessAdmission) + api.GET("/ownership/details", m.handleOwnershipDetails) + api.POST("/ownership/claim", m.handleOwnershipClaim) + api.POST("/ownership/rotate", m.handleCredentialRotate) + api.POST("/ownership/confirm", m.handleCredentialConfirm) + api.POST("/ownership/shutdown", m.handleOwnershipShutdown) } func (m *ControlServer) handleSubprocessAdmission(c *gin.Context) { @@ -104,6 +186,7 @@ func (m *ControlServer) handleHandshake(c *gin.Context) { return } + m.ownership.Renew() m.logger.Info("bootstrap handshake completed, auth token issued", zap.String("nonce_fingerprint", commonconfig.NonceFingerprint(req.Nonce))) c.JSON(http.StatusOK, gin.H{"token": token}) @@ -141,7 +224,7 @@ func (m *ControlServer) handleCreateInstance(c *gin.Context) { func (m *ControlServer) handleListInstances(c *gin.Context) { instances := m.instMgr.ListInstances() - c.JSON(http.StatusOK, instances) + c.JSON(http.StatusOK, gin.H{"instances": instances}) } func (m *ControlServer) handleGetInstance(c *gin.Context) { diff --git a/apps/backend/internal/agentctl/server/api/credential_fencing.go b/apps/backend/internal/agentctl/server/api/credential_fencing.go new file mode 100644 index 0000000000..bd7f173c88 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/credential_fencing.go @@ -0,0 +1,65 @@ +package api + +import ( + "context" + "net/http" + + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" +) + +// Long-lived connections outlive the request that authenticated them, so the +// credential check they passed at connect time is the only one they ever get. +// The credential issued by the highest-numbered rotation is the only one that +// authenticates a stream, which means a connection whose credential has been +// superseded must be torn down rather than left reading and writing the +// workspace after every ordinary request from the same holder is refused. +// +// Both helpers take the invalidation channel from the gin context, where the +// authenticating accept check placed it. Calling Invalidated() afresh here +// would reopen a window in which a rotation between the accept check and the +// lookup hands back the channel for the generation that replaced the one just +// authenticated -- a channel that never closes for the rotation that actually +// superseded this connection's credential. + +// closeOnCredentialInvalidation closes conn once the credential this request +// authenticated under is superseded. The returned function releases the +// watcher and must be called when the handler returns. +func closeOnCredentialInvalidation(c *gin.Context, conn *websocket.Conn) func() { + invalidated := credentialInvalidatedFromContext(c) + if invalidated == nil { + return func() {} + } + done := make(chan struct{}) + go func() { + select { + case <-invalidated: + _ = conn.Close() + case <-done: + } + }() + return func() { close(done) } +} + +// credentialScopedRequest returns c.Request bound to a context that is +// cancelled once the credential it authenticated under is superseded. +// httputil.ReverseProxy closes the upstream connection when the inbound +// request's context is cancelled, including for a connection it has upgraded, +// so a proxied socket terminates with the credential that opened it. The +// returned function releases the watcher and must be called when the handler +// returns. +func credentialScopedRequest(c *gin.Context) (*http.Request, func()) { + invalidated := credentialInvalidatedFromContext(c) + if invalidated == nil { + return c.Request, func() {} + } + ctx, cancel := context.WithCancel(c.Request.Context()) + go func() { + select { + case <-invalidated: + cancel() + case <-ctx.Done(): + } + }() + return c.Request.WithContext(ctx), cancel +} diff --git a/apps/backend/internal/agentctl/server/api/credential_rotation.go b/apps/backend/internal/agentctl/server/api/credential_rotation.go new file mode 100644 index 0000000000..953c67bbd8 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/credential_rotation.go @@ -0,0 +1,307 @@ +package api + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "net/http" + "sync" + + "github.com/gin-gonic/gin" +) + +// errRotationCredentialRejected is returned when Rotate is presented a +// credential outside the acceptable set of +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.8. +var errRotationCredentialRejected = errors.New("credential not accepted for rotation") + +// errRotationIDNotIssued is returned when Confirm names a rotation +// identifier the control server never allocated. +var errRotationIDNotIssued = errors.New("rotation identifier not issued") + +// InstanceCredentialSource is the narrow surface a per-instance Server +// authenticates its own requests and streams against, so that a single +// rotating credential -- not a static per-instance token -- is "the only +// one that authenticates an instance operation or a stream" +// (AC-EXECUTORS-CONTROL-OWNERSHIP-002.6, design 01 "Single driver"). +// Satisfied by *credentialState; ControlServer.CredentialSource exposes it. +type InstanceCredentialSource interface { + // AcceptsFull reports whether token is the current highest-numbered + // rotation's credential. + AcceptsFull(token string) bool + // Invalidated returns a channel that closes the moment the next NEW + // rotation (not an idempotent replay) succeeds. Call it fresh per + // connection: the returned channel is single-shot, and a connection + // established after a rotation must observe a later one, not the + // already-closed channel from before it connected. + Invalidated() <-chan struct{} + // AcceptsFullWithInvalidation atomically reports whether token is the + // current highest-numbered rotation's credential and, when accepted, + // the invalidation channel for that exact generation. A caller that + // holds a connection open past the check (a stream) must capture the + // channel from this same call rather than calling AcceptsFull followed + // by a separate Invalidated(): a rotation landing between two + // independent lock acquisitions could authenticate against the + // generation it just superseded while handing back the channel for the + // generation that replaced it, which never closes for the rotation that + // actually invalidated the accepted credential + // (AC-EXECUTORS-CONTROL-OWNERSHIP-002.2). + AcceptsFullWithInvalidation(token string) (bool, <-chan struct{}) +} + +// credentialState is the two-phase credential rotation machinery of design +// 01's "Single driver" section. It starts holding a single fully- +// authenticating credential (the bootstrap token) and no superseded +// credential -- the state a freshly spawned server is in before any +// adoption has ever rotated it. +// +// A rotation replaces the fully-authenticating credential and demotes the +// previous one to "adoption-only" (it authenticates only a further rotation +// attempt or the ownership-shutdown operation, AC-002.7/AC-002.9) until the +// adopting backend confirms it has durably stored the replacement, at which +// point the superseded credential is dropped entirely. +type credentialState struct { + mu sync.Mutex + rotationID int64 + latest string + superseded string + unconfirmed bool + // invalidated is closed and replaced with a fresh channel on every NEW + // rotation (AC-EXECUTORS-CONTROL-OWNERSHIP-002.2): every per-instance + // stream handler watching the channel it captured at connect time + // observes the close and terminates, so a prior holder cannot keep + // consuming an instance's events once its credential is superseded. An + // idempotent replay (AC-002.10) allocates no new rotation and therefore + // does not invalidate anything -- nothing established since the last + // real rotation could have used a credential the replay superseded. + invalidated chan struct{} +} + +var _ InstanceCredentialSource = (*credentialState)(nil) + +// newCredentialState seeds the machinery with the control server's initial +// (bootstrap-minted) credential. No rotation has happened yet, so there is +// no superseded credential and nothing to confirm. +func newCredentialState(initial string) *credentialState { + return &credentialState{latest: initial, invalidated: make(chan struct{})} +} + +// Invalidated implements InstanceCredentialSource. +func (c *credentialState) Invalidated() <-chan struct{} { + c.mu.Lock() + defer c.mu.Unlock() + return c.invalidated +} + +// Latest returns the credential that authenticates every operation. +func (c *credentialState) Latest() string { + c.mu.Lock() + defer c.mu.Unlock() + return c.latest +} + +// AcceptsFull reports whether token authenticates a normal instance +// operation or stream (AC-EXECUTORS-CONTROL-OWNERSHIP-002.6): only the +// credential issued by the current highest-numbered rotation qualifies. +func (c *credentialState) AcceptsFull(token string) bool { + c.mu.Lock() + defer c.mu.Unlock() + return tokenEqual(token, c.latest) +} + +// AcceptsFullWithInvalidation implements InstanceCredentialSource. +func (c *credentialState) AcceptsFullWithInvalidation(token string) (bool, <-chan struct{}) { + c.mu.Lock() + defer c.mu.Unlock() + if !tokenEqual(token, c.latest) { + return false, nil + } + return true, c.invalidated +} + +// AcceptsAdoptionOnly reports whether token authenticates a further +// rotation attempt or the ownership-shutdown operation +// (AC-EXECUTORS-CONTROL-OWNERSHIP-002.7, -002.9): the latest credential +// always qualifies, and the superseded one qualifies only while its +// rotation remains unconfirmed. +func (c *credentialState) AcceptsAdoptionOnly(token string) bool { + c.mu.Lock() + defer c.mu.Unlock() + if tokenEqual(token, c.latest) { + return true + } + return c.unconfirmed && c.superseded != "" && tokenEqual(token, c.superseded) +} + +// Rotate replaces the latest credential. presented must be the current +// latest credential (an ordinary rotation) or, while the previous rotation +// remains unconfirmed, the credential it superseded -- which is an +// idempotent retry (AC-EXECUTORS-CONTROL-OWNERSHIP-002.10) that allocates +// no new rotation and returns the same identifier and replacement again. +func (c *credentialState) Rotate(presented string) (rotationID int64, replacement string, err error) { + c.mu.Lock() + defer c.mu.Unlock() + + if c.unconfirmed && c.superseded != "" && tokenEqual(presented, c.superseded) { + return c.rotationID, c.latest, nil + } + if !tokenEqual(presented, c.latest) { + return 0, "", errRotationCredentialRejected + } + + replacement = generateRotationCredential() + c.superseded = c.latest + c.latest = replacement + c.rotationID++ + c.unconfirmed = true + close(c.invalidated) + c.invalidated = make(chan struct{}) + return c.rotationID, c.latest, nil +} + +// Confirm names the rotation the caller has durably stored (both the +// replacement credential and the control-server record's reference to it). +// Naming the current highest-numbered rotation -- for the first time or +// repeated -- drops the superseded credential and is accepted. Naming any +// earlier identifier is accepted and has no effect, so a delayed or +// duplicated confirmation can never revoke a credential issued after it. +// Naming an identifier never issued is rejected. +func (c *credentialState) Confirm(rotationID int64) error { + c.mu.Lock() + defer c.mu.Unlock() + + if rotationID <= 0 || rotationID > c.rotationID { + return errRotationIDNotIssued + } + if rotationID < c.rotationID { + return nil + } + c.unconfirmed = false + c.superseded = "" + return nil +} + +func generateRotationCredential() string { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + panic("crypto/rand failed: " + err.Error()) + } + return hex.EncodeToString(b) +} + +// controlCredentialAuth authenticates control-server requests against the +// rotating credential set in creds, distinguishing adoption-only paths +// (where the superseded credential remains valid until confirmed, per +// AC-002.7/-002.9) from every other path (where only the current +// highest-numbered rotation's credential is valid). Mirrors +// bearerTokenAuth's "no credential configured disables auth" behavior when +// creds started out empty. +func controlCredentialAuth(creds *credentialState, adoptionOnlyPaths map[string]bool, exemptPaths ...string) gin.HandlerFunc { + if creds.Latest() == "" { + return func(c *gin.Context) { c.Next() } + } + + exempt := make(map[string]bool, len(exemptPaths)) + for _, p := range exemptPaths { + exempt[p] = true + } + + return func(c *gin.Context) { + if exempt[c.Request.URL.Path] { + c.Next() + return + } + + token := extractBearerToken(c.Request) + if token == "" { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errMissingAuthHeader}) + return + } + + var ok bool + if adoptionOnlyPaths[c.Request.URL.Path] { + ok = creds.AcceptsAdoptionOnly(token) + } else { + ok = creds.AcceptsFull(token) + } + if !ok { + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{errKey: errInvalidAuthToken}) + return + } + + c.Next() + } +} + +type confirmRotationRequest struct { + RotationID int64 `json:"rotation_id"` +} + +// afterCredentialRotateAccepted is invoked, if non-nil, immediately after +// credentials.Rotate succeeds and before ownership.Renew is called in +// handleCredentialRotate. It exists solely so a test can deterministically +// reproduce the window a concurrent unowned-shutdown reaper could latch +// shutdown in between those two calls; always nil in production. +var afterCredentialRotateAccepted func() + +// handleCredentialRotate is the rotate half of the two-phase rotation. +// Refused once the one-way unowned-shutdown door has fired (AC-003.9): a +// rotation is the first step of an adoption attempt, and a decided +// shutdown must never be reopened. +func (m *ControlServer) handleCredentialRotate(c *gin.Context) { + if m.ownership.IsShuttingDown() { + c.JSON(http.StatusConflict, gin.H{errKey: shuttingDownMessage}) + return + } + + presented := extractBearerToken(c.Request) + rotationID, replacement, err := m.credentials.Rotate(presented) + if err != nil { + c.JSON(http.StatusForbidden, gin.H{errKey: err.Error()}) + return + } + + if afterCredentialRotateAccepted != nil { + afterCredentialRotateAccepted() + } + + // A successful rotation -- fresh or an idempotent replay -- renews + // ownership: it is the first authenticated operation an adopting + // backend issues (design 01, "Unowned shutdown"). A shutdown that + // latches in the gap between the IsShuttingDown check above and this + // call must still be honored: Renew reports false once that happens, + // and this response must not claim success over a server that is + // already tearing down. + if !m.ownership.Renew() { + c.JSON(http.StatusConflict, gin.H{errKey: shuttingDownMessage}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "rotation_id": rotationID, + "credential": replacement, + }) +} + +// handleCredentialConfirm is the confirm half of the two-phase rotation. +// It does not itself renew ownership: only the handshake, the claim +// operation, and rotation do (AC-003.8). +func (m *ControlServer) handleCredentialConfirm(c *gin.Context) { + if m.ownership.IsShuttingDown() { + c.JSON(http.StatusConflict, gin.H{errKey: shuttingDownMessage}) + return + } + + var req confirmRotationRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{errKey: "missing or invalid rotation_id"}) + return + } + + if err := m.credentials.Confirm(req.RotationID); err != nil { + c.JSON(http.StatusNotFound, gin.H{errKey: err.Error()}) + return + } + + c.JSON(http.StatusOK, gin.H{lspStatusKey: "confirmed"}) +} diff --git a/apps/backend/internal/agentctl/server/api/credential_rotation_extra_streams_test.go b/apps/backend/internal/agentctl/server/api/credential_rotation_extra_streams_test.go new file mode 100644 index 0000000000..52f07d3432 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/credential_rotation_extra_streams_test.go @@ -0,0 +1,234 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/gorilla/websocket" + "github.com/kandev/kandev/internal/agentctl/server/process" +) + +// The four handlers covered here are the long-lived upgrades that sit behind +// the same instanceAuth group as the agent and workspace streams but forward +// through their own loops. AC-EXECUTORS-CONTROL-OWNERSHIP-002.6 makes the +// credential issued by the highest-numbered rotation the only one that +// authenticates a stream, so each must be torn down when its credential is +// superseded -- otherwise a superseded backend keeps a live PTY, language +// server, or proxied socket into the workspace after being fenced out of +// every ordinary request. + +// upstreamEchoWebSocket starts a server that upgrades and echoes every frame, +// standing in for whatever listens on a proxied port. +func upstreamEchoWebSocket(t *testing.T) *httptest.Server { + t.Helper() + upgrader := websocket.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + return + } + defer func() { _ = conn.Close() }() + for { + messageType, data, readErr := conn.ReadMessage() + if readErr != nil { + return + } + if writeErr := conn.WriteMessage(messageType, data); writeErr != nil { + return + } + } + })) + t.Cleanup(srv.Close) + return srv +} + +// startShellTerminalWithAuth starts a PTY-backed terminal through the routed +// server carrying a bearer credential, so the terminal exists as a specific +// holder before its stream is dialled. +func startShellTerminalWithAuth(t *testing.T, server *httptest.Server, token, terminalID string) { + t.Helper() + body, err := json.Marshal(shellTerminalStartRequest{TerminalID: terminalID, Cols: 80, Rows: 24}) + if err != nil { + t.Fatalf("marshal start request: %v", err) + } + req, err := http.NewRequest(http.MethodPost, server.URL+"/api/v1/shell/terminal/start", bytes.NewReader(body)) + if err != nil { + t.Fatalf("build start request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("start terminal: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(resp.Body) + t.Fatalf("start terminal: status = %d (body %s)", resp.StatusCode, payload) + } +} + +// dialWithAuth dials an arbitrary path on the agentctl test server carrying a +// bearer credential, so the connection is established as a specific holder. +func dialWithAuth(t *testing.T, server *httptest.Server, path, token string) *websocket.Conn { + t.Helper() + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + conn, _, err := websocket.DefaultDialer.Dial( + "ws"+strings.TrimPrefix(server.URL, "http")+path, header) + if err != nil { + t.Fatalf("dial %s: %v", path, err) + } + return conn +} + +// assertEchoRoundTrip proves a proxied socket is genuinely carrying traffic +// before a rotation, so a post-rotation closure cannot be mistaken for a +// connection that never worked in the first place. +func assertEchoRoundTrip(t *testing.T, conn *websocket.Conn) { + t.Helper() + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if err := conn.WriteMessage(websocket.TextMessage, []byte("ping")); err != nil { + t.Fatalf("pre-rotation write: %v", err) + } + _, data, err := conn.ReadMessage() + if err != nil { + t.Fatalf("pre-rotation read: %v", err) + } + if string(data) != "ping" { + t.Fatalf("pre-rotation echo = %q, want %q", data, "ping") + } +} + +// TestShellTerminalStreamTerminatesWhenCredentialRotates is the highest-value +// case: the terminal stream's read loop writes raw bytes into the PTY, so a +// superseded holder that keeps it open retains arbitrary command execution in +// the workspace long after every other route rejects its credential. +func TestShellTerminalStreamTerminatesWhenCredentialRotates(t *testing.T) { + source := newCredentialState("initial-token") + srv := newTestServer(t) + srv.SetCredentialSource(source) + + httpServer := httptest.NewServer(srv.Router()) + defer httpServer.Close() + + t.Cleanup(func() { + if err := srv.procMgr.ShellManager().StopAll(); err != nil { + t.Errorf("stop all terminals: %v", err) + } + }) + const terminalID = "term-fencing" + startShellTerminalWithAuth(t, httpServer, "initial-token", terminalID) + + conn := dialWithAuth(t, httpServer, "/api/v1/shell/terminal/"+terminalID+"/stream", "initial-token") + defer func() { _ = conn.Close() }() + + // Drive the PTY once so the stream is proven live under the current + // credential before rotating. + const marker = "kandev-fencing-marker" + if err := conn.WriteMessage(websocket.BinaryMessage, []byte("echo "+marker+"\r")); err != nil { + t.Fatalf("pre-rotation terminal input: %v", err) + } + readShellTerminalUntil(t, conn, marker) + + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Fatalf("Rotate: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} + +// TestShellTerminalStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture +// pins the same accept/capture window the agent and workspace streams already +// cover: the invalidation channel must come from instanceAuth's own accept +// check, never a fresh Invalidated() call that could hand back the generation +// which replaced the one just authenticated. +func TestShellTerminalStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture(t *testing.T) { + source := newCredentialState("initial-token") + srv := newTestServer(t) + srv.SetCredentialSource(source) + + httpServer := httptest.NewServer(srv.Router()) + defer httpServer.Close() + + t.Cleanup(func() { + if err := srv.procMgr.ShellManager().StopAll(); err != nil { + t.Errorf("stop all terminals: %v", err) + } + }) + const terminalID = "term-fencing-gap" + startShellTerminalWithAuth(t, httpServer, "initial-token", terminalID) + + afterInstanceAuthAccepted = func() { + afterInstanceAuthAccepted = nil + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Errorf("Rotate in accept/capture gap: %v", err) + } + } + t.Cleanup(func() { afterInstanceAuthAccepted = nil }) + + conn := dialWithAuth(t, httpServer, "/api/v1/shell/terminal/"+terminalID+"/stream", "initial-token") + defer func() { _ = conn.Close() }() + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} + +// TestPortProxyTerminatesWhenCredentialRotates covers the proxied socket: the +// port proxy forwards an upgraded connection to an arbitrary local port, so a +// superseded holder keeping one open still reaches whatever listens there. +func TestPortProxyTerminatesWhenCredentialRotates(t *testing.T) { + upstream := upstreamEchoWebSocket(t) + port := portOf(t, upstream.URL) + + source := newCredentialState("initial-token") + srv := newTestServer(t) + srv.SetCredentialSource(source) + + httpServer := httptest.NewServer(srv.Router()) + defer httpServer.Close() + + conn := dialWithAuth(t, httpServer, "/api/v1/port-proxy/"+strconv.Itoa(port)+"/ws", "initial-token") + defer func() { _ = conn.Close() }() + + assertEchoRoundTrip(t, conn) + + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Fatalf("Rotate: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} + +// TestVscodeProxyTerminatesWhenCredentialRotates mirrors the port proxy for +// the code-server proxy, which forwards upgraded connections on the same +// terms. +func TestVscodeProxyTerminatesWhenCredentialRotates(t *testing.T) { + upstream := upstreamEchoWebSocket(t) + port := portOf(t, upstream.URL) + + source := newCredentialState("initial-token") + srv := newTestServer(t) + srv.SetCredentialSource(source) + srv.procMgr.SetVscodeForTest(process.VscodeStatusRunning, port) + + httpServer := httptest.NewServer(srv.Router()) + defer httpServer.Close() + + conn := dialWithAuth(t, httpServer, "/api/v1/vscode/proxy/ws", "initial-token") + defer func() { _ = conn.Close() }() + + assertEchoRoundTrip(t, conn) + + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Fatalf("Rotate: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} diff --git a/apps/backend/internal/agentctl/server/api/credential_rotation_lsp_stream_unix_test.go b/apps/backend/internal/agentctl/server/api/credential_rotation_lsp_stream_unix_test.go new file mode 100644 index 0000000000..2e6d6a90d9 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/credential_rotation_lsp_stream_unix_test.go @@ -0,0 +1,64 @@ +//go:build !windows + +package api + +import ( + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/gorilla/websocket" +) + +// TestLSPStreamTerminatesWhenCredentialRotates completes the fencing coverage +// for the long-lived upgrades: the language-server bridge pipes client frames +// straight into a subprocess with the workspace as its root, so a superseded +// holder keeping one open still reads and edits the workspace through it. +func TestLSPStreamTerminatesWhenCredentialRotates(t *testing.T) { + binDir := t.TempDir() + serverPath := filepath.Join(binDir, "kotlin-lsp") + if err := os.WriteFile(serverPath, []byte("#!/bin/sh\nexec /bin/cat\n"), 0o755); err != nil { + t.Fatalf("write fake language server: %v", err) + } + t.Setenv("HOME", t.TempDir()) + // Prepend rather than replace: the package's later tests resolve git + // through a lookup cached the first time any test triggers it, so a test + // that empties PATH decides that answer for every test after it. + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + source := newCredentialState("initial-token") + srv := newTestServer(t) + srv.SetCredentialSource(source) + + httpServer := httptest.NewServer(srv.Router()) + defer httpServer.Close() + + conn := dialWithAuth(t, httpServer, "/api/v1/lsp/stream?language=kotlin", "initial-token") + defer func() { _ = conn.Close() }() + + // Read the ready frame and echo one request through the bridge, so the + // post-rotation closure cannot be mistaken for a bridge that never came up. + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, _, err := conn.ReadMessage(); err != nil { + t.Fatalf("read ready frame: %v", err) + } + payload := []byte(`{"jsonrpc":"2.0","id":1,"method":"initialize"}`) + if err := conn.WriteMessage(websocket.TextMessage, payload); err != nil { + t.Fatalf("pre-rotation write: %v", err) + } + _, echoed, err := conn.ReadMessage() + if err != nil { + t.Fatalf("pre-rotation read: %v", err) + } + if string(echoed) != string(payload) { + t.Fatalf("echoed payload = %s, want %s", echoed, payload) + } + + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Fatalf("Rotate: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} diff --git a/apps/backend/internal/agentctl/server/api/credential_rotation_stream_test.go b/apps/backend/internal/agentctl/server/api/credential_rotation_stream_test.go new file mode 100644 index 0000000000..41c16bf253 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/credential_rotation_stream_test.go @@ -0,0 +1,290 @@ +package api + +import ( + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/gorilla/websocket" + "github.com/kandev/kandev/internal/agentctl/types" +) + +// fakeCredentialSource is a minimal InstanceCredentialSource test double: it +// accepts exactly one token and exposes a channel the caller can close +// directly, without going through credentialState's rotation bookkeeping. +type fakeCredentialSource struct { + accepted string + invalidated chan struct{} +} + +func newFakeCredentialSource(accepted string) *fakeCredentialSource { + return &fakeCredentialSource{accepted: accepted, invalidated: make(chan struct{})} +} + +func (f *fakeCredentialSource) AcceptsFull(token string) bool { return token == f.accepted } +func (f *fakeCredentialSource) Invalidated() <-chan struct{} { return f.invalidated } + +func (f *fakeCredentialSource) AcceptsFullWithInvalidation(token string) (bool, <-chan struct{}) { + if token != f.accepted { + return false, nil + } + return true, f.invalidated +} + +// TestInstanceAuthAcceptsCurrentCredentialRejectsSuperseded pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.6: when a credentialSource is wired, +// instanceAuth authenticates every per-instance request dynamically against +// it instead of the static token captured at construction, so a rotation +// takes effect on the very next request with no server restart. +func TestInstanceAuthAcceptsCurrentCredentialRejectsSuperseded(t *testing.T) { + source := newFakeCredentialSource("current-token") + s := &Server{credentialSource: source} + r := gin.New() + r.Use(s.instanceAuth("static-token-ignored-when-source-set")) + r.GET("/api/v1/data", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"data": "secret"}) }) + + for _, tc := range []struct { + name string + token string + want int + }{ + {name: "current credential accepted", token: "current-token", want: http.StatusOK}, + {name: "superseded credential rejected", token: "static-token-ignored-when-source-set", want: http.StatusUnauthorized}, + {name: "unknown token rejected", token: "some-other-token", want: http.StatusUnauthorized}, + {name: "missing token rejected", token: "", want: http.StatusUnauthorized}, + } { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/v1/data", nil) + if tc.token != "" { + req.Header.Set("Authorization", "Bearer "+tc.token) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != tc.want { + t.Fatalf("status = %d, want %d", w.Code, tc.want) + } + }) + } + + // Rotation takes effect immediately on the next request: no caching of + // AcceptsFull's answer anywhere in the middleware. + source.accepted = "rotated-token" + req := httptest.NewRequest(http.MethodGet, "/api/v1/data", nil) + req.Header.Set("Authorization", "Bearer current-token") + w := httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusUnauthorized { + t.Fatalf("pre-rotation credential after rotation: status = %d, want %d", w.Code, http.StatusUnauthorized) + } + req = httptest.NewRequest(http.MethodGet, "/api/v1/data", nil) + req.Header.Set("Authorization", "Bearer rotated-token") + w = httptest.NewRecorder() + r.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("post-rotation credential: status = %d, want %d", w.Code, http.StatusOK) + } +} + +// dialTestWSWithAuth connects a WebSocket client to the test server's +// /api/v1/agent/stream endpoint carrying an Authorization header, so a test +// can dial as a specific credential holder rather than relying on +// instanceAuth being disabled (dialTestWS's no-token path). +func dialTestWSWithAuth(t *testing.T, server *httptest.Server, token string) *websocket.Conn { + t.Helper() + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/agent/stream" + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + conn, _, err := websocket.DefaultDialer.Dial(wsURL, header) + if err != nil { + t.Fatalf("failed to dial WebSocket: %v", err) + } + return conn +} + +// assertConnectionClosedByServer reads from conn and requires the read to +// fail with something other than a deadline timeout. A bare "err == nil +// means still open" check cannot tell a deliberate server-side close apart +// from an untouched connection that simply outlived `within`: both produce +// a non-nil error from ReadMessage, but only the timeout case means the +// fencing mechanism never fired. Confirmed empirically: an identical dial +// with no rotation at all still errors with "i/o timeout" once `within` +// elapses, which a bare nil check would have accepted as proof of closure. +func assertConnectionClosedByServer(t *testing.T, conn *websocket.Conn, within time.Duration) { + t.Helper() + _ = conn.SetReadDeadline(time.Now().Add(within)) + _, _, err := conn.ReadMessage() + if err == nil { + t.Fatal("ReadMessage after credential rotation = nil error, want the server to have closed the connection") + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + t.Fatalf("ReadMessage after credential rotation timed out waiting for the server to close (%v), want the server to close immediately", err) + } +} + +// TestAgentStreamTerminatesWhenCredentialRotates pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.2 end to end: a live /agent/stream +// connection authenticated under the current credential must be terminated +// by the server the moment that credential is superseded, so a prior holder +// cannot keep consuming an instance's events. Unlike the credentialState +// unit tests (which only prove the Invalidated channel closes) and the +// instanceAuth test above (which only proves the *next* request is +// rejected), this dials a real, already-open stream and proves the server +// actively tears it down rather than merely refusing new connections. +func TestAgentStreamTerminatesWhenCredentialRotates(t *testing.T) { + source := newCredentialState("initial-token") + s := newTestServer(t) + s.SetCredentialSource(source) + httpServer := httptest.NewServer(s.router) + defer httpServer.Close() + + conn := dialTestWSWithAuth(t, httpServer, "initial-token") + defer func() { _ = conn.Close() }() + + // Prove the connection is alive under the current credential before + // rotating, so a failure after rotation can't be blamed on a connection + // that never worked. + resp := sendWSRequest(t, conn, "agent.stderr", nil) + if resp.Type != "response" { + t.Fatalf("pre-rotation request: got type %q, want a response", resp.Type) + } + + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Fatalf("Rotate: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} + +// TestAgentStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture pins +// Review round 5 finding 1: instanceAuth's accept check and a stream +// handler's own invalidation-channel capture used to be two independent lock +// acquisitions, so a rotation landing in the gap between them could +// authenticate a request against the generation it just superseded while +// handing the handler a channel for the generation that replaced it -- a +// channel that never closes for the rotation that actually invalidated the +// accepted credential. afterInstanceAuthAccepted deterministically lands a +// rotation in exactly that gap; the fix (atomic AcceptsFullWithInvalidation, +// captured by the middleware and read by the handler from gin context) must +// still close the stream. +func TestAgentStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture(t *testing.T) { + source := newCredentialState("initial-token") + s := newTestServer(t) + s.SetCredentialSource(source) + httpServer := httptest.NewServer(s.router) + defer httpServer.Close() + + afterInstanceAuthAccepted = func() { + afterInstanceAuthAccepted = nil + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Errorf("Rotate in accept/capture gap: %v", err) + } + } + t.Cleanup(func() { afterInstanceAuthAccepted = nil }) + + conn := dialTestWSWithAuth(t, httpServer, "initial-token") + defer func() { _ = conn.Close() }() + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} + +// TestWorkspaceStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture +// mirrors TestAgentStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture +// for the workspace stream's own credentialInvalidatedFromContext read. +func TestWorkspaceStreamTerminatesWhenCredentialRotatesBetweenAcceptAndCapture(t *testing.T) { + source := newCredentialState("initial-token") + s := newTestServer(t) + s.SetCredentialSource(source) + httpServer := httptest.NewServer(s.router) + defer httpServer.Close() + + afterInstanceAuthAccepted = func() { + afterInstanceAuthAccepted = nil + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Errorf("Rotate in accept/capture gap: %v", err) + } + } + t.Cleanup(func() { afterInstanceAuthAccepted = nil }) + + conn := dialTestWorkspaceStreamWithAuth(t, httpServer, "initial-token") + defer func() { _ = conn.Close() }() + + // The handler unconditionally sends a "connected" message as its first + // frame before the forwarding loop (and its invalidation check) ever + // runs; consume it before asserting closure, exactly like + // TestWorkspaceStreamTerminatesWhenCredentialRotates does. + var connected types.WorkspaceStreamMessage + if err := conn.ReadJSON(&connected); err != nil { + t.Fatalf("reading connected message: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} + +// dialTestWorkspaceStreamWithAuth connects a WebSocket client to the test +// server's /api/v1/workspace/stream endpoint carrying an Authorization +// header, mirroring dialTestWSWithAuth for the agent stream. +func dialTestWorkspaceStreamWithAuth(t *testing.T, server *httptest.Server, token string) *websocket.Conn { + t.Helper() + wsURL := "ws" + strings.TrimPrefix(server.URL, "http") + "/api/v1/workspace/stream" + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + conn, _, err := websocket.DefaultDialer.Dial(wsURL, header) + if err != nil { + t.Fatalf("failed to dial WebSocket: %v", err) + } + return conn +} + +// TestWorkspaceStreamTerminatesWhenCredentialRotates mirrors +// TestAgentStreamTerminatesWhenCredentialRotates for the workspace stream's +// own select-on-Invalidated arm (workspace.go forwardWorkspaceStream, +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.2). Both streams sit behind the same +// instanceAuth middleware group (server.go) but are forwarded by two +// separate goroutines with two separate select statements, so proving one +// terminates on rotation says nothing about the other. +func TestWorkspaceStreamTerminatesWhenCredentialRotates(t *testing.T) { + source := newCredentialState("initial-token") + s := newTestServer(t) + s.SetCredentialSource(source) + httpServer := httptest.NewServer(s.router) + defer httpServer.Close() + + conn := dialTestWorkspaceStreamWithAuth(t, httpServer, "initial-token") + defer func() { _ = conn.Close() }() + + // The handler sends a "connected" message immediately on upgrade, then + // answers a client ping with a pong -- together they prove the + // connection and its forwarding goroutine are live under the current + // credential before rotating, so a failure after rotation can't be + // blamed on a connection that never worked. + var connected types.WorkspaceStreamMessage + if err := conn.ReadJSON(&connected); err != nil { + t.Fatalf("reading connected message: %v", err) + } + if connected.Type != types.WorkspaceMessageTypeConnected { + t.Fatalf("first message type = %q, want %q", connected.Type, types.WorkspaceMessageTypeConnected) + } + + if err := conn.WriteJSON(types.NewWorkspacePing()); err != nil { + t.Fatalf("writing ping: %v", err) + } + var pong types.WorkspaceStreamMessage + if err := conn.ReadJSON(&pong); err != nil { + t.Fatalf("pre-rotation ping/pong: %v", err) + } + if pong.Type != types.WorkspaceMessageTypePong { + t.Fatalf("pre-rotation response type = %q, want %q", pong.Type, types.WorkspaceMessageTypePong) + } + + if _, _, err := source.Rotate("initial-token"); err != nil { + t.Fatalf("Rotate: %v", err) + } + + assertConnectionClosedByServer(t, conn, 2*time.Second) +} diff --git a/apps/backend/internal/agentctl/server/api/credential_rotation_test.go b/apps/backend/internal/agentctl/server/api/credential_rotation_test.go new file mode 100644 index 0000000000..5833d38e4e --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/credential_rotation_test.go @@ -0,0 +1,467 @@ +package api + +import ( + "net/http/httptest" + "testing" + "time" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/common/logger" +) + +// --- credentialState unit tests --- + +// TestCredentialStateRotatePresentingLatestAllocatesNewRotation pins the +// ordinary rotation path: presenting the current fully-authenticating +// credential allocates rotation 1 and a fresh replacement. +func TestCredentialStateRotatePresentingLatestAllocatesNewRotation(t *testing.T) { + c := newCredentialState("initial-token") + + id, replacement, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("Rotate: %v", err) + } + if id != 1 { + t.Fatalf("rotation id = %d, want 1", id) + } + if replacement == "" || replacement == "initial-token" { + t.Fatalf("replacement = %q, want a fresh non-empty credential", replacement) + } + if got := c.Latest(); got != replacement { + t.Fatalf("Latest() = %q, want the replacement %q", got, replacement) + } +} + +// TestCredentialStateRotateRejectsUnknownCredential pins that a credential +// outside the acceptable set (AC-002.8) is refused. +func TestCredentialStateRotateRejectsUnknownCredential(t *testing.T) { + c := newCredentialState("initial-token") + + if _, _, err := c.Rotate("some-other-token"); err == nil { + t.Fatal("Rotate with unknown credential = nil error, want a rejection") + } +} + +// TestCredentialStateRotateIsIdempotentUnderRetryWithSupersededCredential +// pins AC-EXECUTORS-CONTROL-OWNERSHIP-002.10: presenting the credential the +// highest-numbered rotation superseded, while unconfirmed, returns the same +// rotation again rather than allocating a new one. +func TestCredentialStateRotateIsIdempotentUnderRetryWithSupersededCredential(t *testing.T) { + c := newCredentialState("initial-token") + + id1, replacement1, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("first Rotate: %v", err) + } + + id2, replacement2, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("retry Rotate: %v", err) + } + + if id2 != id1 { + t.Fatalf("retry rotation id = %d, want the same id %d (no new allocation)", id2, id1) + } + if replacement2 != replacement1 { + t.Fatalf("retry replacement = %q, want the same replacement %q", replacement2, replacement1) + } +} + +// TestCredentialStateRotateConcurrentCallersConvergeOnOneRotation pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.10 under genuine concurrency, not just +// sequential retry: two callers present the same current credential to +// Rotate at effectively the same time. The mutex serializes them, so exactly +// one becomes the real rotation and the other -- now presenting what has +// become the superseded credential -- takes the idempotent-retry branch and +// converges on the identical (rotationID, replacement) pair rather than +// erroring, allocating a second rotation, or corrupting the shared state. +// Neither caller is left believing it owns a distinct credential. +// +// This is a property of the state machine, not a claim that two backends can +// reach it: AC-EXECUTORS-CONTROL-OWNERSHIP-002.4 excludes a second backend at +// startup through the runtime-state ownership lock. The two racers here stand +// in for one backend retrying across a crash, and for the ownership lock +// being wrong. +func TestCredentialStateRotateConcurrentCallersConvergeOnOneRotation(t *testing.T) { + c := newCredentialState("initial-token") + + type result struct { + id int64 + replacement string + err error + } + results := make(chan result, 2) + start := make(chan struct{}) + for i := 0; i < 2; i++ { + go func() { + <-start + id, replacement, err := c.Rotate("initial-token") + results <- result{id: id, replacement: replacement, err: err} + }() + } + close(start) + + first := <-results + second := <-results + + if first.err != nil || second.err != nil { + t.Fatalf("concurrent Rotate errors = %v, %v, want both nil", first.err, second.err) + } + if first.id != 1 || second.id != 1 { + t.Fatalf("rotation ids = %d, %d, want both 1 (no second rotation allocated)", first.id, second.id) + } + if first.replacement == "" || first.replacement != second.replacement { + t.Fatalf("replacements = %q, %q, want both equal and non-empty", first.replacement, second.replacement) + } + if got := c.Latest(); got != first.replacement { + t.Fatalf("Latest() = %q, want the converged replacement %q", got, first.replacement) + } +} + +// TestCredentialStateAcceptsAdoptionOnlyForSupersededUntilConfirmed pins +// AC-002.7: the superseded credential authenticates adoption-only +// operations until confirmed, and the replacement authenticates everything +// throughout. +func TestCredentialStateAcceptsAdoptionOnlyForSupersededUntilConfirmed(t *testing.T) { + c := newCredentialState("initial-token") + _, replacement, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("Rotate: %v", err) + } + + if !c.AcceptsAdoptionOnly("initial-token") { + t.Fatal("AcceptsAdoptionOnly(superseded) = false before confirm, want true") + } + if !c.AcceptsAdoptionOnly(replacement) { + t.Fatal("AcceptsAdoptionOnly(latest) = false, want true (latest always qualifies)") + } + if c.AcceptsFull("initial-token") { + t.Fatal("AcceptsFull(superseded) = true, want false (superseded never authenticates a full operation)") + } + if !c.AcceptsFull(replacement) { + t.Fatal("AcceptsFull(latest) = false, want true") + } +} + +// TestCredentialStateConfirmDropsSupersededCredential pins that confirming +// the highest-numbered rotation removes the superseded credential from the +// acceptable set entirely. +func TestCredentialStateConfirmDropsSupersededCredential(t *testing.T) { + c := newCredentialState("initial-token") + id, _, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("Rotate: %v", err) + } + + if err := c.Confirm(id); err != nil { + t.Fatalf("Confirm: %v", err) + } + + if c.AcceptsAdoptionOnly("initial-token") { + t.Fatal("AcceptsAdoptionOnly(superseded) = true after confirm, want false") + } + if _, _, err := c.Rotate("initial-token"); err == nil { + t.Fatal("Rotate(superseded) after confirm = nil error, want a rejection") + } +} + +// TestCredentialStateConfirmOfEarlierRotationIsAcceptedAndHasNoEffect pins +// that a confirmation naming an earlier rotation identifier is accepted but +// does not revoke the credential issued after it (AC-002.6): a delayed or +// duplicated confirmation can never win against a later rotation. +func TestCredentialStateConfirmOfEarlierRotationIsAcceptedAndHasNoEffect(t *testing.T) { + c := newCredentialState("initial-token") + firstID, firstReplacement, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("first Rotate: %v", err) + } + if _, _, err := c.Rotate(firstReplacement); err != nil { + t.Fatalf("second Rotate: %v", err) + } + + if err := c.Confirm(firstID); err != nil { + t.Fatalf("Confirm(earlier id) = %v, want accepted with no effect", err) + } + + // The second rotation's superseded credential (firstReplacement) must + // still be adoption-only-acceptable: confirming the stale first + // identifier must not have dropped it. + if !c.AcceptsAdoptionOnly(firstReplacement) { + t.Fatal("AcceptsAdoptionOnly(second rotation's superseded credential) = false, want true (stale confirm had no effect)") + } +} + +// TestCredentialStateConfirmOfUnissuedRotationIsRejected pins that naming +// an identifier the control server never allocated is rejected. +func TestCredentialStateConfirmOfUnissuedRotationIsRejected(t *testing.T) { + c := newCredentialState("initial-token") + + if err := c.Confirm(1); err == nil { + t.Fatal("Confirm(never-issued id) = nil error, want a rejection") + } +} + +// TestCredentialStateAcceptableSetCappedAtTwo pins AC-002.8: rotating a +// second time invalidates the credential the *first* rotation superseded, +// even though its rotation was never confirmed. +func TestCredentialStateAcceptableSetCappedAtTwo(t *testing.T) { + c := newCredentialState("initial-token") + _, firstReplacement, err := c.Rotate("initial-token") + if err != nil { + t.Fatalf("first Rotate: %v", err) + } + if _, _, err := c.Rotate(firstReplacement); err != nil { + t.Fatalf("second Rotate: %v", err) + } + + if c.AcceptsAdoptionOnly("initial-token") { + t.Fatal("AcceptsAdoptionOnly(original credential) = true after a second rotation, want false (acceptable set capped at two)") + } +} + +// --- HTTP handler tests --- + +func newRotationTestServer(t *testing.T) (*ControlServer, *httptest.Server, string, int) { + t.Helper() + cfg := &config.Config{AuthToken: "initial-token"} + log := logger.Default() + cs := NewControlServer(cfg, &instance.Manager{}, log) + server := httptest.NewServer(cs.Router()) + t.Cleanup(server.Close) + host, port := parseHostPort(t, server.URL) + return cs, server, host, port +} + +// TestHandleCredentialRotateReturnsNewCredentialAndRenewsOwnership pins the +// end-to-end rotate flow through the real handler and client. +func TestHandleCredentialRotateReturnsNewCredentialAndRenewsOwnership(t *testing.T) { + cs, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + backdateOwnership(cs, time.Hour) + + result, err := client.RotateCredential(t.Context()) + if err != nil { + t.Fatalf("RotateCredential: %v", err) + } + if result.RotationID != 1 { + t.Fatalf("RotationID = %d, want 1", result.RotationID) + } + if result.Credential == "" || result.Credential == "initial-token" { + t.Fatalf("Credential = %q, want a fresh non-empty replacement", result.Credential) + } + if elapsed := cs.ownership.UnownedFor(); elapsed >= time.Minute { + t.Fatalf("UnownedFor after rotate = %v, want well under the backdated hour (rotation should have renewed)", elapsed) + } +} + +// TestSupersededCredentialCannotListInstances pins that the superseded +// credential authenticates only the adoption-only paths, not an ordinary +// instance operation. +func TestSupersededCredentialCannotListInstances(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + rotatingClient := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + if _, err := rotatingClient.RotateCredential(t.Context()); err != nil { + t.Fatalf("RotateCredential: %v", err) + } + + supersededClient := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + if _, err := supersededClient.ListInstances(t.Context()); err == nil { + t.Fatal("ListInstances with superseded credential = nil error, want a rejection") + } +} + +// TestSupersededCredentialCanRetryRotateUntilConfirmed pins the +// idempotent-replay path reachable over HTTP: retrying rotate with the +// superseded credential returns the same rotation again. +func TestSupersededCredentialCanRetryRotateUntilConfirmed(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + first, err := client.RotateCredential(t.Context()) + if err != nil { + t.Fatalf("first RotateCredential: %v", err) + } + + retryClient := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + retry, err := retryClient.RotateCredential(t.Context()) + if err != nil { + t.Fatalf("retry RotateCredential: %v", err) + } + + if retry.RotationID != first.RotationID || retry.Credential != first.Credential { + t.Fatalf("retry = %+v, want the same rotation as first %+v", retry, first) + } +} + +// TestHandleCredentialConfirmDropsSupersededCredential pins that after +// confirmation the superseded credential is refused even for rotation +// retries. +func TestHandleCredentialConfirmDropsSupersededCredential(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + result, err := client.RotateCredential(t.Context()) + if err != nil { + t.Fatalf("RotateCredential: %v", err) + } + + client.SetAuthToken(result.Credential) + if err := client.ConfirmCredentialRotation(t.Context(), result.RotationID); err != nil { + t.Fatalf("ConfirmCredentialRotation: %v", err) + } + + supersededClient := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + if _, err := supersededClient.RotateCredential(t.Context()); err == nil { + t.Fatal("RotateCredential with confirmed-away superseded credential = nil error, want a rejection") + } +} + +// TestHandleCredentialConfirmRejectsUnissuedRotationID pins the HTTP-layer +// rejection of an identifier the control server never allocated. +func TestHandleCredentialConfirmRejectsUnissuedRotationID(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if err := client.ConfirmCredentialRotation(t.Context(), 999); err == nil { + t.Fatal("ConfirmCredentialRotation(never-issued id) = nil error, want a rejection") + } +} + +// TestHandleCredentialRotateRefusedAfterShutdownBegun pins the one-way-door +// refusal: rotate is the first step of an adoption attempt and must be +// refused once the unowned-shutdown latch has fired. +func TestHandleCredentialRotateRefusedAfterShutdownBegun(t *testing.T) { + cs, _, host, port := newRotationTestServer(t) + cs.ownership.BeginShutdown() + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if _, err := client.RotateCredential(t.Context()); err == nil { + t.Fatal("RotateCredential after shutdown began = nil error, want a rejection") + } +} + +// TestHandleCredentialRotateRefusesWhenShutdownLatchesBetweenAcceptAndRenew +// pins Review round 2 finding 5: a concurrent unowned-shutdown reaper can +// latch shutdown in the gap between credentials.Rotate succeeding and +// ownership.Renew being called -- Renew reports false once that happens, and +// the handler must not claim success over a server that is already tearing +// down. Deterministically reproduces that exact window via a test-only hook +// rather than relying on real goroutine timing. +func TestHandleCredentialRotateRefusesWhenShutdownLatchesBetweenAcceptAndRenew(t *testing.T) { + cs, _, host, port := newRotationTestServer(t) + t.Cleanup(func() { afterCredentialRotateAccepted = nil }) + afterCredentialRotateAccepted = func() { cs.ownership.BeginShutdown() } + + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if _, err := client.RotateCredential(t.Context()); err == nil { + t.Fatal("RotateCredential racing a shutdown latch = nil error, want a rejection") + } +} + +// --- ownership-shutdown operation (AC-EXECUTORS-CONTROL-OWNERSHIP-002.9) --- + +// TestHandleOwnershipShutdownAcceptsTheLatestCredentialWithNoPriorRotation +// pins that shutdown requires no prior adoption or rotation: a backend +// holding only the original bootstrap credential can still invoke it. +func TestHandleOwnershipShutdownAcceptsTheLatestCredentialWithNoPriorRotation(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if err := client.ShutdownControlServer(t.Context()); err != nil { + t.Fatalf("ShutdownControlServer: %v", err) + } +} + +// TestHandleOwnershipShutdownAcceptsSupersededUnconfirmedCredential pins +// AC-002.9's carve-out: the credential a rotation superseded still +// authenticates shutdown, on its adoption-only terms, until confirmed. +func TestHandleOwnershipShutdownAcceptsSupersededUnconfirmedCredential(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + rotatingClient := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + if _, err := rotatingClient.RotateCredential(t.Context()); err != nil { + t.Fatalf("RotateCredential: %v", err) + } + + supersededClient := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + if err := supersededClient.ShutdownControlServer(t.Context()); err != nil { + t.Fatalf("ShutdownControlServer with superseded credential: %v", err) + } +} + +// TestHandleOwnershipShutdownRejectsUnknownCredential pins that shutdown is +// still authenticated: a token outside the acceptable set is refused. +func TestHandleOwnershipShutdownRejectsUnknownCredential(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("some-other-token")) + + if err := client.ShutdownControlServer(t.Context()); err == nil { + t.Fatal("ShutdownControlServer with unknown credential = nil error, want a rejection") + } +} + +// TestHandleOwnershipShutdownSignalsShutdownRequested pins that a +// successful shutdown call closes the channel the run loop selects on. +func TestHandleOwnershipShutdownSignalsShutdownRequested(t *testing.T) { + cs, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if err := client.ShutdownControlServer(t.Context()); err != nil { + t.Fatalf("ShutdownControlServer: %v", err) + } + + select { + case <-cs.ShutdownRequested(): + case <-time.After(time.Second): + t.Fatal("ShutdownRequested() channel not closed within 1s of a successful shutdown call") + } +} + +// TestHandleOwnershipShutdownLatchesTheOneWayDoor pins that shutdown fires +// the same one-way ownership latch as the unowned-period reaper: once +// invoked, a subsequent claim is refused with the shutting-down outcome. +func TestHandleOwnershipShutdownLatchesTheOneWayDoor(t *testing.T) { + cs, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if err := client.ShutdownControlServer(t.Context()); err != nil { + t.Fatalf("ShutdownControlServer: %v", err) + } + if !cs.ownership.IsShuttingDown() { + t.Fatal("IsShuttingDown() = false after ownership-shutdown call, want true") + } + if err := client.ClaimOwnership(t.Context()); err == nil { + t.Fatal("ClaimOwnership after ownership-shutdown = nil error, want a rejection") + } +} + +// TestHandleOwnershipShutdownIsIdempotent pins that a second shutdown call +// after the first still succeeds without panicking (the channel close is +// guarded by sync.Once) and without re-latching anything. +func TestHandleOwnershipShutdownIsIdempotent(t *testing.T) { + _, _, host, port := newRotationTestServer(t) + log := logger.Default() + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("initial-token")) + + if err := client.ShutdownControlServer(t.Context()); err != nil { + t.Fatalf("first ShutdownControlServer: %v", err) + } + if err := client.ShutdownControlServer(t.Context()); err != nil { + t.Fatalf("second ShutdownControlServer: %v", err) + } +} diff --git a/apps/backend/internal/agentctl/server/api/identity.go b/apps/backend/internal/agentctl/server/api/identity.go new file mode 100644 index 0000000000..e6954add24 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/identity.go @@ -0,0 +1,38 @@ +package api + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +// SurvivalCapabilities lists every named capability this build of agentctl +// advertises on GET /identity. Adoption gates on the backend's required set +// being a SUBSET of this list (design 01, "Capability compatibility") -- a +// set comparison rather than a version match, so a future agentctl build can +// add capabilities without breaking older backends, and an older agentctl +// can be correctly refused by a newer backend that requires one it lacks. +var SurvivalCapabilities = []string{"agent-survival.v1"} + +// handleIdentity reports this launch's opaque identity and capability +// scope. It is deliberately exempt from bearer-token auth (see +// NewControlServer): identity retrieval decides adoption compatibility, so +// it cannot itself be gated on the answer. Because it sits below +// authentication it carries nothing beyond what that decision needs, and in +// particular no filesystem path; adoption reads those from +// /api/v1/ownership/details once authenticated. +func (m *ControlServer) handleIdentity(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "server_identity": m.cfg.ServerIdentity, + "capabilities": SurvivalCapabilities, + // unowned_period_ms is this server's own resolved unowned period + // (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2/.7), not the caller's config: + // an adopting backend's local config can disagree with what this + // process actually enforces (e.g. an operator raised + // agentctl.unownedPeriod between restarts), and that mismatch drives + // the adopting backend to renew on a cadence too slow for this + // server's own reaper. This is the only channel that value crosses + // back to an adopting backend. + "unowned_period_ms": m.unownedPeriod.Milliseconds(), + }) +} diff --git a/apps/backend/internal/agentctl/server/api/identity_test.go b/apps/backend/internal/agentctl/server/api/identity_test.go new file mode 100644 index 0000000000..befff6e985 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/identity_test.go @@ -0,0 +1,149 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/common/logger" +) + +// TestHandleIdentityIsReachableWithoutAuthAndReportsIdentityAndCapabilities +// pins design 01's "Identity retrieval ... is what decides compatibility, so +// it cannot itself be gated on the answer": /identity must answer even when +// AuthToken is configured and the caller presents no bearer token at all, +// and must report this launch's opaque server identity and the advertised +// capability set. +func TestHandleIdentityIsReachableWithoutAuthAndReportsIdentityAndCapabilities(t *testing.T) { + log := logger.Default() + cfg := &config.Config{ + AuthToken: "some-configured-token", + HomeDir: "/home/kandev-test/.kandev", + ServerIdentity: "server-identity-abc", + DiagnosticLogPath: "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", + } + mgr := instance.NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + + cs := NewControlServer(cfg, mgr, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + + resp, err := http.Get(server.URL + "/identity") //nolint:noctx // test-only, hits an ephemeral httptest server + if err != nil { + t.Fatalf("GET /identity: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d (identity must not be gated by auth)", resp.StatusCode, http.StatusOK) + } + + var body struct { + ServerIdentity string `json:"server_identity"` + Capabilities []string `json:"capabilities"` + UnownedPeriodMS int64 `json:"unowned_period_ms"` + } + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode: %v", err) + } + if body.ServerIdentity != cfg.ServerIdentity { + t.Errorf("ServerIdentity = %q, want %q", body.ServerIdentity, cfg.ServerIdentity) + } + if len(body.Capabilities) == 0 { + t.Error("Capabilities is empty, want the advertised capability set") + } + // AC-EXECUTORS-CONTROL-OWNERSHIP-003.2 (Review round 3, finding 5): an + // adopting backend must be able to renew ownership on the cadence this + // server actually enforces, not on whatever its own local config + // resolves to -- the two can disagree across a restart that changed + // agentctl.unownedPeriod. /identity is the only channel that value ever + // crosses back to an adopting backend. + if body.UnownedPeriodMS != cs.unownedPeriod.Milliseconds() { + t.Errorf("UnownedPeriodMS = %d, want %d (this server's own resolved unowned period)", body.UnownedPeriodMS, cs.unownedPeriod.Milliseconds()) + } +} + +// TestGetIdentityRoundTripsThroughTheRealClient drives GET /identity through +// the real handler and the real agentctl.ControlClient, matching the +// end-to-end convention established for ListInstances (a hand-fabricated +// client fixture could never have caught that envelope mismatch). +func TestGetIdentityRoundTripsThroughTheRealClient(t *testing.T) { + log := logger.Default() + cfg := &config.Config{ + HomeDir: "/home/kandev-test/.kandev", + ServerIdentity: "server-identity-xyz", + DiagnosticLogPath: "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", + } + mgr := instance.NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + + cs := NewControlServer(cfg, mgr, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) + + identity, err := client.GetIdentity(t.Context()) + if err != nil { + t.Fatalf("GetIdentity: %v", err) + } + if identity.ServerIdentity != cfg.ServerIdentity { + t.Errorf("ServerIdentity = %q, want %q", identity.ServerIdentity, cfg.ServerIdentity) + } + if len(identity.Capabilities) == 0 { + t.Error("Capabilities is empty, want the advertised capability set") + } + if identity.UnownedPeriodMS != cs.unownedPeriod.Milliseconds() { + t.Errorf("UnownedPeriodMS = %d, want %d", identity.UnownedPeriodMS, cs.unownedPeriod.Milliseconds()) + } +} + +// TestGetServerDetailsRoundTripsThroughTheRealClientAndRequiresAuth pins the +// endpoint the paths moved to. The filesystem values an adopting backend +// records are still reachable, but only once it has authenticated, so a +// caller that never proved possession of the credential learns nothing about +// this machine's layout. +func TestGetServerDetailsRoundTripsThroughTheRealClientAndRequiresAuth(t *testing.T) { + log := logger.Default() + cfg := &config.Config{ + AuthToken: "details-credential", + HomeDir: "/home/kandev-test/.kandev", + ServerIdentity: "server-identity-details", + DiagnosticLogPath: "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", + } + mgr := instance.NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + + cs := NewControlServer(cfg, mgr, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + + unauthenticated, err := http.Get(server.URL + "/api/v1/ownership/details") //nolint:noctx // test-only, hits an ephemeral httptest server + if err != nil { + t.Fatalf("GET /api/v1/ownership/details: %v", err) + } + defer func() { _ = unauthenticated.Body.Close() }() + if unauthenticated.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated status = %d, want %d", unauthenticated.StatusCode, http.StatusUnauthorized) + } + + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) + client.SetAuthToken(cfg.AuthToken) + + details, err := client.GetServerDetails(t.Context()) + if err != nil { + t.Fatalf("GetServerDetails: %v", err) + } + if details.HomeDir != cfg.HomeDir { + t.Errorf("HomeDir = %q, want %q", details.HomeDir, cfg.HomeDir) + } + if details.DiagnosticLogPath != cfg.DiagnosticLogPath { + t.Errorf("DiagnosticLogPath = %q, want %q", details.DiagnosticLogPath, cfg.DiagnosticLogPath) + } +} diff --git a/apps/backend/internal/agentctl/server/api/lsp.go b/apps/backend/internal/agentctl/server/api/lsp.go index c92d2b82fd..8f37964fb8 100644 --- a/apps/backend/internal/agentctl/server/api/lsp.go +++ b/apps/backend/internal/agentctl/server/api/lsp.go @@ -146,6 +146,9 @@ func (s *Server) handleLSPStreamWS(c *gin.Context) { } conn.SetReadLimit(protocol.MaxMessageBytes) + releaseFencing := closeOnCredentialInvalidation(c, conn) + defer releaseFencing() + binaryPath, err := s.lspInstaller.BinaryPath(language) if err != nil { autoInstall := lspAutoInstallRequested(c) diff --git a/apps/backend/internal/agentctl/server/api/ownership.go b/apps/backend/internal/agentctl/server/api/ownership.go new file mode 100644 index 0000000000..b5ffbed8ab --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/ownership.go @@ -0,0 +1,128 @@ +package api + +import ( + "net/http" + "sync" + "time" + + "github.com/gin-gonic/gin" +) + +// ownershipState tracks how long it has been since a backend last proved it +// owns this control server (design 01, "Unowned shutdown"). The claim is a +// single explicit operation that both establishes and renews it; the +// bootstrap handshake and a successful credential rotation also renew it. +// It carries no instance identity: a server with zero instances is still +// owned. +type ownershipState struct { + mu sync.Mutex + lastRenewal time.Time + shuttingDown bool +} + +// newOwnershipState starts the clock at construction (process start), so a +// server whose owning backend dies before ever renewing reaps itself from +// its own launch time without any special-cased "never renewed" baseline. +func newOwnershipState() *ownershipState { + return &ownershipState{lastRenewal: time.Now()} +} + +// Renew records a successful claim, handshake, or rotation. Returns false +// without renewing when a shutdown has already begun (design 02's one-way +// door): a claim arriving after that point must be refused, not treated as +// reviving the ownership the shutdown decided to end. +func (o *ownershipState) Renew() bool { + o.mu.Lock() + defer o.mu.Unlock() + if o.shuttingDown { + return false + } + o.lastRenewal = time.Now() + return true +} + +// UnownedFor returns how long it has been since the last successful +// renewal, judged on this process's own clock per design 01 ("never on a +// wall-clock value either side supplies"). +func (o *ownershipState) UnownedFor() time.Duration { + o.mu.Lock() + defer o.mu.Unlock() + return time.Since(o.lastRenewal) +} + +// BeginShutdown latches the one-way unowned-shutdown decision. Returns false +// if a shutdown had already begun, so a caller can distinguish "this call +// began it" from "it was already in progress" without a second check. +func (o *ownershipState) BeginShutdown() bool { + o.mu.Lock() + defer o.mu.Unlock() + if o.shuttingDown { + return false + } + o.shuttingDown = true + return true +} + +// TryBeginShutdownIfUnownedFor atomically checks whether at least period has +// elapsed since the last renewal and, if so, latches the one-way shutdown +// door in the same critical section (AC-EXECUTORS-CONTROL-OWNERSHIP-003.9). +// Performing the elapsed check and the latch under one lock acquisition +// closes the window a separate UnownedFor()-then-BeginShutdown() sequence +// leaves open: a claim's Renew() landing between those two calls would +// succeed (shuttingDown still false) while the reaper's latch still fires +// right after, killing every instance out from under a backend that was just +// told its claim succeeded. Returns true only when this call itself both +// observed the elapsed period and performed the latch. +func (o *ownershipState) TryBeginShutdownIfUnownedFor(period time.Duration) bool { + o.mu.Lock() + defer o.mu.Unlock() + if o.shuttingDown { + return false + } + if time.Since(o.lastRenewal) < period { + return false + } + o.shuttingDown = true + return true +} + +// IsShuttingDown reports whether the one-way shutdown latch has fired. +func (o *ownershipState) IsShuttingDown() bool { + o.mu.Lock() + defer o.mu.Unlock() + return o.shuttingDown +} + +// shuttingDownMessage is the shared one-way-door refusal message for every +// ownership-lifecycle operation (claim, rotate, confirm) once +// AC-EXECUTORS-CONTROL-OWNERSHIP-003.9's shutdown latch has fired. +const shuttingDownMessage = "control server is shutting down" + +// handleOwnershipClaim establishes or renews ownership. Unlike /identity, +// this is a normal authenticated endpoint: it sits above the identity and +// capability negotiation in design 01's gate ordering. +func (m *ControlServer) handleOwnershipClaim(c *gin.Context) { + if !m.ownership.Renew() { + c.JSON(http.StatusConflict, gin.H{errKey: shuttingDownMessage}) + return + } + c.JSON(http.StatusOK, gin.H{lspStatusKey: "claimed"}) +} + +// handleOwnershipShutdown is the ownership-shutdown operation of +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.9: it sits below capability negotiation +// (registered as an adoption-only path, so the superseded credential +// authenticates it too, per AC-002.7) and requires no prior adoption or +// rotation. It only destroys -- it cannot drive an instance or outlive the +// call -- which is why the wider credential set is safe to admit here. +// +// The actual "stop every instance and exit" work happens in the run loop +// (cmd/agentctl/main.go), which already owns that sequence for signal- and +// parent-death-triggered shutdown; this handler only latches the one-way +// shutdown door and signals the loop to run it, mirroring the existing +// parent-liveness trigger rather than duplicating teardown here. +func (m *ControlServer) handleOwnershipShutdown(c *gin.Context) { + m.ownership.BeginShutdown() + m.requestShutdown() + c.JSON(http.StatusOK, gin.H{lspStatusKey: "shutting_down"}) +} diff --git a/apps/backend/internal/agentctl/server/api/ownership_proof.go b/apps/backend/internal/agentctl/server/api/ownership_proof.go new file mode 100644 index 0000000000..3bd38f0074 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/ownership_proof.go @@ -0,0 +1,59 @@ +package api + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/kandev/kandev/internal/common/ownershipproof" +) + +// ProveOwnership returns one proof per credential currently acceptable to +// this server, replacement first. There are at most two, and the superseded +// one is included while its rotation is unconfirmed because an adopting +// backend that crashed mid-rotation still holds it and must be able to +// adopt again. +func (c *credentialState) ProveOwnership(challenge, binding string) []string { + c.mu.Lock() + defer c.mu.Unlock() + + proofs := []string{ownershipproof.Derive(c.latest, challenge, binding)} + if c.unconfirmed && c.superseded != "" { + proofs = append(proofs, ownershipproof.Derive(c.superseded, challenge, binding)) + } + return proofs +} + +type ownershipProveRequest struct { + Challenge string `json:"challenge"` +} + +// handleOwnershipProve lets an adopting backend verify this process is the +// control server its record names before it sends the credential. It sits +// below authentication because it is what establishes that trust; answering +// it reveals nothing, since a proof is a keyed digest over a challenge the +// caller chose and cannot be inverted to the credential. +func (m *ControlServer) handleOwnershipProve(c *gin.Context) { + if m.ownership.IsShuttingDown() { + c.JSON(http.StatusConflict, gin.H{errKey: shuttingDownMessage}) + return + } + + var req ownershipProveRequest + if err := c.ShouldBindJSON(&req); err != nil || req.Challenge == "" { + c.JSON(http.StatusBadRequest, gin.H{errKey: "missing or invalid challenge"}) + return + } + + c.JSON(http.StatusOK, gin.H{"proofs": m.credentials.ProveOwnership(req.Challenge, m.cfg.HomeDir)}) +} + +// handleOwnershipDetails carries the values an adopting backend records. +// They are filesystem paths, so they are served here, behind authentication, +// rather than on the unauthenticated identity endpoint. +func (m *ControlServer) handleOwnershipDetails(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "home_dir": m.cfg.HomeDir, + "diagnostic_log_path": m.cfg.DiagnosticLogPath, + }) +} diff --git a/apps/backend/internal/agentctl/server/api/ownership_proof_test.go b/apps/backend/internal/agentctl/server/api/ownership_proof_test.go new file mode 100644 index 0000000000..392df55bec --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/ownership_proof_test.go @@ -0,0 +1,195 @@ +package api + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/common/logger" +) + +// expectedProof mirrors the derivation an adopting backend performs +// independently. It is written out longhand here rather than calling the +// production helper so the test pins the wire contract, not the +// implementation's agreement with itself. +func expectedProof(credential, challenge, binding string) string { + mac := hmac.New(sha256.New, []byte(credential)) + mac.Write([]byte(challenge)) + mac.Write([]byte{0}) + mac.Write([]byte(binding)) + return hex.EncodeToString(mac.Sum(nil)) +} + +func postProve(t *testing.T, baseURL, challenge string) (int, []string) { + t.Helper() + body, err := json.Marshal(map[string]string{"challenge": challenge}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + resp, err := http.Post(baseURL+"/ownership/prove", "application/json", bytes.NewReader(body)) //nolint:noctx // test-only, ephemeral httptest server + if err != nil { + t.Fatalf("POST /ownership/prove: %v", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return resp.StatusCode, nil + } + var decoded struct { + Proofs []string `json:"proofs"` + } + if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil { + t.Fatalf("decode: %v", err) + } + return resp.StatusCode, decoded.Proofs +} + +// TestOwnershipProveAnswersOnlyWithAValueDerivedFromTheHeldCredential is the +// server half of AC-EXECUTORS-CONTROL-OWNERSHIP-001.10: a control server +// demonstrates it already holds the credential without ever transmitting it, +// and the demonstration is bound to the caller's challenge so a value +// harvested for one attempt is worthless for the next. +func TestOwnershipProveAnswersOnlyWithAValueDerivedFromTheHeldCredential(t *testing.T) { + log := logger.Default() + cfg := &config.Config{ + AuthToken: "the-real-credential", + HomeDir: "/home/kandev-test/.kandev", + ServerIdentity: "server-identity-abc", + } + mgr := instance.NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + server := httptest.NewServer(NewControlServer(cfg, mgr, log).Router()) + defer server.Close() + + status, proofs := postProve(t, server.URL, "challenge-one") + if status != http.StatusOK { + t.Fatalf("status = %d, want 200", status) + } + if len(proofs) != 1 { + t.Fatalf("len(proofs) = %d, want 1 (only the current credential is acceptable)", len(proofs)) + } + want := expectedProof("the-real-credential", "challenge-one", "/home/kandev-test/.kandev") + if proofs[0] != want { + t.Fatalf("proof = %q, want %q", proofs[0], want) + } + + // The credential itself is never the demonstration. + if proofs[0] == "the-real-credential" { + t.Fatal("the proof is the credential verbatim") + } + + // A different challenge yields a different proof, so a captured proof + // cannot be replayed against the next attempt's challenge. + _, second := postProve(t, server.URL, "challenge-two") + if len(second) != 1 || second[0] == proofs[0] { + t.Fatalf("proof did not vary with the challenge: %v vs %v", proofs, second) + } + + // A server holding a different credential cannot produce this answer. + if proofs[0] == expectedProof("some-other-credential", "challenge-one", "/home/kandev-test/.kandev") { + t.Fatal("proof did not depend on the credential") + } + // Nor can one whose home directory differs, which is bound into the + // derivation precisely because it is no longer disclosed. + if proofs[0] == expectedProof("the-real-credential", "challenge-one", "/some/other/home") { + t.Fatal("proof did not depend on the home directory binding") + } +} + +// TestOwnershipProveCoversEveryAcceptableCredentialDuringTheTwoPhaseWindow: +// AC-EXECUTORS-CONTROL-OWNERSHIP-002.8 keeps at most two credentials +// acceptable while a rotation is unconfirmed, and AC-002.10's replay path +// means the adopting backend may still hold the superseded one. The proof +// must cover both, or that replay would be refused as an impostor. +func TestOwnershipProveCoversEveryAcceptableCredentialDuringTheTwoPhaseWindow(t *testing.T) { + creds := newCredentialState("bootstrap-credential") + + single := creds.ProveOwnership("chal", "/home") + if len(single) != 1 || single[0] != expectedProof("bootstrap-credential", "chal", "/home") { + t.Fatalf("before any rotation, proofs = %v", single) + } + + _, replacement, err := creds.Rotate("bootstrap-credential") + if err != nil { + t.Fatalf("Rotate: %v", err) + } + + both := creds.ProveOwnership("chal", "/home") + if len(both) != 2 { + t.Fatalf("len(proofs) = %d, want 2 while the rotation is unconfirmed", len(both)) + } + wantNew := expectedProof(replacement, "chal", "/home") + wantOld := expectedProof("bootstrap-credential", "chal", "/home") + if both[0] != wantNew || both[1] != wantOld { + t.Fatalf("proofs = %v, want [replacement, superseded]", both) + } + + if err := creds.Confirm(1); err != nil { + t.Fatalf("Confirm: %v", err) + } + after := creds.ProveOwnership("chal", "/home") + if len(after) != 1 || after[0] != wantNew { + t.Fatalf("after confirmation, proofs = %v, want only the replacement", after) + } +} + +// TestOwnershipProveRejectsAnEmptyChallenge: a response must never be +// accepted for a challenge the backend did not generate +// (AC-EXECUTORS-CONTROL-OWNERSHIP-001.10), so a caller that supplies no +// challenge gets no proof rather than one over a fixed empty string. +func TestOwnershipProveRejectsAnEmptyChallenge(t *testing.T) { + log := logger.Default() + cfg := &config.Config{AuthToken: "cred", HomeDir: "/home", ServerIdentity: "id"} + mgr := instance.NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + server := httptest.NewServer(NewControlServer(cfg, mgr, log).Router()) + defer server.Close() + + if status, _ := postProve(t, server.URL, ""); status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for an empty challenge", status) + } +} + +// TestIdentityDisclosesNoFilesystemPath is +// AC-EXECUTORS-CONTROL-OWNERSHIP-001.11: the endpoint that sits below +// authentication carries only what the decision to authenticate needs. +func TestIdentityDisclosesNoFilesystemPath(t *testing.T) { + log := logger.Default() + cfg := &config.Config{ + AuthToken: "cred", + HomeDir: "/home/kandev-test/.kandev", + ServerIdentity: "server-identity-abc", + DiagnosticLogPath: "/home/kandev-test/.kandev/logs/agentctl-diagnostic.log", + } + mgr := instance.NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + server := httptest.NewServer(NewControlServer(cfg, mgr, log).Router()) + defer server.Close() + + resp, err := http.Get(server.URL + "/identity") //nolint:noctx // test-only, ephemeral httptest server + if err != nil { + t.Fatalf("GET /identity: %v", err) + } + defer func() { _ = resp.Body.Close() }() + + var payload map[string]any + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + t.Fatalf("decode: %v", err) + } + for _, forbidden := range []string{"home_dir", "diagnostic_log_path"} { + if _, present := payload[forbidden]; present { + t.Errorf("unauthenticated /identity still discloses %q", forbidden) + } + } + for _, required := range []string{"server_identity", "capabilities", "unowned_period_ms"} { + if _, present := payload[required]; !present { + t.Errorf("/identity no longer reports %q, which adoption needs", required) + } + } +} diff --git a/apps/backend/internal/agentctl/server/api/ownership_test.go b/apps/backend/internal/agentctl/server/api/ownership_test.go new file mode 100644 index 0000000000..dd9d493f69 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/ownership_test.go @@ -0,0 +1,297 @@ +package api + +import ( + "net/http/httptest" + "testing" + "time" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/common/logger" +) + +// TestOwnershipStateRenewUpdatesLastRenewal pins that Renew advances the +// tracked instant, which the unowned-period reaper (Layer 5.5) will measure +// elapsed time against. +func TestOwnershipStateRenewUpdatesLastRenewal(t *testing.T) { + o := newOwnershipState() + time.Sleep(2 * time.Millisecond) + before := o.UnownedFor() + + if !o.Renew() { + t.Fatal("Renew() = false, want true") + } + after := o.UnownedFor() + + if after >= before { + t.Fatalf("UnownedFor after Renew (%v) >= before (%v), want renewal to reset the elapsed duration", after, before) + } +} + +// TestOwnershipStateNewStartsFromProcessLaunch pins design 01's "A server +// spawned by a backend that then dies before completing the handshake has +// no successful renewal at all, so its period runs from process start and it +// reaps itself without special handling" -- the very first UnownedFor call, +// before any Renew, must already report a small nonzero-or-zero duration +// measured from construction, not an unset/zero-time sentinel that would +// read as "just now" forever or as an enormous elapsed duration. +func TestOwnershipStateNewStartsFromProcessLaunch(t *testing.T) { + o := newOwnershipState() + time.Sleep(2 * time.Millisecond) + + elapsed := o.UnownedFor() + if elapsed <= 0 { + t.Fatalf("UnownedFor() = %v, want a small positive duration measured from construction", elapsed) + } + if elapsed > time.Second { + t.Fatalf("UnownedFor() = %v, want a small duration (construction just happened), not a large one implying a zero-time baseline", elapsed) + } +} + +// TestOwnershipStateBeginShutdownIsOneWay pins the one-way door: once a +// shutdown has begun, Renew must refuse (never resurrect a decided +// shutdown), and BeginShutdown itself is idempotent (a second call reports +// it was already begun rather than restarting the decision). +func TestOwnershipStateBeginShutdownIsOneWay(t *testing.T) { + o := newOwnershipState() + + if !o.BeginShutdown() { + t.Fatal("first BeginShutdown() = false, want true") + } + if o.BeginShutdown() { + t.Fatal("second BeginShutdown() = true, want false (idempotent one-way latch)") + } + if !o.IsShuttingDown() { + t.Fatal("IsShuttingDown() = false after BeginShutdown, want true") + } + if o.Renew() { + t.Fatal("Renew() = true after shutdown began, want false (a decided shutdown must never be reversed)") + } +} + +// TestOwnershipStateTryBeginShutdownIfUnownedForRefusesAfterRecentRenewal +// pins the atomic check-and-latch's core correctness: a renewal that landed +// before the call must be observed by the SAME call that would otherwise +// latch shutdown, closing the TOCTOU window a separate UnownedFor()-then- +// BeginShutdown() sequence leaves open (Review round 1, finding 5). +func TestOwnershipStateTryBeginShutdownIfUnownedForRefusesAfterRecentRenewal(t *testing.T) { + o := newOwnershipState() + if !o.Renew() { + t.Fatal("Renew() = false, want true") + } + + if o.TryBeginShutdownIfUnownedFor(time.Hour) { + t.Fatal("TryBeginShutdownIfUnownedFor() = true immediately after a renewal, want false") + } + if o.IsShuttingDown() { + t.Fatal("IsShuttingDown() = true after a refused attempt, want false") + } +} + +// TestOwnershipStateTryBeginShutdownIfUnownedForLatchesOnceElapsed pins the +// success path: once the period has genuinely elapsed, the call both +// reports true and latches the one-way shutdown door. +func TestOwnershipStateTryBeginShutdownIfUnownedForLatchesOnceElapsed(t *testing.T) { + o := newOwnershipState() + o.mu.Lock() + o.lastRenewal = time.Now().Add(-time.Hour) + o.mu.Unlock() + + if !o.TryBeginShutdownIfUnownedFor(time.Minute) { + t.Fatal("TryBeginShutdownIfUnownedFor() = false once the period elapsed, want true") + } + if !o.IsShuttingDown() { + t.Fatal("IsShuttingDown() = false after a successful latch, want true") + } +} + +// TestOwnershipStateTryBeginShutdownIfUnownedForIsOneWay pins idempotency: +// a second call after the door is already latched must not report success +// again, matching BeginShutdown's existing one-way-door contract. +func TestOwnershipStateTryBeginShutdownIfUnownedForIsOneWay(t *testing.T) { + o := newOwnershipState() + o.mu.Lock() + o.lastRenewal = time.Now().Add(-time.Hour) + o.mu.Unlock() + + if !o.TryBeginShutdownIfUnownedFor(time.Minute) { + t.Fatal("first TryBeginShutdownIfUnownedFor() = false, want true") + } + if o.TryBeginShutdownIfUnownedFor(time.Minute) { + t.Fatal("second TryBeginShutdownIfUnownedFor() = true, want false (one-way latch)") + } +} + +// TestHandleOwnershipClaimRenewsOwnership pins that a successful claim +// renews the tracked ownership instant. The claim operation carries no +// instance identity -- a server with zero instances is still owned. +func TestHandleOwnershipClaimRenewsOwnership(t *testing.T) { + cfg := &config.Config{AuthToken: "test-token"} + log := logger.Default() + cs := NewControlServer(cfg, &instance.Manager{}, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("test-token")) + + backdateOwnership(cs, time.Hour) + + if err := client.ClaimOwnership(t.Context()); err != nil { + t.Fatalf("ClaimOwnership: %v", err) + } + + if elapsed := cs.ownership.UnownedFor(); elapsed >= time.Minute { + t.Fatalf("UnownedFor after claim = %v, want well under the backdated hour (claim should have renewed)", elapsed) + } +} + +// backdateOwnership sets a ControlServer's tracked last-renewal instant into +// the past, so a test can assert that a subsequent operation renews it +// without racing HTTP round-trip latency against a tiny real elapsed +// duration. +func backdateOwnership(cs *ControlServer, age time.Duration) { + cs.ownership.mu.Lock() + defer cs.ownership.mu.Unlock() + cs.ownership.lastRenewal = time.Now().Add(-age) +} + +// TestHandleOwnershipClaimRequiresAuth pins that the claim operation sits +// ABOVE identity/capability negotiation in design 01's gate ordering: unlike +// /identity, it is a normal authenticated endpoint. +func TestHandleOwnershipClaimRequiresAuth(t *testing.T) { + cfg := &config.Config{AuthToken: "test-token"} + log := logger.Default() + cs := NewControlServer(cfg, &instance.Manager{}, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) // no auth token + + if err := client.ClaimOwnership(t.Context()); err == nil { + t.Fatal("ClaimOwnership without auth = nil error, want a rejection") + } +} + +// TestHandleOwnershipClaimRefusedAfterShutdownBegun pins the one-way-door +// refusal at the HTTP layer: a claim arriving after an unowned shutdown has +// begun must be refused with a shutting-down outcome rather than reviving +// ownership underneath a teardown in progress (design 02's failure table). +func TestHandleOwnershipClaimRefusedAfterShutdownBegun(t *testing.T) { + cfg := &config.Config{AuthToken: "test-token"} + log := logger.Default() + cs := NewControlServer(cfg, &instance.Manager{}, log) + cs.ownership.BeginShutdown() + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("test-token")) + + if err := client.ClaimOwnership(t.Context()); err == nil { + t.Fatal("ClaimOwnership after shutdown began = nil error, want a rejection") + } +} + +// TestHandleHandshakeRenewsOwnership pins design 01's "two other operations +// renew it: the bootstrap handshake and a successful credential rotation" -- +// a fresh server's first renewal comes from the handshake that mints its +// credential, giving the unowned-period timer a definite start without +// requiring a separate claim call immediately after. +func TestHandleHandshakeRenewsOwnership(t *testing.T) { + cfg := &config.Config{ + AuthToken: "test-generated-token", + BootstrapNonce: "test-nonce", + } + log := logger.Default() + cs := NewControlServer(cfg, &instance.Manager{}, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) + + backdateOwnership(cs, time.Hour) + + if _, err := client.Handshake(t.Context(), "test-nonce"); err != nil { + t.Fatalf("Handshake: %v", err) + } + + if elapsed := cs.ownership.UnownedFor(); elapsed >= time.Minute { + t.Fatalf("UnownedFor after handshake = %v, want well under the backdated hour (handshake should have renewed)", elapsed) + } +} + +// TestNonRenewingControlOperationsDoNotRenewOwnership pins the negative +// clause of AC-EXECUTORS-CONTROL-OWNERSHIP-003.8: beyond the claim, the +// bootstrap handshake and a successful credential rotation, "no operation +// shall renew ownership: neither an instance operation, nor an open stream, +// nor enumeration". Only the three renewing operations are pinned +// positively elsewhere, which leaves the clause that actually carries the +// guarantee unguarded -- a renewal added to the shared credential +// middleware, or to any instance route, would make a backend that is busy +// but has stopped renewing look present forever and defeat the unowned +// shutdown entirely. Every case drives real HTTP through the router, so the +// middleware is on the path under test rather than bypassed. +// +// The clause's "open stream" arm is structural rather than testable here: +// the streaming routes live on the per-instance Server (server.go), which +// holds no ownership state at all, so no stream can reach this clock. +func TestNonRenewingControlOperationsDoNotRenewOwnership(t *testing.T) { + const backdate = time.Hour + + cfg := &config.Config{AuthToken: "test-token"} + log := logger.Default() + cs := NewControlServer(cfg, &instance.Manager{}, log) + server := httptest.NewServer(cs.Router()) + defer server.Close() + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log, agentctl.WithControlAuthToken("test-token")) + + // Each call asserts its own outcome before ownership is examined. A + // request that never reached the handler would also leave ownership + // unrenewed, so without these the test would pass vacuously against a + // server that was not listening at all. + cases := []struct { + name string + call func(t *testing.T) + }{ + {"enumeration", func(t *testing.T) { + if _, err := client.ListInstances(t.Context()); err != nil { + t.Fatalf("ListInstances: %v", err) + } + }}, + {"instance operation", func(t *testing.T) { + _, err := client.GetInstance(t.Context(), "no-such-instance") + // The not-found wording is the handler's 404, distinct from the + // "failed to get instance" a transport error produces, so this + // asserts the request was served rather than merely refused. + if err == nil || err.Error() != `instance "no-such-instance" not found` { + t.Fatalf("GetInstance error = %v, want the handler's not-found response", err) + } + }}, + {"ownership details read", func(t *testing.T) { + if _, err := client.GetServerDetails(t.Context()); err != nil { + t.Fatalf("GetServerDetails: %v", err) + } + }}, + {"health", func(t *testing.T) { + if err := client.Health(t.Context()); err != nil { + t.Fatalf("Health: %v", err) + } + }}, + {"identity", func(t *testing.T) { + if _, err := client.GetIdentity(t.Context()); err != nil { + t.Fatalf("GetIdentity: %v", err) + } + }}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + backdateOwnership(cs, backdate) + tc.call(t) + if elapsed := cs.ownership.UnownedFor(); elapsed < backdate { + t.Fatalf("UnownedFor() after %s = %v, want >= %v (this operation must not renew ownership)", tc.name, elapsed, backdate) + } + }) + } +} diff --git a/apps/backend/internal/agentctl/server/api/port_proxy.go b/apps/backend/internal/agentctl/server/api/port_proxy.go index 10e2be73c4..2db9f6f8c1 100644 --- a/apps/backend/internal/agentctl/server/api/port_proxy.go +++ b/apps/backend/internal/agentctl/server/api/port_proxy.go @@ -78,7 +78,10 @@ func (s *Server) handlePortProxy(c *gin.Context) { } }() - proxy.ServeHTTP(c.Writer, c.Request) + request, releaseFencing := credentialScopedRequest(c) + defer releaseFencing() + + proxy.ServeHTTP(c.Writer, request) } func (s *Server) createPortProxy(port int) *httputil.ReverseProxy { diff --git a/apps/backend/internal/agentctl/server/api/server.go b/apps/backend/internal/agentctl/server/api/server.go index cd8541cadb..fada65c7d5 100644 --- a/apps/backend/internal/agentctl/server/api/server.go +++ b/apps/backend/internal/agentctl/server/api/server.go @@ -40,9 +40,26 @@ type Server struct { metricsCollector *metrics.Collector lspInstaller lspInstallerRegistry + // credentialSource, when set via SetCredentialSource, authenticates this + // instance's requests and streams against the control server's single + // rotating credential instead of the static cfg.AuthToken captured at + // construction (design 01 "Single driver", + // AC-EXECUTORS-CONTROL-OWNERSHIP-002.6). Nil preserves the legacy + // static-token behavior every test constructing a Server without a + // running control server alongside it relies on. + credentialSource InstanceCredentialSource + upgrader websocket.Upgrader } +// SetCredentialSource wires this instance server's authentication and +// stream lifetime to the control server's single rotating credential, +// replacing the static per-instance token captured at construction. Call +// once, before the server starts accepting requests. +func (s *Server) SetCredentialSource(src InstanceCredentialSource) { + s.credentialSource = src +} + // NewServer creates a new API server for an agent instance. // If mcpServer is provided, MCP routes will be registered. // If mcpBackendClient is provided, the agent stream becomes bidirectional for MCP. @@ -71,7 +88,7 @@ func NewServer(cfg *config.InstanceConfig, procMgr *process.Manager, mcpServer * // - /health: liveness probe // - /sse, /message, /mcp: MCP endpoints used by the agent subprocess which // runs in the same trust boundary but does not possess the auth token. - s.router.Use(bearerTokenAuth(cfg.AuthToken, "/health", "/sse", "/message", "/mcp")) + s.router.Use(s.instanceAuth(cfg.AuthToken, "/health", "/sse", "/message", "/mcp")) // Validate X-Instance-ID so a client that holds a stale port (because // the previous instance was deleted and the port recycled to a new // instance) gets a clean 404 instead of accidentally configuring or diff --git a/apps/backend/internal/agentctl/server/api/shell_terminal.go b/apps/backend/internal/agentctl/server/api/shell_terminal.go index 28f951c1bb..a6d348df5d 100644 --- a/apps/backend/internal/agentctl/server/api/shell_terminal.go +++ b/apps/backend/internal/agentctl/server/api/shell_terminal.go @@ -75,6 +75,9 @@ func (s *Server) handleShellTerminalStreamWS(c *gin.Context) { } defer func() { _ = conn.Close() }() + releaseFencing := closeOnCredentialInvalidation(c, conn) + defer releaseFencing() + s.logger.Info("terminal shell WebSocket connected", zap.String("terminal_id", terminalID)) // Subscribe to shell output diff --git a/apps/backend/internal/agentctl/server/api/turn_outcome.go b/apps/backend/internal/agentctl/server/api/turn_outcome.go new file mode 100644 index 0000000000..e8be512cef --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/turn_outcome.go @@ -0,0 +1,53 @@ +package api + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +// handleGetTurnOutcome implements the read side of +// AC-EXECUTORS-SURVIVAL-004.2/.6: a repeatable, non-discarding read of the +// named instance's last retained terminal turn outcome. 404s only when the +// instance itself is unknown; "nothing retained" is a normal 200 answer +// (the case AC-EXECUTORS-SURVIVAL-004.5 maps to "publish as running"). +func (m *ControlServer) handleGetTurnOutcome(c *gin.Context) { + id := c.Param("id") + + outcome, hasOutcome, instanceFound := m.instMgr.PeekTurnOutcome(id) + if !instanceFound { + c.JSON(http.StatusNotFound, gin.H{errKey: instanceNotFoundMessage}) + return + } + if !hasOutcome { + c.JSON(http.StatusOK, gin.H{"retained": false}) + return + } + c.JSON(http.StatusOK, gin.H{ + "retained": true, + "turn_id": outcome.TurnID, + "event": outcome.Event, + }) +} + +type ackTurnOutcomeRequest struct { + TurnID int64 `json:"turn_id" binding:"required"` +} + +// handleAckTurnOutcome implements AC-EXECUTORS-SURVIVAL-004.6's +// acknowledgement: discards the retained outcome only when turn_id names the +// one currently retained. An identifier the server no longer holds, or never +// held, is accepted and changes nothing -- so this always answers 200, +// making a retried acknowledgement safe to send again. +func (m *ControlServer) handleAckTurnOutcome(c *gin.Context) { + id := c.Param("id") + + var req ackTurnOutcomeRequest + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{errKey: "turn_id is required"}) + return + } + + m.instMgr.AckTurnOutcome(id, req.TurnID) + c.JSON(http.StatusOK, gin.H{lspStatusKey: "acked"}) +} diff --git a/apps/backend/internal/agentctl/server/api/turn_outcome_test.go b/apps/backend/internal/agentctl/server/api/turn_outcome_test.go new file mode 100644 index 0000000000..5ff8e54051 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/turn_outcome_test.go @@ -0,0 +1,152 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "testing" + + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/pkg/agent" +) + +// newTurnOutcomeTestServer wires a real instance.Manager with one real +// instance through a real ControlServer, mirroring the ListInstances +// end-to-end pattern (control_instance_list_test.go): the wire contract for +// turn-outcome retain/peek/ack must be proven against real JSON encoding and +// decoding on both ends, not a hand-written fixture. +func newTurnOutcomeTestServer(t *testing.T) (*instance.Manager, *agentctl.ControlClient, string) { + t.Helper() + log := logger.Default() + mgr := instance.NewManager(&config.Config{ + Ports: config.PortConfig{Base: 0, Max: 0}, + Defaults: config.InstanceDefaults{Protocol: agent.ProtocolACP}, + }, log) + t.Cleanup(func() { _ = mgr.Shutdown(t.Context()) }) + mgr.SetServerFactory(func(*config.InstanceConfig, *process.Manager, *logger.Logger) http.Handler { + return http.NotFoundHandler() + }) + + created, err := mgr.CreateInstance(t.Context(), &instance.CreateRequest{ + WorkspacePath: t.TempDir(), + }) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + t.Cleanup(func() { _ = mgr.StopInstance(t.Context(), created.ID) }) + + cs := NewControlServer(&config.Config{}, mgr, log) + server := httptest.NewServer(cs.Router()) + t.Cleanup(server.Close) + host, port := parseHostPort(t, server.URL) + client := agentctl.NewControlClient(host, port, log) + + return mgr, client, created.ID +} + +// TestGetTurnOutcomeReturns404ForUnknownInstance pins that the wire contract +// distinguishes "no such instance" from "instance exists, nothing retained". +func TestGetTurnOutcomeReturns404ForUnknownInstance(t *testing.T) { + _, client, _ := newTurnOutcomeTestServer(t) + + if _, err := client.GetTurnOutcome(t.Context(), "does-not-exist"); err == nil { + t.Fatal("GetTurnOutcome(unknown instance) = nil error, want an error") + } +} + +// TestGetTurnOutcomeReturnsNilWhenNothingRetained pins +// AC-EXECUTORS-SURVIVAL-004.5: a real, live instance with no retained +// terminal outcome yet must answer "nothing retained", not an error. +func TestGetTurnOutcomeReturnsNilWhenNothingRetained(t *testing.T) { + _, client, instanceID := newTurnOutcomeTestServer(t) + + outcome, err := client.GetTurnOutcome(t.Context(), instanceID) + if err != nil { + t.Fatalf("GetTurnOutcome: %v", err) + } + if outcome != nil { + t.Fatalf("outcome = %+v, want nil (nothing retained yet)", outcome) + } +} + +// TestGetTurnOutcomeRoundTripsARetainedOutcomeRepeatedly pins +// AC-EXECUTORS-SURVIVAL-004.2/.6 end to end: a retained outcome round-trips +// through the real HTTP handler and real client, and repeated GETs return +// the identical turn ID and event without discarding it. +func TestGetTurnOutcomeRoundTripsARetainedOutcomeRepeatedly(t *testing.T) { + mgr, client, instanceID := newTurnOutcomeTestServer(t) + + turnID, ok := mgr.RetainTurnOutcome(instanceID, streams.AgentEvent{ + Type: streams.EventTypeComplete, + SessionID: "sess-xyz", + }) + if !ok { + t.Fatal("RetainTurnOutcome() ok = false, want true") + } + + for i := 0; i < 2; i++ { + outcome, err := client.GetTurnOutcome(t.Context(), instanceID) + if err != nil { + t.Fatalf("iteration %d: GetTurnOutcome: %v", i, err) + } + if outcome == nil { + t.Fatalf("iteration %d: outcome = nil, want the retained outcome", i) + } + if outcome.TurnID != turnID { + t.Fatalf("iteration %d: TurnID = %d, want %d", i, outcome.TurnID, turnID) + } + if outcome.Event.SessionID != "sess-xyz" { + t.Fatalf("iteration %d: Event.SessionID = %q, want %q", i, outcome.Event.SessionID, "sess-xyz") + } + } +} + +// TestAckTurnOutcomeClearsTheMatchingOutcome pins the ack half of +// AC-EXECUTORS-SURVIVAL-004.6 end to end: acking the retained turn ID +// discards it, and a subsequent GET reports nothing retained. +func TestAckTurnOutcomeClearsTheMatchingOutcome(t *testing.T) { + mgr, client, instanceID := newTurnOutcomeTestServer(t) + + turnID, _ := mgr.RetainTurnOutcome(instanceID, streams.AgentEvent{Type: streams.EventTypeComplete}) + + if err := client.AckTurnOutcome(t.Context(), instanceID, turnID); err != nil { + t.Fatalf("AckTurnOutcome: %v", err) + } + + outcome, err := client.GetTurnOutcome(t.Context(), instanceID) + if err != nil { + t.Fatalf("GetTurnOutcome after ack: %v", err) + } + if outcome != nil { + t.Fatalf("outcome after ack = %+v, want nil", outcome) + } +} + +// TestAckTurnOutcomeWithUnrelatedIDIsSafeAndChangesNothing pins that acking +// an identifier the server never issued (or already discarded) is accepted +// -- so a retried acknowledgement is always safe -- and does not disturb a +// different outcome that is still retained. +func TestAckTurnOutcomeWithUnrelatedIDIsSafeAndChangesNothing(t *testing.T) { + mgr, client, instanceID := newTurnOutcomeTestServer(t) + + turnID, _ := mgr.RetainTurnOutcome(instanceID, streams.AgentEvent{Type: streams.EventTypeComplete}) + + if err := client.AckTurnOutcome(t.Context(), instanceID, turnID+999); err != nil { + t.Fatalf("AckTurnOutcome(unrelated id): %v", err) + } + + outcome, err := client.GetTurnOutcome(t.Context(), instanceID) + if err != nil { + t.Fatalf("GetTurnOutcome: %v", err) + } + if outcome == nil { + t.Fatal("outcome = nil after an unrelated ack, want the original outcome still retained") + } + if outcome.TurnID != turnID { + t.Fatalf("TurnID = %d, want %d (unaffected by the unrelated ack)", outcome.TurnID, turnID) + } +} diff --git a/apps/backend/internal/agentctl/server/api/unowned_reaper.go b/apps/backend/internal/agentctl/server/api/unowned_reaper.go new file mode 100644 index 0000000000..1524930fb6 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/unowned_reaper.go @@ -0,0 +1,61 @@ +package api + +import ( + "time" + + "go.uber.org/zap" +) + +// unownedReaperScanInterval is how often the reaper checks elapsed unowned +// time. It is a polling granularity, not an operator-facing setting -- like +// F71's log-sink bounds, it is a fixed implementation constant, not one of +// the five numeric tunables in the configuration surface. +const unownedReaperScanInterval = 15 * time.Second + +// StartUnownedReaper starts the background goroutine that stops every +// instance and exits once no ownership claim has been current for the +// resolved unowned period (AC-EXECUTORS-CONTROL-OWNERSHIP-003.1). It fires +// the SAME one-way shutdown latch and ShutdownRequested signal the +// ownership-shutdown operation uses (see ownership.go), so the run loop +// only needs one trigger for both a deliberate shutdown call and a timed-out +// one. +// +// Callers must not start this until the launch is actually running with the +// agent-survival capability engaged AND something is periodically renewing +// ownership -- the backend-side claim loop in +// internal/agent/runtime/lifecycle: starting it unconditionally on every +// agentctl launch would self-terminate a perfectly healthy attached instance +// once the period elapses, since nothing renews ownership without that +// claim loop. Gated by cmd/agentctl/main.go's startUnownedReaperIfEnabled. +func (m *ControlServer) StartUnownedReaper() { + m.reaperWG.Add(1) + go m.runUnownedReaper(m.unownedPeriod, unownedReaperScanInterval) +} + +// StopUnownedReaper signals the reaper goroutine to exit and waits for it +// to drain. Safe to call even if StartUnownedReaper was never called. +func (m *ControlServer) StopUnownedReaper() { + m.reaperStopOnce.Do(func() { close(m.reaperStop) }) + m.reaperWG.Wait() +} + +func (m *ControlServer) runUnownedReaper(period, interval time.Duration) { + defer m.reaperWG.Done() + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-m.reaperStop: + return + case <-ticker.C: + if m.decideUnownedShutdown(period) { + m.requestShutdown() + m.logger.Warn("no ownership claim renewed within the unowned period, shutting down", + zap.Duration("unowned_period", period)) + return + } + } + } +} diff --git a/apps/backend/internal/agentctl/server/api/unowned_reaper_test.go b/apps/backend/internal/agentctl/server/api/unowned_reaper_test.go new file mode 100644 index 0000000000..a611ce0b96 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/unowned_reaper_test.go @@ -0,0 +1,159 @@ +package api + +import ( + "sync" + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/instance" + "github.com/kandev/kandev/internal/common/logger" +) + +// resolveUnownedPeriod's own coverage (AC-EXECUTORS-CONTROL-OWNERSHIP-003.4/ +// .6/.7) moved to internal/common/ownershipperiod, which both this package +// and the backend's ownership-renewal loop now import, so the computation +// cannot drift between the two sides that must agree on it. + +// --- reaper goroutine lifecycle --- + +func newUnownedReaperTestServer(t *testing.T) *ControlServer { + t.Helper() + cfg := &config.Config{AuthToken: "test-token"} + cs := NewControlServer(cfg, &instance.Manager{}, logger.Default()) + t.Cleanup(cs.StopUnownedReaper) + return cs +} + +// TestUnownedReaperFiresShutdownAfterPeriodElapses pins +// AC-EXECUTORS-CONTROL-OWNERSHIP-003.1/.9: once no ownership claim has been +// current for the resolved period, the reaper latches the one-way shutdown +// door and signals the run loop through the SAME mechanism the +// ownership-shutdown operation uses, not a second path. +func TestUnownedReaperFiresShutdownAfterPeriodElapses(t *testing.T) { + cs := newUnownedReaperTestServer(t) + backdateOwnership(cs, time.Hour) + + cs.reaperWG.Add(1) + go cs.runUnownedReaper(20*time.Millisecond, 5*time.Millisecond) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if cs.ownership.IsShuttingDown() { + break + } + time.Sleep(5 * time.Millisecond) + } + + if !cs.ownership.IsShuttingDown() { + t.Fatal("ownership.IsShuttingDown() = false after the unowned period elapsed, want true") + } + + // Latching the one-way door and closing the signal channel are separate + // steps, so the signal is awaited rather than sampled once. + select { + case <-cs.ShutdownRequested(): + case <-time.After(2 * time.Second): + t.Fatal("ShutdownRequested() channel not closed after the reaper fired") + } +} + +// TestUnownedReaperDoesNotFireWhileOwnershipIsRenewed pins that a +// continuously-renewed ownership claim keeps the reaper quiet: it must +// measure elapsed time since the last renewal, not since the reaper itself +// started. +func TestUnownedReaperDoesNotFireWhileOwnershipIsRenewed(t *testing.T) { + cs := newUnownedReaperTestServer(t) + + cs.reaperWG.Add(1) + go cs.runUnownedReaper(200*time.Millisecond, 10*time.Millisecond) + + renewDeadline := time.Now().Add(400 * time.Millisecond) + for time.Now().Before(renewDeadline) { + cs.ownership.Renew() + time.Sleep(5 * time.Millisecond) + } + + if cs.ownership.IsShuttingDown() { + t.Fatal("ownership.IsShuttingDown() = true despite continuous renewal, want false") + } + select { + case <-cs.ShutdownRequested(): + t.Fatal("ShutdownRequested() channel closed despite continuous renewal") + default: + } +} + +// TestStopUnownedReaperSafeWithoutStart pins that Stop is a safe no-op when +// the reaper was never started, mirroring the goroutine-ownership +// convention's idempotent-on-both-ends requirement. +func TestStopUnownedReaperSafeWithoutStart(t *testing.T) { + cs := newUnownedReaperTestServer(t) + + done := make(chan struct{}) + go func() { + cs.StopUnownedReaper() + close(done) + }() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("StopUnownedReaper() blocked when the reaper was never started") + } +} + +// TestStopUnownedReaperIsIdempotent pins that calling Stop twice after a real +// Start does not panic or deadlock. +func TestStopUnownedReaperIsIdempotent(t *testing.T) { + cs := newUnownedReaperTestServer(t) + cs.reaperWG.Add(1) + go cs.runUnownedReaper(time.Hour, 5*time.Millisecond) + + cs.StopUnownedReaper() + cs.StopUnownedReaper() +} + +// TestRunUnownedReaperUsesInjectedDecider pins Review round 2 finding 8: the +// reaper loop must call the injected decideUnownedShutdown hook -- the same +// atomic check-and-latch ownership.TryBeginShutdownIfUnownedFor is wired to +// by default -- rather than a hardcoded call to that method. Overriding the +// hook to a call-counting stub that never fires proves the loop actually +// consults it every tick instead of a reintroduced separate +// check-then-latch pair that would leave this package green even with the +// injection point deleted. +func TestRunUnownedReaperUsesInjectedDecider(t *testing.T) { + cs := newUnownedReaperTestServer(t) + + var calls int + var mu sync.Mutex + cs.decideUnownedShutdown = func(time.Duration) bool { + mu.Lock() + defer mu.Unlock() + calls++ + return false + } + + cs.reaperWG.Add(1) + go cs.runUnownedReaper(time.Hour, 5*time.Millisecond) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + mu.Lock() + n := calls + mu.Unlock() + if n >= 3 { + break + } + time.Sleep(5 * time.Millisecond) + } + + mu.Lock() + defer mu.Unlock() + if calls < 3 { + t.Fatalf("decideUnownedShutdown called %d times, want at least 3", calls) + } + if cs.ownership.IsShuttingDown() { + t.Fatal("ownership.IsShuttingDown() = true, want false since the injected decider always returned false") + } +} diff --git a/apps/backend/internal/agentctl/server/api/vscode_proxy.go b/apps/backend/internal/agentctl/server/api/vscode_proxy.go index 9bab1dfbe7..a12d68a45f 100644 --- a/apps/backend/internal/agentctl/server/api/vscode_proxy.go +++ b/apps/backend/internal/agentctl/server/api/vscode_proxy.go @@ -80,5 +80,8 @@ func (s *Server) handleVscodeProxy(c *gin.Context) { } }() - proxy.ServeHTTP(c.Writer, c.Request) + request, releaseFencing := credentialScopedRequest(c) + defer releaseFencing() + + proxy.ServeHTTP(c.Writer, request) } diff --git a/apps/backend/internal/agentctl/server/api/workspace.go b/apps/backend/internal/agentctl/server/api/workspace.go index 84c5a56688..9ddada8ba2 100644 --- a/apps/backend/internal/agentctl/server/api/workspace.go +++ b/apps/backend/internal/agentctl/server/api/workspace.go @@ -92,21 +92,32 @@ func (s *Server) handleWorkspaceStreamWS(c *gin.Context) { } go s.handleWorkspaceStreamInput(stream, shellWriter, done) - s.forwardWorkspaceStream(stream, sub, shellOutputCh, done) + // AC-EXECUTORS-CONTROL-OWNERSHIP-002.2: the invalidation channel comes + // from instanceAuth's context value, captured atomically with this + // request's own accept check -- not a fresh Invalidated() call here, + // which would be a second, independent lock acquisition racing a + // concurrent rotation. + s.forwardWorkspaceStream(stream, sub, shellOutputCh, done, credentialInvalidatedFromContext(c)) } // forwardWorkspaceStream forwards workspace events and shell output to the -// WebSocket until the client goes away or the handler shuts down. +// WebSocket until the client goes away or the handler shuts down. invalidated +// is nil when credentialSource is unset, which never fires in a select -- +// legacy behavior for every existing test constructing a Server without a +// control server alongside it. func (s *Server) forwardWorkspaceStream( stream *workspaceStreamConn, sub types.WorkspaceStreamSubscriber, shellOutputCh chan []byte, done <-chan struct{}, + invalidated <-chan struct{}, ) { for { select { case <-done: return + case <-invalidated: + return case msg, ok := <-sub: if !ok { return diff --git a/apps/backend/internal/agentctl/server/api/workspace_rescan.go b/apps/backend/internal/agentctl/server/api/workspace_rescan.go index 29cf489afd..9f9003e2a0 100644 --- a/apps/backend/internal/agentctl/server/api/workspace_rescan.go +++ b/apps/backend/internal/agentctl/server/api/workspace_rescan.go @@ -11,6 +11,11 @@ import ( // out to satisfy goconst's repeated-string rule across the api package. const errKey = "error" +// instanceNotFoundMessage is the shared 404 body for every instance-scoped +// handler that looks an instance up by ID. Hoisted out to satisfy goconst's +// repeated-string rule across the api package. +const instanceNotFoundMessage = "instance not found" + // RescanWorkspaceRequest is the body for POST /api/v1/workspace/rescan. // // work_dir is optional. When supplied, the manager updates its tracking diff --git a/apps/backend/internal/agentctl/server/config/config.go b/apps/backend/internal/agentctl/server/config/config.go index 56bd1a9755..b46f45d864 100644 --- a/apps/backend/internal/agentctl/server/config/config.go +++ b/apps/backend/internal/agentctl/server/config/config.go @@ -36,6 +36,17 @@ import ( const ( windowsOS = "windows" pathEnvKey = "PATH" + + defaultUnownedPeriod = 10 * time.Minute + defaultDetachedEventLimit = 100 + + // diagnosticLogFileName is the fixed name of agentctl's own detached + // diagnostic log file (AC-EXECUTORS-SURVIVAL-001.3). It lives under + // HomeDir/logs alongside the backend's own "backend-logs.log" and the + // per-session ACP debug logs under logs/acp/ -- a distinct file, so two + // processes never rotate the same file or share one set of rollover + // journals. + diagnosticLogFileName = "agentctl-diagnostic.log" ) // Config is the agentctl configuration. @@ -111,6 +122,50 @@ type Config struct { // OTLPEndpoint is the resolved endpoint used by agentctl transport tracing. OTLPEndpoint string + // UnownedPeriod is how long this control server tolerates having no + // owning backend before it stops its instances and exits. Sourced from + // KANDEV_ACP_UNOWNED_PERIOD (default 10m). Only meaningful when the + // agent-survival capability is active; unused otherwise. + UnownedPeriod time.Duration + + // DetachedEventLimit bounds the per-instance retained-event count while + // this control server has no owning backend attached. Sourced from + // KANDEV_ACP_DETACHED_EVENT_LIMIT (default 100). Only meaningful when the + // agent-survival capability is active; unused otherwise. + DetachedEventLimit int + + // HomeDir is the resolved Kandev root directory this agentctl process was + // started for. It is installation identity: the adopting backend compares + // it against its own resolved home before ever authenticating, per design + // 01's "Ownership identity and credential". Reported on GET /identity. + HomeDir string + + // ServerIdentity is an opaque value generated once per process launch, + // echoed on GET /identity and compared by the adopting backend, never + // logged. It answers "is this the process a previous launch started", + // distinct from AuthToken which answers "is the caller authorized". + ServerIdentity string + + // DiagnosticLogPath is the durable file agentctl's own diagnostic + // output is written to while it may be running detached from any + // backend (AC-EXECUTORS-SURVIVAL-001.3). Derived from HomeDir with a + // fixed filename -- the sink adds no configuration key and no + // environment variable of its own. Empty only when HomeDir itself + // could not be resolved. Echoed on GET /identity so an adopting or + // freshly-spawning backend records it into the control-server record + // without independently re-deriving the same path formula. + DiagnosticLogPath string + + // AgentSurvivalEnabled is whether the agent-survival capability is + // engaged for this launch, copied directly from the managed startup + // contract (never "unresolved" the way UnownedPeriod/DetachedEventLimit + // can be -- a managed launch always states it, and false is the + // legitimate "disabled" answer). An unmanaged/direct launch (Load(), + // no startup contract) always resolves this to false. Gates + // StartUnownedReaper, the detached diagnostic log sink switch, and the + // launcher-side kill-path removal (Layer 5.9). + AgentSurvivalEnabled bool + // mu protects BootstrapNonce from concurrent access during handshake. mu sync.Mutex } @@ -229,6 +284,11 @@ type InstanceConfig struct { // inherited from the server startup contract. NotificationQueueCapacity int + // DetachedEventLimit bounds the per-instance retained-event count + // (AC-EXECUTORS-SURVIVAL-001.6), inherited from the server startup + // contract. It sizes the process manager's updates channel buffer. + DetachedEventLimit int + // SessionID is the session ID for this agent instance (used in MCP tool calls) SessionID string @@ -355,11 +415,28 @@ func load(startup *commonconfig.AgentctlStartupConfig) *Config { idleReaperInterval := getEnvDuration("KANDEV_ACP_IDLE_REAPER_INTERVAL", time.Minute) notificationQueueCapacity := getEnvInt("KANDEV_ACP_NOTIF_QUEUE", 131072) otlpEndpoint := getEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + unownedPeriod := getEnvDuration("KANDEV_ACP_UNOWNED_PERIOD", defaultUnownedPeriod) + detachedEventLimit := getEnvInt("KANDEV_ACP_DETACHED_EVENT_LIMIT", defaultDetachedEventLimit) + homeDir := resolveHomeDir() + agentSurvivalEnabled := false if startup != nil { idleTimeout = startup.IdleTimeout idleReaperInterval = startup.IdleReaperInterval notificationQueueCapacity = startup.NotificationQueueCapacity otlpEndpoint = startup.OTLPEndpoint + // Zero means the caller did not resolve these (an older backend, or + // one built before agent survival existed) — keep the env/built-in + // value already computed above rather than adopting zero. + if startup.UnownedPeriod != 0 { + unownedPeriod = startup.UnownedPeriod + } + if startup.DetachedEventLimit != 0 { + detachedEventLimit = startup.DetachedEventLimit + } + // Unlike the two tunables above, false is a meaningful resolved + // answer here (every managed launch states it), so it is copied + // unconditionally rather than guarded by a zero-value check. + agentSurvivalEnabled = startup.AgentSurvivalEnabled } cfg := &Config{ @@ -387,6 +464,12 @@ func load(startup *commonconfig.AgentctlStartupConfig) *Config { IdleReaperInterval: idleReaperInterval, NotificationQueueCapacity: notificationQueueCapacity, OTLPEndpoint: otlpEndpoint, + UnownedPeriod: unownedPeriod, + DetachedEventLimit: detachedEventLimit, + HomeDir: homeDir, + ServerIdentity: generateSelfToken(), + DiagnosticLogPath: resolveDiagnosticLogPath(homeDir), + AgentSurvivalEnabled: agentSurvivalEnabled, } // Bootstrap nonce mode: agentctl generates its own token and the backend @@ -459,6 +542,37 @@ func (c *Config) ConsumeNonce(nonce string) string { return c.AuthToken } +// resolveHomeDir determines the Kandev root directory this agentctl process +// was started for, honoring KANDEV_HOME_DIR (already the Kandev root) ahead +// of $HOME so dev/e2e isolation and Docker/K8s roots resolve consistently. +// This duplicates adapter/transport/shared.resolveACPLogDir's precedence +// rather than importing it: config is a leaf package with no dependency on +// adapter/transport/shared, and pulling that dependency in for one four-line +// primitive would invert the layering. +func resolveHomeDir() string { + if home := os.Getenv("KANDEV_HOME_DIR"); home != "" { + return home + } + if home, err := os.UserHomeDir(); err == nil && home != "" { + return filepath.Join(home, ".kandev") + } + if cwd, err := os.Getwd(); err == nil { + return cwd + } + return "" +} + +// resolveDiagnosticLogPath derives agentctl's own detached diagnostic log +// path from the resolved home directory (AC-EXECUTORS-SURVIVAL-001.3): a +// fixed filename under HomeDir/logs, not an operator-configurable value. +// Empty when homeDir itself could not be resolved. +func resolveDiagnosticLogPath(homeDir string) string { + if homeDir == "" { + return "" + } + return filepath.Join(homeDir, "logs", diagnosticLogFileName) +} + // generateSelfToken creates a cryptographically random 32-byte hex-encoded token. func generateSelfToken() string { b := make([]byte, 32) @@ -484,6 +598,7 @@ func (c *Config) NewInstanceConfig(port int, overrides *InstanceOverrides) *Inst LogFormat: c.LogFormat, ProcessBufferMaxBytes: c.Defaults.ProcessBufferMaxBytes, NotificationQueueCapacity: c.NotificationQueueCapacity, + DetachedEventLimit: c.DetachedEventLimit, VscodeCommand: c.VscodeCommand, McpMode: "task", AuthToken: c.AuthToken, diff --git a/apps/backend/internal/agentctl/server/config/config_test.go b/apps/backend/internal/agentctl/server/config/config_test.go index c9b3903dc8..92362471c2 100644 --- a/apps/backend/internal/agentctl/server/config/config_test.go +++ b/apps/backend/internal/agentctl/server/config/config_test.go @@ -41,6 +41,49 @@ func TestLoadWithStartupUsesExplicitManagedValues(t *testing.T) { } } +// TestLoadWithStartupPropagatesAgentSurvivalEnabled pins that +// Config.AgentSurvivalEnabled is copied directly from the managed contract in +// both directions -- unlike UnownedPeriod/DetachedEventLimit, false is not +// "unresolved" here (a managed launch always sets Configured=true), so it +// must not be treated as "keep agentctl's own default". +func TestLoadWithStartupPropagatesAgentSurvivalEnabled(t *testing.T) { + base := commonconfig.AgentctlStartupConfig{ + Configured: true, + IdleReaperInterval: time.Minute, + NotificationQueueCapacity: 4096, + } + + enabled := base + enabled.AgentSurvivalEnabled = true + cfg, err := LoadWithStartup(enabled) + if err != nil { + t.Fatalf("LoadWithStartup: %v", err) + } + if !cfg.AgentSurvivalEnabled { + t.Fatal("AgentSurvivalEnabled = false, want true when the startup contract enables it") + } + + disabled := base + disabled.AgentSurvivalEnabled = false + cfg, err = LoadWithStartup(disabled) + if err != nil { + t.Fatalf("LoadWithStartup: %v", err) + } + if cfg.AgentSurvivalEnabled { + t.Fatal("AgentSurvivalEnabled = true, want false when the startup contract disables it") + } +} + +// TestLoadWithoutStartupLeavesAgentSurvivalDisabled pins that a legacy/direct +// (unmanaged) launch -- Load(), no startup contract -- never engages the +// capability, matching AC-EXECUTORS-SURVIVAL-005.2's "defaults disabled". +func TestLoadWithoutStartupLeavesAgentSurvivalDisabled(t *testing.T) { + cfg := Load() + if cfg.AgentSurvivalEnabled { + t.Fatal("AgentSurvivalEnabled = true from Load() with no startup contract, want false") + } +} + func TestNewInstanceConfigNormalizesMcpProviders(t *testing.T) { cfg := (&Config{}).NewInstanceConfig(0, &InstanceOverrides{ McpProviders: []string{" GITLAB ", "unsupported", "github", "github"}, diff --git a/apps/backend/internal/agentctl/server/config/diagnostic_log_test.go b/apps/backend/internal/agentctl/server/config/diagnostic_log_test.go new file mode 100644 index 0000000000..b52918b55c --- /dev/null +++ b/apps/backend/internal/agentctl/server/config/diagnostic_log_test.go @@ -0,0 +1,63 @@ +package config + +import ( + "path/filepath" + "testing" +) + +// TestResolveDiagnosticLogPath pins AC-EXECUTORS-SURVIVAL-001.3: the +// diagnostic log lives under HomeDir/logs with a fixed filename, and is +// empty only when HomeDir itself could not be resolved -- there is no +// separate configuration key or environment variable for this path. +func TestResolveDiagnosticLogPath(t *testing.T) { + tests := []struct { + name string + homeDir string + want string + }{ + { + name: "derives from home dir", + homeDir: "/home/kandev-test/.kandev", + want: filepath.Join("/home/kandev-test/.kandev", "logs", "agentctl-diagnostic.log"), + }, + { + name: "empty home dir yields empty path", + homeDir: "", + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := resolveDiagnosticLogPath(tt.homeDir); got != tt.want { + t.Fatalf("resolveDiagnosticLogPath(%q) = %q, want %q", tt.homeDir, got, tt.want) + } + }) + } +} + +// TestResolveDiagnosticLogPathDiffersFromBackendLogFile pins the "different +// file from the one the backend writes its own logs to" half of +// AC-EXECUTORS-SURVIVAL-001.3 -- two processes rotating one file corrupts +// both, so the filename must never collide with the backend's own +// "backend-logs.log". +func TestResolveDiagnosticLogPathDiffersFromBackendLogFile(t *testing.T) { + got := resolveDiagnosticLogPath("/home/kandev-test/.kandev") + if filepath.Base(got) == "backend-logs.log" { + t.Fatalf("resolveDiagnosticLogPath() = %q, filename collides with the backend's own log file", got) + } +} + +// TestLoadDerivesDiagnosticLogPathFromTheSameHomeDir pins that Config.HomeDir +// and Config.DiagnosticLogPath are derived from a single resolution, so they +// can never disagree about which installation they belong to. +func TestLoadDerivesDiagnosticLogPathFromTheSameHomeDir(t *testing.T) { + cfg := Load() + if cfg.HomeDir == "" { + t.Skip("HomeDir could not be resolved in this environment") + } + want := filepath.Join(cfg.HomeDir, "logs", "agentctl-diagnostic.log") + if cfg.DiagnosticLogPath != want { + t.Fatalf("DiagnosticLogPath = %q, want %q (derived from HomeDir %q)", cfg.DiagnosticLogPath, want, cfg.HomeDir) + } +} diff --git a/apps/backend/internal/agentctl/server/config/identity_test.go b/apps/backend/internal/agentctl/server/config/identity_test.go new file mode 100644 index 0000000000..65a329cdae --- /dev/null +++ b/apps/backend/internal/agentctl/server/config/identity_test.go @@ -0,0 +1,55 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +// TestLoadPopulatesServerIdentity pins that every launch gets a non-empty, +// unique opaque identity value -- the "opaque per-launch value echoed on an +// identity endpoint" design 01 describes -- regardless of whether bootstrap +// auth is configured. +func TestLoadPopulatesServerIdentity(t *testing.T) { + first := load(nil) + second := load(nil) + + if first.ServerIdentity == "" { + t.Fatal("ServerIdentity is empty, want a generated value") + } + if first.ServerIdentity == second.ServerIdentity { + t.Fatal("ServerIdentity is identical across two loads, want distinct per-launch values") + } +} + +// TestLoadResolvesHomeDirFromKandevHomeDirEnv pins that HomeDir honors +// KANDEV_HOME_DIR directly (already the Kandev root), matching +// adapter/transport/shared.resolveACPLogDir's precedence so agentctl reports +// the same installation identity everywhere it resolves the Kandev home. +func TestLoadResolvesHomeDirFromKandevHomeDirEnv(t *testing.T) { + t.Setenv("KANDEV_HOME_DIR", "/custom/kandev/home") + + cfg := load(nil) + + if cfg.HomeDir != "/custom/kandev/home" { + t.Fatalf("HomeDir = %q, want %q", cfg.HomeDir, "/custom/kandev/home") + } +} + +// TestLoadResolvesHomeDirFallsBackToUserHomeDir pins the fallback when +// KANDEV_HOME_DIR is unset: the user's home directory plus ".kandev". +func TestLoadResolvesHomeDirFallsBackToUserHomeDir(t *testing.T) { + t.Setenv("KANDEV_HOME_DIR", "") + + userHome, err := os.UserHomeDir() + if err != nil { + t.Skipf("no user home dir available in this environment: %v", err) + } + + cfg := load(nil) + + want := filepath.Join(userHome, ".kandev") + if cfg.HomeDir != want { + t.Fatalf("HomeDir = %q, want %q", cfg.HomeDir, want) + } +} diff --git a/apps/backend/internal/agentctl/server/instance/instance.go b/apps/backend/internal/agentctl/server/instance/instance.go index bf4125730c..707e302d42 100644 --- a/apps/backend/internal/agentctl/server/instance/instance.go +++ b/apps/backend/internal/agentctl/server/instance/instance.go @@ -16,6 +16,15 @@ import ( type processManager interface { CloseAdmission() StopForTeardown(context.Context) error + // WorkspaceSourceRoots returns the live, current source-root allowlist a + // rescan/rebind may have changed since creation (AC-EXECUTORS-SURVIVAL- + // 002.14's "workspace source roots" reconstruction row). + WorkspaceSourceRoots() []string + // GetSessionID returns the provider's own session identity, held by the + // live adapter (AC-EXECUTORS-SURVIVAL-002.14's "provider session + // identity" reconstruction row). Empty before the agent's session/new or + // session/load completes. + GetSessionID() string } // Instance represents a single agent instance running as a subprocess. @@ -42,6 +51,13 @@ type Instance struct { // CreatedAt is the timestamp when this instance was created CreatedAt time.Time + // SessionID is the task session ID this instance was created for, if any. + // Used to correlate a recovered instance back to its owning session. + SessionID string + + // TaskID is the task ID this instance was created for, if any. + TaskID string + // manager is the process manager handling the agent subprocess (unexported) manager processManager @@ -64,6 +80,11 @@ type Instance struct { stopMu sync.Mutex statusMu sync.RWMutex portReleased bool + + // turnOutcome holds this instance's single retained last-terminal-turn + // slot (AC-EXECUTORS-SURVIVAL-004.1/.6). Zero value is ready to use -- + // no outcome retained. + turnOutcome turnOutcomeState } // MarkActivity stamps the current time as the most recent activity on this @@ -228,6 +249,26 @@ type InstanceInfo struct { // CreatedAt is the timestamp when this instance was created CreatedAt time.Time `json:"created_at"` + + // SessionID is the task session ID this instance was created for, if any. + SessionID string `json:"session_id,omitempty"` + + // TaskID is the task ID this instance was created for, if any. + TaskID string `json:"task_id,omitempty"` + + // WorkspaceSourceRoots is the live, current source-root allowlist this + // instance is enforcing right now -- a rescan or rebind can change it + // after creation, so this always reflects that, not a creation-time + // snapshot (AC-EXECUTORS-SURVIVAL-002.14's "workspace source roots" + // reconstruction row: read back, never pushed). + WorkspaceSourceRoots []string `json:"workspace_source_roots,omitempty"` + + // ProviderSessionID is the live agent CLI's own session identity, held by + // the adapter (AC-EXECUTORS-SURVIVAL-002.14's "provider session identity" + // reconstruction row: the adopted instance, which holds the provider + // session -- never the database). Distinct from SessionID above, which is + // this Kandev task session's identity, not the underlying agent's. + ProviderSessionID string `json:"provider_session_id,omitempty"` } // Info returns a safe copy of the instance data for API serialization. @@ -245,14 +286,25 @@ func (i *Instance) Info() *InstanceInfo { } } + var sourceRoots []string + var providerSessionID string + if i.manager != nil { + sourceRoots = i.manager.WorkspaceSourceRoots() + providerSessionID = i.manager.GetSessionID() + } + return &InstanceInfo{ - ID: i.ID, - Port: i.Port, - Status: i.Status, - WorkspacePath: i.WorkspacePath, - AgentCommand: i.AgentCommand, - Env: envCopy, - CreatedAt: i.CreatedAt, + ID: i.ID, + Port: i.Port, + Status: i.Status, + WorkspacePath: i.WorkspacePath, + AgentCommand: i.AgentCommand, + Env: envCopy, + CreatedAt: i.CreatedAt, + SessionID: i.SessionID, + TaskID: i.TaskID, + WorkspaceSourceRoots: sourceRoots, + ProviderSessionID: providerSessionID, } } diff --git a/apps/backend/internal/agentctl/server/instance/instance_info_test.go b/apps/backend/internal/agentctl/server/instance/instance_info_test.go new file mode 100644 index 0000000000..d11ffb48aa --- /dev/null +++ b/apps/backend/internal/agentctl/server/instance/instance_info_test.go @@ -0,0 +1,57 @@ +package instance + +import ( + "testing" + "time" +) + +// TestInstanceInfoReflectsLiveWorkspaceSourceRootsAndProviderSessionID pins +// AC-EXECUTORS-SURVIVAL-002.14's "workspace source roots" and "provider +// session identity" reconstruction rows: both are read back live from the +// process manager on every Info() call, not snapshotted at creation, so a +// rescan/rebind or a completed session/new is reflected immediately. +func TestInstanceInfoReflectsLiveWorkspaceSourceRootsAndProviderSessionID(t *testing.T) { + inst := &Instance{ + ID: "instance-1", + Port: 5001, + Status: "running", + CreatedAt: time.Now(), + manager: &fakeProcessManager{ + workspaceSourceRoots: []string{"/ws/task-1/backend", "/ws/task-1/frontend"}, + sessionID: "provider-session-9", + }, + } + + info := inst.Info() + + if len(info.WorkspaceSourceRoots) != 2 || + info.WorkspaceSourceRoots[0] != "/ws/task-1/backend" || + info.WorkspaceSourceRoots[1] != "/ws/task-1/frontend" { + t.Errorf("WorkspaceSourceRoots = %v, want the live manager allowlist", info.WorkspaceSourceRoots) + } + if info.ProviderSessionID != "provider-session-9" { + t.Errorf("ProviderSessionID = %q, want provider-session-9", info.ProviderSessionID) + } +} + +// TestInstanceInfoWithoutManagerLeavesRecoveryRowsEmpty pins the nil-manager +// guard: an Instance never wired to a process manager (defensive test-only +// shape) must not panic Info(), and both rows report empty rather than a +// stale or fabricated value. +func TestInstanceInfoWithoutManagerLeavesRecoveryRowsEmpty(t *testing.T) { + inst := &Instance{ + ID: "instance-1", + Port: 5001, + Status: "running", + CreatedAt: time.Now(), + } + + info := inst.Info() + + if info.WorkspaceSourceRoots != nil { + t.Errorf("WorkspaceSourceRoots = %v, want nil without a manager", info.WorkspaceSourceRoots) + } + if info.ProviderSessionID != "" { + t.Errorf("ProviderSessionID = %q, want empty without a manager", info.ProviderSessionID) + } +} diff --git a/apps/backend/internal/agentctl/server/instance/manager.go b/apps/backend/internal/agentctl/server/instance/manager.go index 7a1c033b78..07bb527c6d 100644 --- a/apps/backend/internal/agentctl/server/instance/manager.go +++ b/apps/backend/internal/agentctl/server/instance/manager.go @@ -10,6 +10,7 @@ import ( "os/exec" "strings" "sync" + "sync/atomic" "time" "github.com/google/uuid" @@ -71,6 +72,14 @@ type Manager struct { // racing cancellation against tracker startup and accepting whichever // outcome it happens to get. afterTrackerStart func() + + // turnIDSeq allocates the turn identifiers retained terminal outcomes + // are keyed by (AC-EXECUTORS-SURVIVAL-004.1). It is shared across every + // instance this Manager supervises and lives for this process's whole + // lifetime, matching the AC's "unique across every turn of every + // instance the control server supervises for as long as that control + // server runs" -- deliberately NOT reset per instance. + turnIDSeq atomic.Int64 } // NewManager creates a new instance manager. @@ -203,6 +212,13 @@ func (m *Manager) CreateInstance(ctx context.Context, req *CreateRequest) (*Crea // Create process manager procMgr := process.NewManager(instanceCfg, m.logger) + // Wire retained-outcome recording (AC-EXECUTORS-SURVIVAL-004) before + // anything that could reach Start(): this manager satisfies + // process.TurnOutcomeRecorder via RetainTurnOutcome above, and nothing + // outside this function can start the process manager until + // CreateInstance returns, so setting it here happens-before any + // terminal event the instance could ever produce. + procMgr.SetTurnOutcomeRecorder(id, m) // Materialize provider-qualified comparison targets before any tracker // polling starts. Failures remain explicit unavailable tracker state. procMgr.PrepareComparisonTargets(ctx) @@ -243,6 +259,8 @@ func (m *Manager) CreateInstance(ctx context.Context, req *CreateRequest) (*Crea AgentCommand: agentCmd, Env: req.Env, CreatedAt: time.Now(), + SessionID: req.SessionID, + TaskID: req.TaskID, manager: procMgr, } inst.MarkActivity() diff --git a/apps/backend/internal/agentctl/server/instance/manager_shutdown_test.go b/apps/backend/internal/agentctl/server/instance/manager_shutdown_test.go index 52059896ff..05f5f99715 100644 --- a/apps/backend/internal/agentctl/server/instance/manager_shutdown_test.go +++ b/apps/backend/internal/agentctl/server/instance/manager_shutdown_test.go @@ -314,11 +314,13 @@ func (s *fakeHTTPServer) Close() error { } type fakeProcessManager struct { - stopErr error - stopped bool - stopStarted chan<- struct{} - stopRelease <-chan struct{} - stopStartOnce sync.Once + stopErr error + stopped bool + stopStarted chan<- struct{} + stopRelease <-chan struct{} + stopStartOnce sync.Once + workspaceSourceRoots []string + sessionID string } // legacyResourceReleaseError proves a process teardown error cannot authorize @@ -335,6 +337,10 @@ func (e legacyResourceReleaseError) CanReleaseInstanceResources() bool { func (m *fakeProcessManager) CloseAdmission() {} +func (m *fakeProcessManager) WorkspaceSourceRoots() []string { return m.workspaceSourceRoots } + +func (m *fakeProcessManager) GetSessionID() string { return m.sessionID } + func (m *fakeProcessManager) StopForTeardown(context.Context) error { m.stopped = true if m.stopStarted != nil { diff --git a/apps/backend/internal/agentctl/server/instance/manager_turn_outcome.go b/apps/backend/internal/agentctl/server/instance/manager_turn_outcome.go new file mode 100644 index 0000000000..b986ff871d --- /dev/null +++ b/apps/backend/internal/agentctl/server/instance/manager_turn_outcome.go @@ -0,0 +1,56 @@ +package instance + +import "github.com/kandev/kandev/internal/agentctl/types/streams" + +// RetainTurnOutcome assigns a fresh turn identifier and records event as the +// named instance's new last terminal outcome (AC-EXECUTORS-SURVIVAL-004.1). +// Returns ok=false without allocating an identifier when the instance is not +// found, so a caller racing instance teardown never burns an identifier for +// an outcome nobody can retrieve. +func (m *Manager) RetainTurnOutcome(instanceID string, event streams.AgentEvent) (turnID int64, ok bool) { + inst, found := m.GetInstance(instanceID) + if !found { + return 0, false + } + turnID = m.turnIDSeq.Add(1) + inst.turnOutcome.Retain(turnID, event) + return turnID, true +} + +// PeekTurnOutcome returns the named instance's retained outcome without +// discarding it (AC-EXECUTORS-SURVIVAL-004.6). instanceFound distinguishes +// "no such instance" from "instance exists, nothing retained" so the HTTP +// handler can 404 only on the former. +func (m *Manager) PeekTurnOutcome(instanceID string) (outcome TurnOutcome, hasOutcome, instanceFound bool) { + inst, found := m.GetInstance(instanceID) + if !found { + return TurnOutcome{}, false, false + } + outcome, hasOutcome = inst.turnOutcome.Peek() + return outcome, hasOutcome, true +} + +// AckTurnOutcome discards the named instance's retained outcome only if +// turnID names the one currently retained. Naming any other identifier -- +// including one for an instance that no longer exists -- is accepted and +// changes nothing (AC-EXECUTORS-SURVIVAL-004.6), so a retried acknowledgement +// is always safe to call again. +func (m *Manager) AckTurnOutcome(instanceID string, turnID int64) { + inst, found := m.GetInstance(instanceID) + if !found { + return + } + inst.turnOutcome.Ack(turnID) +} + +// ClearTurnOutcome retires the prior terminal outcome before a new prompt is +// accepted. The prompt generation floor prevents a terminal event from an +// older turn, still waiting in the process manager's output path, from +// repopulating the slot after it was cleared. +func (m *Manager) ClearTurnOutcome(instanceID string, promptGeneration uint64) { + inst, found := m.GetInstance(instanceID) + if !found { + return + } + inst.turnOutcome.Clear(promptGeneration) +} diff --git a/apps/backend/internal/agentctl/server/instance/manager_turn_outcome_wiring_test.go b/apps/backend/internal/agentctl/server/instance/manager_turn_outcome_wiring_test.go new file mode 100644 index 0000000000..825d45a333 --- /dev/null +++ b/apps/backend/internal/agentctl/server/instance/manager_turn_outcome_wiring_test.go @@ -0,0 +1,70 @@ +package instance + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/pkg/agent" +) + +// TestCreateInstanceWiresTurnOutcomeRecorder pins the end-to-end wiring +// CreateInstance is responsible for (AC-EXECUTORS-SURVIVAL-004.1): a real +// terminal event sent through the process manager it constructs must reach +// this instance's retained-outcome slot, peekable via PeekTurnOutcome. Only +// exported process.Manager API is used (SendErrorEvent) so this exercises +// the production path -- forwardUpdates/sendUpdateBlocking calling back into +// recordTerminalOutcome -- rather than reaching into unexported internals. +func TestCreateInstanceWiresTurnOutcomeRecorder(t *testing.T) { + log := newTestLogger(t) + mgr := NewManager(&config.Config{ + Ports: config.PortConfig{Base: 0, Max: 0}, + Defaults: config.InstanceDefaults{Protocol: agent.ProtocolACP}, + }, log) + t.Cleanup(func() { _ = mgr.Shutdown(context.Background()) }) + + var captured *process.Manager + mgr.SetServerFactory(func(cfg *config.InstanceConfig, procMgr *process.Manager, log *logger.Logger) http.Handler { + captured = procMgr + return http.NotFoundHandler() + }) + + resp, err := mgr.CreateInstance(context.Background(), &CreateRequest{WorkspacePath: t.TempDir()}) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + t.Cleanup(func() { _ = mgr.StopInstance(context.Background(), resp.ID) }) + + if captured == nil { + t.Fatal("server factory was never invoked; no process.Manager captured") + } + + if _, hasOutcome, found := mgr.PeekTurnOutcome(resp.ID); !found || hasOutcome { + t.Fatalf("hasOutcome = %v, found = %v before any terminal event, want found=true hasOutcome=false", hasOutcome, found) + } + + captured.SendErrorEvent("boom", 1) + + deadline := time.After(time.Second) + for { + outcome, hasOutcome, found := mgr.PeekTurnOutcome(resp.ID) + if !found { + t.Fatal("instance not found while polling for the retained outcome") + } + if hasOutcome { + if outcome.Event.Error != "boom" { + t.Fatalf("retained outcome error = %q, want boom", outcome.Event.Error) + } + return + } + select { + case <-deadline: + t.Fatal("timed out waiting for SendErrorEvent to reach the retained outcome slot") + case <-time.After(time.Millisecond): + } + } +} diff --git a/apps/backend/internal/agentctl/server/instance/turn_outcome.go b/apps/backend/internal/agentctl/server/instance/turn_outcome.go new file mode 100644 index 0000000000..16b01c78ab --- /dev/null +++ b/apps/backend/internal/agentctl/server/instance/turn_outcome.go @@ -0,0 +1,81 @@ +package instance + +import ( + "sync" + + "github.com/kandev/kandev/internal/agentctl/types/streams" +) + +// TurnOutcome pairs a terminal agent event with the identifier assigned to +// the turn it belongs to (AC-EXECUTORS-SURVIVAL-004.1). The identifier is +// scoped to this control server's own process lifetime -- it is not the +// durable, cross-restart Kandev turn identity the rest of the backend +// already tracks (streams.AgentEvent.TurnID). Reusing that value would tie +// this retention mechanism to identifiers that already predate any given +// control-server launch, when the AC explicitly only requires uniqueness +// "for as long as that control server runs". +type TurnOutcome struct { + TurnID int64 `json:"turn_id"` + Event streams.AgentEvent `json:"event"` +} + +// turnOutcomeState holds the single retained outcome slot for one instance. +// Only the LAST terminal outcome is kept (design 03: "agentctl therefore +// retains each instance's last terminal turn outcome") -- retaining a +// history is unnecessary because a session can only be in one terminal +// state at re-track time, and design 03 explicitly leaves applying that one +// state as the whole job of AC-EXECUTORS-SURVIVAL-004.2. +type turnOutcomeState struct { + mu sync.Mutex + retained *TurnOutcome + minimumPromptGeneration uint64 +} + +// Retain records event as the instance's new last terminal outcome under +// the given turnID, replacing whatever was retained before. Retrieval never +// clears this state (AC-EXECUTORS-SURVIVAL-004.6) -- only Ack does. +func (s *turnOutcomeState) Retain(turnID int64, event streams.AgentEvent) { + s.mu.Lock() + defer s.mu.Unlock() + if event.PromptGeneration != 0 && event.PromptGeneration < s.minimumPromptGeneration { + return + } + s.retained = &TurnOutcome{TurnID: turnID, Event: event} +} + +// Peek returns the retained outcome, if any, without discarding it. Calling +// it repeatedly returns the same outcome and turn identifier until Ack +// removes it (AC-EXECUTORS-SURVIVAL-004.6's repeatable, non-discarding read). +func (s *turnOutcomeState) Peek() (TurnOutcome, bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.retained == nil { + return TurnOutcome{}, false + } + return *s.retained, true +} + +// Ack discards the retained outcome only when turnID names the one +// currently retained. Naming an identifier that is not (or no longer) +// retained is accepted and changes nothing, so a retried acknowledgement -- +// or one that arrives after a newer outcome has already replaced it -- is +// always safe (AC-EXECUTORS-SURVIVAL-004.6). +func (s *turnOutcomeState) Ack(turnID int64) { + s.mu.Lock() + defer s.mu.Unlock() + if s.retained != nil && s.retained.TurnID == turnID { + s.retained = nil + } +} + +// Clear retires the current outcome and records the generation of the prompt +// that follows it. Terminal events from older generations are ignored when +// they arrive late through the process manager's output path. +func (s *turnOutcomeState) Clear(promptGeneration uint64) { + s.mu.Lock() + defer s.mu.Unlock() + s.retained = nil + if promptGeneration > s.minimumPromptGeneration { + s.minimumPromptGeneration = promptGeneration + } +} diff --git a/apps/backend/internal/agentctl/server/instance/turn_outcome_test.go b/apps/backend/internal/agentctl/server/instance/turn_outcome_test.go new file mode 100644 index 0000000000..dfe1911dca --- /dev/null +++ b/apps/backend/internal/agentctl/server/instance/turn_outcome_test.go @@ -0,0 +1,199 @@ +package instance + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/agentctl/server/config" + "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/pkg/agent" +) + +func newTurnOutcomeTestManager(t *testing.T) *Manager { + t.Helper() + log := newTestLogger(t) + cfg := &config.Config{ + Ports: config.PortConfig{Base: 0, Max: 0}, + Defaults: config.InstanceDefaults{Protocol: agent.ProtocolACP}, + } + mgr := NewManager(cfg, log) + t.Cleanup(func() { _ = mgr.Shutdown(context.Background()) }) + return mgr +} + +func addTestInstance(t *testing.T, mgr *Manager, id string) *Instance { + t.Helper() + inst := &Instance{ID: id} + mgr.mu.Lock() + mgr.instances[id] = inst + mgr.mu.Unlock() + return inst +} + +// TestRetainTurnOutcomeReturnsFalseForUnknownInstance pins that no turn +// identifier is allocated for an instance that does not exist -- a caller +// racing instance teardown should never burn an identifier nobody can +// retrieve. +func TestRetainTurnOutcomeReturnsFalseForUnknownInstance(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + + turnID, ok := mgr.RetainTurnOutcome("missing", streams.AgentEvent{Type: streams.EventTypeComplete}) + if ok { + t.Fatal("RetainTurnOutcome() ok = true for an unknown instance, want false") + } + if turnID != 0 { + t.Fatalf("RetainTurnOutcome() turnID = %d for an unknown instance, want 0", turnID) + } +} + +// TestPeekTurnOutcomeIsRepeatableAndNonDiscarding pins +// AC-EXECUTORS-SURVIVAL-004.6: retrieval never clears the retained outcome, +// so repeated reads return the identical outcome and turn identifier. +func TestPeekTurnOutcomeIsRepeatableAndNonDiscarding(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + addTestInstance(t, mgr, "inst-1") + + turnID, ok := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{Type: streams.EventTypeComplete, SessionID: "sess-1"}) + if !ok { + t.Fatal("RetainTurnOutcome() ok = false, want true") + } + + for i := 0; i < 3; i++ { + outcome, hasOutcome, instanceFound := mgr.PeekTurnOutcome("inst-1") + if !instanceFound { + t.Fatalf("iteration %d: instanceFound = false, want true", i) + } + if !hasOutcome { + t.Fatalf("iteration %d: hasOutcome = false, want true (retrieval must not discard)", i) + } + if outcome.TurnID != turnID { + t.Fatalf("iteration %d: TurnID = %d, want %d", i, outcome.TurnID, turnID) + } + if outcome.Event.SessionID != "sess-1" { + t.Fatalf("iteration %d: Event.SessionID = %q, want %q", i, outcome.Event.SessionID, "sess-1") + } + } +} + +// TestPeekTurnOutcomeDistinguishesUnknownInstanceFromNothingRetained pins +// that a 404-worthy "no such instance" is distinguishable from the normal +// "instance exists, nothing retained yet" answer. +func TestPeekTurnOutcomeDistinguishesUnknownInstanceFromNothingRetained(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + addTestInstance(t, mgr, "inst-1") + + _, hasOutcome, instanceFound := mgr.PeekTurnOutcome("inst-1") + if !instanceFound { + t.Fatal("instanceFound = false for a real instance, want true") + } + if hasOutcome { + t.Fatal("hasOutcome = true before anything was retained, want false") + } + + _, _, instanceFound = mgr.PeekTurnOutcome("missing") + if instanceFound { + t.Fatal("instanceFound = true for an unknown instance, want false") + } +} + +// TestAckTurnOutcomeDiscardsOnlyTheMatchingIdentifier pins the core of +// AC-EXECUTORS-SURVIVAL-004.6: acking the wrong identifier changes nothing, +// while acking the right one discards it. +func TestAckTurnOutcomeDiscardsOnlyTheMatchingIdentifier(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + addTestInstance(t, mgr, "inst-1") + + turnID, _ := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{Type: streams.EventTypeComplete}) + + mgr.AckTurnOutcome("inst-1", turnID+999) + if _, hasOutcome, _ := mgr.PeekTurnOutcome("inst-1"); !hasOutcome { + t.Fatal("acking the wrong turn ID discarded the outcome, want it left retained") + } + + mgr.AckTurnOutcome("inst-1", turnID) + if _, hasOutcome, _ := mgr.PeekTurnOutcome("inst-1"); hasOutcome { + t.Fatal("acking the matching turn ID left the outcome retained, want it discarded") + } +} + +// TestAckTurnOutcomeOnUnknownInstanceIsANoOp pins that acking an instance +// that no longer exists (e.g. already torn down) never panics and never +// errors -- an unknown identifier is always accepted and does nothing. +func TestAckTurnOutcomeOnUnknownInstanceIsANoOp(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + mgr.AckTurnOutcome("missing", 1) +} + +func TestClearTurnOutcomeRetiresPriorOutcomeAndRejectsOlderLateEvents(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + addTestInstance(t, mgr, "inst-1") + firstID, _ := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{ + Type: streams.EventTypeComplete, + PromptGeneration: 1, + }) + if firstID == 0 { + t.Fatal("first retained outcome did not receive an identifier") + } + + mgr.ClearTurnOutcome("inst-1", 2) + if _, hasOutcome, _ := mgr.PeekTurnOutcome("inst-1"); hasOutcome { + t.Fatal("ClearTurnOutcome left the prior terminal outcome retained") + } + + mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{ + Type: streams.EventTypeError, + PromptGeneration: 1, + }) + if _, hasOutcome, _ := mgr.PeekTurnOutcome("inst-1"); hasOutcome { + t.Fatal("late terminal event from the prior generation repopulated the slot") + } + + mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{ + Type: streams.EventTypeComplete, + PromptGeneration: 2, + }) + outcome, hasOutcome, _ := mgr.PeekTurnOutcome("inst-1") + if !hasOutcome || outcome.Event.PromptGeneration != 2 { + t.Fatalf("current-generation outcome = %+v, retained = %v; want generation 2", outcome, hasOutcome) + } +} + +// TestRetainTurnOutcomeAllocatesStrictlyIncreasingIdentifiersAcrossInstances +// pins AC-EXECUTORS-SURVIVAL-004.1: the identifier is unique across every +// turn of every instance this control server supervises, not just within +// one instance. +func TestRetainTurnOutcomeAllocatesStrictlyIncreasingIdentifiersAcrossInstances(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + addTestInstance(t, mgr, "inst-1") + addTestInstance(t, mgr, "inst-2") + + id1, _ := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{Type: streams.EventTypeComplete}) + id2, _ := mgr.RetainTurnOutcome("inst-2", streams.AgentEvent{Type: streams.EventTypeComplete}) + id3, _ := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{Type: streams.EventTypeError}) + + if id1 == id2 || id2 == id3 || id1 == id3 { + t.Fatalf("turn identifiers were not unique: id1=%d id2=%d id3=%d", id1, id2, id3) + } +} + +// TestRetainTurnOutcomeReplacesThePreviouslyRetainedOutcome pins design 03's +// "agentctl therefore retains each instance's last terminal turn outcome" -- +// only the most recent terminal outcome is kept, not a history. +func TestRetainTurnOutcomeReplacesThePreviouslyRetainedOutcome(t *testing.T) { + mgr := newTurnOutcomeTestManager(t) + addTestInstance(t, mgr, "inst-1") + + firstID, _ := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{Type: streams.EventTypeError}) + secondID, _ := mgr.RetainTurnOutcome("inst-1", streams.AgentEvent{Type: streams.EventTypeComplete}) + + outcome, hasOutcome, _ := mgr.PeekTurnOutcome("inst-1") + if !hasOutcome { + t.Fatal("hasOutcome = false, want true") + } + if outcome.TurnID != secondID { + t.Fatalf("TurnID = %d, want the second (most recent) outcome's ID %d (first was %d)", outcome.TurnID, secondID, firstID) + } + if outcome.Event.Type != streams.EventTypeComplete { + t.Fatalf("Event.Type = %q, want the most recent outcome's type %q", outcome.Event.Type, streams.EventTypeComplete) + } +} diff --git a/apps/backend/internal/agentctl/server/process/attachment.go b/apps/backend/internal/agentctl/server/process/attachment.go new file mode 100644 index 0000000000..5ae8df4ba6 --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/attachment.go @@ -0,0 +1,34 @@ +package process + +// attachedCount tracks how many live backend event-stream connections +// (/agent/stream) this instance currently has. It is a count rather than a +// single flag so an overlapping reconnect -- a new connection arriving +// before the old one's teardown has run -- cannot make IsAttached report +// false while a connection is still actually live. +// +// This is the agentctl-local "instance is attached" signal design 03 +// requires: an instance is attached when its event stream to the owning +// backend is established, evaluated at INSTANCE granularity (not "the +// server has an owning backend", which is true during the window after +// adoption and before this instance's own stream reconnects -- exactly the +// window a permission request must not be auto-denied in). + +// MarkAttached records that a backend event-stream connection is now live +// for this instance. Called by the /agent/stream handler once the +// WebSocket upgrade succeeds. +func (m *Manager) MarkAttached() { + m.attachedCount.Add(1) +} + +// MarkDetached records that a backend event-stream connection for this +// instance has ended. Called by the /agent/stream handler when the +// connection closes, for symmetry with MarkAttached. +func (m *Manager) MarkDetached() { + m.attachedCount.Add(-1) +} + +// IsAttached reports whether this instance currently has at least one live +// backend event-stream connection. +func (m *Manager) IsAttached() bool { + return m.attachedCount.Load() > 0 +} diff --git a/apps/backend/internal/agentctl/server/process/attachment_test.go b/apps/backend/internal/agentctl/server/process/attachment_test.go new file mode 100644 index 0000000000..218127cdf2 --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/attachment_test.go @@ -0,0 +1,51 @@ +package process + +import "testing" + +// TestManagerStartsDetached pins that a freshly constructed Manager has no +// attached backend event stream yet -- matching design 03's "the coarser +// [server-owned] granularity would auto-deny exactly [in the window] before +// this instance's streams are up" caveat: a permission request in that +// window must be treated as detached, not attached. +func TestManagerStartsDetached(t *testing.T) { + m := &Manager{} + if m.IsAttached() { + t.Fatal("IsAttached() = true for a freshly constructed Manager, want false") + } +} + +// TestMarkAttachedAndMarkDetachedToggleIsAttached pins the basic connect/ +// disconnect transition the WS stream handler drives. +func TestMarkAttachedAndMarkDetachedToggleIsAttached(t *testing.T) { + m := &Manager{} + m.MarkAttached() + if !m.IsAttached() { + t.Fatal("IsAttached() = false after MarkAttached, want true") + } + m.MarkDetached() + if m.IsAttached() { + t.Fatal("IsAttached() = true after MarkDetached, want false") + } +} + +// TestIsAttachedStaysTrueAcrossAnOverlappingReconnect pins that a second, +// overlapping stream connection (e.g. a reconnect racing the old +// connection's teardown) does not make the instance look detached the +// moment the FIRST connection's defer runs -- attachment is a count, not a +// single flag, so it only reaches zero once every live connection has +// disconnected. +func TestIsAttachedStaysTrueAcrossAnOverlappingReconnect(t *testing.T) { + m := &Manager{} + m.MarkAttached() // first connection + m.MarkAttached() // second, overlapping connection + + m.MarkDetached() // first connection's defer fires + if !m.IsAttached() { + t.Fatal("IsAttached() = false with one connection still live, want true") + } + + m.MarkDetached() // second connection's defer fires + if m.IsAttached() { + t.Fatal("IsAttached() = true after every connection disconnected, want false") + } +} diff --git a/apps/backend/internal/agentctl/server/process/blocking_send.go b/apps/backend/internal/agentctl/server/process/blocking_send.go new file mode 100644 index 0000000000..d754728419 --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/blocking_send.go @@ -0,0 +1,40 @@ +package process + +import "github.com/kandev/kandev/internal/agentctl/server/adapter" + +// sendUpdateBlocking sends event on the updates channel, parking the caller +// while the channel is full rather than discarding the event +// (AC-EXECUTORS-SURVIVAL-001.5/.6). The park selects against the instance's +// current stop signal, mirroring forwardUpdates's own send: stopping the +// instance releases every producer parked on it immediately, without +// waiting for a backend that may never attach. Returns false when the +// instance stopped (or never started) before the event could be delivered. +// +// The fast path never depends on a stop channel: when there is room, the +// send just happens, matching the un-parked behaviour every one of these +// call sites had before conversion. Only a send that would actually block +// needs a stop channel to select against; one that was never established +// (Start has not run) is treated as already stopped rather than parking +// forever with no lifecycle that could ever release it. +func (m *Manager) sendUpdateBlocking(event adapter.AgentEvent) bool { + m.recordTerminalOutcome(&event) + + select { + case m.updatesCh <- event: + return true + default: + } + + stopChVal := m.stopChSnapshot.Load() + if stopChVal == nil { + return false + } + stopCh := stopChVal.(chan struct{}) + + select { + case m.updatesCh <- event: + return true + case <-stopCh: + return false + } +} diff --git a/apps/backend/internal/agentctl/server/process/blocking_send_sites_test.go b/apps/backend/internal/agentctl/server/process/blocking_send_sites_test.go new file mode 100644 index 0000000000..c04e4f2745 --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/blocking_send_sites_test.go @@ -0,0 +1,324 @@ +package process + +import ( + "context" + "os" + "os/exec" + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/server/adapter" +) + +// parkThenAssertRelease is the shared shape for the three "must park, not +// discard" site tests below: fill updatesCh, run the site's send in a +// goroutine, confirm it has not returned while the channel stays full, then +// release it (by draining the channel or closing stopCh) and confirm the +// goroutine finishes promptly afterward. +func parkThenAssertRelease(t *testing.T, run func(), release func()) { + t.Helper() + done := make(chan struct{}) + go func() { + run() + close(done) + }() + + select { + case <-done: + t.Fatal("send returned before the channel had room or a stop was signalled -- it must park, not discard") + case <-time.After(50 * time.Millisecond): + } + + release() + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("send did not return promptly after release") + } +} + +// TestSendErrorEventWithProviderErrorParksOnAFullChannel pins +// AC-EXECUTORS-SURVIVAL-001.5/.6 for the agent-ERROR site: a full channel +// must park the caller rather than silently drop the error, and must +// release once a consumer drains a slot. +func TestSendErrorEventWithProviderErrorParksOnAFullChannel(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + m.stopChSnapshot.Store(make(chan struct{})) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + + parkThenAssertRelease(t, + func() { m.SendErrorEventWithProviderError("boom", 1, nil) }, + func() { <-m.updatesCh }, + ) + + select { + case event := <-m.updatesCh: + if event.Error != "boom" { + t.Fatalf("Error = %q, want %q", event.Error, "boom") + } + default: + t.Fatal("the parked error event was never delivered") + } +} + +// TestSendErrorEventWithProviderErrorReleasesOnStop pins the pause-not-a-hang +// half: a parked send must release the instant stop fires, without ever +// delivering the event. +func TestSendErrorEventWithProviderErrorReleasesOnStop(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + stopCh := make(chan struct{}) + m.stopChSnapshot.Store(stopCh) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} + + parkThenAssertRelease(t, + func() { m.SendErrorEventWithProviderError("boom", 1, nil) }, + func() { close(stopCh) }, + ) +} + +// TestSendErrorEventWithProviderErrorReleasesOnRealManagerStop pins the same +// release guarantee driven through the real Manager.Stop path, not a +// hand-closed stopCh standing in for it: a full channel must still park a +// send in flight when Stop is called on a started manager, and Stop's own +// teardown (closeAdapterAndStdin closing stopCh) must release it and return +// within a bounded time, not hang waiting on the parked goroutine. +func TestSendErrorEventWithProviderErrorReleasesOnRealManagerStop(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + m.stopCh = make(chan struct{}) + m.stopChSnapshot.Store(m.stopCh) + m.status.Store(StatusRunning) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} + + parkThenAssertRelease(t, + func() { m.SendErrorEventWithProviderError("boom", 1, nil) }, + func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := m.Stop(ctx); err != nil { + t.Errorf("Stop: %v", err) + } + }, + ) +} + +// TestSendPermissionCancelledNotificationParksOnAFullChannel pins +// AC-EXECUTORS-SURVIVAL-001.5/.6 for the permission-cancelled site. +func TestSendPermissionCancelledNotificationParksOnAFullChannel(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + m.stopChSnapshot.Store(make(chan struct{})) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} + pending := &PendingPermission{ID: "pending-1", RequestID: "req-1", Request: &adapter.PermissionRequest{}} + + parkThenAssertRelease(t, + func() { m.sendPermissionCancelledNotification(pending) }, + func() { <-m.updatesCh }, + ) + + select { + case event := <-m.updatesCh: + if event.Type != adapter.EventTypePermissionCancelled || event.PendingID != "pending-1" { + t.Fatalf("event = %+v, want a permission-cancelled event for pending-1", event) + } + default: + t.Fatal("the parked permission-cancelled event was never delivered") + } +} + +// newPendingPermissionForNotificationTest builds a PendingPermission with a +// populated Snapshot, matching the shape sendPermissionNotification reads. +func newPendingPermissionForNotificationTest(m *Manager) *PendingPermission { + pending := &PendingPermission{ + ID: "pending-1", + RequestID: "req-1", + Request: &adapter.PermissionRequest{ + SessionID: "session-1", + ToolCallID: "tool-1", + Title: "Run command", + Options: []adapter.PermissionOption{ + {OptionID: "allow-once", Kind: "allow_once"}, + }, + }, + ResponseCh: make(chan *adapter.PermissionResponse, 1), + } + pending.Snapshot = m.permissionSnapshot(pending) + return pending +} + +// TestSendPermissionNotificationParksWhenDetachedOnAFullChannel pins the +// behaviour change design 03 requires for the eighth COVERED site: while +// DETACHED, a full channel must park the permission-request notification +// rather than auto-cancelling after five seconds -- auto-cancelling while +// detached would silently deny every permission the agent asks for. +func TestSendPermissionNotificationParksWhenDetachedOnAFullChannel(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + m.stopChSnapshot.Store(make(chan struct{})) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + pending := newPendingPermissionForNotificationTest(m) + + parkThenAssertRelease(t, + func() { m.sendPermissionNotification(pending) }, + func() { <-m.updatesCh }, + ) + + select { + case event := <-m.updatesCh: + if event.Type != adapter.EventTypePermissionRequest || event.PendingID != "pending-1" { + t.Fatalf("event = %+v, want a permission-request event for pending-1", event) + } + default: + t.Fatal("the parked permission-request event was never delivered") + } + select { + case resp := <-pending.ResponseCh: + t.Fatalf("ResponseCh received %+v, want no auto-cancel while parked", resp) + default: + } +} + +// TestSendPermissionNotificationReleasesOnStopWithoutAutoCancellingWhenDetached +// pins the "pause not a hang" half for the detached branch: a parked +// permission-request notification releases on instance stop like every +// other COVERED site, and -- unlike the attached five-second timer path -- +// does not synthesize a Cancelled response, since nobody is waiting to be +// denied and the instance is being torn down anyway. +func TestSendPermissionNotificationReleasesOnStopWithoutAutoCancellingWhenDetached(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + stopCh := make(chan struct{}) + m.stopChSnapshot.Store(stopCh) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} + pending := newPendingPermissionForNotificationTest(m) + + parkThenAssertRelease(t, + func() { m.sendPermissionNotification(pending) }, + func() { close(stopCh) }, + ) + + select { + case resp := <-pending.ResponseCh: + t.Fatalf("ResponseCh received %+v, want no auto-cancel on a stop release", resp) + default: + } +} + +// TestSendPermissionNotificationSendsImmediatelyWhenAttachedAndRoomAvailable +// pins that the attached branch is unchanged for the common case: with room +// on the channel, the event is delivered the same way it always was, +// regardless of the new attached/detached gate. +func TestSendPermissionNotificationSendsImmediatelyWhenAttachedAndRoomAvailable(t *testing.T) { + m := &Manager{ + updatesCh: make(chan adapter.AgentEvent, 1), + logger: newTestLogger(t), + } + m.MarkAttached() + pending := newPendingPermissionForNotificationTest(m) + + m.sendPermissionNotification(pending) + + select { + case event := <-m.updatesCh: + if event.Type != adapter.EventTypePermissionRequest || event.PendingID != "pending-1" { + t.Fatalf("event = %+v, want a permission-request event for pending-1", event) + } + default: + t.Fatal("attached send with room available did not deliver the event") + } +} + +// TestManagerFailingProcessHelper is a self-exec subprocess helper that +// exits with a nonzero status, used by +// TestWaitForExitParksTheExitErrorEventOnAFullChannel below to drive +// waitForExit's real error branch through a real process exit rather than a +// fabricated exec.Cmd state. +func TestManagerFailingProcessHelper(t *testing.T) { + if os.Getenv("KANDEV_MANAGER_FAILING_PROCESS_HELPER") != "1" { + return + } + os.Exit(3) +} + +// TestWaitForExitParksTheExitErrorEventOnAFullChannel pins +// AC-EXECUTORS-SURVIVAL-001.5/.6 for the agent-process-exit-error site: the +// exit error event carrying exit code and recent stderr must park on a full +// channel rather than being dropped, since it is the only report that the +// agent process died at all. +func TestWaitForExitParksTheExitErrorEventOnAFullChannel(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=TestManagerFailingProcessHelper") + cmd.Env = append(os.Environ(), "KANDEV_MANAGER_FAILING_PROCESS_HELPER=1") + stderr, stderrWriter, err := os.Pipe() + if err != nil { + t.Fatalf("stderr pipe: %v", err) + } + cmd.Stderr = stderrWriter + m := &Manager{ + cmd: cmd, + stderr: stderr, + logger: newTestLogger(t), + doneCh: make(chan struct{}), + updatesCh: make(chan adapter.AgentEvent, 1), + groupAliveFn: func(int) bool { return false }, + } + m.status.Store(StatusRunning) + // sendUpdateBlocking treats an unestablished stopChSnapshot as "already + // stopped" (see its doc comment) so a genuinely never-started Manager + // doesn't park forever. Since this test builds a Manager by struct + // literal instead of through Start(), it must seed the snapshot itself + // the same way Start() does, or the send below would return immediately + // on a fast cmd.Wait() instead of parking -- a real flake this test hit + // before the snapshot was added here. + m.stopChSnapshot.Store(make(chan struct{})) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + t.Cleanup(func() { + if cmd.ProcessState == nil && cmd.Process != nil { + _ = cmd.Process.Kill() + } + _ = stderrWriter.Close() + _ = stderr.Close() + m.wg.Wait() + }) + if err := cmd.Start(); err != nil { + t.Fatalf("start process: %v", err) + } + if err := stderrWriter.Close(); err != nil { + t.Fatalf("close parent stderr pipe: %v", err) + } + + stderrDone := make(chan struct{}) + m.wg.Add(2) + go m.readStderr(stderrDone) + + parkThenAssertRelease(t, + func() { m.waitForExit(stderrDone) }, + func() { <-m.updatesCh }, + ) + + select { + case event := <-m.updatesCh: + if event.Type != adapter.EventTypeError { + t.Fatalf("event type = %q, want error", event.Type) + } + if data, ok := event.Data["exit_code"]; !ok || data != 3 { + t.Fatalf("exit_code = %v, want 3", data) + } + default: + t.Fatal("the parked exit-error event was never delivered") + } +} diff --git a/apps/backend/internal/agentctl/server/process/blocking_send_test.go b/apps/backend/internal/agentctl/server/process/blocking_send_test.go new file mode 100644 index 0000000000..769978b1aa --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/blocking_send_test.go @@ -0,0 +1,154 @@ +package process + +import ( + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/server/adapter" +) + +// TestSendUpdateBlockingSendsImmediatelyWhenChannelHasRoom pins the common +// case: a converted send behaves exactly like the non-blocking one did +// whenever the channel isn't actually full. +func TestSendUpdateBlockingSendsImmediatelyWhenChannelHasRoom(t *testing.T) { + m := &Manager{ + logger: newTestLogger(t), + updatesCh: make(chan adapter.AgentEvent, 1), + } + stopCh := make(chan struct{}) + m.stopChSnapshot.Store(stopCh) + + if ok := m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError, Error: "boom"}); !ok { + t.Fatal("sendUpdateBlocking() = false, want true (channel had room)") + } + + select { + case event := <-m.updatesCh: + if event.Error != "boom" { + t.Fatalf("Error = %q, want %q", event.Error, "boom") + } + default: + t.Fatal("event was not enqueued on updatesCh") + } +} + +// TestSendUpdateBlockingParksUntilRoomFreesUp pins AC-EXECUTORS-SURVIVAL-001.5/.6: +// a full channel must PARK the producer rather than discard the event, and +// deliver it once a consumer drains the channel. +func TestSendUpdateBlockingParksUntilRoomFreesUp(t *testing.T) { + m := &Manager{ + logger: newTestLogger(t), + updatesCh: make(chan adapter.AgentEvent, 1), + } + stopCh := make(chan struct{}) + m.stopChSnapshot.Store(stopCh) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + + done := make(chan bool, 1) + go func() { + done <- m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError, Error: "parked"}) + }() + + select { + case <-done: + t.Fatal("sendUpdateBlocking returned before the channel had room -- it must park, not discard") + case <-time.After(50 * time.Millisecond): + } + + <-m.updatesCh // drain the blocking event, freeing a slot + + select { + case ok := <-done: + if !ok { + t.Fatal("sendUpdateBlocking() = false once room freed up, want true") + } + case <-time.After(time.Second): + t.Fatal("sendUpdateBlocking did not unblock after the channel drained") + } + + select { + case event := <-m.updatesCh: + if event.Error != "parked" { + t.Fatalf("Error = %q, want %q", event.Error, "parked") + } + default: + t.Fatal("the parked event was never delivered") + } +} + +// TestSendUpdateBlockingReleasesOnStop pins the "pause not a hang" half of +// the design: a producer parked on a full channel must be released the +// instant the instance's stop signal fires, without waiting for a consumer +// that may never arrive. +func TestSendUpdateBlockingReleasesOnStop(t *testing.T) { + m := &Manager{ + logger: newTestLogger(t), + updatesCh: make(chan adapter.AgentEvent, 1), + } + stopCh := make(chan struct{}) + m.stopChSnapshot.Store(stopCh) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + + done := make(chan bool, 1) + go func() { + done <- m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError, Error: "released"}) + }() + + select { + case <-done: + t.Fatal("sendUpdateBlocking returned before stop was signalled") + case <-time.After(50 * time.Millisecond): + } + + close(stopCh) + + select { + case ok := <-done: + if ok { + t.Fatal("sendUpdateBlocking() = true after a stop release, want false (event must not be delivered)") + } + case <-time.After(time.Second): + t.Fatal("sendUpdateBlocking did not release on stop") + } +} + +// TestSendUpdateBlockingSendsWithRoomEvenWithNoStopChannelEstablished pins +// that the fast, common path never depends on a stop channel existing at +// all: when there's room, the send just happens. +func TestSendUpdateBlockingSendsWithRoomEvenWithNoStopChannelEstablished(t *testing.T) { + m := &Manager{ + logger: newTestLogger(t), + updatesCh: make(chan adapter.AgentEvent, 1), + } + + if ok := m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError}); !ok { + t.Fatal("sendUpdateBlocking() = false with room available, want true regardless of stop-channel state") + } +} + +// TestSendUpdateBlockingReturnsFalseImmediatelyWhenNeverStartedAndFull pins +// the defensive behaviour for a Manager whose stop channel was never +// established (Start has not run) and whose channel is actually full: treat +// it as already stopped rather than parking forever with no lifecycle that +// could ever release it. +func TestSendUpdateBlockingReturnsFalseImmediatelyWhenNeverStartedAndFull(t *testing.T) { + m := &Manager{ + logger: newTestLogger(t), + updatesCh: make(chan adapter.AgentEvent, 1), + } + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + + done := make(chan bool, 1) + go func() { + done <- m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError}) + }() + + select { + case ok := <-done: + if ok { + t.Fatal("sendUpdateBlocking() = true with no stop channel established, want false") + } + case <-time.After(time.Second): + t.Fatal("sendUpdateBlocking did not return promptly with no stop channel established") + } +} diff --git a/apps/backend/internal/agentctl/server/process/manager.go b/apps/backend/internal/agentctl/server/process/manager.go index ad5786a2ec..3e4be1519a 100644 --- a/apps/backend/internal/agentctl/server/process/manager.go +++ b/apps/backend/internal/agentctl/server/process/manager.go @@ -45,6 +45,14 @@ const ( StatusError Status = "error" ) +// defaultUpdatesChannelCapacity is the updates channel buffer size used when +// InstanceConfig.DetachedEventLimit is left at its Go zero value -- the +// common case for tests and any other caller that builds a config.InstanceConfig +// directly instead of going through Config.NewInstanceConfig. It matches the +// agentctl.detachedEventLimit catalog tunable's own default +// (AC-EXECUTORS-SURVIVAL-001.6). +const defaultUpdatesChannelCapacity = 100 + // errorWrapper wraps an error so it can be stored in atomic.Value (which cannot store nil) type errorWrapper struct { err error @@ -246,18 +254,38 @@ type Manager struct { finalCommand string // Synchronization - mu sync.RWMutex - wg sync.WaitGroup - stopCh chan struct{} - doneCh chan struct{} - startMu sync.Mutex - admissionMu sync.Mutex - admissionCount int - admissionDrained chan struct{} - stopping bool - lifetimeCtx context.Context - lifetimeCancel context.CancelFunc - mainReapPending atomic.Bool + mu sync.RWMutex + wg sync.WaitGroup + stopCh chan struct{} + // stopChSnapshot mirrors stopCh for readers that must not take mu: Stop + // holds mu.Lock() for its entire teardown, including waiting for + // waitForExit to finish, so a stop-aware blocking send reached from that + // same goroutine (e.g. the exit-error send in waitForExit) would deadlock + // against Stop if it read stopCh under mu.RLock() instead. Start and + // startOneShot store the freshly created channel here in the same place + // they assign stopCh itself. + stopChSnapshot atomic.Value // chan struct{} + doneCh chan struct{} + // attachedCount is the live count of backend event-stream connections + // (see attachment.go). Zero value correctly starts an instance detached. + attachedCount atomic.Int32 + // turnOutcomeRecorder and turnOutcomeInstanceID back retained-outcome + // wiring (see turn_outcome.go). Both are guarded by mu: set once by + // SetTurnOutcomeRecorder before any goroutine that could read them is + // spawned (instance.Manager.CreateInstance calls it immediately after + // constructing this Manager, before Start can be reached), then read + // from forwardUpdates and sendUpdateBlocking's callers, neither of which + // otherwise holds mu. + turnOutcomeRecorder TurnOutcomeRecorder + turnOutcomeInstanceID string + startMu sync.Mutex + admissionMu sync.Mutex + admissionCount int + admissionDrained chan struct{} + stopping bool + lifetimeCtx context.Context + lifetimeCancel context.CancelFunc + mainReapPending atomic.Bool // stopChClosed guards close(stopCh), which is the only part of teardown // that is not naturally idempotent. It is reset wherever stopCh itself is // created so the flag always describes the current channel — a Start that @@ -371,11 +399,15 @@ func (m *Manager) BeginStop() { // NewManager creates a new process manager func NewManager(cfg *config.InstanceConfig, log *logger.Logger) *Manager { cfg.WorkDir = resolveExistingWorkDir(cfg.WorkDir, log.WithFields(zap.String("component", "process-manager"))) + updatesChannelCapacity := cfg.DetachedEventLimit + if updatesChannelCapacity <= 0 { + updatesChannelCapacity = defaultUpdatesChannelCapacity + } lifetimeCtx, lifetimeCancel := context.WithCancel(context.Background()) m := &Manager{ cfg: cfg, logger: log.WithFields(zap.String("component", "process-manager")), - updatesCh: make(chan adapter.AgentEvent, 100), + updatesCh: make(chan adapter.AgentEvent, updatesChannelCapacity), pendingPermissions: make(map[string]*PendingPermission), lifetimeCtx: lifetimeCtx, lifetimeCancel: lifetimeCancel, @@ -457,6 +489,15 @@ func (m *Manager) currentWorkspaceSourceRoots() []string { return append([]string(nil), m.workspaceSourceRoots...) } +// WorkspaceSourceRoots returns the live, current source-root allowlist +// (AC-EXECUTORS-SURVIVAL-002.14's "workspace source roots" reconstruction +// row: read back from the adopted instance, never pushed). A rescan or +// rebind changes this in place, so callers always see the value this +// instance is enforcing right now, not a snapshot from creation. +func (m *Manager) WorkspaceSourceRoots() []string { + return m.currentWorkspaceSourceRoots() +} + // lookupBaseBranch reads the task's recorded base branch for a given // repository name from the per-instance map. The empty key "" addresses the // single-repo / root tracker. Falls back to the empty-key entry when the @@ -1312,6 +1353,7 @@ func (m *Manager) Start(ctx context.Context) error { m.processLifecycle = processLifecycle m.stopCh = make(chan struct{}) + m.stopChSnapshot.Store(m.stopCh) m.doneCh = make(chan struct{}) m.stopChClosed.Store(false) @@ -1367,6 +1409,7 @@ func (m *Manager) startOneShot() error { } m.stopCh = make(chan struct{}) + m.stopChSnapshot.Store(m.stopCh) m.doneCh = make(chan struct{}) m.stopChClosed.Store(false) @@ -1904,6 +1947,7 @@ func (m *Manager) forwardUpdates(agentAdapter adapter.AgentAdapter, stopCh <-cha if !ok { return } + m.recordTerminalOutcome(&update) select { case m.updatesCh <- update: case <-stopCh: @@ -1929,21 +1973,21 @@ func (m *Manager) SendErrorEvent(errorMessage string, promptGeneration uint64) { // SendErrorEventWithProviderError sends an error event with optional safe // provider details. The details are already normalized by the adapter and are // never populated from the manager's raw stderr ring. +// +// This is a COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6): the agent ERROR +// event must park on a full channel rather than being dropped, since it is +// one of the events most likely to matter across a detached gap. func (m *Manager) SendErrorEventWithProviderError( errorMessage string, promptGeneration uint64, providerError *streams.ProviderError, ) { - select { - case m.updatesCh <- adapter.AgentEvent{ + m.sendUpdateBlocking(adapter.AgentEvent{ Type: adapter.EventTypeError, Error: errorMessage, PromptGeneration: promptGeneration, ProviderError: providerError, - }: - default: - m.logger.Warn("updates channel full, could not send error event") - } + }) } // PublishMCPAttachment forwards safe MCP attachment evidence through the @@ -2635,23 +2679,26 @@ func (m *Manager) waitForExit(stderrDone <-chan struct{}) { zap.Int("exit_code", exitCode), zap.Strings("recent_stderr", recentStderr)) - // Send error event to the updates channel so UI can display it + // Send error event to the updates channel so UI can display it. This is + // a COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6) and the only report + // that the agent process died at all, so it must park on a full + // channel rather than being dropped. sendUpdateBlocking selects + // against the stop-channel snapshot rather than m.mu-guarded state + // deliberately: Stop holds m.mu for its entire teardown, including + // waiting on this same waitForExit goroutine to finish, so reading + // stopCh under m.mu here would deadlock against it. errorMsg := fmt.Sprintf("Agent process exited with code %d", exitCode) if len(recentStderr) > 0 { errorMsg = fmt.Sprintf("%s: %s", errorMsg, strings.Join(recentStderr, "; ")) } - select { - case m.updatesCh <- adapter.AgentEvent{ + m.sendUpdateBlocking(adapter.AgentEvent{ Type: adapter.EventTypeError, Error: errorMsg, Data: map[string]any{ "exit_code": exitCode, "recent_stderr": recentStderr, }, - }: - default: - m.logger.Warn("updates channel full, could not send exit error event") - } + }) default: m.exitCode.Store(0) m.logger.Info("agent process exited successfully") @@ -2819,8 +2866,17 @@ func (m *Manager) autoApprovePermission(req *adapter.PermissionRequest) (*adapte } // sendPermissionNotification sends a permission request notification through the updates channel. -// Uses a blocking send with timeout to ensure delivery. If delivery fails within 5 seconds, -// auto-cancels the permission so the agent doesn't hang waiting for a response. +// +// This is the eighth COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6) and the +// only one of the eight that must branch on attached/detached state, rather +// than always parking: while ATTACHED, a blocking send with a five-second +// timeout is a deliberate safety valve against a stalled UI, and must +// survive unchanged -- auto-cancelling the permission so the agent doesn't +// hang waiting for a response. While DETACHED, that same auto-cancel would +// silently deny every permission the agent asks for, five seconds apart, so +// it must park instead: the wait ends either because a backend later +// attaches (which starts draining the channel, satisfying the same select +// sendUpdateBlocking already performs) or because the instance stops. func (m *Manager) sendPermissionNotification(pending *PendingPermission) { options := make([]streams.PermissionOption, len(pending.Snapshot.Options)) for i, option := range pending.Snapshot.Options { @@ -2846,6 +2902,11 @@ func (m *Manager) sendPermissionNotification(pending *PendingPermission) { zap.String("pending_id", pending.ID), zap.String("action_type", pending.Request.ActionType)) + if !m.IsAttached() { + m.sendUpdateBlocking(event) + return + } + timer := time.NewTimer(5 * time.Second) defer timer.Stop() select { @@ -2864,6 +2925,9 @@ func (m *Manager) sendPermissionNotification(pending *PendingPermission) { // sendPermissionCancelledNotification sends a notification that a permission request was cancelled. // This happens when the context is cancelled (e.g., agent completes or user stops the task) // before the user responds to the permission request. +// +// This is a COVERED site (AC-EXECUTORS-SURVIVAL-001.5/.6): it must park on a +// full channel rather than being dropped. func (m *Manager) sendPermissionCancelledNotification(pending *PendingPermission) { event := adapter.AgentEvent{ Type: adapter.EventTypePermissionCancelled, @@ -2876,13 +2940,7 @@ func (m *Manager) sendPermissionCancelledNotification(pending *PendingPermission zap.String("pending_id", pending.ID), zap.String("session_id", pending.Request.SessionID)) - select { - case m.updatesCh <- event: - // Sent successfully - default: - m.logger.Warn("updates channel full, dropping permission cancelled notification", - zap.String("pending_id", pending.ID)) - } + m.sendUpdateBlocking(event) } func (m *Manager) permissionSessionID(pending *PendingPermission) string { diff --git a/apps/backend/internal/agentctl/server/process/turn_outcome.go b/apps/backend/internal/agentctl/server/process/turn_outcome.go new file mode 100644 index 0000000000..35af326e37 --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/turn_outcome.go @@ -0,0 +1,86 @@ +package process + +import "github.com/kandev/kandev/internal/agentctl/server/adapter" + +// TurnOutcomeRecorder retains the last terminal outcome observed for an +// instance so a backend that re-attaches after a restart +// (AC-EXECUTORS-SURVIVAL-004) can learn what happened to a turn it never saw +// complete. Satisfied by instance.Manager.RetainTurnOutcome; defined here +// (rather than imported from the instance package) because process must not +// import instance, which already imports process. +type TurnOutcomeRecorder interface { + RetainTurnOutcome(instanceID string, event adapter.AgentEvent) (turnID int64, ok bool) +} + +// turnOutcomeClearer is an optional extension implemented by the instance +// manager. A new prompt retires the previous terminal outcome before it can +// be mistaken for the prompt that is now in flight after a backend restart. +type turnOutcomeClearer interface { + ClearTurnOutcome(instanceID string, promptGeneration uint64) +} + +// SetTurnOutcomeRecorder wires the instance-level outcome retention sink for +// this process manager. Called once by instance.Manager.CreateInstance right +// after constructing the process manager and before Start can be reached +// through any path, mirroring the StderrProviderSetter optional-interface +// pattern used elsewhere in this package. A Manager with no recorder set +// (most existing tests, and the e2e harness) simply never retains -- +// retention is additive, not required for correct event delivery. +func (m *Manager) SetTurnOutcomeRecorder(instanceID string, recorder TurnOutcomeRecorder) { + m.mu.Lock() + defer m.mu.Unlock() + m.turnOutcomeInstanceID = instanceID + m.turnOutcomeRecorder = recorder +} + +// ClearTurnOutcome retires the previous terminal outcome before a new prompt +// is dispatched. The generation floor also rejects a late delivery of an +// older terminal event that races this clear operation. +func (m *Manager) ClearTurnOutcome(promptGeneration uint64) { + m.mu.RLock() + recorder := m.turnOutcomeRecorder + instanceID := m.turnOutcomeInstanceID + m.mu.RUnlock() + clearer, ok := recorder.(turnOutcomeClearer) + if !ok { + return + } + clearer.ClearTurnOutcome(instanceID, promptGeneration) +} + +// recordTerminalOutcome retains event if it is one of the two terminal event +// types AC-EXECUTORS-SURVIVAL-004 cares about (EventTypeComplete, +// EventTypeError) and a recorder has been wired. Every non-terminal event +// (permission request/cancelled, MCP attachment, context window, session +// models, ...) is filtered out here rather than at each producer, since +// EventTypeComplete/EventTypeError is emitted from several independent +// places (this manager's own agent-ERROR and process-exit-error sends, and +// the ACP adapter's session/prompt completion and protocol-error paths) that +// all converge on updatesCh regardless of origin -- this is the single +// filter for all of them, called from every point an event is actually +// delivered onto updatesCh (forwardUpdates, sendUpdateBlocking). +// +// event is a pointer so the assigned turn identifier can be stamped onto +// ControlTurnID before the caller sends this same value onward -- callers +// must call this before the channel send, not after, or the stamp never +// reaches the delivered copy. AC-EXECUTORS-SURVIVAL-004.4 requires that +// identifier to travel on both the retained-read observation (returned via +// TurnOutcome.TurnID) and the live-delivered observation (this stamp); the +// retained copy stored by Retain is deliberately the pre-stamp value, since +// TurnOutcome.TurnID already carries the identifier for that side. +func (m *Manager) recordTerminalOutcome(event *adapter.AgentEvent) { + if event.Type != adapter.EventTypeComplete && event.Type != adapter.EventTypeError { + return + } + m.mu.RLock() + recorder := m.turnOutcomeRecorder + instanceID := m.turnOutcomeInstanceID + m.mu.RUnlock() + if recorder == nil { + return + } + turnID, ok := recorder.RetainTurnOutcome(instanceID, *event) + if ok { + event.ControlTurnID = turnID + } +} diff --git a/apps/backend/internal/agentctl/server/process/turn_outcome_test.go b/apps/backend/internal/agentctl/server/process/turn_outcome_test.go new file mode 100644 index 0000000000..75ae7272ec --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/turn_outcome_test.go @@ -0,0 +1,271 @@ +package process + +import ( + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/server/adapter" + "github.com/kandev/kandev/internal/agentctl/types/streams" +) + +// fakeTurnOutcomeRecorder captures RetainTurnOutcome calls for assertion, +// standing in for instance.Manager without importing it (process must not +// import instance, which already imports process). +type fakeTurnOutcomeRecorder struct { + calls []fakeTurnOutcomeCall + clearCalls []fakeTurnOutcomeClearCall +} + +type fakeTurnOutcomeCall struct { + instanceID string + event adapter.AgentEvent +} + +type fakeTurnOutcomeClearCall struct { + instanceID string + promptGeneration uint64 +} + +func (f *fakeTurnOutcomeRecorder) RetainTurnOutcome(instanceID string, event adapter.AgentEvent) (int64, bool) { + f.calls = append(f.calls, fakeTurnOutcomeCall{instanceID: instanceID, event: event}) + return int64(len(f.calls)), true +} + +func (f *fakeTurnOutcomeRecorder) ClearTurnOutcome(instanceID string, promptGeneration uint64) { + f.clearCalls = append(f.clearCalls, fakeTurnOutcomeClearCall{ + instanceID: instanceID, + promptGeneration: promptGeneration, + }) +} + +func TestClearTurnOutcomeForwardsPromptGenerationToRecorder(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + m := &Manager{} + m.SetTurnOutcomeRecorder("instance-1", recorder) + + m.ClearTurnOutcome(9) + + if len(recorder.clearCalls) != 1 { + t.Fatalf("clear calls = %d, want 1", len(recorder.clearCalls)) + } + call := recorder.clearCalls[0] + if call.instanceID != "instance-1" || call.promptGeneration != 9 { + t.Fatalf("clear call = %+v, want instance-1/generation-9", call) + } +} + +// TestRecordTerminalOutcomeRetainsCompleteAndErrorEvents pins +// AC-EXECUTORS-SURVIVAL-004.1: the two terminal event types must reach the +// wired recorder with the instance ID SetTurnOutcomeRecorder was given. +func TestRecordTerminalOutcomeRetainsCompleteAndErrorEvents(t *testing.T) { + terminalTypes := []string{adapter.EventTypeComplete, adapter.EventTypeError} + for _, eventType := range terminalTypes { + t.Run(eventType, func(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + m := &Manager{} + m.SetTurnOutcomeRecorder("instance-1", recorder) + + event := adapter.AgentEvent{Type: eventType} + m.recordTerminalOutcome(&event) + + if len(recorder.calls) != 1 { + t.Fatalf("recorder calls = %d, want 1", len(recorder.calls)) + } + if recorder.calls[0].instanceID != "instance-1" { + t.Fatalf("instanceID = %q, want instance-1", recorder.calls[0].instanceID) + } + if recorder.calls[0].event.Type != eventType { + t.Fatalf("event type = %q, want %q", recorder.calls[0].event.Type, eventType) + } + if event.ControlTurnID != int64(len(recorder.calls)) { + t.Fatalf("event.ControlTurnID = %d, want %d (stamped from RetainTurnOutcome's return)", + event.ControlTurnID, len(recorder.calls)) + } + }) + } +} + +// TestRecordTerminalOutcomeIgnoresNonTerminalEvents pins that every other +// event type -- including the two excluded MCP-attachment sites' type and +// the permission lifecycle types -- must never reach the recorder. AC-004 is +// specifically "what happened to my last turn," not every lifecycle event. +func TestRecordTerminalOutcomeIgnoresNonTerminalEvents(t *testing.T) { + nonTerminalTypes := []string{ + adapter.EventTypePermissionRequest, + adapter.EventTypePermissionCancelled, + streams.EventTypeMCPAttachment, + "context_window", + "session_models", + "message_chunk", + } + for _, eventType := range nonTerminalTypes { + t.Run(eventType, func(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + m := &Manager{} + m.SetTurnOutcomeRecorder("instance-1", recorder) + + event := adapter.AgentEvent{Type: eventType} + m.recordTerminalOutcome(&event) + + if len(recorder.calls) != 0 { + t.Fatalf("recorder calls = %d for type %q, want 0", len(recorder.calls), eventType) + } + }) + } +} + +// TestRecordTerminalOutcomeNoopWithoutRecorder pins that a Manager with no +// recorder wired (the common case: most tests, and the e2e harness) safely +// ignores terminal events instead of panicking. +func TestRecordTerminalOutcomeNoopWithoutRecorder(t *testing.T) { + m := &Manager{} + event := adapter.AgentEvent{Type: adapter.EventTypeComplete} + m.recordTerminalOutcome(&event) + if event.ControlTurnID != 0 { + t.Fatalf("ControlTurnID = %d, want 0 with no recorder wired", event.ControlTurnID) + } +} + +// TestSendUpdateBlockingRecordsTerminalOutcomeOnDelivery pins that both the +// fast path (room available) and the parked path (room freed up later) of +// sendUpdateBlocking retain a terminal event on successful delivery. +func TestSendUpdateBlockingRecordsTerminalOutcomeOnDelivery(t *testing.T) { + t.Run("fast path", func(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + m := &Manager{updatesCh: make(chan adapter.AgentEvent, 1)} + m.SetTurnOutcomeRecorder("instance-1", recorder) + + if !m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError}) { + t.Fatal("sendUpdateBlocking returned false with room available") + } + if len(recorder.calls) != 1 { + t.Fatalf("recorder calls = %d, want 1", len(recorder.calls)) + } + }) + + t.Run("parked path", func(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + m := &Manager{updatesCh: make(chan adapter.AgentEvent, 1)} + m.SetTurnOutcomeRecorder("instance-1", recorder) + m.stopChSnapshot.Store(make(chan struct{})) + m.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} // fill the buffer + + done := make(chan bool, 1) + go func() { done <- m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError}) }() + + select { + case <-done: + t.Fatal("sendUpdateBlocking returned before the channel had room") + case <-time.After(50 * time.Millisecond): + } + + <-m.updatesCh // drain the pre-filled slot + + select { + case ok := <-done: + if !ok { + t.Fatal("sendUpdateBlocking returned false after room freed up") + } + case <-time.After(time.Second): + t.Fatal("sendUpdateBlocking did not return promptly after room freed up") + } + if len(recorder.calls) != 1 { + t.Fatalf("recorder calls = %d, want 1", len(recorder.calls)) + } + }) +} + +// TestForwardUpdatesRecordsTerminalOutcomeForAdapterOriginatedEvents pins +// that a terminal event emitted by the ACP adapter (session/prompt +// completion, protocol-level error) -- which never goes through +// sendUpdateBlocking, only through forwardUpdates's copy from the adapter's +// own channel -- is also retained. This is the coverage this manager cannot +// get any other way, since the adapter has no direct reference to the +// recorder. +func TestForwardUpdatesRecordsTerminalOutcomeForAdapterOriginatedEvents(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + stopCh := make(chan struct{}) + stub := newStubAdapter() + m := &Manager{ + adapter: stub, + updatesCh: make(chan adapter.AgentEvent, 10), + stopCh: stopCh, + logger: newTestLogger(t), + } + m.SetTurnOutcomeRecorder("instance-1", recorder) + m.wg.Add(1) + go m.forwardUpdates(stub, stopCh) + t.Cleanup(func() { close(stopCh) }) + + stub.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} + stub.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeMessageChunk} + + select { + case <-m.updatesCh: + case <-time.After(time.Second): + t.Fatal("timed out waiting for the complete event to be forwarded") + } + select { + case <-m.updatesCh: + case <-time.After(time.Second): + t.Fatal("timed out waiting for the message_chunk event to be forwarded") + } + + if len(recorder.calls) != 1 { + t.Fatalf("recorder calls = %d, want 1 (only the complete event)", len(recorder.calls)) + } + if recorder.calls[0].event.Type != adapter.EventTypeComplete { + t.Fatalf("retained event type = %q, want complete", recorder.calls[0].event.Type) + } +} + +// TestSendUpdateBlockingStampsControlTurnIDOnDeliveredCopy and +// TestForwardUpdatesStampsControlTurnIDOnDeliveredCopy pin +// AC-EXECUTORS-SURVIVAL-004.4: the control-server-assigned turn identifier +// must reach the copy actually delivered on m.updatesCh (the "delivered +// event" observation), not just the pre-stamp copy Retain stores (the +// "retained turn status" observation, which carries it via +// TurnOutcome.TurnID instead). Without this stamp a backend re-attaching +// after a restart has no way to recognize a live redelivery of the exact +// turn it already applied from GetTurnOutcome. +func TestSendUpdateBlockingStampsControlTurnIDOnDeliveredCopy(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + m := &Manager{updatesCh: make(chan adapter.AgentEvent, 1)} + m.SetTurnOutcomeRecorder("instance-1", recorder) + + if !m.sendUpdateBlocking(adapter.AgentEvent{Type: adapter.EventTypeError}) { + t.Fatal("sendUpdateBlocking returned false with room available") + } + + delivered := <-m.updatesCh + if delivered.ControlTurnID != 1 { + t.Fatalf("delivered.ControlTurnID = %d, want 1", delivered.ControlTurnID) + } +} + +func TestForwardUpdatesStampsControlTurnIDOnDeliveredCopy(t *testing.T) { + recorder := &fakeTurnOutcomeRecorder{} + stopCh := make(chan struct{}) + stub := newStubAdapter() + m := &Manager{ + adapter: stub, + updatesCh: make(chan adapter.AgentEvent, 10), + stopCh: stopCh, + logger: newTestLogger(t), + } + m.SetTurnOutcomeRecorder("instance-1", recorder) + m.wg.Add(1) + go m.forwardUpdates(stub, stopCh) + t.Cleanup(func() { close(stopCh) }) + + stub.updatesCh <- adapter.AgentEvent{Type: adapter.EventTypeComplete} + + select { + case delivered := <-m.updatesCh: + if delivered.ControlTurnID != 1 { + t.Fatalf("delivered.ControlTurnID = %d, want 1", delivered.ControlTurnID) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for the complete event to be forwarded") + } +} diff --git a/apps/backend/internal/agentctl/server/process/updates_channel_capacity_test.go b/apps/backend/internal/agentctl/server/process/updates_channel_capacity_test.go new file mode 100644 index 0000000000..09d2f9deff --- /dev/null +++ b/apps/backend/internal/agentctl/server/process/updates_channel_capacity_test.go @@ -0,0 +1,35 @@ +package process + +import ( + "testing" + + "github.com/kandev/kandev/internal/agentctl/server/config" +) + +// TestNewManagerSizesUpdatesChannelFromDetachedEventLimit pins +// AC-EXECUTORS-SURVIVAL-001.6: the per-instance retained-event limit is the +// updates channel's own buffer, not an incidental hardcoded size, so a +// non-default configured value must be honoured. +func TestNewManagerSizesUpdatesChannelFromDetachedEventLimit(t *testing.T) { + mgr := NewManager(&config.InstanceConfig{ + WorkDir: t.TempDir(), + DetachedEventLimit: 7, + }, newTestLogger(t)) + + if cap(mgr.updatesCh) != 7 { + t.Fatalf("cap(updatesCh) = %d, want 7 (from DetachedEventLimit)", cap(mgr.updatesCh)) + } +} + +// TestNewManagerDefaultsUpdatesChannelCapacityWhenUnset pins the fallback for +// the many call sites (including most of this package's own tests) that +// construct config.InstanceConfig directly without going through +// Config.NewInstanceConfig, so DetachedEventLimit is left at its Go +// zero value rather than the configured default of 100. +func TestNewManagerDefaultsUpdatesChannelCapacityWhenUnset(t *testing.T) { + mgr := NewManager(&config.InstanceConfig{WorkDir: t.TempDir()}, newTestLogger(t)) + + if cap(mgr.updatesCh) != defaultUpdatesChannelCapacity { + t.Fatalf("cap(updatesCh) = %d, want the default %d", cap(mgr.updatesCh), defaultUpdatesChannelCapacity) + } +} diff --git a/apps/backend/internal/agentctl/server/process/workspace_files_test.go b/apps/backend/internal/agentctl/server/process/workspace_files_test.go index ad611f8e75..9bcce1a039 100644 --- a/apps/backend/internal/agentctl/server/process/workspace_files_test.go +++ b/apps/backend/internal/agentctl/server/process/workspace_files_test.go @@ -213,6 +213,43 @@ func TestWorkspaceFileOperationsAllowRegisteredLinkedSource(t *testing.T) { } } +// TestWorkspaceFileOperationsWithNoAllowedSourceRootsFailClosed pins the +// AC-EXECUTORS-SURVIVAL-002.14 recovery contract: when the adopted instance +// reports zero workspace source roots (nil/empty), the tracker must reject +// every durable-source symlink escape rather than treating "no roots +// configured" as "no restriction". Ordinary in-workspace file operations are +// unaffected, since they never need the allowlist. +func TestWorkspaceFileOperationsWithNoAllowedSourceRootsFailClosed(t *testing.T) { + workspace := t.TempDir() + source := t.TempDir() + if err := os.Symlink(source, filepath.Join(workspace, "linked")); err != nil { + t.Skip("symlinks not supported") + } + + log, err := logger.NewFromZap(zap.NewNop()) + if err != nil { + t.Fatal(err) + } + wt := &WorkspaceTracker{workDir: workspace, logger: log} + // Deliberately never call SetAllowedSourceRoots, reproducing the + // zero-value state a freshly recovered tracker has before any roots are + // (re)applied from the adopted instance. + + if err := wt.CreateFile(filepath.Join("linked", "escape.txt")); err == nil { + t.Fatal("CreateFile through an unregistered symlink unexpectedly succeeded with no allowed source roots") + } + if _, err := os.Stat(filepath.Join(source, "escape.txt")); !os.IsNotExist(err) { + t.Fatalf("file leaked into the symlink target despite no allowed source roots: %v", err) + } + + if err := wt.CreateFile("plain.txt"); err != nil { + t.Fatalf("CreateFile for an ordinary in-workspace path: %v", err) + } + if _, err := os.Stat(filepath.Join(workspace, "plain.txt")); err != nil { + t.Fatalf("in-workspace file was not created: %v", err) + } +} + func TestWorkspaceFileMutationsRejectDescendantSymlinkSwap(t *testing.T) { for _, tc := range []struct { name string diff --git a/apps/backend/internal/agentctl/types/streams/agent.go b/apps/backend/internal/agentctl/types/streams/agent.go index eb87eec9dc..d3465f2aeb 100644 --- a/apps/backend/internal/agentctl/types/streams/agent.go +++ b/apps/backend/internal/agentctl/types/streams/agent.go @@ -143,6 +143,18 @@ type AgentEvent struct { // ownership from cross-subject event ordering. TurnID string `json:"turn_id,omitempty"` + // ControlTurnID is the control-server-assigned turn identifier + // (AC-EXECUTORS-SURVIVAL-004.1), stamped onto a terminal event at the + // same point the control server retains it as that instance's turn + // outcome (internal/agentctl/server/process.recordTerminalOutcome). Zero + // means "not retained" -- the sequence that assigns it starts at 1 + // (instance.Manager.turnIDSeq). AC-EXECUTORS-SURVIVAL-004.4 requires the + // dedup decision between a retrieved outcome and a live-delivered event + // for the same turn to be made on this identifier, since it -- unlike + // PromptGeneration -- travels on both observations and survives a + // restart that resets the backend's own in-memory generation counter. + ControlTurnID int64 `json:"control_turn_id,omitempty"` + // --- Message fields (for "message_chunk" type) --- // Text contains streaming text content from the agent. diff --git a/apps/backend/internal/backendapp/adapters.go b/apps/backend/internal/backendapp/adapters.go index 62f5f86bb3..05688fe140 100644 --- a/apps/backend/internal/backendapp/adapters.go +++ b/apps/backend/internal/backendapp/adapters.go @@ -325,6 +325,23 @@ func newLifecycleAdapter(mgr *lifecycle.Manager, reg *registry.Registry, log *lo // LaunchAgent creates a new agentctl instance for a task. // Agent subprocess is NOT started - call StartAgentProcess() explicitly. +// wrapSessionRecoveryGuardError translates the raw lifecycle recovery-guard +// sentinels into an orchestrator.SessionRecoveryGuardError so callers outside +// internal/agent/runtime/ can distinguish a retryable in-progress recovery +// from a non-retryable unstoppable-agent condition without importing +// lifecycle directly. Errors that are not guard-related pass through +// unchanged. +func wrapSessionRecoveryGuardError(err error, sessionID string) error { + switch { + case errors.Is(err, lifecycle.ErrSessionRecoveryGuarded): + return &orchestrator.SessionRecoveryGuardError{Cause: err, SessionID: sessionID, Retryable: true} + case errors.Is(err, lifecycle.ErrSessionUnstoppableAgent): + return &orchestrator.SessionRecoveryGuardError{Cause: err, SessionID: sessionID, Retryable: false} + default: + return err + } +} + func (a *lifecycleAdapter) LaunchAgent(ctx context.Context, req *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) { // WorkspacePath wins when set (repo-less task with picked folder); otherwise // fall back to RepositoryURL (legacy: this carries a local filesystem path @@ -343,7 +360,7 @@ func (a *lifecycleAdapter) LaunchAgent(ctx context.Context, req *executor.Launch // Create the agentctl execution (does NOT start agent process) execution, err := a.mgr.Launch(ctx, launchReq) if err != nil { - return nil, err + return nil, wrapSessionRecoveryGuardError(err, req.SessionID) } // Extract worktree info from metadata if available @@ -633,13 +650,29 @@ func normalizeRuntimeStopError(err error) error { } // RowLiveness classifies the liveness of the OS process backing an -// executors_running row using the runtime-aware host-local probe. It is the -// orchestrator's window into the platform-split liveness check (kept in the -// lifecycle package) used by startup reconciliation -// (#1597 runtime-aware liveness). A local check never runs against a +// executors_running row. It is the orchestrator's window into the +// platform-split liveness check (kept in the lifecycle package) used outside +// a reconciliation pass — e.g. the single-session idle reclaim path — and +// always takes its own fresh adopted-server enumeration for a standalone row +// rather than reusing one a pass cached (see RowLivenessScoped for the +// pass-scoped sibling). A local/enumeration check never runs against a // remote/SSH row — such rows return Unknown. func (a *lifecycleAdapter) RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness { - return lifecycle.RowProcessLiveness(row) + return a.mgr.RowLiveness(row) +} + +// NewStandaloneLivenessScope takes one adopted-server enumeration for reuse +// across every row of a single reconciliation pass (design 02 "Persistence": +// "It has two kinds of caller, and only one of them is a pass"). Satisfies +// the orchestrator's optional standaloneLivenessScoper capability. +func (a *lifecycleAdapter) NewStandaloneLivenessScope(ctx context.Context) interface{} { + return a.mgr.NewStandaloneLivenessScope(ctx) +} + +// RowLivenessScoped classifies row's liveness reusing scope (from +// NewStandaloneLivenessScope) instead of taking a fresh enumeration. +func (a *lifecycleAdapter) RowLivenessScoped(row *models.ExecutorRunning, scope interface{}) models.ProcessLiveness { + return a.mgr.RowLivenessScoped(row, scope) } // GetAgentStatus returns the status of an agent execution diff --git a/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go b/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go index 981e4f0f3e..42990b25d6 100644 --- a/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go +++ b/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go @@ -41,7 +41,7 @@ func (b *capturingKubernetesBackend) CreateInstance( func (b *capturingKubernetesBackend) StopInstance(context.Context, *lifecycle.ExecutorInstance, bool) error { return nil } -func (b *capturingKubernetesBackend) RecoverInstances(context.Context) ([]*lifecycle.ExecutorInstance, error) { +func (b *capturingKubernetesBackend) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*lifecycle.ExecutorInstance, error) { return nil, nil } func (b *capturingKubernetesBackend) GetInteractiveRunner() *process.InteractiveRunner { return nil } diff --git a/apps/backend/internal/backendapp/agentctl.go b/apps/backend/internal/backendapp/agentctl.go index 81c2a288b7..fc6345d693 100644 --- a/apps/backend/internal/backendapp/agentctl.go +++ b/apps/backend/internal/backendapp/agentctl.go @@ -2,12 +2,18 @@ package backendapp import ( "context" + "fmt" + "net" + "strconv" "time" + agentruntime "github.com/kandev/kandev/internal/agent/runtime" agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agent/runtime/agentctl/launcher" "github.com/kandev/kandev/internal/common/config" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/common/ownershipperiod" + "github.com/kandev/kandev/internal/secrets" "go.uber.org/zap" ) @@ -15,18 +21,167 @@ import ( type agentctlLauncherResult struct { cleanup func() error binaryPath string + // recoveryDeadlineStart is the AC-EXECUTORS-SURVIVAL-003.7 recovery + // deadline's clock start (the instant this launch first contacted a + // recorded control endpoint). Zero when this launch never contacted one + // (adoption disabled, no record, or a fresh spawn) -- the lifecycle + // manager falls back to its own Start-time default in that case. + recoveryDeadlineStart time.Time + // inheritedRecordScope is what this launch found at the recorded control + // endpoint. It governs how recovery classifies recovery-inventory records + // left by an earlier launch, which cannot be judged against a server this + // backend started itself. + inheritedRecordScope agentruntime.InheritedRecordScope } -// provideAgentctlLauncher starts the agentctl launcher for standalone runtime. -// agentctl is a core service that always runs - it's used by the Standalone runtime -// for agent execution on the host machine. -// If the configured port is unavailable, the launcher may fall back to an OS-assigned -// port. In that case, cfg.Agent.StandalonePort is updated to reflect the actual port. +// provideAgentctlLauncher starts or adopts the agentctl control server for +// standalone runtime. agentctl is a core service that always runs - it's used +// by the Standalone runtime for agent execution on the host machine. +// +// When the agent-survival capability is enabled (Layer 6.1 of +// agent-survival-across-restart), a surviving control server from a prior +// launch is tried first via agentruntime.AttemptAdoptControlServer; only when +// that fails or there is no record does this spawn a fresh one, same as +// when the capability is disabled. Either way, cfg.Agent.Standalone* is +// updated to point at whichever server this launch ends up using. func provideAgentctlLauncher( ctx context.Context, cfg *config.Config, log *logger.Logger, availability *agentctlclient.Availability, + store agentruntime.AdoptionRecordStore, + secretStore secrets.SecretStore, +) (*agentctlLauncherResult, error) { + result, scope := resolveSurvivingAgentctl(ctx, cfg, log, store, secretStore) + if result != nil { + availability.MarkAvailable() + return result, nil + } + fresh, err := spawnFreshAgentctl(ctx, cfg, log, availability, store, secretStore) + if fresh != nil { + // The server this launch just started never launched the instances a + // prior launch recorded, so recovery needs to know what was found at + // the recorded endpoint rather than judging them against it. + fresh.inheritedRecordScope = scope + } + return fresh, err +} + +// resolveSurvivingAgentctl applies the agent-survival capability gate to a +// control server a prior launch left running, and returns non-nil only when +// this launch took that server over. With the capability disabled nothing is +// ever adopted, but a server an earlier survival-enabled launch detached is +// still running with no backend attached to it, so it is stopped here rather +// than left to its own unowned-shutdown timer. Either way a nil return means +// the caller spawns fresh. +func resolveSurvivingAgentctl( + ctx context.Context, + cfg *config.Config, + log *logger.Logger, + store agentruntime.AdoptionRecordStore, + secretStore secrets.SecretStore, +) (*agentctlLauncherResult, agentruntime.InheritedRecordScope) { + if !cfg.Features.AgentSurvival { + agentruntime.ReclaimUnneededControlServer(ctx, store, secretStore, controlClientFactory(log), cfg.ResolvedHomeDir(), + cfg.Agentctl.RecoveryReadTimeout, cfg.Agentctl.RecoveryReadRetries, log) + return nil, agentruntime.InheritedRecordScopeNoServer + } + return adoptSurvivingAgentctl(ctx, cfg, log, store, secretStore) +} + +// refusedAdoptionScope maps a refused adoption to what recovery may conclude +// about records an earlier launch left behind. A refusal that never reached +// the server -- nothing answered at the recorded endpoint, or no endpoint was +// recorded -- means no control server holds those instances, so they can be +// probed and repaired. Every refusal that did reach a reachable server leaves +// that server running, and an instance may still be alive on it, so nothing +// may be concluded from its absence from a server this launch started. +func refusedAdoptionScope(outcome agentruntime.AdoptionOutcome) agentruntime.InheritedRecordScope { + if outcome.ContactedAt.IsZero() { + return agentruntime.InheritedRecordScopeNoServer + } + return agentruntime.InheritedRecordScopeForeignServer +} + +// adoptSurvivingAgentctl attempts to adopt a control server recorded by a +// prior launch. Returns nil when adoption did not complete for any reason +// (design 02's "Startup" steps 4-6): the caller then falls back to spawning +// a fresh server exactly as it would with the capability disabled. +func adoptSurvivingAgentctl( + ctx context.Context, + cfg *config.Config, + log *logger.Logger, + store agentruntime.AdoptionRecordStore, + secretStore secrets.SecretStore, +) (*agentctlLauncherResult, agentruntime.InheritedRecordScope) { + outcome := agentruntime.AttemptAdoptControlServer(ctx, store, secretStore, controlClientFactory(log), + cfg.ResolvedHomeDir(), agentruntime.RequiredSurvivalCapabilities, + cfg.Agentctl.RecoveryReadTimeout, cfg.Agentctl.RecoveryReadRetries, log) + if !outcome.Adopted { + log.Info("control server adoption did not complete; spawning a fresh one", + zap.String("reason", string(outcome.Reason))) + return nil, refusedAdoptionScope(outcome) + } + + host, port, err := splitEndpoint(outcome.Endpoint) + if err != nil { + log.Warn("adopted control server endpoint is malformed; spawning fresh instead", zap.Error(err)) + return nil, refusedAdoptionScope(outcome) + } + + log.Info("adopted a surviving control server", zap.String("endpoint", outcome.Endpoint)) + cfg.Agent.StandaloneHost = host + cfg.Agent.StandalonePort = port + cfg.Agent.StandaloneAuthToken = outcome.Credential + // This process never spawned the adopted server, so it has no PID to + // record here. Standalone liveness for an adopted record is judged by + // enumeration against the adopted server (design 02, "Persistence"), + // not by a stale PID probe -- a later layer of this feature. + cfg.Agent.StandalonePID = 0 + + // AC-EXECUTORS-CONTROL-OWNERSHIP-003.8: the rotation AttemptAdoptControlServer + // just performed already counted as the first renewal, but nothing renews + // ownership again after that without this loop. The renewal cadence is + // computed from the adopted server's OWN reported unowned period + // (outcome.UnownedPeriod), never this launch's local config: the two can + // disagree across a restart that changed agentctl.unownedPeriod, and + // renewing on a stale/mismatched cadence lets the server's own reaper + // fire while this backend still believes it owns it (Review round 3, + // finding 5). + renewer := startOwnershipRenewal(ctx, log, outcome.Endpoint, outcome.Credential, + resolveAdoptedRenewalPeriod(outcome.UnownedPeriod)) + + return &agentctlLauncherResult{ + // Survival is enabled and this server was adopted, not spawned: the + // registered cleanup must not stop it (AC-EXECUTORS-SURVIVAL-001.1), + // and there is no local process for this launcher to own regardless. + // It must still stop the renewal loop this backend started, or the + // goroutine leaks past backend shutdown. + cleanup: func() error { + if renewer != nil { + renewer.Stop() + } + return nil + }, + binaryPath: launcher.FindAgentctlBinary(), + recoveryDeadlineStart: outcome.ContactedAt, + inheritedRecordScope: agentruntime.InheritedRecordScopeAdopted, + }, agentruntime.InheritedRecordScopeAdopted +} + +// spawnFreshAgentctl is today's unconditional launch path. If the +// agent-survival capability is enabled, the freshly spawned server is also +// durably recorded as the installation's control server (failure-table row +// "Own server started after a refused or failed adoption"); recording +// failure is logged and non-fatal to startup, since the next restart simply +// finds no record and spawns fresh again. +func spawnFreshAgentctl( + ctx context.Context, + cfg *config.Config, + log *logger.Logger, + availability *agentctlclient.Availability, + store agentruntime.AdoptionRecordStore, + secretStore secrets.SecretStore, ) (*agentctlLauncherResult, error) { l, cleanup, err := launcher.Provide(ctx, launcher.Config{ Host: cfg.Agent.StandaloneHost, @@ -45,17 +200,114 @@ func provideAgentctlLauncher( zap.Int("actual_port", actualPort)) cfg.Agent.StandalonePort = actualPort } - // Store the per-launch auth token so downstream clients can authenticate + // Store the per-launch auth token so downstream clients can authenticate. cfg.Agent.StandaloneAuthToken = l.AuthToken() // Store the agentctl control-server PID so local/standalone executor rows can // carry a real host-local liveness handle (executors_running.local_pid). cfg.Agent.StandalonePID = l.Pid() + + var renewer *agentruntime.OwnershipRenewer + if cfg.Features.AgentSurvival { + endpoint := net.JoinHostPort(cfg.Agent.StandaloneHost, strconv.Itoa(cfg.Agent.StandalonePort)) + client, err := controlClientFactory(log)(endpoint) + if err != nil { + log.Warn("failed to build control client for the freshly spawned agentctl; adoption record not written", zap.Error(err)) + } else if err := agentruntime.RecordFreshControlServer(ctx, store, secretStore, client, endpoint, l.AuthToken()); err != nil { + log.Warn("failed to record freshly spawned control server", zap.Error(err)) + } + // AC-EXECUTORS-CONTROL-OWNERSHIP-003.8: the bootstrap handshake this + // launch just completed already counted as the first renewal, but + // nothing renews ownership again after that without this loop. This + // launch spawned the server moments ago from this same config, so + // (unlike the adopted path above) the local resolution IS the value + // the server itself enforces. + period, _ := ownershipperiod.Resolve(cfg.Agentctl.UnownedPeriod, cfg.Agentctl.IdleTimeout) + renewer = startOwnershipRenewal(ctx, log, endpoint, l.AuthToken(), period) + } + return &agentctlLauncherResult{ - cleanup: cleanup, + cleanup: func() error { + if renewer != nil { + renewer.Stop() + } + return cleanup() + }, binaryPath: l.BinaryPath(), }, nil } +// startOwnershipRenewal builds a dedicated control client authenticated with +// the given credential and starts the periodic ownership-claim loop against +// it (AC-EXECUTORS-CONTROL-OWNERSHIP-003.2/.8), computing the renewal +// interval from the given already-resolved unowned period. The caller +// decides where that period comes from: this launch's own local config is +// only correct for a server this launch just spawned itself -- an adopted +// server may have been spawned by a prior launch with different config, so +// its renewal period must come from what that server itself reports (see +// resolveAdoptedRenewalPeriod). Returns nil (logged, not fatal) when the +// endpoint can't be parsed; startup already succeeded by this point, so +// refusing to run the surviving server over a renewal-loop failure would +// trade a smaller problem for a bigger one. +func startOwnershipRenewal( + ctx context.Context, + log *logger.Logger, + endpoint string, + credential string, + period time.Duration, +) *agentruntime.OwnershipRenewer { + host, port, err := splitEndpoint(endpoint) + if err != nil { + log.Warn("failed to start ownership renewal loop; endpoint malformed", zap.Error(err)) + return nil + } + client := agentctlclient.NewControlClient(host, port, log) + client.SetAuthToken(credential) + renewer := agentruntime.NewOwnershipRenewer(client, ownershipperiod.RenewalInterval(period), log) + renewer.Start(ctx) + return renewer +} + +// resolveAdoptedRenewalPeriod picks the unowned period an adopted server's +// ownership-renewal loop renews against: the value that server itself +// reported on /identity during adoption (outcome.UnownedPeriod), never this +// launch's local config -- the two can disagree across a restart that +// changed agentctl.unownedPeriod, and renewing on this launch's own +// (possibly longer) cadence lets the adopted server's own reaper fire while +// this backend still believes it owns it. Falls back to the shared floor +// only for a legacy pre-upgrade server that answered /identity without an +// unowned_period_ms field at all (reported as zero). +func resolveAdoptedRenewalPeriod(reported time.Duration) time.Duration { + if reported <= 0 { + return ownershipperiod.MinPeriod + } + return reported +} + +// controlClientFactory builds the real agentruntime.AdoptionControlClientFactory +// used in production: it parses "host:port" and returns a live +// agentctlclient.ControlClient targeting it. +func controlClientFactory(log *logger.Logger) agentruntime.AdoptionControlClientFactory { + return func(endpoint string) (agentruntime.AdoptionControlClient, error) { + host, port, err := splitEndpoint(endpoint) + if err != nil { + return nil, err + } + return agentctlclient.NewControlClient(host, port, log), nil + } +} + +func splitEndpoint(endpoint string) (host string, port int, err error) { + host, portStr, err := net.SplitHostPort(endpoint) + if err != nil { + return "", 0, fmt.Errorf("split control server endpoint %q: %w", endpoint, err) + } + port, err = strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("parse control server port from endpoint %q: %w", endpoint, err) + } + return host, port, nil +} + // waitForAgentctlControlHealthy waits for the agentctl control server to be healthy. // This is called during startup to ensure agentctl is ready before accepting requests. func waitForAgentctlControlHealthy(ctx context.Context, cfg *config.Config, log *logger.Logger) { diff --git a/apps/backend/internal/backendapp/agentctl_survival_gate_test.go b/apps/backend/internal/backendapp/agentctl_survival_gate_test.go new file mode 100644 index 0000000000..f8833fc811 --- /dev/null +++ b/apps/backend/internal/backendapp/agentctl_survival_gate_test.go @@ -0,0 +1,165 @@ +package backendapp + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/common/ownershipproof" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" +) + +const ( + gateTestHomeDir = "/home/kandev-gate-test" + gateSeedCredential = "bootstrap-credential" +) + +// recordingControlServer stands in for a control server left running by a +// prior launch and records which ownership operation this launch performed +// against it. Adoption rotates the credential; reclamation shuts the server +// down. The two are mutually exclusive, so which one arrives is the whole +// observation. +type recordingControlServer struct { + *httptest.Server + mu sync.Mutex + rotated bool + shutDown bool + identity string + challenges []string +} + +func newRecordingControlServer(t *testing.T) *recordingControlServer { + t.Helper() + rec := &recordingControlServer{identity: "gate-survivor-identity"} + rec.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/identity": + _ = json.NewEncoder(w).Encode(agentctlclient.IdentityInfo{ + ServerIdentity: rec.identity, + Capabilities: lifecycle.RequiredSurvivalCapabilities, + }) + case r.Method == http.MethodPost && r.URL.Path == "/ownership/prove": + var req struct { + Challenge string `json:"challenge"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + rec.mu.Lock() + rec.challenges = append(rec.challenges, req.Challenge) + rec.mu.Unlock() + _ = json.NewEncoder(w).Encode(map[string][]string{ + "proofs": {ownershipproof.Derive(gateSeedCredential, req.Challenge, gateTestHomeDir)}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/ownership/details": + _ = json.NewEncoder(w).Encode(agentctlclient.ServerDetails{HomeDir: gateTestHomeDir}) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/ownership/rotate": + rec.mu.Lock() + rec.rotated = true + rec.mu.Unlock() + _ = json.NewEncoder(w).Encode(agentctlclient.CredentialRotationResult{ + RotationID: 1, + Credential: "gate-rotated-credential", + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/ownership/confirm": + w.WriteHeader(http.StatusOK) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/ownership/shutdown": + rec.mu.Lock() + rec.shutDown = true + rec.mu.Unlock() + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(rec.Close) + return rec +} + +func (r *recordingControlServer) observed() (rotated, shutDown bool) { + r.mu.Lock() + defer r.mu.Unlock() + return r.rotated, r.shutDown +} + +// newGateFixture wires a recorded surviving control server to a store and a +// secret store holding the credential that server was launched with. +func newGateFixture(t *testing.T) (*recordingControlServer, *fakeControlServerStore, secrets.SecretStore, *config.Config) { + t.Helper() + server := newRecordingControlServer(t) + + secretStore := newIDGeneratingSecretStore() + seed := secrets.SecretWithValue{} + seed.Name = "gate-seed" + seed.Value = gateSeedCredential + if err := secretStore.Create(context.Background(), &seed); err != nil { + t.Fatalf("seed secret: %v", err) + } + + store := &fakeControlServerStore{record: &models.ControlServerRecord{ + Endpoint: server.Listener.Addr().String(), + ServerIdentity: server.identity, + CredentialSecretID: seed.ID, + }} + + cfg := &config.Config{} + cfg.HomeDir = gateTestHomeDir + return server, store, secretStore, cfg +} + +// TestResolveSurvivingAgentctlAdoptsWhenSurvivalEnabled pins the open side of +// the capability gate: with the capability on, a recorded server that proves +// its identity is adopted, and this launch takes it over by rotating its +// credential rather than starting a second one. +func TestResolveSurvivingAgentctlAdoptsWhenSurvivalEnabled(t *testing.T) { + server, store, secretStore, cfg := newGateFixture(t) + cfg.Features.AgentSurvival = true + + result, _ := resolveSurvivingAgentctl(context.Background(), cfg, newSurvivalTestLogger(t), store, secretStore) + if result == nil { + t.Fatal("result = nil, want the recorded server to be adopted") + } + t.Cleanup(func() { _ = result.cleanup() }) + + rotated, shutDown := server.observed() + if !rotated { + t.Fatal("the adopted server's credential was never rotated") + } + if shutDown { + t.Fatal("the adopted server was shut down") + } + if cfg.Agent.StandaloneAuthToken != "gate-rotated-credential" { + t.Fatalf("StandaloneAuthToken = %q, want the rotated credential", cfg.Agent.StandaloneAuthToken) + } +} + +// TestResolveSurvivingAgentctlReclaimsDetachedServerWhenSurvivalDisabled pins +// the closed side. With the capability off, nothing may be adopted -- but a +// server a previous survival-enabled launch detached is still running with no +// backend attached to it, so this launch stops it instead of leaving it to +// time out on its own, and then falls through to a fresh spawn. +func TestResolveSurvivingAgentctlReclaimsDetachedServerWhenSurvivalDisabled(t *testing.T) { + server, store, secretStore, cfg := newGateFixture(t) + cfg.Features.AgentSurvival = false + + result, _ := resolveSurvivingAgentctl(context.Background(), cfg, newSurvivalTestLogger(t), store, secretStore) + if result != nil { + t.Fatalf("result = %+v, want nil: nothing may be adopted with the capability disabled", result) + } + + rotated, shutDown := server.observed() + if rotated { + t.Fatal("an adoption rotation was attempted with the capability disabled") + } + if !shutDown { + t.Fatal("the detached control server was left running with the capability disabled") + } + if cfg.Agent.StandaloneAuthToken != "" { + t.Fatalf("StandaloneAuthToken = %q, want empty: no server was adopted", cfg.Agent.StandaloneAuthToken) + } +} diff --git a/apps/backend/internal/backendapp/agentctl_survival_test.go b/apps/backend/internal/backendapp/agentctl_survival_test.go new file mode 100644 index 0000000000..354391099e --- /dev/null +++ b/apps/backend/internal/backendapp/agentctl_survival_test.go @@ -0,0 +1,278 @@ +package backendapp + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/common/ownershipperiod" + "github.com/kandev/kandev/internal/common/ownershipproof" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" +) + +func newSurvivalTestLogger(t *testing.T) *logger.Logger { + t.Helper() + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatalf("new logger: %v", err) + } + return log +} + +// fakeControlServerStore is a minimal lifecycle.AdoptionRecordStore double +// scoped to this file's wiring tests. +type fakeControlServerStore struct { + record *models.ControlServerRecord + upserts []*models.ControlServerRecord +} + +func (s *fakeControlServerStore) GetControlServerRecord(context.Context) (*models.ControlServerRecord, error) { + if s.record == nil { + return nil, models.ErrControlServerRecordNotFound + } + return s.record, nil +} + +func (s *fakeControlServerStore) UpsertControlServerRecord(_ context.Context, record *models.ControlServerRecord) error { + s.upserts = append(s.upserts, record) + s.record = record + return nil +} + +// idGeneratingSecretStore is a minimal secrets.SecretStore double that +// auto-generates an ID on Create, scoped to this file's wiring tests +// (mirrors the lifecycle package's own inMemorySecretStore test double). +type idGeneratingSecretStore struct { + mu sync.Mutex + store map[string]string + next int +} + +func newIDGeneratingSecretStore() *idGeneratingSecretStore { + return &idGeneratingSecretStore{store: make(map[string]string)} +} + +func (s *idGeneratingSecretStore) Create(_ context.Context, secret *secrets.SecretWithValue) error { + s.mu.Lock() + defer s.mu.Unlock() + if secret.ID == "" { + s.next++ + secret.ID = fmt.Sprintf("survival-secret-%d", s.next) + } + s.store[secret.ID] = secret.Value + return nil +} + +func (s *idGeneratingSecretStore) Get(_ context.Context, id string) (*secrets.Secret, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.store[id]; !ok { + return nil, fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + return &secrets.Secret{ID: id}, nil +} + +func (s *idGeneratingSecretStore) Reveal(_ context.Context, id string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + value, ok := s.store[id] + if !ok { + return "", fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + return value, nil +} + +func (s *idGeneratingSecretStore) Update(_ context.Context, id string, req *secrets.UpdateSecretRequest) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.store[id]; !ok { + return fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + if req.Value != nil { + s.store[id] = *req.Value + } + return nil +} + +func (s *idGeneratingSecretStore) Delete(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.store, id) + return nil +} + +func (s *idGeneratingSecretStore) List(context.Context) ([]*secrets.SecretListItem, error) { + return nil, nil +} + +func (*idGeneratingSecretStore) Close() error { return nil } + +func TestSplitEndpointParsesHostAndPort(t *testing.T) { + host, port, err := splitEndpoint("127.0.0.1:39429") + if err != nil { + t.Fatalf("splitEndpoint: %v", err) + } + if host != "127.0.0.1" || port != 39429 { + t.Fatalf("host=%q port=%d, want 127.0.0.1:39429", host, port) + } +} + +func TestSplitEndpointRejectsMalformedEndpoint(t *testing.T) { + if _, _, err := splitEndpoint("not-a-valid-endpoint"); err == nil { + t.Fatal("want error for malformed endpoint, got nil") + } +} + +// TestAdoptSurvivingAgentctlReturnsNilWhenNoRecord pins the fallback +// wiring: with no control-server record at all, adoptSurvivingAgentctl must +// return nil so provideAgentctlLauncher falls through to spawning fresh -- +// exactly like the capability being disabled. +func TestAdoptSurvivingAgentctlReturnsNilWhenNoRecord(t *testing.T) { + cfg := &config.Config{} + store := &fakeControlServerStore{} + + result, _ := adoptSurvivingAgentctl(context.Background(), cfg, newSurvivalTestLogger(t), store, nil) + + if result != nil { + t.Fatalf("result = %+v, want nil", result) + } +} + +// TestAdoptSurvivingAgentctlAdoptsAndUpdatesConfig drives the real wiring +// end to end against an httptest server standing in for a surviving +// agentctl: identity, rotate, and confirm are all real HTTP round trips +// through the production agentctl.ControlClient, not a fake. Verifies the +// adopted endpoint/credential land on cfg.Agent.Standalone* and that the +// returned result's cleanup is a no-op (nothing to stop for an adopted +// server per AC-EXECUTORS-SURVIVAL-001.1). +func TestAdoptSurvivingAgentctlAdoptsAndUpdatesConfig(t *testing.T) { + const homeDir = "/home/kandev-test" + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/identity": + _ = json.NewEncoder(w).Encode(agentctlclient.IdentityInfo{ + ServerIdentity: "survivor-identity", + Capabilities: lifecycle.RequiredSurvivalCapabilities, + }) + case r.Method == http.MethodPost && r.URL.Path == "/ownership/prove": + var req struct { + Challenge string `json:"challenge"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + _ = json.NewEncoder(w).Encode(map[string][]string{ + "proofs": {ownershipproof.Derive("bootstrap-credential", req.Challenge, homeDir)}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/api/v1/ownership/details": + _ = json.NewEncoder(w).Encode(agentctlclient.ServerDetails{ + HomeDir: homeDir, + DiagnosticLogPath: "/home/kandev-test/logs/agentctl-diagnostic.log", + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/ownership/rotate": + _ = json.NewEncoder(w).Encode(agentctlclient.CredentialRotationResult{ + RotationID: 1, + Credential: "rotated-credential", + }) + case r.Method == http.MethodPost && r.URL.Path == "/api/v1/ownership/confirm": + w.WriteHeader(http.StatusOK) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(server.Close) + + secretStore := newIDGeneratingSecretStore() + var seedSecret secrets.SecretWithValue + seedSecret.Name = "seed" + seedSecret.Value = "bootstrap-credential" + if err := secretStore.Create(context.Background(), &seedSecret); err != nil { + t.Fatalf("seed secret: %v", err) + } + + endpoint := server.Listener.Addr().String() + store := &fakeControlServerStore{record: &models.ControlServerRecord{ + Endpoint: endpoint, + ServerIdentity: "survivor-identity", + CredentialSecretID: seedSecret.ID, + }} + cfg := &config.Config{} + cfg.HomeDir = homeDir + + result, _ := adoptSurvivingAgentctl(context.Background(), cfg, newSurvivalTestLogger(t), store, secretStore) + + if result == nil { + t.Fatal("result = nil, want a non-nil adopted result") + } + if err := result.cleanup(); err != nil { + t.Fatalf("cleanup() = %v, want nil (adopted server must not be stopped)", err) + } + if cfg.Agent.StandaloneAuthToken != "rotated-credential" { + t.Fatalf("StandaloneAuthToken = %q, want rotated-credential", cfg.Agent.StandaloneAuthToken) + } + wantHost, wantPort, err := splitEndpoint(endpoint) + if err != nil { + t.Fatalf("splitEndpoint(endpoint): %v", err) + } + if cfg.Agent.StandaloneHost != wantHost || cfg.Agent.StandalonePort != wantPort { + t.Fatalf("StandaloneHost/Port = %s:%d, want %s:%d", cfg.Agent.StandaloneHost, cfg.Agent.StandalonePort, wantHost, wantPort) + } + if cfg.Agent.StandalonePID != 0 { + t.Fatalf("StandalonePID = %d, want 0 (this process never spawned the adopted server)", cfg.Agent.StandalonePID) + } + if len(store.upserts) != 1 { + t.Fatalf("upserts = %d, want 1", len(store.upserts)) + } +} + +// TestStartOwnershipRenewalReturnsNilOnMalformedEndpoint pins that a +// malformed endpoint is logged and non-fatal: startup already succeeded by +// the time this is called, so the caller must not fail the whole launch +// over a renewal-loop wiring problem. +func TestStartOwnershipRenewalReturnsNilOnMalformedEndpoint(t *testing.T) { + renewer := startOwnershipRenewal(context.Background(), newSurvivalTestLogger(t), "not-a-valid-endpoint", "cred", time.Minute) + if renewer != nil { + t.Fatal("renewer = non-nil, want nil for a malformed endpoint") + } +} + +// TestStartOwnershipRenewalStartsAndStopsCleanly pins the Start/Stop wiring +// itself (the renewal cadence and retry behaviour are covered by the +// lifecycle package's own OwnershipRenewer tests): a well-formed endpoint +// yields a running renewer, and Stop returns promptly rather than blocking +// for anything resembling the resolved renewal interval. +func TestStartOwnershipRenewalStartsAndStopsCleanly(t *testing.T) { + renewer := startOwnershipRenewal(context.Background(), newSurvivalTestLogger(t), "127.0.0.1:0", "cred", time.Minute) + if renewer == nil { + t.Fatal("renewer = nil, want a started renewer for a well-formed endpoint") + } + renewer.Stop() +} + +// TestResolveAdoptedRenewalPeriodUsesReportedValue pins Review round 3, +// finding 5's fix: an adopted server's own reported unowned period drives +// its renewal cadence, not this launch's local config. +func TestResolveAdoptedRenewalPeriodUsesReportedValue(t *testing.T) { + if got := resolveAdoptedRenewalPeriod(5 * time.Minute); got != 5*time.Minute { + t.Fatalf("resolveAdoptedRenewalPeriod(5m) = %v, want 5m", got) + } +} + +// TestResolveAdoptedRenewalPeriodFallsBackToFloorWhenUnreported pins the +// legacy/pre-upgrade fallback: a server that reported no unowned period at +// all (zero) still gets a sane, non-zero renewal cadence rather than a +// zero-interval renewal loop. +func TestResolveAdoptedRenewalPeriodFallsBackToFloorWhenUnreported(t *testing.T) { + if got := resolveAdoptedRenewalPeriod(0); got != ownershipperiod.MinPeriod { + t.Fatalf("resolveAdoptedRenewalPeriod(0) = %v, want the shared floor %v", got, ownershipperiod.MinPeriod) + } +} diff --git a/apps/backend/internal/backendapp/agents.go b/apps/backend/internal/backendapp/agents.go index 9abafbe673..7e5b40ff3f 100644 --- a/apps/backend/internal/backendapp/agents.go +++ b/apps/backend/internal/backendapp/agents.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strings" + "time" "go.uber.org/zap" @@ -24,7 +25,6 @@ import ( ) func provideLifecycleManager( - ctx context.Context, cfg *config.Config, log *logger.Logger, eventBus bus.EventBus, @@ -36,6 +36,12 @@ func provideLifecycleManager( managedRuntimeSelections managedruntime.SelectionReader, mcpIdentityScoper lifecycle.MCPIdentityScoper, mcpPrincipalScoper lifecycle.MCPPrincipalScoper, + recoveryDeadlineStart time.Time, + inheritedRecordScope lifecycle.InheritedRecordScope, + workspaceInfoProvider lifecycle.WorkspaceInfoProvider, + passthroughSessionProvider lifecycle.PassthroughSessionProvider, + runningWriter lifecycle.ExecutorRunningWriter, + startupRecoveryGuard *lifecycle.RecoveryGuard, ) (*lifecycle.Manager, error) { log.Info("Initializing Agent Manager...") secretStores := newLifecycleSecretStores(rawSecretStore) @@ -56,6 +62,8 @@ func provideLifecycleManager( cfg.Agent.StandalonePort, log, ) + // Per-instance servers enforce the same single rotating credential as + // the control server -- see the field doc on config.AgentConfig. standaloneExec.SetAuthToken(cfg.Agent.StandaloneAuthToken) // Create InteractiveRunner for passthrough mode (no WorkspaceTracker, uses callbacks) @@ -116,6 +124,36 @@ func provideLifecycleManager( log, ) + // Persistence writer for executors_running, wired before Start runs its + // recovery pass: Start reads ListLiveStandaloneExecutorsRunning to build + // the recovery-inventory correlation guard (AC-EXECUTORS-SURVIVAL-002), and + // that read silently returns no candidates when no writer is set yet. This + // must happen here rather than after provideLifecycleManager returns, or + // every recovery pass sees an empty inventory regardless of what is + // actually durable, misclassifying every live standalone instance as an + // orphan. + lifecycleMgr.SetExecutorRunningWriter(runningWriter) + + // Install the startup recovery guard before the standalone backend gets its + // unstoppable-session sink. Both must refer to the same guard instance, or a + // failed duplicate stop can retain a different guard than the one launch + // admission checks. + lifecycleMgr.SetRecoveryGuard(startupRecoveryGuard) + + // Wire recovery's bounded stop-retry budget (AC-EXECUTORS-SURVIVAL-002.13/ + // 002.15) and the AC-EXECUTORS-SURVIVAL-002.16 unstoppable-session sink + // into the standalone backend before recovery starts. + standaloneExec.SetRecoveryRetryConfig(cfg.Agentctl.RecoveryReadTimeout, cfg.Agentctl.RecoveryReadRetries) + standaloneExec.SetUnstoppableSessionRecorder(lifecycleMgr.RecoveryGuard()) + + // AC-EXECUTORS-SURVIVAL-003.7: the single bound covering this pass's + // adoption+enumeration+reconstruction work, clocked from this launch's + // first contact with a recorded control endpoint (zero when there was + // none -- Start then falls back to its own invocation time). + lifecycleMgr.SetRecoveryDeadline(cfg.Agentctl.RecoveryDeadline) + lifecycleMgr.SetRecoveryDeadlineStart(recoveryDeadlineStart) + lifecycleMgr.SetInheritedRecordScope(inheritedRecordScope) + // Register environment preparers (keyed by ExecutorType — the // "local"/"worktree"/"local_docker"/"sprites" taxonomy, not Runtime). // The Worktree preparer is registered separately in @@ -162,17 +200,47 @@ func provideLifecycleManager( if mcpPrincipalScoper != nil { lifecycleMgr.SetMCPPrincipalScoper(mcpPrincipalScoper) } - - if err := lifecycleMgr.Start(ctx); err != nil { - return nil, err + // AC-EXECUTORS-SURVIVAL-002.14: wire the workspace info provider before + // Start so a recovered execution's task-environment identity is + // reconstructed from the durable store during this same startup pass, + // instead of staying empty until something else happens to resolve it. + if workspaceInfoProvider != nil { + lifecycleMgr.SetWorkspaceInfoProvider(workspaceInfoProvider) } - - log.Info("Agent Manager initialized", + // AC-EXECUTORS-SURVIVAL-005.3: wire the durable passthrough-mode lookup + // before Start so the startup recovery guard can exclude a confirmed + // passthrough session from guarding instead of treating every recovered + // session as guardable. + if passthroughSessionProvider != nil { + lifecycleMgr.SetPassthroughLookup(passthroughLookupFromSessionProvider(passthroughSessionProvider)) + } + // Kill-path #4 (design 01 "Kill paths that must change together"): + // StopAgentWithReason needs the capability state to decide survivable + // detach versus terminating stop on backend shutdown. + lifecycleMgr.SetAgentSurvivalEnabled(cfg.Features.AgentSurvival) + log.Info("Agent Manager configured", zap.Int("runtimes", len(executorRegistry.List())), zap.Int("agent_types", len(agentRegistry.List()))) return lifecycleMgr, nil } +// passthroughLookupFromSessionProvider adapts a durable session reader into +// the lifecycle.PassthroughLookup closure consumed by +// Manager.SetPassthroughLookup (AC-EXECUTORS-SURVIVAL-005.3). A failed or +// empty read answers ok=false so the caller treats the session's mode as +// unknown and falls back to guarding it -- the safe default documented on +// Manager.passthroughLookup: guarding a passthrough session by mistake only +// costs one refused launch, never excluding a real recovery candidate. +func passthroughLookupFromSessionProvider(provider lifecycle.PassthroughSessionProvider) lifecycle.PassthroughLookup { + return func(ctx context.Context, sessionID string) (bool, bool) { + session, err := provider.GetTaskSession(ctx, sessionID) + if err != nil || session == nil { + return false, false + } + return session.IsPassthrough, true + } +} + type lifecycleSecretStores struct { runtime secrets.SecretStore credentials secrets.SecretStore diff --git a/apps/backend/internal/backendapp/helpers_test.go b/apps/backend/internal/backendapp/helpers_test.go index cfc298b0cc..c7b92ecfe6 100644 --- a/apps/backend/internal/backendapp/helpers_test.go +++ b/apps/backend/internal/backendapp/helpers_test.go @@ -472,7 +472,7 @@ func (s *shutdownDeadlineExecutor) StopInstance( return nil } -func (s *shutdownDeadlineExecutor) RecoverInstances(context.Context) ([]*lifecycle.ExecutorInstance, error) { +func (s *shutdownDeadlineExecutor) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*lifecycle.ExecutorInstance, error) { return nil, nil } diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 845bc80d67..e9060792c5 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -507,16 +507,31 @@ func startServices( //nolint:cyclop } log.Info("ACP messages will be stored as comments") + // ============================================ + // STARTUP RECOVERY GUARD (AC-EXECUTORS-SURVIVAL-002.8) + // ============================================ + // Read the live standalone recovery-inventory records and take a guard + // for every session they name (except a confirmed passthrough one) + // before contacting any control server -- including the adoption attempt + // provideAgentctlLauncher makes just below. Deferring this until the + // lifecycle manager's own Start() runs would be too late: adoption is + // this backend's first control-server contact, and design 02 "Startup" + // step 3 must precede step 4. + startupRecoveryGuard := lifecycle.TakeStartupRecoveryGuards( + ctx, repos.Task, passthroughLookupFromSessionProvider(services.Task), log) + // ============================================ // AGENTCTL LAUNCHER (for standalone mode) // ============================================ agentRuntimeAvailability := agentctlclient.NewAvailability(eventBus, log) - agentctlResult, err := provideAgentctlLauncher(ctx, cfg, log, agentRuntimeAvailability) + agentctlResult, err := provideAgentctlLauncher(ctx, cfg, log, agentRuntimeAvailability, repos.Task, repos.Secrets) if err != nil { log.Error("Failed to start agentctl subprocess", zap.Error(err)) return false } var agentctlBinaryPath string + var recoveryDeadlineStart time.Time + var inheritedRecordScope lifecycle.InheritedRecordScope if agentctlResult != nil { addCleanup(agentctlResult.cleanup) defer func() { @@ -532,10 +547,13 @@ func startServices( //nolint:cyclop // Capture the binary path so initOfficeServices can include it in the // ServiceOptions when constructing the office service. agentctlBinaryPath = agentctlResult.binaryPath + recoveryDeadlineStart = agentctlResult.recoveryDeadlineStart + inheritedRecordScope = agentctlResult.inheritedRecordScope } return startAgentInfrastructure(ctx, cfg, log, addCleanup, eventBus, agentRuntimeAvailability, - dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups, cancelWorkers) + dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, recoveryDeadlineStart, inheritedRecordScope, + startupRecoveryGuard, runCleanups, cancelWorkers) } // startAgentInfrastructure initializes the agent lifecycle manager, worktree, orchestrator, @@ -555,6 +573,9 @@ func startAgentInfrastructure( agentSettingsController *agentsettingscontroller.Controller, agentRegistry *registry.Registry, agentctlBinaryPath string, + recoveryDeadlineStart time.Time, + inheritedRecordScope lifecycle.InheritedRecordScope, + startupRecoveryGuard *lifecycle.RecoveryGuard, runCleanups func(), cancelWorkers context.CancelFunc, ) bool { @@ -585,7 +606,6 @@ func startAgentInfrastructure( // AGENT MANAGER // ============================================ lifecycleMgr, err := provideLifecycleManager( - ctx, cfg, log, eventBus, @@ -597,6 +617,12 @@ func startAgentInfrastructure( services.ManagedRuntimeSelections, mcpScopeResolver.Scope, mcpScopeResolver.ScopePrincipal, + recoveryDeadlineStart, + inheritedRecordScope, + services.Task, + services.Task, + repos.Task, + startupRecoveryGuard, ) if err != nil { log.Error("Failed to initialize agent manager", zap.Error(err)) @@ -626,7 +652,6 @@ func startAgentInfrastructure( services.Task.SetAgentComparisonTargetPusher(lifecycleMgr) services.Task.SetExecutorCapabilityProber(lifecycleMgr) - lifecycleMgr.SetWorkspaceInfoProvider(services.Task) // Session/environment-scoped HTTP surfaces (shell, files, ports, vscode, // LSP, terminals) enforce per-user workspace scoping (opt-in auth). The // GetOrEnsure* execution paths run these checks internally; the vscode and @@ -634,18 +659,11 @@ func startAgentInfrastructure( // the handler, and the SSR terminal-list routes call CheckTaskAccess / // CheckEnvironmentAccess / CheckTaskEnvironmentAccess in a route guard. wireLifecycleAccessCheckers(lifecycleMgr, services.Task) - log.Info("Workspace info provider configured for session recovery") // TODO(task-model-unification Phase 2, ADR 0004): wire agentruntime.New(lifecycleMgr) // once a real consumer (workflow-engine / cron-driven trigger handlers) exists. // Allocating the facade in Phase 1 without a caller is dead code. - // Persistence writer for executors_running. This makes the lifecycle manager - // the sole writer of agent_execution_id / container_id / runtime / status — - // the structural fix for the agent-execution-id divergence bug. Must be set - // before any Launch / EnsureWorkspaceExecutionForSession can run. - lifecycleMgr.SetExecutorRunningWriter(repos.Task) - // Lets user shell terminals export the executor profile's env vars, so the // terminal sees the same variables the agent subprocess and the repository // setup script get. @@ -693,6 +711,9 @@ func startAgentInfrastructure( } services.Task.SetWorkflowMovePreflight(orchestratorSvc) orchestratorSvc.SetAgentctlBinaryPath(agentctlBinaryPath) + // The checker is populated by lifecycleMgr.Start below before the + // orchestrator's startup reconciliation runs. + orchestratorSvc.SetRetrackedSessionChecker(lifecycleMgr.WasSessionRetracked) orchestratorSvc.SetRouteActionHandler(dynamicRouteActionHandler( repos.Task, repos.AgentSettings, @@ -867,6 +888,22 @@ func startAgentInfrastructure( // overlapping that would race the pool swap. databaseQuiesce = addRuntimeCleanup(deliveryCleanup) + // Recovery publishes agent events synchronously. Subscribe the orchestrator + // after all event-handler dependencies are wired, but before lifecycle + // recovery, so a retained terminal outcome cannot be published into an empty + // in-memory bus while Service.Start is still doing its startup reconciliation. + // Service.Start calls Watcher.Start again; the watcher is idempotent and keeps + // these subscriptions. + if err := orchestratorSvc.StartEventWatcher(ctx); err != nil { + log.Error("Failed to start orchestrator event watcher for agent recovery", zap.Error(err)) + return false + } + if err := lifecycleMgr.Start(ctx); err != nil { + _ = orchestratorSvc.StopEventWatcher() + log.Error("Failed to recover agent manager", zap.Error(err)) + return false + } + return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services, agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath, func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelWorkers, restoreCleanups, databaseQuiesce) @@ -991,6 +1028,8 @@ func startGatewayAndServe( hostControlClient := agentctlclient.NewControlClient(cfg.Agent.StandaloneHost, cfg.Agent.StandalonePort, log, agentctlclient.WithControlAuthToken(cfg.Agent.StandaloneAuthToken)) hostUtilityMgr := hostutility.NewManager(agentRegistry, cfg.Agent.StandaloneHost, cfg.Agent.StandalonePort, hostControlClient, log) + // Per-instance servers enforce the same single rotating credential as + // the control server -- see config.AgentConfig's field doc. hostUtilityMgr.SetAuthToken(cfg.Agent.StandaloneAuthToken) hostUtilityMgr.SetProfileResolver(profilebinding.New(repos.AgentSettings, func(agentID string) bool { _, ok := agentRegistry.GetInferenceAgent(agentID) diff --git a/apps/backend/internal/backendapp/passthrough_lookup_test.go b/apps/backend/internal/backendapp/passthrough_lookup_test.go new file mode 100644 index 0000000000..c3aeb17e4c --- /dev/null +++ b/apps/backend/internal/backendapp/passthrough_lookup_test.go @@ -0,0 +1,86 @@ +package backendapp + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +type fakePassthroughSessionProvider struct { + session *models.TaskSession + err error +} + +func (f *fakePassthroughSessionProvider) GetTaskSession(_ context.Context, _ string) (*models.TaskSession, error) { + return f.session, f.err +} + +// TestPassthroughLookupFromSessionProviderReturnsSnapshotOnSuccess pins +// Review round 3, finding 6: the closure wired into +// Manager.SetPassthroughLookup must answer with the durable +// TaskSession.IsPassthrough snapshot, not the zero value, when the read +// succeeds. +func TestPassthroughLookupFromSessionProviderReturnsSnapshotOnSuccess(t *testing.T) { + provider := &fakePassthroughSessionProvider{session: &models.TaskSession{IsPassthrough: true}} + lookup := passthroughLookupFromSessionProvider(provider) + + isPassthrough, ok := lookup(context.Background(), "session-1") + + if !ok { + t.Fatal("ok = false, want true on a successful read") + } + if !isPassthrough { + t.Fatal("isPassthrough = false, want true") + } +} + +func TestPassthroughLookupFromSessionProviderReturnsFalseSnapshotOnSuccess(t *testing.T) { + provider := &fakePassthroughSessionProvider{session: &models.TaskSession{IsPassthrough: false}} + lookup := passthroughLookupFromSessionProvider(provider) + + isPassthrough, ok := lookup(context.Background(), "session-1") + + if !ok { + t.Fatal("ok = false, want true on a successful read") + } + if isPassthrough { + t.Fatal("isPassthrough = true, want false") + } +} + +// TestPassthroughLookupFromSessionProviderFailsOpenToUnknownOnError pins the +// safe-default half of finding 6: a failed durable read must answer +// ok=false (unknown), never fabricate isPassthrough=true/false, so the +// recovery-guard caller falls back to guarding the session. +func TestPassthroughLookupFromSessionProviderFailsOpenToUnknownOnError(t *testing.T) { + provider := &fakePassthroughSessionProvider{err: errors.New("db unavailable")} + lookup := passthroughLookupFromSessionProvider(provider) + + isPassthrough, ok := lookup(context.Background(), "session-1") + + if ok { + t.Fatal("ok = true, want false when the durable read fails") + } + if isPassthrough { + t.Fatal("isPassthrough = true, want false alongside ok=false") + } +} + +// TestPassthroughLookupFromSessionProviderFailsOpenToUnknownOnNilSession +// covers a provider answering successfully with a nil session (e.g. scoping +// denial folded into a nil, no-error return by some future implementation). +func TestPassthroughLookupFromSessionProviderFailsOpenToUnknownOnNilSession(t *testing.T) { + provider := &fakePassthroughSessionProvider{session: nil, err: nil} + lookup := passthroughLookupFromSessionProvider(provider) + + isPassthrough, ok := lookup(context.Background(), "session-1") + + if ok { + t.Fatal("ok = true, want false when the provider answers with a nil session") + } + if isPassthrough { + t.Fatal("isPassthrough = true, want false alongside ok=false") + } +} diff --git a/apps/backend/internal/backendapp/session_recovery_guard_wrap_test.go b/apps/backend/internal/backendapp/session_recovery_guard_wrap_test.go new file mode 100644 index 0000000000..5496a3949c --- /dev/null +++ b/apps/backend/internal/backendapp/session_recovery_guard_wrap_test.go @@ -0,0 +1,38 @@ +package backendapp + +import ( + "errors" + "testing" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/orchestrator" + "github.com/stretchr/testify/require" +) + +func TestWrapSessionRecoveryGuardErrorTranslatesRetryableSentinel(t *testing.T) { + err := wrapSessionRecoveryGuardError(lifecycle.ErrSessionRecoveryGuarded, "session-1") + + var guardErr *orchestrator.SessionRecoveryGuardError + require.ErrorAs(t, err, &guardErr) + require.True(t, guardErr.Retryable) + require.Equal(t, "session-1", guardErr.SessionID) + require.ErrorIs(t, err, lifecycle.ErrSessionRecoveryGuarded) +} + +func TestWrapSessionRecoveryGuardErrorTranslatesNonRetryableSentinel(t *testing.T) { + err := wrapSessionRecoveryGuardError(lifecycle.ErrSessionUnstoppableAgent, "session-2") + + var guardErr *orchestrator.SessionRecoveryGuardError + require.ErrorAs(t, err, &guardErr) + require.False(t, guardErr.Retryable) + require.Equal(t, "session-2", guardErr.SessionID) + require.ErrorIs(t, err, lifecycle.ErrSessionUnstoppableAgent) +} + +func TestWrapSessionRecoveryGuardErrorPassesThroughUnrelatedError(t *testing.T) { + cause := errors.New("boom") + + err := wrapSessionRecoveryGuardError(cause, "session-3") + + require.Same(t, cause, err) +} diff --git a/apps/backend/internal/common/config/agentctl.go b/apps/backend/internal/common/config/agentctl.go index dcca562067..9a45a1c39a 100644 --- a/apps/backend/internal/common/config/agentctl.go +++ b/apps/backend/internal/common/config/agentctl.go @@ -21,6 +21,20 @@ type AgentctlStartupConfig struct { IdleReaperInterval time.Duration `json:"idleReaperInterval"` NotificationQueueCapacity int `json:"notificationQueueCapacity"` OTLPEndpoint string `json:"otlpEndpoint"` + // UnownedPeriod and DetachedEventLimit are agentctl-owned agent-survival + // tunables. Zero means "not resolved by an agent-survival-aware backend"; + // agentctl falls back to its own built-in defaults in that case (see + // Validate below), the same compatibility posture already used for a + // backend older than this contract. + UnownedPeriod time.Duration `json:"unownedPeriod"` + DetachedEventLimit int `json:"detachedEventLimit"` + // AgentSurvivalEnabled carries the current value of the + // features.agentSurvival runtime flag for this launch. Unlike + // UnownedPeriod/DetachedEventLimit, false is not "unresolved" -- a + // managed launch always sets Configured=true, so false is a meaningful + // "disabled for this launch" answer agentctl must not second-guess with + // its own default. + AgentSurvivalEnabled bool `json:"agentSurvivalEnabled"` } // ManagedAgentctlStartupConfig returns the agentctl settings resolved by the @@ -35,6 +49,9 @@ func (c *Config) ManagedAgentctlStartupConfig() AgentctlStartupConfig { IdleReaperInterval: c.Agentctl.IdleReaperInterval, NotificationQueueCapacity: c.Agentctl.NotificationQueueCapacity, OTLPEndpoint: c.Observability.OTLPEndpoint, + UnownedPeriod: c.Agentctl.UnownedPeriod, + DetachedEventLimit: c.Agentctl.DetachedEventLimit, + AgentSurvivalEnabled: c.Features.AgentSurvival, } } @@ -52,6 +69,15 @@ func (c AgentctlStartupConfig) Validate() error { if c.NotificationQueueCapacity < 1024 || c.NotificationQueueCapacity > 131072 { return fmt.Errorf("agentctl notification queue capacity must be between 1024 and 131072") } + // Zero means "not resolved by an agent-survival-aware caller"; agentctl + // substitutes its own built-in default in that case. Any other value must + // already be within the contract's accepted range. + if c.UnownedPeriod < 0 { + return fmt.Errorf("agentctl unowned period must be zero (unset) or positive") + } + if c.DetachedEventLimit != 0 && (c.DetachedEventLimit < 1 || c.DetachedEventLimit > 10000) { + return fmt.Errorf("agentctl detached event limit must be zero (unset) or between 1 and 10000") + } return nil } diff --git a/apps/backend/internal/common/config/catalog.go b/apps/backend/internal/common/config/catalog.go index 32dd8fb693..a54a146a70 100644 --- a/apps/backend/internal/common/config/catalog.go +++ b/apps/backend/internal/common/config/catalog.go @@ -100,6 +100,11 @@ var startupCatalog = []CatalogEntry{ {Key: "agentctl.idleTimeout", EnvVars: []string{"KANDEV_ACP_IDLE_TIMEOUT"}, Owner: "agentctl", Default: "1h"}, {Key: "agentctl.idleReaperInterval", EnvVars: []string{"KANDEV_ACP_IDLE_REAPER_INTERVAL"}, Owner: "agentctl", Default: "1m"}, {Key: "agentctl.notificationQueueCapacity", EnvVars: []string{"KANDEV_ACP_NOTIF_QUEUE"}, Owner: "agentctl", Default: "131072"}, + {Key: "agentctl.recoveryDeadline", EnvVars: []string{"KANDEV_ACP_RECOVERY_DEADLINE"}, Owner: "backend", Default: "30s"}, + {Key: "agentctl.recoveryReadTimeout", EnvVars: []string{"KANDEV_ACP_RECOVERY_READ_TIMEOUT"}, Owner: "backend", Default: "2s"}, + {Key: "agentctl.recoveryReadRetries", EnvVars: []string{"KANDEV_ACP_RECOVERY_READ_RETRIES"}, Owner: "backend", Default: "2"}, + {Key: "agentctl.unownedPeriod", EnvVars: []string{"KANDEV_ACP_UNOWNED_PERIOD"}, Owner: "agentctl", Default: "10m"}, + {Key: "agentctl.detachedEventLimit", EnvVars: []string{"KANDEV_ACP_DETACHED_EVENT_LIMIT"}, Owner: "agentctl", Default: "100"}, {Key: "planning.coalesceWindowMs", EnvVars: []string{"KANDEV_PLAN_COALESCE_WINDOW_MS"}, Owner: "planning", Default: "300000"}, {Key: "observability.otlpEndpoint", EnvVars: []string{"OTEL_EXPORTER_OTLP_ENDPOINT"}, Owner: "observability", Default: "", Sensitive: true}, {Key: "launcher.webPort", EnvVars: []string{"KANDEV_WEB_PORT"}, Owner: "launcher", Default: "automatic"}, @@ -133,6 +138,7 @@ var startupExclusions = []CatalogExclusion{ {EnvVar: "KANDEV_FEATURES_CLAUDE_BACKGROUND_PROMPT_HANDOFF", Class: "debug", Reason: "runtime feature flag registry"}, {EnvVar: "KANDEV_FEATURES_CLAUDE_MID_TURN_STEERING", Class: "debug", Reason: "runtime feature flag registry"}, {EnvVar: "KANDEV_FEATURES_OFFICE_SESSION_IDENTITY", Class: "debug", Reason: "runtime feature flag registry"}, + {EnvVar: "KANDEV_FEATURES_AGENT_SURVIVAL", Class: "profile", Reason: "runtime feature flag registry"}, {EnvVar: "KANDEV_DEBUG_AGENT_MESSAGES", Class: "debug", Reason: "ACP frame diagnostics"}, {EnvVar: "KANDEV_DEBUG_ACP_MAX_FILES", Class: "debug", Reason: "ACP debug retention"}, {EnvVar: "KANDEV_DEBUG_ACP_RETENTION_HOURS", Class: "debug", Reason: "ACP debug retention"}, diff --git a/apps/backend/internal/common/config/catalog_test.go b/apps/backend/internal/common/config/catalog_test.go index e29e4d5a25..e9b7aaaec5 100644 --- a/apps/backend/internal/common/config/catalog_test.go +++ b/apps/backend/internal/common/config/catalog_test.go @@ -123,6 +123,11 @@ func auditedStartupEnvironmentInventory() []auditedStartupEnvironment { {envVar: "KANDEV_ACP_IDLE_TIMEOUT", class: "catalog"}, {envVar: "KANDEV_ACP_IDLE_REAPER_INTERVAL", class: "catalog"}, {envVar: "KANDEV_ACP_NOTIF_QUEUE", class: "catalog"}, + {envVar: "KANDEV_ACP_RECOVERY_DEADLINE", class: "catalog"}, + {envVar: "KANDEV_ACP_RECOVERY_READ_TIMEOUT", class: "catalog"}, + {envVar: "KANDEV_ACP_RECOVERY_READ_RETRIES", class: "catalog"}, + {envVar: "KANDEV_ACP_UNOWNED_PERIOD", class: "catalog"}, + {envVar: "KANDEV_ACP_DETACHED_EVENT_LIMIT", class: "catalog"}, {envVar: "KANDEV_PLAN_COALESCE_WINDOW_MS", class: "catalog"}, {envVar: "OTEL_EXPORTER_OTLP_ENDPOINT", class: "catalog"}, {envVar: "KANDEV_WEB_PORT", class: "catalog"}, @@ -153,6 +158,7 @@ func auditedStartupEnvironmentInventory() []auditedStartupEnvironment { {envVar: "KANDEV_FEATURES_CLAUDE_BACKGROUND_PROMPT_HANDOFF", class: "exclusion"}, {envVar: "KANDEV_FEATURES_CLAUDE_MID_TURN_STEERING", class: "exclusion"}, {envVar: "KANDEV_FEATURES_OFFICE_SESSION_IDENTITY", class: "exclusion"}, + {envVar: "KANDEV_FEATURES_AGENT_SURVIVAL", class: "exclusion"}, {envVar: "KANDEV_DEBUG_AGENT_MESSAGES", class: "exclusion"}, {envVar: "KANDEV_DEBUG_ACP_MAX_FILES", class: "exclusion"}, {envVar: "KANDEV_DEBUG_ACP_RETENTION_HOURS", class: "exclusion"}, @@ -230,6 +236,9 @@ func TestAgentctlStartupConfigRoundTripsAndRejectsInvalidValues(t *testing.T) { IdleReaperInterval: 3 * time.Minute, NotificationQueueCapacity: 4096, OTLPEndpoint: "http://collector:4318", + UnownedPeriod: 10 * time.Minute, + DetachedEventLimit: 100, + AgentSurvivalEnabled: true, } raw, err := EncodeAgentctlStartupConfig(want) if err != nil { @@ -248,4 +257,47 @@ func TestAgentctlStartupConfigRoundTripsAndRejectsInvalidValues(t *testing.T) { if _, err := EncodeAgentctlStartupConfig(invalid); err == nil { t.Fatal("EncodeAgentctlStartupConfig accepted an invalid queue capacity") } + + invalidUnownedPeriod := want + invalidUnownedPeriod.UnownedPeriod = -time.Minute + if _, err := EncodeAgentctlStartupConfig(invalidUnownedPeriod); err == nil { + t.Fatal("EncodeAgentctlStartupConfig accepted a negative unowned period") + } + + // Zero is accepted for both fields: an agent-survival-unaware caller (or + // an older backend build) omits them, and agentctl substitutes its own + // built-in default rather than treating an absent value as invalid. + unset := want + unset.UnownedPeriod = 0 + unset.DetachedEventLimit = 0 + if _, err := EncodeAgentctlStartupConfig(unset); err != nil { + t.Fatalf("EncodeAgentctlStartupConfig rejected zero-valued (unset) survival tunables: %v", err) + } + + invalidDetachedEventLimit := want + invalidDetachedEventLimit.DetachedEventLimit = 10001 + if _, err := EncodeAgentctlStartupConfig(invalidDetachedEventLimit); err == nil { + t.Fatal("EncodeAgentctlStartupConfig accepted an out-of-range detached event limit") + } +} + +// TestManagedAgentctlStartupConfigReflectsAgentSurvivalFlag pins that the +// resolved child contract carries the current value of the +// features.agentSurvival runtime flag (Config.Features.AgentSurvival) for +// this launch, mirroring the always-on UnownedPeriod/DetachedEventLimit +// passthrough rather than being gated on either of those being non-zero. +func TestManagedAgentctlStartupConfigReflectsAgentSurvivalFlag(t *testing.T) { + cfg := &Config{} + cfg.Agentctl.NotificationQueueCapacity = 4096 + cfg.Agentctl.IdleReaperInterval = time.Minute + + cfg.Features.AgentSurvival = false + if got := cfg.ManagedAgentctlStartupConfig(); got.AgentSurvivalEnabled { + t.Fatal("AgentSurvivalEnabled = true, want false when the runtime flag is off") + } + + cfg.Features.AgentSurvival = true + if got := cfg.ManagedAgentctlStartupConfig(); !got.AgentSurvivalEnabled { + t.Fatal("AgentSurvivalEnabled = false, want true when the runtime flag is on") + } } diff --git a/apps/backend/internal/common/config/config.go b/apps/backend/internal/common/config/config.go index 22efc38837..471de923cb 100644 --- a/apps/backend/internal/common/config/config.go +++ b/apps/backend/internal/common/config/config.go @@ -83,6 +83,21 @@ type AgentctlConfig struct { IdleTimeout time.Duration `mapstructure:"idleTimeout"` IdleReaperInterval time.Duration `mapstructure:"idleReaperInterval"` NotificationQueueCapacity int `mapstructure:"notificationQueueCapacity"` + + // RecoveryDeadline is the single global bound covering adoption, + // enumeration and reconstruction together during standalone agent + // survival re-tracking at backend startup. + RecoveryDeadline time.Duration `mapstructure:"recoveryDeadline"` + // RecoveryReadTimeout and RecoveryReadRetries bound each read of a + // reconstruction value from the adopted control server during re-tracking. + RecoveryReadTimeout time.Duration `mapstructure:"recoveryReadTimeout"` + RecoveryReadRetries int `mapstructure:"recoveryReadRetries"` + // UnownedPeriod is how long a standalone control server tolerates having + // no owning backend before it stops its instances and exits. + UnownedPeriod time.Duration `mapstructure:"unownedPeriod"` + // DetachedEventLimit bounds the per-instance retained-event count while + // a standalone control server has no owning backend attached. + DetachedEventLimit int `mapstructure:"detachedEventLimit"` } // PlanningConfig contains planning service timing settings. @@ -486,6 +501,13 @@ type FeaturesConfig struct { // kill switch that restores runner-seat binding and task-active-session // decision re-evaluation when disabled. OfficeSessionIdentity bool `mapstructure:"office_session_identity" json:"officeSessionIdentity"` + + // AgentSurvival lets a worktree or local-executor agent session survive a + // backend restart by adopting its still-running standalone control server + // instead of killing it. Off in every embedded profile, and unavailable on + // Windows (survival trades the platform's kill-on-job-close safeguard for + // an adoption handshake, which is untested there). + AgentSurvival bool `mapstructure:"agent_survival" json:"agentSurvival"` } // LoggingConfig holds logging configuration. @@ -536,8 +558,15 @@ type AgentConfig struct { // StandalonePort is the control port for standalone agentctl (default: 39429) StandalonePort int `mapstructure:"standalonePort"` - // StandaloneAuthToken is the per-launch auth token retrieved via handshake. - // Set at runtime after agentctl starts; not persisted in config files. + // StandaloneAuthToken is the single credential that authenticates both + // the agentctl *control server* (instance create/list/delete, health, + // ownership rotate/confirm) and every per-instance agentctl server it + // supervises (/agent/stream, file tree, shell, ...), per design 01 + // "Single driver" (AC-EXECUTORS-CONTROL-OWNERSHIP-002.6). After adopting + // a surviving control server it holds the freshly rotated credential + // (AC-EXECUTORS-CONTROL-OWNERSHIP-002); use it for every client built + // against this control server or any instance it supervises. Set at + // runtime after agentctl starts; not persisted in config files. StandaloneAuthToken string `mapstructure:"-"` // StandalonePID is the OS process id of the standalone agentctl control-server @@ -782,6 +811,9 @@ func loadWithPath(configPath, homeDir string) (*Config, error) { return nil, fmt.Errorf("error unmarshaling config: %w", err) } sources := applyStartupDefaultsAndEnvironment(&cfg, yamlKeys, profileDefaults, envSnapshot) + if err := applySurvivalRecoveryEnv(&cfg, envSnapshot, sources); err != nil { + return nil, fmt.Errorf("config validation failed: %w", err) + } warnings := inspectSecretPermissions(selection, v) cfg.Source = buildConfigSource(selection, v, sources, warnings) diff --git a/apps/backend/internal/common/config/source.go b/apps/backend/internal/common/config/source.go index 169cb2e856..caa33af107 100644 --- a/apps/backend/internal/common/config/source.go +++ b/apps/backend/internal/common/config/source.go @@ -75,6 +75,11 @@ var yamlOnlyStartupKeys = map[string]struct{}{ "agentctl.idleTimeout": {}, "agentctl.idleReaperInterval": {}, "agentctl.notificationQueueCapacity": {}, + "agentctl.recoveryDeadline": {}, + "agentctl.recoveryReadTimeout": {}, + "agentctl.recoveryReadRetries": {}, + "agentctl.unownedPeriod": {}, + "agentctl.detachedEventLimit": {}, "planning.coalesceWindowMs": {}, "office.schedulerTickMs": {}, "observability.otlpEndpoint": {}, @@ -291,6 +296,11 @@ func applyStartupDefaults(cfg *Config, yamlKeys map[string]bool, profileDefaults setDefaultDuration("agentctl.idleTimeout", &cfg.Agentctl.IdleTimeout, time.Hour) setDefaultDuration("agentctl.idleReaperInterval", &cfg.Agentctl.IdleReaperInterval, time.Minute) setDefaultInt("agentctl.notificationQueueCapacity", &cfg.Agentctl.NotificationQueueCapacity, 131072) + setDefaultDuration("agentctl.recoveryDeadline", &cfg.Agentctl.RecoveryDeadline, 30*time.Second) + setDefaultDuration("agentctl.recoveryReadTimeout", &cfg.Agentctl.RecoveryReadTimeout, 2*time.Second) + setDefaultInt("agentctl.recoveryReadRetries", &cfg.Agentctl.RecoveryReadRetries, 2) + setDefaultDuration("agentctl.unownedPeriod", &cfg.Agentctl.UnownedPeriod, 10*time.Minute) + setDefaultInt("agentctl.detachedEventLimit", &cfg.Agentctl.DetachedEventLimit, 100) setDefaultInt("planning.coalesceWindowMs", &cfg.Planning.CoalesceWindowMs, 300000) setDefaultInt("office.schedulerTickMs", &cfg.Office.SchedulerTickMs, 5000) if !yamlKeys["observability.otlpEndpoint"] { @@ -314,6 +324,16 @@ func applyStartupEnvironment(cfg *Config, envSnapshot map[string]string, sources applyNonNegativeDurationEnv("agentctl.idleTimeout", &cfg.Agentctl.IdleTimeout, time.Hour, envSnapshot, sources) applyDurationEnv("agentctl.idleReaperInterval", &cfg.Agentctl.IdleReaperInterval, time.Minute, envSnapshot, sources) applyBoundedIntEnv("agentctl.notificationQueueCapacity", &cfg.Agentctl.NotificationQueueCapacity, 131072, 1024, 131072, envSnapshot, sources) + // UnownedPeriod's ordering-against-idle-timeout clamp and 1-minute floor + // are applied by agentctl itself at control-server start (it is the only + // place both values are simultaneously known); the backend only carries + // a sane positive duration through to the managed-child startup contract. + applyNonNegativeDurationEnv("agentctl.unownedPeriod", &cfg.Agentctl.UnownedPeriod, 10*time.Minute, envSnapshot, sources) + // agentctl.recoveryDeadline, agentctl.recoveryReadTimeout, + // agentctl.recoveryReadRetries and agentctl.detachedEventLimit are + // validated strictly (reject, never clamp) by applySurvivalRecoveryEnv, + // called separately by the caller so an out-of-range value fails startup + // instead of silently substituting the built-in default. applyNonNegativeIntEnv("planning.coalesceWindowMs", &cfg.Planning.CoalesceWindowMs, 300000, envSnapshot, sources) applyPositiveIntEnv("office.schedulerTickMs", &cfg.Office.SchedulerTickMs, 5000, envSnapshot, sources) applyStringEnvAllowEmpty("observability.otlpEndpoint", &cfg.Observability.OTLPEndpoint, envSnapshot, sources) @@ -435,6 +455,68 @@ func applyBoundedIntEnv(key string, target *int, fallback, minimum, maximum int, sources[key] = SourceEnvironment } +// applySurvivalRecoveryEnv validates the four agent-survival tunables whose +// contract requires rejecting an out-of-range value at startup rather than +// silently substituting the built-in default (unlike every other env-sourced +// setting in this file, which falls back to its default on a bad value). +func applySurvivalRecoveryEnv(cfg *Config, env map[string]string, sources map[string]SettingSource) error { + if err := applyStrictDurationEnv("agentctl.recoveryDeadline", &cfg.Agentctl.RecoveryDeadline, time.Second, 5*time.Minute, env, sources); err != nil { + return err + } + if err := applyStrictDurationEnv("agentctl.recoveryReadTimeout", &cfg.Agentctl.RecoveryReadTimeout, 100*time.Millisecond, time.Hour, env, sources); err != nil { + return err + } + if err := applyStrictIntEnv("agentctl.recoveryReadRetries", &cfg.Agentctl.RecoveryReadRetries, 0, 10, env, sources); err != nil { + return err + } + if err := applyStrictIntEnv("agentctl.detachedEventLimit", &cfg.Agentctl.DetachedEventLimit, 1, 10000, env, sources); err != nil { + return err + } + return nil +} + +func applyStrictDurationEnv(key string, target *time.Duration, minimum, maximum time.Duration, env map[string]string, sources map[string]SettingSource) error { + entry, ok := CatalogEntryForKey(key) + if !ok { + return nil + } + raw, found := nonEmptyEnv(env, entry.EnvVars...) + if !found { + return nil + } + parsed, err := time.ParseDuration(strings.TrimSpace(raw)) + if err != nil { + return fmt.Errorf("%s: invalid duration %q from %s", key, raw, entry.EnvVars[0]) + } + if parsed < minimum || parsed > maximum { + return fmt.Errorf("%s: %s is outside the accepted range [%s, %s]", key, parsed, minimum, maximum) + } + *target = parsed + sources[key] = SourceEnvironment + return nil +} + +func applyStrictIntEnv(key string, target *int, minimum, maximum int, env map[string]string, sources map[string]SettingSource) error { + entry, ok := CatalogEntryForKey(key) + if !ok { + return nil + } + raw, found := nonEmptyEnv(env, entry.EnvVars...) + if !found { + return nil + } + parsed, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil { + return fmt.Errorf("%s: invalid integer %q from %s", key, raw, entry.EnvVars[0]) + } + if parsed < minimum || parsed > maximum { + return fmt.Errorf("%s: %d is outside the accepted range [%d, %d]", key, parsed, minimum, maximum) + } + *target = parsed + sources[key] = SourceEnvironment + return nil +} + func applyBoolEnv(key string, target *bool, fallback bool, env map[string]string, sources map[string]SettingSource) { entry, ok := CatalogEntryForKey(key) if !ok { diff --git a/apps/backend/internal/common/config/survival_recovery_config_test.go b/apps/backend/internal/common/config/survival_recovery_config_test.go new file mode 100644 index 0000000000..77f4208698 --- /dev/null +++ b/apps/backend/internal/common/config/survival_recovery_config_test.go @@ -0,0 +1,156 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestSurvivalRecoveryConfigDefaults(t *testing.T) { + cfg, err := LoadWithPath(t.TempDir()) + if err != nil { + t.Fatalf("LoadWithPath: %v", err) + } + if cfg.Agentctl.RecoveryDeadline != 30*time.Second { + t.Errorf("RecoveryDeadline = %v, want 30s", cfg.Agentctl.RecoveryDeadline) + } + if cfg.Agentctl.RecoveryReadTimeout != 2*time.Second { + t.Errorf("RecoveryReadTimeout = %v, want 2s", cfg.Agentctl.RecoveryReadTimeout) + } + if cfg.Agentctl.RecoveryReadRetries != 2 { + t.Errorf("RecoveryReadRetries = %d, want 2", cfg.Agentctl.RecoveryReadRetries) + } + if cfg.Agentctl.UnownedPeriod != 10*time.Minute { + t.Errorf("UnownedPeriod = %v, want 10m", cfg.Agentctl.UnownedPeriod) + } + if cfg.Agentctl.DetachedEventLimit != 100 { + t.Errorf("DetachedEventLimit = %d, want 100", cfg.Agentctl.DetachedEventLimit) + } +} + +func TestSurvivalRecoveryConfigYAMLRoundTrips(t *testing.T) { + dir := t.TempDir() + contents := `agentctl: + recoveryDeadline: 45s + recoveryReadTimeout: 500ms + recoveryReadRetries: 3 + unownedPeriod: 15m + detachedEventLimit: 250 +` + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(contents), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + cfg, err := LoadWithPath(dir) + if err != nil { + t.Fatalf("LoadWithPath: %v", err) + } + if cfg.Agentctl.RecoveryDeadline != 45*time.Second { + t.Errorf("RecoveryDeadline = %v, want 45s", cfg.Agentctl.RecoveryDeadline) + } + if cfg.Agentctl.RecoveryReadTimeout != 500*time.Millisecond { + t.Errorf("RecoveryReadTimeout = %v, want 500ms", cfg.Agentctl.RecoveryReadTimeout) + } + if cfg.Agentctl.RecoveryReadRetries != 3 { + t.Errorf("RecoveryReadRetries = %d, want 3", cfg.Agentctl.RecoveryReadRetries) + } + if cfg.Agentctl.UnownedPeriod != 15*time.Minute { + t.Errorf("UnownedPeriod = %v, want 15m", cfg.Agentctl.UnownedPeriod) + } + if cfg.Agentctl.DetachedEventLimit != 250 { + t.Errorf("DetachedEventLimit = %d, want 250", cfg.Agentctl.DetachedEventLimit) + } +} + +func TestSurvivalRecoveryConfigEnvOverridesYAML(t *testing.T) { + dir := t.TempDir() + contents := "agentctl:\n recoveryDeadline: 45s\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(contents), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + t.Setenv("KANDEV_ACP_RECOVERY_DEADLINE", "1m") + cfg, err := LoadWithPath(dir) + if err != nil { + t.Fatalf("LoadWithPath: %v", err) + } + if cfg.Agentctl.RecoveryDeadline != time.Minute { + t.Errorf("RecoveryDeadline = %v, want 1m (env should win over YAML)", cfg.Agentctl.RecoveryDeadline) + } +} + +func TestSurvivalRecoveryConfigRejectsOutOfRangeEnv(t *testing.T) { + cases := []struct { + name string + envVar string + value string + }{ + {"recoveryDeadline too low", "KANDEV_ACP_RECOVERY_DEADLINE", "500ms"}, + {"recoveryDeadline too high", "KANDEV_ACP_RECOVERY_DEADLINE", "10m"}, + {"recoveryReadTimeout too low", "KANDEV_ACP_RECOVERY_READ_TIMEOUT", "50ms"}, + {"recoveryReadRetries negative", "KANDEV_ACP_RECOVERY_READ_RETRIES", "-1"}, + {"recoveryReadRetries too high", "KANDEV_ACP_RECOVERY_READ_RETRIES", "11"}, + {"detachedEventLimit zero", "KANDEV_ACP_DETACHED_EVENT_LIMIT", "0"}, + {"detachedEventLimit too high", "KANDEV_ACP_DETACHED_EVENT_LIMIT", "10001"}, + {"recoveryDeadline malformed", "KANDEV_ACP_RECOVERY_DEADLINE", "not-a-duration"}, + {"recoveryReadRetries malformed", "KANDEV_ACP_RECOVERY_READ_RETRIES", "not-an-int"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(tc.envVar, tc.value) + if _, err := LoadWithPath(t.TempDir()); err == nil { + t.Fatalf("LoadWithPath: expected rejection for %s=%q, got no error (value must be rejected, not clamped to default)", tc.envVar, tc.value) + } + }) + } +} + +func TestSurvivalRecoveryConfigRejectsOutOfRangeYAML(t *testing.T) { + dir := t.TempDir() + contents := "agentctl:\n recoveryDeadline: 10m\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(contents), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + if _, err := LoadWithPath(dir); err == nil { + t.Fatal("LoadWithPath: expected rejection for out-of-range YAML recoveryDeadline, got no error") + } +} + +func TestSurvivalRecoveryConfigRejectsReadBudgetOverCeiling(t *testing.T) { + dir := t.TempDir() + // timeout(4s) * (retries(2)+1) = 12s exceeds min(6s, deadline(30s)/5=6s) = 6s. + contents := "agentctl:\n recoveryReadTimeout: 4s\n recoveryReadRetries: 2\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(contents), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + if _, err := LoadWithPath(dir); err == nil { + t.Fatal("LoadWithPath: expected rejection when read-timeout*(retries+1) exceeds the ceiling, got no error") + } +} + +func TestSurvivalRecoveryConfigReadBudgetValidatedAgainstConfiguredDeadline(t *testing.T) { + dir := t.TempDir() + // timeout(2s) * (retries(2)+1) = 6s. Against the default 30s deadline this + // is within the 6s ceiling, but against a configured 10s deadline the + // ceiling drops to min(6s, 2s) = 2s, so it must be rejected. + contents := "agentctl:\n recoveryDeadline: 10s\n recoveryReadTimeout: 2s\n recoveryReadRetries: 2\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(contents), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + if _, err := LoadWithPath(dir); err == nil { + t.Fatal("LoadWithPath: expected rejection validated against the actually-configured recoveryDeadline, got no error") + } +} + +func TestSurvivalRecoveryConfigCatalogEntriesPresent(t *testing.T) { + for _, key := range []string{ + "agentctl.recoveryDeadline", + "agentctl.recoveryReadTimeout", + "agentctl.recoveryReadRetries", + "agentctl.unownedPeriod", + "agentctl.detachedEventLimit", + } { + if _, ok := CatalogEntryForKey(key); !ok { + t.Errorf("catalog is missing %q", key) + } + } +} diff --git a/apps/backend/internal/common/config/validation.go b/apps/backend/internal/common/config/validation.go index df12c405e9..e4fdc8b4db 100644 --- a/apps/backend/internal/common/config/validation.go +++ b/apps/backend/internal/common/config/validation.go @@ -4,6 +4,7 @@ import ( "fmt" "net" "strings" + "time" ) func validateStartupSettings(cfg *Config) error { @@ -16,6 +17,11 @@ func validateStartupSettings(cfg *Config) error { errs = append(errs, configuredValidation(cfg, "agentctl.idleTimeout", cfg.Agentctl.IdleTimeout < 0, "agentctl.idleTimeout must be zero or greater")...) errs = append(errs, configuredValidation(cfg, "agentctl.idleReaperInterval", cfg.Agentctl.IdleReaperInterval <= 0, "agentctl.idleReaperInterval must be positive")...) errs = append(errs, configuredValidation(cfg, "agentctl.notificationQueueCapacity", cfg.Agentctl.NotificationQueueCapacity < 1024 || cfg.Agentctl.NotificationQueueCapacity > 131072, "agentctl.notificationQueueCapacity must be between 1024 and 131072")...) + errs = append(errs, configuredValidation(cfg, "agentctl.recoveryDeadline", cfg.Agentctl.RecoveryDeadline < time.Second || cfg.Agentctl.RecoveryDeadline > 5*time.Minute, "agentctl.recoveryDeadline must be between 1s and 5m")...) + errs = append(errs, configuredValidation(cfg, "agentctl.recoveryReadTimeout", cfg.Agentctl.RecoveryReadTimeout < 100*time.Millisecond, "agentctl.recoveryReadTimeout must be at least 100ms")...) + errs = append(errs, configuredValidation(cfg, "agentctl.recoveryReadRetries", cfg.Agentctl.RecoveryReadRetries < 0 || cfg.Agentctl.RecoveryReadRetries > 10, "agentctl.recoveryReadRetries must be between 0 and 10")...) + errs = append(errs, configuredValidation(cfg, "agentctl.detachedEventLimit", cfg.Agentctl.DetachedEventLimit < 1 || cfg.Agentctl.DetachedEventLimit > 10000, "agentctl.detachedEventLimit must be between 1 and 10000")...) + errs = append(errs, validateRecoveryReadBudget(cfg)...) errs = append(errs, configuredValidation(cfg, "planning.coalesceWindowMs", cfg.Planning.CoalesceWindowMs < 0, "planning.coalesceWindowMs must be zero or greater")...) errs = append(errs, configuredValidation(cfg, "office.schedulerTickMs", cfg.Office.SchedulerTickMs <= 0, "office.schedulerTickMs must be positive")...) errs = append(errs, configuredValidation(cfg, "launcher.webPort", cfg.Launcher.WebPort < 1 || cfg.Launcher.WebPort > 65535, "launcher.webPort must be between 1 and 65535")...) @@ -26,6 +32,29 @@ func validateStartupSettings(cfg *Config) error { return nil } +// validateRecoveryReadBudget rejects a configured combination of +// agentctl.recoveryReadTimeout and agentctl.recoveryReadRetries whose total +// attempt duration exceeds the lesser of 6 seconds or one fifth of the +// actually-configured agentctl.recoveryDeadline. Unlike configuredValidation, +// this check is unconditional: the three values it compares can each come +// from a different source (YAML, environment, or built-in default), and the +// contract requires validating the combination regardless of where any one +// of them came from. +func validateRecoveryReadBudget(cfg *Config) []string { + attempts := time.Duration(cfg.Agentctl.RecoveryReadRetries+1) * cfg.Agentctl.RecoveryReadTimeout + ceiling := cfg.Agentctl.RecoveryDeadline / 5 + if 6*time.Second < ceiling { + ceiling = 6 * time.Second + } + if attempts > ceiling { + return []string{fmt.Sprintf( + "agentctl.recoveryReadTimeout * (agentctl.recoveryReadRetries + 1) = %s exceeds the allowed ceiling of %s (the lesser of 6s and one fifth of agentctl.recoveryDeadline = %s)", + attempts, ceiling, cfg.Agentctl.RecoveryDeadline, + )} + } + return nil +} + func configuredValidation(cfg *Config, key string, invalid bool, message string) []string { if cfg.SourceFor(key) == SourceConfiguration && invalid { return []string{message} diff --git a/apps/backend/internal/common/netprobe/port.go b/apps/backend/internal/common/netprobe/port.go new file mode 100644 index 0000000000..0ae4013058 --- /dev/null +++ b/apps/backend/internal/common/netprobe/port.go @@ -0,0 +1,59 @@ +// Package netprobe holds the single contract for deciding whether a local +// TCP port is occupied. +// +// It lives in internal/common because the two callers sit on opposite sides +// of a dependency boundary -- the process launcher that picks the backend, +// web, and agentctl ports, and the runtime-tier launcher that starts a +// standalone agentctl -- and a port answered "free" by one and "busy" by the +// other is how a backend ends up talking to a control server it did not +// start. +package netprobe + +import ( + "fmt" + "net" + "strconv" + "time" +) + +// ConnectProbeTimeout bounds each loopback connect probe so a silently +// dropped SYN (for example under WSL2 mirrored networking to an unbound +// loopback port) cannot hang port selection. A timed-out connect counts as +// "nothing listening". +const ConnectProbeTimeout = 500 * time.Millisecond + +// PortAvailable reports whether a port is free. It is free only when a +// dual-stack loopback connect finds nothing listening AND a fresh loopback +// bind succeeds. +// +// The connect probe is required because a server binds the wildcard address +// (0.0.0.0 and [::]): on macOS/BSD a specific 127.0.0.1 bind succeeds against +// an active wildcard listener (Go also sets SO_REUSEADDR), so a bind-only +// check reports a busy port as free. The bind probe is retained because it +// catches reservations a connect misses (Windows phantom reservations, +// TIME_WAIT). +func PortAvailable(port int) bool { + if HasListener(port) { + return false + } + ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + return false + } + _ = ln.Close() + return true +} + +// HasListener reports whether anything answers a loopback connect on either +// the IPv4 or IPv6 loopback address. A server may hold only the IPv6 wildcard +// socket, so both families are probed. +func HasListener(port int) bool { + for _, host := range []string{"127.0.0.1", "::1"} { + conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), ConnectProbeTimeout) + if err == nil { + _ = conn.Close() + return true + } + } + return false +} diff --git a/apps/backend/internal/common/netprobe/port_test.go b/apps/backend/internal/common/netprobe/port_test.go new file mode 100644 index 0000000000..a34797cb74 --- /dev/null +++ b/apps/backend/internal/common/netprobe/port_test.go @@ -0,0 +1,83 @@ +package netprobe + +import ( + "net" + "testing" +) + +// TestPortAvailableReportsAWildcardListenerAsOccupied is the case a +// bind-only check gets wrong. A control server binds the wildcard address, +// and on macOS/BSD a fresh 127.0.0.1 bind against an active wildcard +// listener succeeds, so a launcher that only tries to bind concludes the +// port is free and starts a second server on a port some durable record +// already names. +func TestPortAvailableReportsAWildcardListenerAsOccupied(t *testing.T) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("wildcard listen: %v", err) + } + defer func() { _ = listener.Close() }() + port := listener.Addr().(*net.TCPAddr).Port + + if PortAvailable(port) { + t.Fatalf("port %d reported free while a wildcard listener holds it", port) + } + if !HasListener(port) { + t.Fatalf("port %d reported as having no listener while one is bound", port) + } +} + +// TestPortAvailableReportsALoopbackListenerAsOccupied covers the narrower +// bind: a server listening only on IPv4 loopback is still occupying the +// port for our purposes. +func TestPortAvailableReportsALoopbackListenerAsOccupied(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("loopback listen: %v", err) + } + defer func() { _ = listener.Close() }() + port := listener.Addr().(*net.TCPAddr).Port + + if PortAvailable(port) { + t.Fatalf("port %d reported free while a loopback listener holds it", port) + } +} + +// TestPortAvailableReportsAClosedPortAsFree is the open side: the probe must +// not be so conservative that it never finds a usable port. +func TestPortAvailableReportsAClosedPortAsFree(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("loopback listen: %v", err) + } + port := listener.Addr().(*net.TCPAddr).Port + if err := listener.Close(); err != nil { + t.Fatalf("close listener: %v", err) + } + + if !PortAvailable(port) { + t.Fatalf("port %d reported occupied after its listener closed", port) + } + if HasListener(port) { + t.Fatalf("port %d reported a listener after its listener closed", port) + } +} + +// TestHasListenerProbesBothLoopbackFamilies pins the dual-stack half of the +// contract: a server holding only the IPv6 wildcard socket is still found. +// Skipped where the host has no usable IPv6 loopback. +func TestHasListenerProbesBothLoopbackFamilies(t *testing.T) { + listener, err := net.Listen("tcp6", "[::1]:0") + if err != nil { + t.Skipf("no usable IPv6 loopback on this host: %v", err) + } + defer func() { _ = listener.Close() }() + port := listener.Addr().(*net.TCPAddr).Port + + if !HasListener(port) { + t.Fatalf("port %d reported as having no listener while an IPv6 listener holds it", port) + } + if PortAvailable(port) { + t.Fatalf("port %d reported free while an IPv6 listener holds it", port) + } +} diff --git a/apps/backend/internal/common/ownershipperiod/period.go b/apps/backend/internal/common/ownershipperiod/period.go new file mode 100644 index 0000000000..7b6d2b5c74 --- /dev/null +++ b/apps/backend/internal/common/ownershipperiod/period.go @@ -0,0 +1,56 @@ +// Package ownershipperiod resolves the effective control-server unowned +// period from its two independent inputs. It is cross-tier shared code +// (both the agentctl control server, which enforces the period, and the +// backend, which must renew ownership strictly inside a third of it, need +// the identical computation) and therefore lives in internal/common rather +// than being owned by either side and imported across the boundary. +package ownershipperiod + +import "time" + +// MinPeriod is the floor of AC-EXECUTORS-CONTROL-OWNERSHIP-003.7: no +// configuration may produce an unowned period at or near zero, which would +// expire ownership before an adopting backend could ever claim it. +const MinPeriod = time.Minute + +// Resolve applies AC-EXECUTORS-CONTROL-OWNERSHIP-003.4/.6/.7 to a +// configured unowned period, given the (possibly disabled, i.e. <= 0) +// per-instance idle timeout. It returns the effective period and a +// human-readable adjustment reason for each rule that fired, so the caller +// can log what happened (both criteria require recording the adjustment). +// +// Order matters: the idle-timeout clamp (003.4) is applied first, then the +// one-minute floor (003.7) is applied second and takes precedence -- when +// the idle timeout is short enough that no value satisfies both, the floor +// wins and the ordering constraint is left unsatisfied on purpose (003.7). +func Resolve(configured, idleTimeout time.Duration) (period time.Duration, adjustments []string) { + period = configured + + // 003.6: a disabled idle timeout (<=0) leaves nothing for the ordering + // constraint to order, so the clamp of 003.4 is skipped entirely rather + // than derived from the disabled sentinel. + if idleTimeout > 0 && period >= idleTimeout { + period = idleTimeout / 2 + adjustments = append(adjustments, + "unowned period was not shorter than the idle timeout; clamped to half the idle timeout") + } + + if period < MinPeriod { + period = MinPeriod + adjustments = append(adjustments, + "unowned period was below the one-minute floor; raised to the floor") + } + + return period, adjustments +} + +// RenewalInterval returns the cadence AC-EXECUTORS-CONTROL-OWNERSHIP-003.2 +// requires a renewing backend to use for a given resolved unowned period: +// strictly shorter than one third of it, so that after two consecutive +// failed renewals a third attempt still falls strictly inside the period. +// Quartering rather than using period/3 leaves headroom against scheduling +// jitter and network latency instead of relying on integer-division +// rounding to satisfy "strictly shorter". +func RenewalInterval(period time.Duration) time.Duration { + return period / 4 +} diff --git a/apps/backend/internal/common/ownershipperiod/period_test.go b/apps/backend/internal/common/ownershipperiod/period_test.go new file mode 100644 index 0000000000..0de9982301 --- /dev/null +++ b/apps/backend/internal/common/ownershipperiod/period_test.go @@ -0,0 +1,98 @@ +package ownershipperiod + +import ( + "testing" + "time" +) + +func TestResolve(t *testing.T) { + tests := []struct { + name string + configured time.Duration + idleTimeout time.Duration + wantPeriod time.Duration + wantAdjustments int + }{ + { + name: "already satisfies ordering, no adjustment", + configured: 5 * time.Minute, + idleTimeout: 10 * time.Minute, + wantPeriod: 5 * time.Minute, + wantAdjustments: 0, + }, + { + name: "AC-003.4 clamps to half the idle timeout when not shorter", + configured: 10 * time.Minute, + idleTimeout: 10 * time.Minute, + wantPeriod: 5 * time.Minute, + wantAdjustments: 1, + }, + { + name: "AC-003.4 clamps when configured exceeds idle timeout", + configured: 20 * time.Minute, + idleTimeout: 4 * time.Minute, + wantPeriod: 2 * time.Minute, + wantAdjustments: 1, + }, + { + name: "AC-003.6 disabled idle reaping (zero) is left unchanged", + configured: 10 * time.Minute, + idleTimeout: 0, + wantPeriod: 10 * time.Minute, + wantAdjustments: 0, + }, + { + name: "AC-003.6 negative idle timeout treated as disabled", + configured: 10 * time.Minute, + idleTimeout: -1, + wantPeriod: 10 * time.Minute, + wantAdjustments: 0, + }, + { + name: "AC-003.7 floor applies when configured value is too small", + configured: 10 * time.Second, + idleTimeout: 0, + wantPeriod: MinPeriod, + wantAdjustments: 1, + }, + { + name: "AC-003.7 floor takes precedence over the AC-003.4 clamp", + configured: time.Minute, + idleTimeout: 30 * time.Second, + wantPeriod: MinPeriod, + wantAdjustments: 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + period, adjustments := Resolve(tt.configured, tt.idleTimeout) + if period != tt.wantPeriod { + t.Fatalf("Resolve(%v, %v) period = %v, want %v", + tt.configured, tt.idleTimeout, period, tt.wantPeriod) + } + if len(adjustments) != tt.wantAdjustments { + t.Fatalf("Resolve(%v, %v) adjustments = %v (len %d), want len %d", + tt.configured, tt.idleTimeout, adjustments, len(adjustments), tt.wantAdjustments) + } + }) + } +} + +// TestRenewalInterval pins AC-EXECUTORS-CONTROL-OWNERSHIP-003.2: three +// consecutive intervals must sum to strictly less than the period, so a +// third renewal attempt after two failures is still due strictly before the +// unowned-period expiry test can fire. +func TestRenewalInterval(t *testing.T) { + periods := []time.Duration{MinPeriod, 90 * time.Second, 5 * time.Minute, 10 * time.Minute, time.Hour} + for _, period := range periods { + interval := RenewalInterval(period) + if interval <= 0 { + t.Fatalf("RenewalInterval(%v) = %v, want > 0", period, interval) + } + if threeIntervals := 3 * interval; threeIntervals >= period { + t.Fatalf("RenewalInterval(%v) = %v; 3x = %v, want strictly less than the period", + period, interval, threeIntervals) + } + } +} diff --git a/apps/backend/internal/common/ownershipproof/proof.go b/apps/backend/internal/common/ownershipproof/proof.go new file mode 100644 index 0000000000..7432c73fad --- /dev/null +++ b/apps/backend/internal/common/ownershipproof/proof.go @@ -0,0 +1,44 @@ +// Package ownershipproof derives the value a control server returns to +// demonstrate it already holds an ownership credential, without disclosing +// it. Both tiers need the identical derivation -- agentctl produces it and +// the backend verifies it -- and agentctl ships as its own binary, so this +// lives in common rather than in either side. +package ownershipproof + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" +) + +// Derive returns the proof for credential over challenge. binding is mixed +// in after a zero separator so a server belonging to a different +// installation cannot answer for this one, and so the challenge and the +// binding cannot run together into an ambiguous input. +func Derive(credential, challenge, binding string) string { + mac := hmac.New(sha256.New, []byte(credential)) + mac.Write([]byte(challenge)) + mac.Write([]byte{0}) + mac.Write([]byte(binding)) + return hex.EncodeToString(mac.Sum(nil)) +} + +// Matches reports whether any offered proof is the one credential produces +// for this challenge and binding. A server may hold either of two +// acceptable credentials while a rotation is unconfirmed, so it offers one +// proof per credential and the verifier accepts on any match. Comparison is +// constant-time. An empty credential, an empty challenge, or no offered +// proofs never match: a caller with nothing to verify has proved nothing. +func Matches(credential, challenge, binding string, offered []string) bool { + if credential == "" || challenge == "" { + return false + } + want := []byte(Derive(credential, challenge, binding)) + matched := false + for _, candidate := range offered { + if hmac.Equal([]byte(candidate), want) { + matched = true + } + } + return matched +} diff --git a/apps/backend/internal/common/ownershipproof/proof_test.go b/apps/backend/internal/common/ownershipproof/proof_test.go new file mode 100644 index 0000000000..7aed35c6bd --- /dev/null +++ b/apps/backend/internal/common/ownershipproof/proof_test.go @@ -0,0 +1,49 @@ +package ownershipproof + +import "testing" + +func TestDeriveDependsOnEveryInput(t *testing.T) { + base := Derive("cred", "chal", "bind") + for _, tc := range []struct{ name, cred, chal, bind string }{ + {"credential", "other", "chal", "bind"}, + {"challenge", "cred", "other", "bind"}, + {"binding", "cred", "chal", "other"}, + } { + if Derive(tc.cred, tc.chal, tc.bind) == base { + t.Errorf("proof did not vary with the %s", tc.name) + } + } + if base == "cred" { + t.Fatal("proof is the credential verbatim") + } + // The separator must keep adjacent fields from running together, so + // these two distinct (challenge, binding) pairs cannot collide. + if Derive("cred", "ab", "c") == Derive("cred", "a", "bc") { + t.Fatal("challenge and binding are not unambiguously separated") + } +} + +func TestMatchesAcceptsAnyOfferedProofAndRefusesNothing(t *testing.T) { + chal, bind := "chal", "bind" + mine := Derive("mine", chal, bind) + theirs := Derive("theirs", chal, bind) + + if !Matches("mine", chal, bind, []string{theirs, mine}) { + t.Error("did not accept a matching proof offered alongside another") + } + if Matches("mine", chal, bind, []string{theirs}) { + t.Error("accepted a proof derived from a different credential") + } + if Matches("mine", chal, bind, nil) { + t.Error("accepted an empty set of proofs") + } + if Matches("mine", chal, bind, []string{""}) { + t.Error("accepted an empty proof") + } + if Matches("", chal, bind, []string{Derive("", chal, bind)}) { + t.Error("accepted a proof when no credential was held") + } + if Matches("mine", "", bind, []string{Derive("mine", "", bind)}) { + t.Error("accepted a proof over an empty challenge") + } +} diff --git a/apps/backend/internal/launcher/ports.go b/apps/backend/internal/launcher/ports.go index 577efc7508..0a92f84421 100644 --- a/apps/backend/internal/launcher/ports.go +++ b/apps/backend/internal/launcher/ports.go @@ -3,12 +3,12 @@ package launcher import ( "fmt" "math/rand" - "net" "os" "strconv" "time" "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/common/netprobe" ) type portConfig struct { @@ -217,41 +217,9 @@ func pickAvailablePortExcept(preferred int, used map[int]bool) (int, error) { return 0, fmt.Errorf("unable to find a free port") } -// connectProbeTimeout bounds each loopback connect probe so a silently dropped -// SYN (for example under WSL2 mirrored networking to an unbound loopback port) -// cannot hang port selection. A timed-out connect counts as "nothing listening". -const connectProbeTimeout = 500 * time.Millisecond - -// canBind reports whether a port is available. It is available only when a -// dual-stack loopback connect finds nothing listening AND a fresh loopback bind -// succeeds. The connect probe is required because a running backend binds the -// wildcard address (0.0.0.0 and [::]): on macOS/BSD a specific 127.0.0.1 bind -// succeeds against an active wildcard listener (Go also sets SO_REUSEADDR), so a -// bind-only check would falsely report the busy port as free. The bind probe is -// retained because it catches reservations a connect misses (Windows phantom -// reservations, TIME_WAIT). +// canBind reports whether a port is available, using the shared probe +// contract in internal/common/netprobe so the launcher and the agentctl +// launcher can never disagree about whether a port is occupied. func canBind(port int) bool { - if portHasListener(port) { - return false - } - ln, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)) - if err != nil { - return false - } - _ = ln.Close() - return true -} - -// portHasListener reports whether anything answers a loopback connect on either -// the IPv4 or IPv6 loopback address. The existing backend may hold only the IPv6 -// wildcard socket, so both families are probed. -func portHasListener(port int) bool { - for _, host := range []string{"127.0.0.1", "::1"} { - conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, strconv.Itoa(port)), connectProbeTimeout) - if err == nil { - _ = conn.Close() - return true - } - } - return false + return netprobe.PortAvailable(port) } diff --git a/apps/backend/internal/orchestrator/event_handlers_test.go b/apps/backend/internal/orchestrator/event_handlers_test.go index 37ecc8dab8..042dffe549 100644 --- a/apps/backend/internal/orchestrator/event_handlers_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_test.go @@ -287,19 +287,23 @@ type mockAgentManager struct { // optional rowLivenessProber so reconciliation tests can drive runtime-aware // liveness per row. Nil → the mock is not a prober and reconciliation treats // every row as Unknown. - rowLivenessFn func(*models.ExecutorRunning) models.ProcessLiveness - resolveProfileInfo *executor.AgentProfileInfo - resolveProfileErr error - restartProcessCalls []string // tracks execution IDs passed to RestartAgentProcess - restartProcessErr error - promptErr error - promptResult *executor.PromptResult - promptAcceptedOnError bool - promptAgentFunc func(context.Context, string, string, []v1.MessageAttachment, bool) (*executor.PromptResult, error) - launchAgentFunc func(context.Context, *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) - startAgentProcessCalls []string - startAgentProcessErr error - startAgentProcessFunc func(context.Context, string) error + rowLivenessFn func(*models.ExecutorRunning) models.ProcessLiveness + // newStandaloneLivenessScopeCalls counts NewStandaloneLivenessScope + // invocations, letting a test assert a reconciliation pass takes exactly + // one enumeration and reuses it across every row. + newStandaloneLivenessScopeCalls int + resolveProfileInfo *executor.AgentProfileInfo + resolveProfileErr error + restartProcessCalls []string // tracks execution IDs passed to RestartAgentProcess + restartProcessErr error + promptErr error + promptResult *executor.PromptResult + promptAcceptedOnError bool + promptAgentFunc func(context.Context, string, string, []v1.MessageAttachment, bool) (*executor.PromptResult, error) + launchAgentFunc func(context.Context, *executor.LaunchAgentRequest) (*executor.LaunchAgentResponse, error) + startAgentProcessCalls []string + startAgentProcessErr error + startAgentProcessFunc func(context.Context, string) error // probeBackgroundWorkloadsFunc, when non-nil, overrides // ProbeBackgroundWorkloads's default Unknown/nil response. @@ -668,6 +672,21 @@ func (m *mockAgentManager) RowLiveness(row *models.ExecutorRunning) models.Proce } return models.ProcessLivenessUnknown } + +// NewStandaloneLivenessScope and RowLivenessScoped make the mock satisfy the +// orchestrator's optional standaloneLivenessScoper alongside rowLivenessProber. +// The mock has no real enumeration to scope, so it returns a nil placeholder +// scope and RowLivenessScoped simply delegates to the existing RowLiveness/ +// rowLivenessFn machinery -- every existing rowLivenessFn-based test keeps its +// exact behavior without change. +func (m *mockAgentManager) NewStandaloneLivenessScope(_ context.Context) interface{} { + m.newStandaloneLivenessScopeCalls++ + return nil +} + +func (m *mockAgentManager) RowLivenessScoped(row *models.ExecutorRunning, _ interface{}) models.ProcessLiveness { + return m.RowLiveness(row) +} func (m *mockAgentManager) ResolveAgentProfile(_ context.Context, _ string) (*executor.AgentProfileInfo, error) { if m.resolveProfileErr != nil { return nil, m.resolveProfileErr diff --git a/apps/backend/internal/orchestrator/executor/executor_host_gh_bridge_test.go b/apps/backend/internal/orchestrator/executor/executor_host_gh_bridge_test.go index 3be846a837..5f9ccd0773 100644 --- a/apps/backend/internal/orchestrator/executor/executor_host_gh_bridge_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_host_gh_bridge_test.go @@ -454,8 +454,9 @@ func TestExecutorHostGHBridgeEligibility(t *testing.T) { } hostsMu.Lock() defer hostsMu.Unlock() - if strings.Join(hosts, ",") != "github.com,ghe.example" { - t.Fatalf("probed hosts = %v, want [github.com ghe.example]", hosts) + sort.Strings(hosts) + if strings.Join(hosts, ",") != "ghe.example,github.com" { + t.Fatalf("probed hosts = %v, want [ghe.example github.com]", hosts) } }) diff --git a/apps/backend/internal/orchestrator/handlers/handlers.go b/apps/backend/internal/orchestrator/handlers/handlers.go index c9a7a86336..095858bbae 100644 --- a/apps/backend/internal/orchestrator/handlers/handlers.go +++ b/apps/backend/internal/orchestrator/handlers/handlers.go @@ -138,10 +138,13 @@ func (h *Handlers) wsLaunchSession(ctx context.Context, msg *ws.Message) (*ws.Me resp, err := h.service.LaunchSession(ctx, &req) if err != nil { - intent := orchestrator.ResolveIntent(&req) - if recoveryResponse, responseErr := taskArchivedConflictResponse(msg, err); recoveryResponse != nil || responseErr != nil { - return recoveryResponse, responseErr + if guardResponse, responseErr := sessionRecoveryGuardConflictResponse(msg, err); guardResponse != nil || responseErr != nil { + return guardResponse, responseErr } + if archivedResponse, responseErr := taskArchivedConflictResponse(msg, err); archivedResponse != nil || responseErr != nil { + return archivedResponse, responseErr + } + intent := orchestrator.ResolveIntent(&req) // A launch failing because the root context was cancelled or the // session is already terminal is an expected shutdown teardown race, // not a fault: log WARN (no stack trace) so it does not masquerade as a @@ -266,6 +269,23 @@ func branchRecoveryConflictResponse(msg *ws.Message, err error) (*ws.Message, er return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeConflict, err.Error(), branchRecoveryErr.Details()) } +// sessionRecoveryGuardConflictResponse surfaces the startup recovery guard's +// two distinct refusal reasons: a retryable in-progress recovery maps to +// CONFLICT (the caller should retry once recovery resolves), and a +// non-retryable unstoppable-agent condition maps to UNAVAILABLE (only a +// backend restart clears it). +func sessionRecoveryGuardConflictResponse(msg *ws.Message, err error) (*ws.Message, error) { + var guardErr *orchestrator.SessionRecoveryGuardError + if !errors.As(err, &guardErr) { + return nil, nil + } + code := ws.ErrorCodeUnavailable + if guardErr.Retryable { + code = ws.ErrorCodeConflict + } + return ws.NewError(msg.ID, msg.Action, code, err.Error(), guardErr.Details()) +} + func taskArchivedConflictResponse(msg *ws.Message, err error) (*ws.Message, error) { if !errors.Is(err, executor.ErrTaskArchived) { return nil, nil @@ -311,6 +331,9 @@ func (h *Handlers) wsRecoverSession(ctx context.Context, msg *ws.Message) (*ws.M if recoveryResponse, responseErr := branchRecoveryConflictResponse(msg, err); recoveryResponse != nil || responseErr != nil { return recoveryResponse, responseErr } + if guardResponse, responseErr := sessionRecoveryGuardConflictResponse(msg, err); guardResponse != nil || responseErr != nil { + return guardResponse, responseErr + } h.logger.Error("failed to recover session", zap.String("task_id", req.TaskID), zap.String("session_id", req.SessionID), diff --git a/apps/backend/internal/orchestrator/handlers/handlers_test.go b/apps/backend/internal/orchestrator/handlers/handlers_test.go index 6d63b4a23f..b7238af906 100644 --- a/apps/backend/internal/orchestrator/handlers/handlers_test.go +++ b/apps/backend/internal/orchestrator/handlers/handlers_test.go @@ -59,6 +59,51 @@ func TestBranchRecoveryConflictResponsePreservesRecoveryDetails(t *testing.T) { require.Equal(t, "resume_new_branch", payload.Details["recovery_action"]) } +func TestSessionRecoveryGuardConflictResponseMapsRetryableToConflict(t *testing.T) { + msg := createTestMessage(t, ws.ActionSessionLaunch, map[string]interface{}{}) + err := &orchestrator.SessionRecoveryGuardError{ + Cause: errors.New("session is guarded pending recovery"), + SessionID: "session-1", + Retryable: true, + } + + response, responseErr := sessionRecoveryGuardConflictResponse(msg, err) + require.NoError(t, responseErr) + require.NotNil(t, response) + payload := parseError(t, response) + require.Equal(t, ws.ErrorCodeConflict, payload.Code) + require.Equal(t, "session_recovery_in_progress", payload.Details["kind"]) + require.Equal(t, true, payload.Details["retryable"]) + require.Equal(t, "session-1", payload.Details["session_id"]) +} + +func TestSessionRecoveryGuardConflictResponseMapsNonRetryableToUnavailable(t *testing.T) { + msg := createTestMessage(t, ws.ActionSessionLaunch, map[string]interface{}{}) + err := &orchestrator.SessionRecoveryGuardError{ + Cause: errors.New("session has an unstoppable agent from a prior launch"), + SessionID: "session-2", + Retryable: false, + } + + response, responseErr := sessionRecoveryGuardConflictResponse(msg, err) + require.NoError(t, responseErr) + require.NotNil(t, response) + payload := parseError(t, response) + require.Equal(t, ws.ErrorCodeUnavailable, payload.Code) + require.Equal(t, "session_recovery_unstoppable", payload.Details["kind"]) + require.Equal(t, false, payload.Details["retryable"]) + require.Equal(t, "session-2", payload.Details["session_id"]) +} + +func TestSessionRecoveryGuardConflictResponseIgnoresUnrelatedError(t *testing.T) { + msg := createTestMessage(t, ws.ActionSessionLaunch, map[string]interface{}{}) + + response, responseErr := sessionRecoveryGuardConflictResponse(msg, errors.New("boom")) + + require.NoError(t, responseErr) + require.Nil(t, response) +} + func TestWsEnsureSessionRequestParsesAutoStartOverride(t *testing.T) { tests := []struct { name string diff --git a/apps/backend/internal/orchestrator/reconcile_liveness.go b/apps/backend/internal/orchestrator/reconcile_liveness.go index bd79dc0098..816b3d63ad 100644 --- a/apps/backend/internal/orchestrator/reconcile_liveness.go +++ b/apps/backend/internal/orchestrator/reconcile_liveness.go @@ -25,6 +25,22 @@ type rowLivenessProber interface { RowLiveness(row *models.ExecutorRunning) models.ProcessLiveness } +// standaloneLivenessScoper is the optional capability the orchestrator uses to +// take one adopted-server enumeration and reuse it across every row of a +// single reconciliation pass (design 02 "Persistence": "it has two kinds of +// caller, and only one of them is a pass"). Satisfied by the lifecycle +// adapter alongside rowLivenessProber. A caller outside a pass (e.g. the idle +// reclaim path) must keep using rowLivenessProber's context-free RowLiveness, +// which always takes its own fresh enumeration. +// +// Kept as its own narrow optional interface rather than widening +// rowLivenessProber's RowLiveness method, so the 17 existing test doubles +// built around that single-row, context-free signature don't have to change. +type standaloneLivenessScoper interface { + NewStandaloneLivenessScope(ctx context.Context) interface{} + RowLivenessScoped(row *models.ExecutorRunning, scope interface{}) models.ProcessLiveness +} + // agentRunningProber preserves runtime probe errors so reclaim can fail // closed. The legacy boolean method remains the fallback for test doubles and // adapters that do not expose the richer probe yet. @@ -152,6 +168,32 @@ func (s *Service) rowLiveness(row *models.ExecutorRunning) models.ProcessLivenes return prober.RowLiveness(row) } +// newStandaloneLivenessScope takes one adopted-server enumeration for reuse +// across every row of a single reconciliation pass, when s.agentManager +// implements standaloneLivenessScoper. Returns nil otherwise (unit tests, +// degraded startup, or a caller that doesn't support scoped enumeration); +// rowLivenessScoped treats a nil scope as "fall back to the unscoped probe". +func (s *Service) newStandaloneLivenessScope(ctx context.Context) interface{} { + scoper, ok := s.agentManager.(standaloneLivenessScoper) + if !ok || scoper == nil { + return nil + } + return scoper.NewStandaloneLivenessScope(ctx) +} + +// rowLivenessScoped classifies row's liveness reusing scope (from +// newStandaloneLivenessScope) when s.agentManager supports it, falling back +// to the ordinary context-free rowLiveness otherwise -- so a caller can +// unconditionally pass whatever newStandaloneLivenessScope returned, even a +// nil one from a prober that doesn't implement the scoped capability. +func (s *Service) rowLivenessScoped(row *models.ExecutorRunning, scope interface{}) models.ProcessLiveness { + scoper, ok := s.agentManager.(standaloneLivenessScoper) + if !ok || scoper == nil { + return s.rowLiveness(row) + } + return scoper.RowLivenessScoped(row, scope) +} + // pruneOrRepairExecutorRow enforces the resume-safety deletion invariant // (#1597 resume-safety invariant) at a reconciliation cleanup site: a row backing // a resumable session, or holding a resume_token, is repaired in place (never diff --git a/apps/backend/internal/orchestrator/reconcile_liveness_scope_test.go b/apps/backend/internal/orchestrator/reconcile_liveness_scope_test.go new file mode 100644 index 0000000000..8b0c78d24f --- /dev/null +++ b/apps/backend/internal/orchestrator/reconcile_liveness_scope_test.go @@ -0,0 +1,84 @@ +package orchestrator + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" +) + +// TestReconcileExecutorSessionsOnStartupReusesOneLivenessScopePerPass pins +// design 02 "Persistence": startup reconciliation is a pass, so it must take +// exactly one adopted-server enumeration and reuse it across every row, +// never enumerate once per row. +func TestReconcileExecutorSessionsOnStartupReusesOneLivenessScopePerPass(t *testing.T) { + repo := setupTestRepo(t) + ctx := context.Background() + + for _, id := range []string{"1", "2"} { + taskID := "task-scope-" + id + sessionID := "session-scope-" + id + seedTaskAndSession(t, repo, taskID, sessionID, models.TaskSessionStateIdle) + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: sessionID, + SessionID: sessionID, + TaskID: taskID, + Runtime: agentruntime.RuntimeStandalone, + Status: models.ExecutorRunningStatusRunning, + LocalPID: 4242, + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("upsert executor row %s: %v", sessionID, err) + } + } + + agentMgr := &mockAgentManager{} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), agentMgr) + + svc.reconcileExecutorSessionsOnStartup(ctx) + + if agentMgr.newStandaloneLivenessScopeCalls != 1 { + t.Fatalf("NewStandaloneLivenessScope calls = %d, want exactly 1 for a two-row reconciliation pass", + agentMgr.newStandaloneLivenessScopeCalls) + } +} + +// TestReconcileExecutorSessionsOnStartupSkipsScopeWithoutStandaloneCapability +// pins the fallback: when s.agentManager doesn't implement +// standaloneLivenessScoper, reconciliation still runs (via the ordinary +// unscoped rowLiveness) without panicking on a nil scope. +func TestReconcileExecutorSessionsOnStartupSkipsScopeWithoutStandaloneCapability(t *testing.T) { + repo := setupTestRepo(t) + ctx := context.Background() + seedTaskAndSession(t, repo, "task-noscope", "session-noscope", models.TaskSessionStateIdle) + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "session-noscope", + SessionID: "session-noscope", + TaskID: "task-noscope", + Runtime: agentruntime.RuntimeStandalone, + Status: models.ExecutorRunningStatusRunning, + LocalPID: 4242, + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("upsert executor row: %v", err) + } + + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), nonScopingAgentManager{&mockAgentManager{}}) + + svc.reconcileExecutorSessionsOnStartup(ctx) +} + +// nonScopingAgentManager embeds executor.AgentManagerClient by interface +// value, not by *mockAgentManager, so only the methods that interface +// declares are promoted -- NewStandaloneLivenessScope/RowLivenessScoped are +// not part of it, so nonScopingAgentManager does not satisfy +// standaloneLivenessScoper even though the wrapped *mockAgentManager has +// those methods. +type nonScopingAgentManager struct { + executor.AgentManagerClient +} diff --git a/apps/backend/internal/orchestrator/reconcile_retracked_sessions_test.go b/apps/backend/internal/orchestrator/reconcile_retracked_sessions_test.go new file mode 100644 index 0000000000..d37a044e5f --- /dev/null +++ b/apps/backend/internal/orchestrator/reconcile_retracked_sessions_test.go @@ -0,0 +1,207 @@ +package orchestrator + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +// countingTurnService is a minimal TurnService stub that only needs to +// record AbandonOpenTurns calls; every other method panics if a test +// exercises a path that was not supposed to reach it. +type countingTurnService struct { + mu sync.Mutex + abandonOpenTurnsCall int +} + +func (c *countingTurnService) AbandonOpenTurns(context.Context, string) error { + c.mu.Lock() + c.abandonOpenTurnsCall++ + c.mu.Unlock() + return nil +} +func (c *countingTurnService) calls() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.abandonOpenTurnsCall +} +func (*countingTurnService) StartTurn(context.Context, string) (*models.Turn, error) { + panic("countingTurnService: StartTurn should not be called") +} +func (*countingTurnService) ReserveTurn(context.Context, string, *models.PromptDispatchRecovery) (*models.Turn, error) { + panic("countingTurnService: ReserveTurn should not be called") +} +func (*countingTurnService) MarkReservedTurnDispatchAttempted(context.Context, *models.Turn) error { + panic("countingTurnService: MarkReservedTurnDispatchAttempted should not be called") +} +func (*countingTurnService) PublishReservedTurn(context.Context, *models.Turn) error { + panic("countingTurnService: PublishReservedTurn should not be called") +} +func (*countingTurnService) RollbackReservedTurn(context.Context, string, string) (bool, error) { + panic("countingTurnService: RollbackReservedTurn should not be called") +} +func (*countingTurnService) ReconcileUnpublishedPromptTurns(context.Context) (int, error) { + return 0, nil +} +func (*countingTurnService) CompleteTurn(context.Context, string) error { return nil } +func (*countingTurnService) GetTurn(context.Context, string) (*models.Turn, error) { + return nil, nil +} +func (*countingTurnService) GetActiveTurn(context.Context, string) (*models.Turn, error) { + return nil, nil +} +func (*countingTurnService) UpdateTurn(context.Context, *models.Turn) error { return nil } +func (*countingTurnService) PatchTurnMetadata(context.Context, string, string, map[string]interface{}) error { + return nil +} + +// TestReconcileSessionsOnStartupSkipsReconciliationWhenRetracked pins +// AC-EXECUTORS-SURVIVAL-003.2/.3/.4/.5: a session the lifecycle manager +// reports as re-tracked keeps its RUNNING state, its task stays IN_PROGRESS, +// its task is never marked interrupted, and its open turns are never +// abandoned -- because its agent actually survived the restart, none of +// today's "backend died mid-turn" cleanup applies to it. +func TestReconcileSessionsOnStartupSkipsReconciliationWhenRetracked(t *testing.T) { + repo := setupTestRepo(t) + ctx := context.Background() + now := time.Now().UTC() + + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) + + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "er1", + SessionID: "session1", + TaskID: "task1", + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + t.Fatalf("failed to upsert executor running: %v", err) + } + + taskRepo := newMockTaskRepo() + taskRepo.tasks["task1"] = &v1.Task{ID: "task1", State: v1.TaskStateInProgress} + + turns := &countingTurnService{} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{}) + svc.turnService = turns + svc.SetRetrackedSessionChecker(func(sessionID string) bool { return sessionID == "session1" }) + + svc.reconcileSessionsOnStartup(ctx) + + session, err := repo.GetTaskSession(ctx, "session1") + if err != nil { + t.Fatalf("failed to load session: %v", err) + } + if session.State != models.TaskSessionStateRunning { + t.Fatalf("session state = %q, want %q (AC-EXECUTORS-SURVIVAL-003.3)", session.State, models.TaskSessionStateRunning) + } + + if state, ok := taskRepo.updatedStates["task1"]; ok { + t.Fatalf("expected task state left untouched (AC-EXECUTORS-SURVIVAL-003.5), got write to %q", state) + } + + task, err := repo.GetTask(ctx, "task1") + if err != nil { + t.Fatalf("failed to load task: %v", err) + } + if _, marked := task.Metadata[models.MetaKeyInterruptedAt]; marked { + t.Fatal("expected task NOT to be marked interrupted (AC-EXECUTORS-SURVIVAL-003.2)") + } + + if got := turns.calls(); got != 0 { + t.Fatalf("AbandonOpenTurns called %d times, want 0 (AC-EXECUTORS-SURVIVAL-003.4)", got) + } +} + +// TestReconcileSessionsOnStartupAppliesFullReconciliationWhenNotRetracked is +// the regression pin for the same fixture as above, but with the checker +// reporting the session as NOT re-tracked -- today's full "backend died +// mid-turn" reconciliation must still apply exactly as before Layer 6.6. +func TestReconcileSessionsOnStartupAppliesFullReconciliationWhenNotRetracked(t *testing.T) { + repo := setupTestRepo(t) + ctx := context.Background() + now := time.Now().UTC() + + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) + + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "er1", + SessionID: "session1", + TaskID: "task1", + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + t.Fatalf("failed to upsert executor running: %v", err) + } + + taskRepo := newMockTaskRepo() + taskRepo.tasks["task1"] = &v1.Task{ID: "task1", State: v1.TaskStateInProgress} + + turns := &countingTurnService{} + svc := createTestServiceWithAgent(repo, newMockStepGetter(), taskRepo, &mockAgentManager{}) + svc.turnService = turns + svc.SetRetrackedSessionChecker(func(sessionID string) bool { return false }) + + svc.reconcileSessionsOnStartup(ctx) + + session, err := repo.GetTaskSession(ctx, "session1") + if err != nil { + t.Fatalf("failed to load session: %v", err) + } + if session.State != models.TaskSessionStateWaitingForInput { + t.Fatalf("session state = %q, want %q", session.State, models.TaskSessionStateWaitingForInput) + } + + if state, ok := taskRepo.updatedStates["task1"]; !ok || state != v1.TaskStateReview { + t.Fatalf("expected task moved to REVIEW, updatedStates[task1] = %q (ok=%v)", state, ok) + } + + task, err := repo.GetTask(ctx, "task1") + if err != nil { + t.Fatalf("failed to load task: %v", err) + } + if _, marked := task.Metadata[models.MetaKeyInterruptedAt]; !marked { + t.Fatal("expected task to be marked interrupted") + } + + if got := turns.calls(); got != 1 { + t.Fatalf("AbandonOpenTurns called %d times, want 1", got) + } +} + +// TestReconcileSessionsOnStartupUnwiredCheckerTreatsEverySessionAsNotRetracked +// pins the nil-checker default: when SetRetrackedSessionChecker is never +// called (agent survival disabled, or backendapp wiring absent), every +// active session is reconciled exactly as before Layer 6.6. +func TestReconcileSessionsOnStartupUnwiredCheckerTreatsEverySessionAsNotRetracked(t *testing.T) { + repo := setupTestRepo(t) + ctx := context.Background() + now := time.Now().UTC() + + seedTaskAndSession(t, repo, "task1", "session1", models.TaskSessionStateRunning) + + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "er1", + SessionID: "session1", + TaskID: "task1", + CreatedAt: now, + UpdatedAt: now, + }); err != nil { + t.Fatalf("failed to upsert executor running: %v", err) + } + + svc := createTestServiceWithAgent(repo, newMockStepGetter(), newMockTaskRepo(), &mockAgentManager{}) + svc.reconcileSessionsOnStartup(ctx) + + session, err := repo.GetTaskSession(ctx, "session1") + if err != nil { + t.Fatalf("failed to load session: %v", err) + } + if session.State != models.TaskSessionStateWaitingForInput { + t.Fatalf("session state = %q, want %q", session.State, models.TaskSessionStateWaitingForInput) + } +} diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go index 119f970aee..c1c10e2321 100644 --- a/apps/backend/internal/orchestrator/service.go +++ b/apps/backend/internal/orchestrator/service.go @@ -744,6 +744,16 @@ type Service struct { // session.ensure). Nil = unscoped. taskAccessCheck func(ctx context.Context, taskID string) error + // retrackedSessionCheck reports whether the lifecycle manager + // successfully re-tracked a session during this backend's own startup + // recovery pass (AC-EXECUTORS-SURVIVAL-003.1). Consulted by startup + // reconciliation so a session whose agent survived the restart skips + // "backend died mid-turn" cleanup. Nil (unwired, or agent survival + // disabled) preserves today's behavior -- every active session is + // reconciled as if its agent died with the backend. See + // SetRetrackedSessionChecker. + retrackedSessionCheck func(sessionID string) bool + // routeActionHandler is owned by the dynamic conductor composition. The // orchestrator only validates/authorizes the request and returns the // authoritative route snapshot; concrete and dynamic callers share this @@ -2017,6 +2027,27 @@ func (s *Service) SetTaskAccessChecker(check func(ctx context.Context, taskID st s.taskAccessCheck = check } +// SetRetrackedSessionChecker installs the lifecycle manager's query for +// AC-EXECUTORS-SURVIVAL-003.1: whether a session was successfully +// re-tracked during this backend's own startup recovery pass. Wired from +// backendapp after the lifecycle manager's synchronous Start() has already +// completed, so every session's outcome is decided before this service's +// own Start() runs startup reconciliation. Nil (unwired) preserves today's +// behavior. +func (s *Service) SetRetrackedSessionChecker(check func(sessionID string) bool) { + s.retrackedSessionCheck = check +} + +// wasSessionRetracked applies the configured retrackedSessionCheck. Nil +// checker or empty sessionID is treated as "not retracked" -- today's +// unconditional reconciliation, the correct default. +func (s *Service) wasSessionRetracked(sessionID string) bool { + if s.retrackedSessionCheck == nil || sessionID == "" { + return false + } + return s.retrackedSessionCheck(sessionID) +} + // authorizeSession applies the configured per-user session check. No-op when // unwired. func (s *Service) authorizeSession(ctx context.Context, sessionID string) error { @@ -3129,6 +3160,27 @@ func (s *Service) Start(ctx context.Context) error { return nil } +// StartEventWatcher subscribes the orchestrator before lifecycle recovery. +// Recovery can publish an agent event before Service.Start completes its +// startup reconciliation, so those subscriptions must exist first. +// Watcher.Start is idempotent; Service.Start can call it again at its normal +// startup point without replacing these subscriptions. +func (s *Service) StartEventWatcher(ctx context.Context) error { + if s.watcher == nil { + return errors.New("orchestrator event watcher is not configured") + } + return s.watcher.Start(ctx) +} + +// StopEventWatcher removes subscriptions created by StartEventWatcher when a +// later startup phase fails before Service.Start owns the watcher lifecycle. +func (s *Service) StopEventWatcher() error { + if s.watcher == nil { + return nil + } + return s.watcher.Stop() +} + // Stop stops all orchestrator components func (s *Service) Stop() error { s.mu.Lock() @@ -3342,6 +3394,10 @@ func (s *Service) reconcileExecutorSessionsOnStartup(ctx context.Context) { s.logger.Info("reconciling sessions on startup (lazy recovery)", zap.Int("count", len(runningExecutors))) report := newStartupCleanupReport() + // One adopted-server enumeration, reused for every row of this pass + // (design 02 "Persistence": "it has two kinds of caller, and only one of + // them is a pass") -- taken once here rather than per-row. + scope := s.newStandaloneLivenessScope(ctx) var remoteRecords []executor.RemoteStatusPollRequest for _, running := range runningExecutors { if models.IsRemoteExecutorType(models.ExecutorType(running.Runtime)) { @@ -3354,7 +3410,7 @@ func (s *Service) reconcileExecutorSessionsOnStartup(ctx context.Context) { Metadata: running.Metadata, }) } - s.reconcileOneSessionOnStartup(ctx, running, report) + s.reconcileOneSessionOnStartup(ctx, running, report, scope) } report.flush(s.logger) @@ -3365,7 +3421,7 @@ func (s *Service) reconcileExecutorSessionsOnStartup(ctx context.Context) { } // reconcileOneSessionOnStartup adjusts DB state for a single session without launching agents. -func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport) { +func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport, scope interface{}) { sessionID := running.SessionID if sessionID == "" { return @@ -3374,7 +3430,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod session, err := s.repo.GetTaskSession(ctx, sessionID) if err != nil { if isTaskSessionNotFound(err) { - s.handleMissingSessionOnStartup(ctx, running, report) + s.handleMissingSessionOnStartup(ctx, running, report, scope) return } s.logger.Warn("failed to load session for reconciliation; preserving executor record", @@ -3386,7 +3442,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod previousState := session.State // Handle terminal and never-started states (reuse existing cleanup logic) - if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState, report); skip { + if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState, report, scope); skip { return } if previousState == models.TaskSessionStateCreated { @@ -3413,7 +3469,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod // dead local_pid. If the local process is confirmed dead, repair the row // in place — resume_token/worktree are preserved (RowMustBePreserved // treats IDLE as resumable). Remote rows report Unknown and are untouched. - if s.rowLiveness(running) == models.ProcessLivenessDead { + if s.rowLivenessScoped(running, scope) == models.ProcessLivenessDead { s.repairDeadRowLiveness(ctx, running) } s.logger.Info("session reconciled for lazy recovery (idle, no state change)", @@ -3424,7 +3480,7 @@ func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *mod return } - s.reconcileActiveSessionOnStartup(ctx, running, sessionID, previousState, session) + s.reconcileActiveSessionOnStartup(ctx, running, sessionID, previousState, session, scope) } func (s *Service) reconcileActiveSessionOnStartup( @@ -3433,10 +3489,23 @@ func (s *Service) reconcileActiveSessionOnStartup( sessionID string, previousState models.TaskSessionState, session *models.TaskSession, + scope interface{}, ) { + // AC-EXECUTORS-SURVIVAL-003.1: the lifecycle manager's re-tracking + // outcome for sessionID is already decided by the time this runs (its + // Start() is synchronous and completes before this service's own Start() + // is ever called). A re-tracked session's agent survived the restart and + // is already running under this backend, so none of the "backend died + // mid-turn" reconciliation below applies to it + // (AC-EXECUTORS-SURVIVAL-003.2/.3/.4/.5). + retracked := s.wasSessionRetracked(sessionID) + // Active states: STARTING, RUNNING, WAITING_FOR_INPUT - // Set session to WAITING_FOR_INPUT (idle, ready for lazy resume when user opens it) - if previousState != models.TaskSessionStateWaitingForInput { + // Set session to WAITING_FOR_INPUT (idle, ready for lazy resume when user + // opens it) -- unless the session was re-tracked, in which case it is + // still genuinely running and AC-EXECUTORS-SURVIVAL-003.3 requires + // leaving it in that state. + if !retracked && previousState != models.TaskSessionStateWaitingForInput { s.updateTaskSessionStateWithHook( ctx, running.TaskID, @@ -3448,7 +3517,11 @@ func (s *Service) reconcileActiveSessionOnStartup( session, ) } - s.abandonOpenTurnsOnStartup(ctx, sessionID, "active session reconciled to waiting") + // AC-EXECUTORS-SURVIVAL-003.4: a re-tracked session's open turn is still + // live, not abandoned. + if !retracked { + s.abandonOpenTurnsOnStartup(ctx, sessionID, "active session reconciled to waiting") + } // PRESERVE executors_running.agent_execution_id post-restart. The in-memory // process is gone, but the stored ID still serves as a "this session was @@ -3461,8 +3534,9 @@ func (s *Service) reconcileActiveSessionOnStartup( // in-memory Add on the next launch, closing the divergence window we set out // to fix in this refactor. - // Ensure task is in REVIEW state (not stuck IN_PROGRESS) - if running.TaskID != "" { + // Ensure task is in REVIEW state (not stuck IN_PROGRESS) -- skipped for a + // re-tracked session's task, per AC-EXECUTORS-SURVIVAL-003.5. + if !retracked && running.TaskID != "" { task, taskErr := s.repo.GetTask(ctx, running.TaskID) if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress && !taskArchived(task) { // UpdateTaskStateIfCurrentIn (not the unconditional UpdateTaskState) @@ -3481,11 +3555,14 @@ func (s *Service) reconcileActiveSessionOnStartup( // Mark the task interrupted when its session was mid-turn when the backend // died (STARTING/RUNNING) so task-list surfaces can show the red // interruption icon. WAITING_FOR_INPUT sessions were idle, not interrupted. + // A re-tracked session's task was never actually interrupted -- its agent + // kept running across the restart -- so this is skipped for it per + // AC-EXECUTORS-SURVIVAL-003.2. // The write is archive-atomic (SetTaskMetadataKeyIfNotArchived), so an // archive that commits after this check cannot leave a stale marker on an // archived task. The marker is cleared when a session of the task next // enters STARTING/RUNNING (see updateTaskSessionStateWithHook). - if running.TaskID != "" && (previousState == models.TaskSessionStateStarting || previousState == models.TaskSessionStateRunning) { + if !retracked && running.TaskID != "" && (previousState == models.TaskSessionStateStarting || previousState == models.TaskSessionStateRunning) { if _, setErr := s.repo.SetTaskMetadataKeyIfNotArchived(ctx, running.TaskID, models.MetaKeyInterruptedAt, time.Now().UTC().Format(time.RFC3339)); setErr != nil { s.logger.Warn("failed to mark task interrupted on startup", zap.String("task_id", running.TaskID), @@ -3500,7 +3577,7 @@ func (s *Service) reconcileActiveSessionOnStartup( // cleared) so it never keeps claiming a live process (#1597 expected behavior). // resume_token/worktree are preserved by the repair. Remote rows report Unknown // and are left to their own runtime's status poll. - if s.rowLiveness(running) == models.ProcessLivenessDead { + if s.rowLivenessScoped(running, scope) == models.ProcessLivenessDead { s.repairDeadRowLiveness(ctx, running) } @@ -3508,17 +3585,18 @@ func (s *Service) reconcileActiveSessionOnStartup( zap.String("session_id", sessionID), zap.String("task_id", running.TaskID), zap.String("previous_state", string(previousState)), + zap.Bool("retracked", retracked), zap.Bool("has_resume_token", running.ResumeToken != ""), zap.Bool("has_worktree", running.WorktreePath != "")) } -func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport) { +func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *models.ExecutorRunning, report *startupCleanupReport, scope interface{}) { sessionID := running.SessionID executionID := strings.TrimSpace(running.AgentExecutionID) if executionID == "" || s.agentManager == nil { decision := startupCleanupDecision{ disposition: startupCleanupDispositionPreserved, - liveness: processLivenessClass(s.rowLiveness(running)), + liveness: processLivenessClass(s.rowLivenessScoped(running, scope)), stopErrorClass: "missing_stop_handle", expected: true, } @@ -3534,7 +3612,7 @@ func (s *Service) handleMissingSessionOnStartup(ctx context.Context, running *mo // invariant, so the orphan row does not survive restarts forever. Any // other error (alive/unknown/remote row, or a non-not-found failure) is // preserved and left for a later attempt. - decision := classifyStartupCleanup(err, s.rowLiveness(running)) + decision := classifyStartupCleanup(err, s.rowLivenessScoped(running, scope)) if !decision.proceed { if decision.expected { report.recordExpected(running, decision) @@ -3578,7 +3656,7 @@ func isTaskSessionNotFound(err error) bool { // handleTerminalSessionOnStartup processes sessions in terminal states during startup. // Returns true if the session should be skipped (no further processing needed). -func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState, report *startupCleanupReport) bool { +func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState, report *startupCleanupReport, scope interface{}) bool { sessionID := session.ID switch previousState { case models.TaskSessionStateCompleted, models.TaskSessionStateCancelled: @@ -3587,7 +3665,7 @@ func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *m zap.String("task_id", session.TaskID), zap.String("state", string(previousState))) s.abandonOpenTurnsOnStartup(ctx, sessionID, "terminal session cleanup") - if !s.stopRuntimeForStartupCleanup(ctx, running, "startup terminal session cleanup", report) { + if !s.stopRuntimeForStartupCleanup(ctx, running, "startup terminal session cleanup", report, scope) { return true } // Resume-safety invariant: prune the row only if it is not still resumable @@ -3597,14 +3675,14 @@ func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *m s.pruneOrRepairExecutorRow(ctx, running, previousState) return true case models.TaskSessionStateFailed: - s.handleFailedSessionOnStartup(ctx, session, running, report) + s.handleFailedSessionOnStartup(ctx, session, running, report, scope) return true } return false } // handleFailedSessionOnStartup handles a failed session during startup recovery. -func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, report *startupCleanupReport) { +func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, report *startupCleanupReport, scope interface{}) { sessionID := session.ID s.abandonOpenTurnsOnStartup(ctx, sessionID, "failed session cleanup") // If session failed, ensure task is in REVIEW state (not stuck IN_PROGRESS) @@ -3632,7 +3710,7 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod s.logger.Info("stopping failed session runtime before cleaning up executor record", zap.String("session_id", sessionID), zap.String("task_id", session.TaskID)) - if !s.stopRuntimeForStartupCleanup(ctx, running, "startup failed session cleanup", report) { + if !s.stopRuntimeForStartupCleanup(ctx, running, "startup failed session cleanup", report, scope) { return } // Prune only subject to the resume-safety invariant (a lingering @@ -3650,10 +3728,10 @@ func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *mod // (alive/unknown/remote row, or a non-not-found failure) preserves the row for a // later attempt. A row with no stoppable handle proceeds only when its local // process liveness is confirmed dead; alive and Unknown rows remain durable. -func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *models.ExecutorRunning, reason string, report *startupCleanupReport) bool { +func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *models.ExecutorRunning, reason string, report *startupCleanupReport, scope interface{}) bool { executionID := strings.TrimSpace(running.AgentExecutionID) if executionID == "" || s.agentManager == nil { - liveness := s.rowLiveness(running) + liveness := s.rowLivenessScoped(running, scope) decision := startupCleanupDecision{ disposition: startupCleanupDispositionPreserved, liveness: processLivenessClass(liveness), @@ -3679,7 +3757,7 @@ func (s *Service) stopRuntimeForStartupCleanup(ctx context.Context, running *mod if err == nil { return true } - decision := classifyStartupCleanup(err, s.rowLiveness(running)) + decision := classifyStartupCleanup(err, s.rowLivenessScoped(running, scope)) if !decision.proceed { fields := append(startupCleanupFields(running, decision), zap.String("reason", reason)) if decision.expected { diff --git a/apps/backend/internal/orchestrator/session_recovery_guard_error.go b/apps/backend/internal/orchestrator/session_recovery_guard_error.go new file mode 100644 index 0000000000..60e1b48d17 --- /dev/null +++ b/apps/backend/internal/orchestrator/session_recovery_guard_error.go @@ -0,0 +1,45 @@ +package orchestrator + +const detailsKeyKind = "kind" + +// SessionRecoveryGuardError identifies a session launch refused by the +// startup recovery guard. It wraps the original sentinel error so callers +// can still use errors.Is/errors.As. +type SessionRecoveryGuardError struct { + Cause error + SessionID string + Retryable bool +} + +func (e *SessionRecoveryGuardError) Error() string { + if e == nil { + return "session launch is blocked by the recovery guard" + } + if e.Retryable { + return "the session is still being reconciled after a backend restart; try again shortly" + } + return "the session's agent could not be stopped after a backend restart and requires another restart before it can be launched" +} + +func (e *SessionRecoveryGuardError) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} + +// Details returns the stable WebSocket recovery contract for this error. +func (e *SessionRecoveryGuardError) Details() map[string]interface{} { + if e == nil { + return nil + } + kind := "session_recovery_unstoppable" + if e.Retryable { + kind = "session_recovery_in_progress" + } + return map[string]interface{}{ + detailsKeyKind: kind, + "retryable": e.Retryable, + metaKeySessionID: e.SessionID, + } +} diff --git a/apps/backend/internal/orchestrator/session_recovery_guard_error_test.go b/apps/backend/internal/orchestrator/session_recovery_guard_error_test.go new file mode 100644 index 0000000000..c02fec3511 --- /dev/null +++ b/apps/backend/internal/orchestrator/session_recovery_guard_error_test.go @@ -0,0 +1,42 @@ +package orchestrator + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSessionRecoveryGuardErrorRetryablePreservesTypedCauseAndDetails(t *testing.T) { + cause := errors.New("session is guarded pending recovery") + err := &SessionRecoveryGuardError{Cause: cause, SessionID: "session-1", Retryable: true} + + require.ErrorIs(t, error(err), cause) + require.Equal(t, map[string]interface{}{ + "kind": "session_recovery_in_progress", + "retryable": true, + "session_id": "session-1", + }, err.Details()) + require.Contains(t, err.Error(), "try again shortly") +} + +func TestSessionRecoveryGuardErrorNonRetryablePreservesTypedCauseAndDetails(t *testing.T) { + cause := errors.New("session has an unstoppable agent from a prior launch") + err := &SessionRecoveryGuardError{Cause: cause, SessionID: "session-2", Retryable: false} + + require.ErrorIs(t, error(err), cause) + require.Equal(t, map[string]interface{}{ + "kind": "session_recovery_unstoppable", + "retryable": false, + "session_id": "session-2", + }, err.Details()) + require.Contains(t, err.Error(), "requires another restart") +} + +func TestSessionRecoveryGuardErrorNilReceiverIsSafe(t *testing.T) { + var err *SessionRecoveryGuardError + + require.Nil(t, err.Unwrap()) + require.Nil(t, err.Details()) + require.Contains(t, err.Error(), "blocked by the recovery guard") +} diff --git a/apps/backend/internal/profiles/profiles.yaml b/apps/backend/internal/profiles/profiles.yaml index f3873f3e28..4ad4c382be 100644 --- a/apps/backend/internal/profiles/profiles.yaml +++ b/apps/backend/internal/profiles/profiles.yaml @@ -106,6 +106,15 @@ features: dev: "true" e2e: "true" + # Agent survival across a backend restart (worktree/local executors only). + # Off in every profile: it replaces the standalone control server's + # kill-on-restart safeguard with an adoption handshake, and is unavailable + # on Windows. E2E specs that need it set the env var per-spec. + KANDEV_FEATURES_AGENT_SURVIVAL: + prod: "false" + dev: "false" + e2e: "false" + # ── Mock providers ──────────────────────────────────────────────── # Swap real third-party services for in-memory fakes. Production # leaves all of these unset (real APIs). Dev opts into the mock agent diff --git a/apps/backend/internal/runtimeflags/availability_test.go b/apps/backend/internal/runtimeflags/availability_test.go new file mode 100644 index 0000000000..fef23e1abc --- /dev/null +++ b/apps/backend/internal/runtimeflags/availability_test.go @@ -0,0 +1,112 @@ +package runtimeflags + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/common/config" +) + +// TestStateForReportsAvailabilityFromProbe pins the additive availability +// mechanism from system-design/agent-survival-across-restart-02.md: a nil-able +// probe on the flag definition, present in the serialized state only when the +// flag is unavailable, and an unavailable flag behaves as disabled regardless +// of its stored value. +func TestStateForReportsAvailabilityFromProbe(t *testing.T) { + svc := NewService(&memoryStore{}, Options{ + DefaultValues: map[string]bool{"features.testProbe": true}, + RuntimeValues: map[string]bool{"features.testProbe": true}, + IsExplicitEnv: func(string) bool { return false }, + }) + def := RuntimeFlagDefinition{ + Key: "features.testProbe", + EnvVar: "KANDEV_FEATURES_TEST_PROBE", + Kind: KindFeature, + Label: "Test probe flag", + Description: "test", + Stability: StabilityExperimental, + RiskLevel: RiskHigh, + RiskDescription: "test", + RestartRequired: true, + Mutable: true, + Available: func() (bool, string) { return false, "platform_unsupported" }, + } + + state := svc.stateFor(def, map[string]bool{"features.testProbe": true}) + + if state.Availability == nil { + t.Fatal("Availability = nil, want a populated availability object for an unavailable flag") + } + if state.Availability.Available { + t.Fatal("Availability.Available = true, want false") + } + if state.Availability.ReasonCode != "platform_unsupported" { + t.Fatalf("Availability.ReasonCode = %q, want %q", state.Availability.ReasonCode, "platform_unsupported") + } + if state.EffectiveValue { + t.Fatal("EffectiveValue = true, want false: an unavailable flag must behave as disabled regardless of its stored override") + } +} + +func TestStateForNilProbeReportsNoAvailabilityObject(t *testing.T) { + svc := NewService(&memoryStore{}, Options{ + DefaultValues: map[string]bool{"features.office": false}, + RuntimeValues: map[string]bool{"features.office": false}, + IsExplicitEnv: func(string) bool { return false }, + }) + def, ok := DefinitionByKey("features.office") + if !ok { + t.Fatal("features.office definition missing") + } + state := svc.stateFor(def, nil) + if state.Availability != nil { + t.Fatalf("Availability = %+v, want nil when the definition has no probe", state.Availability) + } +} + +// TestSetOverrideRefusesEnablingAnUnavailableFlag pins that turning an +// unavailable flag on is refused rather than silently stored, while clearing +// or disabling it is still allowed. The probe is injected on a throwaway +// registration since the real agent-survival probe is host-dependent. +func TestSetOverrideRefusesEnablingAnUnavailableFlag(t *testing.T) { + const key = "features.testProbeRefuse" + registrations = append(registrations, runtimeFlagRegistration{ + definition: RuntimeFlagDefinition{ + Key: key, + EnvVar: "KANDEV_FEATURES_TEST_PROBE_REFUSE", + Kind: KindFeature, + Label: "Test probe refuse flag", + Description: "test", + Stability: StabilityExperimental, + RiskLevel: RiskHigh, + RiskDescription: "test", + RestartRequired: true, + Mutable: true, + Available: func() (bool, string) { return false, "platform_unsupported" }, + }, + read: func(*config.Config) bool { return false }, + apply: func(*config.Config, bool) {}, + }) + t.Cleanup(func() { registrations = registrations[:len(registrations)-1] }) + + svc := NewService(&memoryStore{}, Options{ + DefaultValues: map[string]bool{key: false}, + RuntimeValues: map[string]bool{key: false}, + IsExplicitEnv: func(string) bool { return false }, + }) + + enable := true + if _, err := svc.SetOverride(context.Background(), key, &enable); !errors.Is(err, ErrFlagUnavailable) { + t.Fatalf("SetOverride(true) error = %v, want %v", err, ErrFlagUnavailable) + } + + disable := false + if _, err := svc.SetOverride(context.Background(), key, &disable); err != nil { + t.Fatalf("SetOverride(false) on an unavailable flag: %v", err) + } + + if _, err := svc.SetOverride(context.Background(), key, nil); err != nil { + t.Fatalf("SetOverride(nil) on an unavailable flag: %v", err) + } +} diff --git a/apps/backend/internal/runtimeflags/handlers.go b/apps/backend/internal/runtimeflags/handlers.go index 0cfef37676..bbc263c6a1 100644 --- a/apps/backend/internal/runtimeflags/handlers.go +++ b/apps/backend/internal/runtimeflags/handlers.go @@ -54,6 +54,9 @@ func (h *Handler) patch(c *gin.Context) { } else if errors.Is(err, ErrUnknownFlag) { status = http.StatusNotFound msg = "runtime flag not found" + } else if errors.Is(err, ErrFlagUnavailable) { + status = http.StatusConflict + msg = "runtime flag is unavailable on this host" } c.JSON(status, gin.H{"error": msg}) return diff --git a/apps/backend/internal/runtimeflags/registry.go b/apps/backend/internal/runtimeflags/registry.go index 987ed24b78..6ef786e90b 100644 --- a/apps/backend/internal/runtimeflags/registry.go +++ b/apps/backend/internal/runtimeflags/registry.go @@ -1,6 +1,26 @@ package runtimeflags -import "github.com/kandev/kandev/internal/common/config" +import ( + "runtime" + + "github.com/kandev/kandev/internal/common/config" +) + +// ReasonPlatformUnsupported is the stable, machine-readable reason code for a +// flag that requires a host platform this install does not run on. The +// frontend translates it; it is never shown to the operator as raw text. +const ReasonPlatformUnsupported = "platform_unsupported" + +// agentSurvivalAvailability implements the platform scope decision in +// system-design/agent-survival-across-restart-02.md: survival is supported on +// macOS and Linux, and unavailable on Windows, where it would trade the +// platform's kill-on-job-close safeguard for an untested adoption handshake. +func agentSurvivalAvailability() (bool, string) { + if runtime.GOOS == "windows" { + return false, ReasonPlatformUnsupported + } + return true, "" +} // runtimeFlagRegistration keeps the public metadata and the typed config // binding for a flag together. The function fields stay internal so the HTTP @@ -177,6 +197,25 @@ var registrations = []runtimeFlagRegistration{ read: func(cfg *config.Config) bool { return cfg.Features.OfficeSessionIdentity }, apply: func(cfg *config.Config, value bool) { cfg.Features.OfficeSessionIdentity = value }, }, + { + definition: RuntimeFlagDefinition{ + Key: "features.agentSurvival", + EnvVar: "KANDEV_FEATURES_AGENT_SURVIVAL", + Kind: KindFeature, + Label: "Agent survival across backend restart", + Description: "Lets a worktree or local-executor agent session survive a backend restart by adopting its still-running standalone control server instead of killing it.", + Stability: StabilityExperimental, + RiskLevel: RiskHigh, + RiskDescription: "Replaces the standalone control server's kill-on-restart safeguard with an adoption handshake. " + + "Enable it only after reviewing the recovery and ownership guarantees, since it changes what happens to an agent " + + "process when the backend restarts unexpectedly. Unavailable on Windows, where the removed safeguard is depended on.", + RestartRequired: true, + Mutable: true, + Available: agentSurvivalAvailability, + }, + read: func(cfg *config.Config) bool { return cfg.Features.AgentSurvival }, + apply: func(cfg *config.Config, value bool) { cfg.Features.AgentSurvival = value }, + }, { definition: RuntimeFlagDefinition{ Key: "debug.devMode", diff --git a/apps/backend/internal/runtimeflags/registry_test.go b/apps/backend/internal/runtimeflags/registry_test.go index 66a2824bbe..6f8307784a 100644 --- a/apps/backend/internal/runtimeflags/registry_test.go +++ b/apps/backend/internal/runtimeflags/registry_test.go @@ -2,6 +2,7 @@ package runtimeflags import ( "reflect" + "runtime" "strings" "testing" @@ -476,3 +477,64 @@ func TestDefinitionsIncludeClaudeMidTurnSteeringMetadata(t *testing.T) { t.Fatal("RestartRequired = false, want true") } } + +// TestDefinitionsIncludeAgentSurvivalMetadata pins the registration for +// AC-EXECUTORS-SURVIVAL-005.2/5.4: off by default, restart-required, and +// carrying a host-availability probe (checked separately below). +func TestDefinitionsIncludeAgentSurvivalMetadata(t *testing.T) { + def, ok := DefinitionByKey("features.agentSurvival") + if !ok { + t.Fatal("features.agentSurvival definition missing") + } + if def.EnvVar != "KANDEV_FEATURES_AGENT_SURVIVAL" { + t.Fatalf("EnvVar = %q, want KANDEV_FEATURES_AGENT_SURVIVAL", def.EnvVar) + } + if def.Stability != StabilityExperimental { + t.Fatalf("Stability = %q, want experimental", def.Stability) + } + if def.RiskLevel != RiskHigh { + t.Fatalf("RiskLevel = %q, want high", def.RiskLevel) + } + if def.RiskDescription == "" { + t.Fatal("RiskDescription empty") + } + if !def.RestartRequired { + t.Fatal("RestartRequired = false, want true") + } + if !def.Mutable { + t.Fatal("Mutable = false, want true") + } + if def.Available == nil { + t.Fatal("Available probe missing; AC-EXECUTORS-SURVIVAL-005.4 requires the platform-unsupported state to be reportable") + } + defaults, err := profiles.FeatureFlagDefaults() + if err != nil { + t.Fatalf("profiles.FeatureFlagDefaults: %v", err) + } + if got := defaults["agent_survival"]; got != "false" { + t.Fatalf("profiles.yaml agent_survival default = %q, want %q (AC-EXECUTORS-SURVIVAL-005.2: off by default)", got, "false") + } +} + +// TestAgentSurvivalAvailabilityMatchesPlatformScope pins the platform-scope +// decision (macOS + Linux only) against whatever GOOS this test actually runs +// on, so the Windows CI job exercises the unavailable branch for real instead +// of every job only ever seeing "available". +func TestAgentSurvivalAvailabilityMatchesPlatformScope(t *testing.T) { + available, reasonCode := agentSurvivalAvailability() + if runtime.GOOS == "windows" { + if available { + t.Fatal("agentSurvivalAvailability() = available, want unavailable on Windows") + } + if reasonCode != ReasonPlatformUnsupported { + t.Fatalf("reasonCode = %q, want %q", reasonCode, ReasonPlatformUnsupported) + } + return + } + if !available { + t.Fatalf("agentSurvivalAvailability() = unavailable (%q), want available on %s", reasonCode, runtime.GOOS) + } + if reasonCode != "" { + t.Fatalf("reasonCode = %q, want empty when available", reasonCode) + } +} diff --git a/apps/backend/internal/runtimeflags/service.go b/apps/backend/internal/runtimeflags/service.go index f39fb8d004..74461afb6c 100644 --- a/apps/backend/internal/runtimeflags/service.go +++ b/apps/backend/internal/runtimeflags/service.go @@ -8,9 +8,10 @@ import ( ) var ( - ErrEnvLocked = errors.New("runtime flag is controlled by environment") - ErrStoreUnset = errors.New("runtime flag store is not configured") - ErrUnknownFlag = errors.New("unknown runtime flag") + ErrEnvLocked = errors.New("runtime flag is controlled by environment") + ErrStoreUnset = errors.New("runtime flag store is not configured") + ErrUnknownFlag = errors.New("unknown runtime flag") + ErrFlagUnavailable = errors.New("runtime flag is unavailable on this host") ) type Options struct { @@ -60,6 +61,9 @@ func (s *Service) SetOverride(ctx context.Context, key string, value *bool) ([]R if s.isEnvLocked(def) { return nil, ErrEnvLocked } + if value != nil && *value && !s.isAvailable(def) { + return nil, ErrFlagUnavailable + } if value == nil { if err := s.store.DeleteOverride(ctx, key); err != nil { return nil, err @@ -91,6 +95,17 @@ func (s *Service) stateFor(def RuntimeFlagDefinition, overrides map[string]bool) source = SourceDefault } + var availability *RuntimeFlagAvailability + if def.Available != nil { + if available, reasonCode := def.Available(); !available { + // An unavailable flag behaves as disabled whatever its stored + // value, so an admin override cannot leave a host claiming a + // capability it cannot honour. + effectiveValue = false + availability = &RuntimeFlagAvailability{Available: false, ReasonCode: reasonCode} + } + } + return RuntimeFlagState{ Key: def.Key, Kind: def.Kind, @@ -108,6 +123,7 @@ func (s *Service) stateFor(def RuntimeFlagDefinition, overrides map[string]bool) RestartRequired: def.RestartRequired, RequiresRestartToApply: def.RestartRequired && effectiveValue != runtimeValue, Mutable: def.Mutable, + Availability: availability, } } @@ -116,6 +132,14 @@ func (s *Service) isEnvLocked(def RuntimeFlagDefinition) bool { return locked } +func (s *Service) isAvailable(def RuntimeFlagDefinition) bool { + if def.Available == nil { + return true + } + available, _ := def.Available() + return available +} + func (s *Service) envLockedValue(def RuntimeFlagDefinition, fallback bool) (bool, bool) { if s.opts.IsExplicitEnv(def.EnvVar) { return s.envValue(def.EnvVar, fallback), true diff --git a/apps/backend/internal/runtimeflags/types.go b/apps/backend/internal/runtimeflags/types.go index e53570c2a9..aa912d8465 100644 --- a/apps/backend/internal/runtimeflags/types.go +++ b/apps/backend/internal/runtimeflags/types.go @@ -34,6 +34,11 @@ const ( RiskHigh RuntimeFlagRiskLevel = "high" ) +// RuntimeFlagAvailabilityProbe reports whether a flag can be enabled on the +// current host and, when it cannot, a stable machine-readable reason code the +// frontend translates. Nil means always available. +type RuntimeFlagAvailabilityProbe func() (available bool, reasonCode string) + type RuntimeFlagDefinition struct { Key string `json:"key"` EnvVar string `json:"env_var"` @@ -46,25 +51,37 @@ type RuntimeFlagDefinition struct { RestartRequired bool `json:"restart_required"` Mutable bool `json:"mutable"` ImpliedEnvVars []string `json:"-"` + // Available is an optional host-availability probe. Nil (every existing + // registration) means always available, which keeps this additive: no + // other flag's behavior, serialization, or admin-override path changes. + Available RuntimeFlagAvailabilityProbe `json:"-"` +} + +// RuntimeFlagAvailability reports why a flag cannot be enabled on this host. +// Present in RuntimeFlagState only when the flag is unavailable. +type RuntimeFlagAvailability struct { + Available bool `json:"available"` + ReasonCode string `json:"reason_code,omitempty"` } type RuntimeFlagState struct { - Key string `json:"key"` - Kind RuntimeFlagKind `json:"kind"` - Label string `json:"label"` - Description string `json:"description"` - Stability RuntimeFlagStability `json:"stability"` - RiskLevel RuntimeFlagRiskLevel `json:"risk_level"` - RiskDescription string `json:"risk_description"` - EffectiveValue bool `json:"effective_value"` - DefaultValue bool `json:"default_value"` - OverrideValue *bool `json:"override_value"` - Source RuntimeFlagSource `json:"source"` - EnvVar string `json:"env_var"` - EnvLocked bool `json:"env_locked"` - RestartRequired bool `json:"restart_required"` - RequiresRestartToApply bool `json:"requires_restart_to_apply"` - Mutable bool `json:"mutable"` + Key string `json:"key"` + Kind RuntimeFlagKind `json:"kind"` + Label string `json:"label"` + Description string `json:"description"` + Stability RuntimeFlagStability `json:"stability"` + RiskLevel RuntimeFlagRiskLevel `json:"risk_level"` + RiskDescription string `json:"risk_description"` + EffectiveValue bool `json:"effective_value"` + DefaultValue bool `json:"default_value"` + OverrideValue *bool `json:"override_value"` + Source RuntimeFlagSource `json:"source"` + EnvVar string `json:"env_var"` + EnvLocked bool `json:"env_locked"` + RestartRequired bool `json:"restart_required"` + RequiresRestartToApply bool `json:"requires_restart_to_apply"` + Mutable bool `json:"mutable"` + Availability *RuntimeFlagAvailability `json:"availability,omitempty"` } type Store interface { diff --git a/apps/backend/internal/task/models/control_server_record.go b/apps/backend/internal/task/models/control_server_record.go new file mode 100644 index 0000000000..b64426d403 --- /dev/null +++ b/apps/backend/internal/task/models/control_server_record.go @@ -0,0 +1,46 @@ +package models + +import ( + "errors" + "time" +) + +// ErrControlServerRecordNotFound is returned when no control-server record +// has been written yet, e.g. before the standalone control server is ever +// started or adopted on this installation. +var ErrControlServerRecordNotFound = errors.New("control server record not found") + +// ControlServerRecord is the single installation-scoped durable record of the +// standalone agentctl control server that supervises every worktree and local +// agent instance on this host: where it is, how to prove ownership of it, and +// what it can do. Exactly one row exists per installation. +// +// It is deliberately separate from ExecutorRunning: that table is per-session +// and has no row when no session is live, which would leave a detached, +// instance-less server unlocatable. Written when a server is started or +// adopted, read before any control-server contact at startup. +type ControlServerRecord struct { + // Endpoint is the control server's host:port. The launcher may relocate to + // a free port when the configured one is busy, so this is the only durable + // way a restarted backend can find a relocated survivor. + Endpoint string `json:"endpoint"` + // ServerIdentity is the opaque per-launch value the control server + // echoes on its identity endpoint, compared by the adopting backend to + // confirm it is talking to the server this record describes. + ServerIdentity string `json:"server_identity"` + // CredentialSecretID is a reference into the secret store for the single + // rotating credential that authenticates both control-plane and + // per-instance operations and streams (design 01 "Single driver", + // AC-EXECUTORS-CONTROL-OWNERSHIP-002.6). The credential itself is never + // stored here: this is a reference, not the bearer token. + CredentialSecretID string `json:"credential_secret_id"` + // Capabilities is the named capability set last observed from the + // control server's identity endpoint. + Capabilities []string `json:"capabilities"` + // DiagnosticLogPath is the location of the control server's detached + // diagnostic-output log sink, so both the next backend and an operator + // can find it. + DiagnosticLogPath string `json:"diagnostic_log_path"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/apps/backend/internal/task/repository/sqlite/base_schema.go b/apps/backend/internal/task/repository/sqlite/base_schema.go index 3e71285f54..7d2f0255a8 100644 --- a/apps/backend/internal/task/repository/sqlite/base_schema.go +++ b/apps/backend/internal/task/repository/sqlite/base_schema.go @@ -37,6 +37,7 @@ func (r *Repository) initSchemaContext(ctx context.Context) error { r.initTaskUsageEventsSchema, r.initAttachmentsSchema, r.initTaskResourceCleanupSchema, + r.initControlServerRecordSchema, r.initGitSchema, r.initReviewSchema, r.initTaskReviewSchema, @@ -185,6 +186,28 @@ func (r *Repository) initTaskResourceCleanupSchema() error { return err } +// control_server_records is the single installation-scoped durable record of +// the standalone agentctl control server (see models.ControlServerRecord). +// The id CHECK enforces exactly one row, the same singleton pattern used by +// dynamic_installation_keys above. +const controlServerRecordSchemaDDL = ` + CREATE TABLE IF NOT EXISTS control_server_records ( + id INTEGER PRIMARY KEY CHECK (id = 1), + endpoint TEXT NOT NULL, + server_identity TEXT NOT NULL, + credential_secret_id TEXT NOT NULL, + capabilities TEXT NOT NULL DEFAULT '[]', + diagnostic_log_path TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ); +` + +func (r *Repository) initControlServerRecordSchema() error { + _, err := r.db.Exec(controlServerRecordSchemaDDL) + return err +} + // ensureWorkspaceIndexes creates workspace-related indexes func (r *Repository) ensureWorkspaceIndexes() error { if _, err := r.db.ExecContext(r.migrationContext(), `CREATE INDEX IF NOT EXISTS idx_tasks_workspace_id ON tasks(workspace_id)`); err != nil { diff --git a/apps/backend/internal/task/repository/sqlite/control_server_record.go b/apps/backend/internal/task/repository/sqlite/control_server_record.go new file mode 100644 index 0000000000..0c4b95b199 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/control_server_record.go @@ -0,0 +1,94 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +// GetControlServerRecord reads the single installation-scoped control-server +// record. Startup step 4 reads this before any control-server contact: with +// no record (models.ErrControlServerRecordNotFound), the caller spawns a +// fresh server and reports no recovered instances. +func (r *Repository) GetControlServerRecord(ctx context.Context) (*models.ControlServerRecord, error) { + record := &models.ControlServerRecord{} + var capabilitiesJSON string + + err := r.ro.QueryRowContext(ctx, ` + SELECT endpoint, server_identity, credential_secret_id, capabilities, diagnostic_log_path, created_at, updated_at + FROM control_server_records WHERE id = 1 + `).Scan( + &record.Endpoint, &record.ServerIdentity, &record.CredentialSecretID, &capabilitiesJSON, + &record.DiagnosticLogPath, &record.CreatedAt, &record.UpdatedAt, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, models.ErrControlServerRecordNotFound + } + if err != nil { + return nil, err + } + + if capabilitiesJSON != "" { + if err := json.Unmarshal([]byte(capabilitiesJSON), &record.Capabilities); err != nil { + return nil, fmt.Errorf("failed to deserialize control server record capabilities: %w", err) + } + } + return record, nil +} + +// UpsertControlServerRecord writes the single installation-scoped +// control-server record, replacing whatever record was there before. A +// server started after a refused or failed adoption rewrites the record to +// name the new server: the record cannot keep pointing at a server that is +// about to reap itself. CreatedAt is preserved across a rewrite. +func (r *Repository) UpsertControlServerRecord(ctx context.Context, record *models.ControlServerRecord) error { + if record == nil { + return fmt.Errorf("control server record is nil") + } + + now := time.Now().UTC() + if record.CreatedAt.IsZero() { + record.CreatedAt = now + } + record.UpdatedAt = now + + capabilities := record.Capabilities + if capabilities == nil { + capabilities = []string{} + } + capabilitiesJSON, err := json.Marshal(capabilities) + if err != nil { + return fmt.Errorf("failed to serialize control server record capabilities: %w", err) + } + + _, err = r.db.ExecContext(ctx, r.db.Rebind(` + INSERT INTO control_server_records ( + id, endpoint, server_identity, credential_secret_id, capabilities, diagnostic_log_path, created_at, updated_at + ) VALUES (1, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + endpoint = excluded.endpoint, + server_identity = excluded.server_identity, + credential_secret_id = excluded.credential_secret_id, + capabilities = excluded.capabilities, + diagnostic_log_path = excluded.diagnostic_log_path, + updated_at = excluded.updated_at + `), + record.Endpoint, + record.ServerIdentity, + record.CredentialSecretID, + string(capabilitiesJSON), + record.DiagnosticLogPath, + record.CreatedAt, + record.UpdatedAt, + ) + // created_at is deliberately absent from the UPDATE SET list above, so a + // rewrite after a prior record's row keeps the original creation time in + // the database even though the in-memory record above was stamped with + // this call's now. + return err +} diff --git a/apps/backend/internal/task/repository/sqlite/control_server_record_postgres_test.go b/apps/backend/internal/task/repository/sqlite/control_server_record_postgres_test.go new file mode 100644 index 0000000000..a49c1dd004 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/control_server_record_postgres_test.go @@ -0,0 +1,95 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresControlServerRecordRoundTrips is the PostgreSQL counterpart to +// TestUpsertControlServerRecordThenGetRoundTrips/TestUpsertControlServerRecordIsSingleton +// (SQLite): control_server_records is a new table reached through raw SQL +// (an ON CONFLICT(id) DO UPDATE upsert plus a JSON-encoded capabilities +// column), so schema replay alone does not prove the upsert, singleton, and +// CreatedAt-preservation behavior actually holds on this dialect too. +// Skips unless KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresControlServerRecordRoundTrips(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + + if _, err := repo.GetControlServerRecord(ctx); err == nil { + t.Fatalf("expected ErrControlServerRecordNotFound before any write") + } + + first := &models.ControlServerRecord{ + Endpoint: "127.0.0.1:41123", + ServerIdentity: "server-identity-1", + CredentialSecretID: "secret-ref-1", + Capabilities: []string{"resume", "diagnostics"}, + DiagnosticLogPath: "/home/kandev/logs/agentctl-diagnostic.log", + } + if err := repo.UpsertControlServerRecord(ctx, first); err != nil { + t.Fatalf("UpsertControlServerRecord(first): %v", err) + } + firstCreatedAt := first.CreatedAt + + got, err := repo.GetControlServerRecord(ctx) + if err != nil { + t.Fatalf("GetControlServerRecord: %v", err) + } + if got.Endpoint != first.Endpoint || got.ServerIdentity != first.ServerIdentity || + got.CredentialSecretID != first.CredentialSecretID || got.DiagnosticLogPath != first.DiagnosticLogPath { + t.Fatalf("got = %#v, want match of %#v", got, first) + } + if len(got.Capabilities) != 2 || got.Capabilities[0] != "resume" || got.Capabilities[1] != "diagnostics" { + t.Fatalf("Capabilities = %#v, want [resume diagnostics]", got.Capabilities) + } + + time.Sleep(2 * time.Millisecond) + + second := &models.ControlServerRecord{ + Endpoint: "127.0.0.1:52222", + ServerIdentity: "server-identity-2", + CredentialSecretID: "secret-ref-2", + DiagnosticLogPath: "/home/kandev/logs/agentctl-diagnostic.log", + } + if err := repo.UpsertControlServerRecord(ctx, second); err != nil { + t.Fatalf("UpsertControlServerRecord(second): %v", err) + } + + got2, err := repo.GetControlServerRecord(ctx) + if err != nil { + t.Fatalf("GetControlServerRecord(after rewrite): %v", err) + } + if got2.Endpoint != second.Endpoint || got2.ServerIdentity != second.ServerIdentity { + t.Fatalf("got2 = %#v, want the rewritten (second) record", got2) + } + // Compared at microsecond granularity because PostgreSQL TIMESTAMP stores + // microseconds while Go's time.Time carries nanoseconds, so the first + // write's in-memory stamp reads back truncated (.330254029 -> .330254). + // That truncation is a storage property, not the contract under test: the + // regression this guards is an upsert that RESETS created_at to now, which + // the 2ms sleep above puts milliseconds away and which a microsecond + // comparison still catches with room to spare. + if !got2.CreatedAt.Truncate(time.Microsecond).Equal(firstCreatedAt.Truncate(time.Microsecond)) { + t.Fatalf("CreatedAt = %v, want preserved original %v", got2.CreatedAt, firstCreatedAt) + } + if !got2.UpdatedAt.After(got2.CreatedAt) { + t.Fatalf("UpdatedAt = %v, want after CreatedAt %v", got2.UpdatedAt, got2.CreatedAt) + } + + var count int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM control_server_records`).Scan(&count); err != nil { + t.Fatalf("count rows: %v", err) + } + if count != 1 { + t.Fatalf("row count = %d, want 1", count) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/control_server_record_test.go b/apps/backend/internal/task/repository/sqlite/control_server_record_test.go new file mode 100644 index 0000000000..e3f11910c2 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/control_server_record_test.go @@ -0,0 +1,122 @@ +package sqlite + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestGetControlServerRecordNotFound pins startup step 4's "no record" +// branch: before any server has ever been started or adopted on this +// installation, the read returns the distinguishable not-found sentinel +// rather than an empty record, so the caller can tell "spawn fresh" apart +// from "record read failed". +func TestGetControlServerRecordNotFound(t *testing.T) { + repo := newRepoForSessionTests(t) + + _, err := repo.GetControlServerRecord(context.Background()) + if !errors.Is(err, models.ErrControlServerRecordNotFound) { + t.Fatalf("err = %v, want %v", err, models.ErrControlServerRecordNotFound) + } +} + +// TestUpsertControlServerRecordThenGetRoundTrips pins that every field +// written by UpsertControlServerRecord (endpoint, identity, credential +// reference, capability set, diagnostic log location) survives a round trip +// unchanged. +func TestUpsertControlServerRecordThenGetRoundTrips(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + record := &models.ControlServerRecord{ + Endpoint: "127.0.0.1:41123", + ServerIdentity: "server-identity-1", + CredentialSecretID: "secret-ref-1", + Capabilities: []string{"resume", "diagnostics"}, + DiagnosticLogPath: "/home/kandev/logs/agentctl-diagnostic.log", + } + if err := repo.UpsertControlServerRecord(ctx, record); err != nil { + t.Fatalf("UpsertControlServerRecord: %v", err) + } + + got, err := repo.GetControlServerRecord(ctx) + if err != nil { + t.Fatalf("GetControlServerRecord: %v", err) + } + if got.Endpoint != record.Endpoint { + t.Errorf("Endpoint = %q, want %q", got.Endpoint, record.Endpoint) + } + if got.ServerIdentity != record.ServerIdentity { + t.Errorf("ServerIdentity = %q, want %q", got.ServerIdentity, record.ServerIdentity) + } + if got.CredentialSecretID != record.CredentialSecretID { + t.Errorf("CredentialSecretID = %q, want %q", got.CredentialSecretID, record.CredentialSecretID) + } + if len(got.Capabilities) != 2 || got.Capabilities[0] != "resume" || got.Capabilities[1] != "diagnostics" { + t.Errorf("Capabilities = %#v, want [resume diagnostics]", got.Capabilities) + } + if got.DiagnosticLogPath != record.DiagnosticLogPath { + t.Errorf("DiagnosticLogPath = %q, want %q", got.DiagnosticLogPath, record.DiagnosticLogPath) + } + if got.CreatedAt.IsZero() || got.UpdatedAt.IsZero() { + t.Errorf("CreatedAt/UpdatedAt not set: %#v", got) + } +} + +// TestUpsertControlServerRecordIsSingleton pins the installation-scoped +// invariant: a second write replaces the one existing row (own server started +// after a refused or failed adoption rewrites the record to name the new +// server) rather than creating a second row, and CreatedAt is preserved +// across the rewrite while UpdatedAt advances. +func TestUpsertControlServerRecordIsSingleton(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + first := &models.ControlServerRecord{ + Endpoint: "127.0.0.1:41123", + ServerIdentity: "server-identity-1", + CredentialSecretID: "secret-ref-1", + DiagnosticLogPath: "/home/kandev/logs/agentctl-diagnostic.log", + } + if err := repo.UpsertControlServerRecord(ctx, first); err != nil { + t.Fatalf("UpsertControlServerRecord(first): %v", err) + } + firstCreatedAt := first.CreatedAt + + time.Sleep(2 * time.Millisecond) + + second := &models.ControlServerRecord{ + Endpoint: "127.0.0.1:52222", + ServerIdentity: "server-identity-2", + CredentialSecretID: "secret-ref-2", + DiagnosticLogPath: "/home/kandev/logs/agentctl-diagnostic.log", + } + if err := repo.UpsertControlServerRecord(ctx, second); err != nil { + t.Fatalf("UpsertControlServerRecord(second): %v", err) + } + + got, err := repo.GetControlServerRecord(ctx) + if err != nil { + t.Fatalf("GetControlServerRecord: %v", err) + } + if got.Endpoint != second.Endpoint || got.ServerIdentity != second.ServerIdentity { + t.Fatalf("got = %#v, want the rewritten (second) record", got) + } + if !got.CreatedAt.Equal(firstCreatedAt) { + t.Errorf("CreatedAt = %v, want preserved original %v", got.CreatedAt, firstCreatedAt) + } + if !got.UpdatedAt.After(got.CreatedAt) { + t.Errorf("UpdatedAt = %v, want after CreatedAt %v", got.UpdatedAt, got.CreatedAt) + } + + var count int + if err := repo.ro.QueryRowContext(ctx, `SELECT COUNT(*) FROM control_server_records`).Scan(&count); err != nil { + t.Fatalf("count rows: %v", err) + } + if count != 1 { + t.Fatalf("row count = %d, want 1", count) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/executor.go b/apps/backend/internal/task/repository/sqlite/executor.go index 7166b8a8e5..9d37afde1e 100644 --- a/apps/backend/internal/task/repository/sqlite/executor.go +++ b/apps/backend/internal/task/repository/sqlite/executor.go @@ -11,6 +11,7 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/agentruntime" kandevdb "github.com/kandev/kandev/internal/db" "github.com/kandev/kandev/internal/db/dialect" "github.com/kandev/kandev/internal/task/models" @@ -298,6 +299,29 @@ func (r *Repository) ListExecutorsRunningIdle(ctx context.Context, cutoff time.T return scanExecutorRunningRows(rows) } +// ListExecutorsRunningLiveStandalone returns every non-terminal executors_running +// row whose runtime is the standalone control server (worktree/local executors). +// Used by startup recovery to build the guard/correlation inventory at step 3, +// before any control-server contact (discovery H). +func (r *Repository) ListExecutorsRunningLiveStandalone(ctx context.Context) ([]*models.ExecutorRunning, error) { + rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(` + SELECT id, session_id, task_id, execution_profile_id, executor_id, runtime, status, resumable, resume_token, + last_message_uuid, agent_execution_id, container_id, agentctl_url, agentctl_port, pid, local_pid, + worktree_id, worktree_path, worktree_branch, last_seen_at, error_message, metadata, + created_at, updated_at + FROM executors_running + WHERE runtime = ? AND status NOT IN (?, ?, ?) + ORDER BY updated_at DESC + `), agentruntime.RuntimeStandalone, + models.ExecutorRunningStatusStopped, models.ExecutorRunningStatusComplete, models.ExecutorRunningStatusFailed) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + return scanExecutorRunningRows(rows) +} + func (r *Repository) ListExecutorsRunningByTaskID(ctx context.Context, taskID string) ([]*models.ExecutorRunning, error) { if taskID == "" { return nil, fmt.Errorf("task_id is required") diff --git a/apps/backend/internal/task/repository/sqlite/executor_running_live_standalone_test.go b/apps/backend/internal/task/repository/sqlite/executor_running_live_standalone_test.go new file mode 100644 index 0000000000..918c91a9e4 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/executor_running_live_standalone_test.go @@ -0,0 +1,68 @@ +package sqlite + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +// TestListExecutorsRunningLiveStandaloneFiltersRuntimeAndStatus pins the +// startup recovery inventory query (discovery H): only rows on the standalone +// control server (worktree/local) that are not yet terminal are candidates +// for guard-taking and re-tracking correlation at startup step 3, before any +// control-server contact. +func TestListExecutorsRunningLiveStandaloneFiltersRuntimeAndStatus(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + + seedExecutorRunningCleanupTask(t, repo, "task-1") + + seedLiveStandaloneRow(t, repo, "task-1", "session-live-standalone", agentruntime.RuntimeStandalone, models.ExecutorRunningStatusRunning) + seedLiveStandaloneRow(t, repo, "task-1", "session-stopped-standalone", agentruntime.RuntimeStandalone, models.ExecutorRunningStatusStopped) + seedLiveStandaloneRow(t, repo, "task-1", "session-completed-standalone", agentruntime.RuntimeStandalone, models.ExecutorRunningStatusComplete) + seedLiveStandaloneRow(t, repo, "task-1", "session-failed-standalone", agentruntime.RuntimeStandalone, models.ExecutorRunningStatusFailed) + seedLiveStandaloneRow(t, repo, "task-1", "session-live-docker", agentruntime.RuntimeDocker, models.ExecutorRunningStatusRunning) + seedLiveStandaloneRow(t, repo, "task-1", "session-live-ssh", agentruntime.RuntimeSSH, models.ExecutorRunningStatusRunning) + + rows, err := repo.ListExecutorsRunningLiveStandalone(ctx) + if err != nil { + t.Fatalf("ListExecutorsRunningLiveStandalone: %v", err) + } + if len(rows) != 1 { + t.Fatalf("rows = %d, want 1: %#v", len(rows), rows) + } + if rows[0].SessionID != "session-live-standalone" { + t.Fatalf("SessionID = %q, want %q", rows[0].SessionID, "session-live-standalone") + } + if rows[0].Runtime != agentruntime.RuntimeStandalone { + t.Fatalf("Runtime = %q, want %q", rows[0].Runtime, agentruntime.RuntimeStandalone) + } +} + +func seedLiveStandaloneRow( + t *testing.T, + repo *Repository, + taskID, sessionID string, + runtime agentruntime.Runtime, + status string, +) { + t.Helper() + ctx := context.Background() + + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ID: sessionID, TaskID: taskID, State: models.TaskSessionStateRunning}); err != nil { + t.Fatalf("CreateTaskSession(%s): %v", sessionID, err) + } + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: sessionID, + SessionID: sessionID, + TaskID: taskID, + ExecutorID: "executor-1", + Runtime: runtime, + Status: status, + AgentExecutionID: "exec-" + sessionID, + }); err != nil { + t.Fatalf("UpsertExecutorRunning(%s): %v", sessionID, err) + } +} diff --git a/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go b/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go index cdbca0c5a9..ceeaa22315 100644 --- a/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go +++ b/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go @@ -215,7 +215,7 @@ func (b *persistedKubernetesCleanupBackend) StopInstance( b.pvcDeleted = true return nil } -func (*persistedKubernetesCleanupBackend) RecoverInstances(context.Context) ([]*lifecycle.ExecutorInstance, error) { +func (*persistedKubernetesCleanupBackend) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*lifecycle.ExecutorInstance, error) { return nil, nil } func (*persistedKubernetesCleanupBackend) GetInteractiveRunner() *process.InteractiveRunner { diff --git a/apps/web/components/settings/system/feature-toggle-card.test.tsx b/apps/web/components/settings/system/feature-toggle-card.test.tsx index 83406f7a91..608a7be9ea 100644 --- a/apps/web/components/settings/system/feature-toggle-card.test.tsx +++ b/apps/web/components/settings/system/feature-toggle-card.test.tsx @@ -1,5 +1,6 @@ import { cleanup, fireEvent, render, screen } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { TooltipProvider } from "@kandev/ui/tooltip"; import type { RuntimeFlagState } from "@/lib/types/runtime-flags"; import { FeatureToggleCard } from "./feature-toggle-card"; @@ -27,6 +28,8 @@ vi.mock("@kandev/ui/switch", () => ({ afterEach(cleanup); +const TOGGLE_LABEL = "Toggle Office mode"; + describe("FeatureToggleCard", () => { it("shows risk copy as supporting text instead of a warning alert", () => { render( @@ -56,7 +59,7 @@ describe("FeatureToggleCard", () => { />, ); - fireEvent.click(screen.getByLabelText("Toggle Office mode")); + fireEvent.click(screen.getByLabelText(TOGGLE_LABEL)); expect(onChange).toHaveBeenCalledWith(true); }); @@ -89,7 +92,7 @@ describe("FeatureToggleCard", () => { />, ); - expect(screen.getByLabelText("Toggle Office mode")).toHaveProperty("disabled", true); + expect(screen.getByLabelText(TOGGLE_LABEL)).toHaveProperty("disabled", true); expect(screen.getByRole("button", { name: /use default/i })).toHaveProperty("disabled", true); expect(screen.getByText("Controlled by launch environment")).not.toBeNull(); }); @@ -106,7 +109,7 @@ describe("FeatureToggleCard", () => { />, ); - expect(screen.getByLabelText("Toggle Office mode")).toHaveProperty("disabled", true); + expect(screen.getByLabelText(TOGGLE_LABEL)).toHaveProperty("disabled", true); expect(screen.getByRole("button", { name: /use default/i })).toHaveProperty("disabled", true); }); @@ -125,6 +128,49 @@ describe("FeatureToggleCard", () => { }); }); +describe("FeatureToggleCard - unavailable state", () => { + it("disables the switch and shows the reason when the flag is unavailable on this host", () => { + const onChange = vi.fn(); + render( + + undefined} + /> + , + ); + + expect(screen.getByLabelText(TOGGLE_LABEL)).toHaveProperty("disabled", true); + expect(screen.getByText("Unavailable")).not.toBeNull(); + expect( + screen.getByText("This feature isn't supported on this operating system."), + ).not.toBeNull(); + }); + + it("falls back to a generic unavailable message for an unrecognized reason code", () => { + render( + + undefined} + onReset={() => undefined} + /> + , + ); + + expect( + screen.getByText("This feature can't be enabled on this install right now."), + ).not.toBeNull(); + }); +}); + function flagState(overrides: Partial = {}): RuntimeFlagState { return { key: "features.office", diff --git a/apps/web/components/settings/system/feature-toggle-card.tsx b/apps/web/components/settings/system/feature-toggle-card.tsx index c172339537..278fa6a2e9 100644 --- a/apps/web/components/settings/system/feature-toggle-card.tsx +++ b/apps/web/components/settings/system/feature-toggle-card.tsx @@ -5,7 +5,8 @@ import { Badge } from "@kandev/ui/badge"; import { Button } from "@kandev/ui/button"; import { CardContent, CardHeader, CardTitle } from "@kandev/ui/card"; import { Switch } from "@kandev/ui/switch"; -import { IconFlask, IconLock, IconRefresh } from "@tabler/icons-react"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@kandev/ui/tooltip"; +import { IconAlertTriangle, IconFlask, IconLock, IconRefresh } from "@tabler/icons-react"; import type { TFunction } from "i18next"; import type { RuntimeFlagState } from "@/lib/types/runtime-flags"; import { SettingsCard } from "@/components/settings/settings-card"; @@ -27,7 +28,8 @@ export function FeatureToggleCard({ onReset, }: FeatureToggleCardProps) { const { t } = useTranslation(); - const disabled = saving || flag.env_locked || !flag.mutable; + const unavailable = flag.availability?.available === false; + const disabled = saving || flag.env_locked || !flag.mutable || unavailable; return ( @@ -41,19 +43,18 @@ export function FeatureToggleCard({

{flag.description}

- {flag.risk_description && (

{flag.risk_description}

)} + {unavailable && }