Files
kandev/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go
T

465 lines
15 KiB
Go

package lifecycle
import (
"context"
"encoding/json"
"errors"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/kandev/kandev/internal/agent/executor"
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/agentctl/server/process"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
func TestManager_MarkCompleted_Success(t *testing.T) {
mgr := newTestManager(t)
execution := &AgentExecution{
ID: "test-execution-id",
TaskID: "test-task-id",
AgentProfileID: "test-agent",
ContainerID: "container-123",
Status: v1.AgentStatusRunning,
StartedAt: time.Now(),
}
mgr.executionStore.Add(execution)
// Mark as completed successfully (exit code 0)
err := mgr.MarkCompleted("test-execution-id", 0, "")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got, _ := mgr.GetExecution("test-execution-id")
if got.Status != v1.AgentStatusCompleted {
t.Errorf("expected status %v, got %v", v1.AgentStatusCompleted, got.Status)
}
if got.FinishedAt == nil {
t.Error("expected FinishedAt to be set")
}
if got.ExitCode == nil || *got.ExitCode != 0 {
t.Errorf("expected exit code 0, got %v", got.ExitCode)
}
}
func TestManager_MarkCompleted_Failure(t *testing.T) {
mgr := newTestManager(t)
execution := &AgentExecution{
ID: "test-execution-id",
TaskID: "test-task-id",
AgentProfileID: "test-agent",
ContainerID: "container-123",
Status: v1.AgentStatusRunning,
StartedAt: time.Now(),
}
mgr.executionStore.Add(execution)
// Mark as failed
err := mgr.MarkCompleted("test-execution-id", 1, "process failed")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got, _ := mgr.GetExecution("test-execution-id")
if got.Status != v1.AgentStatusFailed {
t.Errorf("expected status %v, got %v", v1.AgentStatusFailed, got.Status)
}
if got.ErrorMessage != "process failed" {
t.Errorf("expected error message 'process failed', got %q", got.ErrorMessage)
}
if got.ExitCode == nil || *got.ExitCode != 1 {
t.Errorf("expected exit code 1, got %v", got.ExitCode)
}
}
func TestManager_MarkCompleted_Idempotent(t *testing.T) {
mgr := newTestManager(t)
execution := &AgentExecution{
ID: "test-execution-id",
TaskID: "test-task-id",
AgentProfileID: "test-agent",
Status: v1.AgentStatusFailed,
StartedAt: time.Now(),
}
exitCode := 1
execution.ExitCode = &exitCode
execution.ErrorMessage = "first error"
mgr.executionStore.Add(execution)
// Second MarkCompleted should be a no-op (already terminal)
err := mgr.MarkCompleted("test-execution-id", 1, "second error")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
got, _ := mgr.GetExecution("test-execution-id")
if got.ErrorMessage != "first error" {
t.Errorf("expected original error message preserved, got %q", got.ErrorMessage)
}
}
func TestManager_MarkCompleted_NotFound(t *testing.T) {
mgr := newTestManager(t)
err := mgr.MarkCompleted("non-existent", 0, "")
if err == nil {
t.Error("expected error for non-existent execution")
}
}
func TestManager_RemoveExecution(t *testing.T) {
mgr := newTestManager(t)
execution := &AgentExecution{
ID: "test-execution-id",
TaskID: "test-task-id",
SessionID: "test-session-id",
ContainerID: "container-123",
}
mgr.executionStore.Add(execution)
// Remove execution
mgr.RemoveExecution("test-execution-id")
// Verify it's gone from all maps
if _, found := mgr.GetExecution("test-execution-id"); found {
t.Error("execution should be removed from executions map")
}
if _, found := mgr.GetExecutionBySessionID("test-session-id"); found {
t.Error("execution should be removed from bySession map")
}
// Remove non-existent should not panic
mgr.RemoveExecution("non-existent")
}
func TestManager_CleanupStaleExecution_StopsRuntimeInstance(t *testing.T) {
log := newTestRegistryLogger()
reg := newTestRegistry()
eventBus := &MockEventBus{}
credsMgr := &MockCredentialsManager{}
profileResolver := &MockProfileResolver{}
// Create executor registry with a mock backend that tracks StopInstance calls
execRegistry := NewExecutorRegistry(log)
mock := &mockStopTracker{name: "standalone"}
execRegistry.Register(mock)
mgr := NewManager(reg, eventBus, execRegistry, credsMgr, profileResolver, nil, ExecutorFallbackWarn, "", log)
execution := &AgentExecution{
ID: "exec-1",
SessionID: "session-1",
RuntimeName: "standalone",
}
mgr.executionStore.Add(execution)
err := mgr.CleanupStaleExecutionBySessionID(context.Background(), "session-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Verify StopInstance was called on the backend
if !mock.stopCalled {
t.Error("expected StopInstance to be called on the executor backend")
}
if mock.stoppedInstanceID != "exec-1" {
t.Errorf("expected StopInstance with instance ID exec-1, got %q", mock.stoppedInstanceID)
}
// Verify execution was removed from store
if _, found := mgr.GetExecutionBySessionID("session-1"); found {
t.Error("expected execution to be removed from store")
}
}
func TestManager_CleanupStaleExecution_NoopForMissingSession(t *testing.T) {
mgr := newTestManager(t)
err := mgr.CleanupStaleExecutionBySessionID(context.Background(), "non-existent")
if err != nil {
t.Fatalf("expected nil error for non-existent session, got: %v", err)
}
}
func TestManager_CleanupStaleExecution_SkipsStopWhenNoRuntime(t *testing.T) {
mgr := newTestManager(t) // no executor registry
execution := &AgentExecution{
ID: "exec-1",
SessionID: "session-1",
RuntimeName: "standalone",
}
mgr.executionStore.Add(execution)
err := mgr.CleanupStaleExecutionBySessionID(context.Background(), "session-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Should still remove from store even without executor registry
if _, found := mgr.GetExecutionBySessionID("session-1"); found {
t.Error("expected execution to be removed from store")
}
}
func TestManager_CleanupStaleExecution_ReleasesAgentctlLockBeforeRemove(t *testing.T) {
mgr := newTestManager(t)
modes := make(chan string, 1)
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/api/v1/workspace/poll-mode" {
modes <- r.Method
}
w.WriteHeader(http.StatusOK)
}))
t.Cleanup(srv.Close)
port := srv.Listener.Addr().(*net.TCPAddr).Port
client := agentctl.NewClient("127.0.0.1", port, newTestLogger())
t.Cleanup(func() { client.Close() })
execution := &AgentExecution{
ID: "exec-with-poll-client",
SessionID: "session-with-poll-client",
WorkspacePath: "/tmp/workspace-with-poll-client",
agentctl: client,
}
if err := mgr.executionStore.Add(execution); err != nil {
t.Fatalf("add execution: %v", err)
}
// Runtime interest makes RemoveExecution publish the transition to paused.
// The cleanup path must detach the client before that transition attempts to
// acquire the client read lock.
mgr.pollAggregator.HandleRuntimeInterest(execution.SessionID, true)
select {
case <-modes:
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for initial poll-mode push")
}
done := make(chan error, 1)
go func() {
done <- mgr.CleanupStaleExecutionBySessionID(context.Background(), execution.SessionID)
}()
select {
case err := <-done:
if err != nil {
t.Fatalf("cleanup stale execution: %v", err)
}
case <-time.After(500 * time.Millisecond):
t.Fatal("cleanup stale execution blocked while removing execution")
}
if _, found := mgr.GetExecutionBySessionID(execution.SessionID); found {
t.Fatal("expected stale execution to be removed")
}
}
func TestManager_CleanupStaleExecution_PreservesTrackingOnStopError(t *testing.T) {
log := newTestRegistryLogger()
reg := newTestRegistry()
execRegistry := NewExecutorRegistry(log)
mock := &mockStopTracker{name: "standalone", stopErr: errors.New("runtime unavailable")}
execRegistry.Register(mock)
mgr := NewManager(reg, &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log)
execution := &AgentExecution{ID: "exec-error", SessionID: "session-error", RuntimeName: "standalone"}
if err := mgr.executionStore.Add(execution); err != nil {
t.Fatalf("failed to seed execution: %v", err)
}
if err := mgr.CleanupStaleExecutionBySessionID(context.Background(), "session-error"); err == nil {
t.Fatal("cleanup must report a runtime stop failure")
}
if _, found := mgr.GetExecutionBySessionID("session-error"); !found {
t.Fatal("failed cleanup must retain execution tracking for a retry")
}
}
// mockStopTracker is a minimal ExecutorBackend that records StopInstance calls.
type mockStopTracker struct {
name executor.Name
stopCalled bool
stoppedInstanceID string
stoppedSessionID string
stopReason string
stopForce bool
stopErr error
}
func (m *mockStopTracker) Name() executor.Name { return m.name }
func (m *mockStopTracker) HealthCheck(ctx context.Context) error {
return nil
}
func (m *mockStopTracker) CreateInstance(ctx context.Context, req *ExecutorCreateRequest) (*ExecutorInstance, error) {
return nil, nil
}
func (m *mockStopTracker) StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error {
m.stopCalled = true
m.stoppedInstanceID = instance.InstanceID
m.stoppedSessionID = instance.SessionID
m.stopReason = instance.StopReason
m.stopForce = force
return m.stopErr
}
func TestManagerStopAllAgentsPassesBackendShutdownReason(t *testing.T) {
log := newTestRegistryLogger()
execRegistry := NewExecutorRegistry(log)
mock := &mockStopTracker{name: "standalone"}
execRegistry.Register(mock)
mgr := NewManager(nil, &MockEventBus{}, execRegistry, nil, nil, nil, ExecutorFallbackWarn, "", log)
if err := mgr.executionStore.Add(&AgentExecution{
ID: "exec-1",
SessionID: "session-1",
RuntimeName: "standalone",
}); err != nil {
t.Fatalf("add execution: %v", err)
}
if err := mgr.StopAllAgents(context.Background()); err != nil {
t.Fatalf("StopAllAgents: %v", err)
}
if mock.stopReason != StopReasonBackendShutdown {
t.Fatalf("StopInstance reason = %q, want %q", mock.stopReason, StopReasonBackendShutdown)
}
}
func (m *mockStopTracker) RecoverInstances(ctx context.Context, records []*models.ExecutorRunning) ([]*ExecutorInstance, error) {
return nil, nil
}
func (m *mockStopTracker) GetInteractiveRunner() *process.InteractiveRunner {
return nil
}
func (m *mockStopTracker) RequiresCloneURL() bool { return false }
func (m *mockStopTracker) ShouldApplyPreferredShell() bool { return false }
func (m *mockStopTracker) IsAlwaysResumable() bool { return false }
func TestManager_StartStop(t *testing.T) {
mgr := newTestManager(t)
ctx := context.Background()
// Test Start
err := mgr.Start(ctx)
if err != nil {
t.Fatalf("unexpected error starting manager: %v", err)
}
// Test Stop
err = mgr.Stop()
if err != nil {
t.Fatalf("unexpected error stopping manager: %v", err)
}
}
// TestManager_StartSeedsRecoveredExecution verifies the recovery path itself,
// not only the readiness helper. A recovered agentctl never passes through
// waitForAgentctlReady, so startup must seed its base-branch map before stream
// reconnection begins.
func TestManager_StartSeedsRecoveredExecution(t *testing.T) {
log := newTestRegistryLogger()
branchesCh := make(chan map[string]string, 1)
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
server := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/workspace/base-branches" {
http.NotFound(w, r)
return
}
var body struct {
BaseBranches map[string]string `json:"base_branches"`
}
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
branchesCh <- body.BaseBranches
w.WriteHeader(http.StatusOK)
})}
go func() { _ = server.Serve(listener) }()
t.Cleanup(func() {
_ = server.Close()
})
port := listener.Addr().(*net.TCPAddr).Port
client := agentctl.NewClient("127.0.0.1", port, log)
execRegistry := NewExecutorRegistry(log)
execRegistry.Register(&MockExecutor{
name: executor.NameStandalone,
recoverInstances: []*ExecutorInstance{{
InstanceID: "exec-recovered",
TaskID: "task-recovered",
AgentProfileID: "profile-1",
Client: client,
}},
})
mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log)
t.Cleanup(func() { _ = mgr.Stop() })
mgr.SetBaseBranchProvider(func(context.Context, string) (map[string]string, error) {
return map[string]string{"frontend": "develop"}, nil
})
if err := mgr.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
select {
case got := <-branchesCh:
if got["frontend"] != "develop" {
t.Fatalf("recovered base branch = %q, want develop; map=%v", got["frontend"], got)
}
case <-time.After(time.Second):
t.Fatal("timed out waiting for recovered execution base-branch seed")
}
}
// TestManagerStartRefusesRecoveredExecutionWithEmptyTaskID pins Review round
// 3, finding 3: a recovered instance whose TaskID is empty (the
// recovery-inventory record's declared source answered with nothing --
// executor_standalone.go's buildRecoveredInstances no longer falls back to
// the instance's own claim) is authoritatively missing a required value
// (AC-EXECUTORS-SURVIVAL-002.4): it must never be published as a tracked
// execution, and is instead stopped through the AC-EXECUTORS-SURVIVAL-002.6
// stop path.
func TestManagerStartRefusesRecoveredExecutionWithEmptyTaskID(t *testing.T) {
log := newTestRegistryLogger()
execRegistry := NewExecutorRegistry(log)
mockExec := &MockExecutor{
name: executor.NameStandalone,
recoverInstances: []*ExecutorInstance{{
InstanceID: "exec-no-task",
TaskID: "",
SessionID: "session-no-task",
RuntimeName: executor.NameStandalone,
}},
}
execRegistry.Register(mockExec)
mgr := NewManager(newTestRegistry(), &MockEventBus{}, execRegistry, &MockCredentialsManager{}, &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log)
t.Cleanup(func() { _ = mgr.Stop() })
if err := mgr.Start(context.Background()); err != nil {
t.Fatalf("Start: %v", err)
}
if _, ok := mgr.GetExecution("exec-no-task"); ok {
t.Fatal("an execution with an empty TaskID must not be tracked")
}
if len(mockExec.stopInstanceCalls) != 1 || mockExec.stopInstanceCalls[0].InstanceID != "exec-no-task" {
t.Fatalf("stopInstanceCalls = %+v, want the unreconstructable instance stopped", mockExec.stopInstanceCalls)
}
}