diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go new file mode 100644 index 0000000000..9f6fe53286 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_sprites_remote_test.go @@ -0,0 +1,738 @@ +package lifecycle + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + sprites "github.com/superfly/sprites-go" + + "github.com/kandev/kandev/internal/agent/agents" + "github.com/kandev/kandev/internal/agent/executor" + spritesutil "github.com/kandev/kandev/internal/sprites" +) + +// spritesAPIServer stands in for the Sprites control-plane REST API. Only the +// plain-HTTP endpoints the executor uses are served; the WebSocket exec channel +// is deliberately out of scope (see the package test notes). +type spritesAPIServer struct { + mu sync.Mutex + created []string + getStatus int + getBody string + createStatus int + server *httptest.Server +} + +func newSpritesAPIServer(t *testing.T) *spritesAPIServer { + t.Helper() + s := &spritesAPIServer{ + getStatus: http.StatusOK, + createStatus: http.StatusCreated, + getBody: `{"name":"kandev-abc","id":"sprite-1","status":"running"}`, + } + s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + switch { + case r.Method == http.MethodPost && r.URL.Path == "/v1/sprites": + var body struct { + Name string `json:"name"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + s.created = append(s.created, body.Name) + w.WriteHeader(s.createStatus) + _, _ = w.Write([]byte(`{"name":"` + body.Name + `"}`)) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/v1/sprites/"): + w.WriteHeader(s.getStatus) + if s.getStatus == http.StatusOK { + _, _ = w.Write([]byte(s.getBody)) + } + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(s.server.Close) + return s +} + +func (s *spritesAPIServer) client() *sprites.Client { + return sprites.NewClient(s.server.URL, "test-token") +} + +func TestSpritesExecutorStaticSurface(t *testing.T) { + exec := newTestSpritesExecutor(nil) + if exec.Name() != executor.NameSprites { + t.Fatalf("Name() = %q", exec.Name()) + } + if err := exec.HealthCheck(context.Background()); err != nil { + t.Fatalf("HealthCheck = %v, want nil", err) + } + if !exec.RequiresCloneURL() { + t.Fatal("sprites clone the repository inside the sandbox") + } + if exec.ShouldApplyPreferredShell() { + t.Fatal("the host shell must not be applied to a cloud sandbox") + } + if !exec.IsAlwaysResumable() { + t.Fatal("sprite sandboxes are always resumable") + } + if exec.GetInteractiveRunner() != nil { + t.Fatal("sprites have no host-side interactive runner") + } + instances, err := exec.RecoverInstances(context.Background()) + if err != nil || instances != nil { + t.Fatalf("RecoverInstances() = %v, %v", instances, err) + } + if err := exec.ResumeRemoteInstance(context.Background(), &ExecutorCreateRequest{}); err != nil { + t.Fatalf("ResumeRemoteInstance = %v, want nil", err) + } + if err := exec.ResumeRemoteInstance(context.Background(), nil); err == nil { + t.Fatal("a nil request must be rejected") + } +} + +func TestSpritesExecutorCreateSprite(t *testing.T) { + api := newSpritesAPIServer(t) + exec := newTestSpritesExecutor(nil) + + sprite, err := exec.createSprite(context.Background(), api.client(), "kandev-abc123456789") + if err != nil { + t.Fatalf("createSprite: %v", err) + } + if sprite == nil { + t.Fatal("createSprite returned no sprite") + } + api.mu.Lock() + created := append([]string(nil), api.created...) + api.mu.Unlock() + if len(created) != 1 || created[0] != "kandev-abc123456789" { + t.Fatalf("created sprites = %v", created) + } + + t.Run("API failure is wrapped", func(t *testing.T) { + api.mu.Lock() + api.createStatus = http.StatusInternalServerError + api.mu.Unlock() + _, err := exec.createSprite(context.Background(), api.client(), "kandev-fail") + if err == nil || !strings.Contains(err.Error(), "failed to create sprite") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestSpritesExecutorReconnectSprite(t *testing.T) { + api := newSpritesAPIServer(t) + exec := newTestSpritesExecutor(nil) + + sprite, err := exec.reconnectSprite(context.Background(), api.client(), "kandev-abc") + if err != nil { + t.Fatalf("reconnectSprite: %v", err) + } + if sprite.Status != "running" { + t.Fatalf("sprite status = %q", sprite.Status) + } + + t.Run("a missing sandbox is classified as not-found", func(t *testing.T) { + api.mu.Lock() + api.getStatus = http.StatusNotFound + api.mu.Unlock() + _, err := exec.reconnectSprite(context.Background(), api.client(), "kandev-gone") + if !errors.Is(err, spritesutil.ErrSpriteNotFound) { + t.Fatalf("error = %v, want ErrSpriteNotFound so CreateInstance can reprovision", err) + } + }) +} + +func TestSpritesExecutorStepCreateSpriteReportsProgress(t *testing.T) { + api := newSpritesAPIServer(t) + exec := newTestSpritesExecutor(nil) + + t.Run("fresh provisioning", func(t *testing.T) { + var steps []PrepareStep + report := newSpritesStepReporter(func(step PrepareStep, _, _ int) { + steps = append(steps, step) + }, newSpritesProgressPlan(false)) + + _, err := exec.stepCreateSprite(context.Background(), api.client(), "kandev-abc", false, report) + if err != nil { + t.Fatalf("stepCreateSprite: %v", err) + } + if len(steps) != 2 || steps[0].Name != "Creating cloud sandbox" { + t.Fatalf("steps = %+v", steps) + } + if steps[1].Status != PrepareStepCompleted { + t.Fatalf("final step = %+v", steps[1]) + } + }) + + t.Run("reconnect names the step differently", func(t *testing.T) { + var steps []PrepareStep + report := newSpritesStepReporter(func(step PrepareStep, _, _ int) { + steps = append(steps, step) + }, newSpritesProgressPlan(true)) + + _, err := exec.stepCreateSprite(context.Background(), api.client(), "kandev-abc", true, report) + if err != nil { + t.Fatalf("stepCreateSprite: %v", err) + } + if steps[0].Name != "Reconnecting cloud sandbox" { + t.Fatalf("step name = %q", steps[0].Name) + } + }) + + t.Run("failure is reported on the step", func(t *testing.T) { + api.mu.Lock() + api.getStatus = http.StatusNotFound + api.mu.Unlock() + var steps []PrepareStep + report := newSpritesStepReporter(func(step PrepareStep, _, _ int) { + steps = append(steps, step) + }, newSpritesProgressPlan(true)) + + _, err := exec.stepCreateSprite(context.Background(), api.client(), "kandev-gone", true, report) + if err == nil { + t.Fatal("expected the reconnect to fail") + } + last := steps[len(steps)-1] + if last.Status != PrepareStepFailed || last.Error == "" { + t.Fatalf("final step = %+v, want a failed step carrying the error", last) + } + }) +} + +func TestSpritesResolveSpriteName(t *testing.T) { + exec := newTestSpritesExecutor(nil) + + t.Run("fresh launches derive the name from the instance ID", func(t *testing.T) { + got := exec.resolveSpriteName(&ExecutorCreateRequest{InstanceID: "0123456789abcdef"}, false) + if got != "kandev-0123456789ab" { + t.Fatalf("sprite name = %q, want the 12-char instance prefix", got) + } + }) + + t.Run("reconnect prefers the recorded sprite name", func(t *testing.T) { + got := exec.resolveSpriteName(&ExecutorCreateRequest{ + InstanceID: "0123456789abcdef", + Metadata: map[string]interface{}{MetadataKeySpriteName: "kandev-existing"}, + }, true) + if got != "kandev-existing" { + t.Fatalf("sprite name = %q", got) + } + }) + + t.Run("reconnect without a recorded name falls back to the previous execution", func(t *testing.T) { + got := exec.resolveSpriteName(&ExecutorCreateRequest{ + InstanceID: "0123456789abcdef", + PreviousExecutionID: "fedcba9876543210", + }, true) + if got != "kandev-fedcba987654" { + t.Fatalf("sprite name = %q", got) + } + short := exec.resolveSpriteName(&ExecutorCreateRequest{PreviousExecutionID: "short"}, true) + if short != "kandev-short" { + t.Fatalf("sprite name = %q", short) + } + }) +} + +func TestSpritesShouldReconnect(t *testing.T) { + if spritesShouldReconnect(nil) { + t.Fatal("a nil request must not reconnect") + } + if spritesShouldReconnect(&ExecutorCreateRequest{}) { + t.Fatal("a fresh request must not reconnect") + } + if !spritesShouldReconnect(&ExecutorCreateRequest{PreviousExecutionID: "prev"}) { + t.Fatal("a previous execution implies reconnect") + } + if !spritesShouldReconnect(&ExecutorCreateRequest{ + Metadata: map[string]interface{}{MetadataKeySpriteName: "kandev-x"}, + }) { + t.Fatal("a recorded sprite name implies reconnect") + } +} + +func TestSpritesAgentInstanceID(t *testing.T) { + if got := spritesAgentInstanceID(nil); got != "" { + t.Fatalf("nil request = %q", got) + } + if got := spritesAgentInstanceID(&ExecutorCreateRequest{InstanceID: "cur", PreviousExecutionID: "prev"}); got != "cur" { + t.Fatalf("instance id = %q, want the current instance", got) + } + if got := spritesAgentInstanceID(&ExecutorCreateRequest{PreviousExecutionID: "prev"}); got != "prev" { + t.Fatalf("instance id = %q", got) + } +} + +func TestSpritesFallbackToFreshSandboxRewritesThePlan(t *testing.T) { + exec := newTestSpritesExecutor(nil) + plan := newSpritesProgressPlan(true) + var steps []PrepareStep + var totals []int + report := newSpritesStepReporter(func(step PrepareStep, _, total int) { + steps = append(steps, step) + totals = append(totals, total) + }, plan) + + newName := exec.fallbackToFreshSandbox(&ExecutorCreateRequest{ + InstanceID: "0123456789abcdef", + Metadata: map[string]interface{}{MetadataKeyWorktreeBranch: "feature/task-1"}, + }, plan, report, "kandev-old") + + if newName != "kandev-0123456789ab" { + t.Fatalf("new sprite name = %q", newName) + } + if plan.total() != 7 { + t.Fatalf("plan total = %d, want the full fresh-provision plan", plan.total()) + } + if !plan.has(spriteStepUploadAgentctl) || !plan.has(spriteStepApplyNetworkPolicy) { + t.Fatal("the fresh plan must include the setup and network-policy steps") + } + if len(steps) != 1 { + t.Fatalf("steps = %+v, want one explanatory notice", steps) + } + notice := steps[0] + if notice.Name != "Reconnecting cloud sandbox" || notice.Status != PrepareStepSkipped { + t.Fatalf("notice = %+v", notice) + } + if notice.Warning == "" || !strings.Contains(notice.WarningDetail, "kandev-old") || + !strings.Contains(notice.WarningDetail, newName) { + t.Fatalf("notice must explain the swap: %+v", notice) + } + if !strings.Contains(notice.WarningDetail, "feature/task-1") { + t.Fatalf("notice must name the branch: %q", notice.WarningDetail) + } + if totals[0] != 7 { + t.Fatalf("reported total = %d, want the replaced plan's total", totals[0]) + } +} + +func TestSpritesFallbackToFreshSandboxUsesBaseBranchAndPlaceholder(t *testing.T) { + exec := newTestSpritesExecutor(nil) + plan := newSpritesProgressPlan(true) + var steps []PrepareStep + report := newSpritesStepReporter(func(step PrepareStep, _, _ int) { steps = append(steps, step) }, plan) + + exec.fallbackToFreshSandbox(&ExecutorCreateRequest{ + InstanceID: "0123456789abcdef", + Metadata: map[string]interface{}{MetadataKeyBaseBranch: "main"}, + }, plan, report, "kandev-old") + if !strings.Contains(steps[0].WarningDetail, "main") { + t.Fatalf("detail = %q, want the base branch when no worktree branch is set", steps[0].WarningDetail) + } + + if got := branchOrPlaceholder(""); got != "(unknown)" { + t.Fatalf("branchOrPlaceholder(\"\") = %q", got) + } + if got := branchOrPlaceholder("dev"); got != "dev" { + t.Fatalf("branchOrPlaceholder(\"dev\") = %q", got) + } +} + +func TestSpritesBuildInstanceResult(t *testing.T) { + exec := newTestSpritesExecutor(nil) + createdAt := time.Date(2026, 8, 10, 12, 0, 0, 0, time.UTC) + sprite := &sprites.Sprite{Status: " running ", CreatedAt: createdAt} + + instance := exec.buildInstanceResult(&ExecutorCreateRequest{ + InstanceID: "instance-1", TaskID: "task-1", SessionID: "session-1", + }, "kandev-abc", sprite, 51234, 8765, true) + + if instance.InstanceID != "instance-1" || instance.TaskID != "task-1" || instance.SessionID != "session-1" { + t.Fatalf("identity = %+v", instance) + } + if instance.RuntimeName != executor.NameSprites { + t.Fatalf("RuntimeName = %q", instance.RuntimeName) + } + if instance.WorkspacePath != spritesWorkspacePath { + t.Fatalf("WorkspacePath = %q", instance.WorkspacePath) + } + if instance.AuthToken != "" { + t.Fatalf("AuthToken = %q; sprites are isolated VMs and use no agentctl token", instance.AuthToken) + } + if instance.Metadata[MetadataKeySpriteName] != "kandev-abc" { + t.Fatalf("sprite name metadata = %v", instance.Metadata[MetadataKeySpriteName]) + } + if instance.Metadata[MetadataKeySpriteState] != "running" { + t.Fatalf("sprite state metadata = %v, want it trimmed", instance.Metadata[MetadataKeySpriteState]) + } + if instance.Metadata[MetadataKeySpriteCreatedAt] != createdAt { + t.Fatalf("created-at metadata = %v", instance.Metadata[MetadataKeySpriteCreatedAt]) + } + if instance.Metadata[MetadataKeyLocalPort] != 51234 { + t.Fatalf("local port metadata = %v", instance.Metadata[MetadataKeyLocalPort]) + } + if instance.Metadata["reuse_existing_process"] != true || instance.Metadata[MetadataKeyIsRemote] != true { + t.Fatalf("metadata = %+v", instance.Metadata) + } + if instance.Client == nil { + t.Fatal("instance must carry an agentctl client on the forwarded local port") + } +} + +func TestSpriteCreateInstanceRequestMapsEveryField(t *testing.T) { + autoApprove := false + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + Protocol: "acp", + McpMode: "task", + McpProviders: []string{"github"}, + AutoApprovePermissions: true, + AutoApprovePermissionsOverride: &autoApprove, + AgentConfig: &fakeRuntimeAgent{ + MockAgent: agents.NewMockAgent(), id: "codex", + requiresKill: true, stripEnv: []string{"NODE_OPTIONS"}, + }, + Env: map[string]string{"OPENAI_API_KEY": "sk-1"}, + Metadata: map[string]interface{}{MetadataKeyBaseBranches: map[string]string{"": "main"}}, + } + + got := spriteCreateInstanceRequest(req) + + if got.ID != "instance-1" || got.WorkspacePath != spritesWorkspacePath { + t.Fatalf("request = %+v", got) + } + if got.AgentType != "codex" || !got.RequiresProcessKill { + t.Fatalf("agent fields = %+v", got) + } + if !equalStrings(got.StripEnv, []string{"NODE_OPTIONS"}) { + t.Fatalf("StripEnv = %v", got.StripEnv) + } + if got.AutoApprovePermissions == nil || *got.AutoApprovePermissions { + t.Fatalf("AutoApprovePermissions = %v, want the explicit profile override to win", got.AutoApprovePermissions) + } + if got.BaseBranches[""] != "main" { + t.Fatalf("BaseBranches = %+v", got.BaseBranches) + } + // Unlike SSH, the sandbox is isolated so the full env is forwarded. + if got.Env["OPENAI_API_KEY"] != "sk-1" { + t.Fatalf("Env = %+v", got.Env) + } + req.Env["MUTATED"] = "after" + if _, leaked := got.Env["MUTATED"]; leaked { + t.Fatalf("the request env must be cloned, not aliased: %+v", got.Env) + } +} + +func TestAgentAndRuntimeFieldsFromRequest(t *testing.T) { + if got := agentTypeFromReq(&ExecutorCreateRequest{}); got != "" { + t.Fatalf("agent type = %q", got) + } + if got := agentTypeFromReq(&ExecutorCreateRequest{AgentConfig: newFakeIdentityAgent("codex")}); got != "codex" { + t.Fatalf("agent type = %q", got) + } + + if requiresProcessKillFromReq(nil) || requiresProcessKillFromReq(&ExecutorCreateRequest{}) { + t.Fatal("missing agent config must not require a process kill") + } + if stripEnvFromReq(nil) != nil || stripEnvFromReq(&ExecutorCreateRequest{}) != nil { + t.Fatal("missing agent config must yield no strip list") + } + req := &ExecutorCreateRequest{AgentConfig: &fakeRuntimeAgent{ + MockAgent: agents.NewMockAgent(), id: "opencode", + requiresKill: true, stripEnv: []string{"NODE_OPTIONS"}, + }} + if !requiresProcessKillFromReq(req) { + t.Fatal("the agent runtime config must be honoured") + } + if !equalStrings(stripEnvFromReq(req), []string{"NODE_OPTIONS"}) { + t.Fatalf("strip env = %v", stripEnvFromReq(req)) + } +} + +// fakeReconnectControl records the reconnect delete/create sequence. +type fakeReconnectControl struct { + deleted []string + created int + deleteErr error + createErr error + port int +} + +func (c *fakeReconnectControl) Delete(_ context.Context, instanceID string) error { + c.deleted = append(c.deleted, instanceID) + return c.deleteErr +} + +func (c *fakeReconnectControl) Create(context.Context, *ExecutorCreateRequest) (int, error) { + c.created++ + return c.port, c.createErr +} + +func TestReplaceSpriteReconnectInstance(t *testing.T) { + t.Run("deletes before creating", func(t *testing.T) { + control := &fakeReconnectControl{port: 8123} + port, err := replaceSpriteReconnectInstance(context.Background(), control, + &ExecutorCreateRequest{InstanceID: "instance-1"}, "instance-1") + if err != nil { + t.Fatalf("replaceSpriteReconnectInstance: %v", err) + } + if port != 8123 { + t.Fatalf("port = %d", port) + } + if len(control.deleted) != 1 || control.deleted[0] != "instance-1" || control.created != 1 { + t.Fatalf("control = %+v", control) + } + }) + + t.Run("a failed delete skips the create", func(t *testing.T) { + wantErr := errors.New("delete failed") + control := &fakeReconnectControl{deleteErr: wantErr} + _, err := replaceSpriteReconnectInstance(context.Background(), control, + &ExecutorCreateRequest{}, "instance-1") + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v", err) + } + if control.created != 0 { + t.Fatal("a failed delete must not be followed by a create") + } + }) + + t.Run("a failed create bubbles up", func(t *testing.T) { + wantErr := errors.New("create failed") + control := &fakeReconnectControl{createErr: wantErr} + _, err := replaceSpriteReconnectInstance(context.Background(), control, + &ExecutorCreateRequest{}, "instance-1") + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v", err) + } + }) +} + +func TestSpritesExecutorStopInstanceEarlyReturns(t *testing.T) { + t.Run("nil instance", func(t *testing.T) { + if err := newTestSpritesExecutor(nil).StopInstance(context.Background(), nil, false); err != nil { + t.Fatalf("StopInstance(nil): %v", err) + } + }) + + t.Run("no sprite name recorded", func(t *testing.T) { + exec := newTestSpritesExecutor(nil) + if err := exec.StopInstance(context.Background(), &ExecutorInstance{InstanceID: "i"}, false); err != nil { + t.Fatalf("StopInstance: %v", err) + } + }) + + t.Run("a non-terminal stop preserves the sandbox and drops the proxy", func(t *testing.T) { + exec := newTestSpritesExecutor(nil) + cancelled := false + exec.proxies["i"] = &SpritesProxySession{ + spriteName: "kandev-abc", + cancel: func() { cancelled = true }, + } + exec.tokens["i"] = "token" + + err := exec.StopInstance(context.Background(), &ExecutorInstance{ + InstanceID: "i", + StopReason: "stopped via API", + Metadata: map[string]interface{}{MetadataKeySpriteName: "kandev-abc"}, + }, false) + if err != nil { + t.Fatalf("StopInstance: %v", err) + } + if !cancelled { + t.Fatal("the local proxy session must be torn down on every stop") + } + if _, tracked := exec.proxies["i"]; tracked { + t.Fatal("the proxy must be dropped from the map") + } + if exec.tokens["i"] != "token" { + t.Fatal("a preserved sandbox must keep its cached token for resume") + } + }) + + t.Run("a terminal stop without a token cannot destroy and says so", func(t *testing.T) { + exec := newTestSpritesExecutor(nil) + err := exec.StopInstance(context.Background(), &ExecutorInstance{ + InstanceID: "i", + StopReason: StopReasonTaskDeleted, + Metadata: map[string]interface{}{MetadataKeySpriteName: "kandev-abc"}, + }, false) + if err != nil { + t.Fatalf("StopInstance without a token must not error: %v", err) + } + }) +} + +func TestSpritesExecutorGetRemoteStatusWithoutAToken(t *testing.T) { + exec := newTestSpritesExecutor(nil) + + t.Run("nil instance", func(t *testing.T) { + if _, err := exec.GetRemoteStatus(context.Background(), nil); err == nil { + t.Fatal("expected an error for a nil instance") + } + }) + + t.Run("no sprite name yields unknown", func(t *testing.T) { + status, err := exec.GetRemoteStatus(context.Background(), &ExecutorInstance{InstanceID: "i"}) + if err != nil { + t.Fatalf("GetRemoteStatus: %v", err) + } + if status.State != "unknown" || status.RuntimeName != executor.NameSprites { + t.Fatalf("status = %+v", status) + } + if status.LastCheckedAt.IsZero() { + t.Fatal("LastCheckedAt must be stamped") + } + }) + + t.Run("no resolvable token yields unknown but names the sandbox", func(t *testing.T) { + status, err := exec.GetRemoteStatus(context.Background(), &ExecutorInstance{ + InstanceID: "i", + Metadata: map[string]interface{}{MetadataKeySpriteName: " kandev-abc "}, + }) + if err != nil { + t.Fatalf("GetRemoteStatus: %v", err) + } + if status.State != "unknown" || status.RemoteName != "kandev-abc" { + t.Fatalf("status = %+v", status) + } + }) +} + +func TestSpritesExecutorCleanupOnFailure(t *testing.T) { + t.Run("nil sprite is a no-op", func(t *testing.T) { + newTestSpritesExecutor(nil).cleanupOnFailure(context.Background(), nil, "i", true) + }) + + t.Run("reconnect flow preserves the sandbox but drops local state", func(t *testing.T) { + exec := newTestSpritesExecutor(nil) + cancelled := false + exec.proxies["i"] = &SpritesProxySession{cancel: func() { cancelled = true }} + exec.tokens["i"] = "token" + + exec.cleanupOnFailure(context.Background(), &sprites.Sprite{}, "i", false) + + if !cancelled { + t.Fatal("the proxy session must be cancelled") + } + if _, tracked := exec.proxies["i"]; tracked { + t.Fatal("the proxy must be dropped") + } + if _, tracked := exec.tokens["i"]; tracked { + t.Fatal("the cached token must be dropped") + } + }) +} + +func TestSpritesCloseProxySessionIsNilSafe(t *testing.T) { + exec := newTestSpritesExecutor(nil) + exec.closeProxySession(nil) + exec.closeProxySession(&SpritesProxySession{}) + cancelled := false + exec.closeProxySession(&SpritesProxySession{cancel: func() { cancelled = true }}) + if !cancelled { + t.Fatal("cancel must be invoked when present") + } +} + +func TestNonZeroTimePtr(t *testing.T) { + if nonZeroTimePtr(time.Time{}) != nil { + t.Fatal("a zero time must map to nil") + } + stamp := time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC) + got := nonZeroTimePtr(stamp) + if got == nil || !got.Equal(stamp) { + t.Fatalf("nonZeroTimePtr = %v", got) + } +} + +func TestGetFreePortReturnsABindablePort(t *testing.T) { + port, err := getFreePort() + if err != nil { + t.Fatalf("getFreePort: %v", err) + } + if port < 1 || port > 65535 { + t.Fatalf("port = %d, out of range", port) + } +} + +func TestSpritesApplyNetworkPolicy(t *testing.T) { + exec := newTestSpritesExecutor(nil) + + t.Run("no rules configured is a no-op", func(t *testing.T) { + api := newSpritesAPIServer(t) + if err := exec.applyNetworkPolicy(context.Background(), api.client(), "kandev-abc", + &ExecutorCreateRequest{Metadata: map[string]interface{}{}}); err != nil { + t.Fatalf("applyNetworkPolicy: %v", err) + } + }) + + t.Run("an empty rule list is a no-op", func(t *testing.T) { + api := newSpritesAPIServer(t) + if err := exec.applyNetworkPolicy(context.Background(), api.client(), "kandev-abc", + &ExecutorCreateRequest{Metadata: map[string]interface{}{ + "sprites_network_policy_rules": `[]`, + }}); err != nil { + t.Fatalf("applyNetworkPolicy: %v", err) + } + }) + + t.Run("malformed rules are rejected", func(t *testing.T) { + api := newSpritesAPIServer(t) + err := exec.applyNetworkPolicy(context.Background(), api.client(), "kandev-abc", + &ExecutorCreateRequest{Metadata: map[string]interface{}{ + "sprites_network_policy_rules": `{not json`, + }}) + if err == nil || !strings.Contains(err.Error(), "failed to parse network policy rules") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestSpritesResolvePrepareScript(t *testing.T) { + claude := &fakeIdentityAgent{ + MockAgent: agents.NewMockAgent(), id: "claude-code", installScript: "npm i -g claude", + } + exec := NewSpritesExecutor(nil, &mockAgentLister{agents: []agents.Agent{claude}}, + nil, spritesAgentctlPort, newTestLogger()) + + t.Run("an explicit setup script gets the branch-checkout postlude", func(t *testing.T) { + script, err := exec.resolvePrepareScript(&ExecutorCreateRequest{ + TaskID: "task-1", + Metadata: map[string]interface{}{ + MetadataKeySetupScript: "printf setup > {{workspace.path}}/marker", + MetadataKeyBaseBranch: "main", + MetadataKeyWorktreeBranch: "feature/task-1", + }, + }) + if err != nil { + t.Fatalf("resolvePrepareScript: %v", err) + } + if !strings.Contains(script, "printf setup > '/workspace'/marker") { + t.Fatalf("workspace placeholder unresolved:\n%s", script) + } + if !strings.Contains(script, "kandev-managed: ensure session feature branch is checked out") { + t.Fatalf("branch-checkout postlude missing:\n%s", script) + } + if !strings.Contains(script, "feature/task-1") { + t.Fatalf("session branch missing:\n%s", script) + } + }) + + t.Run("selected agents contribute their install scripts", func(t *testing.T) { + script, err := exec.resolvePrepareScript(&ExecutorCreateRequest{ + TaskID: "task-1", + AgentConfig: claude, + Metadata: map[string]interface{}{ + MetadataKeySetupScript: "{{kandev.agents.install}}", + MetadataKeyBaseBranch: "main", + }, + }) + if err != nil { + t.Fatalf("resolvePrepareScript: %v", err) + } + if !strings.Contains(script, "npm i -g claude") { + t.Fatalf("agent install script missing:\n%s", script) + } + }) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_control_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_control_test.go new file mode 100644 index 0000000000..d77a7018bd --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_control_test.go @@ -0,0 +1,412 @@ +package lifecycle + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/agents" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/githubauth" +) + +// Tests for the HTTP-over-SSH control plane (handshake + create-instance), the +// local port forwarder, and the environment actually forwarded to the remote +// agent instance. + +func TestRemoteControlHandshake(t *testing.T) { + t.Run("returns the issued bearer token", func(t *testing.T) { + var gotPath, gotContentType, gotBody string + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + gotPath, gotContentType, gotBody = r.URL.Path, r.Header.Get("Content-Type"), string(body) + _, _ = io.WriteString(w, `{"token":"handshake-token"}`) + })) + defer httpServer.Close() + + server := newFakeSSHServer(t, nil) + server.forwardTo(strings.TrimPrefix(httpServer.URL, "http://")) + + token, err := remoteControlHandshake(context.Background(), server.dial(t), 39429, "nonce-abc") + if err != nil { + t.Fatalf("remoteControlHandshake: %v", err) + } + if token != "handshake-token" { + t.Fatalf("token = %q", token) + } + if gotPath != "/auth/handshake" { + t.Fatalf("path = %q", gotPath) + } + if gotContentType != "application/json" { + t.Fatalf("content type = %q", gotContentType) + } + if gotBody != `{"nonce":"nonce-abc"}` { + t.Fatalf("body = %q", gotBody) + } + }) + + t.Run("non-200 is an error", func(t *testing.T) { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusForbidden) + })) + defer httpServer.Close() + server := newFakeSSHServer(t, nil) + server.forwardTo(strings.TrimPrefix(httpServer.URL, "http://")) + + _, err := remoteControlHandshake(context.Background(), server.dial(t), 39429, "nonce") + if err == nil || !strings.Contains(err.Error(), "403") { + t.Fatalf("error = %v, want the status code", err) + } + }) + + t.Run("empty token is an error", func(t *testing.T) { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"token":""}`) + })) + defer httpServer.Close() + server := newFakeSSHServer(t, nil) + server.forwardTo(strings.TrimPrefix(httpServer.URL, "http://")) + + _, err := remoteControlHandshake(context.Background(), server.dial(t), 39429, "nonce") + if err == nil || !strings.Contains(err.Error(), "no token") { + t.Fatalf("error = %v, want missing-token error", err) + } + }) + + t.Run("unreachable control port is an error", func(t *testing.T) { + server := newFakeSSHServer(t, nil) // no forward target configured + _, err := remoteControlHandshake(context.Background(), server.dial(t), 39429, "nonce") + if err == nil || !strings.Contains(err.Error(), "agentctl handshake") { + t.Fatalf("error = %v, want dial failure", err) + } + }) +} + +func TestCreateRemoteAgentInstancePostsTheBuiltRequest(t *testing.T) { + var gotPath, gotAuth string + var gotReq agentctl.CreateInstanceRequest + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotAuth = r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&gotReq) + _, _ = io.WriteString(w, `{"id":"instance-1","port":41234}`) + })) + defer httpServer.Close() + + server := newFakeSSHServer(t, nil) + server.forwardTo(strings.TrimPrefix(httpServer.URL, "http://")) + + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + Protocol: "acp", + McpMode: "task", + AgentConfig: &fakeIdentityAgent{MockAgent: agents.NewMockAgent(), id: "claude-code", name: "Claude Code"}, + Env: map[string]string{envKeyAnthropicAPIKey: "sk-test"}, + Metadata: map[string]interface{}{ + MetadataKeyBaseBranches: map[string]string{"": "main"}, + }, + } + + port, err := createRemoteAgentInstance(context.Background(), server.dial(t), + 39429, "/remote/task", req, "bearer-token", newTestLogger()) + if err != nil { + t.Fatalf("createRemoteAgentInstance: %v", err) + } + if port != 41234 { + t.Fatalf("port = %d, want 41234", port) + } + if gotPath != "/api/v1/instances" { + t.Fatalf("path = %q", gotPath) + } + if gotAuth != "Bearer bearer-token" { + t.Fatalf("authorization = %q", gotAuth) + } + if gotReq.ID != "instance-1" || gotReq.SessionID != "session-1" || gotReq.TaskID != "task-1" { + t.Fatalf("identity fields = %+v", gotReq) + } + if gotReq.WorkspacePath != "/remote/task" { + t.Fatalf("WorkspacePath = %q, want the remote task dir (never the host path)", gotReq.WorkspacePath) + } + if gotReq.AgentType != "claude-code" || gotReq.Protocol != "acp" || gotReq.McpMode != "task" { + t.Fatalf("agent routing fields = %+v", gotReq) + } + if gotReq.BaseBranches[""] != "main" { + t.Fatalf("BaseBranches = %+v", gotReq.BaseBranches) + } + if gotReq.Env[envKeyAnthropicAPIKey] != "sk-test" { + t.Fatalf("Env = %+v, want the allowlisted credential forwarded", gotReq.Env) + } +} + +func TestCreateRemoteAgentInstanceErrors(t *testing.T) { + tests := []struct { + name string + status int + body string + wantErr string + }{ + {name: "server error carries the body", status: http.StatusInternalServerError, body: "boom", wantErr: "returned 500: boom"}, + {name: "unparseable body", status: http.StatusOK, body: "not json", wantErr: "parse create-instance response"}, + {name: "port zero is rejected", status: http.StatusOK, body: `{"id":"x","port":0}`, wantErr: "returned port 0"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(tc.status) + _, _ = io.WriteString(w, tc.body) + })) + defer httpServer.Close() + server := newFakeSSHServer(t, nil) + server.forwardTo(strings.TrimPrefix(httpServer.URL, "http://")) + + _, err := createRemoteAgentInstance(context.Background(), server.dial(t), 39429, + "/remote/task", &ExecutorCreateRequest{InstanceID: "i"}, "", newTestLogger()) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %v, want %q", err, tc.wantErr) + } + }) + } +} + +func TestBuildSSHCreateInstanceRequestMapsEveryField(t *testing.T) { + autoApprove := true + req := &ExecutorCreateRequest{ + InstanceID: "instance-9", + TaskID: "task-9", + SessionID: "session-9", + Protocol: "acp", + McpMode: "office", + McpProviders: []string{"github"}, + AutoApprovePermissions: false, + AutoApprovePermissionsOverride: &autoApprove, + AgentConfig: &fakeRuntimeAgent{MockAgent: agents.NewMockAgent(), id: "opencode", requiresKill: true, stripEnv: []string{"NODE_OPTIONS"}}, + Env: map[string]string{envKeyOpenAIAPIKey: "sk-1", "UNRELATED": "x"}, + Metadata: map[string]interface{}{ + MetadataKeyBaseBranches: map[string]string{"repo-a": "develop"}, + }, + } + + got := buildSSHCreateInstanceRequest(req, "/remote/task") + + if got.ID != "instance-9" || got.SessionID != "session-9" || got.TaskID != "task-9" { + t.Fatalf("identity = %+v", got) + } + if got.WorkspacePath != "/remote/task" { + t.Fatalf("WorkspacePath = %q", got.WorkspacePath) + } + if got.AgentType != "opencode" { + t.Fatalf("AgentType = %q", got.AgentType) + } + if got.AutoApprovePermissions == nil || !*got.AutoApprovePermissions { + t.Fatalf("AutoApprovePermissions = %v, want the profile override to win", got.AutoApprovePermissions) + } + if !got.RequiresProcessKill { + t.Fatal("RequiresProcessKill must come from the agent runtime config") + } + if len(got.StripEnv) != 1 || got.StripEnv[0] != "NODE_OPTIONS" { + t.Fatalf("StripEnv = %v", got.StripEnv) + } + if got.BaseBranches["repo-a"] != "develop" { + t.Fatalf("BaseBranches = %+v", got.BaseBranches) + } + if got.McpMode != "office" || len(got.McpProviders) != 1 || got.McpProviders[0] != "github" { + t.Fatalf("mcp routing = %q / %v", got.McpMode, got.McpProviders) + } + if got.Env[envKeyOpenAIAPIKey] != "sk-1" { + t.Fatalf("Env = %+v, want the allowlisted key", got.Env) + } + if _, leaked := got.Env["UNRELATED"]; leaked { + t.Fatalf("Env leaked a non-allowlisted key: %+v", got.Env) + } +} + +func TestSSHRemoteAgentEnvForwardsOnlyScopedCredentials(t *testing.T) { + t.Run("nil request and nil env produce nil", func(t *testing.T) { + if got := sshRemoteAgentEnv(nil); got != nil { + t.Fatalf("sshRemoteAgentEnv(nil) = %+v", got) + } + if got := sshRemoteAgentEnv(&ExecutorCreateRequest{}); got != nil { + t.Fatalf("sshRemoteAgentEnv(no env) = %+v", got) + } + }) + + t.Run("only the credential allowlist is forwarded", func(t *testing.T) { + env := sshRemoteAgentEnv(&ExecutorCreateRequest{Env: map[string]string{ + envKeyClaudeCodeOAuthToken: "oauth", + envKeyGitHubToken: "ghp", + envKeyGitLabHost: "gitlab.example", + envKeyAnthropicAPIKey: "", // empty must never clobber a remote value + "HOME": "/root", + "PATH": "/usr/bin", + }}) + want := map[string]string{ + envKeyClaudeCodeOAuthToken: "oauth", + envKeyGitHubToken: "ghp", + envKeyGitLabHost: "gitlab.example", + } + if !equalStringMaps(env, want) { + t.Fatalf("env = %+v, want %+v", env, want) + } + }) + + t.Run("managed broker env and indexed git config ride along", func(t *testing.T) { + env := sshRemoteAgentEnv(&ExecutorCreateRequest{Env: map[string]string{ + githubauth.CredentialBrokerURLEnv: "http://127.0.0.1:9/broker", + githubauth.CredentialLeaseEnv: "lease-1", + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "credential.helper", + "GIT_CONFIG_VALUE_0": "kandev", + }}) + if env[githubauth.CredentialBrokerURLEnv] != "http://127.0.0.1:9/broker" || + env[githubauth.CredentialLeaseEnv] != "lease-1" { + t.Fatalf("broker env not forwarded: %+v", env) + } + if env["GIT_CONFIG_COUNT"] != "1" || env["GIT_CONFIG_KEY_0"] != "credential.helper" || + env["GIT_CONFIG_VALUE_0"] != "kandev" { + t.Fatalf("indexed git config not forwarded: %+v", env) + } + }) + + t.Run("approved secret keys are additive and never override", func(t *testing.T) { + env := sshRemoteAgentEnv(&ExecutorCreateRequest{ + Env: map[string]string{ + envKeyGitHubToken: "managed", + "DEPLOY_KEY": "approved-value", + "EMPTY_KEY": "", + "bad key": "ignored", + }, + ApprovedSecretEnvKeys: []string{"DEPLOY_KEY", "EMPTY_KEY", "bad key", envKeyGitHubToken}, + }) + if env["DEPLOY_KEY"] != "approved-value" { + t.Fatalf("approved key not forwarded: %+v", env) + } + if _, present := env["EMPTY_KEY"]; present { + t.Fatalf("empty approved value must be skipped: %+v", env) + } + if _, present := env["bad key"]; present { + t.Fatalf("non-POSIX identifier must be skipped: %+v", env) + } + if env[envKeyGitHubToken] != "managed" { + t.Fatalf("approval must not replace the executor-selected credential: %+v", env) + } + }) +} + +func TestSSHAgentTypeFromReq(t *testing.T) { + if got := sshAgentTypeFromReq(nil); got != "" { + t.Fatalf("nil request agent type = %q", got) + } + if got := sshAgentTypeFromReq(&ExecutorCreateRequest{}); got != "" { + t.Fatalf("missing agent config type = %q", got) + } + got := sshAgentTypeFromReq(&ExecutorCreateRequest{AgentConfig: newFakeIdentityAgent("codex")}) + if got != "codex" { + t.Fatalf("agent type = %q, want codex", got) + } +} + +func TestStartPortForwardTunnelsToTheRemotePort(t *testing.T) { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + _, _ = io.WriteString(w, "ok") + return + } + w.WriteHeader(http.StatusTeapot) + })) + defer httpServer.Close() + + server := newFakeSSHServer(t, nil) + var forwardedPort uint32 + server.tcpTarget = func(port uint32) string { + forwardedPort = port + return strings.TrimPrefix(httpServer.URL, "http://") + } + + fwd, err := StartPortForward(server.dial(t), 41234, newTestLogger()) + if err != nil { + t.Fatalf("StartPortForward: %v", err) + } + defer func() { _ = fwd.Close() }() + + if fwd.LocalPort() == 0 { + t.Fatal("LocalPort must be a real bound port") + } + if err := waitAgentctlHealthy(context.Background(), fwd.LocalPort(), 5*time.Second); err != nil { + t.Fatalf("waitAgentctlHealthy through the forward: %v", err) + } + if forwardedPort != 41234 { + t.Fatalf("forwarded remote port = %d, want 41234", forwardedPort) + } + + if err := fwd.Close(); err != nil { + t.Fatalf("first Close: %v", err) + } + if err := fwd.Close(); err != nil { + t.Fatalf("Close must be idempotent, got %v", err) + } +} + +func TestWaitAgentctlHealthyFailsOnNonSuccessStatus(t *testing.T) { + httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer httpServer.Close() + + server := newFakeSSHServer(t, nil) + server.forwardTo(strings.TrimPrefix(httpServer.URL, "http://")) + fwd, err := StartPortForward(server.dial(t), 41234, newTestLogger()) + if err != nil { + t.Fatalf("StartPortForward: %v", err) + } + defer func() { _ = fwd.Close() }() + + // waitAgentctlHealthy sleeps 200ms between attempts, so the deadline has to + // span at least two windows for the retry loop itself to be exercised. + err = waitAgentctlHealthy(context.Background(), fwd.LocalPort(), 600*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "not reachable") { + t.Fatalf("error = %v, want unreachable after the deadline", err) + } +} + +func TestWaitAgentctlHealthyHonoursContextCancellation(t *testing.T) { + // Port 1 on loopback refuses connections, so the probe loop always retries + // and only the context can end it. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := waitAgentctlHealthy(ctx, 1, 5*time.Second) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } +} + +func equalStrings(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +func equalStringMaps(got, want map[string]string) bool { + if len(got) != len(want) { + return false + } + for key, value := range want { + if got[key] != value { + return false + } + } + return true +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_credentials_remote_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_credentials_remote_test.go new file mode 100644 index 0000000000..a0caf6cee6 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_credentials_remote_test.go @@ -0,0 +1,352 @@ +package lifecycle + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/kandev/kandev/internal/agent/agents" + "github.com/kandev/kandev/internal/agent/remoteauth" +) + +// remoteAuthAgent declares a RemoteAuth block so BuildCatalog produces real +// method entries for it. +type remoteAuthAgent struct { + *agents.MockAgent + id string + remoteAuth *agents.RemoteAuth +} + +func (a *remoteAuthAgent) ID() string { return a.id } +func (a *remoteAuthAgent) DisplayName() string { return a.id } +func (a *remoteAuthAgent) RemoteAuth() *agents.RemoteAuth { return a.remoteAuth } +func (a *remoteAuthAgent) InstallScript() string { return "" } +func (a *remoteAuthAgent) Name() string { return a.id } +func (a *remoteAuthAgent) newCatalog() remoteauth.Catalog { + return remoteauth.BuildCatalog([]agents.Agent{a}) +} +func (a *remoteAuthAgent) lister() RemoteAuthAgentLister { + return &mockAgentLister{agents: []agents.Agent{a}} +} +func (a *remoteAuthAgent) fileMethodID(index string) string { + return "agent:" + a.id + ":files:" + index +} + +var _ agents.Agent = (*remoteAuthAgent)(nil) + +// newCredentialAgent declares one file method (a credentials file under the +// home dir) and one env method carrying a setup script. +func newCredentialAgent(sourceFile string) *remoteAuthAgent { + return &remoteAuthAgent{ + MockAgent: agents.NewMockAgent(), + id: "credential-agent", + remoteAuth: &agents.RemoteAuth{Methods: []agents.RemoteAuthMethod{ + { + Type: "files", + SourceFiles: map[string][]string{runtime.GOOS: {sourceFile}}, + TargetRelDir: ".credential-agent", + Label: "Copy credentials", + }, + { + Type: "env", + EnvVar: "CREDENTIAL_AGENT_TOKEN", + SetupScript: "printf %s \"$CREDENTIAL_AGENT_TOKEN\" > ~/.credential-agent/token", + }, + }}, + } +} + +func TestSSHExecutorBuildRemoteAuthCatalog(t *testing.T) { + t.Run("no agent list still yields the built-in gh spec", func(t *testing.T) { + exec := &SSHExecutor{logger: newTestLogger()} + catalog := exec.buildRemoteAuthCatalog() + if _, ok := catalog.FindMethod("gh_cli_env"); !ok { + t.Fatalf("catalog must always carry gh_cli_env, got %+v", catalog.Specs) + } + }) + + t.Run("enabled agents contribute their methods", func(t *testing.T) { + agent := newCredentialAgent(".credential-agent/creds.json") + exec := &SSHExecutor{logger: newTestLogger(), agentList: agent.lister()} + catalog := exec.buildRemoteAuthCatalog() + if _, ok := catalog.FindMethod(agent.fileMethodID("0")); !ok { + t.Fatalf("catalog missing the agent file method, got %+v", catalog.Specs) + } + if _, ok := catalog.FindMethod("agent:credential-agent:env:CREDENTIAL_AGENT_TOKEN"); !ok { + t.Fatal("catalog missing the agent env method") + } + }) +} + +func TestSelectFileMethodsKeepsOnlySelectedFileMethods(t *testing.T) { + agent := newCredentialAgent(".credential-agent/creds.json") + catalog := agent.newCatalog() + + methods := selectFileMethods(catalog, []string{ + agent.fileMethodID("0"), // file method — kept + "agent:credential-agent:env:CREDENTIAL_AGENT_TOKEN", // env method — dropped + "totally-unknown", // unknown — warned and dropped + }, newTestLogger()) + + if len(methods) != 1 { + t.Fatalf("selected methods = %+v, want only the file method", methods) + } + if methods[0].MethodID != agent.fileMethodID("0") || methods[0].Type != authMethodTypeFiles { + t.Fatalf("selected method = %+v", methods[0]) + } + if got := selectFileMethods(catalog, nil, newTestLogger()); len(got) != 0 { + t.Fatalf("empty selection = %+v, want none", got) + } +} + +func TestSSHExecutorResolveRemoteAuthHomeDir(t *testing.T) { + t.Run("metadata override wins without probing", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := &SSHExecutor{logger: newTestLogger()} + home, err := exec.resolveRemoteAuthHomeDir(context.Background(), server.dial(t), + &ExecutorCreateRequest{Metadata: map[string]interface{}{ + MetadataKeyRemoteAuthHome: " /srv/home ", + }}, SSHRemotePlatform{}) + if err != nil { + t.Fatalf("resolveRemoteAuthHomeDir: %v", err) + } + if home != "/srv/home" { + t.Fatalf("home = %q, want the trimmed override", home) + } + if len(server.commands()) != 0 { + t.Fatalf("the override must skip the probe, commands: %v", server.commands()) + } + }) + + t.Run("probes the remote through the platform shell", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("/Users/kandev\n") }) + exec := &SSHExecutor{logger: newTestLogger()} + home, err := exec.resolveRemoteAuthHomeDir(context.Background(), server.dial(t), + &ExecutorCreateRequest{}, SSHRemotePlatform{GOOS: "darwin", GOARCH: "arm64"}) + if err != nil { + t.Fatalf("resolveRemoteAuthHomeDir: %v", err) + } + if home != "/Users/kandev" { + t.Fatalf("home = %q", home) + } + // Darwin must probe under zsh, not bash. + if got := server.commands(); len(got) != 1 || !strings.HasPrefix(got[0], "zsh -lc ") { + t.Fatalf("probe command = %v, want a zsh login shell on darwin", got) + } + }) + + t.Run("empty and failing probes are errors", func(t *testing.T) { + blank := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut(" \n") }) + exec := &SSHExecutor{logger: newTestLogger()} + _, err := exec.resolveRemoteAuthHomeDir(context.Background(), blank.dial(t), + &ExecutorCreateRequest{}, SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), "empty string") { + t.Fatalf("error = %v", err) + } + + broken := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("closed") }) + _, err = exec.resolveRemoteAuthHomeDir(context.Background(), broken.dial(t), + &ExecutorCreateRequest{}, SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), "resolve remote $HOME for credentials") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestSSHExecutorRunAuthSetupScripts(t *testing.T) { + agent := newCredentialAgent(".credential-agent/creds.json") + + t.Run("runs only for env methods whose env var is present", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := &SSHExecutor{logger: newTestLogger(), agentList: agent.lister()} + + req := &ExecutorCreateRequest{Env: map[string]string{ + "CREDENTIAL_AGENT_TOKEN": "trigger-value", + envKeyGitHubToken: "ghp-s3cr3t", + }} + exec.runAuthSetupScripts(context.Background(), server.dial(t), req, agent.newCatalog(), + SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"}) + + calls := server.execCalls() + if len(calls) != 1 { + t.Fatalf("setup script runs = %d, want 1", len(calls)) + } + script := unwrapLoginShell(t, "bash", calls[0].Command) + if !strings.HasPrefix(script, "set -a; . /dev/stdin; set +a\n") { + t.Fatalf("setup script must source stdin under set -a:\n%s", script) + } + if !strings.Contains(script, `printf %s "$CREDENTIAL_AGENT_TOKEN"`) { + t.Fatalf("setup script body missing:\n%s", script) + } + // Secrets ride on stdin, never on the remote argv, so they stay out of + // `ps aux` / /proc/PID/cmdline while the script runs. + if strings.Contains(calls[0].Command, "ghp-s3cr3t") { + t.Fatalf("the secret must never reach argv:\n%s", calls[0].Command) + } + if calls[0].Stdin != "GITHUB_TOKEN='ghp-s3cr3t'\n" { + t.Fatalf("stdin = %q, want the allowlisted credential as an env assignment", calls[0].Stdin) + } + }) + + t.Run("absent env var skips the script", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := &SSHExecutor{logger: newTestLogger(), agentList: agent.lister()} + exec.runAuthSetupScripts(context.Background(), server.dial(t), + &ExecutorCreateRequest{Env: map[string]string{}}, agent.newCatalog(), SSHRemotePlatform{}) + if len(server.commands()) != 0 { + t.Fatalf("no setup script should run, commands: %v", server.commands()) + } + }) + + t.Run("a failing setup script is best-effort", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("boom") }) + exec := &SSHExecutor{logger: newTestLogger(), agentList: agent.lister()} + exec.runAuthSetupScripts(context.Background(), server.dial(t), + &ExecutorCreateRequest{Env: map[string]string{"CREDENTIAL_AGENT_TOKEN": "tok"}}, + agent.newCatalog(), SSHRemotePlatform{}) + if len(server.commands()) != 1 { + t.Fatalf("commands = %v, want the failure swallowed after one attempt", server.commands()) + } + }) + +} + +func TestSSHExecutorUploadCredentials(t *testing.T) { + t.Run("no selection uploads nothing", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.uploadCredentials(context.Background(), server.dial(t), + &ExecutorCreateRequest{Metadata: map[string]interface{}{}}, SSHRemotePlatform{}) + if err != nil { + t.Fatalf("uploadCredentials: %v", err) + } + }) + + t.Run("malformed remote_credentials is an error", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.uploadCredentials(context.Background(), server.dial(t), + &ExecutorCreateRequest{Metadata: map[string]interface{}{"remote_credentials": "{"}}, + SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), "failed to parse remote_credentials") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("selected file methods land under the remote home", func(t *testing.T) { + localHome := t.TempDir() + t.Setenv("HOME", localHome) + if err := os.MkdirAll(filepath.Join(localHome, ".credential-agent"), 0o755); err != nil { + t.Fatalf("seed local credential dir: %v", err) + } + if err := os.WriteFile(filepath.Join(localHome, ".credential-agent", "creds.json"), + []byte(`{"token":"local"}`), 0o600); err != nil { + t.Fatalf("seed local credential file: %v", err) + } + + remoteHome := t.TempDir() + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + if strings.Contains(command, remoteHomeCommand) { + return sshOut(remoteHome) + } + return sshOK + }) + server.enableSFTP() + + agent := newCredentialAgent(".credential-agent/creds.json") + exec := &SSHExecutor{logger: newTestLogger(), agentList: agent.lister()} + req := &ExecutorCreateRequest{Metadata: map[string]interface{}{ + "remote_credentials": `["` + agent.fileMethodID("0") + `","gh_cli_token"]`, + }} + + if err := exec.uploadCredentials(context.Background(), server.dial(t), req, SSHRemotePlatform{}); err != nil { + t.Fatalf("uploadCredentials: %v", err) + } + + uploaded := filepath.Join(remoteHome, ".credential-agent", "creds.json") + data, err := os.ReadFile(uploaded) + if err != nil { + t.Fatalf("read uploaded credential: %v", err) + } + if string(data) != `{"token":"local"}` { + t.Fatalf("uploaded credential = %q", data) + } + }) + + t.Run("unresolvable remote home is the one hard failure", func(t *testing.T) { + localHome := t.TempDir() + t.Setenv("HOME", localHome) + if err := os.MkdirAll(filepath.Join(localHome, ".credential-agent"), 0o755); err != nil { + t.Fatalf("seed local credential dir: %v", err) + } + if err := os.WriteFile(filepath.Join(localHome, ".credential-agent", "creds.json"), + []byte("x"), 0o600); err != nil { + t.Fatalf("seed local credential file: %v", err) + } + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("no shell") }) + agent := newCredentialAgent(".credential-agent/creds.json") + exec := &SSHExecutor{logger: newTestLogger(), agentList: agent.lister()} + err := exec.uploadCredentials(context.Background(), server.dial(t), + &ExecutorCreateRequest{Metadata: map[string]interface{}{ + "remote_credentials": `["` + agent.fileMethodID("0") + `"]`, + }}, SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), "resolve remote $HOME for credentials") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestSSHFileUploaderWriteFileCreatesParents(t *testing.T) { + server := newFakeSSHServer(t, nil) + server.enableSFTP() + uploader := &sshFileUploader{client: server.dial(t)} + target := filepath.Join(t.TempDir(), "a", "b", "c", "creds.json") + + if err := uploader.WriteFile(context.Background(), target, []byte("secret"), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + data, err := os.ReadFile(target) + if err != nil { + t.Fatalf("read written file: %v", err) + } + if string(data) != "secret" { + t.Fatalf("content = %q", data) + } + info, err := os.Stat(target) + if err != nil { + t.Fatalf("stat: %v", err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("mode = %v, want 0600", info.Mode().Perm()) + } + + // Writing again over the existing tree must still succeed. + if err := uploader.WriteFile(context.Background(), target, []byte("rotated"), 0o644); err != nil { + t.Fatalf("second WriteFile: %v", err) + } + data, _ = os.ReadFile(target) + if string(data) != "rotated" { + t.Fatalf("rotated content = %q", data) + } +} + +func TestSSHFileUploaderWriteFileFailsOnUncreatableParent(t *testing.T) { + server := newFakeSSHServer(t, nil) + server.enableSFTP() + uploader := &sshFileUploader{client: server.dial(t)} + + // A regular file cannot become a directory, so mkdir -p must fail. + root := t.TempDir() + blocker := filepath.Join(root, "blocker") + if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil { + t.Fatalf("seed blocker: %v", err) + } + err := uploader.WriteFile(context.Background(), filepath.Join(blocker, "nested", "creds"), + []byte("x"), 0o600) + if err == nil { + t.Fatal("expected the write to fail when the parent cannot be created") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_dial_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_dial_test.go new file mode 100644 index 0000000000..8189a62184 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_dial_test.go @@ -0,0 +1,655 @@ +package lifecycle + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/pem" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "golang.org/x/crypto/ssh" +) + +// newTestHomeDir points $HOME at a fresh temp dir so ~/.ssh lookups in this +// package are hermetic instead of reading the developer's real config. +func newTestHomeDir(t *testing.T) string { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + if err := os.MkdirAll(filepath.Join(home, ".ssh"), 0o700); err != nil { + t.Fatalf("create fake ~/.ssh: %v", err) + } + return home +} + +func writeSSHConfig(t *testing.T, home, contents string) { + t.Helper() + if err := os.WriteFile(filepath.Join(home, ".ssh", "config"), []byte(contents), 0o600); err != nil { + t.Fatalf("write ssh config: %v", err) + } +} + +func TestInheritFromSSHConfig(t *testing.T) { + t.Run("empty fields inherit from the matching Host block", func(t *testing.T) { + home := newTestHomeDir(t) + writeSSHConfig(t, home, `Host prod + HostName prod.internal + Port 2222 + User deploy + IdentityFile ~/.ssh/prod_key + ProxyJump bastion.example +`) + target, err := ResolveSSHTarget(SSHConnConfig{HostAlias: "prod", PinnedFingerprint: "SHA256:x"}) + if err != nil { + t.Fatalf("ResolveSSHTarget: %v", err) + } + if target.Host != "prod.internal" || target.Port != 2222 || target.User != "deploy" { + t.Fatalf("target = %+v", target) + } + if target.IdentitySource != SSHIdentitySourceFile { + t.Fatalf("IdentitySource = %q, want file", target.IdentitySource) + } + if target.IdentityFile != filepath.Join(home, ".ssh", "prod_key") { + t.Fatalf("IdentityFile = %q, want the expanded ~ path", target.IdentityFile) + } + if target.ProxyJump != "bastion.example" { + t.Fatalf("ProxyJump = %q", target.ProxyJump) + } + }) + + t.Run("explicit form values always win", func(t *testing.T) { + home := newTestHomeDir(t) + writeSSHConfig(t, home, `Host prod + HostName prod.internal + Port 2222 + User deploy + ProxyJump config-bastion +`) + target, err := ResolveSSHTarget(SSHConnConfig{ + HostAlias: "prod", + Host: "override.example", + Port: 2200, + User: "override-user", + IdentitySource: SSHIdentitySourceAgent, + ProxyJump: "form-bastion", + PinnedFingerprint: "SHA256:x", + }) + if err != nil { + t.Fatalf("ResolveSSHTarget: %v", err) + } + if target.Host != "override.example" || target.Port != 2200 || target.User != "override-user" { + t.Fatalf("target = %+v, want the explicit values", target) + } + if target.ProxyJump != "form-bastion" || target.IdentitySource != SSHIdentitySourceAgent { + t.Fatalf("target = %+v", target) + } + }) + + t.Run("an unreadable config leaves the alias as the hostname", func(t *testing.T) { + home := newTestHomeDir(t) + writeSSHConfig(t, home, "this is not { a valid ssh config\n") + target, err := ResolveSSHTarget(SSHConnConfig{ + HostAlias: "prod", + User: "deploy", + IdentitySource: SSHIdentitySourceAgent, + PinnedFingerprint: "SHA256:x", + }) + if err != nil { + t.Fatalf("ResolveSSHTarget: %v", err) + } + if target.Host != "prod" { + t.Fatalf("Host = %q, want the alias fallback", target.Host) + } + }) + + t.Run("a non-numeric config port is ignored and the default applies", func(t *testing.T) { + home := newTestHomeDir(t) + writeSSHConfig(t, home, "Host prod\n HostName prod.internal\n Port not-a-port\n") + target, err := ResolveSSHTarget(SSHConnConfig{ + HostAlias: "prod", + User: "deploy", + IdentitySource: SSHIdentitySourceAgent, + PinnedFingerprint: "SHA256:x", + }) + if err != nil { + t.Fatalf("ResolveSSHTarget: %v", err) + } + if target.Port != 22 { + t.Fatalf("Port = %d, want the default 22", target.Port) + } + }) +} + +func TestApplyTargetDefaultsIdentityFallbacks(t *testing.T) { + t.Run("ssh-agent is preferred when SSH_AUTH_SOCK is set", func(t *testing.T) { + newTestHomeDir(t) + t.Setenv("SSH_AUTH_SOCK", "/tmp/agent.sock") + target, err := ResolveSSHTarget(SSHConnConfig{ + Host: "build.example", User: "deploy", PinnedFingerprint: "SHA256:x", + }) + if err != nil { + t.Fatalf("ResolveSSHTarget: %v", err) + } + if target.IdentitySource != SSHIdentitySourceAgent { + t.Fatalf("IdentitySource = %q, want agent", target.IdentitySource) + } + }) + + t.Run("no agent falls back to ~/.ssh/id_ed25519", func(t *testing.T) { + home := newTestHomeDir(t) + t.Setenv("SSH_AUTH_SOCK", "") + target, err := ResolveSSHTarget(SSHConnConfig{ + Host: "build.example", User: "deploy", PinnedFingerprint: "SHA256:x", + }) + if err != nil { + t.Fatalf("ResolveSSHTarget: %v", err) + } + if target.IdentitySource != SSHIdentitySourceFile { + t.Fatalf("IdentitySource = %q, want file", target.IdentitySource) + } + if target.IdentityFile != filepath.Join(home, ".ssh", "id_ed25519") { + t.Fatalf("IdentityFile = %q", target.IdentityFile) + } + }) + + t.Run("user is required when $USER is unset", func(t *testing.T) { + newTestHomeDir(t) + t.Setenv("USER", "") + _, err := ResolveSSHTarget(SSHConnConfig{Host: "build.example", IdentitySource: SSHIdentitySourceAgent}) + if err == nil || !strings.Contains(err.Error(), "user is required") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestBuildAuthMethods(t *testing.T) { + t.Run("file identity loads the key", func(t *testing.T) { + path := writeTestIdentityFile(t) + methods, cleanup, err := buildAuthMethods(&SSHTarget{ + IdentitySource: SSHIdentitySourceFile, + IdentityFile: path, + }) + if err != nil { + t.Fatalf("buildAuthMethods: %v", err) + } + defer cleanup() + if len(methods) != 1 { + t.Fatalf("methods = %d, want 1", len(methods)) + } + }) + + t.Run("file identity requires a path", func(t *testing.T) { + _, cleanup, err := buildAuthMethods(&SSHTarget{IdentitySource: SSHIdentitySourceFile}) + cleanup() + if err == nil || !strings.Contains(err.Error(), "identity file path is required") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("missing identity file is reported with its path", func(t *testing.T) { + missing := filepath.Join(t.TempDir(), "absent_key") + _, cleanup, err := buildAuthMethods(&SSHTarget{ + IdentitySource: SSHIdentitySourceFile, + IdentityFile: missing, + }) + cleanup() + if err == nil || !strings.Contains(err.Error(), "failed to read identity file") { + t.Fatalf("error = %v", err) + } + if !strings.Contains(err.Error(), missing) { + t.Fatalf("error = %v, want the path", err) + } + }) + + t.Run("an unparseable identity file is reported", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "garbage_key") + if err := os.WriteFile(path, []byte("not a key"), 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + _, cleanup, err := buildAuthMethods(&SSHTarget{ + IdentitySource: SSHIdentitySourceFile, + IdentityFile: path, + }) + cleanup() + if err == nil || !strings.Contains(err.Error(), "failed to parse identity file") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("a passphrase-protected key points the user at ssh-agent", func(t *testing.T) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + block, err := ssh.MarshalPrivateKeyWithPassphrase(priv, "", []byte("hunter2")) + if err != nil { + t.Fatalf("marshal encrypted key: %v", err) + } + path := filepath.Join(t.TempDir(), "encrypted_key") + if err := os.WriteFile(path, pem.EncodeToMemory(block), 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + _, cleanup, err := buildAuthMethods(&SSHTarget{ + IdentitySource: SSHIdentitySourceFile, + IdentityFile: path, + }) + cleanup() + if err == nil || !strings.Contains(err.Error(), "passphrase-protected") { + t.Fatalf("error = %v", err) + } + if !strings.Contains(err.Error(), "ssh-add") { + t.Fatalf("error = %v, want the ssh-agent remediation hint", err) + } + }) + + t.Run("agent identity requires SSH_AUTH_SOCK", func(t *testing.T) { + t.Setenv("SSH_AUTH_SOCK", "") + _, cleanup, err := buildAuthMethods(&SSHTarget{IdentitySource: SSHIdentitySourceAgent}) + cleanup() + if err == nil || !strings.Contains(err.Error(), "SSH_AUTH_SOCK is not set") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("agent identity connects to the socket", func(t *testing.T) { + sock := filepath.Join(t.TempDir(), "agent.sock") + listener, err := net.Listen("unix", sock) + if err != nil { + t.Skipf("unix sockets unavailable: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + t.Setenv("SSH_AUTH_SOCK", sock) + + methods, cleanup, err := buildAuthMethods(&SSHTarget{IdentitySource: SSHIdentitySourceAgent}) + if err != nil { + t.Fatalf("buildAuthMethods: %v", err) + } + if len(methods) != 1 { + t.Fatalf("methods = %d, want 1", len(methods)) + } + cleanup() // releases the agent socket + }) + + t.Run("an unreachable agent socket is an error", func(t *testing.T) { + t.Setenv("SSH_AUTH_SOCK", filepath.Join(t.TempDir(), "absent.sock")) + _, cleanup, err := buildAuthMethods(&SSHTarget{IdentitySource: SSHIdentitySourceAgent}) + cleanup() + if err == nil || !strings.Contains(err.Error(), "failed to connect to ssh-agent") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("an unknown identity source is rejected", func(t *testing.T) { + _, cleanup, err := buildAuthMethods(&SSHTarget{IdentitySource: "smartcard"}) + cleanup() + if err == nil || !strings.Contains(err.Error(), `unsupported identity source: "smartcard"`) { + t.Fatalf("error = %v", err) + } + }) +} + +func TestBuildClientConfigHostKeyCallback(t *testing.T) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("signer: %v", err) + } + hostKey := signer.PublicKey() + fingerprint := SSHFingerprint(hostKey) + if !strings.HasPrefix(fingerprint, "SHA256:") { + t.Fatalf("SSHFingerprint = %q, want the OpenSSH SHA256 form", fingerprint) + } + + identity := writeTestIdentityFile(t) + + t.Run("test mode records the observed fingerprint and accepts", func(t *testing.T) { + target := &SSHTarget{ + User: "deploy", IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + } + cfg, cleanup, err := buildClientConfig(target) + if err != nil { + t.Fatalf("buildClientConfig: %v", err) + } + defer cleanup() + if cfg.User != "deploy" { + t.Fatalf("User = %q", cfg.User) + } + if err := cfg.HostKeyCallback("host:22", nil, hostKey); err != nil { + t.Fatalf("unpinned host key must be accepted: %v", err) + } + if target.ObservedFingerprint != fingerprint { + t.Fatalf("ObservedFingerprint = %q, want %q", target.ObservedFingerprint, fingerprint) + } + }) + + t.Run("a pinned fingerprint accepts a match", func(t *testing.T) { + target := &SSHTarget{ + User: "deploy", IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: fingerprint, + } + cfg, cleanup, err := buildClientConfig(target) + if err != nil { + t.Fatalf("buildClientConfig: %v", err) + } + defer cleanup() + if err := cfg.HostKeyCallback("host:22", nil, hostKey); err != nil { + t.Fatalf("matching host key must be accepted: %v", err) + } + }) + + t.Run("a pinned fingerprint rejects a mismatch", func(t *testing.T) { + target := &SSHTarget{ + User: "deploy", IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: "SHA256:something-else", + } + cfg, cleanup, err := buildClientConfig(target) + if err != nil { + t.Fatalf("buildClientConfig: %v", err) + } + defer cleanup() + err = cfg.HostKeyCallback("host:22", nil, hostKey) + var mismatch *errHostKeyMismatch + if !errors.As(err, &mismatch) { + t.Fatalf("error = %v, want errHostKeyMismatch", err) + } + if mismatch.Expected != "SHA256:something-else" || mismatch.Got != fingerprint { + t.Fatalf("mismatch = %+v", mismatch) + } + if !strings.Contains(mismatch.Error(), "host key changed") { + t.Fatalf("message = %q", mismatch.Error()) + } + }) + + t.Run("an auth failure surfaces before any dial", func(t *testing.T) { + _, cleanup, err := buildClientConfig(&SSHTarget{IdentitySource: "smartcard"}) + cleanup() + if err == nil { + t.Fatal("expected the auth-method failure to abort config building") + } + }) +} + +func TestDialSSHConnectsAndEnforcesThePinnedFingerprint(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("hello\n") }) + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + t.Run("matching fingerprint connects", func(t *testing.T) { + target := &SSHTarget{ + Host: "127.0.0.1", Port: server.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: server.hostKeyFingerprint(), + } + client, err := DialSSH(context.Background(), target) + if err != nil { + t.Fatalf("DialSSH: %v", err) + } + defer func() { _ = client.Close() }() + + out, _, err := runSSHCommand(context.Background(), client, "echo hello") + if err != nil { + t.Fatalf("run over the dialled client: %v", err) + } + if out != "hello\n" { + t.Fatalf("stdout = %q", out) + } + if target.ObservedFingerprint != server.hostKeyFingerprint() { + t.Fatalf("ObservedFingerprint = %q", target.ObservedFingerprint) + } + }) + + t.Run("a changed host key aborts the handshake", func(t *testing.T) { + target := &SSHTarget{ + Host: "127.0.0.1", Port: server.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: "SHA256:not-the-server-key", + } + client, err := dialSSH(context.Background(), target) + if err == nil { + _ = client.Close() + t.Fatal("expected the pinned-fingerprint mismatch to abort the dial") + } + if !strings.Contains(err.Error(), "host key changed") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("an unreachable host fails at the TCP dial", func(t *testing.T) { + target := &SSHTarget{ + Host: "127.0.0.1", Port: 1, User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: server.hostKeyFingerprint(), + } + if _, err := dialSSH(context.Background(), target); err == nil || + !strings.Contains(err.Error(), "tcp dial") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("a bad identity aborts before dialling", func(t *testing.T) { + target := &SSHTarget{ + Host: "127.0.0.1", Port: server.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, + IdentityFile: filepath.Join(t.TempDir(), "absent"), + } + if _, err := dialSSH(context.Background(), target); err == nil || + !strings.Contains(err.Error(), "failed to read identity file") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestDialViaJumpTunnelsThroughTheBastion(t *testing.T) { + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + + final := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("behind-the-bastion\n") }) + bastion := newFakeSSHServer(t, nil) + // The bastion forwards every direct-tcpip channel to the final host. + bastion.forwardTo(final.addr()) + + target := &SSHTarget{ + Host: "127.0.0.1", Port: final.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: final.hostKeyFingerprint(), + ProxyJump: "jump@127.0.0.1:" + strconv.Itoa(bastion.port()), + } + + client, err := dialSSH(context.Background(), target) + if err != nil { + t.Fatalf("dialSSH via jump: %v", err) + } + defer func() { _ = client.Close() }() + + out, _, err := runSSHCommand(context.Background(), client, "echo hi") + if err != nil { + t.Fatalf("run through the tunnel: %v", err) + } + if out != "behind-the-bastion\n" { + t.Fatalf("stdout = %q, want the final host's output", out) + } +} + +func TestDialViaJumpFailsWhenTheBastionIsUnreachable(t *testing.T) { + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + final := newFakeSSHServer(t, nil) + + target := &SSHTarget{ + Host: "127.0.0.1", Port: final.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: final.hostKeyFingerprint(), + ProxyJump: "jump@127.0.0.1:1", + } + _, err := dialSSH(context.Background(), target) + if err == nil || !strings.Contains(err.Error(), "bastion dial") { + t.Fatalf("error = %v, want a bastion dial failure", err) + } +} + +func TestDialViaJumpFailsWhenTheTunnelTargetIsClosed(t *testing.T) { + newTestHomeDir(t) + identity := writeTestIdentityFile(t) + final := newFakeSSHServer(t, nil) + bastion := newFakeSSHServer(t, nil) // no forward target configured + + target := &SSHTarget{ + Host: "127.0.0.1", Port: final.port(), User: "kandev", + IdentitySource: SSHIdentitySourceFile, IdentityFile: identity, + PinnedFingerprint: final.hostKeyFingerprint(), + ProxyJump: "jump@127.0.0.1:" + strconv.Itoa(bastion.port()), + } + _, err := dialSSH(context.Background(), target) + if err == nil || !strings.Contains(err.Error(), "bastion tunnel") { + t.Fatalf("error = %v, want a tunnel failure", err) + } +} + +func TestResolveBastion(t *testing.T) { + home := newTestHomeDir(t) + writeSSHConfig(t, home, `Host jump-alias + HostName jump.internal + Port 2022 + User jumper +`) + + t.Run("a literal user@host:port is parsed inline", func(t *testing.T) { + bastion, err := resolveBastion(&SSHTarget{ + ProxyJump: "jumper@bastion.example:2222", + IdentitySource: SSHIdentitySourceFile, + IdentityFile: "/keys/id_ed25519", + }) + if err != nil { + t.Fatalf("resolveBastion: %v", err) + } + if bastion.Host != "bastion.example" || bastion.Port != 2222 || bastion.User != "jumper" { + t.Fatalf("bastion = %+v", bastion) + } + if bastion.IdentityFile != "/keys/id_ed25519" { + t.Fatalf("bastion identity = %q, want it inherited from the target", bastion.IdentityFile) + } + }) + + t.Run("an alias is resolved through ~/.ssh/config", func(t *testing.T) { + bastion, err := resolveBastion(&SSHTarget{ + ProxyJump: "jump-alias", + IdentitySource: SSHIdentitySourceAgent, + }) + if err != nil { + t.Fatalf("resolveBastion: %v", err) + } + if bastion.Host != "jump.internal" || bastion.Port != 2022 || bastion.User != "jumper" { + t.Fatalf("bastion = %+v", bastion) + } + }) + + t.Run("an IPv6 literal keeps its brackets off the hostname", func(t *testing.T) { + bastion, err := resolveBastion(&SSHTarget{ + ProxyJump: "jumper@[2001:db8::1]:2222", + IdentitySource: SSHIdentitySourceAgent, + }) + if err != nil { + t.Fatalf("resolveBastion: %v", err) + } + if bastion.Host != "2001:db8::1" || bastion.Port != 2222 { + t.Fatalf("bastion = %+v", bastion) + } + }) +} + +func TestBastionHostKeyCallback(t *testing.T) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("signer: %v", err) + } + _, otherPriv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate other key: %v", err) + } + otherSigner, err := ssh.NewSignerFromKey(otherPriv) + if err != nil { + t.Fatalf("other signer: %v", err) + } + remote := &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 2222} + + t.Run("absent known_hosts accepts on trust-on-first-use", func(t *testing.T) { + newTestHomeDir(t) + callback := bastionHostKeyCallback("bastion.example") + if err := callback("bastion.example:2222", remote, signer.PublicKey()); err != nil { + t.Fatalf("TOFU callback must accept: %v", err) + } + }) + + t.Run("an unknown host in known_hosts is accepted", func(t *testing.T) { + home := newTestHomeDir(t) + writeKnownHosts(t, home, "other.example", otherSigner.PublicKey()) + callback := bastionHostKeyCallback("bastion.example") + if err := callback("bastion.example:2222", remote, signer.PublicKey()); err != nil { + t.Fatalf("unknown host must be accepted: %v", err) + } + }) + + t.Run("a matching known_hosts entry is accepted", func(t *testing.T) { + home := newTestHomeDir(t) + writeKnownHosts(t, home, "[bastion.example]:2222", signer.PublicKey()) + callback := bastionHostKeyCallback("bastion.example") + if err := callback("bastion.example:2222", remote, signer.PublicKey()); err != nil { + t.Fatalf("matching key must be accepted: %v", err) + } + }) + + t.Run("a changed known_hosts key is rejected", func(t *testing.T) { + home := newTestHomeDir(t) + writeKnownHosts(t, home, "[bastion.example]:2222", otherSigner.PublicKey()) + callback := bastionHostKeyCallback("bastion.example") + err := callback("bastion.example:2222", remote, signer.PublicKey()) + if err == nil || !strings.Contains(err.Error(), "host key changed against known_hosts") { + t.Fatalf("error = %v, want a loud mismatch", err) + } + if !strings.Contains(err.Error(), "bastion.example") { + t.Fatalf("error = %v, want the ProxyJump value for context", err) + } + }) + + t.Run("an unparseable known_hosts falls back to accepting", func(t *testing.T) { + home := newTestHomeDir(t) + if err := os.WriteFile(filepath.Join(home, ".ssh", "known_hosts"), + []byte("garbage line without a key\n"), 0o600); err != nil { + t.Fatalf("write known_hosts: %v", err) + } + callback := bastionHostKeyCallback("bastion.example") + if err := callback("bastion.example:2222", remote, signer.PublicKey()); err != nil { + t.Fatalf("unparseable known_hosts must degrade to accept: %v", err) + } + }) + + t.Run("the log-only callback accepts anything", func(t *testing.T) { + callback := tofuLogOnlyCallback("bastion.example", "") + if err := callback("bastion.example:2222", remote, signer.PublicKey()); err != nil { + t.Fatalf("tofuLogOnlyCallback must accept: %v", err) + } + }) +} + +func writeKnownHosts(t *testing.T, home, host string, key ssh.PublicKey) { + t.Helper() + line := fmt.Sprintf("%s %s %s\n", host, key.Type(), + strings.TrimSpace(strings.TrimPrefix(string(ssh.MarshalAuthorizedKey(key)), key.Type()+" "))) + if err := os.WriteFile(filepath.Join(home, ".ssh", "known_hosts"), []byte(line), 0o600); err != nil { + t.Fatalf("write known_hosts: %v", err) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go new file mode 100644 index 0000000000..8f0cb19c35 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_lifecycle_test.go @@ -0,0 +1,605 @@ +package lifecycle + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + "golang.org/x/crypto/ssh" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/githubauth" +) + +// writeTestIdentityFile writes an unencrypted ed25519 private key so the SSH +// executor's "file" identity source has something real to parse. The fake +// server accepts any auth, so the key only has to load. +func writeTestIdentityFile(t *testing.T) string { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate identity key: %v", err) + } + block, err := ssh.MarshalPrivateKey(priv, "") + if err != nil { + t.Fatalf("marshal identity key: %v", err) + } + path := filepath.Join(t.TempDir(), "id_ed25519") + if err := os.WriteFile(path, pem.EncodeToMemory(block), 0o600); err != nil { + t.Fatalf("write identity key: %v", err) + } + return path +} + +// sshConnectionMetadata builds the executor-config metadata that points the +// SSH executor at the in-process fake server. +func sshConnectionMetadata(t *testing.T, server *fakeSSHServer) map[string]interface{} { + t.Helper() + return map[string]interface{}{ + MetadataKeySSHHost: "127.0.0.1", + MetadataKeySSHPort: strconv.Itoa(server.port()), + MetadataKeySSHUser: "kandev", + MetadataKeySSHHostFingerprint: server.hostKeyFingerprint(), + MetadataKeySSHIdentitySource: string(SSHIdentitySourceFile), + MetadataKeySSHIdentityFile: writeTestIdentityFile(t), + } +} + +func TestSSHExecutorStaticSurface(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + if exec.Name() != executor.NameSSH { + t.Fatalf("Name() = %q", exec.Name()) + } + if err := exec.HealthCheck(context.Background()); err != nil { + t.Fatalf("HealthCheck() = %v, want nil (per-host reachability is checked elsewhere)", err) + } + if !exec.RequiresCloneURL() { + t.Fatal("SSH remotes must clone the repository themselves") + } + if exec.ShouldApplyPreferredShell() { + t.Fatal("the host's preferred shell must not be applied to a remote") + } + if !exec.IsAlwaysResumable() { + t.Fatal("SSH executions are always resumable") + } + if exec.GetInteractiveRunner() != nil { + t.Fatal("SSH has no host-side interactive runner") + } + instances, err := exec.RecoverInstances(context.Background()) + if err != nil || instances != nil { + t.Fatalf("RecoverInstances() = %v, %v; want nil, nil", instances, err) + } +} + +func TestSSHExecutorTargetFromMetadata(t *testing.T) { + t.Run("maps every connection field", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + target, err := exec.targetFromMetadata(map[string]interface{}{ + MetadataKeySSHHost: "build.example", + MetadataKeySSHPort: "2222", + MetadataKeySSHUser: "deploy", + MetadataKeySSHHostFingerprint: "SHA256:pinned", + MetadataKeySSHIdentitySource: string(SSHIdentitySourceFile), + MetadataKeySSHIdentityFile: "/keys/id_ed25519", + MetadataKeySSHProxyJump: "bastion.example", + }) + if err != nil { + t.Fatalf("targetFromMetadata: %v", err) + } + if target.Host != "build.example" || target.Port != 2222 || target.User != "deploy" { + t.Fatalf("target = %+v", target) + } + if target.PinnedFingerprint != "SHA256:pinned" { + t.Fatalf("PinnedFingerprint = %q", target.PinnedFingerprint) + } + if target.IdentitySource != SSHIdentitySourceFile || target.IdentityFile != "/keys/id_ed25519" { + t.Fatalf("identity = %q / %q", target.IdentitySource, target.IdentityFile) + } + if target.ProxyJump != "bastion.example" { + t.Fatalf("ProxyJump = %q", target.ProxyJump) + } + }) + + t.Run("host alias alone is accepted", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + target, err := exec.targetFromMetadata(map[string]interface{}{ + MetadataKeySSHHostAlias: "prod-box", + MetadataKeySSHUser: "deploy", + MetadataKeySSHHostFingerprint: "SHA256:pinned", + MetadataKeySSHIdentitySource: string(SSHIdentitySourceAgent), + }) + if err != nil { + t.Fatalf("targetFromMetadata: %v", err) + } + if target.Host != "prod-box" { + t.Fatalf("Host = %q, want the alias used as hostname", target.Host) + } + }) + + tests := []struct { + name string + metadata map[string]interface{} + wantErr string + }{ + { + name: "host is required", + metadata: map[string]interface{}{MetadataKeySSHHostFingerprint: "SHA256:x"}, + wantErr: "host (or host_alias) is required", + }, + { + name: "fingerprint is required", + metadata: map[string]interface{}{ + MetadataKeySSHHost: "build.example", + }, + wantErr: "host_fingerprint is required", + }, + { + name: "non-numeric port is rejected loudly", + metadata: map[string]interface{}{ + MetadataKeySSHHost: "build.example", + MetadataKeySSHPort: "abc", + MetadataKeySSHHostFingerprint: "SHA256:x", + }, + wantErr: `invalid ssh_port "abc"`, + }, + { + name: "port zero is rejected rather than defaulted to 22", + metadata: map[string]interface{}{ + MetadataKeySSHHost: "build.example", + MetadataKeySSHPort: "0", + MetadataKeySSHHostFingerprint: "SHA256:x", + }, + wantErr: `invalid ssh_port "0"`, + }, + { + name: "out-of-range port is rejected", + metadata: map[string]interface{}{ + MetadataKeySSHHost: "build.example", + MetadataKeySSHPort: "70000", + MetadataKeySSHHostFingerprint: "SHA256:x", + }, + wantErr: `invalid ssh_port "70000"`, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + _, err := exec.targetFromMetadata(tc.metadata) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %v, want %q", err, tc.wantErr) + } + }) + } +} + +func TestSSHExecutorWorkdirRoot(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + if got := exec.workdirRoot(nil); got != sshDefaultWorkdir { + t.Fatalf("default workdir = %q, want %q", got, sshDefaultWorkdir) + } + got := exec.workdirRoot(map[string]interface{}{MetadataKeySSHWorkdirRoot: "/srv/kandev"}) + if got != "/srv/kandev" { + t.Fatalf("override workdir = %q", got) + } +} + +func TestSSHExecutorCloseTearsDownEveryTrackedSession(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + var closed int + var mu sync.Mutex + exec.closeClient = func(*ssh.Client) error { + mu.Lock() + defer mu.Unlock() + closed++ + return nil + } + forwardServer := newFakeSSHServer(t, nil) + fwd, err := StartPortForward(forwardServer.dial(t), 41234, newTestLogger()) + if err != nil { + t.Fatalf("StartPortForward: %v", err) + } + exec.sessions["a"] = &sshSessionState{client: &ssh.Client{}, forwarder: fwd} + exec.sessions["b"] = &sshSessionState{client: &ssh.Client{}} + + if err := exec.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if closed != 2 { + t.Fatalf("closed clients = %d, want 2", closed) + } + if len(exec.sessions) != 0 { + t.Fatalf("sessions still tracked after Close: %+v", exec.sessions) + } + // The forwarder must be closed: a second Close is a no-op error-free call, + // and its listener must no longer accept. + if err := fwd.Close(); err != nil { + t.Fatalf("forwarder Close after executor Close: %v", err) + } +} + +func TestSSHExecutorCloseSSHClientFallsBackToTheRealClose(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + if err := exec.closeSSHClient(nil); err != nil { + t.Fatalf("closing a nil client must be a no-op, got %v", err) + } + exec.closeClient = nil + server := newFakeSSHServer(t, nil) + client := server.dial(t) + if err := exec.closeSSHClient(client); err != nil { + t.Fatalf("closeSSHClient: %v", err) + } + if _, err := client.NewSession(); err == nil { + t.Fatal("client must actually be closed when no closeClient seam is set") + } +} + +// sshLaunchHarness wires a fake SSH server plus an in-process agentctl control +// server so CreateInstance / ResumeRemoteInstance can run end to end. +type sshLaunchHarness struct { + server *fakeSSHServer + remoteHome string + // createRequests captures each create-instance body the control server saw. + createRequests []string + handshakes int + mu sync.Mutex +} + +func newSSHLaunchHarness(t *testing.T, pid string) *sshLaunchHarness { + t.Helper() + h := &sshLaunchHarness{remoteHome: t.TempDir()} + + control := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() + defer h.mu.Unlock() + switch r.URL.Path { + case "/auth/handshake": + h.handshakes++ + _, _ = io.WriteString(w, `{"token":"launch-token"}`) + case "/api/v1/instances": + body, _ := io.ReadAll(r.Body) + h.createRequests = append(h.createRequests, string(body)) + _, _ = io.WriteString(w, `{"id":"remote-instance","port":41234}`) + case "/health": + _, _ = io.WriteString(w, "ok") + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(control.Close) + + platform := SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"} + _, sha := writeFakeAgentctl(t, platform, "agentctl-bytes") + if err := os.MkdirAll(filepath.Join(h.remoteHome, ".kandev", "bin"), 0o755); err != nil { + t.Fatalf("seed remote bin dir: %v", err) + } + + h.server = newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch { + case command == "uname -a": + return sshOut("Linux remote 6.1.0\n") + case command == "uname -m": + return sshOut("x86_64\n") + case command == "uname -s": + return sshOut("Linux\n") + case command == "git --version": + return sshOut("git version 2.44.0\n") + case strings.Contains(command, remoteHomeCommand): + return sshOut(h.remoteHome) + case strings.Contains(command, "agentctl.sha256"): + return sshOut(sha + "\n") + case strings.Contains(command, "test -x"): + return sshOK + case strings.Contains(command, "nohup"): + return sshOut(pid + "\n") + case strings.Contains(command, "agentctl.log"): + return sshOut("HTTP server bound successfully\n") + case strings.Contains(command, "kill -0"): + return sshOK + default: + // mkdir -p, prepare script, stop script, … + return sshOK + } + }) + h.server.enableSFTP() + h.server.forwardTo(strings.TrimPrefix(control.URL, "http://")) + return h +} + +func (h *sshLaunchHarness) createInstanceBodies() []string { + h.mu.Lock() + defer h.mu.Unlock() + return append([]string(nil), h.createRequests...) +} + +func TestSSHExecutorCreateInstanceProvisionsAndTracksTheSession(t *testing.T) { + harness := newSSHLaunchHarness(t, "4242") + exec := NewSSHExecutor(nil, nil, NewAgentctlResolver(newTestLogger()), newTestLogger()) + t.Cleanup(func() { _ = exec.Close() }) + + metadata := sshConnectionMetadata(t, harness.server) + metadata[MetadataKeySetupScript] = "echo prepared" + metadata[MetadataKeyBaseBranch] = "main" + metadata[MetadataKeyWorktreeBranch] = "feature/task-1" + + var steps []string + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + Protocol: "acp", + Metadata: metadata, + OnProgress: func(step PrepareStep, _, _ int) { + steps = append(steps, step.Name+":"+string(step.Status)) + }, + } + + instance, err := exec.CreateInstance(context.Background(), req) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + + wantTaskDir := harness.remoteHome + "/.kandev/tasks/task-task-1" + wantSessionDir := wantTaskDir + "/.kandev/sessions/session-1" + if instance.InstanceID != "instance-1" || instance.TaskID != "task-1" || instance.SessionID != "session-1" { + t.Fatalf("instance identity = %+v", instance) + } + if instance.RuntimeName != executor.NameSSH { + t.Fatalf("RuntimeName = %q", instance.RuntimeName) + } + if instance.WorkspacePath != wantTaskDir { + t.Fatalf("WorkspacePath = %q, want the remote task dir %q", instance.WorkspacePath, wantTaskDir) + } + if instance.AuthToken != "launch-token" { + t.Fatalf("AuthToken = %q, want the handshake token", instance.AuthToken) + } + if instance.Client == nil { + t.Fatal("instance must carry an agentctl client bound to the local forward") + } + if got := instance.Metadata[MetadataKeySSHRemoteTaskDir]; got != wantTaskDir { + t.Fatalf("remote task dir metadata = %v", got) + } + if got := instance.Metadata[MetadataKeySSHRemoteSessionDir]; got != wantSessionDir { + t.Fatalf("remote session dir metadata = %v", got) + } + if got := instance.Metadata[MetadataKeySSHRemoteAgentctlPort]; got != "41234" { + t.Fatalf("remote agentctl port metadata = %v, want the per-instance port", got) + } + if got := instance.Metadata[MetadataKeySSHRemoteAgentctlPID]; got != "4242" { + t.Fatalf("remote pid metadata = %v", got) + } + if got := instance.Metadata[MetadataKeySSHHost]; got != "127.0.0.1" { + t.Fatalf("host metadata = %v", got) + } + if got := instance.Metadata[MetadataKeySSHHostFingerprint]; got != harness.server.hostKeyFingerprint() { + t.Fatalf("fingerprint metadata = %v", got) + } + if got := instance.Metadata[MetadataKeySSHWorkdirRoot]; got != sshDefaultWorkdir { + t.Fatalf("workdir root metadata = %v", got) + } + if got := instance.Metadata[MetadataKeyIsRemote]; got != true { + t.Fatalf("is_remote metadata = %v", got) + } + forwardPort, _ := strconv.Atoi(instance.Metadata[MetadataKeySSHLocalForwardPort].(string)) + if forwardPort == 0 { + t.Fatal("local forward port metadata must be a real bound port") + } + + if len(harness.createInstanceBodies()) != 1 { + t.Fatalf("create-instance calls = %d, want exactly one", len(harness.createInstanceBodies())) + } + if !strings.Contains(harness.createInstanceBodies()[0], `"workspace_path":"`+wantTaskDir+`"`) { + t.Fatalf("create-instance body = %s", harness.createInstanceBodies()[0]) + } + + for _, want := range []string{ + "Connecting to SSH host:completed", + "Detecting remote OS:completed", + "Uploading agent controller:completed", + "Preparing task directory:completed", + "Running prepare script:completed", + "Preparing session directory:completed", + "Starting agent controller:completed", + "Creating agent instance:completed", + "Connecting to agent controller:completed", + } { + if !sshContainsStep(steps, want) { + t.Fatalf("progress steps %v missing %q", steps, want) + } + } + + exec.mu.Lock() + state := exec.sessions["instance-1"] + exec.mu.Unlock() + if state == nil { + t.Fatal("CreateInstance must track the session state for teardown") + } + if state.pid != 4242 || state.remoteDir != wantSessionDir || state.remoteTaskDir != wantTaskDir { + t.Fatalf("session state = %+v", state) + } + if state.authToken != "launch-token" { + t.Fatalf("session auth token = %q", state.authToken) + } + if state.platform.GOOS != "linux" || state.platform.GOARCH != "amd64" { + t.Fatalf("session platform = %+v", state.platform) + } + + // StopInstance releases the forwarder and SSH client and forgets the state. + if err := exec.StopInstance(context.Background(), instance, true); err != nil { + t.Fatalf("StopInstance: %v", err) + } + exec.mu.Lock() + _, stillTracked := exec.sessions["instance-1"] + exec.mu.Unlock() + if stillTracked { + t.Fatal("StopInstance must drop the tracked session") + } + if _, ok := harness.server.lastCommandContaining("kill 4242"); !ok { + t.Fatalf("expected the remote agentctl to be killed, commands: %v", harness.server.commands()) + } +} + +func TestSSHExecutorCreateInstanceReusesResumedState(t *testing.T) { + server := newFakeSSHServer(t, nil) + fwd, err := StartPortForward(server.dial(t), 41234, newTestLogger()) + if err != nil { + t.Fatalf("StartPortForward: %v", err) + } + t.Cleanup(func() { _ = fwd.Close() }) + + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.sessions["instance-1"] = &sshSessionState{ + target: &SSHTarget{Host: "build.example", Port: 2222, User: "deploy", PinnedFingerprint: "SHA256:pinned"}, + forwarder: fwd, + pid: 999, + remoteDir: "/remote/session", + authToken: "resumed-token", + reusingProcess: true, + } + + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + Metadata: map[string]interface{}{ + MetadataKeySSHRemoteTaskDir: "/remote/task", + MetadataKeySSHRemoteAgentctlPort: "41234", + }, + } + instance, err := exec.CreateInstance(context.Background(), req) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + if instance.AuthToken != "resumed-token" { + t.Fatalf("AuthToken = %q, want the resumed token", instance.AuthToken) + } + if instance.WorkspacePath != "/remote/task" { + t.Fatalf("WorkspacePath = %q", instance.WorkspacePath) + } + if got := instance.Metadata["reuse_existing_process"]; got != true { + t.Fatalf("reuse_existing_process = %v, want true", got) + } + if got := instance.Metadata[MetadataKeySSHRemoteAgentctlPID]; got != "999" { + t.Fatalf("pid metadata = %v", got) + } + if got := instance.Metadata[MetadataKeySSHHost]; got != "build.example" { + t.Fatalf("host metadata = %v, want it mirrored from the resumed target", got) + } + if got := instance.Metadata[MetadataKeySSHLocalForwardPort]; got != strconv.Itoa(fwd.LocalPort()) { + t.Fatalf("local forward port = %v, want the live forwarder's port", got) + } + if len(server.commands()) != 0 { + t.Fatalf("resumed create must not re-run remote setup, commands: %v", server.commands()) + } +} + +func TestSSHExecutorCreateInstanceIgnoresResumedStateForBrokerRefresh(t *testing.T) { + // A managed-broker launch must not silently reuse the previous session's + // agentctl: the broker lease in its environment is stale. + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.sessions["instance-1"] = &sshSessionState{authToken: "old"} + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + Env: map[string]string{ + githubauth.CredentialBrokerURLEnv: "http://127.0.0.1:9/broker", + githubauth.CredentialLeaseEnv: "lease-1", + }, + } + if _, ok := exec.resumedStateForCreate(req); ok { + t.Fatal("managed broker env must force a fresh create") + } + if _, ok := exec.resumedStateForCreate(nil); ok { + t.Fatal("nil request must not resume") + } +} + +func TestSSHExecutorCreateInstanceRejectsUnsupportedRemotePlatform(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch command { + case "uname -m": + return sshOut("riscv64\n") + case "uname -s": + return sshOut("FreeBSD\n") + default: + return sshOK + } + }) + exec := NewSSHExecutor(nil, nil, NewAgentctlResolver(newTestLogger()), newTestLogger()) + + var failedSteps []PrepareStep + _, err := exec.CreateInstance(context.Background(), &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + Metadata: sshConnectionMetadata(t, server), + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepFailed { + failedSteps = append(failedSteps, step) + } + }, + }) + if err == nil { + t.Fatal("expected an unsupported-platform failure") + } + if !strings.Contains(err.Error(), "FreeBSD/riscv64") { + t.Fatalf("error = %v, want the raw remote platform preserved", err) + } + if len(failedSteps) != 1 || failedSteps[0].Name != "Detecting remote OS" { + t.Fatalf("failed steps = %+v, want a failed OS-detection step", failedSteps) + } + if len(exec.sessions) != 0 { + t.Fatal("a failed launch must leave no tracked session") + } +} + +func TestSSHExecutorCreateInstanceRejectsBadConnectionMetadata(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + _, err := exec.CreateInstance(context.Background(), &ExecutorCreateRequest{ + InstanceID: "instance-1", + Metadata: map[string]interface{}{MetadataKeySSHHost: "build.example"}, + }) + if err == nil || !strings.Contains(err.Error(), "host_fingerprint is required") { + t.Fatalf("error = %v", err) + } +} + +func TestSSHExecutorPrepareRemoteHostReportsUploadFailure(t *testing.T) { + // The agentctl resolver cannot find a helper for this platform, so the + // upload step fails and must be reported as such. + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch command { + case "uname -m": + return sshOut("arm64\n") + case "uname -s": + return sshOut("Darwin\n") + default: + return sshOK + } + }) + t.Setenv("KANDEV_AGENTCTL_DARWIN_ARM64_BINARY", filepath.Join(t.TempDir(), "absent")) + exec := NewSSHExecutor(nil, nil, NewAgentctlResolver(newTestLogger()), newTestLogger()) + + var failed []string + _, _, err := exec.prepareRemoteHost(context.Background(), server.dial(t), &ExecutorCreateRequest{ + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepFailed { + failed = append(failed, step.Name) + } + }, + }) + if err == nil { + t.Fatal("expected the missing agentctl helper to fail the launch") + } + if !sshContainsStep(failed, "Uploading agent controller") { + t.Fatalf("failed steps = %v, want the upload step", failed) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go index 55ceed91be..0eb24ae37f 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations.go @@ -123,7 +123,9 @@ func runSSHCommandStdin(ctx context.Context, client *ssh.Client, cmd string, std } defer func() { _ = session.Close() }() - var outBuf, errBuf bytes.Buffer + // Synchronized because the cancellation path below reads these while + // session.Run is still going — see syncBuffer. + var outBuf, errBuf syncBuffer session.Stdout = &outBuf session.Stderr = &errBuf if stdin != nil { @@ -143,6 +145,34 @@ func runSSHCommandStdin(ctx context.Context, client *ssh.Client, cmd string, std } } +// syncBuffer is a bytes.Buffer safe for concurrent use by one writer and one +// reader. It exists for runSSHCommandStdin's cancellation path: that path +// returns while session.Run is still executing, so golang.org/x/crypto/ssh's +// stdout/stderr copier goroutines are still writing into these buffers when we +// read them — a data race on a plain bytes.Buffer, and one that outlives the +// call, since the copiers keep writing until the remote command actually ends. +// +// Deliberately not an io.ReaderFrom: bytes.Buffer implements ReadFrom, and +// io.Copy prefers it, which would let the copier reach the underlying buffer +// without taking the lock. Exposing only Write keeps every mutation guarded. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +// String returns a consistent snapshot of whatever the remote has sent so far. +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + // detectRemoteInfo runs a tiny probe to learn about the host. The support gate // happens at the caller — this function only reports. func detectRemoteInfo(ctx context.Context, client *ssh.Client) (*SSHRemoteInfo, error) { diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations_remote_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations_remote_test.go new file mode 100644 index 0000000000..6bf4ed2dd2 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_operations_remote_test.go @@ -0,0 +1,886 @@ +package lifecycle + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// remoteHomeCommand is the exact probe expandRemoteHome issues; matching on it +// keeps the fake-server rules honest about the real command string. +const remoteHomeCommand = `printf %s "$HOME"` + +func TestDetectRemoteInfoNormalizesPlatform(t *testing.T) { + tests := []struct { + name string + unameOS string + unameArch string + wantGOOS string + wantGOARCH string + }{ + {name: "linux x86_64", unameOS: "Linux", unameArch: "x86_64", wantGOOS: "linux", wantGOARCH: "amd64"}, + {name: "linux aarch64", unameOS: "Linux", unameArch: "aarch64", wantGOOS: "linux", wantGOARCH: "arm64"}, + {name: "darwin arm64", unameOS: "Darwin", unameArch: "arm64", wantGOOS: "darwin", wantGOARCH: "arm64"}, + {name: "darwin amd64", unameOS: "Darwin", unameArch: "x86_64", wantGOOS: "darwin", wantGOARCH: "amd64"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: "uname -a", result: sshOut(" " + tc.unameOS + " host 6.1.0 \n")}, + sshScriptRule{match: "uname -m", result: sshOut(tc.unameArch + "\n")}, + sshScriptRule{match: "uname -s", result: sshOut(tc.unameOS + "\n")}, + sshScriptRule{match: "git --version", result: sshOut("git version 2.44.0\n")}, + ).handle) + client := server.dial(t) + + info, err := SSHProbeRemote(context.Background(), client) + if err != nil { + t.Fatalf("SSHProbeRemote: %v", err) + } + if info.OS != tc.unameOS || info.Arch != tc.unameArch { + t.Fatalf("raw uname = %q/%q, want %q/%q", info.OS, info.Arch, tc.unameOS, tc.unameArch) + } + if info.UnameAll != tc.unameOS+" host 6.1.0" { + t.Fatalf("UnameAll = %q, want trimmed uname -a output", info.UnameAll) + } + if info.GitVer != "git version 2.44.0" { + t.Fatalf("GitVer = %q", info.GitVer) + } + if info.Platform.GOOS != tc.wantGOOS || info.Platform.GOARCH != tc.wantGOARCH { + t.Fatalf("platform = %s, want %s/%s", info.Platform, tc.wantGOOS, tc.wantGOARCH) + } + if info.Platform.UnameOS != tc.unameOS || info.Platform.UnameArch != tc.unameArch { + t.Fatalf("platform retained raw = %q/%q", info.Platform.UnameOS, info.Platform.UnameArch) + } + if err := SSHRequireSupportedRemotePlatform(info.Platform); err != nil { + t.Fatalf("normalized platform %s must be supported: %v", info.Platform, err) + } + wantCommands := []string{"uname -a", "uname -m", "uname -s", "git --version"} + if got := server.commands(); !equalStrings(got, wantCommands) { + t.Fatalf("probe commands = %v, want %v", got, wantCommands) + } + }) + } +} + +func TestDetectRemoteInfoFailsWhenUnameUnavailable(t *testing.T) { + tests := []struct { + name string + failing string + wantErr string + }{ + {name: "arch probe fails", failing: "uname -m", wantErr: "ssh: uname -m"}, + {name: "os probe fails", failing: "uname -s", wantErr: "ssh: uname -s"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + if command == tc.failing { + return sshFail("uname: not found") + } + return sshOut("Linux\n") + }) + client := server.dial(t) + + info, err := detectRemoteInfo(context.Background(), client) + if err == nil { + t.Fatalf("expected failure, got info %+v", info) + } + if !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %v, want it to mention %q", err, tc.wantErr) + } + }) + } +} + +func TestRequireSupportedRemotePlatformPreservesRawDetail(t *testing.T) { + t.Run("supported matrix", func(t *testing.T) { + for _, platform := range []SSHRemotePlatform{ + {GOOS: "linux", GOARCH: "amd64"}, + {GOOS: "linux", GOARCH: "arm64"}, + {GOOS: "darwin", GOARCH: "amd64"}, + {GOOS: "darwin", GOARCH: "arm64"}, + } { + if err := requireSupportedRemotePlatform(platform); err != nil { + t.Fatalf("%s must be supported: %v", platform, err) + } + } + }) + + t.Run("unsupported os keeps raw uname detail", func(t *testing.T) { + platform, ok := normalizeSSHRemotePlatform("FreeBSD", "riscv64") + if ok { + t.Fatal("FreeBSD/riscv64 must not normalize as supported") + } + if platform.String() != "unknown" { + t.Fatalf("normalized platform = %q, want unknown", platform.String()) + } + err := requireSupportedRemotePlatform(platform) + if err == nil { + t.Fatal("expected unsupported-platform error") + } + if !strings.Contains(err.Error(), `"FreeBSD/riscv64"`) { + t.Fatalf("error must preserve the raw reported platform: %v", err) + } + if !strings.Contains(err.Error(), "linux/{amd64,arm64}") || + !strings.Contains(err.Error(), "darwin/{amd64,arm64}") { + t.Fatalf("error must list the supported matrix: %v", err) + } + }) + + t.Run("half-unsupported keeps the normalized value", func(t *testing.T) { + // Recognised OS, unrecognised arch: String() is "unknown" because + // GOARCH is empty, so the raw pair is surfaced instead. + platform, ok := normalizeSSHRemotePlatform("Linux", "ppc64le") + if ok { + t.Fatal("linux/ppc64le must not normalize as supported") + } + if platform.GOOS != "linux" || platform.GOARCH != "" { + t.Fatalf("platform = %+v, want linux with empty arch", platform) + } + err := requireSupportedRemotePlatform(platform) + if err == nil || !strings.Contains(err.Error(), `"Linux/ppc64le"`) { + t.Fatalf("error = %v, want raw Linux/ppc64le detail", err) + } + }) + + t.Run("fully unknown platform has no raw detail to add", func(t *testing.T) { + err := requireSupportedRemotePlatform(SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), `"unknown"`) { + t.Fatalf("error = %v, want unknown", err) + } + }) +} + +func TestNormalizeSSHRemotePlatformAcceptsUnameAliases(t *testing.T) { + tests := []struct { + osName, arch string + wantGOOS, wantArch string + wantSupported bool + }{ + {osName: "linux", arch: "amd64", wantGOOS: "linux", wantArch: "amd64", wantSupported: true}, + {osName: " Linux ", arch: " X86_64 ", wantGOOS: "linux", wantArch: "amd64", wantSupported: true}, + {osName: "DARWIN", arch: "AARCH64", wantGOOS: "darwin", wantArch: "arm64", wantSupported: true}, + {osName: "Windows_NT", arch: "x86_64", wantGOOS: "", wantArch: "amd64", wantSupported: false}, + {osName: "Linux", arch: "i686", wantGOOS: "linux", wantArch: "", wantSupported: false}, + } + for _, tc := range tests { + t.Run(tc.osName+"/"+tc.arch, func(t *testing.T) { + platform, ok := normalizeSSHRemotePlatform(tc.osName, tc.arch) + if ok != tc.wantSupported { + t.Fatalf("supported = %v, want %v", ok, tc.wantSupported) + } + if platform.GOOS != tc.wantGOOS || platform.GOARCH != tc.wantArch { + t.Fatalf("normalized = %q/%q, want %q/%q", + platform.GOOS, platform.GOARCH, tc.wantGOOS, tc.wantArch) + } + }) + } +} + +func TestSSHDefaultShellForPlatformIsPlatformAware(t *testing.T) { + // End-to-end contract: Darwin defaults to zsh, everything else to bash. + if got := SSHDefaultShellForPlatform(SSHRemotePlatform{GOOS: "darwin", GOARCH: "arm64"}); got != "zsh" { + t.Fatalf("darwin default shell = %q, want zsh", got) + } + if got := SSHDefaultShellForPlatform(SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"}); got != "bash" { + t.Fatalf("linux default shell = %q, want bash", got) + } + if got := SSHDefaultShellForPlatform(SSHRemotePlatform{}); got != "bash" { + t.Fatalf("unknown-platform default shell = %q, want bash", got) + } +} + +func TestExpandRemoteHome(t *testing.T) { + t.Run("rewrites tilde prefix", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut("/home/kandev\n")}, + ).handle) + client := server.dial(t) + + got, err := expandRemoteHome(context.Background(), client, "~/.kandev/bin/agentctl") + if err != nil { + t.Fatalf("expandRemoteHome: %v", err) + } + if got != "/home/kandev/.kandev/bin/agentctl" { + t.Fatalf("expanded = %q", got) + } + }) + + t.Run("bare tilde resolves to home", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut("/home/kandev")}, + ).handle) + got, err := expandRemoteHome(context.Background(), server.dial(t), "~") + if err != nil { + t.Fatalf("expandRemoteHome: %v", err) + } + if got != "/home/kandev" { + t.Fatalf("expanded = %q", got) + } + }) + + t.Run("absolute path issues no remote command", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + got, err := expandRemoteHome(context.Background(), server.dial(t), "/opt/agentctl") + if err != nil { + t.Fatalf("expandRemoteHome: %v", err) + } + if got != "/opt/agentctl" { + t.Fatalf("expanded = %q, want unchanged", got) + } + if cmds := server.commands(); len(cmds) != 0 { + t.Fatalf("absolute path must not probe the remote, got %v", cmds) + } + }) + + t.Run("empty remote home is an error", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut(" \n ") }) + _, err := expandRemoteHome(context.Background(), server.dial(t), "~/x") + if err == nil || !strings.Contains(err.Error(), "remote $HOME is empty") { + t.Fatalf("error = %v, want empty-$HOME error", err) + } + }) + + t.Run("probe failure is wrapped", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("boom") }) + _, err := expandRemoteHome(context.Background(), server.dial(t), "~/x") + if err == nil || !strings.Contains(err.Error(), "ssh: resolve $HOME") { + t.Fatalf("error = %v, want wrapped resolve error", err) + } + }) +} + +func TestRunSSHCommandStdinFeedsRemoteStdin(t *testing.T) { + server := newFakeSSHServer(t, func(_, stdin string) sshExecResult { + return sshExecResult{Stdout: "received:" + stdin} + }) + stdout, _, err := runSSHCommandStdin(context.Background(), server.dial(t), + "cat", strings.NewReader("SECRET='x y'\n")) + if err != nil { + t.Fatalf("runSSHCommandStdin: %v", err) + } + if stdout != "received:SECRET='x y'\n" { + t.Fatalf("stdout = %q", stdout) + } + if calls := server.execCalls(); len(calls) != 1 || calls[0].Stdin != "SECRET='x y'\n" { + t.Fatalf("recorded stdin = %+v", calls) + } +} + +func TestRunSSHCommandAbandonsARunningCommandOnCancellation(t *testing.T) { + // The remote command is held open across the cancellation for two reasons: + // it makes the select land on ctx.Done() deterministically (cancelling up + // front leaves both arms ready, and Go picks a ready arm at random), and it + // keeps x/crypto/ssh's stdout/stderr copier goroutines writing while the + // buffers are read — so under -race this doubles as the regression test for + // the syncBuffer guarding those buffers. + started := make(chan struct{}) + release := make(chan struct{}) + finished := make(chan struct{}) + var releaseOnce, startOnce sync.Once + releaseRemote := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(releaseRemote) + + server := newFakeSSHServer(t, func(string, string) sshExecResult { + startOnce.Do(func() { close(started) }) + <-release + close(finished) + return sshOut("output produced after the caller gave up") + }) + client := server.dial(t) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { + <-started + cancel() + }() + + _, _, err := runSSHCommand(ctx, client, "sleep 60") + + // Returning while the command is still running is the whole point: the + // caller must not be pinned to a remote command that outlives its context. + select { + case <-finished: + t.Fatal("runSSHCommand waited for the remote command instead of abandoning it") + default: + } + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } + + // Let the remote command finish and drain the server handler before the + // test returns, so goleak sees no stragglers. + releaseRemote() + <-finished +} + +func TestSyncBufferSnapshotsConcurrentWrites(t *testing.T) { + var buf syncBuffer + const writers, perWriter = 8, 50 + + var wg sync.WaitGroup + wg.Add(writers) + for range writers { + go func() { + defer wg.Done() + for range perWriter { + _, _ = buf.Write([]byte("x")) + } + }() + } + // Read concurrently with the writes; under -race this is the assertion. + for range 100 { + _ = buf.String() + } + wg.Wait() + + if got := len(buf.String()); got != writers*perWriter { + t.Fatalf("buffered bytes = %d, want %d", got, writers*perWriter) + } +} + +func TestRunSSHCommandSurfacesRemoteExitStatus(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { + return sshExecResult{Stdout: "partial", Stderr: "fatal: repo not found", ExitCode: 128} + }) + stdout, stderr, err := runSSHCommand(context.Background(), server.dial(t), "git clone x") + if err == nil { + t.Fatal("expected non-zero exit to be an error") + } + if stdout != "partial" || stderr != "fatal: repo not found" { + t.Fatalf("stdout/stderr = %q/%q, want both captured alongside the error", stdout, stderr) + } +} + +// writeFakeAgentctl writes a stand-in agentctl binary and points the resolver +// at it via the platform env var, returning the path and its hex sha256. +func writeFakeAgentctl(t *testing.T, platform SSHRemotePlatform, content string) (path, sha string) { + t.Helper() + path = filepath.Join(t.TempDir(), "agentctl-"+platform.GOOS+"-"+platform.GOARCH) + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatalf("write fake agentctl: %v", err) + } + sum := sha256.Sum256([]byte(content)) + envName := fmt.Sprintf("KANDEV_AGENTCTL_%s_%s_BINARY", + strings.ToUpper(platform.GOOS), strings.ToUpper(platform.GOARCH)) + t.Setenv(envName, path) + return path, hex.EncodeToString(sum[:]) +} + +func TestLocalAgentctlSha256(t *testing.T) { + platform := SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"} + path, wantSha := writeFakeAgentctl(t, platform, "agentctl-bytes") + resolver := NewAgentctlResolver(newTestLogger()) + + gotSha, data, gotPath, err := localAgentctlSha256(resolver, platform) + if err != nil { + t.Fatalf("localAgentctlSha256: %v", err) + } + if gotSha != wantSha { + t.Fatalf("sha = %q, want %q", gotSha, wantSha) + } + if string(data) != "agentctl-bytes" { + t.Fatalf("data = %q", data) + } + if gotPath != path { + t.Fatalf("path = %q, want %q", gotPath, path) + } + + t.Run("unresolvable platform bubbles up", func(t *testing.T) { + if _, _, _, err := localAgentctlSha256(resolver, SSHRemotePlatform{GOOS: "plan9"}); err == nil { + t.Fatal("expected unsupported-platform error") + } + }) +} + +func TestSSHCheckAgentctlCached(t *testing.T) { + platform := SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"} + _, sha := writeFakeAgentctl(t, platform, "agentctl-bytes") + resolver := NewAgentctlResolver(newTestLogger()) + + t.Run("matching remote sha reports cached", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut("/home/kandev")}, + sshScriptRule{match: "cat '/home/kandev/.kandev/bin/agentctl.sha256'", result: sshOut(sha + "\n")}, + ).handle) + cached, err := SSHCheckAgentctlCached(context.Background(), server.dial(t), resolver, platform) + if err != nil { + t.Fatalf("SSHCheckAgentctlCached: %v", err) + } + if !cached { + t.Fatal("matching sha must report cached") + } + }) + + t.Run("missing sidecar reports not cached", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut("/home/kandev")}, + sshScriptRule{match: "agentctl.sha256", result: sshOK}, + ).handle) + cached, err := SSHCheckAgentctlCached(context.Background(), server.dial(t), resolver, platform) + if err != nil { + t.Fatalf("SSHCheckAgentctlCached: %v", err) + } + if cached { + t.Fatal("absent sidecar must report not cached") + } + }) + + t.Run("an ssh-level sha256 read failure is an error, not 'needs upload'", func(t *testing.T) { + // Production appends `|| true`, so a missing file exits 0 (covered + // above). A non-zero exit here therefore means the SSH call itself + // failed, which must surface rather than be read as "not cached". + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + if strings.Contains(command, "agentctl.sha256") { + return sshFail("connection reset") + } + return sshOut("/home/kandev") + }) + if _, err := SSHCheckAgentctlCached(context.Background(), server.dial(t), resolver, platform); err == nil { + t.Fatal("expected transport failure to bubble up") + } + }) +} + +func TestEnsureAgentctlOnHostSkipsUploadWhenShaMatches(t *testing.T) { + platform := SSHRemotePlatform{GOOS: "linux", GOARCH: "arm64"} + _, sha := writeFakeAgentctl(t, platform, "cached-agentctl") + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut("/home/kandev")}, + sshScriptRule{match: "cat '/home/kandev/.kandev/bin/agentctl.sha256'", result: sshOut(sha + "\n")}, + sshScriptRule{match: "test -x '/home/kandev/.kandev/bin/agentctl'", result: sshOK}, + ).handle) + + remoteBin, err := ensureAgentctlOnHost(context.Background(), server.dial(t), + NewAgentctlResolver(newTestLogger()), platform, newTestLogger()) + if err != nil { + t.Fatalf("ensureAgentctlOnHost: %v", err) + } + if remoteBin != "/home/kandev/.kandev/bin/agentctl" { + t.Fatalf("remote binary = %q", remoteBin) + } + for _, command := range server.commands() { + if strings.Contains(command, "mkdir -p") { + t.Fatalf("cached agentctl must not re-create the bin dir, saw %q", command) + } + } +} + +func TestEnsureAgentctlOnHostUploadsWhenShaDiffers(t *testing.T) { + platform := SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"} + _, sha := writeFakeAgentctl(t, platform, "fresh-agentctl") + // The SFTP subsystem serves the real filesystem, so the "remote home" is a + // temp dir and the uploaded bytes are assertable on disk. + remoteHome := t.TempDir() + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut(remoteHome)}, + sshScriptRule{match: "agentctl.sha256", result: sshOut("stale-sha\n")}, + sshScriptRule{match: "mkdir -p", result: sshOK}, + sshScriptRule{match: "test -x", result: sshOK}, + ).handle) + server.enableSFTP() + if err := os.MkdirAll(filepath.Join(remoteHome, ".kandev", "bin"), 0o755); err != nil { + t.Fatalf("seed remote bin dir: %v", err) + } + + remoteBin, err := ensureAgentctlOnHost(context.Background(), server.dial(t), + NewAgentctlResolver(newTestLogger()), platform, newTestLogger()) + if err != nil { + t.Fatalf("ensureAgentctlOnHost: %v", err) + } + wantBin := filepath.Join(remoteHome, ".kandev", "bin", "agentctl") + if remoteBin != wantBin { + t.Fatalf("remote binary = %q, want %q", remoteBin, wantBin) + } + uploaded, err := os.ReadFile(wantBin) + if err != nil { + t.Fatalf("read uploaded agentctl: %v", err) + } + if string(uploaded) != "fresh-agentctl" { + t.Fatalf("uploaded bytes = %q", uploaded) + } + info, err := os.Stat(wantBin) + if err != nil { + t.Fatalf("stat uploaded agentctl: %v", err) + } + if info.Mode().Perm() != 0o755 { + t.Fatalf("uploaded mode = %v, want 0755", info.Mode().Perm()) + } + sidecar, err := os.ReadFile(wantBin + ".sha256") + if err != nil { + t.Fatalf("read sha sidecar: %v", err) + } + if string(sidecar) != sha+"\n" { + t.Fatalf("sidecar = %q, want %q", sidecar, sha+"\n") + } + if _, ok := server.lastCommandContaining("mkdir -p '" + filepath.Join(remoteHome, ".kandev", "bin") + "'"); !ok { + t.Fatalf("expected the bin dir to be created before upload, commands: %v", server.commands()) + } +} + +func TestEnsureAgentctlOnHostFailsWhenBinaryNotExecutable(t *testing.T) { + platform := SSHRemotePlatform{GOOS: "linux", GOARCH: "amd64"} + writeFakeAgentctl(t, platform, "fresh-agentctl") + remoteHome := t.TempDir() + if err := os.MkdirAll(filepath.Join(remoteHome, ".kandev", "bin"), 0o755); err != nil { + t.Fatalf("seed remote bin dir: %v", err) + } + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, remoteHomeCommand): + return sshOut(remoteHome) + case strings.Contains(command, "test -x"): + return sshFail("not executable") + default: + return sshOK + } + }) + server.enableSFTP() + + _, err := ensureAgentctlOnHost(context.Background(), server.dial(t), + NewAgentctlResolver(newTestLogger()), platform, newTestLogger()) + if err == nil || !strings.Contains(err.Error(), "not executable after upload") { + t.Fatalf("error = %v, want post-upload executability failure", err) + } +} + +func TestSftpUploadBytesWritesAtomically(t *testing.T) { + server := newFakeSSHServer(t, nil) + server.enableSFTP() + client := server.dial(t) + root := t.TempDir() + target := filepath.Join(root, "creds.json") + + if err := os.WriteFile(target, []byte("previous"), 0o600); err != nil { + t.Fatalf("seed target: %v", err) + } + if err := sftpUploadBytes(client, target, []byte("replacement"), 0o640); err != nil { + t.Fatalf("sftpUploadBytes: %v", err) + } + got, err := os.ReadFile(target) + if err != nil { + t.Fatalf("read target: %v", err) + } + if string(got) != "replacement" { + t.Fatalf("content = %q", got) + } + info, err := os.Stat(target) + if err != nil { + t.Fatalf("stat target: %v", err) + } + if info.Mode().Perm() != 0o640 { + t.Fatalf("mode = %v, want 0640", info.Mode().Perm()) + } + entries, err := os.ReadDir(root) + if err != nil { + t.Fatalf("read dir: %v", err) + } + if len(entries) != 1 { + t.Fatalf("temp files left behind: %d entries", len(entries)) + } +} + +func TestSftpUploadBytesFailsOnMissingParent(t *testing.T) { + server := newFakeSSHServer(t, nil) + server.enableSFTP() + target := filepath.Join(t.TempDir(), "absent", "file") + if err := sftpUploadBytes(server.dial(t), target, []byte("x"), 0o644); err == nil { + t.Fatal("expected create to fail when the parent directory is missing") + } +} + +func TestEnsureRemoteTaskDir(t *testing.T) { + t.Run("creates the dir under the expanded workdir root", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: remoteHomeCommand, result: sshOut("/home/kandev")}, + sshScriptRule{match: "mkdir -p", result: sshOK}, + ).handle) + taskDir, err := ensureRemoteTaskDir(context.Background(), server.dial(t), "~/.kandev", "task-42") + if err != nil { + t.Fatalf("ensureRemoteTaskDir: %v", err) + } + if taskDir != "/home/kandev/.kandev/tasks/task-42" { + t.Fatalf("taskDir = %q", taskDir) + } + call, ok := server.lastCommandContaining("mkdir -p") + if !ok || call.Command != "mkdir -p '/home/kandev/.kandev/tasks/task-42'" { + t.Fatalf("mkdir command = %q", call.Command) + } + }) + + t.Run("empty task dir name is rejected before dialing", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + _, err := ensureRemoteTaskDir(context.Background(), server.dial(t), "~/.kandev", "") + if err == nil || !strings.Contains(err.Error(), "task dir name is empty") { + t.Fatalf("error = %v", err) + } + if cmds := server.commands(); len(cmds) != 0 { + t.Fatalf("no remote command should run, got %v", cmds) + } + }) + + t.Run("mkdir failure is wrapped with the path", func(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + if strings.HasPrefix(command, "mkdir") { + return sshFail("permission denied") + } + return sshOut("/home/kandev") + }) + _, err := ensureRemoteTaskDir(context.Background(), server.dial(t), "~/.kandev", "task-42") + if err == nil || !strings.Contains(err.Error(), "/home/kandev/.kandev/tasks/task-42") { + t.Fatalf("error = %v, want the task dir path in the message", err) + } + }) +} + +func TestEnsureRemoteSessionDir(t *testing.T) { + t.Run("creates the per-session runtime dir", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: "mkdir -p", result: sshOK}, + ).handle) + sessionDir, err := ensureRemoteSessionDir(context.Background(), server.dial(t), "/remote/task", "sess-7") + if err != nil { + t.Fatalf("ensureRemoteSessionDir: %v", err) + } + if sessionDir != "/remote/task/.kandev/sessions/sess-7" { + t.Fatalf("sessionDir = %q", sessionDir) + } + if got := server.commands(); len(got) != 1 || + got[0] != "mkdir -p '/remote/task/.kandev/sessions/sess-7'" { + t.Fatalf("commands = %v", got) + } + }) + + t.Run("empty session ID is rejected", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + _, err := ensureRemoteSessionDir(context.Background(), server.dial(t), "/remote/task", "") + if err == nil || !strings.Contains(err.Error(), "session ID is empty") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("mkdir failure is wrapped", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("read-only fs") }) + _, err := ensureRemoteSessionDir(context.Background(), server.dial(t), "/remote/task", "sess-7") + if err == nil || !strings.Contains(err.Error(), "mkdir session dir") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestProbeRemoteBinary(t *testing.T) { + t.Run("resolves through a login shell and trims output", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: "command -v", result: sshOut("/usr/bin/npx\n")}, + ).handle) + got, err := ProbeRemoteBinary(context.Background(), server.dial(t), "zsh", "npx") + if err != nil { + t.Fatalf("ProbeRemoteBinary: %v", err) + } + if got != "/usr/bin/npx" { + t.Fatalf("resolved path = %q", got) + } + want := `zsh -lc 'command -v '\''npx'\'' 2>/dev/null || true'` + if cmds := server.commands(); len(cmds) != 1 || cmds[0] != want { + t.Fatalf("probe command = %v, want %q", cmds, want) + } + }) + + t.Run("missing binary is an empty string, not an error", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("\n") }) + got, err := ProbeRemoteBinary(context.Background(), server.dial(t), "", "npx") + if err != nil { + t.Fatalf("ProbeRemoteBinary: %v", err) + } + if got != "" { + t.Fatalf("resolved path = %q, want empty", got) + } + }) + + t.Run("transport failure is an error", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("closed") }) + if _, err := ProbeRemoteBinary(context.Background(), server.dial(t), "bash", "npx"); err == nil { + t.Fatal("expected transport error") + } + }) +} + +func TestIsRemoteAgentctlAlive(t *testing.T) { + t.Run("live pid", func(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: "kill -0 4242", result: sshOK}, + ).handle) + if !isRemoteAgentctlAlive(context.Background(), server.dial(t), 4242) { + t.Fatal("expected pid to be reported alive") + } + }) + + t.Run("dead pid", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("no such process") }) + if isRemoteAgentctlAlive(context.Background(), server.dial(t), 4242) { + t.Fatal("failed kill -0 must report not alive") + } + }) + + t.Run("non-positive pid short-circuits", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + client := server.dial(t) + if isRemoteAgentctlAlive(context.Background(), client, 0) || + isRemoteAgentctlAlive(context.Background(), client, -1) { + t.Fatal("non-positive pids must report not alive") + } + if cmds := server.commands(); len(cmds) != 0 { + t.Fatalf("no remote command expected, got %v", cmds) + } + }) +} + +func TestStopRemoteAgentctlSendsTheStopScript(t *testing.T) { + server := newFakeSSHServer(t, newSSHScriptedHandler(t, + sshScriptRule{match: "rm -rf", result: sshOK}, + ).handle) + if err := stopRemoteAgentctl(context.Background(), server.dial(t), "/remote/session", 4242); err != nil { + t.Fatalf("stopRemoteAgentctl: %v", err) + } + cmds := server.commands() + if len(cmds) != 1 || cmds[0] != remoteAgentctlStopCommand("/remote/session", 4242) { + t.Fatalf("stop command = %v", cmds) + } +} + +func TestRemoteAgentctlStopCommandWithoutPidOnlyRemovesSessionDir(t *testing.T) { + got := remoteAgentctlStopCommand("/remote/session", 0) + if got != "rm -rf '/remote/session'" { + t.Fatalf("stop command = %q, want session-dir removal only", got) + } + if strings.Contains(got, "kill") { + t.Fatalf("no pid must mean no kill: %q", got) + } +} + +func TestStartRemoteAgentctlOnPortBuildsLaunchScript(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "nohup"): + return sshOut("4242\n") + case strings.Contains(command, "cat "): + return sshOut("... HTTP server bound successfully on 127.0.0.1:45000\n") + default: + return sshOK + } + }) + + pid, err := startRemoteAgentctlOnPort(context.Background(), server.dial(t), + "zsh", "/home/kandev/.kandev/bin/agentctl", "/remote/task", "/remote/session", + "GITHUB_TOKEN='tok'\n", 45000, newTestLogger()) + if err != nil { + t.Fatalf("startRemoteAgentctlOnPort: %v", err) + } + if pid != 4242 { + t.Fatalf("pid = %d, want 4242", pid) + } + + launch, ok := server.lastCommandContaining("nohup") + if !ok { + t.Fatalf("no launch command recorded: %v", server.commands()) + } + script := unwrapLoginShell(t, "zsh", launch.Command) + for _, want := range []string{ + "AGENTCTL_PORT=45000 nohup '/home/kandev/.kandev/bin/agentctl' --workdir '/remote/task'", + ">> '/remote/session'/agentctl.log 2>&1 < /dev/null &", + `echo "$AGENTCTL_PID" > '/remote/session'/agentctl.pid`, + "mkdir -p '/remote/session'", + ". /dev/stdin", + } { + if !strings.Contains(script, want) { + t.Fatalf("launch script missing %q:\n%s", want, script) + } + } + if launch.Stdin != "GITHUB_TOKEN='tok'\n" { + t.Fatalf("launch stdin = %q, want the env init script (never argv)", launch.Stdin) + } + if strings.Contains(launch.Command, "GITHUB_TOKEN") { + t.Fatalf("secrets must never reach the remote argv:\n%s", launch.Command) + } +} + +func TestStartRemoteAgentctlOnPortDetectsBindCollision(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + if strings.Contains(command, "nohup") { + return sshOut("4242\n") + } + return sshOut("HTTP server failed to bind: address already in use\n") + }) + + _, err := startRemoteAgentctlOnPort(context.Background(), server.dial(t), + "bash", "/bin/agentctl", "/remote/task", "/remote/session", "", 45000, newTestLogger()) + if !errors.Is(err, errSSHAgentctlPortInUse) { + t.Fatalf("error = %v, want errSSHAgentctlPortInUse", err) + } + if !strings.Contains(err.Error(), "address already in use") { + t.Fatalf("error must carry the remote log tail: %v", err) + } +} + +func TestStartRemoteAgentctlOnPortFailsWhenProcessDies(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "nohup"): + return sshOut("4242\n") + case strings.Contains(command, "kill -0"): + return sshFail("no such process") + default: + return sshOut("panic: bad config\n") + } + }) + + _, err := startRemoteAgentctlOnPort(context.Background(), server.dial(t), + "bash", "/bin/agentctl", "/remote/task", "/remote/session", "", 45000, newTestLogger()) + if err == nil || !strings.Contains(err.Error(), "exited before becoming ready") { + t.Fatalf("error = %v, want early-exit detection", err) + } + if !strings.Contains(err.Error(), "panic: bad config") { + t.Fatalf("error must include the log tail: %v", err) + } +} + +func TestStartRemoteAgentctlOnPortRejectsNonNumericPid(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("not-a-pid\n") }) + _, err := startRemoteAgentctlOnPort(context.Background(), server.dial(t), + "bash", "/bin/agentctl", "/remote/task", "/remote/session", "", 45000, newTestLogger()) + if err == nil || !strings.Contains(err.Error(), "non-numeric pid") { + t.Fatalf("error = %v, want non-numeric pid rejection", err) + } +} + +func TestStartRemoteAgentctlOnPortWrapsLaunchFailure(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { + return sshExecResult{Stderr: "zsh: command not found", ExitCode: 127} + }) + _, err := startRemoteAgentctlOnPort(context.Background(), server.dial(t), + "zsh", "/bin/agentctl", "/remote/task", "/remote/session", "", 45000, newTestLogger()) + if err == nil || !strings.Contains(err.Error(), "ssh: launch agentctl") { + t.Fatalf("error = %v, want wrapped launch failure", err) + } + if !strings.Contains(err.Error(), "zsh: command not found") { + t.Fatalf("error must carry remote stderr: %v", err) + } +} + +func TestPickRemoteAgentctlPortStaysInRange(t *testing.T) { + for range 200 { + port := pickRemoteAgentctlPort() + if port < 40000 || port >= 60000 { + t.Fatalf("pickRemoteAgentctlPort() = %d, want [40000, 60000)", port) + } + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_scripts_remote_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_scripts_remote_test.go new file mode 100644 index 0000000000..ccef3c5597 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_scripts_remote_test.go @@ -0,0 +1,569 @@ +package lifecycle + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/kandev/kandev/internal/agent/agents" +) + +func TestCloneAndMergeSSHMetadata(t *testing.T) { + t.Run("clone is a copy, not an alias", func(t *testing.T) { + if got := cloneSSHMetadata(nil); got != nil { + t.Fatalf("cloneSSHMetadata(nil) = %+v, want nil", got) + } + if got := cloneSSHMetadata(map[string]interface{}{}); got != nil { + t.Fatalf("cloneSSHMetadata(empty) = %+v, want nil", got) + } + source := map[string]interface{}{"a": 1} + clone := cloneSSHMetadata(source) + clone["a"] = 2 + if source["a"] != 1 { + t.Fatal("clone must not alias the source map") + } + }) + + t.Run("merge overlays the instance metadata on the launch metadata", func(t *testing.T) { + merged := mergeSSHMetadata( + map[string]interface{}{"a": "launch", "b": "launch"}, + map[string]interface{}{"b": "instance", "c": "instance"}, + ) + if merged["a"] != "launch" || merged["b"] != "instance" || merged["c"] != "instance" { + t.Fatalf("merged = %+v", merged) + } + }) + + t.Run("merge with no base still returns a usable map", func(t *testing.T) { + merged := mergeSSHMetadata(nil, map[string]interface{}{"c": "instance"}) + if merged["c"] != "instance" { + t.Fatalf("merged = %+v", merged) + } + }) +} + +func TestNormalizeSSHRemotePath(t *testing.T) { + cases := []struct{ in, want string }{ + {in: " /remote/task ", want: "/remote/task"}, + {in: "/remote/task/", want: "/remote/task"}, + {in: "/remote/task///", want: "/remote/task"}, + {in: "/", want: "/"}, + {in: "///", want: "/"}, + {in: " ", want: ""}, + {in: "", want: ""}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + if got := normalizeSSHRemotePath(c.in); got != c.want { + t.Fatalf("normalizeSSHRemotePath(%q) = %q, want %q", c.in, got, c.want) + } + }) + } +} + +func TestSSHScriptWithEnvironment(t *testing.T) { + got := sshScriptWithEnvironment("echo hi") + if got != "set -ae\n. /dev/stdin\nset +a\nset -e\necho hi" { + t.Fatalf("sshScriptWithEnvironment = %q", got) + } +} + +func TestRedactSSHScriptOutput(t *testing.T) { + t.Run("every non-empty env value is redacted", func(t *testing.T) { + got := redactSSHScriptOutput( + "cloning with ghp-secret and token sk-secret\n", + map[string]string{"GITHUB_TOKEN": "ghp-secret", "OPENAI_API_KEY": "sk-secret", "EMPTY": ""}, + ) + if strings.Contains(got, "ghp-secret") || strings.Contains(got, "sk-secret") { + t.Fatalf("secrets survived redaction: %q", got) + } + if strings.Count(got, "") != 2 { + t.Fatalf("redacted output = %q, want both values replaced", got) + } + }) + + t.Run("output is tail-trimmed to the max line count", func(t *testing.T) { + var lines []string + for i := range 40 { + lines = append(lines, string(rune('a'+i%26))+"-line") + } + got := redactSSHScriptOutput(strings.Join(lines, "\n"), nil) + if count := strings.Count(got, "\n") + 1; count != sshPrepareOutputMaxLines { + t.Fatalf("kept %d lines, want %d", count, sshPrepareOutputMaxLines) + } + if !strings.HasSuffix(got, lines[len(lines)-1]) { + t.Fatalf("tail line missing from %q", got) + } + }) +} + +func TestSSHCommandFailureDetail(t *testing.T) { + cases := []struct { + name string + stdout, stderr string + err error + want string + }{ + {name: "stdout and stderr joined", stdout: "out", stderr: "err", want: "out\nerr"}, + {name: "error appended on its own line", stdout: "out", stderr: "", err: errStub, want: "out\nstub failure"}, + {name: "error alone", err: errStub, want: "stub failure"}, + {name: "nothing at all", want: ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := sshCommandFailureDetail(c.stdout, c.stderr, c.err); got != c.want { + t.Fatalf("sshCommandFailureDetail = %q, want %q", got, c.want) + } + }) + } +} + +func TestSSHExecutorCollectAgentInstallScripts(t *testing.T) { + claude := &fakeIdentityAgent{MockAgent: agents.NewMockAgent(), id: "claude-code", installScript: "npm i -g claude"} + codex := &fakeIdentityAgent{MockAgent: agents.NewMockAgent(), id: "codex", installScript: "pip install codex"} + silent := &fakeIdentityAgent{MockAgent: agents.NewMockAgent(), id: "silent", installScript: " "} + lister := &mockAgentLister{agents: []agents.Agent{claude, codex, silent}} + + t.Run("no agent list means no scripts", func(t *testing.T) { + exec := &SSHExecutor{logger: newTestLogger()} + if got := exec.collectAgentInstallScripts(&ExecutorCreateRequest{}); got != nil { + t.Fatalf("scripts = %v, want nil", got) + } + exec.agentList = lister + if got := exec.collectAgentInstallScripts(nil); got != nil { + t.Fatalf("scripts for a nil request = %v, want nil", got) + } + }) + + t.Run("the task agent contributes its script", func(t *testing.T) { + exec := &SSHExecutor{logger: newTestLogger(), agentList: lister} + got := exec.collectAgentInstallScripts(&ExecutorCreateRequest{AgentConfig: claude}) + if len(got) != 1 || got[0] != "npm i -g claude" { + t.Fatalf("scripts = %v", got) + } + }) + + t.Run("agents referenced by selected credentials are included", func(t *testing.T) { + exec := &SSHExecutor{logger: newTestLogger(), agentList: lister} + got := exec.collectAgentInstallScripts(&ExecutorCreateRequest{ + Metadata: map[string]interface{}{ + "remote_credentials": `["agent:codex:files:0"]`, + "remote_auth_secrets": `{"agent:claude-code:env:ANTHROPIC_API_KEY":"secret-1"}`, + }, + }) + if len(got) != 2 { + t.Fatalf("scripts = %v, want both agents", got) + } + }) + + t.Run("blank install scripts and malformed metadata are skipped", func(t *testing.T) { + exec := &SSHExecutor{logger: newTestLogger(), agentList: lister} + got := exec.collectAgentInstallScripts(&ExecutorCreateRequest{ + AgentConfig: silent, + Metadata: map[string]interface{}{ + "remote_credentials": `not json`, + "remote_auth_secrets": `also not json`, + }, + }) + if len(got) != 0 { + t.Fatalf("scripts = %v, want none", got) + } + }) +} + +func TestAddMethodAgentIDs(t *testing.T) { + ids := map[string]bool{} + addMethodAgentIDs(ids, map[string]interface{}{"key": `["agent:codex:files:0","system:other"]`}, "key", + func(raw string) []string { + if raw == `["agent:codex:files:0","system:other"]` { + return []string{"agent:codex:files:0", "system:other"} + } + return nil + }) + if !ids["codex"] { + t.Fatalf("ids = %+v, want codex", ids) + } + if len(ids) != 1 { + t.Fatalf("ids = %+v, want only agent-prefixed IDs", ids) + } +} + +func TestSSHExecutorRunPrepareScript(t *testing.T) { + baseMetadata := func() map[string]interface{} { + return map[string]interface{}{ + MetadataKeySetupScript: "echo prepared", + MetadataKeyBaseBranch: "main", + MetadataKeyWorktreeBranch: "feature/task-1", + } + } + + t.Run("runs the resolved script under the platform shell with env on stdin", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { + return sshExecResult{Stdout: "prepared\n"} + }) + exec := &SSHExecutor{logger: newTestLogger()} + var steps []PrepareStep + req := &ExecutorCreateRequest{ + TaskID: "task-1", + Metadata: baseMetadata(), + Env: map[string]string{envKeyGitHubToken: "ghp-s3cr3t"}, + OnProgress: func(step PrepareStep, _, _ int) { + steps = append(steps, step) + }, + } + + err := exec.runPrepareScript(context.Background(), server.dial(t), "/remote/task", req, + SSHRemotePlatform{GOOS: "darwin", GOARCH: "arm64"}, "/remote/bin/agentctl") + if err != nil { + t.Fatalf("runPrepareScript: %v", err) + } + + calls := server.execCalls() + if len(calls) != 1 { + t.Fatalf("prepare runs = %d, want 1", len(calls)) + } + script := unwrapLoginShell(t, "zsh", calls[0].Command) + if !strings.HasPrefix(script, "set -ae\n. /dev/stdin\nset +a\nset -e\n") { + t.Fatalf("prepare script preamble missing:\n%s", script) + } + if !strings.Contains(script, "echo prepared") { + t.Fatalf("prepare script body missing:\n%s", script) + } + if !strings.Contains(calls[0].Stdin, "ghp-s3cr3t") { + t.Fatalf("prepare stdin = %q, want the resolved env", calls[0].Stdin) + } + if strings.Contains(calls[0].Command, "ghp-s3cr3t") { + t.Fatalf("prepare argv leaked a secret:\n%s", calls[0].Command) + } + if len(steps) != 2 || steps[0].Status != PrepareStepRunning || steps[1].Status != PrepareStepCompleted { + t.Fatalf("progress steps = %+v, want running then completed", steps) + } + }) + + t.Run("an unresolvable placeholder aborts before any SSH work", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.runPrepareScript(context.Background(), server.dial(t), "/remote/task", + &ExecutorCreateRequest{Metadata: map[string]interface{}{ + MetadataKeySetupScript: "echo {{unknown.placeholder}}", + }}, SSHRemotePlatform{}, "") + if err == nil || !strings.Contains(err.Error(), "unresolved prepare script placeholder") { + t.Fatalf("error = %v", err) + } + if len(server.commands()) != 0 { + t.Fatalf("no remote command expected, got %v", server.commands()) + } + }) + + t.Run("a failing script reports redacted output and a generic error", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { + return sshExecResult{Stdout: "cloning with ghp-s3cr3t", Stderr: "fatal: auth failed", ExitCode: 1} + }) + exec := &SSHExecutor{logger: newTestLogger()} + var failed []PrepareStep + req := &ExecutorCreateRequest{ + TaskID: "task-1", + Metadata: baseMetadata(), + Env: map[string]string{envKeyGitHubToken: "ghp-s3cr3t"}, + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepFailed { + failed = append(failed, step) + } + }, + } + err := exec.runPrepareScript(context.Background(), server.dial(t), "/remote/task", req, + SSHRemotePlatform{}, "") + if err == nil || err.Error() != "ssh: prepare script failed" { + t.Fatalf("error = %v, want the generic prepare failure", err) + } + if len(failed) != 1 { + t.Fatalf("failed steps = %+v", failed) + } + if strings.Contains(failed[0].Output, "ghp-s3cr3t") { + t.Fatalf("failure output leaked a secret: %q", failed[0].Output) + } + if !strings.Contains(failed[0].Output, "") || + !strings.Contains(failed[0].Output, "fatal: auth failed") { + t.Fatalf("failure output = %q, want redacted stdout plus stderr", failed[0].Output) + } + }) + + t.Run("an invalid env key aborts before running anything remote", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := &SSHExecutor{logger: newTestLogger()} + req := &ExecutorCreateRequest{ + TaskID: "task-1", + Metadata: baseMetadata(), + Env: map[string]string{"BAD KEY": "x"}, + ApprovedSecretEnvKeys: []string{"BAD KEY"}, + } + // sshRemoteContributionEnv is what feeds buildSSHEnvInitScript here; if + // it ever forwards an invalid identifier the launch must fail loudly + // rather than shipping a broken script. + if err := exec.runPrepareScript(context.Background(), server.dial(t), "/remote/task", req, + SSHRemotePlatform{}, ""); err != nil { + if !strings.Contains(err.Error(), "prepare script environment") { + t.Fatalf("error = %v", err) + } + return + } + if len(server.commands()) != 1 { + t.Fatalf("commands = %v, want the prepare script to have run once", server.commands()) + } + }) +} + +func TestSSHExecutorVerifyPrimaryCheckout(t *testing.T) { + const cloneURL = "https://github.com/acme/widgets.git" + + t.Run("no configured repository skips verification", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.verifyPrimaryCheckout(context.Background(), server.dial(t), "/remote/task", + &ExecutorCreateRequest{Metadata: map[string]interface{}{}}, SSHRemotePlatform{}) + if err != nil { + t.Fatalf("verifyPrimaryCheckout: %v", err) + } + if len(server.commands()) != 0 { + t.Fatalf("no remote command expected, got %v", server.commands()) + } + }) + + t.Run("a matching checkout passes and reports the canonical dir", func(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "pwd -P"): + return sshOut("/remote/task\n") + case strings.Contains(command, "rev-parse --show-toplevel"): + return sshOut("/remote/task/\n") + case strings.Contains(command, "config --get remote.origin.url"): + return sshOut(cloneURL + "\n") + case strings.Contains(command, "rev-parse --verify HEAD"): + return sshOut("abc123\n") + default: + return sshOK + } + }) + exec := &SSHExecutor{logger: newTestLogger()} + var completed []PrepareStep + err := exec.verifyPrimaryCheckout(context.Background(), server.dial(t), "/remote/task", + &ExecutorCreateRequest{ + Metadata: map[string]interface{}{"repository_clone_url": cloneURL}, + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepCompleted { + completed = append(completed, step) + } + }, + }, SSHRemotePlatform{}) + if err != nil { + t.Fatalf("verifyPrimaryCheckout: %v", err) + } + if len(completed) != 1 || completed[0].Output != "/remote/task" { + t.Fatalf("completed step = %+v", completed) + } + }) + + tests := []struct { + name string + handler sshExecHandler + wantErr string + wantMsg string + }{ + { + name: "missing task dir", + handler: func(command, _ string) sshExecResult { + if strings.Contains(command, "pwd -P") { + return sshFail("cd: no such directory") + } + return sshOK + }, + wantErr: "primary repository checkout is missing", + wantMsg: "cd: no such directory", + }, + { + name: "not a git checkout", + handler: func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "pwd -P"): + return sshOut("/remote/task\n") + case strings.Contains(command, "rev-parse --show-toplevel"): + return sshFail("fatal: not a git repository") + default: + return sshOK + } + }, + wantErr: "primary repository checkout is missing", + wantMsg: "fatal: not a git repository", + }, + { + name: "checkout root is a nested directory", + handler: func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "pwd -P"): + return sshOut("/remote/task\n") + case strings.Contains(command, "rev-parse --show-toplevel"): + return sshOut("/remote\n") + default: + return sshOK + } + }, + wantErr: "primary repository checkout is missing", + wantMsg: `does not match task directory`, + }, + { + name: "origin points somewhere else", + handler: func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "pwd -P"): + return sshOut("/remote/task\n") + case strings.Contains(command, "rev-parse --show-toplevel"): + return sshOut("/remote/task\n") + case strings.Contains(command, "config --get remote.origin.url"): + return sshOut("https://github.com/acme/other.git\n") + default: + return sshOK + } + }, + wantErr: "primary repository origin does not match configured repository", + }, + { + name: "no commit checked out", + handler: func(command, _ string) sshExecResult { + switch { + case strings.Contains(command, "pwd -P"): + return sshOut("/remote/task\n") + case strings.Contains(command, "rev-parse --show-toplevel"): + return sshOut("/remote/task\n") + case strings.Contains(command, "config --get remote.origin.url"): + return sshOut(cloneURL + "\n") + case strings.Contains(command, "rev-parse --verify HEAD"): + return sshFail("fatal: ambiguous argument 'HEAD'") + default: + return sshOK + } + }, + wantErr: "primary repository has no checkout", + wantMsg: "ambiguous argument", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + server := newFakeSSHServer(t, tc.handler) + exec := &SSHExecutor{logger: newTestLogger()} + var failed []PrepareStep + err := exec.verifyPrimaryCheckout(context.Background(), server.dial(t), "/remote/task", + &ExecutorCreateRequest{ + Metadata: map[string]interface{}{"repository_clone_url": cloneURL}, + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepFailed { + failed = append(failed, step) + } + }, + }, SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("error = %v, want %q", err, tc.wantErr) + } + if len(failed) != 1 || failed[0].Name != "Verifying primary checkout" { + t.Fatalf("failed steps = %+v", failed) + } + if tc.wantMsg != "" && !strings.Contains(failed[0].Output, tc.wantMsg) { + t.Fatalf("failure detail = %q, want %q", failed[0].Output, tc.wantMsg) + } + }) + } +} + +func TestSSHExecutorRunCleanupScript(t *testing.T) { + t.Run("no cleanup script is a no-op", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.runCleanupScript(context.Background(), server.dial(t), "/remote/task", + map[string]interface{}{}, nil, SSHRemotePlatform{}, "instance-1", StopReasonTaskDeleted) + if err != nil { + t.Fatalf("runCleanupScript: %v", err) + } + if len(server.commands()) != 0 { + t.Fatalf("no remote command expected, got %v", server.commands()) + } + }) + + t.Run("a missing task directory is an error", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.runCleanupScript(context.Background(), server.dial(t), "", + map[string]interface{}{MetadataKeyCleanupScript: "echo bye"}, nil, + SSHRemotePlatform{}, "instance-1", StopReasonTaskDeleted) + if err == nil || !strings.Contains(err.Error(), "cleanup task directory is missing") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("resolves the workspace placeholder and runs under the platform shell", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.runCleanupScript(context.Background(), server.dial(t), "/remote/task", + map[string]interface{}{MetadataKeyCleanupScript: "rm -rf {{workspace.path}}/build"}, + map[string]string{envKeyGitHubToken: "ghp-s3cr3t"}, + SSHRemotePlatform{GOOS: "darwin", GOARCH: "arm64"}, "instance-1", StopReasonTaskDeleted) + if err != nil { + t.Fatalf("runCleanupScript: %v", err) + } + calls := server.execCalls() + if len(calls) != 1 { + t.Fatalf("cleanup runs = %d, want 1", len(calls)) + } + script := unwrapLoginShell(t, "zsh", calls[0].Command) + if !strings.Contains(script, "rm -rf '/remote/task'/build") { + t.Fatalf("cleanup script = %q, want the resolved workspace path", script) + } + if calls[0].Stdin != "GITHUB_TOKEN='ghp-s3cr3t'\n" { + t.Fatalf("cleanup stdin = %q", calls[0].Stdin) + } + }) + + t.Run("a failing cleanup returns a generic error", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("rm: permission denied") }) + exec := &SSHExecutor{logger: newTestLogger()} + err := exec.runCleanupScript(context.Background(), server.dial(t), "/remote/task", + map[string]interface{}{MetadataKeyCleanupScript: "rm -rf {{workspace.path}}/build"}, nil, + SSHRemotePlatform{}, "instance-1", StopReasonTaskDeleted) + if err == nil || err.Error() != "ssh: cleanup script failed" { + t.Fatalf("error = %v", err) + } + }) + + t.Run("cleanup survives a cancelled parent context", func(t *testing.T) { + // StopInstance's context is often already cancelled by the time + // cleanup runs, so runCleanupScript must detach from it. + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := &SSHExecutor{logger: newTestLogger()} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := exec.runCleanupScript(ctx, server.dial(t), "/remote/task", + map[string]interface{}{MetadataKeyCleanupScript: "echo bye"}, nil, + SSHRemotePlatform{}, "instance-1", StopReasonTaskDeleted) + if err != nil { + t.Fatalf("runCleanupScript with a cancelled parent: %v", err) + } + if len(server.commands()) != 1 { + t.Fatalf("cleanup must still run, commands: %v", server.commands()) + } + }) +} + +func TestSSHExecutorResolvePrepareScriptRequiresACloneURLForContributions(t *testing.T) { + exec := &SSHExecutor{logger: newTestLogger()} + _, err := exec.resolvePrepareScript(nil, "/remote/task", "") + if err == nil || !strings.Contains(err.Error(), "prepare request is required") { + t.Fatalf("error = %v", err) + } + + _, err = exec.resolveSSHScript(nil, "/remote/task", "", "echo hi") + if err == nil || !strings.Contains(err.Error(), "script request is required") { + t.Fatalf("error = %v", err) + } +} + +// errStub is a fixed error used by the failure-detail table. +var errStub = errors.New("stub failure") diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_stop_resume_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_stop_resume_test.go new file mode 100644 index 0000000000..221c2dacc0 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_ssh_stop_resume_test.go @@ -0,0 +1,653 @@ +package lifecycle + +import ( + "context" + "errors" + "strconv" + "strings" + "testing" + + "golang.org/x/crypto/ssh" + + "github.com/kandev/kandev/internal/githubauth" +) + +// Tests for the SSH executor's status, stop, resume and per-launch preflight +// paths. + +func TestSSHExecutorGetRemoteStatus(t *testing.T) { + t.Run("nil instance is an error", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + if _, err := exec.GetRemoteStatus(context.Background(), nil); err == nil { + t.Fatal("expected an error for a nil instance") + } + }) + + t.Run("untracked instance reports unknown", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + status, err := exec.GetRemoteStatus(context.Background(), &ExecutorInstance{InstanceID: "gone"}) + if err != nil { + t.Fatalf("GetRemoteStatus: %v", err) + } + if status.State != sshStatusUnknown { + t.Fatalf("State = %q, want %q", status.State, sshStatusUnknown) + } + if status.ErrorMessage == "" { + t.Fatal("unknown state must explain why") + } + }) + + t.Run("tracked session with no client reports disconnected", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.sessions["i"] = &sshSessionState{ + target: &SSHTarget{Host: "build.example", User: "deploy", PinnedFingerprint: "SHA256:x"}, + pid: 77, + } + status, err := exec.GetRemoteStatus(context.Background(), &ExecutorInstance{InstanceID: "i"}) + if err != nil { + t.Fatalf("GetRemoteStatus: %v", err) + } + if status.State != sshStatusDisconnected { + t.Fatalf("State = %q", status.State) + } + if status.RemoteName != "build.example" { + t.Fatalf("RemoteName = %q", status.RemoteName) + } + if status.Details["pid"] != 77 || status.Details["user"] != "deploy" || + status.Details["fingerprint"] != "SHA256:x" { + t.Fatalf("Details = %+v", status.Details) + } + }) + + t.Run("live agentctl reports running, dead one reports agentctl-down", func(t *testing.T) { + alive := true + server := newFakeSSHServer(t, func(string, string) sshExecResult { + if alive { + return sshOK + } + return sshFail("no such process") + }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.sessions["i"] = &sshSessionState{ + target: &SSHTarget{Host: "build.example"}, + client: server.dial(t), + pid: 4242, + } + status, err := exec.GetRemoteStatus(context.Background(), &ExecutorInstance{InstanceID: "i"}) + if err != nil { + t.Fatalf("GetRemoteStatus: %v", err) + } + if status.State != sshStatusRunning { + t.Fatalf("State = %q, want %q", status.State, sshStatusRunning) + } + + alive = false + status, err = exec.GetRemoteStatus(context.Background(), &ExecutorInstance{InstanceID: "i"}) + if err != nil { + t.Fatalf("GetRemoteStatus: %v", err) + } + if status.State != sshStatusAgentctlDown { + t.Fatalf("State = %q, want %q", status.State, sshStatusAgentctlDown) + } + }) +} + +func TestSSHShouldStopRemoteAgentctl(t *testing.T) { + tests := []struct { + name string + instance *ExecutorInstance + force bool + want bool + }{ + {name: "nil instance", instance: nil, want: false}, + {name: "backend shutdown preserves agentctl", + instance: &ExecutorInstance{StopReason: StopReasonBackendShutdown}, want: false}, + {name: "forced backend shutdown still stops", + instance: &ExecutorInstance{StopReason: StopReasonBackendShutdown}, force: true, want: true}, + {name: "failed agent stop escalates", + instance: &ExecutorInstance{StopReason: StopReasonBackendShutdown, AgentStopFailed: true}, want: true}, + {name: "user stop kills agentctl", + instance: &ExecutorInstance{StopReason: "stopped via API"}, want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sshShouldStopRemoteAgentctl(tc.instance, tc.force); got != tc.want { + t.Fatalf("sshShouldStopRemoteAgentctl = %v, want %v", got, tc.want) + } + }) + } +} + +func TestSSHExecutorStopInstancePreservesAgentctlOnBackendShutdown(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + stopCalls := 0 + closeCalls := 0 + exec.stopRemote = func(context.Context, *ssh.Client, string, int) error { stopCalls++; return nil } + exec.closeClient = func(*ssh.Client) error { closeCalls++; return nil } + exec.sessions["i"] = &sshSessionState{client: &ssh.Client{}, remoteDir: "/remote/session", pid: 4242} + + err := exec.StopInstance(context.Background(), &ExecutorInstance{ + InstanceID: "i", + StopReason: StopReasonBackendShutdown, + }, false) + if err != nil { + t.Fatalf("StopInstance: %v", err) + } + if stopCalls != 0 { + t.Fatalf("remote stop calls = %d, want 0 so a restart can re-attach", stopCalls) + } + if closeCalls != 1 { + t.Fatalf("client close calls = %d, want 1", closeCalls) + } +} + +func TestSSHExecutorStopInstanceIsSafeWithoutTrackedState(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + if err := exec.StopInstance(context.Background(), nil, false); err != nil { + t.Fatalf("StopInstance(nil): %v", err) + } + if err := exec.StopInstance(context.Background(), &ExecutorInstance{InstanceID: "gone"}, false); err != nil { + t.Fatalf("StopInstance(untracked): %v", err) + } +} + +func TestSSHExecutorStopInstanceFallsBackToTheRealStopRemote(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.stopRemote = nil + exec.sessions["i"] = &sshSessionState{ + client: server.dial(t), + remoteDir: "/remote/session", + pid: 4242, + } + err := exec.StopInstance(context.Background(), &ExecutorInstance{ + InstanceID: "i", + StopReason: "stopped via API", + }, false) + if err != nil { + t.Fatalf("StopInstance: %v", err) + } + if _, ok := server.lastCommandContaining("kill 4242"); !ok { + t.Fatalf("expected the default stopRemoteAgentctl to run, commands: %v", server.commands()) + } +} + +func TestSSHExecutorResumeRemoteInstance(t *testing.T) { + t.Run("incomplete metadata is not a resume", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{Metadata: map[string]interface{}{ + MetadataKeySSHRemoteAgentctlPID: "4242", + }} + if err := exec.ResumeRemoteInstance(context.Background(), req); err != nil { + t.Fatalf("ResumeRemoteInstance = %v, want nil so a fresh create proceeds", err) + } + }) + + t.Run("missing auth token is rejected", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{Metadata: map[string]interface{}{ + MetadataKeySSHRemoteAgentctlPID: "4242", + MetadataKeySSHRemoteAgentctlPort: "41234", + MetadataKeySSHRemoteSessionDir: "/remote/session", + MetadataKeySSHRemoteTaskDir: "/remote/task", + }} + err := exec.ResumeRemoteInstance(context.Background(), req) + if err == nil || !strings.Contains(err.Error(), "missing agentctl auth token") { + t.Fatalf("error = %v", err) + } + }) + + t.Run("already-tracked instance short-circuits", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.sessions["instance-1"] = &sshSessionState{authToken: "tok"} + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + AuthToken: "tok", + Metadata: map[string]interface{}{ + MetadataKeySSHRemoteAgentctlPID: "4242", + MetadataKeySSHRemoteAgentctlPort: "41234", + MetadataKeySSHRemoteSessionDir: "/remote/session", + MetadataKeySSHRemoteTaskDir: "/remote/task", + }, + } + if err := exec.ResumeRemoteInstance(context.Background(), req); err != nil { + t.Fatalf("ResumeRemoteInstance: %v", err) + } + if _, set := req.Metadata[MetadataKeySSHLocalForwardPort]; set { + t.Fatal("an idempotent resume must not rewrite the forward port") + } + }) + + t.Run("re-attaches to a live remote agentctl", func(t *testing.T) { + harness := newSSHLaunchHarness(t, "4242") + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + t.Cleanup(func() { _ = exec.Close() }) + + metadata := sshConnectionMetadata(t, harness.server) + metadata[MetadataKeySSHRemoteAgentctlPID] = "4242" + metadata[MetadataKeySSHRemoteAgentctlPort] = "41234" + metadata[MetadataKeySSHRemoteSessionDir] = "/remote/session" + metadata[MetadataKeySSHRemoteTaskDir] = "/remote/task" + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + SessionID: "session-1", + AuthToken: "persisted-token", + Metadata: metadata, + } + + if err := exec.ResumeRemoteInstance(context.Background(), req); err != nil { + t.Fatalf("ResumeRemoteInstance: %v", err) + } + exec.mu.Lock() + state := exec.sessions["instance-1"] + exec.mu.Unlock() + if state == nil { + t.Fatal("resume must track the session") + } + if state.pid != 4242 || state.remoteDir != "/remote/session" || state.remoteTaskDir != "/remote/task" { + t.Fatalf("state = %+v", state) + } + if state.authToken != "persisted-token" { + t.Fatalf("authToken = %q", state.authToken) + } + forwardPort := req.Metadata[MetadataKeySSHLocalForwardPort] + if forwardPort != strconv.Itoa(state.forwarder.LocalPort()) { + t.Fatalf("metadata forward port = %v, want the new local port %d", + forwardPort, state.forwarder.LocalPort()) + } + }) + + t.Run("dead remote pid fails the resume", func(t *testing.T) { + server := newFakeSSHServer(t, func(command, _ string) sshExecResult { + if strings.Contains(command, "kill -0") { + return sshFail("no such process") + } + return sshOut(t.TempDir()) + }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + metadata := sshConnectionMetadata(t, server) + metadata[MetadataKeySSHRemoteAgentctlPID] = "4242" + metadata[MetadataKeySSHRemoteAgentctlPort] = "41234" + metadata[MetadataKeySSHRemoteSessionDir] = "/remote/session" + metadata[MetadataKeySSHRemoteTaskDir] = "/remote/task" + + err := exec.ResumeRemoteInstance(context.Background(), &ExecutorCreateRequest{ + InstanceID: "instance-1", + AuthToken: "persisted-token", + Metadata: metadata, + }) + if err == nil || !strings.Contains(err.Error(), "not alive on remote") { + t.Fatalf("error = %v, want a dead-pid resume failure", err) + } + if len(exec.sessions) != 0 { + t.Fatal("a failed resume must not track state") + } + }) +} + +func TestSSHExecutorResetManagedBrokerResume(t *testing.T) { + brokerEnv := map[string]string{ + githubauth.CredentialBrokerURLEnv: "http://127.0.0.1:9/broker", + githubauth.CredentialLeaseEnv: "lease-1", + } + baseMetadata := func() map[string]interface{} { + return map[string]interface{}{ + MetadataKeySSHRemoteAgentctlPID: "4242", + MetadataKeySSHRemoteAgentctlPort: "41234", + MetadataKeySSHRemoteSessionDir: "/remote/session", + MetadataKeySSHRemoteTaskDir: "/remote/task", + MetadataKeySSHLocalForwardPort: "5000", + MetadataKeySSHRemoteAgentctlURL: "http://127.0.0.1:5000", + } + } + + t.Run("tracked session is stopped and its runtime metadata cleared", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + var preflightCalls, stopCalls, closeCalls int + var stoppedDir string + var stoppedPid int + exec.brokerPreflight = func(context.Context, *ssh.Client, *ExecutorCreateRequest, SSHRemotePlatform) error { + preflightCalls++ + return nil + } + exec.stopRemote = func(_ context.Context, _ *ssh.Client, dir string, pid int) error { + stopCalls++ + stoppedDir, stoppedPid = dir, pid + return nil + } + exec.closeClient = func(*ssh.Client) error { closeCalls++; return nil } + exec.sessions["instance-1"] = &sshSessionState{ + client: &ssh.Client{}, + remoteDir: "/remote/session", + pid: 4242, + } + + req := &ExecutorCreateRequest{InstanceID: "instance-1", Env: brokerEnv, Metadata: baseMetadata()} + if err := exec.ResumeRemoteInstance(context.Background(), req); err != nil { + t.Fatalf("ResumeRemoteInstance: %v", err) + } + if preflightCalls != 1 || stopCalls != 1 || closeCalls != 1 { + t.Fatalf("calls = preflight %d / stop %d / close %d", preflightCalls, stopCalls, closeCalls) + } + if stoppedDir != "/remote/session" || stoppedPid != 4242 { + t.Fatalf("stopped %q pid %d", stoppedDir, stoppedPid) + } + if len(exec.sessions) != 0 { + t.Fatal("the stale session must be dropped") + } + for _, key := range []string{ + MetadataKeySSHRemoteSessionDir, MetadataKeySSHRemoteAgentctlPort, + MetadataKeySSHRemoteAgentctlPID, MetadataKeySSHLocalForwardPort, + MetadataKeySSHRemoteAgentctlURL, + } { + if _, present := req.Metadata[key]; present { + t.Fatalf("runtime metadata key %q must be cleared so a fresh create runs", key) + } + } + if _, present := req.Metadata[MetadataKeySSHRemoteTaskDir]; !present { + t.Fatal("the task dir must survive so the fresh create reuses the checkout") + } + }) + + t.Run("preflight failure aborts before stopping anything", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + wantErr := errors.New("broker unreachable") + exec.brokerPreflight = func(context.Context, *ssh.Client, *ExecutorCreateRequest, SSHRemotePlatform) error { + return wantErr + } + stopCalls := 0 + exec.stopRemote = func(context.Context, *ssh.Client, string, int) error { stopCalls++; return nil } + exec.sessions["instance-1"] = &sshSessionState{client: &ssh.Client{}} + + err := exec.ResumeRemoteInstance(context.Background(), + &ExecutorCreateRequest{InstanceID: "instance-1", Env: brokerEnv, Metadata: baseMetadata()}) + if !errors.Is(err, wantErr) { + t.Fatalf("error = %v, want %v", err, wantErr) + } + if stopCalls != 0 { + t.Fatalf("stop calls = %d, want 0", stopCalls) + } + }) + + t.Run("stop failure closes the client and surfaces the error", func(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.brokerPreflight = func(context.Context, *ssh.Client, *ExecutorCreateRequest, SSHRemotePlatform) error { + return nil + } + exec.stopRemote = func(context.Context, *ssh.Client, string, int) error { + return errors.New("kill failed") + } + closeCalls := 0 + exec.closeClient = func(*ssh.Client) error { closeCalls++; return nil } + exec.sessions["instance-1"] = &sshSessionState{client: &ssh.Client{}} + + err := exec.ResumeRemoteInstance(context.Background(), + &ExecutorCreateRequest{InstanceID: "instance-1", Env: brokerEnv, Metadata: baseMetadata()}) + if err == nil || !strings.Contains(err.Error(), "stop stale broker-backed agentctl") { + t.Fatalf("error = %v", err) + } + if closeCalls != 1 { + t.Fatalf("close calls = %d, want the client released anyway", closeCalls) + } + }) + + t.Run("untracked session dials the host to refresh the broker", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOK }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + preflightCalls := 0 + exec.brokerPreflight = func(context.Context, *ssh.Client, *ExecutorCreateRequest, SSHRemotePlatform) error { + preflightCalls++ + return nil + } + metadata := sshConnectionMetadata(t, server) + for key, value := range baseMetadata() { + metadata[key] = value + } + + req := &ExecutorCreateRequest{InstanceID: "instance-1", Env: brokerEnv, Metadata: metadata} + if err := exec.ResumeRemoteInstance(context.Background(), req); err != nil { + t.Fatalf("ResumeRemoteInstance: %v", err) + } + if preflightCalls != 1 { + t.Fatalf("preflight calls = %d", preflightCalls) + } + if _, ok := server.lastCommandContaining("kill 4242"); !ok { + t.Fatalf("expected the stale remote agentctl to be stopped, commands: %v", server.commands()) + } + }) +} + +func TestSSHTaskDirName(t *testing.T) { + tests := []struct { + name string + req *ExecutorCreateRequest + wantName string + }{ + { + name: "falls back to the task ID", + req: &ExecutorCreateRequest{TaskID: "abc"}, + wantName: "task-abc", + }, + { + name: "task_dir_name hint wins over the task ID", + req: &ExecutorCreateRequest{TaskID: "abc", + Metadata: map[string]interface{}{"task_dir_name": "kandev-widgets"}}, + wantName: "kandev-widgets", + }, + { + name: "ssh_task_dir_name wins over task_dir_name", + req: &ExecutorCreateRequest{TaskID: "abc", Metadata: map[string]interface{}{ + "task_dir_name": "kandev-widgets", + "ssh_task_dir_name": "ssh-specific", + }}, + wantName: "ssh-specific", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sshTaskDirName(tc.req); got != tc.wantName { + t.Fatalf("sshTaskDirName = %q, want %q", got, tc.wantName) + } + }) + } +} + +func TestClearSSHResumeRuntimeMetadata(t *testing.T) { + metadata := map[string]interface{}{ + MetadataKeySSHRemoteSessionDir: "/remote/session", + MetadataKeySSHRemoteAgentctlPort: "41234", + MetadataKeySSHRemoteAgentctlPID: "4242", + MetadataKeySSHLocalForwardPort: "5000", + MetadataKeySSHRemoteAgentctlURL: "http://127.0.0.1:5000", + MetadataKeySSHRemoteTaskDir: "/remote/task", + MetadataKeySSHHost: "build.example", + } + clearSSHResumeRuntimeMetadata(metadata) + if len(metadata) != 2 { + t.Fatalf("remaining metadata = %+v, want only the durable keys", metadata) + } + if metadata[MetadataKeySSHRemoteTaskDir] != "/remote/task" || metadata[MetadataKeySSHHost] != "build.example" { + t.Fatalf("durable metadata was cleared: %+v", metadata) + } + clearSSHResumeRuntimeMetadata(nil) // must not panic +} + +func TestSSHExecutorReportIsANoOpWithoutACallback(t *testing.T) { + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + exec.report(nil, "Step", PrepareStepCompleted, "output") // must not panic + + var got PrepareStep + exec.report(func(step PrepareStep, _, _ int) { got = step }, "Step", PrepareStepFailed, "boom") + if got.Name != "Step" || got.Status != PrepareStepFailed || got.Output != "boom" { + t.Fatalf("reported step = %+v", got) + } +} + +func TestSSHExecutorPreflightAgentBinary(t *testing.T) { + t.Run("no agent config skips the probe", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + err := exec.preflightAgentBinary(context.Background(), server.dial(t), + &ExecutorCreateRequest{}, SSHRemotePlatform{}) + if err != nil { + t.Fatalf("preflightAgentBinary: %v", err) + } + if len(server.commands()) != 0 { + t.Fatalf("no probe expected, commands: %v", server.commands()) + } + }) + + t.Run("missing binary surfaces the install hint", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + agent := newFakeIdentityAgent("phantom") + agent.name = "Phantom Agent" + agent.installScript = "npm i -g phantom" + + var failed []PrepareStep + err := exec.preflightAgentBinary(context.Background(), server.dial(t), &ExecutorCreateRequest{ + AgentConfig: agent, + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepFailed { + failed = append(failed, step) + } + }, + }, SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), "Phantom Agent") { + t.Fatalf("error = %v, want the agent name", err) + } + if !strings.Contains(err.Error(), "npm i -g phantom") { + t.Fatalf("error = %v, want the install hint", err) + } + if len(failed) != 1 || failed[0].Name != "Verifying agent binary" { + t.Fatalf("failed steps = %+v", failed) + } + }) + + t.Run("present binary passes and reports the resolved path", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("/usr/bin/mock\n") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + var completed []PrepareStep + err := exec.preflightAgentBinary(context.Background(), server.dial(t), &ExecutorCreateRequest{ + AgentConfig: newFakeIdentityAgent("mock"), + OnProgress: func(step PrepareStep, _, _ int) { + if step.Status == PrepareStepCompleted { + completed = append(completed, step) + } + }, + }, SSHRemotePlatform{}) + if err != nil { + t.Fatalf("preflightAgentBinary: %v", err) + } + if len(completed) != 1 || completed[0].Output != "/usr/bin/mock" { + t.Fatalf("completed steps = %+v", completed) + } + }) + + t.Run("probe transport failure fails the launch", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("closed") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + err := exec.preflightAgentBinary(context.Background(), server.dial(t), + &ExecutorCreateRequest{AgentConfig: newFakeIdentityAgent("mock")}, SSHRemotePlatform{}) + if err == nil || !strings.Contains(err.Error(), "probe") { + t.Fatalf("error = %v", err) + } + }) +} + +func TestSSHExecutorProbeNativeBinary(t *testing.T) { + t.Run("agent without a native binary is skipped", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{AgentConfig: newFakeIdentityAgent("mock")} + if exec.probeNativeBinary(context.Background(), server.dial(t), "bash", req, "step") { + t.Fatal("a non-NativeBinaryAgent must not report a hit") + } + }) + + t.Run("empty native binary name is skipped", func(t *testing.T) { + server := newFakeSSHServer(t, nil) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{AgentConfig: &fakeNativeBinaryAgent{ + fakeIdentityAgent: newFakeIdentityAgent("copilot"), + }} + if exec.probeNativeBinary(context.Background(), server.dial(t), "bash", req, "step") { + t.Fatal("an empty native binary name must disable the preference") + } + if len(server.commands()) != 0 { + t.Fatalf("no probe expected, commands: %v", server.commands()) + } + }) + + t.Run("hit records the binary in metadata", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("/usr/bin/copilot\n") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{AgentConfig: &fakeNativeBinaryAgent{ + fakeIdentityAgent: newFakeIdentityAgent("copilot"), + nativeBinary: "copilot", + }} + if !exec.probeNativeBinary(context.Background(), server.dial(t), "bash", req, "step") { + t.Fatal("a resolved native binary must report a hit") + } + if req.Metadata[MetadataKeyNativeBinary] != "copilot" { + t.Fatalf("metadata = %+v, want the native binary recorded", req.Metadata) + } + }) + + t.Run("miss clears any stale metadata value", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshOut("") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{ + AgentConfig: &fakeNativeBinaryAgent{ + fakeIdentityAgent: newFakeIdentityAgent("copilot"), + nativeBinary: "copilot", + }, + Metadata: map[string]interface{}{MetadataKeyNativeBinary: "copilot"}, + } + if exec.probeNativeBinary(context.Background(), server.dial(t), "bash", req, "step") { + t.Fatal("an absent binary must not report a hit") + } + if _, present := req.Metadata[MetadataKeyNativeBinary]; present { + t.Fatalf("stale native binary must be cleared: %+v", req.Metadata) + } + }) + + t.Run("transport error falls through without aborting", func(t *testing.T) { + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("closed") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{ + AgentConfig: &fakeNativeBinaryAgent{ + fakeIdentityAgent: newFakeIdentityAgent("copilot"), + nativeBinary: "copilot", + }, + Metadata: map[string]interface{}{MetadataKeyNativeBinary: "copilot"}, + } + if exec.probeNativeBinary(context.Background(), server.dial(t), "bash", req, "step") { + t.Fatal("a transport error must not report a hit") + } + if _, present := req.Metadata[MetadataKeyNativeBinary]; present { + t.Fatalf("a failed probe must not leave the preference set: %+v", req.Metadata) + } + }) +} + +func TestSSHExecutorMaybeUploadCredentialsSwallowsFailures(t *testing.T) { + // A credential-upload failure is best-effort: the launch continues. + server := newFakeSSHServer(t, func(string, string) sshExecResult { return sshFail("no home") }) + exec := NewSSHExecutor(nil, nil, nil, newTestLogger()) + req := &ExecutorCreateRequest{ + TaskID: "task-1", + SessionID: "session-1", + Metadata: map[string]interface{}{"remote_credentials": `["agent:mock:files:0"]`}, + } + exec.maybeUploadCredentials(context.Background(), server.dial(t), req, SSHRemotePlatform{}) +} + +func sshContainsStep(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go new file mode 100644 index 0000000000..7a0e9d4509 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_standalone_test.go @@ -0,0 +1,327 @@ +package lifecycle + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/kandev/kandev/internal/agent/agents" + "github.com/kandev/kandev/internal/agent/executor" + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/process" +) + +// standaloneControlServer is an in-process stand-in for the host agentctl +// control API the standalone executor talks to. +type standaloneControlServer struct { + mu sync.Mutex + healthy bool + healthCalls int + createRequests []agentctlclient.CreateInstanceRequest + deleted []string + createStatus int + server *httptest.Server +} + +func newStandaloneControlServer(t *testing.T, healthy bool) *standaloneControlServer { + t.Helper() + s := &standaloneControlServer{healthy: healthy, createStatus: http.StatusOK} + s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + switch { + case r.URL.Path == "/health": + s.healthCalls++ + if !s.healthy { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write([]byte(`{"status":"ok"}`)) + case r.URL.Path == "/api/v1/instances" && r.Method == http.MethodPost: + var req agentctlclient.CreateInstanceRequest + _ = json.NewDecoder(r.Body).Decode(&req) + s.createRequests = append(s.createRequests, req) + if s.createStatus != http.StatusOK { + w.WriteHeader(s.createStatus) + _, _ = w.Write([]byte(`{"error":"cannot create"}`)) + return + } + _ = json.NewEncoder(w).Encode(agentctlclient.CreateInstanceResponse{ID: "std-1", Port: 45678}) + case strings.HasPrefix(r.URL.Path, "/api/v1/instances/") && r.Method == http.MethodDelete: + s.deleted = append(s.deleted, strings.TrimPrefix(r.URL.Path, "/api/v1/instances/")) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(s.server.Close) + return s +} + +func (s *standaloneControlServer) executor(t *testing.T) *StandaloneExecutor { + t.Helper() + parsed, err := url.Parse(s.server.URL) + if err != nil { + t.Fatalf("parse control URL: %v", err) + } + port, _ := strconv.Atoi(parsed.Port()) + ctl := agentctlclient.NewControlClient(parsed.Hostname(), port, newTestLogger()) + return NewStandaloneExecutor(ctl, parsed.Hostname(), port, newTestLogger()) +} + +func (s *standaloneControlServer) lastCreateRequest(t *testing.T) agentctlclient.CreateInstanceRequest { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.createRequests) == 0 { + t.Fatal("no create-instance request recorded") + } + return s.createRequests[len(s.createRequests)-1] +} + +func TestStandaloneExecutorStaticSurface(t *testing.T) { + exec := newStandaloneControlServer(t, true).executor(t) + if exec.Name() != executor.NameStandalone { + t.Fatalf("Name() = %q", exec.Name()) + } + if exec.RequiresCloneURL() { + t.Fatal("standalone runs in an existing host workspace, so no clone URL is required") + } + if !exec.ShouldApplyPreferredShell() { + t.Fatal("standalone runs on the host, so the user's preferred shell applies") + } + if exec.IsAlwaysResumable() { + t.Fatal("standalone instances are transient and not always resumable") + } + instances, err := exec.RecoverInstances(context.Background()) + if err != nil || instances != nil { + t.Fatalf("RecoverInstances() = %v, %v; want nil, nil", instances, err) + } + + runner := &process.InteractiveRunner{} + exec.SetInteractiveRunner(runner) + if exec.GetInteractiveRunner() != runner { + t.Fatal("interactive runner round-trip failed") + } +} + +func TestStandaloneExecutorHealthCheck(t *testing.T) { + healthy := newStandaloneControlServer(t, true) + if err := healthy.executor(t).HealthCheck(context.Background()); err != nil { + t.Fatalf("HealthCheck: %v", err) + } + + unhealthy := newStandaloneControlServer(t, false) + if err := unhealthy.executor(t).HealthCheck(context.Background()); err == nil { + t.Fatal("expected an unhealthy control server to fail the health check") + } +} + +func TestStandaloneExecutorCreateInstance(t *testing.T) { + control := newStandaloneControlServer(t, true) + exec := control.executor(t) + exec.SetAuthToken("launch-token") + + agent := &fakeRuntimeAgent{ + MockAgent: agents.NewMockAgent(), + id: "opencode", + requiresKill: true, + stripEnv: []string{"NODE_OPTIONS"}, + } + req := &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + WorkspacePath: "/host/workspace", + WorkspaceSourceRoots: []string{"/host/sources"}, + Protocol: "acp", + McpMode: "task", + AgentConfig: agent, + Env: map[string]string{"EXISTING": "value"}, + Metadata: map[string]interface{}{ + MetadataKeyWorktreeID: "wt-1", + MetadataKeyWorktreeBranch: "feature/task-1", + MetadataKeyBaseBranches: map[string]string{"": "main"}, + }, + } + + instance, err := exec.CreateInstance(context.Background(), req) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + + if instance.StandaloneInstanceID != "std-1" || instance.StandalonePort != 45678 { + t.Fatalf("instance = %+v", instance) + } + if instance.RuntimeName != executor.NameStandalone { + t.Fatalf("RuntimeName = %q", instance.RuntimeName) + } + if instance.WorkspacePath != "/host/workspace" { + t.Fatalf("WorkspacePath = %q, want the host path verbatim", instance.WorkspacePath) + } + if instance.Metadata["standalone_port"] != 45678 { + t.Fatalf("standalone_port metadata = %v", instance.Metadata["standalone_port"]) + } + if instance.Metadata["worktree_id"] != "wt-1" || + instance.Metadata["worktree_path"] != "/host/workspace" || + instance.Metadata["worktree_branch"] != "feature/task-1" { + t.Fatalf("worktree metadata = %+v", instance.Metadata) + } + + got := control.lastCreateRequest(t) + if got.ID != "instance-1" || got.SessionID != "session-1" || got.TaskID != "task-1" { + t.Fatalf("create request identity = %+v", got) + } + if got.AgentCommand != "" { + t.Fatalf("AgentCommand = %q, want empty (the agent starts via a later Configure call)", got.AgentCommand) + } + if got.AutoStart { + t.Fatal("standalone instances must not auto-start the agent") + } + if got.AgentType != "opencode" || !got.RequiresProcessKill { + t.Fatalf("agent runtime fields = %+v", got) + } + if !equalStrings(got.StripEnv, []string{"NODE_OPTIONS"}) { + t.Fatalf("StripEnv = %v", got.StripEnv) + } + if got.BaseBranches[""] != "main" { + t.Fatalf("BaseBranches = %+v", got.BaseBranches) + } + if !equalStrings(got.WorkspaceSourceRoots, []string{"/host/sources"}) { + t.Fatalf("WorkspaceSourceRoots = %v", got.WorkspaceSourceRoots) + } + // The executor injects the task/session identity into the child env. + if got.Env["KANDEV_TASK_ID"] != "task-1" || got.Env["KANDEV_SESSION_ID"] != "session-1" { + t.Fatalf("Env = %+v, want the task and session IDs injected", got.Env) + } + if got.Env["EXISTING"] != "value" { + t.Fatalf("Env = %+v, want the caller's env preserved", got.Env) + } +} + +func TestStandaloneExecutorCreateInstanceWithoutWorktreeMetadata(t *testing.T) { + control := newStandaloneControlServer(t, true) + instance, err := control.executor(t).CreateInstance(context.Background(), &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + WorkspacePath: "/host/workspace", + }) + if err != nil { + t.Fatalf("CreateInstance: %v", err) + } + for _, key := range []string{"worktree_id", "worktree_path", "worktree_branch"} { + if _, present := instance.Metadata[key]; present { + t.Fatalf("metadata %q must be absent without a worktree: %+v", key, instance.Metadata) + } + } + if got := control.lastCreateRequest(t); got.AgentType != "" { + t.Fatalf("AgentType = %q, want empty without an agent config", got.AgentType) + } +} + +func TestStandaloneExecutorCreateInstanceSurfacesControlFailure(t *testing.T) { + control := newStandaloneControlServer(t, true) + control.createStatus = http.StatusInternalServerError + _, err := control.executor(t).CreateInstance(context.Background(), &ExecutorCreateRequest{ + InstanceID: "instance-1", + }) + if err == nil || !strings.Contains(err.Error(), "failed to create standalone instance") { + t.Fatalf("error = %v", err) + } +} + +func TestStandaloneExecutorCreateInstanceFailsWhenAgentctlNeverBecomesReady(t *testing.T) { + control := newStandaloneControlServer(t, false) + // waitForReady polls every 500ms; give the deadline a wide enough margin + // that a loaded CI runner still observes several retries. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := control.executor(t).CreateInstance(ctx, &ExecutorCreateRequest{InstanceID: "instance-1"}) + if err == nil || !strings.Contains(err.Error(), "agentctl not ready") { + t.Fatalf("error = %v, want the readiness wait to fail", err) + } + control.mu.Lock() + calls := control.healthCalls + control.mu.Unlock() + if calls < 2 { + t.Fatalf("health calls = %d, want the wait loop to retry", calls) + } +} + +func TestStandaloneExecutorStopInstance(t *testing.T) { + control := newStandaloneControlServer(t, true) + exec := control.executor(t) + + t.Run("no standalone instance is a no-op", func(t *testing.T) { + if err := exec.StopInstance(context.Background(), &ExecutorInstance{}, false); err != nil { + t.Fatalf("StopInstance: %v", err) + } + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 0 { + t.Fatalf("deleted = %v, want none", control.deleted) + } + }) + + t.Run("deletes the tracked instance", func(t *testing.T) { + err := exec.StopInstance(context.Background(), + &ExecutorInstance{StandaloneInstanceID: "std-1"}, false) + if err != nil { + t.Fatalf("StopInstance: %v", err) + } + control.mu.Lock() + defer control.mu.Unlock() + if len(control.deleted) != 1 || control.deleted[0] != "std-1" { + t.Fatalf("deleted = %v", control.deleted) + } + }) +} + +func TestBuildStandaloneCreateInstanceRequestMapsEveryField(t *testing.T) { + autoApprove := true + req := &ExecutorCreateRequest{ + InstanceID: "instance-2", + TaskID: "task-2", + SessionID: "session-2", + WorkspacePath: "/host/workspace", + WorkspaceSourceRoots: []string{"/host/sources"}, + Protocol: "acp", + McpMode: "office", + McpProviders: []string{"github"}, + AutoApprovePermissionsOverride: &autoApprove, + Metadata: map[string]interface{}{MetadataKeyBaseBranches: map[string]string{"repo": "dev"}}, + } + + got := buildStandaloneCreateInstanceRequest(req, map[string]string{"K": "V"}, "codex", + true, true, false, true, []string{"STRIP"}) + + if got.ID != "instance-2" || got.WorkspacePath != "/host/workspace" || got.AgentType != "codex" { + t.Fatalf("request = %+v", got) + } + if !got.DisableAskQuestion || !got.AssumeMcpSse || got.AssumeMcpHttp || !got.RequiresProcessKill { + t.Fatalf("capability flags = %+v", got) + } + if got.AutoApprovePermissions == nil || !*got.AutoApprovePermissions { + t.Fatalf("AutoApprovePermissions = %v", got.AutoApprovePermissions) + } + if got.Env["K"] != "V" || !equalStrings(got.StripEnv, []string{"STRIP"}) { + t.Fatalf("env/strip = %+v / %v", got.Env, got.StripEnv) + } + if got.BaseBranches["repo"] != "dev" { + t.Fatalf("BaseBranches = %+v", got.BaseBranches) + } + if got.McpMode != "office" || !equalStrings(got.McpProviders, []string{"github"}) { + t.Fatalf("mcp routing = %q / %v", got.McpMode, got.McpProviders) + } + if !equalStrings(got.WorkspaceSourceRoots, []string{"/host/sources"}) { + t.Fatalf("WorkspaceSourceRoots = %v", got.WorkspaceSourceRoots) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/ssh_fake_server_test.go b/apps/backend/internal/agent/runtime/lifecycle/ssh_fake_server_test.go new file mode 100644 index 0000000000..5b9321cfa2 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/ssh_fake_server_test.go @@ -0,0 +1,380 @@ +package lifecycle + +import ( + "crypto/ed25519" + "crypto/rand" + "errors" + "io" + "net" + "strings" + "sync" + "testing" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// This file provides an in-process SSH server so the SSH executor's remote +// paths can be exercised end to end without any real infrastructure. It is the +// SSH analogue of httptest.NewServer: a real protocol implementation speaking +// to a real *ssh.Client, with canned command results supplied by the test. +// +// It supports the three channel shapes the SSH executor actually uses: +// - "session" + exec — every runSSHCommand / runSSHCommandStdin call +// - "direct-tcpip" — StartPortForward and the HTTP-over-SSH control calls +// - "sftp" subsystem — sftpUploadBytes and sshFileUploader.WriteFile +// +// Every goroutine it starts is tracked on a WaitGroup and drained by the +// t.Cleanup hook, because this package runs goleak.VerifyTestMain. + +// sshExecCall records one exec request observed by the fake SSH server. +type sshExecCall struct { + Command string + Stdin string +} + +// sshExecResult is the canned response a handler returns for one exec request. +type sshExecResult struct { + Stdout string + Stderr string + ExitCode int +} + +// sshExecHandler answers a single remote command. +type sshExecHandler func(command, stdin string) sshExecResult + +type fakeSSHServer struct { + t *testing.T + listener net.Listener + signer ssh.Signer + + wg sync.WaitGroup + closeOnce sync.Once + + mu sync.Mutex + calls []sshExecCall + handler sshExecHandler + + // sftpEnabled serves the "sftp" subsystem against the real filesystem, so + // tests point remote paths at a t.TempDir() and assert on real bytes. + sftpEnabled bool + + // tcpTarget maps a requested direct-tcpip port onto a real local address. + // Returning "" rejects the channel. + tcpTarget func(port uint32) string +} + +// newFakeSSHServer starts a listener on 127.0.0.1 and serves SSH until the +// test finishes. handler may be nil, in which case every command succeeds with +// empty output. +func newFakeSSHServer(t *testing.T, handler sshExecHandler) *fakeSSHServer { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate host key: %v", err) + } + signer, err := ssh.NewSignerFromKey(priv) + if err != nil { + t.Fatalf("host signer: %v", err) + } + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + s := &fakeSSHServer{t: t, listener: listener, signer: signer, handler: handler} + t.Cleanup(s.Close) + s.wg.Add(1) + go s.acceptLoop() + return s +} + +// enableSFTP turns on the "sftp" subsystem. The server serves the real +// filesystem, so tests must use paths under t.TempDir(). +func (s *fakeSSHServer) enableSFTP() { s.sftpEnabled = true } + +// forwardTo makes every direct-tcpip channel connect to addr regardless of the +// port the client asked for. Used to point remote "127.0.0.1:" dials at +// an httptest server. +func (s *fakeSSHServer) forwardTo(addr string) { + s.tcpTarget = func(uint32) string { return addr } +} + +func (s *fakeSSHServer) addr() string { return s.listener.Addr().String() } + +func (s *fakeSSHServer) port() int { return s.listener.Addr().(*net.TCPAddr).Port } + +func (s *fakeSSHServer) hostKeyFingerprint() string { + return ssh.FingerprintSHA256(s.signer.PublicKey()) +} + +// commands returns the command strings seen so far, in order. +func (s *fakeSSHServer) commands() []string { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]string, 0, len(s.calls)) + for _, call := range s.calls { + out = append(out, call.Command) + } + return out +} + +// execCalls returns the full recorded calls (command plus stdin), in order. +func (s *fakeSSHServer) execCalls() []sshExecCall { + s.mu.Lock() + defer s.mu.Unlock() + return append([]sshExecCall(nil), s.calls...) +} + +// lastCommandContaining returns the most recent command containing substr. +func (s *fakeSSHServer) lastCommandContaining(substr string) (sshExecCall, bool) { + s.mu.Lock() + defer s.mu.Unlock() + for i := len(s.calls) - 1; i >= 0; i-- { + if strings.Contains(s.calls[i].Command, substr) { + return s.calls[i], true + } + } + return sshExecCall{}, false +} + +func (s *fakeSSHServer) Close() { + s.closeOnce.Do(func() { + _ = s.listener.Close() + }) + s.wg.Wait() +} + +// dial returns a client connected to the fake server. The client is closed by +// t.Cleanup so the SSH mux goroutines drain before goleak inspects them. +func (s *fakeSSHServer) dial(t *testing.T) *ssh.Client { + t.Helper() + client, err := ssh.Dial("tcp", s.addr(), &ssh.ClientConfig{ + User: "kandev", + HostKeyCallback: ssh.FixedHostKey(s.signer.PublicKey()), + }) + if err != nil { + t.Fatalf("dial fake ssh server: %v", err) + } + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func (s *fakeSSHServer) acceptLoop() { + defer s.wg.Done() + for { + conn, err := s.listener.Accept() + if err != nil { + return + } + s.wg.Add(1) + go s.serveConn(conn) + } +} + +func (s *fakeSSHServer) serveConn(conn net.Conn) { + defer s.wg.Done() + defer func() { _ = conn.Close() }() + + cfg := &ssh.ServerConfig{NoClientAuth: true} + cfg.AddHostKey(s.signer) + serverConn, chans, reqs, err := ssh.NewServerConn(conn, cfg) + if err != nil { + return + } + defer func() { _ = serverConn.Close() }() + + s.wg.Add(1) + go func() { + defer s.wg.Done() + ssh.DiscardRequests(reqs) + }() + + for newChan := range chans { + switch newChan.ChannelType() { + case "session": + s.wg.Add(1) + go s.serveSession(newChan) + case "direct-tcpip": + s.wg.Add(1) + go s.serveDirectTCPIP(newChan) + default: + _ = newChan.Reject(ssh.UnknownChannelType, newChan.ChannelType()) + } + } +} + +func (s *fakeSSHServer) serveSession(newChan ssh.NewChannel) { + defer s.wg.Done() + channel, requests, err := newChan.Accept() + if err != nil { + return + } + defer func() { _ = channel.Close() }() + + for req := range requests { + switch req.Type { + case "exec": + var payload struct{ Command string } + if err := ssh.Unmarshal(req.Payload, &payload); err != nil { + _ = req.Reply(false, nil) + return + } + _ = req.Reply(true, nil) + s.runExec(channel, payload.Command) + return + case "subsystem": + var payload struct{ Name string } + if err := ssh.Unmarshal(req.Payload, &payload); err != nil || payload.Name != "sftp" || !s.sftpEnabled { + _ = req.Reply(false, nil) + return + } + _ = req.Reply(true, nil) + s.runSFTP(channel) + return + default: + // pty-req / env / signal — accepted but ignored. + if req.WantReply { + _ = req.Reply(true, nil) + } + } + } +} + +func (s *fakeSSHServer) runExec(channel ssh.Channel, command string) { + stdin, _ := io.ReadAll(channel) + + s.mu.Lock() + s.calls = append(s.calls, sshExecCall{Command: command, Stdin: string(stdin)}) + handler := s.handler + s.mu.Unlock() + + result := sshExecResult{} + if handler != nil { + result = handler(command, string(stdin)) + } + if result.Stdout != "" { + _, _ = io.WriteString(channel, result.Stdout) + } + if result.Stderr != "" { + _, _ = io.WriteString(channel.Stderr(), result.Stderr) + } + _, _ = channel.SendRequest("exit-status", false, ssh.Marshal(struct{ Status uint32 }{uint32(result.ExitCode)})) +} + +func (s *fakeSSHServer) runSFTP(channel ssh.Channel) { + server, err := sftp.NewServer(channel) + if err != nil { + return + } + defer func() { _ = server.Close() }() + if err := server.Serve(); err != nil && !errors.Is(err, io.EOF) { + s.t.Logf("fake sftp server: %v", err) + } +} + +func (s *fakeSSHServer) serveDirectTCPIP(newChan ssh.NewChannel) { + defer s.wg.Done() + var payload struct { + DestAddr string + DestPort uint32 + OrigAddr string + OrigPort uint32 + } + if err := ssh.Unmarshal(newChan.ExtraData(), &payload); err != nil { + _ = newChan.Reject(ssh.ConnectionFailed, "bad direct-tcpip payload") + return + } + target := "" + if s.tcpTarget != nil { + target = s.tcpTarget(payload.DestPort) + } + if target == "" { + _ = newChan.Reject(ssh.ConnectionFailed, "no forward target configured") + return + } + upstream, err := net.Dial("tcp", target) + if err != nil { + _ = newChan.Reject(ssh.ConnectionFailed, err.Error()) + return + } + channel, requests, err := newChan.Accept() + if err != nil { + _ = upstream.Close() + return + } + s.wg.Add(1) + go func() { + defer s.wg.Done() + ssh.DiscardRequests(requests) + }() + + var copies sync.WaitGroup + copies.Add(2) + go func() { + defer copies.Done() + _, _ = io.Copy(upstream, channel) + if tcp, ok := upstream.(*net.TCPConn); ok { + _ = tcp.CloseWrite() + } + }() + go func() { + defer copies.Done() + _, _ = io.Copy(channel, upstream) + _ = channel.CloseWrite() + }() + copies.Wait() + _ = upstream.Close() + _ = channel.Close() +} + +// sshScriptedHandler routes commands to canned results by substring match, in +// declaration order, and fails the test on an unmatched command so a changed +// remote command string surfaces as a test failure rather than silent success. +type sshScriptedHandler struct { + t *testing.T + rules []sshScriptRule +} + +type sshScriptRule struct { + match string + result sshExecResult +} + +func newSSHScriptedHandler(t *testing.T, rules ...sshScriptRule) *sshScriptedHandler { + return &sshScriptedHandler{t: t, rules: rules} +} + +func (h *sshScriptedHandler) handle(command, _ string) sshExecResult { + for _, rule := range h.rules { + if strings.Contains(command, rule.match) { + return rule.result + } + } + h.t.Errorf("fake ssh server: no rule matched command %q", command) + return sshExecResult{ExitCode: 127, Stderr: "no rule matched"} +} + +// unwrapLoginShell asserts that command is a ` -lc '