package runtime import ( "context" "errors" "os" "path/filepath" goruntime "runtime" "sync" "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" "github.com/kandev/kandev/internal/plugins/manifest" "github.com/kandev/kandev/internal/plugins/store" "github.com/kandev/kandev/pkg/pluginsdk" ) // fakeHost is a minimal in-memory pluginsdk.Host for end-to-end tests: it // only implements SetState (with a channel so tests can synchronize on // delivery without a sleep), and errors on everything else. It embeds // UnimplementedHostData to satisfy the Host data API (ADR 0043) // sub-accessors without wiring them; see docs/plans/plugins/host-data-api/ // task-04-host-data-impl.md for the real implementation. type fakeHost struct { pluginsdk.UnimplementedHostData mu sync.Mutex states map[string]map[string]any setCh chan struct{} // Host data API write round-trip recording (ADR 0043 phase 2). writeCh // signals a CreateTask arrived so tests can synchronize without a sleep; // createdTitle/lastSendText capture what the plugin sent over the real // subprocess transport. writeCh chan struct{} createdTitle string lastSendTask string lastSendText string } func newFakeHost() *fakeHost { return &fakeHost{states: map[string]map[string]any{}, setCh: make(chan struct{}, 16), writeCh: make(chan struct{}, 16)} } // Tasks/Messages override UnimplementedHostData so the fixture's write probe // (CreateTask + SendMessage) round-trips to this fake host over the real // subprocess gRPC transport. This fake does not enforce capabilities — that is // covered against a real *pluginHost in internal/plugins' wire tests; here we // only prove the write RPC reaches the host and its result returns. func (h *fakeHost) Tasks() pluginsdk.TaskReader { return fakeHostTaskReader{h} } func (h *fakeHost) Messages() pluginsdk.MessageReader { return fakeHostMessageReader{h} } type fakeHostTaskReader struct{ h *fakeHost } func (fakeHostTaskReader) List(context.Context, pluginsdk.TaskFilter, pluginsdk.Page) ([]pluginsdk.Task, *pluginsdk.PageInfo, error) { return nil, nil, nil } func (fakeHostTaskReader) Get(context.Context, string) (*pluginsdk.Task, error) { return nil, nil } func (fakeHostTaskReader) Update(context.Context, pluginsdk.UpdateTaskInput) (*pluginsdk.Task, error) { return nil, nil } func (fakeHostTaskReader) Move(context.Context, pluginsdk.MoveTaskInput) (*pluginsdk.MoveTaskOutcome, error) { return nil, nil } func (r fakeHostTaskReader) Create(_ context.Context, in pluginsdk.CreateTaskInput) (*pluginsdk.Task, error) { r.h.mu.Lock() r.h.createdTitle = in.Title r.h.mu.Unlock() select { case r.h.writeCh <- struct{}{}: default: } return &pluginsdk.Task{ID: "task-from-host", Title: in.Title, WorkspaceID: in.WorkspaceID, WorkflowID: in.WorkflowID}, nil } type fakeHostMessageReader struct{ h *fakeHost } func (fakeHostMessageReader) List(context.Context, pluginsdk.MessageFilter, pluginsdk.Page) ([]pluginsdk.Message, *pluginsdk.PageInfo, error) { return nil, nil, nil } func (r fakeHostMessageReader) Send(_ context.Context, taskID, sessionID, text string) (*pluginsdk.MessageDispatch, error) { r.h.mu.Lock() r.h.lastSendTask = taskID r.h.lastSendText = text r.h.mu.Unlock() return &pluginsdk.MessageDispatch{SessionID: "session-from-host", Status: "queued"}, nil } func (h *fakeHost) GetState(context.Context, string, string, string) (map[string]any, bool, error) { return nil, false, nil } func (h *fakeHost) SetState(_ context.Context, scope, scopeID, key string, value map[string]any) error { h.mu.Lock() h.states[scope+"|"+scopeID+"|"+key] = value h.mu.Unlock() select { case h.setCh <- struct{}{}: default: } return nil } func (h *fakeHost) DeleteState(context.Context, string, string, string) error { return nil } func (h *fakeHost) ListState(context.Context, string, string) ([]pluginsdk.StateEntry, error) { return nil, nil } func (h *fakeHost) GetConfig(context.Context) (map[string]any, error) { return nil, nil } func (h *fakeHost) GetSecret(context.Context, string) (string, bool, error) { return "", false, nil } func (h *fakeHost) SetSecret(context.Context, string, string) error { return nil } func (h *fakeHost) DeleteSecret(context.Context, string) error { return nil } func (h *fakeHost) RevealSecret(context.Context, string) (string, error) { return "", nil } func (h *fakeHost) EmitEvent(context.Context, string, map[string]any) error { return nil } func (h *fakeHost) get(scope, scopeID, key string) (map[string]any, bool) { h.mu.Lock() defer h.mu.Unlock() v, ok := h.states[scope+"|"+scopeID+"|"+key] return v, ok } func (h *fakeHost) writeProbe() (title, sendTask, sendText string) { h.mu.Lock() defer h.mu.Unlock() return h.createdTitle, h.lastSendTask, h.lastSendText } // buildFixtureRecord copies the real fixture plugin binary into a fresh // InstallPath and returns a *store.Record ready for Manager.Start, // skipping the test if -short suppressed the fixture build. func buildFixtureRecord(t *testing.T, id string) *store.Record { t.Helper() if fixtureBinPath == "" { t.Skip("fixture plugin binary not built (test run with -short)") } installPath := t.TempDir() platformKey := goruntime.GOOS + "-" + goruntime.GOARCH relExec := filepath.Join("server", "plugin-"+platformKey) destExec := filepath.Join(installPath, relExec) if err := os.MkdirAll(filepath.Dir(destExec), 0o755); err != nil { t.Fatalf("mkdir server dir: %v", err) } copyFile(t, fixtureBinPath, destExec, 0o755) return &store.Record{ Manifest: manifest.Manifest{ ID: id, APIVersion: 1, Version: "1.0.0", Runtime: manifest.Runtime{ Type: "binary", Executables: map[string]string{platformKey: relExec}, }, }, Status: store.StatusRegistered, InstallPath: installPath, } } func copyFile(t *testing.T, src, dst string, mode os.FileMode) { t.Helper() data, err := os.ReadFile(src) if err != nil { t.Fatalf("read %s: %v", src, err) } if err := os.WriteFile(dst, data, mode); err != nil { t.Fatalf("write %s: %v", dst, err) } } func TestManager_StartDeliverEventWebhookStop(t *testing.T) { rec := buildFixtureRecord(t, "kandev-fixture-plugin") pluginsDir := t.TempDir() host := newFakeHost() m := NewManager(pluginsDir, nil, testLogger(t)) t.Cleanup(m.StopAll) ctx := context.Background() if err := m.Start(ctx, rec, func(string) pluginsdk.Host { return host }); err != nil { t.Fatalf("Start() unexpected error: %v", err) } if !m.Running(rec.ID) { t.Fatal("Running() = false right after a successful Start()") } remote, ok := m.Get(rec.ID) if !ok { t.Fatal("Get() ok = false right after a successful Start()") } t.Run("HandleWebhook echoes the body over the real subprocess", func(t *testing.T) { resp, err := remote.HandleWebhook(ctx, &pluginsdk.WebhookRequest{WebhookKey: "echo", Body: []byte("hi")}) if err != nil { t.Fatalf("HandleWebhook() unexpected error: %v", err) } if string(resp.Body) != "hi" { t.Fatalf("HandleWebhook().Body = %q, want echoed body", resp.Body) } }) t.Run("HandleAction echoes the verified action payload over the real subprocess", func(t *testing.T) { resp, err := remote.HandleAction(ctx, &pluginsdk.PluginActionRequest{ ActionKey: "connection.get", Context: pluginsdk.VerifiedActionContext{ActorID: "user-1", WorkspaceID: "ws-1"}, Body: []byte(`{"request":"action"}`), }) if err != nil { t.Fatalf("HandleAction() unexpected error: %v", err) } if string(resp.Body) != `{"request":"action"}` { t.Fatalf("HandleAction().Body = %q, want echoed action body", resp.Body) } }) t.Run("DeliverEvent reaches the plugin, which calls back into Host.SetState", func(t *testing.T) { require.Eventually(t, func() bool { if err := remote.DeliverEvent(ctx, &pluginsdk.Event{ EventID: "e1", EventType: "task.created", }); err != nil { return false } select { case <-host.setCh: return true default: return false } }, 5*time.Second, 20*time.Millisecond, "plugin should call Host.SetState after asynchronous Host injection completes") value, found := host.get("instance", "", "last_event") if !found { t.Fatal("fakeHost never recorded the delivered event") } if value["event_type"] != "task.created" { t.Fatalf("recorded event_type = %v, want %q", value["event_type"], "task.created") } }) t.Run("HandleWebhook write probe round-trips CreateTask/SendMessage to Host", func(t *testing.T) { resp, err := remote.HandleWebhook(ctx, &pluginsdk.WebhookRequest{WebhookKey: "write"}) if err != nil { t.Fatalf("HandleWebhook(write) unexpected error: %v", err) } if string(resp.Body) != "ok" { t.Fatalf("HandleWebhook(write).Body = %q, want ok", resp.Body) } select { case <-host.writeCh: case <-time.After(5 * time.Second): t.Fatal("plugin never called Host.Tasks().Create over the subprocess transport") } title, sendTask, sendText := host.writeProbe() if title != "fixture write probe" { t.Fatalf("recorded created task title = %q, want %q", title, "fixture write probe") } // The message send targets the task id the host returned from Create, // proving the two write RPCs chain over the real transport. if sendTask != "task-from-host" || sendText != "fixture probe message" { t.Fatalf("recorded message = (task %q, text %q), want (task-from-host, fixture probe message)", sendTask, sendText) } }) t.Run("Ping succeeds against the live subprocess", func(t *testing.T) { if err := m.Ping(rec.ID); err != nil { t.Fatalf("Ping() unexpected error: %v", err) } }) m.Stop(rec.ID) if m.Running(rec.ID) { t.Fatal("Running() = true after Stop()") } } func TestManager_CrashTriggersAutomaticRestart(t *testing.T) { rec := buildFixtureRecord(t, "kandev-fixture-plugin-crash") pluginsDir := t.TempDir() host := newFakeHost() var mu sync.Mutex var transitions []bool onStatusChange := func(_ string, healthy bool, _ error) { mu.Lock() transitions = append(transitions, healthy) mu.Unlock() } m := NewManager(pluginsDir, onStatusChange, testLogger(t), WithPingInterval(20*time.Millisecond), WithMaxConsecutiveFailures(1), WithRestartBackoff([]time.Duration{10 * time.Millisecond}), WithMaxRestartAttempts(3), ) t.Cleanup(m.StopAll) ctx := context.Background() if err := m.Start(ctx, rec, func(string) pluginsdk.Host { return host }); err != nil { t.Fatalf("Start() unexpected error: %v", err) } remote, ok := m.Get(rec.ID) if !ok { t.Fatal("Get() ok = false right after Start()") } // Invoking the "crash" webhook exits the subprocess immediately; the RPC // call itself is expected to error (connection dropped mid-call) or // hang up cleanly - either way we only assert on the recovery below. _, _ = remote.HandleWebhook(ctx, &pluginsdk.WebhookRequest{WebhookKey: "crash"}) require.Eventually(t, func() bool { mu.Lock() defer mu.Unlock() return len(transitions) >= 2 && !transitions[0] && transitions[1] }, 10*time.Second, 20*time.Millisecond, "onStatusChange should observe degraded (false) then recovered (true) after the crash") } // TestManager_ConcurrentStartOnlySpawnsOnce pins the fix for the TOCTOU // double-spawn race: two Start calls for the same plugin id racing the // "already running" check must not both reach spawnFn. The first goroutine // is deliberately paused *inside* its (slow) spawn — the exact window the // old code released the manager lock across — before the second goroutine's // startProcess call is allowed to race in. func TestManager_ConcurrentStartOnlySpawnsOnce(t *testing.T) { m := NewManager(t.TempDir(), nil, testLogger(t)) t.Cleanup(m.StopAll) var spawnCalls int32 firstEntered := make(chan struct{}) release := make(chan struct{}) var closeOnce sync.Once spawnFn := func() (spawnedProcess, error) { if atomic.AddInt32(&spawnCalls, 1) == 1 { closeOnce.Do(func() { close(firstEntered) }) <-release // hold the first spawn open so the second Start can race in } return &fakeSpawnedProcess{}, nil } var wg sync.WaitGroup errs := make([]error, 2) wg.Add(1) go func() { defer wg.Done() errs[0] = m.startProcess("race-plugin", spawnFn) }() <-firstEntered // the first Start is now mid-spawn, before it has registered anything errs[1] = m.startProcess("race-plugin", spawnFn) close(release) wg.Wait() if got := atomic.LoadInt32(&spawnCalls); got != 1 { t.Fatalf("spawnFn called %d times for a concurrent Start of the same id, want exactly 1 (no double-spawn)", got) } if errs[0] != nil { t.Fatalf("first startProcess() unexpected error: %v", errs[0]) } if errs[1] == nil { t.Fatal("second concurrent startProcess() for the same id succeeded, want an error (already starting)") } if !m.Running("race-plugin") { t.Fatal("Running() = false after the winning Start() completed") } } // TestManager_StopDuringStartAbortsAndLeavesNoOrphanProcess pins the fix for // the Stop-during-Start orphan race: Stop previously only consulted // m.processes, so a Disable/Uninstall racing an in-flight Start was a // silent no-op — the spawn then completed and inserted a live supervised // process for a plugin whose record/files may already be gone by the time // the spawn finished. claimStart's "starting" reservation now doubles as // the hook Stop uses to request an abort: startProcess checks it right // after the (slow) spawn succeeds and, if set, kills the freshly spawned // process and never inserts it into m.processes. func TestManager_StopDuringStartAbortsAndLeavesNoOrphanProcess(t *testing.T) { m := NewManager(t.TempDir(), nil, testLogger(t)) t.Cleanup(m.StopAll) entered := make(chan struct{}) release := make(chan struct{}) var closeOnce sync.Once spawned := &fakeSpawnedProcess{} spawnFn := func() (spawnedProcess, error) { closeOnce.Do(func() { close(entered) }) <-release // hold the spawn open until the test has called Stop return spawned, nil } errCh := make(chan error, 1) go func() { errCh <- m.startProcess("racing-plugin", spawnFn) }() <-entered // the spawn is in flight, before startProcess has registered anything m.Stop("racing-plugin") // races the in-flight Start: must not be a no-op close(release) // let the spawn finish err := <-errCh if err == nil { t.Fatal("startProcess() expected an error when Stop() raced an in-flight Start, got nil") } if m.Running("racing-plugin") { t.Fatal("Running() = true after Stop() raced an in-flight Start; an orphan process was left registered") } if !spawned.isKilled() { t.Fatal("the spawned process was never killed after Stop() raced an in-flight Start (orphan subprocess)") } } // TestManager_StopAllDuringStartAbortsAndLeavesNoOrphanProcess is the same // race as TestManager_StopDuringStartAbortsAndLeavesNoOrphanProcess, but // through StopAll (graceful shutdown), which previously only iterated // m.processes too. func TestManager_StopAllDuringStartAbortsAndLeavesNoOrphanProcess(t *testing.T) { m := NewManager(t.TempDir(), nil, testLogger(t)) t.Cleanup(m.StopAll) entered := make(chan struct{}) release := make(chan struct{}) var closeOnce sync.Once spawned := &fakeSpawnedProcess{} spawnFn := func() (spawnedProcess, error) { closeOnce.Do(func() { close(entered) }) <-release return spawned, nil } errCh := make(chan error, 1) go func() { errCh <- m.startProcess("racing-plugin-2", spawnFn) }() <-entered m.StopAll() close(release) err := <-errCh if err == nil { t.Fatal("startProcess() expected an error when StopAll() raced an in-flight Start, got nil") } if m.Running("racing-plugin-2") { t.Fatal("Running() = true after StopAll() raced an in-flight Start; an orphan process was left registered") } if !spawned.isKilled() { t.Fatal("the spawned process was never killed after StopAll() raced an in-flight Start (orphan subprocess)") } } // TestManager_StartAfterExhaustionRespawns pins the fix that lets a fresh // Start cleanly replace an entry whose restart attempts were exhausted // (gaveUp==true, current==nil): before the fix, Start saw the stale map // entry and always errored "already running" instead of respawning. func TestManager_StartAfterExhaustionRespawns(t *testing.T) { m := NewManager(t.TempDir(), nil, testLogger(t), WithPingInterval(5*time.Millisecond), WithMaxConsecutiveFailures(1), WithRestartBackoff([]time.Duration{0}), WithMaxRestartAttempts(1), ) t.Cleanup(m.StopAll) var mu sync.Mutex firstSpawnDone := false spawnFn := func() (spawnedProcess, error) { mu.Lock() defer mu.Unlock() if !firstSpawnDone { firstSpawnDone = true return &fakeSpawnedProcess{exited: true}, nil // dies immediately, forcing an exhausting restart loop } return nil, errors.New("respawn always fails") // every restart attempt fails too } if err := m.startProcess("exhaust-plugin", spawnFn); err != nil { t.Fatalf("startProcess() unexpected error: %v", err) } // Wait for the supervision loop to actually finish giving up (gaveUp), // not just for current to go nil — current is cleared at the start of // handleFailureAndRestart, before the (failing) respawn attempt(s) run, // so polling only Running()==false races the loop's own completion. require.Eventually(t, func() bool { m.mu.Lock() p, ok := m.processes["exhaust-plugin"] m.mu.Unlock() return ok && p.isGaveUp() }, 2*time.Second, 5*time.Millisecond, "plugin's process should have given up once restart attempts are exhausted") healthy := &fakeSpawnedProcess{} respawnCalls := 0 freshSpawnFn := func() (spawnedProcess, error) { respawnCalls++ return healthy, nil } if err := m.startProcess("exhaust-plugin", freshSpawnFn); err != nil { t.Fatalf("startProcess() after exhaustion unexpected error: %v", err) } if respawnCalls != 1 { t.Fatalf("respawn spawnFn called %d times, want 1", respawnCalls) } if !m.Running("exhaust-plugin") { t.Fatal("Running() = false after a fresh Start() following exhaustion") } } // TestManager_ConcurrentEnableDuringRestartExhaustionDoesNotDeadlock pins the // lock ordering between the final supervision callback and a manual Enable. // The callback deliberately re-enters Manager.RestartCount after a barrier; // Enable must be able to stop the exhausted process without holding m.mu while // it waits for that callback to return. func TestManager_ConcurrentEnableDuringRestartExhaustionDoesNotDeadlock(t *testing.T) { callbackEntered := make(chan struct{}) releaseCallback := make(chan struct{}) var m *Manager m = NewManager(t.TempDir(), func(id string, healthy bool, reason error) { if healthy || reason == nil || reason.Error() != "plugin restart attempts exhausted" { return } close(callbackEntered) <-releaseCallback _ = m.RestartCount(id) }, testLogger(t)) t.Cleanup(m.StopAll) const id = "deadlock-plugin" p := newProcess(id, testLogger(t), nil, m.onStatusChange) p.maxRestartAttempts = 0 p.current = &fakeSpawnedProcess{} m.mu.Lock() m.processes[id] = p m.mu.Unlock() p.wg.Add(1) go func() { defer p.wg.Done() _ = p.handleFailureAndRestart(errors.New("initial failure")) }() <-callbackEntered startDone := make(chan error, 1) go func() { startDone <- m.startProcess(id, func() (spawnedProcess, error) { return &fakeSpawnedProcess{}, nil }) }() <-p.stopCh close(releaseCallback) select { case err := <-startDone: if err != nil { t.Fatalf("startProcess() returned unexpected error: %v", err) } case <-time.After(time.Second): t.Fatal("concurrent Enable remained blocked after restart-exhaustion callback was released") } m.Stop(id) } // TestManager_DefaultStartTimeoutIs30s pins the fix for bounded startup: // Manager.Start previously spawned via a hcplugin.Client with no configured // StartTimeout, defaulting to go-plugin's own 1-minute handshake timeout, so // a hung plugin binary could block Enable/Install for up to a minute. A // fresh Manager must default to a much shorter, explicit bound. func TestManager_DefaultStartTimeoutIs30s(t *testing.T) { m := NewManager(t.TempDir(), nil, testLogger(t)) if m.startTimeout != 30*time.Second { t.Fatalf("default startTimeout = %v, want 30s", m.startTimeout) } } // TestManager_WithStartTimeoutOverridesDefault pins the escape hatch for // callers (and this test file) that need a different bound. func TestManager_WithStartTimeoutOverridesDefault(t *testing.T) { m := NewManager(t.TempDir(), nil, testLogger(t), WithStartTimeout(5*time.Second)) if m.startTimeout != 5*time.Second { t.Fatalf("startTimeout = %v, want 5s (WithStartTimeout override)", m.startTimeout) } } // TestManager_StartFailsFastOnHandshakeTimeout proves the configured // startTimeout actually reaches the underlying hcplugin.Client: an // unreasonably short timeout must make Start() return an error quickly // instead of ever completing the handshake, even against a real, working // fixture binary. func TestManager_StartFailsFastOnHandshakeTimeout(t *testing.T) { rec := buildFixtureRecord(t, "kandev-fixture-plugin-timeout") m := NewManager(t.TempDir(), nil, testLogger(t), WithStartTimeout(1*time.Nanosecond)) t.Cleanup(m.StopAll) started := time.Now() err := m.Start(context.Background(), rec, func(string) pluginsdk.Host { return newFakeHost() }) elapsed := time.Since(started) if err == nil { t.Fatal("Start() with a 1ns startTimeout unexpectedly succeeded") } if elapsed > 10*time.Second { t.Fatalf("Start() took %v to fail, want well under go-plugin's 1-minute default handshake timeout", elapsed) } }