9aaf3170e0
* fix(lifecycle): drain StreamManager goroutines in tests Goroutines spawned by `agentctl.Client.StreamUpdates` / `StreamWorkspace` and by `lifecycle.StreamManager.connectWorkspaceStream` could outlive the tests that created them on slow CI runners, causing intermittent `goleak.VerifyTestMain` failures in `internal/agent/runtime/lifecycle` even though every individual test passed. The leak required the test to race: `client.Close()` could return while a workspace dial was still in flight, leaving the just-spawned WS read/write loops with nobody to drain them, and `StreamManager.Wait()` only fired the drain when the external `stopCh` had been closed first. Make the close paths absolute drain barriers: - `Client.Close()` now tracks every stream goroutine it spawns (`streamWG` + a per-stream `WorkspaceStream` reference) and blocks until all of them have exited. A `closed` flag flipped under the client mutex makes subsequent `StreamUpdates` / `StreamWorkspace` calls reject the dial, so a Close that races a dial in flight cannot leave behind a stranded WS connection. `manager_interaction.go` switches the restart path to per-stream Close helpers so it can keep reusing the client after teardown. - `StreamManager.Wait()` closes a new internal `waitCh` that the retry backoff (`sleepOrStop`) and the connected `<-ws.Done() / <-stop>` select also observe. This makes `Wait` an absolute drain barrier regardless of whether the caller ever closes the external `stopCh`. `streamContext` carries both stops through `stopChannelContext` so in-flight WebSocket dials cancel on either signal. Local stress (`make test-lifecycle-goleak LIFECYCLE_GOLEAK_COUNT=20`) passes 20/20 with `-race` after the fix. The new `TestClientClose_DrainsStreamGoroutines` covers the drain barrier and the post-Close rejection path. Added a `test-lifecycle-goleak` Makefile target as the repro hook for the CI-only flake. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix(lifecycle): track mergeStops goroutine and guard workspace stream races Address PR review feedback: - stopChannelContext now registers the mergeStops goroutine on an optional *sync.WaitGroup so sm.wg.Wait stays a true drain barrier even for the connectUpdatesStream path (outer goroutine returns immediately after dial). - StreamWorkspace re-checks workspaceStreamConn after dial to seal the concurrent-callers race that could orphan a conn. - readWorkspaceStream's defer now guards the workspaceStreamConn reset by identity so an old read goroutine cannot zero out a newer conn pointer. - Clarify comments on streamWG, Client.Close, and the mock.connected wait in the close-barrier test. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> * fix(lifecycle): drop agent-stream drain barrier to unblock workflow cascade The previous Close drain barrier for the agent (updates) stream blocked the on_turn_complete cascade flow. completeAndStopSession calls Close synchronously on the current session's client, and Close was waiting on streamWG (which tracked the read goroutine). The wait stalled the cascade long enough that Step2's auto-started session never got its profile name into the tab — surfacing as a flaky "Profile B tab not visible" failure in workflow-agent-switch E2E shard 7. Drain barrier is still in place for the workspace stream (where the original goleak flake lived). Agent stream is closed but not awaited: the cascade legitimately stops + restarts the agent stream on the same client, and the workspace stream's existing barrier is sufficient for the leak coverage. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: Kandev Agent <agent@kandev.dev>
243 lines
7.5 KiB
Go
243 lines
7.5 KiB
Go
package client
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
|
|
"github.com/gorilla/websocket"
|
|
"github.com/kandev/kandev/internal/agentctl/tracing"
|
|
"github.com/kandev/kandev/internal/agentctl/types"
|
|
"github.com/kandev/kandev/internal/common/logger"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// WorkspaceStreamCallbacks defines callbacks for workspace stream events
|
|
type WorkspaceStreamCallbacks struct {
|
|
OnShellOutput func(data string)
|
|
OnShellExit func(code int)
|
|
OnGitStatus func(update *GitStatusUpdate)
|
|
OnGitCommit func(notification *GitCommitNotification)
|
|
OnGitReset func(notification *GitResetNotification)
|
|
OnBranchSwitch func(notification *GitBranchSwitchNotification)
|
|
OnFileChange func(notification *FileChangeNotification)
|
|
OnProcessOutput func(output *types.ProcessOutput)
|
|
OnProcessStatus func(status *types.ProcessStatusUpdate)
|
|
OnConnected func()
|
|
OnError func(err string)
|
|
}
|
|
|
|
// WorkspaceStream represents an active workspace stream connection
|
|
type WorkspaceStream struct {
|
|
conn *websocket.Conn
|
|
inputCh chan types.WorkspaceStreamMessage
|
|
closeCh chan struct{}
|
|
closeOnce sync.Once
|
|
// wg tracks the read + write goroutines so Wait() can block until they have
|
|
// fully unwound. Close()/Done() only signal shutdown; the read loop may
|
|
// still be returning from a blocked ReadJSON when they fire, so callers
|
|
// that need a true drain barrier (StreamManager shutdown, leak-sensitive
|
|
// tests) must call Wait after Close/Done.
|
|
wg sync.WaitGroup
|
|
logger *logger.Logger
|
|
}
|
|
|
|
// StreamWorkspace opens a unified WebSocket connection for all workspace events
|
|
func (c *Client) StreamWorkspace(ctx context.Context, callbacks WorkspaceStreamCallbacks) (*WorkspaceStream, error) {
|
|
c.mu.Lock()
|
|
if c.closed {
|
|
c.mu.Unlock()
|
|
return nil, fmt.Errorf("agentctl client closed")
|
|
}
|
|
if c.workspaceStreamConn != nil {
|
|
c.mu.Unlock()
|
|
return nil, fmt.Errorf("workspace stream already connected")
|
|
}
|
|
c.mu.Unlock()
|
|
|
|
wsURL := "ws" + c.baseURL[4:] + "/api/v1/workspace/stream"
|
|
conn, _, err := websocket.DefaultDialer.DialContext(ctx, wsURL, c.wsAuthHeaders())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to connect to workspace stream: %w", err)
|
|
}
|
|
|
|
stream := &WorkspaceStream{
|
|
conn: conn,
|
|
inputCh: make(chan types.WorkspaceStreamMessage, 64),
|
|
closeCh: make(chan struct{}),
|
|
logger: c.logger,
|
|
}
|
|
|
|
// Race: Close may have fired between the dial returning and us re-acquiring
|
|
// the lock. Drop the new conn + stream instead of leaking the read/write
|
|
// goroutines past Client.Close's drain barrier.
|
|
c.mu.Lock()
|
|
if c.closed {
|
|
c.mu.Unlock()
|
|
_ = conn.Close()
|
|
return nil, fmt.Errorf("agentctl client closed during workspace stream dial")
|
|
}
|
|
// Re-check after dial: two concurrent StreamWorkspace callers can both pass
|
|
// the pre-dial guard and race here. The later one would orphan the first
|
|
// conn and its goroutines without this check.
|
|
if c.workspaceStreamConn != nil {
|
|
c.mu.Unlock()
|
|
_ = conn.Close()
|
|
return nil, fmt.Errorf("workspace stream already connected")
|
|
}
|
|
c.workspaceStreamConn = conn
|
|
c.workspaceStream = stream
|
|
c.mu.Unlock()
|
|
|
|
c.logger.Info("connected to workspace stream", zap.String("url", wsURL))
|
|
|
|
// Track both goroutines on the per-stream wg so WorkspaceStream.Wait can
|
|
// block until they have fully unwound. The workspace read loop only invokes
|
|
// data callbacks (shell/git/process) and self-closes on exit — it never
|
|
// re-enters manager teardown — so draining it is side-effect-free.
|
|
stream.wg.Add(2)
|
|
go func() {
|
|
defer stream.wg.Done()
|
|
c.readWorkspaceStream(conn, stream, callbacks)
|
|
}()
|
|
go func() {
|
|
defer stream.wg.Done()
|
|
stream.writeLoop(conn)
|
|
}()
|
|
|
|
return stream, nil
|
|
}
|
|
|
|
// workspaceTracedTypes contains message types that are low-volume and worth tracing.
|
|
// High-volume types (shell_output, process_output, shell_input, ping, pong) are excluded.
|
|
var workspaceTracedTypes = map[types.WorkspaceMessageType]bool{
|
|
types.WorkspaceMessageTypeGitStatus: true,
|
|
types.WorkspaceMessageTypeGitCommit: true,
|
|
types.WorkspaceMessageTypeGitReset: true,
|
|
types.WorkspaceMessageTypeBranchSwitch: true,
|
|
types.WorkspaceMessageTypeFileChange: true,
|
|
types.WorkspaceMessageTypeProcessStatus: true,
|
|
types.WorkspaceMessageTypeConnected: true,
|
|
types.WorkspaceMessageTypeError: true,
|
|
}
|
|
|
|
// readWorkspaceStream is the read loop for the workspace WebSocket stream.
|
|
func (c *Client) readWorkspaceStream(conn *websocket.Conn, stream *WorkspaceStream, callbacks WorkspaceStreamCallbacks) {
|
|
defer func() {
|
|
c.mu.Lock()
|
|
// Guard both resets by identity — a concurrent StreamWorkspace caller
|
|
// may have replaced the conn/stream pointers since this read loop started.
|
|
if c.workspaceStreamConn == conn {
|
|
c.workspaceStreamConn = nil
|
|
}
|
|
if c.workspaceStream == stream {
|
|
c.workspaceStream = nil
|
|
}
|
|
c.mu.Unlock()
|
|
stream.Close()
|
|
}()
|
|
|
|
for {
|
|
var msg types.WorkspaceStreamMessage
|
|
if err := conn.ReadJSON(&msg); err != nil {
|
|
if !websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway) {
|
|
c.logger.Debug("workspace stream read error", zap.Error(err))
|
|
}
|
|
return
|
|
}
|
|
if workspaceTracedTypes[msg.Type] {
|
|
tracing.TraceWorkspaceEvent(c.getTraceCtx(), string(msg.Type), c.executionID, c.sessionID)
|
|
}
|
|
dispatchWorkspaceMessage(msg, callbacks)
|
|
}
|
|
}
|
|
|
|
// writeLoop reads from the inputCh and writes messages to the WebSocket connection.
|
|
func (ws *WorkspaceStream) writeLoop(conn *websocket.Conn) {
|
|
for {
|
|
select {
|
|
case <-ws.closeCh:
|
|
return
|
|
case msg, ok := <-ws.inputCh:
|
|
if !ok {
|
|
return
|
|
}
|
|
if err := conn.WriteJSON(msg); err != nil {
|
|
ws.logger.Debug("workspace stream write error", zap.Error(err))
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// WriteShellInput sends input to the shell through the workspace stream
|
|
func (ws *WorkspaceStream) WriteShellInput(data string) error {
|
|
msg := types.NewWorkspaceShellInput(data)
|
|
select {
|
|
case ws.inputCh <- msg:
|
|
return nil
|
|
case <-ws.closeCh:
|
|
return fmt.Errorf("workspace stream closed")
|
|
}
|
|
}
|
|
|
|
// ResizeShell sends a shell resize command through the workspace stream
|
|
func (ws *WorkspaceStream) ResizeShell(cols, rows int) error {
|
|
msg := types.NewWorkspaceShellResize(cols, rows)
|
|
select {
|
|
case ws.inputCh <- msg:
|
|
return nil
|
|
case <-ws.closeCh:
|
|
return fmt.Errorf("workspace stream closed")
|
|
}
|
|
}
|
|
|
|
// Ping sends a ping message through the workspace stream
|
|
func (ws *WorkspaceStream) Ping() error {
|
|
msg := types.NewWorkspacePing()
|
|
select {
|
|
case ws.inputCh <- msg:
|
|
return nil
|
|
case <-ws.closeCh:
|
|
return fmt.Errorf("workspace stream closed")
|
|
}
|
|
}
|
|
|
|
// Close closes the workspace stream
|
|
func (ws *WorkspaceStream) Close() {
|
|
ws.closeOnce.Do(func() {
|
|
close(ws.closeCh)
|
|
if ws.conn != nil {
|
|
if err := ws.conn.Close(); err != nil {
|
|
ws.logger.Debug("failed to close workspace stream connection", zap.Error(err))
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// Done returns a channel that is closed when the stream is closed
|
|
func (ws *WorkspaceStream) Done() <-chan struct{} {
|
|
return ws.closeCh
|
|
}
|
|
|
|
// Wait blocks until the stream's read and write goroutines have fully exited.
|
|
// Done() only reports that shutdown was *requested* (closeCh closed); Wait
|
|
// reports that the goroutines have actually drained. Call after Close (or after
|
|
// Done fires) to make teardown a true barrier for leak detection.
|
|
func (ws *WorkspaceStream) Wait() {
|
|
ws.wg.Wait()
|
|
}
|
|
|
|
// CloseWorkspaceStream closes the workspace stream connection
|
|
func (c *Client) CloseWorkspaceStream() {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
|
|
if c.workspaceStreamConn != nil {
|
|
if err := c.workspaceStreamConn.Close(); err != nil {
|
|
c.logger.Debug("failed to close workspace stream", zap.Error(err))
|
|
}
|
|
c.workspaceStreamConn = nil
|
|
}
|
|
}
|