fix: recover workspace operations after backend restart (#475)

This commit is contained in:
Carlos Florêncio
2026-03-26 18:50:28 +00:00
committed by GitHub
parent 7ac5b3bb17
commit 0074cfbaa9
13 changed files with 339 additions and 54 deletions
+1
View File
@@ -290,6 +290,7 @@ Static analysis runs in CI and pre-commit. New code **must** stay within these l
### Backend
- Provider pattern for DI; stderr for logs, stdout for ACP only
- Pass context through chains; event bus for cross-component comm
- **Execution access:** Workspace-oriented handlers (files, shell, inference, ports, vscode, LSP) MUST use `GetOrEnsureExecution(ctx, sessionID)` — it recovers from backend restarts by creating executions on-demand. Only use `GetExecutionBySessionID` for operations that require a running agent process (prompt, cancel, mode).
### Frontend
- **Data:** SSR fetch → hydrate → read store. Never fetch in components
+1 -1
View File
@@ -25,6 +25,7 @@ require (
go.opentelemetry.io/otel/sdk v1.40.0
go.opentelemetry.io/otel/trace v1.40.0
go.uber.org/zap v1.26.0
golang.org/x/sync v0.19.0
)
require (
@@ -99,7 +100,6 @@ require (
golang.org/x/arch v0.3.0 // indirect
golang.org/x/crypto v0.46.0 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/term v0.38.0 // indirect
golang.org/x/text v0.32.0 // indirect
@@ -61,9 +61,9 @@ func (h *PortHandlers) wsPortList(ctx context.Context, msg *ws.Message) (*ws.Mes
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil)
}
execution, ok := h.lifecycleMgr.GetExecutionBySessionID(req.SessionID)
if !ok {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution", nil)
execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil)
}
client := execution.GetAgentCtlClient()
@@ -66,12 +66,12 @@ func (h *ShellHandlers) wsShellStatus(ctx context.Context, msg *ws.Message) (*ws
return nil, fmt.Errorf("session_id is required")
}
// Get the agent execution for this session
execution, ok := h.lifecycleMgr.GetExecutionBySessionID(req.SessionID)
if !ok {
// Get or create execution on-demand (survives backend restart)
execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{
"available": false,
"error": "no agent running for this session",
"error": "no agent running for this session: " + err.Error(),
})
}
@@ -3,6 +3,7 @@ package handlers
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
@@ -208,8 +209,9 @@ func TestWsShellStatus_NoInstanceFound(t *testing.T) {
if payload["available"] != false {
t.Errorf("expected available=false, got %v", payload["available"])
}
if payload["error"] != "no agent running for this session" {
t.Errorf("expected 'no agent running for this session', got %v", payload["error"])
errorStr, ok := payload["error"].(string)
if !ok || !strings.Contains(errorStr, "no agent running for this session") {
t.Errorf("expected error to contain 'no agent running for this session', got %v", payload["error"])
}
}
@@ -75,9 +75,9 @@ func (h *VscodeHandlers) wsVscodeStart(ctx context.Context, msg *ws.Message) (*w
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil)
}
execution, ok := h.lifecycleMgr.GetExecutionBySessionID(req.SessionID)
if !ok {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution", nil)
execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil)
}
client := execution.GetAgentCtlClient()
@@ -123,9 +123,9 @@ func (h *VscodeHandlers) wsVscodeStop(ctx context.Context, msg *ws.Message) (*ws
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil)
}
execution, ok := h.lifecycleMgr.GetExecutionBySessionID(req.SessionID)
if !ok {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution", nil)
execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil)
}
client := execution.GetAgentCtlClient()
@@ -161,8 +161,8 @@ func (h *VscodeHandlers) wsVscodeStatus(ctx context.Context, msg *ws.Message) (*
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil)
}
execution, ok := h.lifecycleMgr.GetExecutionBySessionID(req.SessionID)
if !ok {
execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewResponse(msg.ID, msg.Action, types.VscodeStatusResponse{Status: "stopped"})
}
@@ -221,9 +221,9 @@ func (h *VscodeHandlers) wsVscodeOpenFile(ctx context.Context, msg *ws.Message)
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "path is required", nil)
}
execution, ok := h.lifecycleMgr.GetExecutionBySessionID(req.SessionID)
if !ok {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution", nil)
execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil)
}
client := execution.GetAgentCtlClient()
@@ -54,9 +54,9 @@ func (h *WorkspaceFileHandlers) wsGetFileTree(ctx context.Context, msg *ws.Messa
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil)
}
// Get agent execution for this session
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, err := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+err.Error(), nil)
}
// Get agentctl client
@@ -94,9 +94,9 @@ func (h *WorkspaceFileHandlers) wsGetFileContent(ctx context.Context, msg *ws.Me
}
// Get agent execution for this session
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, err := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+err.Error(), nil)
}
// Get agentctl client
@@ -138,9 +138,9 @@ func (h *WorkspaceFileHandlers) wsGetFileContentAtRef(ctx context.Context, msg *
}
// Get agent execution for this session
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, err := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+err.Error(), nil)
}
// Get agentctl client
@@ -190,9 +190,9 @@ func (h *WorkspaceFileHandlers) wsUpdateFileContent(ctx context.Context, msg *ws
}
// Get agent execution for this session
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, err := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+err.Error(), nil)
}
// Get agentctl client
@@ -214,7 +214,7 @@ func (h *WorkspaceFileHandlers) wsUpdateFileContent(ctx context.Context, msg *ws
// resolveSessionFileClient parses session_id + path from the message payload
// and returns the agentctl client, or an error response.
func (h *WorkspaceFileHandlers) resolveSessionFileClient(
msg *ws.Message,
ctx context.Context, msg *ws.Message,
) (sessionID, path string, client *agentctl.Client, errResp *ws.Message) {
var req struct {
SessionID string `json:"session_id"`
@@ -234,9 +234,9 @@ func (h *WorkspaceFileHandlers) resolveSessionFileClient(
return "", "", nil, errResp
}
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, resolveErr := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if resolveErr != nil {
errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+resolveErr.Error(), nil)
return "", "", nil, errResp
}
c := execution.GetAgentCtlClient()
@@ -249,7 +249,7 @@ func (h *WorkspaceFileHandlers) resolveSessionFileClient(
// wsCreateFile handles workspace.file.create action
func (h *WorkspaceFileHandlers) wsCreateFile(ctx context.Context, msg *ws.Message) (*ws.Message, error) {
sessionID, path, client, errResp := h.resolveSessionFileClient(msg)
sessionID, path, client, errResp := h.resolveSessionFileClient(ctx, msg)
if errResp != nil {
return errResp, nil
}
@@ -265,7 +265,7 @@ func (h *WorkspaceFileHandlers) wsCreateFile(ctx context.Context, msg *ws.Messag
// wsDeleteFile handles workspace.file.delete action
func (h *WorkspaceFileHandlers) wsDeleteFile(ctx context.Context, msg *ws.Message) (*ws.Message, error) {
sessionID, path, client, errResp := h.resolveSessionFileClient(msg)
sessionID, path, client, errResp := h.resolveSessionFileClient(ctx, msg)
if errResp != nil {
return errResp, nil
}
@@ -296,9 +296,9 @@ func (h *WorkspaceFileHandlers) wsSearchFiles(ctx context.Context, msg *ws.Messa
}
// Get agent execution for this session
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, err := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+err.Error(), nil)
}
// Get agentctl client
@@ -346,9 +346,9 @@ func (h *WorkspaceFileHandlers) wsRenameFile(ctx context.Context, msg *ws.Messag
}
// Get agent execution for this session
execution, found := h.lifecycle.GetExecutionBySessionID(req.SessionID)
if !found {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session", nil)
execution, err := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID)
if err != nil {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+err.Error(), nil)
}
// Get agentctl client
@@ -7,6 +7,7 @@ import (
"time"
"go.uber.org/zap"
"golang.org/x/sync/singleflight"
"github.com/kandev/kandev/internal/agent/docker"
"github.com/kandev/kandev/internal/agent/executor"
@@ -72,6 +73,9 @@ type Manager struct {
// from agentctl instances through the agent stream.
mcpHandler agentctl.MCPHandler
// singleflight deduplicates concurrent GetOrEnsureExecution calls for the same session
ensureExecutionGroup singleflight.Group
// Background remote status polling
remoteStatusPollInterval time.Duration
remoteStatusMu sync.RWMutex
@@ -17,6 +17,36 @@ import (
// have a resolved workspace path (typically while worktree preparation is in progress).
var ErrSessionWorkspaceNotReady = errors.New("session workspace not ready")
// GetOrEnsureExecution returns an existing execution or creates one on-demand.
// Use this for workspace-oriented operations (files, shell, inference, ports, vscode, LSP)
// that should survive backend restarts. For operations requiring a running agent
// process (prompt, cancel, mode), use GetExecutionBySessionID instead.
//
// Concurrent calls for the same sessionID are deduplicated via singleflight.
func (m *Manager) GetOrEnsureExecution(ctx context.Context, sessionID string) (*AgentExecution, error) {
if sessionID == "" {
return nil, fmt.Errorf("session_id is required")
}
// Fast path: execution already in memory
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
return execution, nil
}
// Slow path: create on-demand, deduplicated by singleflight
v, err, _ := m.ensureExecutionGroup.Do(sessionID, func() (interface{}, error) {
// Double-check after acquiring singleflight slot
if execution, exists := m.executionStore.GetBySessionID(sessionID); exists {
return execution, nil
}
return m.EnsureWorkspaceExecutionForSession(ctx, "", sessionID)
})
if err != nil {
return nil, err
}
return v.(*AgentExecution), nil
}
// EnsureWorkspaceExecutionForSession ensures an agentctl execution exists for a specific task session.
// This is used when the frontend provides a session ID (e.g., from URL path /task/[id]/[sessionId]).
// If an execution already exists for the session, it returns it. Otherwise, it creates a new execution
@@ -37,6 +67,11 @@ func (m *Manager) EnsureWorkspaceExecutionForSession(ctx context.Context, taskID
return nil, fmt.Errorf("failed to get workspace info for session %s: %w", sessionID, err)
}
// Resolve taskID from provider when caller doesn't have it (e.g., GetOrEnsureExecution)
if taskID == "" {
taskID = info.TaskID
}
if info.WorkspacePath == "" {
return nil, fmt.Errorf("no workspace path available for session %s", sessionID)
}
@@ -1,9 +1,15 @@
package lifecycle
import (
"context"
"errors"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
func TestErrSessionWorkspaceNotReady_ErrorsIs(t *testing.T) {
@@ -31,3 +37,236 @@ func TestErrSessionWorkspaceNotReady_UnrelatedError(t *testing.T) {
t.Errorf("expected errors.Is to be false for unrelated error")
}
}
func TestGetOrEnsureExecution(t *testing.T) {
t.Run("returns existing execution from store", func(t *testing.T) {
store := NewExecutionStore()
execution := &AgentExecution{
ID: "exec-1",
SessionID: "session-1",
TaskID: "task-1",
Status: v1.AgentStatusRunning,
}
store.Add(execution)
providerCalled := false
provider := &mockWorkspaceInfoProvider{
infos: map[string]*WorkspaceInfo{},
}
// Wrap to detect calls
mgr := &Manager{
executionStore: store,
workspaceInfoProvider: provider,
logger: newTestLogger(),
}
// Override provider to track calls
trackingProvider := &trackingWorkspaceInfoProvider{
delegate: provider,
called: &providerCalled,
}
mgr.workspaceInfoProvider = trackingProvider
got, err := mgr.GetOrEnsureExecution(context.Background(), "session-1")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got.ID != "exec-1" {
t.Errorf("expected execution ID %q, got %q", "exec-1", got.ID)
}
if providerCalled {
t.Error("provider should not be called when execution exists in store")
}
})
t.Run("empty session ID returns error", func(t *testing.T) {
mgr := &Manager{
executionStore: NewExecutionStore(),
logger: newTestLogger(),
}
_, err := mgr.GetOrEnsureExecution(context.Background(), "")
if err == nil {
t.Fatal("expected error for empty session ID")
}
if err.Error() != "session_id is required" {
t.Errorf("unexpected error message: %v", err)
}
})
t.Run("no provider returns error", func(t *testing.T) {
mgr := &Manager{
executionStore: NewExecutionStore(),
workspaceInfoProvider: nil,
logger: newTestLogger(),
}
_, err := mgr.GetOrEnsureExecution(context.Background(), "session-1")
if err == nil {
t.Fatal("expected error when provider is nil")
}
})
t.Run("provider error is propagated", func(t *testing.T) {
provider := &mockWorkspaceInfoProvider{
err: fmt.Errorf("database connection failed"),
}
mgr := &Manager{
executionStore: NewExecutionStore(),
workspaceInfoProvider: provider,
logger: newTestLogger(),
}
_, err := mgr.GetOrEnsureExecution(context.Background(), "session-1")
if err == nil {
t.Fatal("expected error from provider")
}
if !containsString(err.Error(), "database connection failed") {
t.Errorf("expected error to contain provider error, got: %v", err)
}
})
t.Run("concurrent calls use singleflight", func(t *testing.T) {
store := NewExecutionStore()
var callCount atomic.Int32
// Slow provider to create a race window
provider := &slowWorkspaceInfoProvider{
delay: 50 * time.Millisecond,
callCount: &callCount,
info: &WorkspaceInfo{
TaskID: "task-1",
SessionID: "session-1",
WorkspacePath: "/tmp/test",
AgentID: "auggie",
},
}
mgr := &Manager{
executionStore: store,
workspaceInfoProvider: provider,
logger: newTestLogger(),
}
// Both calls will fail at createExecution (no executor backend),
// but singleflight should ensure the provider is called at most once.
var wg sync.WaitGroup
wg.Add(2)
for range 2 {
go func() {
defer wg.Done()
_, _ = mgr.GetOrEnsureExecution(context.Background(), "session-1")
}()
}
wg.Wait()
if callCount.Load() > 1 {
t.Errorf("expected provider to be called at most once (singleflight), got %d calls", callCount.Load())
}
})
}
func TestEnsureWorkspaceExecutionForSession_EmptyTaskID(t *testing.T) {
t.Run("resolves taskID from provider when empty", func(t *testing.T) {
provider := &mockWorkspaceInfoProvider{
infos: map[string]*WorkspaceInfo{
"session-1": {
TaskID: "resolved-task-id",
SessionID: "session-1",
WorkspacePath: "/tmp/test",
AgentID: "auggie",
},
},
}
mgr := &Manager{
executionStore: NewExecutionStore(),
workspaceInfoProvider: provider,
logger: newTestLogger(),
}
// This will fail at createExecution (no executor backend),
// but we can verify the taskID resolution by checking the error path.
// The error should NOT be about empty taskID.
_, err := mgr.EnsureWorkspaceExecutionForSession(context.Background(), "", "session-1")
if err == nil {
t.Fatal("expected error (no executor backend)")
}
// Should fail at createExecution, not at taskID validation
if containsString(err.Error(), "task_id") || containsString(err.Error(), "taskID") {
t.Errorf("unexpected taskID-related error: %v", err)
}
})
t.Run("uses provided taskID when not empty", func(t *testing.T) {
provider := &mockWorkspaceInfoProvider{
infos: map[string]*WorkspaceInfo{
"session-1": {
TaskID: "provider-task-id",
SessionID: "session-1",
WorkspacePath: "/tmp/test",
AgentID: "auggie",
},
},
}
mgr := &Manager{
executionStore: NewExecutionStore(),
workspaceInfoProvider: provider,
logger: newTestLogger(),
}
// This will fail at createExecution (no executor backend),
// but the explicit taskID should be passed through.
_, err := mgr.EnsureWorkspaceExecutionForSession(context.Background(), "explicit-task-id", "session-1")
if err == nil {
t.Fatal("expected error (no executor backend)")
}
// Should fail at createExecution, not at taskID
if containsString(err.Error(), "task_id") || containsString(err.Error(), "taskID") {
t.Errorf("unexpected taskID-related error: %v", err)
}
})
}
// --- test helpers ---
func containsString(s, substr string) bool {
return len(s) >= len(substr) && searchString(s, substr)
}
func searchString(s, substr string) bool {
for i := 0; i <= len(s)-len(substr); i++ {
if s[i:i+len(substr)] == substr {
return true
}
}
return false
}
// trackingWorkspaceInfoProvider wraps a provider and tracks whether it was called.
type trackingWorkspaceInfoProvider struct {
delegate WorkspaceInfoProvider
called *bool
}
func (p *trackingWorkspaceInfoProvider) GetWorkspaceInfoForSession(ctx context.Context, taskID, sessionID string) (*WorkspaceInfo, error) {
*p.called = true
return p.delegate.GetWorkspaceInfoForSession(ctx, taskID, sessionID)
}
// slowWorkspaceInfoProvider adds a delay to simulate slow DB lookups for concurrency tests.
type slowWorkspaceInfoProvider struct {
delay time.Duration
callCount *atomic.Int32
info *WorkspaceInfo
err error
}
func (p *slowWorkspaceInfoProvider) GetWorkspaceInfoForSession(_ context.Context, _, _ string) (*WorkspaceInfo, error) {
p.callCount.Add(1)
time.Sleep(p.delay)
if p.err != nil {
return nil, p.err
}
return p.info, nil
}
@@ -515,12 +515,15 @@ func (m *Manager) GetExecution(executionID string) (*AgentExecution, bool) {
return m.executionStore.Get(executionID)
}
// GetExecutionBySessionID returns the agent execution for a session.
// GetExecutionBySessionID returns the agent execution for a session from the in-memory store only.
//
// Returns (execution, true) if found, or (nil, false) if not found.
// A session can have at most one active execution at a time. If a session exists
// but has no active execution, this returns (nil, false).
//
// For workspace-oriented operations that need restart recovery,
// use GetOrEnsureExecution instead.
//
// Thread-safe: Can be called concurrently from multiple goroutines.
func (m *Manager) GetExecutionBySessionID(sessionID string) (*AgentExecution, bool) {
return m.executionStore.GetBySessionID(sessionID)
@@ -29,10 +29,10 @@ func (m *Manager) ExecuteInferencePrompt(ctx context.Context, sessionID, agentID
return nil, fmt.Errorf("agent %q inference not supported", agentID)
}
// Get the agentctl client
execution, exists := m.executionStore.GetBySessionID(sessionID)
if !exists {
return nil, fmt.Errorf("no execution found for session %s", sessionID)
// Get or create execution on-demand (survives backend restart)
execution, err := m.GetOrEnsureExecution(ctx, sessionID)
if err != nil {
return nil, fmt.Errorf("no execution available for session %s: %w", sessionID, err)
}
client := execution.GetAgentCtlClient()
@@ -96,11 +96,12 @@ func (h *LSPHandler) HandleLSPConnection(c *gin.Context) {
zap.String("session_id", sessionID),
zap.String("language", language))
// Get workspace path from execution
execution, exists := h.lifecycleMgr.GetExecutionBySessionID(sessionID)
if !exists {
// Get or create execution on-demand (survives backend restart)
execution, err := h.lifecycleMgr.GetOrEnsureExecution(c.Request.Context(), sessionID)
if err != nil {
h.logger.Warn("LSP: session not found in lifecycle manager",
zap.String("session_id", sessionID))
zap.String("session_id", sessionID),
zap.Error(err))
h.closeWithCode(c, lspCloseSessionNotFound, "session not found")
return
}