diff --git a/apps/backend/internal/agent/agents/managed_npm_runtime.go b/apps/backend/internal/agent/agents/managed_npm_runtime.go index f45c4e915e..8893a7c78b 100644 --- a/apps/backend/internal/agent/agents/managed_npm_runtime.go +++ b/apps/backend/internal/agent/agents/managed_npm_runtime.go @@ -1,8 +1,7 @@ package agents import ( - "crypto/sha512" - "encoding/hex" + "github.com/kandev/kandev/internal/agent/managedruntime" ) // ManagedNPMRuntimeSpec defines a built-in npm-distributed ACP runtime. @@ -26,14 +25,24 @@ func (s ManagedNPMRuntimeSpec) PackageSpec(version string) string { // SHA-512 and the first 16 lowercase hexadecimal characters. The optional // argument preserves the unversioned legacy key when omitted. func (s ManagedNPMRuntimeSpec) ExecutionCacheKey(versions ...string) string { - digest := sha512.Sum512([]byte(s.PackageSpec(firstVersion(versions)))) - return hex.EncodeToString(digest[:])[:16] + return managedruntime.NpxExecutionCacheKey(s.PackageSpec(firstVersion(versions))) } // ACPCommand returns the normal launch command for the exact version when one // is supplied. An empty version keeps the legacy unversioned behavior. func (s ManagedNPMRuntimeSpec) ACPCommand(version string) Command { - args := []string{"npx", "--yes", "--prefer-offline", s.PackageSpec(version)} + return s.ACPCommandWithNpmPreference(version, false) +} + +// ACPCommandWithNpmPreference builds a managed runtime launch command. The +// package spec and ACP arguments remain trusted agent metadata; recovery only +// changes npm's metadata freshness preference. +func (s ManagedNPMRuntimeSpec) ACPCommandWithNpmPreference(version string, preferOnline bool) Command { + preference := "--prefer-offline" + if preferOnline { + preference = "--prefer-online" + } + args := []string{"npx", "--yes", preference, s.PackageSpec(version)} args = append(args, s.ACPArgs...) return NewCommand(args...) } diff --git a/apps/backend/internal/agent/agents/managed_npm_runtime_test.go b/apps/backend/internal/agent/agents/managed_npm_runtime_test.go index abeb89ab87..8b6915dfc6 100644 --- a/apps/backend/internal/agent/agents/managed_npm_runtime_test.go +++ b/apps/backend/internal/agent/agents/managed_npm_runtime_test.go @@ -1,6 +1,7 @@ package agents import ( + "reflect" "slices" "strings" "testing" @@ -80,6 +81,24 @@ func TestManagedNPMRuntimeBuildsExactVersionCommandsAndCacheKey(t *testing.T) { } } +func TestManagedNPMRuntimeOnlineCommandChangesOnlyNpmFreshnessFlag(t *testing.T) { + spec := ManagedNPMRuntimeSpec{ + Package: "@scope/managed-acp", + ACPArgs: []string{"--acp", "--model", "fast"}, + } + + offline := spec.ACPCommand("1.2.3").Args() + online := spec.ACPCommandWithNpmPreference("1.2.3", true).Args() + want := []string{"npx", "--yes", "--prefer-online", "@scope/managed-acp@1.2.3", "--acp", "--model", "fast"} + if !reflect.DeepEqual(online, want) { + t.Fatalf("online argv = %#v, want %#v", online, want) + } + offline[2] = "--prefer-online" + if !reflect.DeepEqual(offline, online) { + t.Fatalf("online command changed more than npm preference: offline=%#v online=%#v", offline, online) + } +} + func TestManagedNPMRuntimeExactVersionSupportsScopedPackages(t *testing.T) { spec := ManagedNPMRuntimeSpec{Package: "@scope/managed-acp", ACPArgs: []string{"--acp"}} want := []string{"npx", "--yes", "--prefer-offline", "@scope/managed-acp@3.4.5", "--acp"} diff --git a/apps/backend/internal/agent/managedruntime/cache.go b/apps/backend/internal/agent/managedruntime/cache.go new file mode 100644 index 0000000000..f61c93de8a --- /dev/null +++ b/apps/backend/internal/agent/managedruntime/cache.go @@ -0,0 +1,60 @@ +package managedruntime + +import ( + "crypto/sha512" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const npxExecutionKeyLength = 16 + +// NpxExecutionCacheKey returns npm's deterministic _npx execution-tree key +// for one trusted package spec. The package spec must be the exact value that +// the managed runtime command passes to npx. +func NpxExecutionCacheKey(packageSpec string) string { + digest := sha512.Sum512([]byte(packageSpec)) + return hex.EncodeToString(digest[:])[:npxExecutionKeyLength] +} + +// RemoveNpxExecutionTree removes only the _npx tree derived from packageSpec. +// It refuses broad roots and symlinked paths so a cache repair cannot escape +// the exact trusted execution key. +func RemoveNpxExecutionTree(cacheRoot, packageSpec string) error { + root, err := validateNpmCacheRoot(cacheRoot) + if err != nil { + return err + } + packageSpec = strings.TrimSpace(packageSpec) + if packageSpec == "" || strings.ContainsRune(packageSpec, '\x00') || strings.Contains(packageSpec, "..") { + return errors.New("managed runtime package spec is invalid") + } + + key := NpxExecutionCacheKey(packageSpec) + return removeNpxExecutionTree(root, key) +} + +func validateNpmCacheRoot(cacheRoot string) (string, error) { + cacheRoot = strings.TrimSpace(cacheRoot) + if cacheRoot == "" || !filepath.IsAbs(cacheRoot) { + return "", fmt.Errorf("npm cache root is not absolute: %q", cacheRoot) + } + root := filepath.Clean(cacheRoot) + if root == string(filepath.Separator) || filepath.Base(root) == "_npx" { + return "", errors.New("refusing to invalidate npm cache root") + } + if info, err := os.Lstat(root); err == nil { + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("refusing to invalidate execution cache through symlink: %s", root) + } + if !info.IsDir() { + return "", fmt.Errorf("npm cache root is not a directory: %s", root) + } + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("inspect npm cache root: %w", err) + } + return root, nil +} diff --git a/apps/backend/internal/agent/managedruntime/cache_test.go b/apps/backend/internal/agent/managedruntime/cache_test.go new file mode 100644 index 0000000000..a280375236 --- /dev/null +++ b/apps/backend/internal/agent/managedruntime/cache_test.go @@ -0,0 +1,97 @@ +package managedruntime + +import ( + "os" + "path/filepath" + "testing" +) + +func TestRemoveNpxExecutionTreeDeletesOnlyExactTrustedKey(t *testing.T) { + cacheRoot := t.TempDir() + npxRoot := filepath.Join(cacheRoot, "_npx") + if err := os.MkdirAll(npxRoot, 0o755); err != nil { + t.Fatal(err) + } + key := NpxExecutionCacheKey("@scope/managed-acp@1.2.3") + target := filepath.Join(npxRoot, key) + other := filepath.Join(npxRoot, "0123456789abcdef") + if err := os.MkdirAll(target, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(other, 0o755); err != nil { + t.Fatal(err) + } + + if err := RemoveNpxExecutionTree(cacheRoot, "@scope/managed-acp@1.2.3"); err != nil { + t.Fatalf("RemoveNpxExecutionTree: %v", err) + } + if _, err := os.Stat(target); !os.IsNotExist(err) { + t.Fatalf("target stat error = %v, want not-exist", err) + } + if _, err := os.Stat(other); err != nil { + t.Fatalf("unrelated key was removed: %v", err) + } +} + +func TestRemoveNpxExecutionTreeIsIdempotentAndRejectsUnsafeTargets(t *testing.T) { + cacheRoot := t.TempDir() + if err := RemoveNpxExecutionTree(cacheRoot, "managed-acp@1.2.3"); err != nil { + t.Fatalf("absent key should be idempotent: %v", err) + } + if err := RemoveNpxExecutionTree(cacheRoot, ""); err == nil { + t.Fatal("empty package spec should be rejected") + } + if err := RemoveNpxExecutionTree(cacheRoot, "../escape"); err == nil { + t.Fatal("path-like package spec should be rejected") + } + + npxRoot := filepath.Join(cacheRoot, "_npx") + if err := os.MkdirAll(npxRoot, 0o755); err != nil { + t.Fatal(err) + } + escape := filepath.Join(t.TempDir(), "outside") + if err := os.WriteFile(escape, []byte("do not delete"), 0o600); err != nil { + t.Fatal(err) + } + key := NpxExecutionCacheKey("managed-acp@1.2.3") + if err := os.Symlink(escape, filepath.Join(npxRoot, key)); err != nil { + t.Fatal(err) + } + if err := RemoveNpxExecutionTree(cacheRoot, "managed-acp@1.2.3"); err == nil { + t.Fatal("symlinked execution tree should be rejected") + } + if _, err := os.Stat(escape); err != nil { + t.Fatalf("symlink escape target was touched: %v", err) + } + + regularFile := filepath.Join(npxRoot, NpxExecutionCacheKey("managed-acp@1.2.4")) + if err := os.WriteFile(regularFile, []byte("not a tree"), 0o600); err != nil { + t.Fatal(err) + } + if err := RemoveNpxExecutionTree(cacheRoot, "managed-acp@1.2.4"); err == nil { + t.Fatal("regular-file execution tree should be rejected") + } + if _, err := os.Stat(regularFile); err != nil { + t.Fatalf("regular-file execution tree was removed: %v", err) + } +} + +func TestRemoveNpxExecutionTreeRejectsCacheAndNpxRootSymlinks(t *testing.T) { + cacheTarget := t.TempDir() + cacheLink := filepath.Join(t.TempDir(), "cache-link") + if err := os.Symlink(cacheTarget, cacheLink); err != nil { + t.Fatal(err) + } + if err := RemoveNpxExecutionTree(cacheLink, "managed-acp@1.2.3"); err == nil { + t.Fatal("symlinked npm cache root should be rejected") + } + + cacheRoot := t.TempDir() + npxTarget := t.TempDir() + if err := os.Symlink(npxTarget, filepath.Join(cacheRoot, "_npx")); err != nil { + t.Fatal(err) + } + if err := RemoveNpxExecutionTree(cacheRoot, "managed-acp@1.2.3"); err == nil { + t.Fatal("symlinked _npx root should be rejected") + } +} diff --git a/apps/backend/internal/agent/managedruntime/cache_unix.go b/apps/backend/internal/agent/managedruntime/cache_unix.go new file mode 100644 index 0000000000..1b29d0dd5c --- /dev/null +++ b/apps/backend/internal/agent/managedruntime/cache_unix.go @@ -0,0 +1,145 @@ +//go:build !windows + +package managedruntime + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +const managedRuntimeDirectoryOpenFlags = unix.O_RDONLY | unix.O_DIRECTORY | unix.O_NOFOLLOW | unix.O_CLOEXEC + +// removeNpxExecutionTree walks the cache using directory descriptors. Every +// component is opened with O_NOFOLLOW and the final unlink is relative to the +// still-open _npx descriptor, so a concurrent replacement cannot redirect the +// deletion through a symlink. +func removeNpxExecutionTree(cacheRoot, key string) error { + rootFD, err := openManagedRuntimeDirectoryPath(cacheRoot) + if errors.Is(err, unix.ENOENT) { + return nil + } + if err != nil { + return fmt.Errorf("open npm cache root: %w", err) + } + defer func() { _ = unix.Close(rootFD) }() + + npxFD, err := unix.Openat(rootFD, "_npx", managedRuntimeDirectoryOpenFlags, 0) + if errors.Is(err, unix.ENOENT) { + return nil + } + if err != nil { + return fmt.Errorf("open npm execution cache: %w", err) + } + defer func() { _ = unix.Close(npxFD) }() + + targetFD, err := unix.Openat(npxFD, key, managedRuntimeDirectoryOpenFlags, 0) + if errors.Is(err, unix.ENOENT) { + return nil + } + if err != nil { + return fmt.Errorf("open npm execution tree: %w", err) + } + defer func() { _ = unix.Close(targetFD) }() + + if err := removeManagedRuntimeDirectoryContents(targetFD); err != nil { + return fmt.Errorf("remove npm execution tree contents: %w", err) + } + if err := unix.Unlinkat(npxFD, key, unix.AT_REMOVEDIR); err != nil && !errors.Is(err, unix.ENOENT) { + return fmt.Errorf("remove npm execution tree: %w", err) + } + return nil +} + +func openManagedRuntimeDirectoryPath(path string) (int, error) { + if !filepath.IsAbs(path) { + return -1, fmt.Errorf("npm cache root must be absolute") + } + fd, err := unix.Open(string(filepath.Separator), managedRuntimeDirectoryOpenFlags, 0) + if err != nil { + return -1, err + } + for _, part := range managedRuntimeAbsolutePathComponents(path) { + next, openErr := unix.Openat(fd, part, managedRuntimeDirectoryOpenFlags, 0) + if openErr != nil { + _ = unix.Close(fd) + return -1, openErr + } + _ = unix.Close(fd) + fd = next + } + return fd, nil +} + +func managedRuntimeAbsolutePathComponents(path string) []string { + trimmed := strings.TrimPrefix(filepath.ToSlash(filepath.Clean(path)), "/") + if trimmed == "" { + return nil + } + parts := strings.Split(trimmed, "/") + components := make([]string, 0, len(parts)) + for _, part := range parts { + if part != "" && part != "." { + components = append(components, part) + } + } + return components +} + +func removeManagedRuntimeDirectoryContents(dirFD int) error { + names, err := readManagedRuntimeDirectoryNames(dirFD) + if err != nil { + return err + } + for _, name := range names { + var stat unix.Stat_t + if err := unix.Fstatat(dirFD, name, &stat, unix.AT_SYMLINK_NOFOLLOW); err != nil { + if errors.Is(err, unix.ENOENT) { + continue + } + return err + } + if stat.Mode&unix.S_IFMT == unix.S_IFDIR { + childFD, err := unix.Openat(dirFD, name, managedRuntimeDirectoryOpenFlags, 0) + if err != nil { + return err + } + err = removeManagedRuntimeDirectoryContents(childFD) + _ = unix.Close(childFD) + if err != nil { + return err + } + if err := unix.Unlinkat(dirFD, name, unix.AT_REMOVEDIR); err != nil && !errors.Is(err, unix.ENOENT) { + return err + } + continue + } + if err := unix.Unlinkat(dirFD, name, 0); err != nil && !errors.Is(err, unix.ENOENT) { + return err + } + } + return nil +} + +func readManagedRuntimeDirectoryNames(dirFD int) ([]string, error) { + readFD, err := unix.Dup(dirFD) + if err != nil { + return nil, err + } + dir := os.NewFile(uintptr(readFD), "managed-runtime-cache") + if dir == nil { + _ = unix.Close(readFD) + return nil, fmt.Errorf("open npm execution directory descriptor") + } + names, err := dir.Readdirnames(-1) + _ = dir.Close() + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + return names, nil +} diff --git a/apps/backend/internal/agent/managedruntime/cache_windows.go b/apps/backend/internal/agent/managedruntime/cache_windows.go new file mode 100644 index 0000000000..68cffdce80 --- /dev/null +++ b/apps/backend/internal/agent/managedruntime/cache_windows.go @@ -0,0 +1,201 @@ +//go:build windows + +package managedruntime + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "unsafe" + + "golang.org/x/sys/windows" +) + +// Windows uses native handles rooted at the cache. OBJ_DONT_REPARSE and +// FILE_OPEN_REPARSE_POINT reject junctions and symlinks while each component +// is opened; deletion is then requested on the opened handle. +func removeNpxExecutionTree(cacheRoot, key string) error { + rootHandle, err := openManagedRuntimeDirectoryPath(cacheRoot) + if errors.Is(err, windows.ERROR_PATH_NOT_FOUND) || errors.Is(err, windows.ERROR_FILE_NOT_FOUND) { + return nil + } + if err != nil { + return fmt.Errorf("open npm cache root: %w", err) + } + defer windows.CloseHandle(rootHandle) + + npxHandle, err := openManagedRuntimeDirectoryRelative(rootHandle, "_npx") + if errors.Is(err, windows.ERROR_PATH_NOT_FOUND) || errors.Is(err, windows.ERROR_FILE_NOT_FOUND) { + return nil + } + if err != nil { + return fmt.Errorf("open npm execution cache: %w", err) + } + if err := rejectManagedRuntimeReparse(npxHandle); err != nil { + return err + } + defer windows.CloseHandle(npxHandle) + + targetHandle, err := openManagedRuntimeDirectoryRelative(npxHandle, key) + if errors.Is(err, windows.ERROR_PATH_NOT_FOUND) || errors.Is(err, windows.ERROR_FILE_NOT_FOUND) { + return nil + } + if err != nil { + return fmt.Errorf("open npm execution tree: %w", err) + } + if err := rejectManagedRuntimeReparse(targetHandle); err != nil { + return err + } + defer windows.CloseHandle(targetHandle) + + if err := removeManagedRuntimeWindowsContents(targetHandle); err != nil { + return fmt.Errorf("remove npm execution tree contents: %w", err) + } + return markManagedRuntimeForDelete(targetHandle) +} + +func openManagedRuntimeDirectoryPath(path string) (windows.Handle, error) { + return openManagedRuntimeHandle(0, managedRuntimeWindowsNTPath(path), windows.FILE_DIRECTORY_FILE) +} + +func openManagedRuntimeDirectoryRelative(parent windows.Handle, name string) (windows.Handle, error) { + return openManagedRuntimeHandle(parent, name, windows.FILE_DIRECTORY_FILE) +} + +func openManagedRuntimeEntryRelative(parent windows.Handle, name string) (windows.Handle, error) { + return openManagedRuntimeHandle(parent, name, windows.FILE_NON_DIRECTORY_FILE) +} + +func openManagedRuntimeHandle(parent windows.Handle, name string, createOption uint32) (windows.Handle, error) { + objectName, err := windows.NewNTUnicodeString(name) + if err != nil { + return 0, err + } + objectAttributes := &windows.OBJECT_ATTRIBUTES{ + Length: uint32(unsafe.Sizeof(windows.OBJECT_ATTRIBUTES{})), + RootDirectory: parent, + ObjectName: objectName, + Attributes: windows.OBJ_CASE_INSENSITIVE | windows.OBJ_DONT_REPARSE, + } + var ioStatus windows.IO_STATUS_BLOCK + var allocationSize int64 + var handle windows.Handle + err = windows.NtCreateFile( + &handle, + windows.FILE_GENERIC_READ|windows.FILE_GENERIC_WRITE|windows.DELETE, + objectAttributes, + &ioStatus, + &allocationSize, + 0, + windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE, + windows.FILE_OPEN, + createOption|windows.FILE_OPEN_REPARSE_POINT, + 0, + 0, + ) + if err != nil { + return 0, err + } + return handle, nil +} + +func managedRuntimeWindowsNTPath(path string) string { + clean := filepath.Clean(path) + if strings.HasPrefix(clean, `\\?\`) { + return `\??\` + strings.TrimPrefix(clean, `\\?\`) + } + if strings.HasPrefix(clean, `\\`) { + return `\??\UNC\` + strings.TrimPrefix(clean, `\\`) + } + return `\??\` + clean +} + +func rejectManagedRuntimeReparse(handle windows.Handle) error { + var info windows.ByHandleFileInformation + if err := windows.GetFileInformationByHandle(handle, &info); err != nil { + _ = windows.CloseHandle(handle) + return err + } + if info.FileAttributes&windows.FILE_ATTRIBUTE_REPARSE_POINT != 0 { + _ = windows.CloseHandle(handle) + return fmt.Errorf("refusing to invalidate execution cache through reparse point") + } + return nil +} + +func removeManagedRuntimeWindowsContents(dirHandle windows.Handle) error { + readHandle, err := duplicateManagedRuntimeHandle(dirHandle) + if err != nil { + return err + } + dir := os.NewFile(uintptr(readHandle), "managed-runtime-cache") + if dir == nil { + _ = windows.CloseHandle(readHandle) + return fmt.Errorf("open npm execution directory handle") + } + names, err := dir.Readdirnames(-1) + _ = dir.Close() + if err != nil && !errors.Is(err, io.EOF) { + return err + } + for _, name := range names { + child, dirErr := openManagedRuntimeDirectoryRelative(dirHandle, name) + if dirErr == nil { + if err := rejectManagedRuntimeReparse(child); err != nil { + return err + } + if err := removeManagedRuntimeWindowsContents(child); err != nil { + _ = windows.CloseHandle(child) + return err + } + err := markManagedRuntimeForDelete(child) + _ = windows.CloseHandle(child) + if err != nil { + return err + } + continue + } + + file, fileErr := openManagedRuntimeEntryRelative(dirHandle, name) + if fileErr != nil { + return fmt.Errorf("open npm execution entry %s: %w", name, fileErr) + } + if err := rejectManagedRuntimeReparse(file); err != nil { + return err + } + err = markManagedRuntimeForDelete(file) + _ = windows.CloseHandle(file) + if err != nil { + return err + } + } + return nil +} + +func duplicateManagedRuntimeHandle(handle windows.Handle) (windows.Handle, error) { + var duplicate windows.Handle + if err := windows.DuplicateHandle( + windows.CurrentProcess(), handle, windows.CurrentProcess(), &duplicate, 0, false, + windows.DUPLICATE_SAME_ACCESS, + ); err != nil { + return 0, err + } + return duplicate, nil +} + +func markManagedRuntimeForDelete(handle windows.Handle) error { + info := struct{ Flags uint32 }{ + Flags: windows.FILE_DISPOSITION_DELETE | + windows.FILE_DISPOSITION_POSIX_SEMANTICS | + windows.FILE_DISPOSITION_IGNORE_READONLY_ATTRIBUTE, + } + return windows.SetFileInformationByHandle( + handle, + windows.FileDispositionInfoEx, + (*byte)(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ) +} diff --git a/apps/backend/internal/agent/runtime/agentctl/agent.go b/apps/backend/internal/agent/runtime/agentctl/agent.go index b075f36d6e..bf58e52760 100644 --- a/apps/backend/internal/agent/runtime/agentctl/agent.go +++ b/apps/backend/internal/agent/runtime/agentctl/agent.go @@ -398,7 +398,7 @@ func (c *Client) readUpdatesStream( var lastErr error defer func() { - // Clean up pending requests BEFORE draining the worker. On a + // Clean up this connection's pending requests BEFORE draining the worker. On a // connection drop mid-cancel, a worker handler // (orchestrator.handleAgentReady) can block acquiring the per-session // cancelInFlight guard that an in-flight Service.CancelAgent holds @@ -408,8 +408,8 @@ func (c *Client) readUpdatesStream( // the blocked handler finish and the worker exit. Draining first // (<-workerDone) would wait on that handler forever, so cleanup could // never run — the exact deadlock class this stream rework fixes, but on - // the disconnect path. - c.cleanupPendingRequests() + // the disconnect path. A replacement stream's requests remain pending. + c.cleanupPendingRequests(conn) // Stop the worker and wait for the in-flight handler to unwind before // signaling disconnect, so the drain barrier semantics callers rely on @@ -418,7 +418,12 @@ func (c *Client) readUpdatesStream( <-workerDone c.mu.Lock() - c.agentStreamConn = nil + // A startup retry can install a replacement connection before the + // first reader's deferred cleanup runs. Only the reader that owns the + // current connection may clear the shared handle. + if c.agentStreamConn == conn { + c.agentStreamConn = nil + } c.mu.Unlock() if err := conn.Close(); err != nil { c.logger.Debug("failed to close updates websocket", zap.Error(err)) diff --git a/apps/backend/internal/agent/runtime/agentctl/agent_test.go b/apps/backend/internal/agent/runtime/agentctl/agent_test.go index 5cc1bb3337..429deda103 100644 --- a/apps/backend/internal/agent/runtime/agentctl/agent_test.go +++ b/apps/backend/internal/agent/runtime/agentctl/agent_test.go @@ -264,7 +264,7 @@ func TestCleanupPendingRequests(t *testing.T) { c.pendingRequests["req-2"] = ch2 c.pendingMu.Unlock() - c.cleanupPendingRequests() + c.cleanupPendingRequests(nil) c.pendingMu.Lock() count := len(c.pendingRequests) @@ -305,7 +305,7 @@ func TestCleanupPendingRequests_ConcurrentSafe(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - c.cleanupPendingRequests() + c.cleanupPendingRequests(nil) }() wg.Wait() @@ -318,6 +318,42 @@ func TestCleanupPendingRequests_ConcurrentSafe(t *testing.T) { } } +func TestCleanupPendingRequests_ScopesToRetiredStream(t *testing.T) { + c := &Client{ + pendingRequests: make(map[string]chan *ws.Message), + pendingRequestConns: make(map[string]*websocket.Conn), + } + oldConn := &websocket.Conn{} + newConn := &websocket.Conn{} + oldCh := make(chan *ws.Message, 1) + newCh := make(chan *ws.Message, 1) + c.pendingMu.Lock() + c.pendingRequests["old"] = oldCh + c.pendingRequests["new"] = newCh + c.pendingRequestConns["old"] = oldConn + c.pendingRequestConns["new"] = newConn + c.pendingMu.Unlock() + + c.cleanupPendingRequests(oldConn) + + if _, ok := <-oldCh; ok { + t.Fatal("expected retired stream request channel to be closed") + } + select { + case _, ok := <-newCh: + if !ok { + t.Fatal("replacement stream request channel was closed") + } + t.Fatal("replacement stream request channel received an unexpected response") + default: + } + c.pendingMu.Lock() + defer c.pendingMu.Unlock() + if _, ok := c.pendingRequests["new"]; !ok { + t.Fatal("replacement stream request was removed") + } +} + // --- Tests for the 7 migrated methods --- func TestInitialize_Success(t *testing.T) { @@ -819,6 +855,8 @@ func TestStreamUpdates_DisconnectCleansPending(t *testing.T) { ch := make(chan *ws.Message, 1) c.pendingMu.Lock() c.pendingRequests["pending-test"] = ch + c.pendingRequestConns = make(map[string]*websocket.Conn) + c.pendingRequestConns["pending-test"] = c.agentStreamConn c.pendingMu.Unlock() // Wait for disconnect diff --git a/apps/backend/internal/agent/runtime/agentctl/client.go b/apps/backend/internal/agent/runtime/agentctl/client.go index a8984c51bf..3fc355fc98 100644 --- a/apps/backend/internal/agent/runtime/agentctl/client.go +++ b/apps/backend/internal/agent/runtime/agentctl/client.go @@ -56,8 +56,9 @@ type Client struct { streamWriteMu sync.Mutex // Pending request/response tracking for agent stream - pendingRequests map[string]chan *ws.Message - pendingMu sync.Mutex + pendingRequests map[string]chan *ws.Message + pendingRequestConns map[string]*websocket.Conn + pendingMu sync.Mutex } // ClientOption configures optional Client settings. @@ -127,8 +128,9 @@ func NewClient(host string, port int, log *logger.Logger, opts ...ClientOption) longRunningHTTPClient: &http.Client{ Timeout: 5 * time.Minute, }, - logger: log.WithFields(zap.String("component", "agentctl-client")), - pendingRequests: make(map[string]chan *ws.Message), + logger: log.WithFields(zap.String("component", "agentctl-client")), + pendingRequests: make(map[string]chan *ws.Message), + pendingRequestConns: make(map[string]*websocket.Conn), } for _, opt := range opts { opt(c) diff --git a/apps/backend/internal/agent/runtime/agentctl/client_stream.go b/apps/backend/internal/agent/runtime/agentctl/client_stream.go index 48e00ee800..7354423a54 100644 --- a/apps/backend/internal/agent/runtime/agentctl/client_stream.go +++ b/apps/backend/internal/agent/runtime/agentctl/client_stream.go @@ -50,13 +50,21 @@ func (c *Client) sendStreamRequest(ctx context.Context, action string, payload i // Register pending request respCh := make(chan *ws.Message, 1) c.pendingMu.Lock() + if c.pendingRequests == nil { + c.pendingRequests = make(map[string]chan *ws.Message) + } + if c.pendingRequestConns == nil { + c.pendingRequestConns = make(map[string]*websocket.Conn) + } c.pendingRequests[reqID] = respCh + c.pendingRequestConns[reqID] = conn c.pendingMu.Unlock() // Clean up on exit defer func() { c.pendingMu.Lock() delete(c.pendingRequests, reqID) + delete(c.pendingRequestConns, reqID) c.pendingMu.Unlock() }() @@ -109,27 +117,35 @@ func (c *Client) resolvePendingRequest(msg *ws.Message) bool { c.pendingMu.Lock() ch, ok := c.pendingRequests[msg.ID] + if ok { + // Resolve while holding pendingMu so a stream cleanup cannot close the + // channel between the lookup and this send. + select { + case ch <- msg: + default: + } + } c.pendingMu.Unlock() if !ok { return false } - - // Send response to waiting caller (non-blocking since channel is buffered) - select { - case ch <- msg: - default: - } return true } -// cleanupPendingRequests unblocks all pending requests with nil (signaling disconnect). -func (c *Client) cleanupPendingRequests() { +// cleanupPendingRequests unblocks pending requests owned by conn with nil +// (signaling disconnect). A nil connection retains the all-stream cleanup +// behavior used by tests and shutdown paths. +func (c *Client) cleanupPendingRequests(conn *websocket.Conn) { c.pendingMu.Lock() defer c.pendingMu.Unlock() for id, ch := range c.pendingRequests { + if conn != nil && c.pendingRequestConns[id] != conn { + continue + } close(ch) delete(c.pendingRequests, id) + delete(c.pendingRequestConns, id) } } diff --git a/apps/backend/internal/agent/runtime/lifecycle/event_types.go b/apps/backend/internal/agent/runtime/lifecycle/event_types.go index 1a12392b56..fd3e7dd20e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/event_types.go +++ b/apps/backend/internal/agent/runtime/lifecycle/event_types.go @@ -21,6 +21,8 @@ type AgentEventPayload struct { StartedAt time.Time `json:"started_at"` FinishedAt *time.Time `json:"finished_at,omitempty"` ErrorMessage string `json:"error_message,omitempty"` + FailureCode string `json:"failure_code,omitempty"` + FailureDetails string `json:"failure_details,omitempty"` ProviderError *streams.ProviderError `json:"provider_error,omitempty"` ExitCode *int `json:"exit_code,omitempty"` PromptGeneration uint64 `json:"prompt_generation,omitempty"` @@ -49,6 +51,8 @@ type AgentctlEventPayload struct { TaskEnvironmentID string `json:"task_environment_id,omitempty"` AgentExecutionID string `json:"agent_execution_id"` ErrorMessage string `json:"error_message,omitempty"` + FailureCode string `json:"failure_code,omitempty"` + FailureDetails string `json:"failure_details,omitempty"` WorktreeID string `json:"worktree_id,omitempty"` WorktreePath string `json:"worktree_path,omitempty"` WorktreeBranch string `json:"worktree_branch,omitempty"` diff --git a/apps/backend/internal/agent/runtime/lifecycle/events.go b/apps/backend/internal/agent/runtime/lifecycle/events.go index ab4a652a69..39cc9a152f 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/events.go +++ b/apps/backend/internal/agent/runtime/lifecycle/events.go @@ -100,6 +100,8 @@ func newAgentEventPayload(execution *AgentExecution) AgentEventPayload { StartedAt: execution.StartedAt, FinishedAt: execution.FinishedAt, ErrorMessage: execution.ErrorMessage, + FailureCode: execution.FailureCode, + FailureDetails: execution.FailureDetails, ProviderError: execution.ProviderError, ExitCode: execution.ExitCode, PromptGeneration: execution.promptGeneration, @@ -127,6 +129,8 @@ func (p *EventPublisher) PublishAgentctlEvent(ctx context.Context, eventType str TaskEnvironmentID: execution.TaskEnvironmentID, AgentExecutionID: execution.ID, ErrorMessage: errMsg, + FailureCode: execution.FailureCode, + FailureDetails: execution.FailureDetails, WorktreeID: worktreeID, WorktreePath: execution.WorkspacePath, WorktreeBranch: worktreeBranch, diff --git a/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_recovery.go b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_recovery.go new file mode 100644 index 0000000000..04e3ebdae3 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_recovery.go @@ -0,0 +1,49 @@ +package lifecycle + +import ( + "strings" + + "github.com/kandev/kandev/internal/agent/agents" + "github.com/kandev/kandev/internal/agent/managedruntime" +) + +// onlineManagedRuntimeArgs returns a copy of a trusted managed-npm launch +// command with only npm's metadata preference changed. The package spec is +// returned separately so cache invalidation can target the exact execution +// tree used by the failed launch. +func onlineManagedRuntimeArgs(args []string, spec agents.ManagedNPMRuntimeSpec) ([]string, string, bool) { + packageName := strings.TrimSpace(spec.Package) + if packageName == "" || len(args) < 4 { + return nil, "", false + } + + for npxIndex, arg := range args { + if arg != "npx" || npxIndex+3 >= len(args) { + continue + } + if args[npxIndex+1] != "--yes" || args[npxIndex+2] != "--prefer-offline" { + continue + } + + packageSpec := args[npxIndex+3] + if packageSpec != packageName { + versionPrefix := packageName + "@" + if !strings.HasPrefix(packageSpec, versionPrefix) { + return nil, "", false + } + version := strings.TrimPrefix(packageSpec, versionPrefix) + if version == "" { + return nil, "", false + } + if _, err := managedruntime.ParseStableVersion(version); err != nil { + return nil, "", false + } + } + + recovered := append([]string(nil), args...) + recovered[npxIndex+2] = "--prefer-online" + return recovered, packageSpec, true + } + + return nil, "", false +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go new file mode 100644 index 0000000000..ac4d2cb62d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go @@ -0,0 +1,326 @@ +package lifecycle + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/kandev/kandev/internal/agent/agents" + "github.com/kandev/kandev/internal/agent/managedruntime" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + agentctltypes "github.com/kandev/kandev/internal/agentctl/types" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/pkg/api/v1" + "go.uber.org/zap" +) + +const managedRuntimeStartupStderrTimeout = 2 * time.Second + +const managedRuntimeStartupSettleDelay = 500 * time.Millisecond + +// managedRuntimeNpmStartupFailure captures only the bounded stderr needed to +// classify an ACP initialization failure. The command is never built from +// this data. +func (m *Manager) managedRuntimeNpmStartupFailure( + ctx context.Context, + execution *AgentExecution, + initErr error, +) *routingerr.Error { + if execution == nil || execution.agentctl == nil || initErr == nil { + return nil + } + stderrCtx, cancel := context.WithTimeout(ctx, managedRuntimeStartupStderrTimeout) + defer cancel() + lines, err := execution.agentctl.GetAgentStderr(stderrCtx) + if err != nil { + return nil + } + evidence := strings.Join(lines, "\n") + return routingerr.Classify(routingerr.Input{ + Phase: routingerr.PhaseSessionInit, + ProviderID: execution.AgentID, + Stderr: strings.TrimSpace(evidence + "\n" + initErr.Error()), + }) +} + +func managedRuntimeRecoveryAborted(ctx context.Context, m *Manager) error { + if err := context.Cause(ctx); err != nil { + return err + } + if m.IsShuttingDown() { + return errors.New("managed runtime recovery interrupted by shutdown") + } + return nil +} + +func invalidateManagedRuntimeExecutionTree( + ctx context.Context, + invalidator ManagedRuntimeCacheInvalidator, + spec agents.ManagedNPMRuntimeSpec, + packageSpec string, +) error { + if invalidator == nil { + return errors.New("managed runtime cache invalidator is unavailable") + } + if packageSpec == spec.Package { + return invalidator.InvalidateExecutionCache(ctx, spec.Package) + } + version := strings.TrimPrefix(packageSpec, spec.Package+"@") + if version == "" || version == packageSpec { + return errors.New("managed runtime package spec is not versioned") + } + if _, err := managedruntime.ParseStableVersion(version); err != nil { + return err + } + return invalidator.InvalidateExecutionCacheVersion(ctx, spec.Package, version) +} + +func (m *Manager) updateExecutionFailure(executionID, message, code, details string) { + if m.executionStore == nil { + return + } + _ = m.executionStore.WithLock(executionID, func(execution *AgentExecution) { + execution.ErrorMessage = message + execution.FailureCode = code + execution.FailureDetails = details + execution.Status = v1.AgentStatusFailed + }) +} + +func (m *Manager) publishManagedRuntimeStartupFailure( + execution *AgentExecution, + message string, + code routingerr.Code, + details string, + cause error, +) error { + m.updateExecutionFailure(execution.ID, message, string(code), details) + return &routingerr.ManagedRuntimeStartupError{ + Code: code, + Details: details, + Cause: cause, + } +} + +type managedRuntimeStartupRetry struct { + managed agents.ManagedNPMRuntimeAgent + preferOnlineArgs []string + packageSpec string + failureDetails string +} + +func managedRuntimeRetryFailureClassification( + initialCode routingerr.Code, + initialDetails string, + retryErr error, + initializationFailed bool, + second *routingerr.Error, +) (routingerr.Code, string) { + if retryErr == nil { + return initialCode, initialDetails + } + if !initializationFailed || second == nil { + return routingerr.CodeAgentRuntime, routingerr.Sanitize(retryErr.Error()) + } + details := second.RawExcerpt + if details == "" { + details = routingerr.Sanitize(retryErr.Error()) + } + return second.Code, details +} + +func (m *Manager) prepareManagedRuntimeStartupRetry( + ctx context.Context, + execution *AgentExecution, + initErr error, + agentConfig agents.Agent, +) (*managedRuntimeStartupRetry, bool) { + if execution == nil || execution.RuntimeName != agentruntime.RuntimeStandalone || execution.agentctl == nil { + return nil, false + } + managed, ok := agentConfig.(agents.ManagedNPMRuntimeAgent) + if !ok || m.managedRuntimeCacheInvalidator == nil { + return nil, false + } + classification := m.managedRuntimeNpmStartupFailure(ctx, execution, initErr) + if classification == nil || classification.Code != routingerr.CodeManagedRuntimeNpmResolution { + return nil, false + } + preferOnlineArgs, packageSpec, ok := onlineManagedRuntimeArgs(execution.AgentArgs, managed.ManagedNPMRuntime()) + if !ok { + return nil, false + } + return &managedRuntimeStartupRetry{ + managed: managed, + preferOnlineArgs: preferOnlineArgs, + packageSpec: packageSpec, + failureDetails: classification.RawExcerpt, + }, true +} + +// stopAndInvalidateManagedRuntime stops the failed child and removes only its +// trusted npm execution tree. needsFailure distinguishes an ordinary repair +// error from cancellation or shutdown, which must win over recovery. +func (m *Manager) stopAndInvalidateManagedRuntime( + ctx context.Context, + execution *AgentExecution, + retry *managedRuntimeStartupRetry, +) (needsFailure bool, err error) { + if stopErr := execution.agentctl.Stop(ctx); stopErr != nil { + if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { + return false, aborted + } + m.logger.Warn("managed runtime process stop failed before retry; continuing", + zap.String("execution_id", execution.ID), zap.Error(stopErr)) + } + execution.agentctl.CloseUpdatesStream() + if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { + return false, aborted + } + + spec := retry.managed.ManagedNPMRuntime() + if err := invalidateManagedRuntimeExecutionTree(ctx, m.managedRuntimeCacheInvalidator, spec, retry.packageSpec); err != nil { + if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { + return false, aborted + } + return true, err + } + if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { + return false, aborted + } + return false, nil +} + +func (m *Manager) startManagedRuntimeRetry( + ctx context.Context, + execution *AgentExecution, + agentConfig agents.Agent, + approvalPolicy string, + taskDescription string, + attachments []MessageAttachment, + mcpServers []agentctltypes.McpServer, +) (initializationFailed bool, err error) { + if _, err := m.configureAndStartAgent(ctx, execution, approvalPolicy); err != nil { + return false, err + } + if err := managedRuntimeRecoveryAborted(ctx, m); err != nil { + return false, err + } + settleTimer := time.NewTimer(managedRuntimeStartupSettleDelay) + defer settleTimer.Stop() + select { + case <-settleTimer.C: + case <-ctx.Done(): + if cause := context.Cause(ctx); cause != nil { + return false, cause + } + return false, ctx.Err() + } + if err := managedRuntimeRecoveryAborted(ctx, m); err != nil { + return false, err + } + if err := m.initializeACPSession(ctx, execution, agentConfig, taskDescription, attachments, mcpServers); err != nil { + return true, err + } + return false, nil +} + +func (m *Manager) resetManagedRuntimeExecutionForRetry(execution *AgentExecution, args []string) { + _ = m.executionStore.WithLock(execution.ID, func(current *AgentExecution) { + current.AgentArgs = append([]string(nil), args...) + current.AgentCommand = strings.Join(args, " ") + current.Status = v1.AgentStatusStarting + current.ErrorMessage = "" + current.FailureCode = "" + current.FailureDetails = "" + current.setSessionInitialized(false) + m.resetStreamingStateWithHistory(current) + select { + case <-current.promptDoneCh: + default: + } + }) +} + +// retryManagedRuntimeStartup performs the single online metadata retry for a +// host-local managed npm runtime. attempted is true only after the replacement +// process has been selected, so callers can preserve the original path when a +// failure is unrelated or recovery is unavailable. +func (m *Manager) retryManagedRuntimeStartup( + ctx context.Context, + execution *AgentExecution, + initErr error, + agentConfig agents.Agent, + approvalPolicy string, + taskDescription string, + attachments []MessageAttachment, + mcpServers []agentctltypes.McpServer, +) (attempted bool, err error) { + retry, ok := m.prepareManagedRuntimeStartupRetry(ctx, execution, initErr, agentConfig) + if !ok { + return false, initErr + } + failureCode := routingerr.CodeManagedRuntimeNpmResolution + failureDetails := retry.failureDetails + var failureCause error + if err := managedRuntimeRecoveryAborted(ctx, m); err != nil { + return false, err + } + retryGeneration, ok := execution.beginStartupRecovery() + if !ok { + return false, initErr + } + defer execution.finishStartupRecovery() + + m.logger.Warn("retrying managed npm runtime startup with online metadata", + zap.String("execution_id", execution.ID), + zap.String("agent_id", execution.AgentID), + zap.Uint64("startup_generation", retryGeneration)) + + // Stop only the child process. The agentctl server remains alive so the + // same execution can reconnect its streams and retain its identity. + if needsFailure, err := m.stopAndInvalidateManagedRuntime(ctx, execution, retry); err != nil { + if needsFailure { + failureCode = routingerr.CodeAgentRuntime + failureDetails = routingerr.Sanitize(err.Error()) + failureCause = err + return false, m.publishManagedRuntimeStartupFailure( + execution, "managed runtime cache repair failed", failureCode, failureDetails, failureCause, + ) + } + return false, err + } + if err := managedRuntimeRecoveryAborted(ctx, m); err != nil { + return false, err + } + + m.resetManagedRuntimeExecutionForRetry(execution, retry.preferOnlineArgs) + + initializationFailed, retryErr := m.startManagedRuntimeRetry( + ctx, + execution, + agentConfig, + approvalPolicy, + taskDescription, + attachments, + mcpServers, + ) + if retryErr != nil { + if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { + return false, aborted + } + failureCause = retryErr + var second *routingerr.Error + if initializationFailed { + second = m.managedRuntimeNpmStartupFailure(ctx, execution, retryErr) + } + failureCode, failureDetails = managedRuntimeRetryFailureClassification( + failureCode, failureDetails, retryErr, initializationFailed, second, + ) + return true, m.publishManagedRuntimeStartupFailure( + execution, "managed npm runtime failed to prepare", failureCode, failureDetails, failureCause, + ) + } + return true, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup_recovery_test.go b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup_recovery_test.go new file mode 100644 index 0000000000..eceb7c8810 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup_recovery_test.go @@ -0,0 +1,326 @@ +package lifecycle + +import ( + "context" + "errors" + "reflect" + "slices" + "strings" + "sync" + "testing" + + "github.com/kandev/kandev/internal/agent/agents" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + "github.com/kandev/kandev/internal/agentruntime" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +func TestStartupAttemptGenerationRejectsStaleEventsAfterRecovery(t *testing.T) { + execution := &AgentExecution{} + first := execution.beginStartupAttempt() + if !execution.acceptsStartupAttempt(first) { + t.Fatal("initial startup generation should be current") + } + + second, ok := execution.beginStartupRecovery() + if !ok { + t.Fatal("expected startup recovery generation") + } + if second == first || execution.acceptsStartupAttempt(first) { + t.Fatal("first child generation remained current after recovery") + } + if !execution.acceptsStartupAttempt(second) { + t.Fatal("retry startup generation should be current") + } + if _, ok := execution.beginStartupRecovery(); ok { + t.Fatal("startup recovery should be attempted at most once") + } + execution.finishStartupRecovery() +} + +func TestOnlineManagedRuntimeArgsPreserveTrustedLaunchIdentity(t *testing.T) { + spec := agents.ManagedNPMRuntimeSpec{ + Package: "@scope/managed-acp", + ACPArgs: []string{"--acp", "--model", "fast"}, + } + initial := []string{"greywall", "--", "npx", "--yes", "--prefer-offline", "@scope/managed-acp@1.2.3", "--acp", "--model", "fast"} + + got, packageSpec, ok := onlineManagedRuntimeArgs(initial, spec) + if !ok { + t.Fatal("expected managed runtime recovery command") + } + want := []string{"greywall", "--", "npx", "--yes", "--prefer-online", "@scope/managed-acp@1.2.3", "--acp", "--model", "fast"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("online args = %#v, want %#v", got, want) + } + if packageSpec != "@scope/managed-acp@1.2.3" { + t.Fatalf("package spec = %q, want exact selected spec", packageSpec) + } +} + +func TestOnlineManagedRuntimeArgsRejectsNonManagedCommands(t *testing.T) { + spec := agents.ManagedNPMRuntimeSpec{Package: "managed-acp"} + for _, args := range [][]string{ + {"native-agent", "--acp"}, + {"npx", "--yes", "--prefer-offline", "other-agent@1.2.3"}, + {"npx", "--yes", "--prefer-online", "managed-acp@1.2.3"}, + } { + if _, _, ok := onlineManagedRuntimeArgs(args, spec); ok { + t.Fatalf("command %#v should not be eligible for managed runtime recovery", args) + } + } +} + +func TestOnlineManagedRuntimeArgsAcceptsTrustedUnversionedPackage(t *testing.T) { + spec := agents.ManagedNPMRuntimeSpec{Package: "managed-acp"} + args := []string{"npx", "--yes", "--prefer-offline", "managed-acp", "--acp"} + + got, packageSpec, ok := onlineManagedRuntimeArgs(args, spec) + if !ok { + t.Fatal("expected the trusted unversioned command to be eligible") + } + if packageSpec != "managed-acp" { + t.Fatalf("package spec = %q, want managed-acp", packageSpec) + } + want := []string{"npx", "--yes", "--prefer-online", "managed-acp", "--acp"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("online args = %#v, want %#v", got, want) + } +} + +func TestManagedRuntimeRetryFailureClassificationPreservesSecondResult(t *testing.T) { + retryErr := errors.New("failed to initialize ACP: secret=abcdefghijklmnopqrstuvwxyz123456") + second := &routingerr.Error{ + Code: routingerr.CodeAuthRequired, + RawExcerpt: "authentication required", + } + + code, details := managedRuntimeRetryFailureClassification( + routingerr.CodeManagedRuntimeNpmResolution, + "initial npm details", + retryErr, + true, + second, + ) + if code != routingerr.CodeAuthRequired || details != "authentication required" { + t.Fatalf("retry classification = (%q, %q), want auth_required and sanitized second excerpt", code, details) + } + + code, details = managedRuntimeRetryFailureClassification( + routingerr.CodeManagedRuntimeNpmResolution, + "initial npm details", + retryErr, + false, + nil, + ) + if code != routingerr.CodeAgentRuntime { + t.Fatalf("configure failure code = %q, want %q", code, routingerr.CodeAgentRuntime) + } + if details == retryErr.Error() { + t.Fatal("configure failure details must be sanitized") + } +} + +type managedRuntimeRetryInvalidator struct { + mu sync.Mutex + calls []string + err error + cancel context.CancelFunc + canceled bool +} + +func (i *managedRuntimeRetryInvalidator) InvalidateExecutionCache(_ context.Context, packageName string) error { + i.mu.Lock() + defer i.mu.Unlock() + i.calls = append(i.calls, packageName) + if i.cancel != nil && !i.canceled { + i.canceled = true + i.cancel() + } + return i.err +} + +func (i *managedRuntimeRetryInvalidator) InvalidateExecutionCacheVersion(_ context.Context, packageName, version string) error { + i.mu.Lock() + defer i.mu.Unlock() + i.calls = append(i.calls, packageName+"@"+version) + if i.cancel != nil && !i.canceled { + i.canceled = true + i.cancel() + } + return i.err +} + +func (i *managedRuntimeRetryInvalidator) Calls() []string { + i.mu.Lock() + defer i.mu.Unlock() + return append([]string(nil), i.calls...) +} + +func newManagedRuntimeRetryFixture(t *testing.T, failSessionNew bool) (*Manager, *AgentExecution, *restartMockAgentctlServer, agents.Agent, *managedRuntimeRetryInvalidator) { + t.Helper() + mgr := newTestManager(t) + mock := newRestartMockAgentctlServer(t, false, failSessionNew) + client := createTestClient(t, mock.server.URL) + t.Cleanup(client.Close) + if err := client.StreamUpdates(context.Background(), func(agentctl.AgentEvent) {}, nil, nil); err != nil { + t.Fatalf("connect initial agent stream: %v", err) + } + + agentConfig := agents.NewOpenCodeACP() + execution := &AgentExecution{ + ID: "managed-runtime-retry-execution", + TaskID: "task-1", + SessionID: "session-1", + AgentID: agentConfig.ID(), + RuntimeName: agentruntime.RuntimeStandalone, + AgentCommand: "npx", + AgentArgs: []string{"npx", "--yes", "--prefer-offline", "opencode-ai@1.2.3", "acp", "--print-logs", "--log-level", "ERROR"}, + WorkspacePath: "/workspace", + Status: v1.AgentStatusStarting, + agentctl: client, + promptDoneCh: make(chan PromptCompletionSignal, 1), + } + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatalf("add execution: %v", err) + } + execution.beginStartupAttempt() + invalidator := &managedRuntimeRetryInvalidator{} + mgr.SetManagedRuntimeCacheInvalidator(invalidator) + return mgr, execution, mock, agentConfig, invalidator +} + +func TestRetryManagedRuntimeStartupLifecycle(t *testing.T) { + initialErr := errors.New("ACP session initialization failed") + + t.Run("successful recovery uses one online replacement", func(t *testing.T) { + mgr, execution, mock, agentConfig, invalidator := newManagedRuntimeRetryFixture(t, false) + + attempted, err := mgr.retryManagedRuntimeStartup( + context.Background(), execution, initialErr, agentConfig, "", "", nil, nil, + ) + if err != nil { + t.Fatalf("retryManagedRuntimeStartup: %v", err) + } + if !attempted { + t.Fatal("expected one managed runtime retry") + } + if got := invalidator.Calls(); !slices.Equal(got, []string{"opencode-ai@1.2.3"}) { + t.Fatalf("invalidator calls = %#v", got) + } + if got := execution.AgentArgs; !slices.Contains(got, "--prefer-online") || slices.Contains(got, "--prefer-offline") { + t.Fatalf("replacement args = %#v, want online preference only", got) + } + if got := mock.getHTTPActions(); !slices.Equal(got, []string{"stop", "configure", "start"}) { + t.Fatalf("HTTP actions = %#v", got) + } + if execution.FailureCode != "" || execution.Status == v1.AgentStatusFailed { + t.Fatalf("successful recovery left failure state: code=%q status=%q", execution.FailureCode, execution.Status) + } + }) + + t.Run("repeated initialization failure is terminal after one retry", func(t *testing.T) { + mgr, execution, mock, agentConfig, invalidator := newManagedRuntimeRetryFixture(t, true) + + attempted, err := mgr.retryManagedRuntimeStartup( + context.Background(), execution, initialErr, agentConfig, "", "", nil, nil, + ) + if !attempted { + t.Fatal("expected one managed runtime retry") + } + var startupErr *routingerr.ManagedRuntimeStartupError + if !errors.As(err, &startupErr) { + t.Fatalf("retry error = %v, want structured startup error", err) + } + if startupErr.Code != routingerr.CodeManagedRuntimeNpmResolution { + t.Fatalf("retry code = %q", startupErr.Code) + } + if execution.FailureCode != string(routingerr.CodeManagedRuntimeNpmResolution) { + t.Fatalf("execution failure code = %q", execution.FailureCode) + } + if got := invalidator.Calls(); len(got) != 1 { + t.Fatalf("invalidator calls = %#v, want one call", got) + } + if got := mock.getHTTPActions(); !slices.Equal(got, []string{"stop", "configure", "start"}) { + t.Fatalf("HTTP actions = %#v", got) + } + }) + + t.Run("invalidation failure is generic and does not start a replacement", func(t *testing.T) { + mgr, execution, mock, agentConfig, invalidator := newManagedRuntimeRetryFixture(t, false) + invalidator.err = errors.New("permission denied writing /home/alice/.npm/_npx") + + attempted, err := mgr.retryManagedRuntimeStartup( + context.Background(), execution, initialErr, agentConfig, "", "", nil, nil, + ) + if attempted { + t.Fatal("cache repair failure must not count as a started retry") + } + var startupErr *routingerr.ManagedRuntimeStartupError + if !errors.As(err, &startupErr) { + t.Fatalf("repair error = %v, want structured startup error", err) + } + if startupErr.Code != routingerr.CodeAgentRuntime { + t.Fatalf("repair code = %q, want generic runtime code", startupErr.Code) + } + if strings.Contains(startupErr.Details, "/home/alice/") { + t.Fatalf("repair details exposed a host path: %q", startupErr.Details) + } + if !strings.Contains(startupErr.Details, "permission denied") { + t.Fatalf("repair details = %q, want the sanitized repair error", startupErr.Details) + } + if got := mock.getHTTPActions(); !slices.Equal(got, []string{"stop"}) { + t.Fatalf("HTTP actions = %#v, want no configure/start", got) + } + if execution.FailureCode != string(routingerr.CodeAgentRuntime) { + t.Fatalf("execution failure code = %q", execution.FailureCode) + } + }) + + t.Run("cancellation wins over recovery", func(t *testing.T) { + mgr, execution, mock, agentConfig, invalidator := newManagedRuntimeRetryFixture(t, false) + ctx, cancel := context.WithCancel(context.Background()) + invalidator.cancel = cancel + + attempted, err := mgr.retryManagedRuntimeStartup(ctx, execution, initialErr, agentConfig, "", "", nil, nil) + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation error = %v", err) + } + if got := mock.getHTTPActions(); !slices.Equal(got, []string{"stop"}) { + t.Fatalf("HTTP actions = %#v, want no configure/start", got) + } + if attempted { + t.Fatal("cancellation before replacement start must not report a retry") + } + }) + + t.Run("remote and native launches are excluded", func(t *testing.T) { + t.Run("remote", func(t *testing.T) { + mgr, execution, mock, agentConfig, _ := newManagedRuntimeRetryFixture(t, false) + execution.RuntimeName = agentruntime.RuntimeSSH + attempted, err := mgr.retryManagedRuntimeStartup(context.Background(), execution, initialErr, agentConfig, "", "", nil, nil) + if attempted || !errors.Is(err, initialErr) { + t.Fatalf("remote result = (%v, %v), want no retry and original error", attempted, err) + } + if got := mock.getHTTPActions(); len(got) != 0 { + t.Fatalf("remote HTTP actions = %#v", got) + } + }) + + t.Run("native command", func(t *testing.T) { + mgr, execution, mock, _, _ := newManagedRuntimeRetryFixture(t, false) + agentConfig := agents.NewCopilotACP() + execution.AgentID = agentConfig.ID() + execution.AgentCommand = "copilot" + execution.AgentArgs = []string{"copilot", "--acp"} + attempted, err := mgr.retryManagedRuntimeStartup(context.Background(), execution, initialErr, agentConfig, "", "", nil, nil) + if attempted || !errors.Is(err, initialErr) { + t.Fatalf("native result = (%v, %v), want no retry and original error", attempted, err) + } + if got := mock.getHTTPActions(); len(got) != 0 { + t.Fatalf("native HTTP actions = %#v", got) + } + }) + }) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager.go b/apps/backend/internal/agent/runtime/lifecycle/manager.go index cc882fa033..3c9726d2fd 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager.go @@ -163,6 +163,9 @@ type Manager struct { // managedRuntimeSelections supplies exact versions for host-local managed // npm runtimes. Remote/container runtimes intentionally do not consult it. managedRuntimeSelections managedruntime.SelectionReader + // managedRuntimeCacheInvalidator removes one trusted npm execution tree + // during bounded host-local startup recovery. + managedRuntimeCacheInvalidator ManagedRuntimeCacheInvalidator activityCoordinator *activity.Coordinator activityMu sync.Mutex @@ -178,6 +181,13 @@ type ManagedGoCacheEnvironmentProvider interface { ExecutionEnvironment(ctx context.Context) (map[string]string, error) } +// ManagedRuntimeCacheInvalidator owns the host npm cache boundary. Lifecycle +// recovery never discovers or deletes cache paths directly. +type ManagedRuntimeCacheInvalidator interface { + InvalidateExecutionCache(context.Context, string) error + InvalidateExecutionCacheVersion(ctx context.Context, packageName, version string) error +} + // SetManagedGoCacheEnvironmentProvider wires install-wide managed cache settings. func (m *Manager) SetManagedGoCacheEnvironmentProvider(provider ManagedGoCacheEnvironmentProvider) { m.managedGoCache = provider @@ -189,6 +199,12 @@ func (m *Manager) SetManagedRuntimeSelectionStore(store managedruntime.Selection m.managedRuntimeSelections = store } +// SetManagedRuntimeCacheInvalidator wires the settings-owned exact cache +// deletion boundary. It is optional for embedded/test managers. +func (m *Manager) SetManagedRuntimeCacheInvalidator(invalidator ManagedRuntimeCacheInvalidator) { + m.managedRuntimeCacheInvalidator = invalidator +} + // SetActivityCoordinator wires the install-wide host-resource activity gate. // It is optional so embedded and test configurations retain legacy behavior. func (m *Manager) SetActivityCoordinator(coordinator *activity.Coordinator) { @@ -286,17 +302,19 @@ func NewManager( // mcpHandler will be set later via SetMCPHandler. // stopCh is shared with the manager so workspace-stream backoff drains on Stop. mgr.streamManager = NewStreamManager(log, StreamCallbacks{ - OnAgentEvent: mgr.handleAgentEvent, - OnStreamDisconnect: mgr.handleStreamDisconnect, - OnGitStatus: mgr.handleGitStatusUpdate, - OnGitCommit: mgr.handleGitCommitCreated, - OnGitReset: mgr.handleGitResetDetected, - OnBranchSwitch: mgr.handleBranchSwitch, - OnFileChange: mgr.handleFileChangeNotification, - OnShellOutput: mgr.handleShellOutput, - OnShellExit: mgr.handleShellExit, - OnProcessOutput: mgr.handleProcessOutput, - OnProcessStatus: mgr.handleProcessStatus, + OnAgentEvent: mgr.handleAgentEvent, + OnStreamDisconnect: mgr.handleStreamDisconnect, + OnAgentEventWithGeneration: mgr.handleAgentEventWithStartupGeneration, + OnStreamDisconnectWithGeneration: mgr.handleStreamDisconnectWithStartupGeneration, + OnGitStatus: mgr.handleGitStatusUpdate, + OnGitCommit: mgr.handleGitCommitCreated, + OnGitReset: mgr.handleGitResetDetected, + OnBranchSwitch: mgr.handleBranchSwitch, + OnFileChange: mgr.handleFileChangeNotification, + OnShellOutput: mgr.handleShellOutput, + OnShellExit: mgr.handleShellExit, + OnProcessOutput: mgr.handleProcessOutput, + OnProcessStatus: mgr.handleProcessStatus, }, nil, stopCh) // Set session manager dependencies for full orchestration diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_events.go b/apps/backend/internal/agent/runtime/lifecycle/manager_events.go index 9eb23e2579..047f0dd71e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_events.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_events.go @@ -177,16 +177,15 @@ func handleCompleteEventSignal(execution *AgentExecution, event *agentctl.AgentE stopReason = sr } } - select { - case execution.promptDoneCh <- PromptCompletionSignal{ - StopReason: stopReason, - IsError: isError, - Error: errorMsg, - PromptGeneration: event.PromptGeneration, - }: - default: - // Channel full or no one waiting — that's fine (e.g., initial prompt in goroutine) - } + execution.signalPromptCompletionForStartupGeneration( + execution.startupAttemptSnapshot(), + PromptCompletionSignal{ + StopReason: stopReason, + IsError: isError, + Error: errorMsg, + PromptGeneration: event.PromptGeneration, + }, + ) } type promptCompletionClaim struct { @@ -620,6 +619,32 @@ func (m *Manager) handleStreamDisconnect( m.publishStreamDisconnectError(execution, err) } +func (m *Manager) handleStreamDisconnectWithStartupGeneration( + execution *AgentExecution, + err error, + promptGeneration uint64, + startupGeneration uint64, +) { + if !execution.acceptsStartupAttempt(startupGeneration) { + m.logger.Debug("ignoring stale managed-runtime stream disconnect", + zap.String("execution_id", execution.ID), + zap.Uint64("stream_startup_generation", startupGeneration), + zap.Uint64("current_startup_generation", execution.startupAttemptSnapshot())) + return + } + // A startup stream can disconnect before ACP session initialization. The + // startup caller owns that failure and may still perform the bounded npm + // recovery, so do not publish an intermediate failed state here. + if promptGeneration == 0 && !execution.isSessionInitialized() { + m.logger.Debug("ignoring startup stream disconnect before ACP initialization", + zap.String("execution_id", execution.ID), + zap.Uint64("startup_generation", startupGeneration), + zap.Error(err)) + return + } + m.handleStreamDisconnect(execution, err, promptGeneration) +} + func (m *Manager) publishStreamDisconnectError(execution *AgentExecution, err error) { m.eventPublisher.PublishAgentctlEvent( context.Background(), events.AgentctlError, execution, @@ -662,16 +687,13 @@ func (m *Manager) handlePromptHandoffEvent( execution.messageMu.Unlock() m.flushAssistantHistory(execution) - select { - case execution.promptDoneCh <- PromptCompletionSignal{ - StopReason: streams.EventTypeForegroundIdle, - PromptGeneration: event.PromptGeneration, - }: - default: - m.logger.Warn("prompt handoff could not signal lifecycle waiter", - zap.String("execution_id", execution.ID), - zap.Uint64("prompt_generation", event.PromptGeneration)) - } + execution.signalPromptCompletionForStartupGeneration( + execution.startupAttemptSnapshot(), + PromptCompletionSignal{ + StopReason: streams.EventTypeForegroundIdle, + PromptGeneration: event.PromptGeneration, + }, + ) } // handleAgentEvent processes incoming agent events from the agent @@ -788,6 +810,28 @@ func (m *Manager) handleAgentEvent(execution *AgentExecution, event agentctl.Age m.eventPublisher.PublishAgentStreamEvent(execution, event) } +func (m *Manager) handleAgentEventWithStartupGeneration( + execution *AgentExecution, + event agentctl.AgentEvent, + startupGeneration uint64, +) { + if !execution.acceptsStartupAttempt(startupGeneration) { + m.logger.Debug("ignoring stale managed-runtime agent event", + zap.String("execution_id", execution.ID), + zap.String("event_type", event.Type), + zap.Uint64("event_startup_generation", startupGeneration), + zap.Uint64("current_startup_generation", execution.startupAttemptSnapshot())) + return + } + if !execution.isSessionInitialized() && event.PromptGeneration == 0 && event.Type == toolStatusComplete { + m.logger.Debug("ignoring startup completion before ACP initialization", + zap.String("execution_id", execution.ID), + zap.Uint64("startup_generation", startupGeneration)) + return + } + m.handleAgentEvent(execution, event) +} + func agentEventDataString(data map[string]any, key string) string { value, _ := data[key].(string) return value diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go index 0d4712ec28..dc98627097 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go @@ -27,7 +27,7 @@ func (m *Manager) WasSessionInitialized(executionID string) bool { if !exists { return false } - return exec.sessionInitialized + return exec.isSessionInitialized() } // GetSessionAuthMethods returns auth methods for a session's execution. @@ -254,15 +254,14 @@ func (m *Manager) escalateStuckCancel(ctx context.Context, execution *AgentExecu zap.String("execution_id", execution.ID), zap.String("session_id", execution.SessionID)) - select { - case execution.promptDoneCh <- PromptCompletionSignal{ - IsError: true, - Error: "cancel escalated: agent did not complete turn within timeout", - PromptGeneration: execution.promptGenerationSnapshot(), - }: - default: - // Channel already has a pending signal; SendPrompt will pick that up instead. - } + execution.signalPromptCompletionForStartupGeneration( + execution.startupAttemptSnapshot(), + PromptCompletionSignal{ + IsError: true, + Error: "cancel escalated: agent did not complete turn within timeout", + PromptGeneration: execution.promptGenerationSnapshot(), + }, + ) select { case <-ch: @@ -310,7 +309,7 @@ func (m *Manager) SetSessionMode(ctx context.Context, executionID, _ string, mod if execution.agentctl == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } - if !execution.sessionInitialized || execution.ACPSessionID == "" { + if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return fmt.Errorf("execution %q ACP session is not ready", executionID) } return execution.agentctl.SetMode(ctx, execution.ACPSessionID, modeID) @@ -369,7 +368,7 @@ func (m *Manager) SetSessionConfigOption(ctx context.Context, executionID, confi if execution.agentctl == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } - if !execution.sessionInitialized || execution.ACPSessionID == "" { + if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return fmt.Errorf("execution %q ACP session is not ready", executionID) } return execution.agentctl.SetConfigOption(ctx, configID, value) @@ -839,7 +838,7 @@ func (m *Manager) resetAgentRestartState(executionID string, commands agentComma exec.ErrorMessage = "" exec.needsResumeContext = false exec.resumeContextInjected = false - exec.sessionInitialized = false + exec.setSessionInitialized(false) exec.AgentCommand = commands.initial exec.ContinueCommand = commands.continue_ exec.AgentArgs = commands.args @@ -1142,7 +1141,7 @@ func (m *Manager) IsAgentReadyForPrompt(ctx context.Context, sessionID string) b if execution.Status != v1.AgentStatusReady || execution.agentctl == nil { return false } - if !execution.sessionInitialized || execution.ACPSessionID == "" { + if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return false } @@ -1160,7 +1159,7 @@ func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string // InitializeAndPrompt owns the first updates stream. Starting a recovery // stream before ACP initialization finishes creates competing consumers and // can split one prompt's events across them. - if !execution.sessionInitialized || execution.ACPSessionID == "" { + if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return nil } if execution.agentctl.HasAgentStream() { @@ -1182,7 +1181,7 @@ func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string if !execution.agentctl.HasAgentStream() { return fmt.Errorf("agent stream not connected") } - if execution.Status == v1.AgentStatusFailed && execution.sessionInitialized && execution.ACPSessionID != "" { + if execution.Status == v1.AgentStatusFailed && execution.isSessionInitialized() && execution.ACPSessionID != "" { return m.restoreRecoveredFailedExecution(ctx, execution) } return nil @@ -1528,7 +1527,7 @@ func (m *Manager) markStoppedDuringShutdown(execution *AgentExecution, exitCode // stub it to avoid touching the real filesystem. func (m *Manager) classifyAndMaybeRemediate(execution *AgentExecution, exitCode int, errorMessage string) { phase := routingerr.PhaseSessionInit - if execution.sessionInitialized { + if execution.isSessionInitialized() { phase = routingerr.PhasePromptSend } var exitPtr *int diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go index 732f294712..ed9c4f08f0 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go @@ -151,6 +151,13 @@ func newRestartMockAgentctlServer(t *testing.T, failStop, failSessionNew bool) * resp, _ = ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ "success": true, }) + case "agent.stderr": + resp, _ = ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ + "lines": []string{ + "npm error code ETARGET", + "npm error notarget No matching version found for opencode-ai@1.2.3", + }, + }) default: resp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeUnknownAction, "unknown action", nil) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go index 4c5cf11445..7d78e9be89 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go @@ -1865,7 +1865,7 @@ func runtimeEnvFromMetadata(metadata map[string]interface{}) map[string]string { // initializeAgentSession handles post-startup initialization: boot message, ACP session, // MCP servers. It finalizes the boot message on success or failure. -func (m *Manager) initializeAgentSession(ctx context.Context, execution *AgentExecution, bootCommand, agentDisplayName, taskDescription string) error { +func (m *Manager) initializeAgentSession(ctx context.Context, execution *AgentExecution, bootCommand, agentDisplayName, taskDescription, approvalPolicy string) error { bootMsg, bootStopCh := m.createBootMessage(ctx, execution, bootCommand, agentDisplayName) // Give the agent process a moment to initialize @@ -1886,6 +1886,25 @@ func (m *Manager) initializeAgentSession(ctx context.Context, execution *AgentEx attachments := getAttachmentsFromMetadata(execution) if err := m.initializeACPSession(ctx, execution, agentConfig, taskDescription, attachments, mcpServers); err != nil { + attempted, retryErr := m.retryManagedRuntimeStartup( + ctx, + execution, + err, + agentConfig, + approvalPolicy, + taskDescription, + attachments, + mcpServers, + ) + if attempted { + if retryErr == nil { + m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, containerStateExited) + return nil + } + err = retryErr + } else if retryErr != nil { + err = retryErr + } m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, "failed") m.updateExecutionError(execution.ID, "failed to initialize ACP: "+err.Error()) return fmt.Errorf("failed to initialize ACP: %w", err) diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go b/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go index ef2a358164..ab8f736fbe 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go @@ -123,6 +123,7 @@ func (m *Manager) StartAgentProcess(ctx context.Context, executionID string) (re if isPassthrough { return m.startPassthroughExecution(operationCtx, execution, profileInfo) } + execution.beginStartupAttempt() if execution.agentctl == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) @@ -181,7 +182,7 @@ func (m *Manager) StartAgentProcess(ctx context.Context, executionID string) (re zap.String("command", bootCommand)) } - return m.initializeAgentSession(operationCtx, execution, bootCommand, agentDisplayName, taskDescription) + return m.initializeAgentSession(operationCtx, execution, bootCommand, agentDisplayName, taskDescription, approvalPolicy) } func (m *Manager) preflightRemoteContributionPushes(ctx context.Context, execution *AgentExecution) error { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go index 36b4e5d20a..ec06ef4bd8 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go @@ -145,7 +145,7 @@ func (m *Manager) createReboundACPSession(ctx context.Context, execution *AgentE return fmt.Errorf("create ACP session in rebound workspace: %w", err) } execution.ACPSessionID = newSessionID - execution.sessionInitialized = true + execution.setSessionInitialized(true) execution.resumeContextInjected = false execution.needsResumeContext = m.historyManager != nil && m.historyManager.HasHistory(execution.SessionID) diff --git a/apps/backend/internal/agent/runtime/lifecycle/session.go b/apps/backend/internal/agent/runtime/lifecycle/session.go index ab02851ff6..21666b6091 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/session.go +++ b/apps/backend/internal/agent/runtime/lifecycle/session.go @@ -394,7 +394,7 @@ func (sm *SessionManager) InitializeAndPromptWithLayers( // Only mark the launch initialized once the start-model policy has // succeeded — a strict unavailable model fails here, and a failed launch // must not look initialized. - execution.sessionInitialized = true + execution.setSessionInitialized(true) finalConfigID, profileModelApplied, profileModeApplied, profileConfigOptionsApplied := sm.applyProfileSessionLayers( ctx, execution, result.SessionID, profileModel, profileMode, profileConfigOptions, @@ -913,10 +913,19 @@ func (sm *SessionManager) waitForPromptDone( stallTicker := time.NewTicker(time.Minute) defer stallTicker.Stop() stallReported := false + startupGeneration := execution.startupAttemptSnapshot() for { select { case signal := <-execution.promptDoneCh: + if signal.StartupGeneration != startupGeneration && + (signal.StartupGeneration != 0 || startupGeneration != 0) { + sm.logger.Debug("ignoring completion signal for superseded startup generation", + zap.String("execution_id", execution.ID), + zap.Uint64("signal_startup_generation", signal.StartupGeneration), + zap.Uint64("active_startup_generation", startupGeneration)) + continue + } if signal.PromptGeneration != 0 && promptGeneration != 0 && signal.PromptGeneration != promptGeneration { diff --git a/apps/backend/internal/agent/runtime/lifecycle/startup_generation_race_test.go b/apps/backend/internal/agent/runtime/lifecycle/startup_generation_race_test.go new file mode 100644 index 0000000000..161d7adb91 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/startup_generation_race_test.go @@ -0,0 +1,87 @@ +package lifecycle + +import ( + "context" + "testing" + "time" +) + +func TestStreamDisconnectBeforeReplacementCarriesStartupGeneration(t *testing.T) { + execution := &AgentExecution{ + ID: "exec-before-replacement", + promptDoneCh: make(chan PromptCompletionSignal, 1), + } + startupGeneration := execution.beginStartupAttempt() + streamManager := NewStreamManager(newTestLogger(), StreamCallbacks{}, nil, nil) + + streamManager.handleUpdatesDisconnectWithGeneration( + execution, + context.Canceled, + startupGeneration, + ) + + select { + case signal := <-execution.promptDoneCh: + if signal.StartupGeneration != startupGeneration { + t.Fatalf("startup generation = %d, want %d", signal.StartupGeneration, startupGeneration) + } + case <-time.After(time.Second): + t.Fatal("disconnect did not signal the owning prompt") + } +} + +func TestDelayedStreamDisconnectCannotSignalReplacementPrompt(t *testing.T) { + execution := &AgentExecution{ + ID: "exec-after-replacement", + promptDoneCh: make(chan PromptCompletionSignal, 1), + } + oldStartupGeneration := execution.beginStartupAttempt() + newStartupGeneration, ok := execution.beginStartupRecovery() + if !ok { + t.Fatal("expected startup recovery generation") + } + + streamManager := NewStreamManager(newTestLogger(), StreamCallbacks{}, nil, nil) + streamManager.handleUpdatesDisconnectWithGeneration( + execution, + context.Canceled, + oldStartupGeneration, + ) + + select { + case signal := <-execution.promptDoneCh: + t.Fatalf("stale disconnect signaled generation %d after replacement generation %d", signal.StartupGeneration, newStartupGeneration) + default: + } +} + +func TestWaitForPromptDoneRejectsStaleStartupGenerationWithWildcardPrompt(t *testing.T) { + execution := &AgentExecution{ + ID: "exec-wildcard-prompt", + promptDoneCh: make(chan PromptCompletionSignal, 2), + } + oldStartupGeneration := execution.beginStartupAttempt() + newStartupGeneration, ok := execution.beginStartupRecovery() + if !ok { + t.Fatal("expected startup recovery generation") + } + + execution.promptDoneCh <- PromptCompletionSignal{ + IsError: true, + Error: "old stream disconnected", + StartupGeneration: oldStartupGeneration, + } + execution.promptDoneCh <- PromptCompletionSignal{ + StopReason: "end_turn", + StartupGeneration: newStartupGeneration, + } + + sessionManager := NewSessionManager(newSessionTestLogger(), make(chan struct{})) + result, err := sessionManager.waitForPromptDone(context.Background(), execution, 0) + if err != nil { + t.Fatalf("waitForPromptDone: %v", err) + } + if result == nil || result.StopReason != "end_turn" { + t.Fatalf("result = %+v, want replacement completion", result) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/streams.go b/apps/backend/internal/agent/runtime/lifecycle/streams.go index 353c38aff8..b7d06d5ba0 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/streams.go +++ b/apps/backend/internal/agent/runtime/lifecycle/streams.go @@ -15,15 +15,19 @@ import ( type StreamCallbacks struct { OnAgentEvent func(execution *AgentExecution, event agentctl.AgentEvent) OnStreamDisconnect func(execution *AgentExecution, err error, promptGeneration uint64) - OnGitStatus func(execution *AgentExecution, update *agentctl.GitStatusUpdate) - OnGitCommit func(execution *AgentExecution, commit *agentctl.GitCommitNotification) - OnGitReset func(execution *AgentExecution, reset *agentctl.GitResetNotification) - OnBranchSwitch func(execution *AgentExecution, branchSwitch *agentctl.GitBranchSwitchNotification) - OnFileChange func(execution *AgentExecution, notification *agentctl.FileChangeNotification) - OnShellOutput func(execution *AgentExecution, data string) - OnShellExit func(execution *AgentExecution, code int) - OnProcessOutput func(execution *AgentExecution, output *agentctl.ProcessOutput) - OnProcessStatus func(execution *AgentExecution, status *agentctl.ProcessStatusUpdate) + // Generation-aware callbacks are used for startup replacement streams. + // The legacy callbacks remain available to isolated callers and tests. + OnAgentEventWithGeneration func(execution *AgentExecution, event agentctl.AgentEvent, startupGeneration uint64) + OnStreamDisconnectWithGeneration func(execution *AgentExecution, err error, promptGeneration, startupGeneration uint64) + OnGitStatus func(execution *AgentExecution, update *agentctl.GitStatusUpdate) + OnGitCommit func(execution *AgentExecution, commit *agentctl.GitCommitNotification) + OnGitReset func(execution *AgentExecution, reset *agentctl.GitResetNotification) + OnBranchSwitch func(execution *AgentExecution, branchSwitch *agentctl.GitBranchSwitchNotification) + OnFileChange func(execution *AgentExecution, notification *agentctl.FileChangeNotification) + OnShellOutput func(execution *AgentExecution, data string) + OnShellExit func(execution *AgentExecution, code int) + OnProcessOutput func(execution *AgentExecution, output *agentctl.ProcessOutput) + OnProcessStatus func(execution *AgentExecution, status *agentctl.ProcessStatusUpdate) } // StreamManager manages WebSocket streams to agent executions @@ -281,14 +285,17 @@ func (sm *StreamManager) streamContext(execution *AgentExecution) context.Contex // connectUpdatesStream handles the updates WebSocket stream with ready signaling func (sm *StreamManager) connectUpdatesStream(execution *AgentExecution, ready chan<- struct{}) { ctx := sm.streamContext(execution) + startupGeneration := execution.startupAttemptSnapshot() err := execution.agentctl.StreamUpdates(ctx, func(event agentctl.AgentEvent) { - if sm.callbacks.OnAgentEvent != nil { + if sm.callbacks.OnAgentEventWithGeneration != nil { + sm.callbacks.OnAgentEventWithGeneration(execution, event, startupGeneration) + } else if sm.callbacks.OnAgentEvent != nil { sm.callbacks.OnAgentEvent(execution, event) } }, sm.mcpHandlerFor(execution), func(disconnectErr error) { if disconnectErr != nil { - sm.handleUpdatesDisconnect(execution, disconnectErr) + sm.handleUpdatesDisconnectWithGeneration(execution, disconnectErr, startupGeneration) } }) @@ -307,18 +314,32 @@ func (sm *StreamManager) connectUpdatesStream(execution *AgentExecution, ready c } func (sm *StreamManager) handleUpdatesDisconnect(execution *AgentExecution, disconnectErr error) { - // Capture prompt ownership before signaling the waiter. A replacement may - // advance the generation before the lifecycle callback runs. + sm.handleUpdatesDisconnectWithGeneration(execution, disconnectErr, execution.startupAttemptSnapshot()) +} + +func (sm *StreamManager) handleUpdatesDisconnectWithGeneration( + execution *AgentExecution, + disconnectErr error, + startupGeneration uint64, +) { promptGeneration := execution.promptGenerationSnapshot() - select { - case execution.promptDoneCh <- PromptCompletionSignal{ - IsError: true, - Error: "agent stream disconnected: " + disconnectErr.Error(), - PromptGeneration: promptGeneration, - }: - default: + if !execution.signalPromptCompletionForStartupGeneration( + startupGeneration, + PromptCompletionSignal{ + IsError: true, + Error: "agent stream disconnected: " + disconnectErr.Error(), + PromptGeneration: promptGeneration, + }, + ) { + sm.logger.Debug("ignoring stale updates stream disconnect", + zap.String("execution_id", execution.ID), + zap.Uint64("stream_startup_generation", startupGeneration), + zap.Uint64("current_startup_generation", execution.startupAttemptSnapshot())) + return } - if sm.callbacks.OnStreamDisconnect != nil { + if sm.callbacks.OnStreamDisconnectWithGeneration != nil { + sm.callbacks.OnStreamDisconnectWithGeneration(execution, disconnectErr, promptGeneration, startupGeneration) + } else if sm.callbacks.OnStreamDisconnect != nil { sm.callbacks.OnStreamDisconnect(execution, disconnectErr, promptGeneration) } } diff --git a/apps/backend/internal/agent/runtime/lifecycle/types.go b/apps/backend/internal/agent/runtime/lifecycle/types.go index 812d14c45e..7d8ca7203c 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/types.go +++ b/apps/backend/internal/agent/runtime/lifecycle/types.go @@ -55,7 +55,12 @@ type AgentExecution struct { FinishedAt *time.Time ExitCode *int ErrorMessage string - ProviderError *streams.ProviderError + // FailureCode and FailureDetails carry a bounded, structured startup + // diagnostic to the orchestrator. They remain separate from the generic + // error message so user-facing recovery can choose a stable presentation. + FailureCode string + FailureDetails string + ProviderError *streams.ProviderError // metadata is unexported on purpose: it is touched from the launch, prompt // and stop paths concurrently, so all access must go through the metadataMu // helpers in execution_metadata.go. @@ -169,7 +174,8 @@ type AgentExecution struct { // sessionInitialized is set to true after InitializeAndPrompt completes successfully. // Used to distinguish launch-phase failures from normal prompt failures. - sessionInitialized bool + sessionInitialized bool + sessionInitializedMu sync.RWMutex // Available commands from the agent (for slash command menu) availableCommands []streams.AvailableCommand @@ -216,6 +222,26 @@ type AgentExecution struct { // Session-level trace span for grouping all operations under one trace sessionSpan trace.Span sessionSpanMu sync.RWMutex + + // Startup attempts are generation-bearing so callbacks from the first + // process cannot fail a replacement process after npm recovery starts. + // This state has its own mutex because stream setup can run while + // promptLifecycleMu is held by workspace rebind waiting for readiness. + startupAttemptGeneration uint64 + startupRecoveryStarted bool + startupLifecycleMu sync.Mutex +} + +func (e *AgentExecution) isSessionInitialized() bool { + e.sessionInitializedMu.RLock() + defer e.sessionInitializedMu.RUnlock() + return e.sessionInitialized +} + +func (e *AgentExecution) setSessionInitialized(value bool) { + e.sessionInitializedMu.Lock() + e.sessionInitialized = value + e.sessionInitializedMu.Unlock() } type activeTopLevelTool struct { @@ -300,10 +326,11 @@ func (e *AgentExecution) officeProfileID() string { // PromptCompletionSignal carries the result from a complete event or disconnect. type PromptCompletionSignal struct { - StopReason string - IsError bool - Error string - PromptGeneration uint64 + StopReason string + IsError bool + Error string + PromptGeneration uint64 + StartupGeneration uint64 } func (e *AgentExecution) promptGenerationSnapshot() uint64 { @@ -312,6 +339,72 @@ func (e *AgentExecution) promptGenerationSnapshot() uint64 { return e.promptGeneration } +// beginStartupAttempt starts a generation for a new ACP process. Generation +// zero is reserved for executions that predate startup tracking. +func (e *AgentExecution) beginStartupAttempt() uint64 { + e.startupLifecycleMu.Lock() + defer e.startupLifecycleMu.Unlock() + e.startupAttemptGeneration++ + e.startupRecoveryStarted = false + return e.startupAttemptGeneration +} + +// beginStartupRecovery advances the startup generation exactly once. The +// caller uses the returned generation when wiring the replacement streams. +func (e *AgentExecution) beginStartupRecovery() (uint64, bool) { + e.startupLifecycleMu.Lock() + defer e.startupLifecycleMu.Unlock() + if e.startupRecoveryStarted { + return e.startupAttemptGeneration, false + } + e.startupRecoveryStarted = true + e.startupAttemptGeneration++ + return e.startupAttemptGeneration, true +} + +func (e *AgentExecution) finishStartupRecovery() { + e.startupLifecycleMu.Lock() + e.startupRecoveryStarted = false + e.startupLifecycleMu.Unlock() +} + +func (e *AgentExecution) startupAttemptSnapshot() uint64 { + e.startupLifecycleMu.Lock() + defer e.startupLifecycleMu.Unlock() + return e.startupAttemptGeneration +} + +func (e *AgentExecution) acceptsStartupAttempt(generation uint64) bool { + e.startupLifecycleMu.Lock() + defer e.startupLifecycleMu.Unlock() + return e.startupAttemptGeneration == generation +} + +// signalPromptCompletionForStartupGeneration claims the current startup +// generation and enqueues its completion signal as one ownership operation. +// A stream can finish its generation check just before recovery advances the +// execution, so checking and sending under the same mutex prevents an old +// stream from publishing into the replacement prompt's channel. +func (e *AgentExecution) signalPromptCompletionForStartupGeneration( + startupGeneration uint64, + signal PromptCompletionSignal, +) bool { + if e == nil { + return false + } + e.startupLifecycleMu.Lock() + defer e.startupLifecycleMu.Unlock() + if e.startupAttemptGeneration != startupGeneration { + return false + } + signal.StartupGeneration = startupGeneration + select { + case e.promptDoneCh <- signal: + default: + } + return true +} + func (e *AgentExecution) promptTurnIDSnapshot() string { if e == nil { return "" diff --git a/apps/backend/internal/agent/runtime/routingerr/npm_resolution_test.go b/apps/backend/internal/agent/runtime/routingerr/npm_resolution_test.go new file mode 100644 index 0000000000..161eca427b --- /dev/null +++ b/apps/backend/internal/agent/runtime/routingerr/npm_resolution_test.go @@ -0,0 +1,72 @@ +package routingerr + +import ( + "strings" + "testing" +) + +func TestManagedRuntimeNpmResolutionRequiresBoundedETargetEvidence(t *testing.T) { + tests := []struct { + name string + input string + match bool + }{ + { + name: "new npm prefix", + input: "npm error code ETARGET\nnpm error notarget No matching version found for @kandev/agent@1.2.3.", + match: true, + }, + { + name: "old npm prefix", + input: "npm ERR! code ETARGET\nnpm ERR! notarget No matching version found for @kandev/agent@1.2.3.", + match: true, + }, + { + name: "generic disconnect", + input: "peer disconnected while starting the agent", + }, + { + name: "unrelated npm failure", + input: "npm error code EAI_AGAIN\nnpm error network request failed", + }, + { + name: "missing matching version line", + input: "npm error code ETARGET\nnpm error notarget No matching version found for another-package.", + }, + { + name: "evidence beyond bounded sample", + input: "npm error code ETARGET\n" + strings.Repeat("filler\n", MaxRawExcerptBytes) + + "\nnpm error notarget No matching version found for @kandev/agent@1.2.3.", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := matchRuntimeEnvironmentRules(Sanitize(tt.input)) + managedMatch := ok && got.Code == CodeManagedRuntimeNpmResolution + if managedMatch != tt.match { + t.Fatalf("managed match = %v, want %v; error = %#v", managedMatch, tt.match, got) + } + if tt.match && got.Code != CodeManagedRuntimeNpmResolution { + t.Fatalf("code = %q, want %q", got.Code, CodeManagedRuntimeNpmResolution) + } + }) + } +} + +func TestManagedRuntimeNpmResolutionSanitizesDetails(t *testing.T) { + input := "npm error code ETARGET\n" + + "npm error notarget No matching version found for @kandev/agent@1.2.3.\n" + + "token=super-secret-token-value" + + classified := Classify(Input{Phase: PhaseSessionInit, Stderr: input}) + if classified.Code != CodeManagedRuntimeNpmResolution { + t.Fatalf("code = %q, want %q", classified.Code, CodeManagedRuntimeNpmResolution) + } + if len(classified.RawExcerpt) > MaxRawExcerptBytes { + t.Fatalf("excerpt length = %d, want <= %d", len(classified.RawExcerpt), MaxRawExcerptBytes) + } + if strings.Contains(classified.RawExcerpt, "super-secret-token-value") { + t.Fatalf("raw excerpt contains an unsanitized secret: %q", classified.RawExcerpt) + } +} diff --git a/apps/backend/internal/agent/runtime/routingerr/routingerr.go b/apps/backend/internal/agent/runtime/routingerr/routingerr.go index 67afde5710..55af7d5267 100644 --- a/apps/backend/internal/agent/runtime/routingerr/routingerr.go +++ b/apps/backend/internal/agent/runtime/routingerr/routingerr.go @@ -16,25 +16,26 @@ import ( type Code string const ( - CodeAuthRequired Code = "auth_required" - CodeMissingCredentials Code = "missing_credentials" - CodeSubscriptionRequired Code = "subscription_required" - CodeQuotaLimited Code = "quota_limited" - CodeRateLimited Code = "rate_limited" - CodeNetworkUnavailable Code = "network_unavailable" - CodeProviderUnavailable Code = "provider_unavailable" - CodeProviderOverloaded Code = "provider_overloaded" - CodeModelCapacity Code = "model_capacity" - CodeModelUnavailable Code = "model_unavailable" - CodeProviderNotConfigured Code = "provider_not_configured" - CodeUnknownProvider Code = "unknown_provider_error" - CodeAgentRuntime Code = "agent_runtime_error" - CodeTask Code = "task_error" - CodeRepo Code = "repo_error" - CodePermissionDeniedByUser Code = "permission_denied_by_user" - CodeNpxCacheCorrupted Code = "npx_cache_corrupted" - CodeResumeCorrupted Code = "resume_corrupted" - CodeAgentTransportLost Code = "agent_transport_lost" + CodeAuthRequired Code = "auth_required" + CodeMissingCredentials Code = "missing_credentials" + CodeSubscriptionRequired Code = "subscription_required" + CodeQuotaLimited Code = "quota_limited" + CodeRateLimited Code = "rate_limited" + CodeNetworkUnavailable Code = "network_unavailable" + CodeProviderUnavailable Code = "provider_unavailable" + CodeProviderOverloaded Code = "provider_overloaded" + CodeModelCapacity Code = "model_capacity" + CodeModelUnavailable Code = "model_unavailable" + CodeProviderNotConfigured Code = "provider_not_configured" + CodeUnknownProvider Code = "unknown_provider_error" + CodeAgentRuntime Code = "agent_runtime_error" + CodeTask Code = "task_error" + CodeRepo Code = "repo_error" + CodePermissionDeniedByUser Code = "permission_denied_by_user" + CodeNpxCacheCorrupted Code = "npx_cache_corrupted" + CodeManagedRuntimeNpmResolution Code = "managed_runtime_npm_resolution" + CodeResumeCorrupted Code = "resume_corrupted" + CodeAgentTransportLost Code = "agent_transport_lost" ) // RemediationStartFreshSession is the symbolic RemediationPath value for @@ -138,24 +139,39 @@ func Classify(in Input) *Error { e.ResetHint = in.ResetHint return applyInvariants(e) } - if e, ok := matchProviderRules(in.ProviderID, in.Stderr+"\n"+in.Stdout); ok { + if e, ok := matchProviderRules(in.ProviderID, excerpt); ok { e.Phase = in.Phase e.ExitCode = in.ExitCode e.ResetHint = in.ResetHint e.RawExcerpt = excerpt return applyInvariants(e) } - if e, ok := matchProviderNeutralRules(in.Stderr + "\n" + in.Stdout); ok { + if e, ok := matchProviderNeutralRules(excerpt); ok { e.Phase = in.Phase e.ExitCode = in.ExitCode e.ResetHint = in.ResetHint e.RawExcerpt = excerpt return applyInvariants(e) } - if e, ok := matchRuntimeEnvironmentRules(in.Stderr + "\n" + in.Stdout); ok { + if e, ok := matchRuntimeEnvironmentRules(excerpt); ok { e.Phase = in.Phase e.ExitCode = in.ExitCode e.ResetHint = in.ResetHint + if e.Code == CodeNpxCacheCorrupted { + // Preserve the legacy path for the path-aware remediation guard; + // the path is validated again before deletion. + e.RemediationPath = extractNpxCachePath(in.Stderr + "\n" + in.Stdout) + } + e.RawExcerpt = excerpt + return applyInvariants(e) + } + if e, ok := matchLegacyRuntimeEnvironmentRules(in.Stderr + "\n" + in.Stdout); ok { + e.Phase = in.Phase + e.ExitCode = in.ExitCode + e.ResetHint = in.ResetHint + if e.Code == CodeNpxCacheCorrupted { + e.RemediationPath = extractNpxCachePath(in.Stderr + "\n" + in.Stdout) + } e.RawExcerpt = excerpt return applyInvariants(e) } @@ -274,6 +290,10 @@ func applyInvariants(e *Error) *Error { case CodeNpxCacheCorrupted: e.AutoRetryable = true e.FallbackAllowed = true + case CodeManagedRuntimeNpmResolution: + e.UserAction = true + e.AutoRetryable = false + e.FallbackAllowed = false case CodeResumeCorrupted: // The agent's persisted reasoning state is poisoned. Retrying the // resume hits the same 400, and falling back to another provider diff --git a/apps/backend/internal/agent/runtime/routingerr/runtime_rules.go b/apps/backend/internal/agent/runtime/routingerr/runtime_rules.go index d11a1cc8d4..feb28edb47 100644 --- a/apps/backend/internal/agent/runtime/routingerr/runtime_rules.go +++ b/apps/backend/internal/agent/runtime/routingerr/runtime_rules.go @@ -12,6 +12,21 @@ import "regexp" // Each entry pairs a regex matcher with a builder that can extract // signal-specific metadata (e.g. RemediationPath) from the raw text. var runtimeEnvironmentRules = []runtimeRule{ + { + // npm 9 and npm 10 use different casing for the error prefix. Require + // both the ETARGET code and the matching package@version diagnostic in + // one bounded sample so generic disconnects and registry errors do not + // trigger managed-runtime recovery. + id: "npm.etarget.managed_runtime.v1", + pattern: managedRuntimeNpmResolutionRe, + sanitizedOnly: true, + build: func(string) *Error { + return &Error{ + Code: CodeManagedRuntimeNpmResolution, + Confidence: ConfHigh, + } + }, + }, { id: "npm.enotempty.npx.v1", pattern: regexp.MustCompile(`(?s)npm error code ENOTEMPTY.*?_npx/[0-9a-f]+`), @@ -85,6 +100,10 @@ var runtimeEnvironmentRules = []runtimeRule{ }, } +var managedRuntimeNpmResolutionRe = regexp.MustCompile( + `(?ism)^\s*npm\s+(ERR!|error)\s+code\s+ETARGET\b[\s\S]{0,999}^\s*npm\s+(ERR!|error)\s+notarget\s+No matching version found for\s+\S+@\S+`, +) + // cancellationOrDeadlineRe matches context-cancellation and deadline // signatures that can co-occur with the transport-lost wording in the same // error string. Kept separate from transportLostRe so the two can be @@ -148,9 +167,10 @@ func IsResumeCorrupted(message string) bool { } type runtimeRule struct { - id string - pattern *regexp.Regexp - build func(text string) *Error + id string + pattern *regexp.Regexp + build func(text string) *Error + sanitizedOnly bool } // npxCachePathRe captures the cache root, e.g. @@ -171,10 +191,21 @@ func extractNpxCachePath(text string) string { } func matchRuntimeEnvironmentRules(text string) (*Error, bool) { + return matchRuntimeRules(text, false) +} + +func matchLegacyRuntimeEnvironmentRules(text string) (*Error, bool) { + return matchRuntimeRules(text, true) +} + +func matchRuntimeRules(text string, legacyOnly bool) (*Error, bool) { if text == "" { return nil, false } for _, r := range runtimeEnvironmentRules { + if legacyOnly && r.sanitizedOnly { + continue + } if !r.pattern.MatchString(text) { continue } diff --git a/apps/backend/internal/agent/runtime/routingerr/startup_error.go b/apps/backend/internal/agent/runtime/routingerr/startup_error.go new file mode 100644 index 0000000000..36609a430c --- /dev/null +++ b/apps/backend/internal/agent/runtime/routingerr/startup_error.go @@ -0,0 +1,29 @@ +package routingerr + +import "fmt" + +// ManagedRuntimeStartupError carries the final normalized classification from +// a managed-runtime startup attempt through the lifecycle and orchestrator +// error wrappers. Details are already sanitized and bounded by the caller. +type ManagedRuntimeStartupError struct { + Code Code + Details string + Cause error +} + +func (e *ManagedRuntimeStartupError) Error() string { + if e == nil { + return "" + } + if e.Details == "" { + return string(e.Code) + } + return fmt.Sprintf("%s: %s", e.Code, e.Details) +} + +func (e *ManagedRuntimeStartupError) Unwrap() error { + if e == nil { + return nil + } + return e.Cause +} diff --git a/apps/backend/internal/agent/settings/controller/agent_update.go b/apps/backend/internal/agent/settings/controller/agent_update.go index 2bd940a6f6..edbc36b01b 100644 --- a/apps/backend/internal/agent/settings/controller/agent_update.go +++ b/apps/backend/internal/agent/settings/controller/agent_update.go @@ -7,9 +7,7 @@ import ( "errors" "fmt" "io" - "os" "os/exec" - "path/filepath" "strings" "sync" "time" @@ -186,6 +184,29 @@ type ExactRuntimeCacheInvalidator interface { InvalidateExecutionCacheVersion(context.Context, string, string) error } +// InvalidateExecutionCache delegates the exact managed-runtime cache cleanup +// to the configured host updater. Lifecycle startup recovery uses this +// boundary instead of discovering npm paths itself. +func (c *Controller) InvalidateExecutionCache(ctx context.Context, packageName string) error { + if c.runtimeUpdater == nil { + return ErrRuntimeUpdaterUnavailable + } + return c.runtimeUpdater.InvalidateExecutionCache(ctx, packageName) +} + +// InvalidateExecutionCacheVersion delegates version-specific cleanup when the +// host updater supports the exact-tree contract. +func (c *Controller) InvalidateExecutionCacheVersion(ctx context.Context, packageName, version string) error { + if c.runtimeUpdater == nil { + return ErrRuntimeUpdaterUnavailable + } + invalidator, ok := c.runtimeUpdater.(ExactRuntimeCacheInvalidator) + if !ok { + return ErrRuntimeUpdateUnsupported + } + return invalidator.InvalidateExecutionCacheVersion(ctx, packageName, version) +} + type hostRuntimeUpdater struct { host *hostutility.Manager executor directCommandExecutor @@ -290,33 +311,7 @@ func (u *hostRuntimeUpdater) npxCacheRoot(ctx context.Context) (string, error) { if err != nil { return "", fmt.Errorf("resolve npm cache root: %w", err) } - cacheRoot := strings.TrimSpace(output) - if !filepath.IsAbs(cacheRoot) { - return "", fmt.Errorf("npm cache root is not absolute: %q", cacheRoot) - } - cacheRoot = filepath.Clean(cacheRoot) - if cacheRoot == string(filepath.Separator) { - return "", errors.New("refusing to invalidate execution cache under filesystem root") - } - npxRoot := filepath.Join(cacheRoot, "_npx") - if info, statErr := os.Lstat(npxRoot); statErr == nil && info.Mode()&os.ModeSymlink != 0 { - return "", fmt.Errorf("refusing to invalidate execution cache through symlink: %s", npxRoot) - } else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) { - return "", fmt.Errorf("inspect npm execution cache: %w", statErr) - } - return npxRoot, nil -} - -func removeNpmExecutionCacheKey(npxRoot, key string) error { - target := filepath.Join(npxRoot, key) - rel, err := filepath.Rel(npxRoot, target) - if err != nil || rel != key { - return fmt.Errorf("invalid npm execution cache target: %s", target) - } - if err := os.RemoveAll(target); err != nil { - return fmt.Errorf("remove npm execution cache %s: %w", target, err) - } - return nil + return strings.TrimSpace(output), nil } func (u *hostRuntimeUpdater) InvalidateExecutionCache(ctx context.Context, packageName string) error { @@ -324,12 +319,11 @@ func (u *hostRuntimeUpdater) InvalidateExecutionCache(ctx context.Context, packa if packageName == "" { return errors.New("managed runtime package is empty") } - npxRoot, err := u.npxCacheRoot(ctx) + cacheRoot, err := u.npxCacheRoot(ctx) if err != nil { return err } - key := (agents.ManagedNPMRuntimeSpec{Package: packageName}).ExecutionCacheKey() - return removeNpmExecutionCacheKey(npxRoot, key) + return managedruntime.RemoveNpxExecutionTree(cacheRoot, packageName) } func (u *hostRuntimeUpdater) InvalidateExecutionCacheVersion( @@ -345,12 +339,12 @@ func (u *hostRuntimeUpdater) InvalidateExecutionCacheVersion( if _, err := managedruntime.ParseStableVersion(version); err != nil { return err } - npxRoot, err := u.npxCacheRoot(ctx) + cacheRoot, err := u.npxCacheRoot(ctx) if err != nil { return err } spec := agents.ManagedNPMRuntimeSpec{Package: packageName} - return removeNpmExecutionCacheKey(npxRoot, spec.ExecutionCacheKey(version)) + return managedruntime.RemoveNpxExecutionTree(cacheRoot, spec.PackageSpec(version)) } func (u *hostRuntimeUpdater) Refresh( diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 0d2e88dc6b..1bd4d10c34 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -816,6 +816,9 @@ func startGatewayAndServe( // Wire the host utility manager into the settings controller so // /api/v1/agent-models/:agentName reads live capability data. agentSettingsController.SetHostUtility(hostUtilityMgr) + // Reuse the settings-owned exact npm cache boundary for managed-runtime + // startup recovery. The lifecycle manager never derives cache paths. + lifecycleMgr.SetManagedRuntimeCacheInvalidator(agentSettingsController) profileReconciler := agentsettingscontroller.NewProfileReconciler(hostUtilityMgr, agentRegistry, repos.AgentSettings, log) go func() { if err := hostUtilityMgr.Start(ctx); err != nil { diff --git a/apps/backend/internal/orchestrator/event_handlers_agent.go b/apps/backend/internal/orchestrator/event_handlers_agent.go index 01d0fcb47a..652dc7735f 100644 --- a/apps/backend/internal/orchestrator/event_handlers_agent.go +++ b/apps/backend/internal/orchestrator/event_handlers_agent.go @@ -1558,6 +1558,7 @@ func (s *Service) persistLastAgentError(ctx context.Context, data watcher.AgentE if errMsg == "" { errMsg = "agent failed" } + details := routingerr.Sanitize(data.FailureDetails) // Keep this metadata until the user dismisses the UI notice locally or a // later recoverable failure replaces it. A successful turn should not erase // the investigation breadcrumb that explains why the task was marked REVIEW. @@ -1566,6 +1567,8 @@ func (s *Service) persistLastAgentError(ctx context.Context, data watcher.AgentE OccurredAt: time.Now().UTC(), AgentExecutionID: data.AgentExecutionID, RemediationURL: providerRemediationURL(data), + Code: data.FailureCode, + Details: details, } if err := s.repo.SetSessionMetadataKey(ctx, data.SessionID, models.SessionMetaKeyLastAgentError, lastErr); err != nil { s.logger.Warn("failed to persist last agent error", @@ -1587,6 +1590,12 @@ func (s *Service) persistLastAgentError(ctx context.Context, data watcher.AgentE if lastErr.RemediationURL != "" { eventData["remediation_url"] = lastErr.RemediationURL } + if lastErr.Code != "" { + eventData["code"] = lastErr.Code + } + if lastErr.Details != "" { + eventData["details"] = lastErr.Details + } if err := s.eventBus.Publish(ctx, events.TaskSessionErrorChanged, bus.NewEvent( events.TaskSessionErrorChanged, "orchestrator", @@ -1698,6 +1707,13 @@ func (s *Service) createRecoveryStatusMessage(ctx context.Context, data watcher. "is_auth_error": authErr, "resume_corrupted": resumeCorrupted, } + managedRuntimeNpmFailure := data.FailureCode == string(routingerr.CodeManagedRuntimeNpmResolution) + if managedRuntimeNpmFailure { + meta["failure_kind"] = string(routingerr.CodeManagedRuntimeNpmResolution) + if details := routingerr.Sanitize(data.FailureDetails); details != "" { + meta["error_output"] = details + } + } // The validated remediation URL is carried independently of quota // classification so the generic recoverable card can still show the link. if remediationURL := providerRemediationURL(data); remediationURL != "" { @@ -1712,7 +1728,21 @@ func (s *Service) createRecoveryStatusMessage(ctx context.Context, data watcher. } } - meta["actions"] = buildRecoveryActions(data.TaskID, data.SessionID, hasResumeToken, authErr, resumeCorrupted) + if managedRuntimeNpmFailure { + meta["actions"] = []map[string]interface{}{ + wsRecoveryAction( + data.TaskID, + data.SessionID, + "runtime_retry", + "Retry runtime", + "refresh", + "", + "managed-runtime-npm-retry-button", + ), + } + } else { + meta["actions"] = buildRecoveryActions(data.TaskID, data.SessionID, hasResumeToken, authErr, resumeCorrupted) + } if err := s.messageCreator.CreateSessionMessage( ctx, @@ -1785,6 +1815,17 @@ func (s *Service) isOfficeSession(ctx context.Context, sessionID string) bool { // error on focus / auto-resume. // Returns true if the failure was handled (caller should skip default FAILED logic). func (s *Service) handleAgentStartFailed(ctx context.Context, taskID, sessionID, agentExecutionID string, err error, fromResume bool) bool { + failureData := watcher.AgentEventData{ + TaskID: taskID, + SessionID: sessionID, + AgentExecutionID: agentExecutionID, + ErrorMessage: err.Error(), + } + if classified := classifyManagedRuntimeNpmStartFailure(err); classified != nil { + failureData.ErrorMessage = "managed npm runtime failed to prepare" + failureData.FailureCode = string(routingerr.CodeManagedRuntimeNpmResolution) + failureData.FailureDetails = classified.RawExcerpt + } if sessionID != "" { lock, release := s.acquireCancelInFlightGuard(sessionID) defer release() @@ -1798,12 +1839,7 @@ func (s *Service) handleAgentStartFailed(ctx context.Context, taskID, sessionID, return true } - if drop, terminalState := s.shouldDropSessionFailure(ctx, watcher.AgentEventData{ - TaskID: taskID, - SessionID: sessionID, - AgentExecutionID: agentExecutionID, - ErrorMessage: err.Error(), - }, "agent process start", false); drop { + if drop, terminalState := s.shouldDropSessionFailure(ctx, failureData, "agent process start", false); drop { // A cancellation that landed after the executor's first terminal-state // read still needs its exact-execution cleanup path. Returning false lets // the executor observe the final CANCELLED state and arbitrate teardown. @@ -1813,6 +1849,14 @@ func (s *Service) handleAgentStartFailed(ctx context.Context, taskID, sessionID, return true } } + if failureData.FailureCode == string(routingerr.CodeManagedRuntimeNpmResolution) { + s.logger.Info("managed npm runtime startup failure is recoverable", + zap.String("task_id", taskID), + zap.String("session_id", sessionID), + zap.String("agent_execution_id", agentExecutionID)) + s.handleRecoverableFailureLocked(ctx, failureData) + return true + } if !isAuthError(err.Error()) { if fromResume { @@ -1827,15 +1871,29 @@ func (s *Service) handleAgentStartFailed(ctx context.Context, taskID, sessionID, s.logger.Info("agent start failure is auth error, treating as recoverable", zap.String("task_id", taskID), zap.String("session_id", sessionID)) - s.handleRecoverableFailureLocked(ctx, watcher.AgentEventData{ - TaskID: taskID, - SessionID: sessionID, - AgentExecutionID: agentExecutionID, - ErrorMessage: err.Error(), - }) + s.handleRecoverableFailureLocked(ctx, failureData) return true } +func classifyManagedRuntimeNpmStartFailure(err error) *routingerr.Error { + if err == nil { + return nil + } + var structured *routingerr.ManagedRuntimeStartupError + if errors.As(err, &structured) { + if structured.Code != routingerr.CodeManagedRuntimeNpmResolution { + return nil + } + return &routingerr.Error{ + Code: structured.Code, + Confidence: routingerr.ConfHigh, + Phase: routingerr.PhaseSessionInit, + RawExcerpt: structured.Details, + } + } + return nil +} + // actionMetaKey* are the shared keys of the frontend ActionMessage button // descriptor (see apps/web .../messages/action-message.tsx). Defined as // constants because the shape is built in more than one place in this package. @@ -1857,17 +1915,20 @@ const ( // the map keys in one place avoids drift between the buttons and keeps the // metadata shape consistent. func wsRecoveryAction(taskID, sessionID, recoverAction, label, icon, tooltip, testID string) map[string]interface{} { - return map[string]interface{}{ - actionMetaKeyType: "ws_request", - actionMetaKeyLabel: label, - actionMetaKeyIcon: icon, - actionMetaKeyTooltip: tooltip, - actionMetaKeyTestID: testID, + action := map[string]interface{}{ + actionMetaKeyType: "ws_request", + actionMetaKeyLabel: label, + actionMetaKeyIcon: icon, + actionMetaKeyTestID: testID, "params": map[string]interface{}{ "method": "session.recover", "payload": map[string]interface{}{"task_id": taskID, "session_id": sessionID, "action": recoverAction}, }, } + if tooltip != "" { + action[actionMetaKeyTooltip] = tooltip + } + return action } // buildRecoveryActions creates the generic actions array for agent error diff --git a/apps/backend/internal/orchestrator/event_handlers_test.go b/apps/backend/internal/orchestrator/event_handlers_test.go index 47bc368e73..7994ca66b9 100644 --- a/apps/backend/internal/orchestrator/event_handlers_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_test.go @@ -18,6 +18,7 @@ import ( "github.com/kandev/kandev/internal/agent/registry" "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/db" @@ -2503,6 +2504,39 @@ func TestHandleRecoverableFailure(t *testing.T) { } }) + t.Run("persists structured managed runtime failure fields safely", func(t *testing.T) { + repo := setupTestRepo(t) + seedSession(t, repo, "t1", "s1", "step1") + + taskRepo := newMockTaskRepo() + agentMgr := &mockAgentManager{repoForExecutionLookup: repo} + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), taskRepo, agentMgr) + + svc.handleRecoverableFailure(ctx, watcher.AgentEventData{ + TaskID: "t1", + SessionID: "s1", + AgentExecutionID: "exec-1", + ErrorMessage: "managed npm runtime failed to prepare", + FailureCode: "managed_runtime_npm_resolution", + FailureDetails: "npm error code ETARGET\nsecret=super-secret-value", + }) + + session, err := repo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("failed to get session: %v", err) + } + lastErr, ok := models.LoadLastAgentError(session.Metadata) + if !ok { + t.Fatalf("expected last agent error metadata, got %#v", session.Metadata) + } + if lastErr.Code != "managed_runtime_npm_resolution" { + t.Fatalf("failure code = %q, want managed runtime code", lastErr.Code) + } + if !strings.Contains(lastErr.Details, "secret: ***") || strings.Contains(lastErr.Details, "super-secret-value") { + t.Fatalf("failure details were not sanitized: %q", lastErr.Details) + } + }) + t.Run("sets session to WAITING_FOR_INPUT with error message", func(t *testing.T) { repo := setupTestRepo(t) seedSession(t, repo, "t1", "s1", "step1") @@ -2615,6 +2649,38 @@ func TestIsOfficeSessionUsesCanonicalTaskOwnership(t *testing.T) { } } +func TestClassifyManagedRuntimeNpmStartFailureUsesStructuredError(t *testing.T) { + err := fmt.Errorf("failed to initialize ACP: %w", &routingerr.ManagedRuntimeStartupError{ + Code: routingerr.CodeManagedRuntimeNpmResolution, + Details: "npm error code ETARGET\nnpm error notarget No matching version found for managed-acp@1.2.3", + }) + + classified := classifyManagedRuntimeNpmStartFailure(err) + if classified == nil { + t.Fatal("expected structured npm startup error to classify") + } + if classified.Code != routingerr.CodeManagedRuntimeNpmResolution { + t.Fatalf("code = %q, want %q", classified.Code, routingerr.CodeManagedRuntimeNpmResolution) + } + if !strings.Contains(classified.RawExcerpt, "No matching version found") { + t.Fatalf("raw excerpt = %q, want structured details", classified.RawExcerpt) + } + + generic := fmt.Errorf("failed to initialize ACP: %w", &routingerr.ManagedRuntimeStartupError{ + Code: routingerr.CodeAgentRuntime, + Details: "sanitized runtime failure", + }) + if classifyManagedRuntimeNpmStartFailure(generic) != nil { + t.Fatal("generic structured startup failure must not use the npm recovery card") + } + + if classifyManagedRuntimeNpmStartFailure(errors.New( + "npm error code ETARGET\nnpm error notarget No matching version found for managed-acp@1.2.3", + )) != nil { + t.Fatal("unstructured npm text must not select the managed runtime recovery card") + } +} + func TestHandleAgentStartFailed(t *testing.T) { ctx := context.Background() diff --git a/apps/backend/internal/orchestrator/handlers/handlers.go b/apps/backend/internal/orchestrator/handlers/handlers.go index f3f1d4656e..b80c3f8f62 100644 --- a/apps/backend/internal/orchestrator/handlers/handlers.go +++ b/apps/backend/internal/orchestrator/handlers/handlers.go @@ -199,7 +199,7 @@ func (h *Handlers) wsSetPlanMode(ctx context.Context, msg *ws.Message) (*ws.Mess type wsRecoverSessionRequest struct { TaskID string `json:"task_id"` SessionID string `json:"session_id"` - Action string `json:"action"` // "resume", "fresh_start", or "cancel_retry" + Action string `json:"action"` // "resume", "fresh_start", "runtime_retry", or "cancel_retry" } // wsRecoverSession recovers a session by id (resume or fresh start). @@ -222,8 +222,8 @@ func (h *Handlers) wsRecoverSession(ctx context.Context, msg *ws.Message) (*ws.M return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{"cancelled": cancelled}) } - if req.Action != "resume" && req.Action != "fresh_start" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "action must be 'resume', 'fresh_start', or 'cancel_retry'", nil) + if req.Action != "resume" && req.Action != "fresh_start" && req.Action != "runtime_retry" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "action must be 'resume', 'fresh_start', 'runtime_retry', or 'cancel_retry'", nil) } resp, err := h.service.RecoverSession(ctx, req.TaskID, req.SessionID, req.Action) diff --git a/apps/backend/internal/orchestrator/recovery_actions_test.go b/apps/backend/internal/orchestrator/recovery_actions_test.go index 1fda8411fc..6b97937ace 100644 --- a/apps/backend/internal/orchestrator/recovery_actions_test.go +++ b/apps/backend/internal/orchestrator/recovery_actions_test.go @@ -69,6 +69,48 @@ func TestBuildRecoveryActions_ResumeCorruptedWithoutToken(t *testing.T) { } } +func TestCreateRecoveryStatusMessage_ManagedRuntimeNpmUsesOneRetryAction(t *testing.T) { + ctx := context.Background() + repo := setupTestRepo(t) + seedSession(t, repo, "t-npm", "s-npm", "step1") + svc := createTestServiceWithScheduler(repo, newMockStepGetter(), newMockTaskRepo(), &mockAgentManager{repoForExecutionLookup: repo}) + mc := &mockMessageCreator{} + svc.messageCreator = mc + + svc.createRecoveryStatusMessage(ctx, watcher.AgentEventData{ + TaskID: "t-npm", + SessionID: "s-npm", + ErrorMessage: "managed npm runtime failed to prepare", + FailureCode: "managed_runtime_npm_resolution", + FailureDetails: "npm error code ETARGET\nnpm error notarget No matching version found", + }) + + if len(mc.sessionMessages) != 1 { + t.Fatalf("expected one recovery message, got %d", len(mc.sessionMessages)) + } + meta := mc.sessionMessages[0].metadata + if meta["failure_kind"] != "managed_runtime_npm_resolution" { + t.Fatalf("failure_kind = %#v", meta["failure_kind"]) + } + if meta["error_output"] != "npm error code ETARGET\nnpm error notarget No matching version found" { + t.Fatalf("error_output = %#v", meta["error_output"]) + } + actions, ok := meta["actions"].([]map[string]interface{}) + if !ok || len(actions) != 1 { + t.Fatalf("actions = %#v, want one action", meta["actions"]) + } + if actions[0]["test_id"] != "managed-runtime-npm-retry-button" { + t.Fatalf("action test_id = %#v", actions[0]["test_id"]) + } + if _, ok := actions[0]["tooltip"]; ok { + t.Fatal("managed runtime recovery action must not carry an untranslated tooltip") + } + payload := actions[0]["params"].(map[string]interface{})["payload"].(map[string]interface{}) + if payload["action"] != "runtime_retry" { + t.Fatalf("action payload = %#v", payload) + } +} + func TestCreateRecoveryStatusMessage_ResumeCorrupted(t *testing.T) { ctx := context.Background() repo := setupTestRepo(t) diff --git a/apps/backend/internal/orchestrator/session_launch.go b/apps/backend/internal/orchestrator/session_launch.go index eb67cb6cf2..ac6334733c 100644 --- a/apps/backend/internal/orchestrator/session_launch.go +++ b/apps/backend/internal/orchestrator/session_launch.go @@ -369,6 +369,13 @@ func (s *Service) RecoverSession(ctx context.Context, taskID, sessionID, action if err := s.authorizeTask(ctx, taskID); err != nil { return nil, err } + if action == "runtime_retry" { + if s.wasResumeAttempt(ctx, sessionID) { + action = "resume" + } else { + action = "fresh_start" + } + } switch action { case "fresh_start": s.clearResumeToken(ctx, sessionID) diff --git a/apps/backend/internal/orchestrator/watcher/watcher.go b/apps/backend/internal/orchestrator/watcher/watcher.go index c893b81023..0abda79e5a 100644 --- a/apps/backend/internal/orchestrator/watcher/watcher.go +++ b/apps/backend/internal/orchestrator/watcher/watcher.go @@ -38,6 +38,8 @@ type AgentEventData struct { ExecutionProfileID string `json:"execution_profile_id,omitempty"` ExitCode *int `json:"exit_code,omitempty"` ErrorMessage string `json:"error_message,omitempty"` + FailureCode string `json:"failure_code,omitempty"` + FailureDetails string `json:"failure_details,omitempty"` ProviderError *streams.ProviderError `json:"provider_error,omitempty"` PromptGeneration uint64 `json:"prompt_generation,omitempty"` } diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index 2d95aea69b..a2ef389268 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -461,6 +461,8 @@ type LastAgentError struct { OccurredAt time.Time `json:"occurred_at"` AgentExecutionID string `json:"agent_execution_id,omitempty"` RemediationURL string `json:"remediation_url,omitempty"` + Code string `json:"code,omitempty"` + Details string `json:"details,omitempty"` DismissedAt *time.Time `json:"dismissed_at,omitempty"` } diff --git a/apps/web/app/office/tasks/[id]/types.ts b/apps/web/app/office/tasks/[id]/types.ts index 1b6805797b..aa5e595536 100644 --- a/apps/web/app/office/tasks/[id]/types.ts +++ b/apps/web/app/office/tasks/[id]/types.ts @@ -103,6 +103,8 @@ export type RunError = { failedAt: string; /** Adapter-validated provider remediation URL from last_agent_error metadata. */ remediationUrl?: string; + failureCode?: string; + failureDetails?: string; }; export type TaskLabelLocal = { diff --git a/apps/web/components/task/chat/messages/action-message-details.tsx b/apps/web/components/task/chat/messages/action-message-details.tsx new file mode 100644 index 0000000000..d38317ba9f --- /dev/null +++ b/apps/web/components/task/chat/messages/action-message-details.tsx @@ -0,0 +1,88 @@ +import { useCallback, useState } from "react"; +import { IconChevronDown } from "@tabler/icons-react"; +import { useTranslation } from "react-i18next"; +import { AuthMethodsPanel, GenericAuthPanel } from "./auth-methods-panel"; +import { RemediationLink } from "@/components/task/remediation-link"; +import { HostShellDialog } from "@/components/settings/host-shell-dialog"; +import type { MessageAction, RecoveryAuthMethod } from "@/components/task/chat/types"; + +export type ActionMeta = { + actions?: MessageAction[]; + action_visibility?: "running"; + variant?: string; + recovery_actions?: boolean; + is_auth_error?: boolean; + auth_methods?: RecoveryAuthMethod[]; + error_output?: string; + failure_kind?: string; + missing_branch?: string; + provider_name?: string; + model_id?: string; + reset_at?: string; + remediation_url?: string; + retrying?: boolean; + attempt?: number; + max_attempts?: number; + retry_in_seconds?: number; + retry_at?: string; + failure_code?: string; + failure_details?: string; +}; + +export function TechnicalDetails({ children }: { children: string }) { + const { t } = useTranslation(); + return ( +
+ + + {t("chat:technicalDetails")} + +
+        {children}
+      
+
+ ); +} + +export function ActionMessageDetails({ + metadata, + technicalDetails, +}: { + metadata: ActionMeta | undefined; + technicalDetails?: string; +}) { + const [hostShellOpen, setHostShellOpen] = useState(false); + const [hostShellCommand, setHostShellCommand] = useState(undefined); + + const openHostShellWithCommand = useCallback((command: string) => { + setHostShellCommand(command + "\n"); + setHostShellOpen(true); + }, []); + const openHostShell = useCallback(() => { + setHostShellCommand(undefined); + setHostShellOpen(true); + }, []); + + if (!metadata) return null; + const errorOutput = metadata.error_output || technicalDetails; + return ( + <> + {metadata.remediation_url && } + {errorOutput && {errorOutput}} + {metadata.is_auth_error && metadata.auth_methods && metadata.auth_methods.length > 0 && ( + + )} + {metadata.is_auth_error && (!metadata.auth_methods || metadata.auth_methods.length === 0) && ( + + )} + + + ); +} diff --git a/apps/web/components/task/chat/messages/action-message.test.tsx b/apps/web/components/task/chat/messages/action-message.test.tsx index c10c33b8bb..3de2918b25 100644 --- a/apps/web/components/task/chat/messages/action-message.test.tsx +++ b/apps/web/components/task/chat/messages/action-message.test.tsx @@ -32,12 +32,15 @@ const RECOVERY_MESSAGE = "Agent encountered an error"; const RESUME_LABEL = "Resume session"; const RESUME_TEST_ID = "recovery-resume-button"; const STALL_CANCEL_TEST_ID = "stall-cancel-turn-button"; +const TEST_SESSION_ID = "sess-1"; +const TEST_TASK_ID = "task-1"; +const SESSION_RECOVER_METHOD = "session.recover"; function retryMessage(overrides: Partial = {}): Message { return { id: "msg-1", - session_id: toSessionId("sess-1"), - task_id: toTaskId("task-1"), + session_id: toSessionId(TEST_SESSION_ID), + task_id: toTaskId(TEST_TASK_ID), author_type: "system", content: "Provider overloaded — retrying in 5s (attempt 1/5)", type: "status", @@ -48,8 +51,8 @@ function retryMessage(overrides: Partial = {}): Message { attempt: 1, max_attempts: 5, retry_in_seconds: 5, - session_id: "sess-1", - task_id: "task-1", + session_id: TEST_SESSION_ID, + task_id: TEST_TASK_ID, actions: [ { type: "ws_request", @@ -57,8 +60,8 @@ function retryMessage(overrides: Partial = {}): Message { icon: "x", test_id: CANCEL_TEST_ID, params: { - method: "session.recover", - payload: { task_id: "task-1", session_id: "sess-1", action: "cancel_retry" }, + method: SESSION_RECOVER_METHOD, + payload: { task_id: TEST_TASK_ID, session_id: TEST_SESSION_ID, action: "cancel_retry" }, }, }, ], @@ -74,8 +77,8 @@ function transientRetryMetadata(attempt: number, retryInSeconds: number) { attempt, max_attempts: 5, retry_in_seconds: retryInSeconds, - session_id: "sess-1", - task_id: "task-1", + session_id: TEST_SESSION_ID, + task_id: TEST_TASK_ID, actions: [], }; } @@ -94,8 +97,8 @@ function recoveryMessage(withParams = false): Message { ...(withParams ? { params: { - method: "session.recover", - payload: { task_id: "task-1", session_id: "sess-1" }, + method: SESSION_RECOVER_METHOD, + payload: { task_id: TEST_TASK_ID, session_id: TEST_SESSION_ID }, }, } : {}), @@ -116,7 +119,7 @@ function stalledMessage(turnId = "turn-1"): Message { type: "ws_request", label: "Cancel turn", test_id: STALL_CANCEL_TEST_ID, - params: { method: "agent.cancel", payload: { session_id: "sess-1" } }, + params: { method: "agent.cancel", payload: { session_id: TEST_SESSION_ID } }, }, ], }, @@ -135,12 +138,12 @@ function renderAction( ? { taskSessions: { items: { - "sess-1": { state: sessionState, error_message: sessionError } as TaskSession, + [TEST_SESSION_ID]: { state: sessionState, error_message: sessionError } as TaskSession, }, }, turns: { bySession: {}, - activeBySession: activeTurnId ? { "sess-1": activeTurnId } : {}, + activeBySession: activeTurnId ? { [TEST_SESSION_ID]: activeTurnId } : {}, }, } : {}; @@ -176,8 +179,8 @@ describe("ActionMessage — transient retry (warning variant)", () => { failure_code: "model_capacity", provider_name: "Codex", model_id: "gpt-5", - session_id: "sess-1", - task_id: "task-1", + session_id: TEST_SESSION_ID, + task_id: TEST_TASK_ID, actions: [], }, }), @@ -196,9 +199,9 @@ describe("ActionMessage — transient retry (warning variant)", () => { renderAction(retryMessage(), "WAITING_FOR_INPUT"); fireEvent.click(screen.getByTestId(CANCEL_TEST_ID)); await waitFor(() => expect(requestMock).toHaveBeenCalledTimes(1)); - expect(requestMock).toHaveBeenCalledWith("session.recover", { - task_id: "task-1", - session_id: "sess-1", + expect(requestMock).toHaveBeenCalledWith(SESSION_RECOVER_METHOD, { + task_id: TEST_TASK_ID, + session_id: TEST_SESSION_ID, action: "cancel_retry", }); }); @@ -309,7 +312,7 @@ describe("ActionMessage — running stall notice", () => { fireEvent.click(screen.getByTestId(STALL_CANCEL_TEST_ID)); await waitFor(() => - expect(requestMock).toHaveBeenCalledWith("agent.cancel", { session_id: "sess-1" }), + expect(requestMock).toHaveBeenCalledWith("agent.cancel", { session_id: TEST_SESSION_ID }), ); }); @@ -461,6 +464,57 @@ describe("ActionMessage — provider quota recovery", () => { }); }); +describe("ActionMessage — managed npm runtime recovery", () => { + it("renders one localized retry action with collapsed technical details", async () => { + renderAction( + retryMessage({ + content: "managed runtime failed", + metadata: { + variant: "error", + recovery_actions: true, + failure_kind: "managed_runtime_npm_resolution", + error_output: "npm error code ETARGET\nnpm error notarget No matching version found", + actions: [ + { + type: "ws_request", + label: "backend label is ignored", + test_id: "managed-runtime-npm-retry-button", + params: { + method: SESSION_RECOVER_METHOD, + payload: { + task_id: TEST_TASK_ID, + session_id: TEST_SESSION_ID, + action: "runtime_retry", + }, + }, + }, + ], + }, + } as Partial), + "WAITING_FOR_INPUT", + ); + + const card = screen.getByTestId("managed-runtime-npm-recovery"); + expect(card.textContent).toContain("npm could not prepare the runtime"); + expect(card.textContent).toContain("Kandev refreshed package data"); + expect(card.textContent).not.toMatch(/ACP/i); + expect(screen.getByText(TECHNICAL_DETAILS).closest("details")?.open).toBe(false); + expect(screen.getAllByRole("button")).toHaveLength(1); + expect(screen.getByTestId("managed-runtime-npm-retry-button").textContent).toContain( + "Retry runtime", + ); + + fireEvent.click(screen.getByTestId("managed-runtime-npm-retry-button")); + await waitFor(() => + expect(requestMock).toHaveBeenCalledWith(SESSION_RECOVER_METHOD, { + task_id: TEST_TASK_ID, + session_id: TEST_SESSION_ID, + action: "runtime_retry", + }), + ); + }); +}); + describe("ActionMessage — remediation link", () => { const REMEDIATION_URL = "https://opencode.ai/workspace/wrk_01KQM7K5CYT715264YKKFB17ZY/go"; const QUOTA_OUTPUT = "5-hour usage limit reached"; diff --git a/apps/web/components/task/chat/messages/action-message.tsx b/apps/web/components/task/chat/messages/action-message.tsx index c028b3c7f1..dd470e6022 100644 --- a/apps/web/components/task/chat/messages/action-message.tsx +++ b/apps/web/components/task/chat/messages/action-message.tsx @@ -11,7 +11,6 @@ import { IconSparkles, IconGitCommit, IconX, - IconChevronDown, } from "@tabler/icons-react"; import { cn } from "@/lib/utils"; import { Button } from "@kandev/ui/button"; @@ -21,11 +20,9 @@ import { useAppStore, useAppStoreApi } from "@/components/state-provider"; import { useArchiveAndSwitchTask } from "@/hooks/use-task-actions"; import { useTaskRemoval } from "@/hooks/use-task-removal"; import { deleteTask } from "@/lib/api/domains/kanban-api"; -import { AuthMethodsPanel, GenericAuthPanel } from "./auth-methods-panel"; -import { RemediationLink } from "@/components/task/remediation-link"; -import { HostShellDialog } from "@/components/settings/host-shell-dialog"; import type { Message, TaskSessionState } from "@/lib/types/http"; -import type { MessageAction, RecoveryAuthMethod } from "@/components/task/chat/types"; +import type { MessageAction } from "@/components/task/chat/types"; +import { ActionMessageDetails, type ActionMeta } from "./action-message-details"; import { formatDateTime } from "@/lib/i18n/formats"; import { parseRetryAt, retryCountdownLabel } from "./transient-retry"; @@ -40,28 +37,6 @@ const ICON_MAP: Record = { x: IconX, }; -type ActionMeta = { - actions?: MessageAction[]; - action_visibility?: "running"; - variant?: string; - recovery_actions?: boolean; - is_auth_error?: boolean; - auth_methods?: RecoveryAuthMethod[]; - error_output?: string; - failure_kind?: string; - missing_branch?: string; - provider_name?: string; - model_id?: string; - reset_at?: string; - remediation_url?: string; - retrying?: boolean; - attempt?: number; - max_attempts?: number; - retry_in_seconds?: number; - retry_at?: string; - failure_code?: string; -}; - function isSessionActive(state?: TaskSessionState) { return state === "RUNNING" || state === "STARTING" || state === "COMPLETED"; } @@ -302,9 +277,62 @@ function renderSpecialRecovery({ /> ); } + if (metadata?.failure_kind === "managed_runtime_npm_resolution") { + return ( + + ); + } return null; } +function ManagedRuntimeNpmRecovery({ + metadata, + taskId, + onRecoveryRequested, +}: { + metadata: ActionMeta; + taskId?: string; + onRecoveryRequested: () => void; +}) { + const { t } = useTranslation(); + const actions = metadata.actions?.slice(0, 1) ?? []; + return ( +
+
+
+
+ ); +} + function ProviderQuotaRecovery({ metadata, taskId, @@ -441,80 +469,18 @@ function MissingBranchRecovery({ ); } -function TechnicalDetails({ children }: { children: string }) { - const { t } = useTranslation(); - return ( -
- - - {t("chat:technicalDetails")} - -
-        {children}
-      
-
- ); -} - -function ActionMessageDetails({ - metadata, - technicalDetails, -}: { - metadata: ActionMeta | undefined; - technicalDetails?: string; -}) { - const [hostShellOpen, setHostShellOpen] = useState(false); - const [hostShellCommand, setHostShellCommand] = useState(undefined); - - // Auth recovery uses the kandev host shell (where the agent CLIs are - // installed), not the task environment shell - the task env often isn't - // ready when an auth error fires (no workspace path yet), and the user's - // agent auth state lives in their home dir on the host anyway. - const openHostShellWithCommand = useCallback((command: string) => { - // Trailing newline runs the command immediately. Drop it if you'd rather - // let the user review first. - setHostShellCommand(command + "\n"); - setHostShellOpen(true); - }, []); - const openHostShell = useCallback(() => { - setHostShellCommand(undefined); - setHostShellOpen(true); - }, []); - - if (!metadata) return null; - const errorOutput = metadata.error_output || technicalDetails; - return ( - <> - {metadata.remediation_url && } - {errorOutput && {errorOutput}} - {metadata.is_auth_error && metadata.auth_methods && metadata.auth_methods.length > 0 && ( - - )} - {metadata.is_auth_error && (!metadata.auth_methods || metadata.auth_methods.length === 0) && ( - - )} - - - ); -} - function ActionButtons({ actions, taskId, onRecoveryRequested, compact = false, + labelOverride, }: { actions: MessageAction[]; taskId?: string; onRecoveryRequested?: () => void; compact?: boolean; + labelOverride?: string; }) { return (
))}
@@ -542,11 +509,13 @@ function ActionButton({ messageTaskId, onCompleted, compact = false, + labelOverride, }: { action: MessageAction; messageTaskId?: string; onCompleted?: () => void; compact?: boolean; + labelOverride?: string; }): ReactElement | null { const [state, setState] = useState<"idle" | "busy" | "done" | "error">("idle"); const activeTaskId = useAppStore((s) => s.tasks.activeTaskId); @@ -618,7 +587,7 @@ function ActionButton({ data-testid={action.test_id} > {Icon && } - {action.label} + {labelOverride ?? action.label} ); diff --git a/apps/web/components/task/chat/types.ts b/apps/web/components/task/chat/types.ts index e7e3935bac..257b8f7a1a 100644 --- a/apps/web/components/task/chat/types.ts +++ b/apps/web/components/task/chat/types.ts @@ -198,6 +198,8 @@ export type RecoveryMetadata = StatusMetadata & { model_id?: string; reset_at?: string; error_output?: string; + failure_code?: string; + failure_details?: string; }; export type MessageAction = { diff --git a/apps/web/components/task/simple/chat-entries.test.ts b/apps/web/components/task/simple/chat-entries.test.ts index d25050e9df..a900ec7953 100644 --- a/apps/web/components/task/simple/chat-entries.test.ts +++ b/apps/web/components/task/simple/chat-entries.test.ts @@ -22,6 +22,22 @@ function session(overrides: Partial): TaskSession { } describe("buildRunErrorsFromSessions", () => { + it("preserves structured managed runtime failure metadata", () => { + const errors = buildRunErrorsFromSessions([ + session({ + metadata: { + last_agent_error: { + message: "managed npm runtime failed to prepare", + failure_code: "managed_runtime_npm_resolution", + failure_details: "npm error code ETARGET", + }, + }, + }), + ]); + expect(errors[0].failureCode).toBe("managed_runtime_npm_resolution"); + expect(errors[0].failureDetails).toBe("npm error code ETARGET"); + }); + it("preserves the remediation URL from last_agent_error metadata", () => { const errors = buildRunErrorsFromSessions([ session({ diff --git a/apps/web/components/task/simple/chat-entries.ts b/apps/web/components/task/simple/chat-entries.ts index fc4b24ee8e..209e4ba140 100644 --- a/apps/web/components/task/simple/chat-entries.ts +++ b/apps/web/components/task/simple/chat-entries.ts @@ -57,12 +57,15 @@ export function buildRunErrorsFromSessions(sessions: TaskSession[]): RunError[] for (const s of sessions) { if (s.state !== "FAILED") continue; const failedAt = s.completedAt ?? s.updatedAt ?? s.startedAt ?? ""; + const lastError = lastAgentErrorFromSessionMetadata(s.metadata); errors.push({ id: `re-${s.id}`, sessionId: s.id, agentProfileId: s.agentProfileId, rawPayload: s.errorMessage ?? "", failedAt, + failureCode: lastError?.code, + failureDetails: lastError?.details, remediationUrl: normalizeRemediationUrl(remediationUrlFromSessionMetadata(s.metadata)) ?? undefined, }); @@ -106,6 +109,21 @@ function remediationUrlFromSessionMetadata(metadata: Record | n return typeof raw === "string" && raw !== "" ? raw : undefined; } +function lastAgentErrorFromSessionMetadata( + metadata: Record | null | undefined, +): { code?: string; details?: string } | undefined { + const lastError = metadata?.last_agent_error; + if (!lastError || typeof lastError !== "object") return undefined; + const record = lastError as Record; + const code = record.code ?? record.failure_code ?? record.failureCode; + const details = + record.details ?? record.failure_details ?? record.failureDetails ?? record.error_output; + return { + code: typeof code === "string" && code !== "" ? code : undefined, + details: typeof details === "string" && details !== "" ? details : undefined, + }; +} + export type MergeChatEntriesArgs = { comments: TaskComment[]; timeline: TimelineEvent[]; diff --git a/apps/web/components/task/simple/components/managed-runtime-npm-run-error.tsx b/apps/web/components/task/simple/components/managed-runtime-npm-run-error.tsx new file mode 100644 index 0000000000..e77b13d22f --- /dev/null +++ b/apps/web/components/task/simple/components/managed-runtime-npm-run-error.tsx @@ -0,0 +1,74 @@ +"use client"; + +import { useState } from "react"; +import { IconAlertTriangle, IconChevronDown, IconRefresh } from "@tabler/icons-react"; +import { Button } from "@kandev/ui/button"; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@kandev/ui/collapsible"; +import { AgentAvatar } from "@/app/office/components/agent-avatar"; +import type { RunError } from "@/app/office/tasks/[id]/types"; +import { formatRelativeTime } from "@/lib/utils"; +import { useTranslation } from "react-i18next"; + +export function ManagedRuntimeNpmRunError({ + error, + agentName, + onRetry, +}: { + error: RunError; + agentName: string; + onRetry: () => void; +}) { + const { t } = useTranslation(); + const [showDetails, setShowDetails] = useState(false); + const technicalDetails = error.failureDetails; + + return ( +
+ +
+
+ {agentName} + + + + {formatRelativeTime(error.failedAt)} + +
+

{t("chat:managedRuntimeNpmBody")}

+ {technicalDetails && ( + + + + {t("chat:technicalDetails")} + + +
+                {technicalDetails}
+              
+
+
+ )} +
+ +
+
+
+ ); +} diff --git a/apps/web/components/task/simple/components/run-error-entry.tsx b/apps/web/components/task/simple/components/run-error-entry.tsx index 2e062df115..d546716147 100644 --- a/apps/web/components/task/simple/components/run-error-entry.tsx +++ b/apps/web/components/task/simple/components/run-error-entry.tsx @@ -16,6 +16,7 @@ import { formatRelativeTime } from "@/lib/utils"; import { AgentAvatar } from "@/app/office/components/agent-avatar"; import { RemediationLink } from "@/components/task/remediation-link"; import type { RunError } from "@/app/office/tasks/[id]/types"; +import { ManagedRuntimeNpmRunError } from "./managed-runtime-npm-run-error"; import { useTranslation } from "react-i18next"; type RunErrorEntryProps = { @@ -41,7 +42,7 @@ export function RunErrorEntry({ taskId, error }: RunErrorEntryProps) { ); const [showDetails, setShowDetails] = useState(false); - const handleRecover = async (action: "resume" | "fresh_start") => { + const handleRecover = async (action: "resume" | "fresh_start" | "runtime_retry") => { const client = getWebSocketClient(); if (!client) return; try { @@ -55,6 +56,16 @@ export function RunErrorEntry({ taskId, error }: RunErrorEntryProps) { } }; + if (error.failureCode === "managed_runtime_npm_resolution") { + return ( + handleRecover("runtime_retry")} + /> + ); + } + return (
diff --git a/apps/web/components/task/simple/task-chat.test.tsx b/apps/web/components/task/simple/task-chat.test.tsx index d8d25944db..6a67195e80 100644 --- a/apps/web/components/task/simple/task-chat.test.tsx +++ b/apps/web/components/task/simple/task-chat.test.tsx @@ -592,6 +592,29 @@ describe("TaskChat run error entries", () => { expect(screen.getByTestId("run-error-raw-payload").textContent).toContain("Internal error"); }); + it("renders the managed npm recovery entry with one touch-safe action", () => { + const sessions: TaskSession[] = [ + failedSession("s-npm", "agent-npm", "2026-05-01T11:30:00Z", "runtime failed"), + ]; + sessions[0].metadata = { + last_agent_error: { + message: "managed npm runtime failed to prepare", + code: "managed_runtime_npm_resolution", + details: "npm error code ETARGET", + }, + }; + render(wrap()); + + expect(screen.getByTestId("run-error-managed-runtime-npm-recovery")).toBeTruthy(); + expect(screen.getByText(/npm could not prepare the runtime/i)).toBeTruthy(); + expect(screen.getByTestId("run-error-managed-runtime-retry-button")).toBeTruthy(); + expect(screen.getByTestId("run-error-managed-runtime-retry-button").className).toContain( + "min-h-11", + ); + expect(screen.queryByTestId("run-error-resume-button")).toBeNull(); + expect(screen.queryByTestId("run-error-fresh-button")).toBeNull(); + }); + it("does not render an error entry for non-FAILED sessions", () => { const sessions: TaskSession[] = [ { diff --git a/apps/web/e2e/tests/office/managed-runtime-npm-recovery.spec.ts b/apps/web/e2e/tests/office/managed-runtime-npm-recovery.spec.ts new file mode 100644 index 0000000000..a245da65bb --- /dev/null +++ b/apps/web/e2e/tests/office/managed-runtime-npm-recovery.spec.ts @@ -0,0 +1,55 @@ +import { test, expect } from "../../fixtures/office-fixture"; + +test("keeps managed npm recovery visible in Office chat", async ({ + testPage, + apiClient, + officeSeed, +}) => { + const sentFrames: string[] = []; + testPage.on("websocket", (ws) => { + if (!ws.url().endsWith("/ws")) return; + ws.on("framesent", (event) => { + if (typeof event.payload === "string") sentFrames.push(event.payload); + }); + }); + + const task = await apiClient.createTask(officeSeed.workspaceId, "Office managed npm recovery", { + workflow_id: officeSeed.workflowId, + }); + await apiClient.rawRequest("PATCH", `/api/v1/office/tasks/${task.id}`, { + assignee_agent_profile_id: officeSeed.agentId, + }); + await apiClient.seedTaskSession(task.id, { + state: "FAILED", + agentProfileId: officeSeed.agentId, + completedAt: "2026-08-16T09:15:44Z", + metadata: { + last_agent_error: { + message: "managed npm runtime failed to prepare", + code: "managed_runtime_npm_resolution", + details: "npm error code ETARGET\nnpm error notarget No matching version found", + }, + }, + }); + + await testPage.goto(`/office/tasks/${task.id}`); + await expect( + testPage.getByRole("heading", { name: "Office managed npm recovery" }), + ).toBeVisible(); + + const recovery = testPage.getByTestId("run-error-managed-runtime-npm-recovery"); + await expect(recovery).toBeVisible(); + await expect(recovery.getByText("npm could not prepare the runtime")).toBeVisible(); + await expect(recovery.getByTestId("run-error-managed-runtime-retry-button")).toHaveCount(1); + await expect(recovery.getByTestId("run-error-resume-button")).toHaveCount(0); + await expect(recovery.getByTestId("run-error-fresh-button")).toHaveCount(0); + await expect(recovery.getByRole("button", { name: "Technical details" })).toHaveAttribute( + "aria-expanded", + "false", + ); + + await recovery.getByTestId("run-error-managed-runtime-retry-button").click(); + await expect + .poll(() => sentFrames.find((frame) => frame.includes('"action":"session.recover"')) ?? "") + .toContain('"action":"runtime_retry"'); +}); diff --git a/apps/web/e2e/tests/session/managed-runtime-npm-recovery.spec.ts b/apps/web/e2e/tests/session/managed-runtime-npm-recovery.spec.ts new file mode 100644 index 0000000000..0b9c7884d6 --- /dev/null +++ b/apps/web/e2e/tests/session/managed-runtime-npm-recovery.spec.ts @@ -0,0 +1,89 @@ +import { test, expect } from "../../fixtures/test-base"; +import type { ApiClient } from "../../helpers/api-client"; +import type { SeedData } from "../../fixtures/test-base"; +import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions"; +import { SessionPage } from "../../pages/session-page"; + +async function seedManagedRuntimeFailure(apiClient: ApiClient, seedData: SeedData) { + const task = await apiClient.createTask(seedData.workspaceId, "Managed npm runtime recovery", { + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + }); + const { session_id: sessionId } = await apiClient.seedTaskSession(task.id, { + state: "FAILED", + completedAt: "2026-08-16T09:15:44Z", + }); + await apiClient.seedSessionMessage(sessionId, { + type: "status", + content: "managed npm runtime failed to prepare", + metadata: { + recovery_actions: true, + failure_kind: "managed_runtime_npm_resolution", + error_output: "npm error code ETARGET\nnpm error notarget No matching version found", + actions: [ + { + type: "ws_request", + label: "Retry runtime", + test_id: "managed-runtime-npm-retry-button", + params: { + method: "session.recover", + payload: { task_id: task.id, session_id: sessionId, action: "runtime_retry" }, + }, + }, + ], + }, + }); + return task; +} + +test("renders managed npm recovery with one retry action", async ({ + testPage, + apiClient, + seedData, + prCapture, +}, testInfo) => { + const sentFrames: string[] = []; + testPage.on("websocket", (ws) => { + if (!ws.url().endsWith("/ws")) return; + ws.on("framesent", (event) => { + if (typeof event.payload === "string") sentFrames.push(event.payload); + }); + }); + + const task = await seedManagedRuntimeFailure(apiClient, seedData); + await testPage.goto(`/t/${task.id}`); + const session = new SessionPage(testPage); + await session.waitForLoad(); + + const recovery = session.activeChat().getByTestId("managed-runtime-npm-recovery"); + await expect(recovery).toBeVisible(); + await expect( + recovery.getByRole("heading", { name: "npm could not prepare the runtime" }), + ).toBeVisible(); + await expect(recovery).toContainText( + "Kandev refreshed package data and retried the same runtime once.", + ); + await expect(recovery).not.toContainText("ACP"); + await expect(recovery.locator("details")).not.toHaveAttribute("open"); + await expect(recovery.getByTestId("managed-runtime-npm-retry-button")).toHaveCount(1); + await expect(recovery.getByRole("button")).toHaveCount(1); + + await assertNoDocumentHorizontalOverflow(testPage, "managed npm recovery"); + await testPage.screenshot({ + path: testInfo.outputPath("managed-runtime-npm-recovery-desktop.png"), + fullPage: true, + }); + await prCapture.screenshot("managed-runtime-npm-recovery-desktop", { + caption: "Managed npm runtime recovery card", + fullPage: true, + }); + + await recovery.getByText("Technical details", { exact: true }).click(); + await expect(recovery.locator("pre")).toBeVisible(); + await expect(recovery.locator("pre")).toContainText("ETARGET"); + + await recovery.getByTestId("managed-runtime-npm-retry-button").click(); + await expect + .poll(() => sentFrames.find((frame) => frame.includes('"action":"session.recover"')) ?? "") + .toContain('"action":"runtime_retry"'); +}); diff --git a/apps/web/e2e/tests/session/mobile-managed-runtime-npm-recovery.spec.ts b/apps/web/e2e/tests/session/mobile-managed-runtime-npm-recovery.spec.ts new file mode 100644 index 0000000000..ad1b18c8b6 --- /dev/null +++ b/apps/web/e2e/tests/session/mobile-managed-runtime-npm-recovery.spec.ts @@ -0,0 +1,85 @@ +// Filename starts with "mobile-" so this runs on the mobile-chrome project. +import { test, expect } from "../../fixtures/test-base"; +import type { ApiClient } from "../../helpers/api-client"; +import type { SeedData } from "../../fixtures/test-base"; +import { assertNoDocumentHorizontalOverflow } from "../../helpers/layout-assertions"; +import { SessionPage } from "../../pages/session-page"; + +async function seedManagedRuntimeFailure(apiClient: ApiClient, seedData: SeedData) { + const task = await apiClient.createTask(seedData.workspaceId, "Mobile managed npm recovery", { + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + }); + const { session_id: sessionId } = await apiClient.seedTaskSession(task.id, { + state: "FAILED", + completedAt: "2026-08-16T09:15:44Z", + }); + await apiClient.seedSessionMessage(sessionId, { + type: "status", + content: "managed npm runtime failed to prepare", + metadata: { + recovery_actions: true, + failure_kind: "managed_runtime_npm_resolution", + error_output: "npm error code ETARGET\nnpm error notarget No matching version found", + actions: [ + { + type: "ws_request", + label: "Retry runtime", + test_id: "managed-runtime-npm-retry-button", + params: { + method: "session.recover", + payload: { task_id: task.id, session_id: sessionId, action: "runtime_retry" }, + }, + }, + ], + }, + }); + return task; +} + +test("keeps managed npm recovery touch-safe on mobile", async ({ + testPage, + apiClient, + seedData, + prCapture, +}, testInfo) => { + const sentFrames: string[] = []; + testPage.on("websocket", (ws) => { + if (!ws.url().endsWith("/ws")) return; + ws.on("framesent", (event) => { + if (typeof event.payload === "string") sentFrames.push(event.payload); + }); + }); + + const task = await seedManagedRuntimeFailure(apiClient, seedData); + await testPage.goto(`/t/${task.id}`); + const session = new SessionPage(testPage); + await session.waitForLoad(); + + const recovery = session.activeChat().getByTestId("managed-runtime-npm-recovery"); + await expect(recovery).toBeVisible(); + await expect(recovery.getByRole("button")).toHaveCount(1); + const retry = recovery.getByTestId("managed-runtime-npm-retry-button"); + await expect(retry).toBeInViewport(); + const box = await retry.boundingBox(); + expect(box).not.toBeNull(); + expect(box!.height).toBeGreaterThanOrEqual(44); + await expect(recovery.locator("details")).not.toHaveAttribute("open"); + await expect(testPage.getByRole("dialog")).toHaveCount(0); + + await retry.tap(); + await expect + .poll(() => sentFrames.find((frame) => frame.includes('"action":"session.recover"')) ?? "") + .toContain('"action":"runtime_retry"'); + await expect(testPage.getByRole("dialog")).toHaveCount(0); + await assertNoDocumentHorizontalOverflow(testPage, "mobile managed npm recovery"); + + await testPage.screenshot({ + path: testInfo.outputPath("managed-runtime-npm-recovery-mobile.png"), + fullPage: true, + }); + await prCapture.screenshot("managed-runtime-npm-recovery-mobile", { + caption: "Managed npm runtime recovery remains touch-safe on mobile", + fullPage: true, + }); +}); diff --git a/apps/web/lib/session-last-agent-error.test.ts b/apps/web/lib/session-last-agent-error.test.ts index 8c5776c844..2b3f216a16 100644 --- a/apps/web/lib/session-last-agent-error.test.ts +++ b/apps/web/lib/session-last-agent-error.test.ts @@ -16,8 +16,42 @@ const OTHER_TAB_SESSION_ID = "session-other-tab"; const THIS_TAB_SESSION_ID = "session-this-tab"; const OTHER_TAB_STAMP = "stamp-other"; const THIS_TAB_STAMP = "stamp-this"; +const MANAGED_RUNTIME_NPM_FAILURE = "managed_runtime_npm_resolution"; +const NPM_ERROR_CODE_ETARGET = "npm error code ETARGET"; describe("readLastAgentError", () => { + it("reads structured failure fields in snake_case", () => { + expect( + readLastAgentError({ + last_agent_error: { + message: AGENT_ERROR_MESSAGE, + failure_code: MANAGED_RUNTIME_NPM_FAILURE, + failure_details: NPM_ERROR_CODE_ETARGET, + }, + }), + ).toMatchObject({ + code: MANAGED_RUNTIME_NPM_FAILURE, + details: NPM_ERROR_CODE_ETARGET, + }); + }); + + it("reads structured failure fields in camelCase", () => { + expect( + readLastAgentError({ + last_agent_error: { + message: AGENT_ERROR_MESSAGE, + code: MANAGED_RUNTIME_NPM_FAILURE, + details: NPM_ERROR_CODE_ETARGET, + }, + }), + ).toMatchObject({ + code: MANAGED_RUNTIME_NPM_FAILURE, + details: NPM_ERROR_CODE_ETARGET, + }); + }); +}); + +describe("readLastAgentError optional metadata", () => { it("reads snake_case metadata and keeps occurredAt optional", () => { expect( readLastAgentError({ diff --git a/apps/web/lib/session-last-agent-error.ts b/apps/web/lib/session-last-agent-error.ts index f0637eb71d..5f4fad8fd6 100644 --- a/apps/web/lib/session-last-agent-error.ts +++ b/apps/web/lib/session-last-agent-error.ts @@ -6,6 +6,8 @@ export type LastAgentError = { agentExecutionId?: string; /** Adapter-validated provider remediation URL; never derived from prose. */ remediationUrl?: string; + code?: string; + details?: string; dismissedAt?: string; }; @@ -49,19 +51,37 @@ export function readLastAgentError(metadata: Record | null | un const record = raw as Record; const message = typeof record.message === "string" ? record.message : ""; if (!message) return null; - const dismissedAt = - readOptionalString(record.dismissed_at) ?? readOptionalString(record.dismissedAt); + const dismissedAt = readFirstOptionalString(record, ["dismissed_at", "dismissedAt"]); if (dismissedAt) return null; return { message, - occurredAt: readOptionalString(record.occurred_at) ?? readOptionalString(record.occurredAt), - agentExecutionId: - readOptionalString(record.agent_execution_id) ?? readOptionalString(record.agentExecutionId), - remediationUrl: - readOptionalString(record.remediation_url) ?? readOptionalString(record.remediationUrl), + occurredAt: readFirstOptionalString(record, ["occurred_at", "occurredAt"]), + agentExecutionId: readFirstOptionalString(record, ["agent_execution_id", "agentExecutionId"]), + remediationUrl: readFirstOptionalString(record, ["remediation_url", "remediationUrl"]), + ...readStructuredFailureMetadata(record), } satisfies LastAgentError; } +function readStructuredFailureMetadata(record: Record) { + return { + code: readFirstOptionalString(record, ["code", "failure_code", "failureCode"]), + details: readFirstOptionalString(record, [ + "details", + "failure_details", + "failureDetails", + "error_output", + ]), + }; +} + +function readFirstOptionalString(record: Record, keys: string[]) { + for (const key of keys) { + const value = readOptionalString(record[key]); + if (value) return value; + } + return undefined; +} + /** * Stable identifier for a specific error event. Two errors share a stamp iff * they have the same occurredAt timestamp and message. Used to decide whether diff --git a/apps/web/src/locales/en/chat.json b/apps/web/src/locales/en/chat.json index c5380898b1..05149ddfd8 100644 --- a/apps/web/src/locales/en/chat.json +++ b/apps/web/src/locales/en/chat.json @@ -130,6 +130,9 @@ "sendNowReferenceOverflow": "Combined entity references exceed the message limit.", "sortable": "sortable", "technicalDetails": "Technical details", + "managedRuntimeNpmTitle": "npm could not prepare the runtime", + "managedRuntimeNpmBody": "Kandev refreshed package data and retried the same runtime once.", + "managedRuntimeRetry": "Retry runtime", "theAssociatedWorktree": "The associated worktree", "theTaskAndItsData": "The task and its data", "thisActionCannotBeUndone": "This action cannot be undone.", diff --git a/apps/web/src/locales/pseudo/chat.json b/apps/web/src/locales/pseudo/chat.json index 9a7fa85981..993b661708 100644 --- a/apps/web/src/locales/pseudo/chat.json +++ b/apps/web/src/locales/pseudo/chat.json @@ -130,6 +130,9 @@ "sendNowReferenceOverflow": "Ćōḿƀĩńēď ēńţĩţŷ ŕēƒēŕēńćēś ēxćēēď ţĥē ḿēśśàĝē ĺĩḿĩţ.", "sortable": "śōŕţàƀĺē", "technicalDetails": "Ţēćĥńĩćàĺ ďēţàĩĺś", + "managedRuntimeNpmTitle": "ńƥḿ ćōũĺď ńōţ ƥŕēƥàŕē ţĥē ŕũńţĩḿē", + "managedRuntimeNpmBody": "Ķàńďēvē ƒŕēśĥēď ƥàćķàĝē ďàţà àńď ŕēţŕĩēď ţĥē śàḿē ŕũńţĩḿē ōńćē.", + "managedRuntimeRetry": "Ŕēţŕŷ ŕũńţĩḿē", "theAssociatedWorktree": "Ţĥē àśśōćĩàţēď ŵōŕķţŕēē", "theTaskAndItsData": "Ţĥē ţàśķ àńď ĩţś ďàţà", "thisActionCannotBeUndone": "Ţĥĩś àćţĩōń ćàńńōţ ƀē ũńďōńē.", diff --git a/apps/web/src/locales/pt-pt/chat.json b/apps/web/src/locales/pt-pt/chat.json index ac713a1319..16c202bab0 100644 --- a/apps/web/src/locales/pt-pt/chat.json +++ b/apps/web/src/locales/pt-pt/chat.json @@ -114,6 +114,9 @@ "sendNowAttachmentOverflow": "Os anexos combinados excedem os limites da mensagem.", "sendNowReferenceOverflow": "As referências a entidades combinadas excedem o limite da mensagem.", "technicalDetails": "Detalhes técnicos", + "managedRuntimeNpmTitle": "O npm não conseguiu preparar o runtime", + "managedRuntimeNpmBody": "O Kandev atualizou os dados do pacote e repetiu o mesmo runtime uma vez.", + "managedRuntimeRetry": "Repetir runtime", "theAssociatedWorktree": "O worktree associado", "theTaskAndItsData": "A tarefa e os respetivos dados", "thisActionCannotBeUndone": "Esta ação não pode ser anulada.", diff --git a/apps/web/src/locales/zh-cn/chat.json b/apps/web/src/locales/zh-cn/chat.json index 702a636ec4..7a113eb9b2 100644 --- a/apps/web/src/locales/zh-cn/chat.json +++ b/apps/web/src/locales/zh-cn/chat.json @@ -56,6 +56,9 @@ "reorderQueuedMessage": "重新排序队列消息", "sortable": "可排序", "technicalDetails": "技术详情", + "managedRuntimeNpmTitle": "npm 无法准备运行时", + "managedRuntimeNpmBody": "Kandev 已刷新软件包数据,并重试了同一运行时一次。", + "managedRuntimeRetry": "重试运行时", "providerQuotaTitle": "{{provider}} 使用额度已达上限", "newChatTab": "新建聊天", "newQuickChat": "新建快速聊天", diff --git a/apps/web/src/locales/zh-hk/chat.json b/apps/web/src/locales/zh-hk/chat.json index db08706c5a..4383d3a40b 100644 --- a/apps/web/src/locales/zh-hk/chat.json +++ b/apps/web/src/locales/zh-hk/chat.json @@ -56,6 +56,9 @@ "reorderQueuedMessage": "重新排序佇列訊息", "sortable": "可排序", "technicalDetails": "技術詳情", + "managedRuntimeNpmTitle": "npm 無法準備執行環境", + "managedRuntimeNpmBody": "Kandev 已重新整理套件資料,並重試相同的執行環境一次。", + "managedRuntimeRetry": "重試執行環境", "providerQuotaTitle": "{{provider}} 使用額度已達上限", "newChatTab": "新建聊天", "newQuickChat": "新建快速聊天", diff --git a/apps/web/src/locales/zh-tw/chat.json b/apps/web/src/locales/zh-tw/chat.json index 5537fc4eae..2cc80ae15c 100644 --- a/apps/web/src/locales/zh-tw/chat.json +++ b/apps/web/src/locales/zh-tw/chat.json @@ -56,6 +56,9 @@ "reorderQueuedMessage": "重新排序佇列訊息", "sortable": "可排序", "technicalDetails": "技術詳情", + "managedRuntimeNpmTitle": "npm 無法準備執行環境", + "managedRuntimeNpmBody": "Kandev 已重新整理套件資料,並重試相同的執行環境一次。", + "managedRuntimeRetry": "重試執行環境", "providerQuotaTitle": "{{provider}} 使用額度已達上限", "newChatTab": "新建聊天", "newQuickChat": "新建快速聊天", diff --git a/docs/plans/managed-runtime-stale-npm-recovery/plan.md b/docs/plans/managed-runtime-stale-npm-recovery/plan.md new file mode 100644 index 0000000000..097a4d5f71 --- /dev/null +++ b/docs/plans/managed-runtime-stale-npm-recovery/plan.md @@ -0,0 +1,207 @@ +--- +spec: docs/specs/agents/runtime-updates.md +created: 2026-08-16 +status: implemented +--- + +# Implementation Plan: Recover Stale npm Runtime Metadata + +## Overview + +Managed npm runtimes normally start with `npx --prefer-offline`. A stale npm +packument can then hide a dependency version that the configured registry +contains. The managed process exits, and ACP reports only that its peer +disconnected. This plan detects the strict npm `ETARGET` signature in captured +stderr, invalidates only the trusted runtime's deterministic `_npx` tree, and +retries the same package and version once with `--prefer-online`. If recovery +fails, Kanban and Office show an npm-specific inline card with one runtime retry +action and collapsed technical details. + +The implementation does not run `npm cache clean --force`. It does not change +the registry, package, active version, model, permissions, or session identity. + +## Backend + +### Classification and trusted command construction + +- `apps/backend/internal/agent/runtime/routingerr/runtime_rules.go`: add a + stable code for a managed npm version-resolution failure. Require npm + `ETARGET` and `No matching version found for package@version` in the same + bounded stderr sample. Match the supported old and new npm error prefixes. + Do not classify a generic ACP disconnect or unrelated npm failure. +- `apps/backend/internal/agent/agents/managed_npm_runtime.go`: add a trusted + command option for online metadata refresh. The option changes only + `--prefer-offline` to `--prefer-online`. The managed runtime spec still owns + the package and ACP arguments. +- Move deterministic `_npx` cache-key discovery and safe deletion behind one + helper in `apps/backend/internal/agent/managedruntime`. Reuse it from runtime + update jobs and launch recovery. Reject the npm cache root, the `_npx` root, + symlink escapes, and every path outside the exact derived key. + +### Bounded launch recovery + +- `apps/backend/internal/agent/runtime/lifecycle/manager_startup.go`: after a + host-local managed runtime fails ACP initialization, fetch its bounded stderr + and classify the combined failure evidence. Retry only the new exact + classifier. +- Stop the first child, invalidate the exact execution tree, rebuild the same + command in online mode, reconnect streams, and initialize once more. Preserve + the active exact version, command prefix, model, permissions, and session ID. +- Track a startup attempt generation on `AgentExecution`. Completion and stream + events from the first child must not publish a terminal state for the retry. + Cancellation and shutdown must win over recovery. +- Emit no user-facing failure when the online retry succeeds. If it fails, + publish one final event with a stable failure code and bounded sanitized + details. Never include raw host paths or use stderr values as command input. + +### Failure persistence and recovery actions + +- Extend the lifecycle and watcher event data with optional failure code and + details fields. Keep existing generic error fields compatible. +- Extend `models.LastAgentError` with optional code and details fields. Store + the fields in existing session JSON metadata, so no schema migration is + required. +- `apps/backend/internal/orchestrator/event_handlers_agent.go`: recognize the + structured npm failure. Persist one recovery message with + `failure_kind = managed_runtime_npm_resolution`, sanitized `error_output`, + and one **Retry runtime** action. Use resume mode when a resume token exists. + Otherwise, use the existing fresh-run recovery mode behind the same label. + +## Frontend + +### Shared recovery presentation + +- Extend `RecoveryMetadata` and `LastAgentError` parsers with the optional + structured failure fields. Accept snake case from the API and camel case + after store serialization. +- `apps/web/components/task/chat/messages/action-message.tsx`: render the + managed npm failure through the existing inline special-recovery path. State + that npm could not prepare the runtime and that Kandev refreshed package data + and retried once. Keep technical details collapsed and show only + **Retry runtime**. +- `apps/web/components/task/simple/chat-entries.ts` and + `apps/web/components/task/simple/components/run-error-entry.tsx`: carry the + same failure code and details to Office chat and render the same meaning and + action set. +- Add all copy to the existing i18n catalog. Do not mention ACP in user-facing + text. + +### Mobile design contract + +Kanban and Office keep their current single chat scroll owner. The recovery +card remains inline on desktop and phone. It does not open a dialog, sheet, or +drawer. On phone widths, actions stack when needed and each control is at least +44 px high. The nearest shipped mobile exemplar is the provider recovery card +in `action-message.tsx`. Business state, recovery requests, error parsing, and +copy keys remain shared across viewports. + +## Documentation + +- Update the troubleshooting and managed runtime sections in + `docs/public/agents-and-profiles.md`. Explain the automatic same-version + retry, the configured-registry check, and why global npm cache cleanup is not + the normal recovery path. +- Treat the page as a combined how-to and reference page. Keep commands and UI + names synchronized with the shipped behavior. + +## Tests + +- Classifier tests cover the reported npm error, old and new npm prefixes, + bounded sanitization, and false positives. +- Managed command tests prove that normal launches stay offline-preferred and + recovery changes only the npm freshness flag. +- Cache helper tests prove exact-key deletion, absent-key idempotency, and path + and symlink refusal. +- Lifecycle tests prove one successful retry, one terminal repeated failure, + preserved command identity, no intermediate failed event, stale event + rejection, and cancel or shutdown precedence. They also prove no retry for + remote, native, passthrough, and unrelated failures. +- Orchestrator tests prove structured error persistence and the single runtime + retry action for sessions with and without resume tokens. +- Frontend unit tests prove structured metadata parsing and the specialized + Kanban and Office presentations. + +## E2E Tests + +- **Scenario:** GIVEN a terminal failure during managed npm resolution in Kanban chat, + WHEN the recovery card renders on desktop, THEN it names npm, keeps details + collapsed, offers only **Retry runtime**, and sends the expected recovery + request. + **File:** `apps/web/e2e/tests/session/managed-runtime-npm-recovery.spec.ts`. +- **Scenario:** GIVEN the same failure on a phone viewport, WHEN the user opens + the task and taps **Retry runtime**, THEN the action remains touch-safe, the + chat has no horizontal overflow, and no nested overlay appears. + **File:** + `apps/web/e2e/tests/session/mobile-managed-runtime-npm-recovery.spec.ts`. +- **Scenario:** GIVEN the same persisted `last_agent_error` in Office chat, + WHEN the task reloads, THEN the npm-specific copy and one retry action remain + visible. + **File:** `apps/web/e2e/tests/office/managed-runtime-npm-recovery.spec.ts`. + +## Verification Commands + +Bootstrap a fresh worktree before frontend checks: + +```bash +cd apps +pnpm install --frozen-lockfile +``` + +Run focused backend tests: + +```bash +cd apps/backend +go test ./internal/agent/runtime/routingerr ./internal/agent/agents ./internal/agent/managedruntime +go test ./internal/agent/runtime/lifecycle -run 'Test.*ManagedRuntime.*Npm|Test.*StartupRecovery' +go test ./internal/orchestrator -run 'Test.*ManagedRuntime.*Recovery|Test.*RecoveryActions|Test.*LastAgentError' +``` + +Run focused frontend and contract checks: + +```bash +cd apps/web +pnpm exec vitest run components/task/chat/messages/action-message.test.tsx components/task/simple/chat-entries.test.ts components/task/simple/components/run-error-entry.test.tsx lib/session-last-agent-error.test.ts +pnpm run typecheck +pnpm run i18n:check +pnpm run i18n:ratchet +``` + +Build once and run the desktop and mobile browser scenarios: + +```bash +make build-web +make build-backend +cd apps/web +pnpm e2e:run --no-build tests/session/managed-runtime-npm-recovery.spec.ts tests/office/managed-runtime-npm-recovery.spec.ts +pnpm e2e:run --no-build --project mobile-chrome tests/session/mobile-managed-runtime-npm-recovery.spec.ts +``` + +Validate public documentation: + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` + +## Implementation Waves And Parallel Candidates + +Wave 1: + +- [x] [task-01-classify-and-build-recovery-command](task-01-classify-and-build-recovery-command.md) + +Wave 2: + +- [x] [task-02-retry-managed-runtime-launch](task-02-retry-managed-runtime-launch.md) + +Wave 3: + +- [x] [task-03-present-npm-recovery](task-03-present-npm-recovery.md) + +Wave 4: + +- [x] [task-04-prove-responsive-recovery](task-04-prove-responsive-recovery.md) +- [x] [task-05-document-runtime-recovery](task-05-document-runtime-recovery.md) + +Tasks 04 and 05 can run in parallel after the backend and frontend contracts +are stable. The default execution order remains sequential. No subagent +delegation is authorized by this plan. diff --git a/docs/plans/managed-runtime-stale-npm-recovery/task-01-classify-and-build-recovery-command.md b/docs/plans/managed-runtime-stale-npm-recovery/task-01-classify-and-build-recovery-command.md new file mode 100644 index 0000000000..1ffeb4da43 --- /dev/null +++ b/docs/plans/managed-runtime-stale-npm-recovery/task-01-classify-and-build-recovery-command.md @@ -0,0 +1,49 @@ +--- +id: "01-classify-and-build-recovery-command" +title: "Classify stale npm metadata" +status: completed +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/agents/runtime-updates.md" +--- + +# Task 01: Classify stale npm metadata + +Create the strict error signal and trusted recovery primitives before changing +the launch lifecycle. + +- **Acceptance:** Classification requires npm `ETARGET` and the matching + missing `package@version` message in bounded stderr. Generic ACP disconnects, + network errors, and unrelated npm failures do not match. +- **Acceptance:** Normal managed runtime commands keep `--prefer-offline`. A + trusted recovery option changes only that flag to `--prefer-online` and keeps + the same package, version, and ACP arguments. +- **Acceptance:** One shared cache helper derives and removes only the exact + `_npx` execution key for the trusted package spec. It rejects broad, escaped, + and symlinked targets and is idempotent when the key is absent. +- **Acceptance:** Runtime update jobs use the shared helper without changing + their current one-retry behavior. +- **Verification:** Add failing tests first, then run: + + ```bash + cd apps/backend + go test ./internal/agent/runtime/routingerr ./internal/agent/agents ./internal/agent/managedruntime + go test ./internal/agent/settings/controller -run 'Test.*ExecutionCache|Test.*AgentUpdate' + ``` + +- **Files likely touched:** + `apps/backend/internal/agent/runtime/routingerr/runtime_rules.go`, + `apps/backend/internal/agent/runtime/routingerr/runtime_rules_test.go`, + `apps/backend/internal/agent/agents/managed_npm_runtime.go`, + `apps/backend/internal/agent/agents/managed_npm_runtime_test.go`, + `apps/backend/internal/agent/managedruntime/cache.go`, + `apps/backend/internal/agent/managedruntime/cache_test.go`, and + `apps/backend/internal/agent/settings/controller/agent_update.go`. +- **Dependencies:** None. +- **Parallelism:** sequential foundation. +- **Inputs:** Launch-time stale metadata recovery and failure behavior in the + managed runtime spec. +- **Output contract:** Report files changed, RED and GREEN commands and results, + classifier false-positive evidence, cache deletion safety evidence, and + synchronized task and plan status. diff --git a/docs/plans/managed-runtime-stale-npm-recovery/task-02-retry-managed-runtime-launch.md b/docs/plans/managed-runtime-stale-npm-recovery/task-02-retry-managed-runtime-launch.md new file mode 100644 index 0000000000..2439554ac1 --- /dev/null +++ b/docs/plans/managed-runtime-stale-npm-recovery/task-02-retry-managed-runtime-launch.md @@ -0,0 +1,50 @@ +--- +id: "02-retry-managed-runtime-launch" +title: "Retry managed runtime launch" +status: done +wave: 2 +depends_on: ["01-classify-and-build-recovery-command"] +plan: "plan.md" +spec: "../../specs/agents/runtime-updates.md" +--- + +# Task 02: Retry managed runtime launch + +Add one race-safe online retry to host-local managed runtime startup. + +- **Acceptance:** A strict npm version-resolution failure during ACP + initialization stops the first child, invalidates the exact execution key, + and retries the same trusted runtime once with current npm metadata. +- **Acceptance:** The retry preserves the active exact version, command prefix, + model, permissions, session identity, and ACP arguments. +- **Acceptance:** A startup attempt generation prevents delayed completion and + stream events from the first child from publishing a terminal state for the + retry. +- **Acceptance:** Cancellation and shutdown win. Native, remote, passthrough, + unrelated, and repeated online failures do not start another recovery. +- **Acceptance:** Success continues the original launch without a failure + event. Repeated failure publishes one stable failure code and bounded + sanitized details. +- **Verification:** Add failing lifecycle tests first, then run: + + ```bash + cd apps/backend + go test ./internal/agent/runtime/lifecycle -run 'Test.*ManagedRuntime.*Npm|Test.*StartupRecovery' + go test ./internal/agent/runtime/lifecycle + ``` + +- **Files likely touched:** + `apps/backend/internal/agent/runtime/lifecycle/types.go`, + `apps/backend/internal/agent/runtime/lifecycle/manager_startup.go`, + `apps/backend/internal/agent/runtime/lifecycle/manager_events.go`, + `apps/backend/internal/agent/runtime/lifecycle/event_types.go`, + `apps/backend/internal/agent/runtime/lifecycle/events.go`, and focused new or + existing lifecycle test files. +- **Dependencies:** Task 01. +- **Parallelism:** sequential because this consumes the classifier, command, + and cache contracts. +- **Inputs:** The Task 01 classifier and trusted helpers, plus lifecycle process + identity and shutdown rules. +- **Output contract:** Report files changed, RED and GREEN commands and results, + attempt-generation race evidence, exact retry-count evidence, and + synchronized task and plan status. diff --git a/docs/plans/managed-runtime-stale-npm-recovery/task-03-present-npm-recovery.md b/docs/plans/managed-runtime-stale-npm-recovery/task-03-present-npm-recovery.md new file mode 100644 index 0000000000..6f15804334 --- /dev/null +++ b/docs/plans/managed-runtime-stale-npm-recovery/task-03-present-npm-recovery.md @@ -0,0 +1,59 @@ +--- +id: "03-present-npm-recovery" +title: "Present npm runtime recovery" +status: done +wave: 3 +depends_on: ["02-retry-managed-runtime-launch"] +plan: "plan.md" +spec: "../../specs/agents/runtime-updates.md" +--- + +# Task 03: Present npm runtime recovery + +Carry the structured terminal failure to Kanban and Office, then render one +clear runtime retry path. + +- **Acceptance:** Lifecycle and watcher events carry optional failure code and + sanitized details without changing generic error compatibility. +- **Acceptance:** `last_agent_error` stores those optional fields in existing + JSON metadata and reads both API snake case and store camel case. +- **Acceptance:** The orchestrator creates a specialized recovery message with + `failure_kind = managed_runtime_npm_resolution`, collapsed technical details, + and only **Retry runtime**. It uses resume mode when possible and the existing + fresh-run mode otherwise. +- **Acceptance:** Kanban and Office explain that npm could not prepare the + runtime and that Kandev refreshed package data and retried once. User copy + does not mention ACP. +- **Acceptance:** All new copy uses i18n. Desktop and mobile use the same + parsing, state, actions, and translation keys. +- **Verification:** Add failing backend and frontend tests first, then run: + + ```bash + cd apps/backend + go test ./internal/orchestrator -run 'Test.*ManagedRuntime.*Recovery|Test.*RecoveryActions|Test.*LastAgentError' + + cd ../../apps/web + pnpm exec vitest run components/task/chat/messages/action-message.test.tsx components/task/simple/chat-entries.test.ts components/task/simple/components/run-error-entry.test.tsx lib/session-last-agent-error.test.ts + pnpm run typecheck + pnpm run i18n:check + pnpm run i18n:ratchet + ``` + +- **Files likely touched:** + `apps/backend/internal/orchestrator/event_handlers_agent.go`, + `apps/backend/internal/orchestrator/watcher/watcher.go`, + `apps/backend/internal/task/models/models.go`, + `apps/web/lib/session-last-agent-error.ts`, + `apps/web/components/task/chat/types.ts`, + `apps/web/components/task/chat/messages/action-message.tsx`, + `apps/web/components/task/simple/chat-entries.ts`, + `apps/web/components/task/simple/components/run-error-entry.tsx`, relevant + tests, and the matching locale catalogs. +- **Dependencies:** Task 02. +- **Parallelism:** sequential because UI metadata depends on the final backend + event contract. +- **Inputs:** The terminal failure payload from Task 02 and the existing + provider recovery card pattern. +- **Output contract:** Report files changed, RED and GREEN commands and results, + persisted metadata shape, action payloads for resumable and non-resumable + sessions, i18n results, and synchronized task and plan status. diff --git a/docs/plans/managed-runtime-stale-npm-recovery/task-04-prove-responsive-recovery.md b/docs/plans/managed-runtime-stale-npm-recovery/task-04-prove-responsive-recovery.md new file mode 100644 index 0000000000..e2e6b0812e --- /dev/null +++ b/docs/plans/managed-runtime-stale-npm-recovery/task-04-prove-responsive-recovery.md @@ -0,0 +1,42 @@ +--- +id: "04-prove-responsive-recovery" +title: "Prove responsive npm recovery" +status: done +wave: 4 +depends_on: ["03-present-npm-recovery"] +plan: "plan.md" +spec: "../../specs/agents/runtime-updates.md" +--- + +# Task 04: Prove responsive npm recovery + +Exercise the shipped Kanban and Office recovery surfaces in real browser +layouts. + +- **Acceptance:** Desktop Kanban shows npm-specific copy, collapsed details, + one **Retry runtime** action, and the expected recovery request. +- **Acceptance:** A phone viewport keeps the inline card within the chat, has no + horizontal overflow, uses a touch target of at least 44 px, and opens no + nested overlay. +- **Acceptance:** Office restores the specialized card from persisted + `last_agent_error` metadata after reload and offers the same single action. +- **Verification:** Build once, add the browser scenarios, and run: + + ```bash + make build-web + make build-backend + cd apps/web + pnpm e2e:run --no-build tests/session/managed-runtime-npm-recovery.spec.ts tests/office/managed-runtime-npm-recovery.spec.ts + pnpm e2e:run --no-build --project mobile-chrome tests/session/mobile-managed-runtime-npm-recovery.spec.ts + ``` + +- **Files likely touched:** + `apps/web/e2e/tests/session/managed-runtime-npm-recovery.spec.ts`, + `apps/web/e2e/tests/session/mobile-managed-runtime-npm-recovery.spec.ts`, and + `apps/web/e2e/tests/office/managed-runtime-npm-recovery.spec.ts`. +- **Dependencies:** Task 03. +- **Parallelism:** Can run in parallel with Task 05 after Task 03. +- **Inputs:** The shipped structured failure metadata and shared recovery UI. +- **Output contract:** Report files changed, desktop and mobile command results, + viewport and touch-target evidence, overflow evidence, Office reload evidence, + and synchronized task and plan status. diff --git a/docs/plans/managed-runtime-stale-npm-recovery/task-05-document-runtime-recovery.md b/docs/plans/managed-runtime-stale-npm-recovery/task-05-document-runtime-recovery.md new file mode 100644 index 0000000000..67efbe5f02 --- /dev/null +++ b/docs/plans/managed-runtime-stale-npm-recovery/task-05-document-runtime-recovery.md @@ -0,0 +1,35 @@ +--- +id: "05-document-runtime-recovery" +title: "Document npm runtime recovery" +status: done +wave: 4 +depends_on: ["03-present-npm-recovery"] +plan: "plan.md" +spec: "../../specs/agents/runtime-updates.md" +--- + +# Task 05: Document npm runtime recovery + +Update public troubleshooting guidance after the behavior and labels are +stable. + +- **Acceptance:** Public docs explain the automatic same-version online retry + and its one-attempt limit. +- **Acceptance:** The troubleshooting path explains how to inspect the + configured npm registry when the focused error remains. +- **Acceptance:** The docs state that global npm cache cleanup is not the normal + recovery path and do not instruct users to run `npm cache clean --force`. +- **Acceptance:** Commands, labels, and behavior match the implementation. +- **Verification:** Run: + + ```bash + node --test scripts/validate-public-docs.test.mjs + node scripts/validate-public-docs.mjs + ``` + +- **Files likely touched:** `docs/public/agents-and-profiles.md`. +- **Dependencies:** Task 03. +- **Parallelism:** Can run in parallel with Task 04 after Task 03. +- **Inputs:** Final retry behavior and UI labels from Tasks 01 through 03. +- **Output contract:** Report files changed, validation commands and results, + verified links and commands, and synchronized task and plan status. diff --git a/docs/public/agents-and-profiles.md b/docs/public/agents-and-profiles.md index 2a622fe67c..69c77269e5 100644 --- a/docs/public/agents-and-profiles.md +++ b/docs/public/agents-and-profiles.md @@ -69,6 +69,27 @@ configuration options, commands, and runtime version without a page reload. - If preparation, ACP validation, authentication, or persistence fails, Kandev keeps the previous active version and capability catalogue. Select another stable version or retry the same target. - Kandev may prepare the exact version again if npm removes its cache entry. Kandev does not own an offline package inventory, and global npm cache cleanup is not required. +#### Recover a stale npm runtime lookup + +Host-local managed runtimes normally start with npm's offline-preferred lookup. +If npm has stale package metadata and cannot resolve the selected +`package@version`, the first ACP startup can fail even though the configured +registry contains that exact version. Kandev recognizes this specific npm +resolution error, removes only the deterministic `_npx` execution tree for the +selected package and version, then retries the same command once with an +online-preferred metadata lookup. + +The retry keeps the selected package, exact version, command prefix, model, +permissions, and session identity. It does not change the npm registry or +silently select another version. When the retry succeeds, no recovery card is +shown. When it fails again, Kanban and Office show one **Retry runtime** action +with collapsed technical details. + +Do not use `npm cache clean --force` as the normal recovery step. It removes +unrelated npm data and does not target the stale execution tree. If the +specialized retry cannot resolve the runtime, check that the Kandev service +uses the expected npm installation and configured registry. Run `npm config get registry` as the Kandev service user to inspect the registry used by that process. Then use the runtime update controls to select and prepare another trusted stable version. +
Add a custom terminal agent @@ -250,6 +271,7 @@ Only custom TUI agents can be deleted from the agent list. Built-in definitions - **Login required:** use the agent card's login terminal or sign in under Kandev's operating-system user; signing in as another user does not help the service. - **Model, mode, or command probe fails:** authenticate first, refresh discovery, and choose a value advertised by the installed version. - **Launch fails after editing flags:** inspect the command preview, remove stale arguments, and correct unmatched quotes or trailing escapes. +- **Managed npm runtime cannot resolve its selected version:** Kandev checks the configured registry and retries the same version once after refreshing its exact `_npx` execution tree. If the retry fails, verify the service user's npm configuration and registry, then use **Settings > Agents** to prepare another trusted stable version. Do not start with `npm cache clean --force`. - **Environment value is absent:** confirm the secret still exists, the key is not reserved, and an executor/runtime variable is not already taking precedence. - **MCP server is absent:** confirm agent MCP support, valid JSON, transport mode, executor policy, and the session warning logs. - **MCP tools are missing from one agent session:** open the neutral plug button in that session's chat toolbar. **Delivered** means Kandev included the server in the agent launch but cannot yet observe a connection; **Connected** means the built-in server saw MCP initialize; **Active** means it served `tools/list`. A gray filtered/unavailable row explains an intentional omission, while red is reserved for an explicit sanitized error. Third-party profile servers normally remain Delivered because they connect directly to the agent and Kandev cannot inspect them. The report is scoped to that session and its current execution, so compare the affected agent's own toolbar rather than another agent on the task. diff --git a/docs/specs/agents/runtime-updates.md b/docs/specs/agents/runtime-updates.md index bb757cdeaf..a69a697713 100644 --- a/docs/specs/agents/runtime-updates.md +++ b/docs/specs/agents/runtime-updates.md @@ -15,6 +15,12 @@ partly published, incompatible with ACP, or otherwise cannot start. Rebuilding an npm cache is not sufficient when an unversioned command selects the same broken release again. +Managed runtimes can also fail when npm has stale package metadata. In that +case, npm can report `ETARGET` and `No matching version found` for a dependency +that the configured registry now contains. Kandev currently reports the later +ACP disconnect instead of the npm cause. The user then has no useful recovery +action unless they inspect backend logs. + ## What - Settings exposes version management for the built-in managed npm runtimes @@ -46,6 +52,11 @@ broken release again. recipe. - Jobs for one agent are idempotent while queued or running. Installation and version management for the same agent cannot run concurrently. +- A host-local managed runtime launch detects a strict npm `ETARGET` version + resolution failure before ACP initialization. Kandev refreshes npm metadata + and retries the same trusted package and version once. +- If the retry also fails, Kandev reports an npm runtime preparation error and + offers one runtime retry action. The user does not need to understand ACP. The managed package set is: @@ -209,6 +220,33 @@ version while its current probe is unknown produces a repair job. - Candidate validation bypasses the active selection only for the trusted exact candidate command created by the version job. +## Launch-time stale metadata recovery + +- Normal host-local managed runtime commands continue to use + `--prefer-offline`. +- Kandev inspects bounded process stderr when ACP initialization ends before a + response. Automatic recovery requires both npm `ETARGET` and a matching + `No matching version found for package@version` message. +- Kandev classifies the npm error from stderr. It does not build a command from + package names, versions, paths, or registry values found in stderr. +- Recovery removes only the deterministic `_npx` execution tree for the + trusted built-in package specification. It never clears npm's full cache. +- Kandev starts the same managed runtime once with `--prefer-online`. The + command keeps the same trusted package, exact active version when one exists, + ACP arguments, configured npm registry, command prefix, permissions, model, + and session identity. +- Recovery is limited to one retry for each launch attempt. A delayed event + from the first child process cannot fail or complete the replacement process. +- User cancellation and backend shutdown stop recovery. Kandev does not retry + a remote executor, a native runtime, a passthrough command, an unrelated npm + error, or a second failed online attempt. +- A successful retry continues the original session without a failure card. + Kandev records structured recovery telemetry without exposing host paths or + raw process logs. +- A failed retry emits a stable npm runtime failure code and bounded sanitized + details. This evidence is stored with the last agent error so the focused UI + survives a page reload. + ## Failure and recovery behavior - Registry failure during preview keeps approval disabled and runs no command. @@ -224,6 +262,8 @@ version while its current probe is unknown produces a repair job. - Browser disconnect does not cancel a running job. The jobs endpoint can recover process-local progress while the backend remains running. - Active sessions are never restarted, replaced, or hot-swapped. +- A launch-time stale metadata retry is not a version rollback. It prepares the + same package selection again and does not change the active version. ## Persistence guarantees @@ -241,6 +281,9 @@ version while its current probe is unknown produces a repair job. version again; it must not advance to another version. - Dialog selection, output, and result remain page-local after a browser page restart. +- A terminal launch-time npm resolution error stores its stable failure code + and bounded sanitized details in `last_agent_error`. No database migration is + required because the record is JSON metadata with optional fields. ## Desktop and mobile behavior @@ -256,6 +299,16 @@ version while its current probe is unknown produces a repair job. output remain viewport-contained. - Selection state, preview loading, operation labels, request payloads, and terminal results are shared across desktop and mobile presentations. +- A failed automatic launch retry uses the existing inline recovery card in + Kanban chat and Office chat. It does not open a dialog or drawer. +- The card states that npm could not prepare the agent runtime. It states that + Kandev refreshed package data and retried once. Technical details are + collapsed initially. +- The card offers one **Retry runtime** action. When a resume token exists, the + action resumes the session. Otherwise, it starts a replacement run. The card + does not present session history loss as a fix for an npm problem. +- Phone actions stack when needed, remain at least 44 px high, and do not add a + second scroll container. ## Scenarios @@ -284,10 +337,27 @@ version while its current probe is unknown produces a repair job. - **GIVEN** a phone viewport and a long version catalogue or process log, **WHEN** the operator selects and activates a version, **THEN** the drawer remains contained and the primary action remains touch-reachable. +- **GIVEN** a host-local managed runtime exits before ACP initialization with + npm `ETARGET` and a matching missing dependency version, **WHEN** Kandev + reads the captured stderr, **THEN** it removes only the trusted package's + deterministic `_npx` tree and retries the same runtime once with current npm + metadata. +- **GIVEN** that online retry starts successfully, **WHEN** ACP initialization + completes, **THEN** the original session continues and no recovery card is + shown. +- **GIVEN** that online retry fails with the same npm resolution error, + **WHEN** the failure reaches Kanban or Office chat, **THEN** the UI explains + the npm cause, keeps sanitized technical details collapsed, and offers only + **Retry runtime**. +- **GIVEN** a native, SSH, container, passthrough, or unrelated failed launch, + **WHEN** Kandev classifies the error, **THEN** this automatic npm recovery + does not run. ## Out of scope - Scheduled or automatic updates and automatic rollback after launch failure. +- Global npm cache cleanup, registry replacement, dependency substitution, or + automatic selection of another package version. - Prerelease, tag, arbitrary package-spec, registry, or shell-command input. - Kandev-owned npm artifact retention or a package lockfile. - Applying host selections to SSH, container, or other remote runtimes.