diff --git a/apps/backend/internal/agent/lifecycle/executor_sprites.go b/apps/backend/internal/agent/lifecycle/executor_sprites.go index ba9a0b8f16..8ae634bce8 100644 --- a/apps/backend/internal/agent/lifecycle/executor_sprites.go +++ b/apps/backend/internal/agent/lifecycle/executor_sprites.go @@ -2,6 +2,7 @@ package lifecycle import ( "context" + "errors" "fmt" "os" "strings" @@ -17,6 +18,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/server/process" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/secrets" + spritesutil "github.com/kandev/kandev/internal/sprites" ) type RemoteAuthAgentLister interface { @@ -137,11 +139,21 @@ func (r *SpritesExecutor) CreateInstance(ctx context.Context, req *ExecutorCreat zap.String("sprite_name", spriteName), zap.Bool("reconnect_required", reconnect)) - // Step 0: Create or reconnect sprite + // Step 0: Create or reconnect sprite. On reconnect-then-not-found we fall + // through to fresh provisioning under a new name on the same branch. sprite, err := r.stepCreateSprite(ctx, client, spriteName, reconnect, report) if err != nil { - r.cleanupOnFailure(ctx, sprite, req.InstanceID, destroyOnFailure) - return nil, err + if reconnect && errors.Is(err, spritesutil.ErrSpriteNotFound) { + oldName := spriteName + spriteName = r.fallbackToFreshSandbox(req, progressPlan, report, oldName) + reconnect = false + destroyOnFailure = true + sprite, err = r.stepCreateSprite(ctx, client, spriteName, false, report) + } + if err != nil { + r.cleanupOnFailure(ctx, sprite, req.InstanceID, destroyOnFailure) + return nil, err + } } // Steps 1-3: Upload agentctl, credentials, prepare script @@ -199,6 +211,62 @@ func (r *SpritesExecutor) resolveSpriteName(req *ExecutorCreateRequest, reconnec return spritesNamePrefix + suffix } +// fallbackToFreshSandbox flips a stalled reconnect into a fresh provision. It +// emits a skipped "Reconnecting cloud sandbox" step carrying the user-facing +// warning that explains why we're reprovisioning, then mutates the live plan +// in place so the rest of CreateInstance streams the normal fresh-provision +// progress rows. Returns the new sprite name. The user-visible explanation is +// the only place we surface this transition. +func (r *SpritesExecutor) fallbackToFreshSandbox( + req *ExecutorCreateRequest, + plan *spritesProgressPlan, + report func(spritesStepKey, PrepareStep), + oldName string, +) string { + newName := spritesNamePrefix + req.InstanceID[:12] + branch := getMetadataString(req.Metadata, MetadataKeyWorktreeBranch) + if branch == "" { + branch = getMetadataString(req.Metadata, MetadataKeyBaseBranch) + } + + r.logger.Info("sprite missing on reconnect, provisioning fresh", + zap.String("instance_id", req.InstanceID), + zap.String("old_sprite_name", oldName), + zap.String("new_sprite_name", newName), + zap.String("branch", branch)) + + plan.replacePlan([]spritesStepKey{ + spriteStepCreateSprite, + spriteStepUploadAgentctl, + spriteStepUploadCredentials, + spriteStepRunPrepareScript, + spriteStepWaitHealthy, + spriteStepAgentInstance, + spriteStepApplyNetworkPolicy, + }) + + notice := beginStep("Reconnecting cloud sandbox") + notice.Warning = "Previous sandbox is no longer available — provisioning a fresh one for this branch." + notice.WarningDetail = fmt.Sprintf( + "The Sprites sandbox %s could not be reached (it was likely destroyed or expired). "+ + "Kandev is starting a fresh sandbox %s on the same branch %s; this typically takes 30–60 seconds.", + oldName, newName, branchOrPlaceholder(branch), + ) + notice.Output = fmt.Sprintf("Old sandbox: %s\nNew sandbox: %s\nBranch: %s", + oldName, newName, branchOrPlaceholder(branch)) + completeStepSkipped(¬ice) + report(spriteStepCreateSprite, notice) + + return newName +} + +func branchOrPlaceholder(branch string) string { + if branch == "" { + return "(unknown)" + } + return branch +} + func spritesShouldReconnect(req *ExecutorCreateRequest) bool { if req == nil { return false diff --git a/apps/backend/internal/agent/lifecycle/executor_sprites_fallback_test.go b/apps/backend/internal/agent/lifecycle/executor_sprites_fallback_test.go new file mode 100644 index 0000000000..28f95ac8f5 --- /dev/null +++ b/apps/backend/internal/agent/lifecycle/executor_sprites_fallback_test.go @@ -0,0 +1,139 @@ +package lifecycle + +import ( + "strings" + "testing" +) + +func TestSpritesProgressPlanGrowsOnFallback(t *testing.T) { + plan := newSpritesProgressPlan(true) + if plan.total() != 3 { + t.Fatalf("reconnect plan total = %d, want 3", plan.total()) + } + + plan.replacePlan([]spritesStepKey{ + spriteStepCreateSprite, + spriteStepUploadAgentctl, + spriteStepUploadCredentials, + spriteStepRunPrepareScript, + spriteStepWaitHealthy, + spriteStepAgentInstance, + spriteStepApplyNetworkPolicy, + }) + + if plan.total() != 7 { + t.Fatalf("after fallback, plan total = %d, want 7", plan.total()) + } + for _, key := range []spritesStepKey{ + spriteStepUploadAgentctl, + spriteStepUploadCredentials, + spriteStepRunPrepareScript, + spriteStepApplyNetworkPolicy, + } { + if !plan.has(key) { + t.Fatalf("after fallback, plan should include %s", key) + } + } +} + +func TestSpritesStepReporterFollowsPlanReplacement(t *testing.T) { + plan := newSpritesProgressPlan(true) + type reportEvent struct { + index int + total int + } + var events []reportEvent + cb := func(_ PrepareStep, idx, total int) { + events = append(events, reportEvent{index: idx, total: total}) + } + report := newSpritesStepReporter(cb, plan) + + report(spriteStepWaitHealthy, beginStep("Waiting for agent controller")) + if len(events) != 1 || events[0].index != 1 || events[0].total != 3 { + t.Fatalf("pre-replace event = %+v, want index 1 of 3", events) + } + + plan.replacePlan([]spritesStepKey{ + spriteStepCreateSprite, + spriteStepUploadAgentctl, + spriteStepUploadCredentials, + spriteStepRunPrepareScript, + spriteStepWaitHealthy, + spriteStepAgentInstance, + spriteStepApplyNetworkPolicy, + }) + report(spriteStepWaitHealthy, beginStep("Waiting for agent controller")) + if len(events) != 2 || events[1].index != 4 || events[1].total != 7 { + t.Fatalf("post-replace event = %+v, want index 4 of 7", events) + } +} + +func TestSpritesFallbackToFreshSandboxEmitsWarningAndMutatesPlan(t *testing.T) { + r := newTestSpritesExecutor(nil) + plan := newSpritesProgressPlan(true) + + var emitted []PrepareStep + report := func(_ spritesStepKey, step PrepareStep) { + emitted = append(emitted, step) + } + + req := &ExecutorCreateRequest{ + InstanceID: "abcdef0123456789aaaa", + Metadata: map[string]interface{}{ + MetadataKeyWorktreeBranch: "feature/something", + "sprite_name": "kandev-old1234567890", + }, + } + + newName := r.fallbackToFreshSandbox(req, plan, report, "kandev-old1234567890") + + wantName := spritesNamePrefix + req.InstanceID[:12] + if newName != wantName { + t.Fatalf("new sprite name = %q, want %q", newName, wantName) + } + if plan.total() != 7 { + t.Fatalf("plan total after fallback = %d, want 7", plan.total()) + } + if len(emitted) != 1 { + t.Fatalf("expected exactly 1 progress event from fallback, got %d", len(emitted)) + } + + notice := emitted[0] + if notice.Status != PrepareStepSkipped { + t.Fatalf("notice status = %s, want skipped", notice.Status) + } + if !strings.Contains(notice.Warning, "Previous sandbox is no longer available") { + t.Fatalf("notice warning missing user-facing copy: %q", notice.Warning) + } + if !strings.Contains(notice.WarningDetail, "kandev-old1234567890") || + !strings.Contains(notice.WarningDetail, wantName) || + !strings.Contains(notice.WarningDetail, "feature/something") { + t.Fatalf("notice detail missing context: %q", notice.WarningDetail) + } + if !strings.Contains(notice.Output, "Old sandbox: kandev-old1234567890") || + !strings.Contains(notice.Output, "New sandbox: "+wantName) || + !strings.Contains(notice.Output, "Branch: feature/something") { + t.Fatalf("notice output missing structured block: %q", notice.Output) + } +} + +func TestSpritesFallbackUsesPlaceholderBranchWhenMissing(t *testing.T) { + r := newTestSpritesExecutor(nil) + plan := newSpritesProgressPlan(true) + var emitted []PrepareStep + + req := &ExecutorCreateRequest{ + InstanceID: "abcdef0123456789aaaa", + Metadata: map[string]interface{}{}, + } + r.fallbackToFreshSandbox(req, plan, func(_ spritesStepKey, step PrepareStep) { + emitted = append(emitted, step) + }, "kandev-old1234567890") + + if len(emitted) != 1 { + t.Fatalf("expected 1 event, got %d", len(emitted)) + } + if !strings.Contains(emitted[0].Output, "Branch: (unknown)") { + t.Fatalf("expected placeholder branch in output, got %q", emitted[0].Output) + } +} diff --git a/apps/backend/internal/agent/lifecycle/executor_sprites_helpers.go b/apps/backend/internal/agent/lifecycle/executor_sprites_helpers.go index 24ab4ecc12..6f21b186e5 100644 --- a/apps/backend/internal/agent/lifecycle/executor_sprites_helpers.go +++ b/apps/backend/internal/agent/lifecycle/executor_sprites_helpers.go @@ -131,42 +131,61 @@ const ( spriteStepApplyNetworkPolicy spritesStepKey = "apply_network_policy" ) +// spritesProgressPlan is mutable so the reconnect path can swap to the fresh +// plan mid-flight when a stale sandbox falls back to provisioning a new one. +// The reporter created by newSpritesStepReporter resolves indexes lazily off +// the live plan, so callers don't need to re-create it after replacePlan. type spritesProgressPlan struct { steps []spritesStepKey indexes map[spritesStepKey]int } -func newSpritesProgressPlan(reconnect bool) spritesProgressPlan { - steps := []spritesStepKey{spriteStepCreateSprite} - if !reconnect { - steps = append(steps, - spriteStepUploadAgentctl, - spriteStepUploadCredentials, - spriteStepRunPrepareScript, - ) - } - steps = append(steps, spriteStepWaitHealthy, spriteStepAgentInstance) - if !reconnect { - steps = append(steps, spriteStepApplyNetworkPolicy) +func newSpritesProgressPlan(reconnect bool) *spritesProgressPlan { + if reconnect { + return buildPlan([]spritesStepKey{ + spriteStepCreateSprite, + spriteStepWaitHealthy, + spriteStepAgentInstance, + }) } + return buildPlan([]spritesStepKey{ + spriteStepCreateSprite, + spriteStepUploadAgentctl, + spriteStepUploadCredentials, + spriteStepRunPrepareScript, + spriteStepWaitHealthy, + spriteStepAgentInstance, + spriteStepApplyNetworkPolicy, + }) +} +func buildPlan(steps []spritesStepKey) *spritesProgressPlan { indexes := make(map[spritesStepKey]int, len(steps)) for i, key := range steps { indexes[key] = i } - return spritesProgressPlan{steps: steps, indexes: indexes} + return &spritesProgressPlan{steps: steps, indexes: indexes} } -func (p spritesProgressPlan) total() int { +func (p *spritesProgressPlan) total() int { + if p == nil { + return 0 + } return len(p.steps) } -func (p spritesProgressPlan) has(key spritesStepKey) bool { +func (p *spritesProgressPlan) has(key spritesStepKey) bool { + if p == nil { + return false + } _, ok := p.indexes[key] return ok } -func (p spritesProgressPlan) index(key spritesStepKey) int { +func (p *spritesProgressPlan) index(key spritesStepKey) int { + if p == nil { + return -1 + } idx, ok := p.indexes[key] if !ok { return -1 @@ -174,8 +193,25 @@ func (p spritesProgressPlan) index(key spritesStepKey) int { return idx } -// newSpritesStepReporter creates a reporting function that calls OnProgress if non-nil. -func newSpritesStepReporter(onProgress PrepareProgressCallback, plan spritesProgressPlan) func(spritesStepKey, PrepareStep) { +// replacePlan swaps the active plan in place (used by the reconnect-fell-back- +// to-fresh path). The reporter dispatches off the live indexes, so existing +// closures keep working after replacement. +func (p *spritesProgressPlan) replacePlan(steps []spritesStepKey) { + if p == nil { + return + } + indexes := make(map[spritesStepKey]int, len(steps)) + for i, key := range steps { + indexes[key] = i + } + p.steps = steps + p.indexes = indexes +} + +// newSpritesStepReporter creates a reporting function that calls OnProgress if +// non-nil. Index/total are resolved against the live plan on each call so +// replacePlan is reflected in subsequent reports. +func newSpritesStepReporter(onProgress PrepareProgressCallback, plan *spritesProgressPlan) func(spritesStepKey, PrepareStep) { return func(key spritesStepKey, step PrepareStep) { if onProgress == nil { return diff --git a/apps/backend/internal/agent/lifecycle/executor_sprites_lifecycle.go b/apps/backend/internal/agent/lifecycle/executor_sprites_lifecycle.go index d7807f9b45..e3ad56ba5e 100644 --- a/apps/backend/internal/agent/lifecycle/executor_sprites_lifecycle.go +++ b/apps/backend/internal/agent/lifecycle/executor_sprites_lifecycle.go @@ -18,7 +18,7 @@ func (r *SpritesExecutor) reconnectSprite(ctx context.Context, client *sprites.C defer cancel() sprite, err := client.GetSprite(stepCtx, name) if err != nil { - return nil, fmt.Errorf("failed to reconnect sprite %q: %w", name, err) + return nil, fmt.Errorf("failed to reconnect sprite %q: %w", name, spritesutil.WrapNotFound(err)) } return sprite, nil } diff --git a/apps/backend/internal/sprites/errors.go b/apps/backend/internal/sprites/errors.go new file mode 100644 index 0000000000..5788eb6f39 --- /dev/null +++ b/apps/backend/internal/sprites/errors.go @@ -0,0 +1,48 @@ +package sprites + +import ( + "errors" + "fmt" + "strings" +) + +// ErrSpriteNotFound is returned when an upstream Sprites API call indicates the +// sandbox does not exist. The upstream SDK currently returns plain errors with +// a "sprite not found: " message for 404 responses; this sentinel lets +// callers branch on that case without ad-hoc string matching. +var ErrSpriteNotFound = errors.New("sprite not found") + +// WrapNotFound wraps err with ErrSpriteNotFound when err looks like an upstream +// "sprite not found" / 404 response. Other errors are returned unchanged. Pass +// the returned error to errors.Is(err, ErrSpriteNotFound) to detect the case. +func WrapNotFound(err error) error { + if err == nil { + return nil + } + if errors.Is(err, ErrSpriteNotFound) { + return err + } + if !looksLikeNotFound(err.Error()) { + return err + } + return fmt.Errorf("%w: %w", ErrSpriteNotFound, err) +} + +// IsNotFound reports whether err originated from an upstream "not found" +// response, even when WrapNotFound has not been applied. Useful for callers +// that receive a raw upstream error. +func IsNotFound(err error) bool { + if err == nil { + return false + } + if errors.Is(err, ErrSpriteNotFound) { + return true + } + return looksLikeNotFound(err.Error()) +} + +func looksLikeNotFound(msg string) bool { + lower := strings.ToLower(msg) + return strings.Contains(lower, "sprite not found") || + strings.Contains(lower, "not found") +} diff --git a/apps/backend/internal/sprites/errors_test.go b/apps/backend/internal/sprites/errors_test.go new file mode 100644 index 0000000000..01eb60e702 --- /dev/null +++ b/apps/backend/internal/sprites/errors_test.go @@ -0,0 +1,62 @@ +package sprites + +import ( + "errors" + "fmt" + "testing" +) + +func TestWrapNotFoundWraps404Errors(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {name: "sprite not found exact", err: fmt.Errorf("sprite not found: kandev-abc"), want: true}, + {name: "uppercase variant", err: fmt.Errorf("Sprite Not Found: kandev-abc"), want: true}, + {name: "wrapped not found", err: fmt.Errorf("get sprite: %w", fmt.Errorf("sprite not found: kandev-abc")), want: true}, + {name: "generic 404 mention", err: fmt.Errorf("HTTP 404: not found"), want: true}, + {name: "auth failure", err: fmt.Errorf("401 unauthorized"), want: false}, + {name: "transient 500", err: fmt.Errorf("internal server error"), want: false}, + {name: "nil", err: nil, want: false}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + wrapped := WrapNotFound(tc.err) + got := errors.Is(wrapped, ErrSpriteNotFound) + if got != tc.want { + t.Fatalf("errors.Is(WrapNotFound(%v), ErrSpriteNotFound) = %v, want %v", tc.err, got, tc.want) + } + if got != IsNotFound(tc.err) { + t.Fatalf("IsNotFound(%v) disagrees with errors.Is on wrapped form", tc.err) + } + }) + } +} + +func TestWrapNotFoundPreservesOriginalError(t *testing.T) { + orig := fmt.Errorf("sprite not found: kandev-abc") + wrapped := WrapNotFound(orig) + + if !errors.Is(wrapped, orig) { + t.Fatalf("WrapNotFound dropped the original error from the chain") + } + if !errors.Is(wrapped, ErrSpriteNotFound) { + t.Fatalf("WrapNotFound dropped the sentinel from the chain") + } +} + +func TestWrapNotFoundIsIdempotent(t *testing.T) { + once := WrapNotFound(fmt.Errorf("sprite not found: kandev-abc")) + twice := WrapNotFound(once) + if once.Error() != twice.Error() { + t.Fatalf("WrapNotFound is not idempotent: once=%q, twice=%q", once, twice) + } +} + +func TestWrapNotFoundNilPassthrough(t *testing.T) { + if WrapNotFound(nil) != nil { + t.Fatalf("WrapNotFound(nil) should return nil") + } +}