feat(sprites): provision a fresh sandbox when the persisted one is gone

Resuming a Sprites task whose sandbox was destroyed upstream returned
"sprite not found" and dead-ended the user. Detect the upstream 404 via
a typed ErrSpriteNotFound sentinel, mark the reconnect step skipped with
a user-facing warning explaining what happened, swap the prepare plan
in place to the fresh-provision shape, and continue the launch under a
new sandbox name on the same branch.

The progress plan is now mutable so the reporter can stream the fresh
steps live after the reconnect-then-fresh transition. Only the explicit
not-found case triggers fallback; other reconnect failures still
surface as before, and the fallback fires at most once per launch.
This commit is contained in:
José Almeida
2026-05-05 21:19:39 +01:00
parent 3790fc148f
commit 85885dd866
6 changed files with 375 additions and 22 deletions
@@ -2,6 +2,7 @@ package lifecycle
import (
"context"
"errors"
"fmt"
"os"
"strings"
@@ -17,6 +18,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/server/process"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/secrets"
spritesutil "github.com/kandev/kandev/internal/sprites"
)
type RemoteAuthAgentLister interface {
@@ -137,11 +139,21 @@ func (r *SpritesExecutor) CreateInstance(ctx context.Context, req *ExecutorCreat
zap.String("sprite_name", spriteName),
zap.Bool("reconnect_required", reconnect))
// Step 0: Create or reconnect sprite
// Step 0: Create or reconnect sprite. On reconnect-then-not-found we fall
// through to fresh provisioning under a new name on the same branch.
sprite, err := r.stepCreateSprite(ctx, client, spriteName, reconnect, report)
if err != nil {
r.cleanupOnFailure(ctx, sprite, req.InstanceID, destroyOnFailure)
return nil, err
if reconnect && errors.Is(err, spritesutil.ErrSpriteNotFound) {
oldName := spriteName
spriteName = r.fallbackToFreshSandbox(req, progressPlan, report, oldName)
reconnect = false
destroyOnFailure = true
sprite, err = r.stepCreateSprite(ctx, client, spriteName, false, report)
}
if err != nil {
r.cleanupOnFailure(ctx, sprite, req.InstanceID, destroyOnFailure)
return nil, err
}
}
// Steps 1-3: Upload agentctl, credentials, prepare script
@@ -199,6 +211,62 @@ func (r *SpritesExecutor) resolveSpriteName(req *ExecutorCreateRequest, reconnec
return spritesNamePrefix + suffix
}
// fallbackToFreshSandbox flips a stalled reconnect into a fresh provision. It
// emits a skipped "Reconnecting cloud sandbox" step carrying the user-facing
// warning that explains why we're reprovisioning, then mutates the live plan
// in place so the rest of CreateInstance streams the normal fresh-provision
// progress rows. Returns the new sprite name. The user-visible explanation is
// the only place we surface this transition.
func (r *SpritesExecutor) fallbackToFreshSandbox(
req *ExecutorCreateRequest,
plan *spritesProgressPlan,
report func(spritesStepKey, PrepareStep),
oldName string,
) string {
newName := spritesNamePrefix + req.InstanceID[:12]
branch := getMetadataString(req.Metadata, MetadataKeyWorktreeBranch)
if branch == "" {
branch = getMetadataString(req.Metadata, MetadataKeyBaseBranch)
}
r.logger.Info("sprite missing on reconnect, provisioning fresh",
zap.String("instance_id", req.InstanceID),
zap.String("old_sprite_name", oldName),
zap.String("new_sprite_name", newName),
zap.String("branch", branch))
plan.replacePlan([]spritesStepKey{
spriteStepCreateSprite,
spriteStepUploadAgentctl,
spriteStepUploadCredentials,
spriteStepRunPrepareScript,
spriteStepWaitHealthy,
spriteStepAgentInstance,
spriteStepApplyNetworkPolicy,
})
notice := beginStep("Reconnecting cloud sandbox")
notice.Warning = "Previous sandbox is no longer available — provisioning a fresh one for this branch."
notice.WarningDetail = fmt.Sprintf(
"The Sprites sandbox %s could not be reached (it was likely destroyed or expired). "+
"Kandev is starting a fresh sandbox %s on the same branch %s; this typically takes 30–60 seconds.",
oldName, newName, branchOrPlaceholder(branch),
)
notice.Output = fmt.Sprintf("Old sandbox: %s\nNew sandbox: %s\nBranch: %s",
oldName, newName, branchOrPlaceholder(branch))
completeStepSkipped(&notice)
report(spriteStepCreateSprite, notice)
return newName
}
func branchOrPlaceholder(branch string) string {
if branch == "" {
return "(unknown)"
}
return branch
}
func spritesShouldReconnect(req *ExecutorCreateRequest) bool {
if req == nil {
return false
@@ -0,0 +1,139 @@
package lifecycle
import (
"strings"
"testing"
)
func TestSpritesProgressPlanGrowsOnFallback(t *testing.T) {
plan := newSpritesProgressPlan(true)
if plan.total() != 3 {
t.Fatalf("reconnect plan total = %d, want 3", plan.total())
}
plan.replacePlan([]spritesStepKey{
spriteStepCreateSprite,
spriteStepUploadAgentctl,
spriteStepUploadCredentials,
spriteStepRunPrepareScript,
spriteStepWaitHealthy,
spriteStepAgentInstance,
spriteStepApplyNetworkPolicy,
})
if plan.total() != 7 {
t.Fatalf("after fallback, plan total = %d, want 7", plan.total())
}
for _, key := range []spritesStepKey{
spriteStepUploadAgentctl,
spriteStepUploadCredentials,
spriteStepRunPrepareScript,
spriteStepApplyNetworkPolicy,
} {
if !plan.has(key) {
t.Fatalf("after fallback, plan should include %s", key)
}
}
}
func TestSpritesStepReporterFollowsPlanReplacement(t *testing.T) {
plan := newSpritesProgressPlan(true)
type reportEvent struct {
index int
total int
}
var events []reportEvent
cb := func(_ PrepareStep, idx, total int) {
events = append(events, reportEvent{index: idx, total: total})
}
report := newSpritesStepReporter(cb, plan)
report(spriteStepWaitHealthy, beginStep("Waiting for agent controller"))
if len(events) != 1 || events[0].index != 1 || events[0].total != 3 {
t.Fatalf("pre-replace event = %+v, want index 1 of 3", events)
}
plan.replacePlan([]spritesStepKey{
spriteStepCreateSprite,
spriteStepUploadAgentctl,
spriteStepUploadCredentials,
spriteStepRunPrepareScript,
spriteStepWaitHealthy,
spriteStepAgentInstance,
spriteStepApplyNetworkPolicy,
})
report(spriteStepWaitHealthy, beginStep("Waiting for agent controller"))
if len(events) != 2 || events[1].index != 4 || events[1].total != 7 {
t.Fatalf("post-replace event = %+v, want index 4 of 7", events)
}
}
func TestSpritesFallbackToFreshSandboxEmitsWarningAndMutatesPlan(t *testing.T) {
r := newTestSpritesExecutor(nil)
plan := newSpritesProgressPlan(true)
var emitted []PrepareStep
report := func(_ spritesStepKey, step PrepareStep) {
emitted = append(emitted, step)
}
req := &ExecutorCreateRequest{
InstanceID: "abcdef0123456789aaaa",
Metadata: map[string]interface{}{
MetadataKeyWorktreeBranch: "feature/something",
"sprite_name": "kandev-old1234567890",
},
}
newName := r.fallbackToFreshSandbox(req, plan, report, "kandev-old1234567890")
wantName := spritesNamePrefix + req.InstanceID[:12]
if newName != wantName {
t.Fatalf("new sprite name = %q, want %q", newName, wantName)
}
if plan.total() != 7 {
t.Fatalf("plan total after fallback = %d, want 7", plan.total())
}
if len(emitted) != 1 {
t.Fatalf("expected exactly 1 progress event from fallback, got %d", len(emitted))
}
notice := emitted[0]
if notice.Status != PrepareStepSkipped {
t.Fatalf("notice status = %s, want skipped", notice.Status)
}
if !strings.Contains(notice.Warning, "Previous sandbox is no longer available") {
t.Fatalf("notice warning missing user-facing copy: %q", notice.Warning)
}
if !strings.Contains(notice.WarningDetail, "kandev-old1234567890") ||
!strings.Contains(notice.WarningDetail, wantName) ||
!strings.Contains(notice.WarningDetail, "feature/something") {
t.Fatalf("notice detail missing context: %q", notice.WarningDetail)
}
if !strings.Contains(notice.Output, "Old sandbox: kandev-old1234567890") ||
!strings.Contains(notice.Output, "New sandbox: "+wantName) ||
!strings.Contains(notice.Output, "Branch: feature/something") {
t.Fatalf("notice output missing structured block: %q", notice.Output)
}
}
func TestSpritesFallbackUsesPlaceholderBranchWhenMissing(t *testing.T) {
r := newTestSpritesExecutor(nil)
plan := newSpritesProgressPlan(true)
var emitted []PrepareStep
req := &ExecutorCreateRequest{
InstanceID: "abcdef0123456789aaaa",
Metadata: map[string]interface{}{},
}
r.fallbackToFreshSandbox(req, plan, func(_ spritesStepKey, step PrepareStep) {
emitted = append(emitted, step)
}, "kandev-old1234567890")
if len(emitted) != 1 {
t.Fatalf("expected 1 event, got %d", len(emitted))
}
if !strings.Contains(emitted[0].Output, "Branch: (unknown)") {
t.Fatalf("expected placeholder branch in output, got %q", emitted[0].Output)
}
}
@@ -131,42 +131,61 @@ const (
spriteStepApplyNetworkPolicy spritesStepKey = "apply_network_policy"
)
// spritesProgressPlan is mutable so the reconnect path can swap to the fresh
// plan mid-flight when a stale sandbox falls back to provisioning a new one.
// The reporter created by newSpritesStepReporter resolves indexes lazily off
// the live plan, so callers don't need to re-create it after replacePlan.
type spritesProgressPlan struct {
steps []spritesStepKey
indexes map[spritesStepKey]int
}
func newSpritesProgressPlan(reconnect bool) spritesProgressPlan {
steps := []spritesStepKey{spriteStepCreateSprite}
if !reconnect {
steps = append(steps,
spriteStepUploadAgentctl,
spriteStepUploadCredentials,
spriteStepRunPrepareScript,
)
}
steps = append(steps, spriteStepWaitHealthy, spriteStepAgentInstance)
if !reconnect {
steps = append(steps, spriteStepApplyNetworkPolicy)
func newSpritesProgressPlan(reconnect bool) *spritesProgressPlan {
if reconnect {
return buildPlan([]spritesStepKey{
spriteStepCreateSprite,
spriteStepWaitHealthy,
spriteStepAgentInstance,
})
}
return buildPlan([]spritesStepKey{
spriteStepCreateSprite,
spriteStepUploadAgentctl,
spriteStepUploadCredentials,
spriteStepRunPrepareScript,
spriteStepWaitHealthy,
spriteStepAgentInstance,
spriteStepApplyNetworkPolicy,
})
}
func buildPlan(steps []spritesStepKey) *spritesProgressPlan {
indexes := make(map[spritesStepKey]int, len(steps))
for i, key := range steps {
indexes[key] = i
}
return spritesProgressPlan{steps: steps, indexes: indexes}
return &spritesProgressPlan{steps: steps, indexes: indexes}
}
func (p spritesProgressPlan) total() int {
func (p *spritesProgressPlan) total() int {
if p == nil {
return 0
}
return len(p.steps)
}
func (p spritesProgressPlan) has(key spritesStepKey) bool {
func (p *spritesProgressPlan) has(key spritesStepKey) bool {
if p == nil {
return false
}
_, ok := p.indexes[key]
return ok
}
func (p spritesProgressPlan) index(key spritesStepKey) int {
func (p *spritesProgressPlan) index(key spritesStepKey) int {
if p == nil {
return -1
}
idx, ok := p.indexes[key]
if !ok {
return -1
@@ -174,8 +193,25 @@ func (p spritesProgressPlan) index(key spritesStepKey) int {
return idx
}
// newSpritesStepReporter creates a reporting function that calls OnProgress if non-nil.
func newSpritesStepReporter(onProgress PrepareProgressCallback, plan spritesProgressPlan) func(spritesStepKey, PrepareStep) {
// replacePlan swaps the active plan in place (used by the reconnect-fell-back-
// to-fresh path). The reporter dispatches off the live indexes, so existing
// closures keep working after replacement.
func (p *spritesProgressPlan) replacePlan(steps []spritesStepKey) {
if p == nil {
return
}
indexes := make(map[spritesStepKey]int, len(steps))
for i, key := range steps {
indexes[key] = i
}
p.steps = steps
p.indexes = indexes
}
// newSpritesStepReporter creates a reporting function that calls OnProgress if
// non-nil. Index/total are resolved against the live plan on each call so
// replacePlan is reflected in subsequent reports.
func newSpritesStepReporter(onProgress PrepareProgressCallback, plan *spritesProgressPlan) func(spritesStepKey, PrepareStep) {
return func(key spritesStepKey, step PrepareStep) {
if onProgress == nil {
return
@@ -18,7 +18,7 @@ func (r *SpritesExecutor) reconnectSprite(ctx context.Context, client *sprites.C
defer cancel()
sprite, err := client.GetSprite(stepCtx, name)
if err != nil {
return nil, fmt.Errorf("failed to reconnect sprite %q: %w", name, err)
return nil, fmt.Errorf("failed to reconnect sprite %q: %w", name, spritesutil.WrapNotFound(err))
}
return sprite, nil
}
+48
View File
@@ -0,0 +1,48 @@
package sprites
import (
"errors"
"fmt"
"strings"
)
// ErrSpriteNotFound is returned when an upstream Sprites API call indicates the
// sandbox does not exist. The upstream SDK currently returns plain errors with
// a "sprite not found: <name>" message for 404 responses; this sentinel lets
// callers branch on that case without ad-hoc string matching.
var ErrSpriteNotFound = errors.New("sprite not found")
// WrapNotFound wraps err with ErrSpriteNotFound when err looks like an upstream
// "sprite not found" / 404 response. Other errors are returned unchanged. Pass
// the returned error to errors.Is(err, ErrSpriteNotFound) to detect the case.
func WrapNotFound(err error) error {
if err == nil {
return nil
}
if errors.Is(err, ErrSpriteNotFound) {
return err
}
if !looksLikeNotFound(err.Error()) {
return err
}
return fmt.Errorf("%w: %w", ErrSpriteNotFound, err)
}
// IsNotFound reports whether err originated from an upstream "not found"
// response, even when WrapNotFound has not been applied. Useful for callers
// that receive a raw upstream error.
func IsNotFound(err error) bool {
if err == nil {
return false
}
if errors.Is(err, ErrSpriteNotFound) {
return true
}
return looksLikeNotFound(err.Error())
}
func looksLikeNotFound(msg string) bool {
lower := strings.ToLower(msg)
return strings.Contains(lower, "sprite not found") ||
strings.Contains(lower, "not found")
}
@@ -0,0 +1,62 @@
package sprites
import (
"errors"
"fmt"
"testing"
)
func TestWrapNotFoundWraps404Errors(t *testing.T) {
cases := []struct {
name string
err error
want bool
}{
{name: "sprite not found exact", err: fmt.Errorf("sprite not found: kandev-abc"), want: true},
{name: "uppercase variant", err: fmt.Errorf("Sprite Not Found: kandev-abc"), want: true},
{name: "wrapped not found", err: fmt.Errorf("get sprite: %w", fmt.Errorf("sprite not found: kandev-abc")), want: true},
{name: "generic 404 mention", err: fmt.Errorf("HTTP 404: not found"), want: true},
{name: "auth failure", err: fmt.Errorf("401 unauthorized"), want: false},
{name: "transient 500", err: fmt.Errorf("internal server error"), want: false},
{name: "nil", err: nil, want: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
wrapped := WrapNotFound(tc.err)
got := errors.Is(wrapped, ErrSpriteNotFound)
if got != tc.want {
t.Fatalf("errors.Is(WrapNotFound(%v), ErrSpriteNotFound) = %v, want %v", tc.err, got, tc.want)
}
if got != IsNotFound(tc.err) {
t.Fatalf("IsNotFound(%v) disagrees with errors.Is on wrapped form", tc.err)
}
})
}
}
func TestWrapNotFoundPreservesOriginalError(t *testing.T) {
orig := fmt.Errorf("sprite not found: kandev-abc")
wrapped := WrapNotFound(orig)
if !errors.Is(wrapped, orig) {
t.Fatalf("WrapNotFound dropped the original error from the chain")
}
if !errors.Is(wrapped, ErrSpriteNotFound) {
t.Fatalf("WrapNotFound dropped the sentinel from the chain")
}
}
func TestWrapNotFoundIsIdempotent(t *testing.T) {
once := WrapNotFound(fmt.Errorf("sprite not found: kandev-abc"))
twice := WrapNotFound(once)
if once.Error() != twice.Error() {
t.Fatalf("WrapNotFound is not idempotent: once=%q, twice=%q", once, twice)
}
}
func TestWrapNotFoundNilPassthrough(t *testing.T) {
if WrapNotFound(nil) != nil {
t.Fatalf("WrapNotFound(nil) should return nil")
}
}