feat(sprites): provision a fresh sandbox when the persisted one is gone
Resuming a Sprites task whose sandbox was destroyed upstream returned "sprite not found" and dead-ended the user. Detect the upstream 404 via a typed ErrSpriteNotFound sentinel, mark the reconnect step skipped with a user-facing warning explaining what happened, swap the prepare plan in place to the fresh-provision shape, and continue the launch under a new sandbox name on the same branch. The progress plan is now mutable so the reporter can stream the fresh steps live after the reconnect-then-fresh transition. Only the explicit not-found case triggers fallback; other reconnect failures still surface as before, and the fallback fires at most once per launch.
This commit is contained in:
@@ -2,6 +2,7 @@ package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/kandev/kandev/internal/agentctl/server/process"
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
"github.com/kandev/kandev/internal/secrets"
|
||||
spritesutil "github.com/kandev/kandev/internal/sprites"
|
||||
)
|
||||
|
||||
type RemoteAuthAgentLister interface {
|
||||
@@ -137,11 +139,21 @@ func (r *SpritesExecutor) CreateInstance(ctx context.Context, req *ExecutorCreat
|
||||
zap.String("sprite_name", spriteName),
|
||||
zap.Bool("reconnect_required", reconnect))
|
||||
|
||||
// Step 0: Create or reconnect sprite
|
||||
// Step 0: Create or reconnect sprite. On reconnect-then-not-found we fall
|
||||
// through to fresh provisioning under a new name on the same branch.
|
||||
sprite, err := r.stepCreateSprite(ctx, client, spriteName, reconnect, report)
|
||||
if err != nil {
|
||||
r.cleanupOnFailure(ctx, sprite, req.InstanceID, destroyOnFailure)
|
||||
return nil, err
|
||||
if reconnect && errors.Is(err, spritesutil.ErrSpriteNotFound) {
|
||||
oldName := spriteName
|
||||
spriteName = r.fallbackToFreshSandbox(req, progressPlan, report, oldName)
|
||||
reconnect = false
|
||||
destroyOnFailure = true
|
||||
sprite, err = r.stepCreateSprite(ctx, client, spriteName, false, report)
|
||||
}
|
||||
if err != nil {
|
||||
r.cleanupOnFailure(ctx, sprite, req.InstanceID, destroyOnFailure)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Steps 1-3: Upload agentctl, credentials, prepare script
|
||||
@@ -199,6 +211,62 @@ func (r *SpritesExecutor) resolveSpriteName(req *ExecutorCreateRequest, reconnec
|
||||
return spritesNamePrefix + suffix
|
||||
}
|
||||
|
||||
// fallbackToFreshSandbox flips a stalled reconnect into a fresh provision. It
|
||||
// emits a skipped "Reconnecting cloud sandbox" step carrying the user-facing
|
||||
// warning that explains why we're reprovisioning, then mutates the live plan
|
||||
// in place so the rest of CreateInstance streams the normal fresh-provision
|
||||
// progress rows. Returns the new sprite name. The user-visible explanation is
|
||||
// the only place we surface this transition.
|
||||
func (r *SpritesExecutor) fallbackToFreshSandbox(
|
||||
req *ExecutorCreateRequest,
|
||||
plan *spritesProgressPlan,
|
||||
report func(spritesStepKey, PrepareStep),
|
||||
oldName string,
|
||||
) string {
|
||||
newName := spritesNamePrefix + req.InstanceID[:12]
|
||||
branch := getMetadataString(req.Metadata, MetadataKeyWorktreeBranch)
|
||||
if branch == "" {
|
||||
branch = getMetadataString(req.Metadata, MetadataKeyBaseBranch)
|
||||
}
|
||||
|
||||
r.logger.Info("sprite missing on reconnect, provisioning fresh",
|
||||
zap.String("instance_id", req.InstanceID),
|
||||
zap.String("old_sprite_name", oldName),
|
||||
zap.String("new_sprite_name", newName),
|
||||
zap.String("branch", branch))
|
||||
|
||||
plan.replacePlan([]spritesStepKey{
|
||||
spriteStepCreateSprite,
|
||||
spriteStepUploadAgentctl,
|
||||
spriteStepUploadCredentials,
|
||||
spriteStepRunPrepareScript,
|
||||
spriteStepWaitHealthy,
|
||||
spriteStepAgentInstance,
|
||||
spriteStepApplyNetworkPolicy,
|
||||
})
|
||||
|
||||
notice := beginStep("Reconnecting cloud sandbox")
|
||||
notice.Warning = "Previous sandbox is no longer available — provisioning a fresh one for this branch."
|
||||
notice.WarningDetail = fmt.Sprintf(
|
||||
"The Sprites sandbox %s could not be reached (it was likely destroyed or expired). "+
|
||||
"Kandev is starting a fresh sandbox %s on the same branch %s; this typically takes 30–60 seconds.",
|
||||
oldName, newName, branchOrPlaceholder(branch),
|
||||
)
|
||||
notice.Output = fmt.Sprintf("Old sandbox: %s\nNew sandbox: %s\nBranch: %s",
|
||||
oldName, newName, branchOrPlaceholder(branch))
|
||||
completeStepSkipped(¬ice)
|
||||
report(spriteStepCreateSprite, notice)
|
||||
|
||||
return newName
|
||||
}
|
||||
|
||||
func branchOrPlaceholder(branch string) string {
|
||||
if branch == "" {
|
||||
return "(unknown)"
|
||||
}
|
||||
return branch
|
||||
}
|
||||
|
||||
func spritesShouldReconnect(req *ExecutorCreateRequest) bool {
|
||||
if req == nil {
|
||||
return false
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSpritesProgressPlanGrowsOnFallback(t *testing.T) {
|
||||
plan := newSpritesProgressPlan(true)
|
||||
if plan.total() != 3 {
|
||||
t.Fatalf("reconnect plan total = %d, want 3", plan.total())
|
||||
}
|
||||
|
||||
plan.replacePlan([]spritesStepKey{
|
||||
spriteStepCreateSprite,
|
||||
spriteStepUploadAgentctl,
|
||||
spriteStepUploadCredentials,
|
||||
spriteStepRunPrepareScript,
|
||||
spriteStepWaitHealthy,
|
||||
spriteStepAgentInstance,
|
||||
spriteStepApplyNetworkPolicy,
|
||||
})
|
||||
|
||||
if plan.total() != 7 {
|
||||
t.Fatalf("after fallback, plan total = %d, want 7", plan.total())
|
||||
}
|
||||
for _, key := range []spritesStepKey{
|
||||
spriteStepUploadAgentctl,
|
||||
spriteStepUploadCredentials,
|
||||
spriteStepRunPrepareScript,
|
||||
spriteStepApplyNetworkPolicy,
|
||||
} {
|
||||
if !plan.has(key) {
|
||||
t.Fatalf("after fallback, plan should include %s", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpritesStepReporterFollowsPlanReplacement(t *testing.T) {
|
||||
plan := newSpritesProgressPlan(true)
|
||||
type reportEvent struct {
|
||||
index int
|
||||
total int
|
||||
}
|
||||
var events []reportEvent
|
||||
cb := func(_ PrepareStep, idx, total int) {
|
||||
events = append(events, reportEvent{index: idx, total: total})
|
||||
}
|
||||
report := newSpritesStepReporter(cb, plan)
|
||||
|
||||
report(spriteStepWaitHealthy, beginStep("Waiting for agent controller"))
|
||||
if len(events) != 1 || events[0].index != 1 || events[0].total != 3 {
|
||||
t.Fatalf("pre-replace event = %+v, want index 1 of 3", events)
|
||||
}
|
||||
|
||||
plan.replacePlan([]spritesStepKey{
|
||||
spriteStepCreateSprite,
|
||||
spriteStepUploadAgentctl,
|
||||
spriteStepUploadCredentials,
|
||||
spriteStepRunPrepareScript,
|
||||
spriteStepWaitHealthy,
|
||||
spriteStepAgentInstance,
|
||||
spriteStepApplyNetworkPolicy,
|
||||
})
|
||||
report(spriteStepWaitHealthy, beginStep("Waiting for agent controller"))
|
||||
if len(events) != 2 || events[1].index != 4 || events[1].total != 7 {
|
||||
t.Fatalf("post-replace event = %+v, want index 4 of 7", events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpritesFallbackToFreshSandboxEmitsWarningAndMutatesPlan(t *testing.T) {
|
||||
r := newTestSpritesExecutor(nil)
|
||||
plan := newSpritesProgressPlan(true)
|
||||
|
||||
var emitted []PrepareStep
|
||||
report := func(_ spritesStepKey, step PrepareStep) {
|
||||
emitted = append(emitted, step)
|
||||
}
|
||||
|
||||
req := &ExecutorCreateRequest{
|
||||
InstanceID: "abcdef0123456789aaaa",
|
||||
Metadata: map[string]interface{}{
|
||||
MetadataKeyWorktreeBranch: "feature/something",
|
||||
"sprite_name": "kandev-old1234567890",
|
||||
},
|
||||
}
|
||||
|
||||
newName := r.fallbackToFreshSandbox(req, plan, report, "kandev-old1234567890")
|
||||
|
||||
wantName := spritesNamePrefix + req.InstanceID[:12]
|
||||
if newName != wantName {
|
||||
t.Fatalf("new sprite name = %q, want %q", newName, wantName)
|
||||
}
|
||||
if plan.total() != 7 {
|
||||
t.Fatalf("plan total after fallback = %d, want 7", plan.total())
|
||||
}
|
||||
if len(emitted) != 1 {
|
||||
t.Fatalf("expected exactly 1 progress event from fallback, got %d", len(emitted))
|
||||
}
|
||||
|
||||
notice := emitted[0]
|
||||
if notice.Status != PrepareStepSkipped {
|
||||
t.Fatalf("notice status = %s, want skipped", notice.Status)
|
||||
}
|
||||
if !strings.Contains(notice.Warning, "Previous sandbox is no longer available") {
|
||||
t.Fatalf("notice warning missing user-facing copy: %q", notice.Warning)
|
||||
}
|
||||
if !strings.Contains(notice.WarningDetail, "kandev-old1234567890") ||
|
||||
!strings.Contains(notice.WarningDetail, wantName) ||
|
||||
!strings.Contains(notice.WarningDetail, "feature/something") {
|
||||
t.Fatalf("notice detail missing context: %q", notice.WarningDetail)
|
||||
}
|
||||
if !strings.Contains(notice.Output, "Old sandbox: kandev-old1234567890") ||
|
||||
!strings.Contains(notice.Output, "New sandbox: "+wantName) ||
|
||||
!strings.Contains(notice.Output, "Branch: feature/something") {
|
||||
t.Fatalf("notice output missing structured block: %q", notice.Output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSpritesFallbackUsesPlaceholderBranchWhenMissing(t *testing.T) {
|
||||
r := newTestSpritesExecutor(nil)
|
||||
plan := newSpritesProgressPlan(true)
|
||||
var emitted []PrepareStep
|
||||
|
||||
req := &ExecutorCreateRequest{
|
||||
InstanceID: "abcdef0123456789aaaa",
|
||||
Metadata: map[string]interface{}{},
|
||||
}
|
||||
r.fallbackToFreshSandbox(req, plan, func(_ spritesStepKey, step PrepareStep) {
|
||||
emitted = append(emitted, step)
|
||||
}, "kandev-old1234567890")
|
||||
|
||||
if len(emitted) != 1 {
|
||||
t.Fatalf("expected 1 event, got %d", len(emitted))
|
||||
}
|
||||
if !strings.Contains(emitted[0].Output, "Branch: (unknown)") {
|
||||
t.Fatalf("expected placeholder branch in output, got %q", emitted[0].Output)
|
||||
}
|
||||
}
|
||||
@@ -131,42 +131,61 @@ const (
|
||||
spriteStepApplyNetworkPolicy spritesStepKey = "apply_network_policy"
|
||||
)
|
||||
|
||||
// spritesProgressPlan is mutable so the reconnect path can swap to the fresh
|
||||
// plan mid-flight when a stale sandbox falls back to provisioning a new one.
|
||||
// The reporter created by newSpritesStepReporter resolves indexes lazily off
|
||||
// the live plan, so callers don't need to re-create it after replacePlan.
|
||||
type spritesProgressPlan struct {
|
||||
steps []spritesStepKey
|
||||
indexes map[spritesStepKey]int
|
||||
}
|
||||
|
||||
func newSpritesProgressPlan(reconnect bool) spritesProgressPlan {
|
||||
steps := []spritesStepKey{spriteStepCreateSprite}
|
||||
if !reconnect {
|
||||
steps = append(steps,
|
||||
spriteStepUploadAgentctl,
|
||||
spriteStepUploadCredentials,
|
||||
spriteStepRunPrepareScript,
|
||||
)
|
||||
}
|
||||
steps = append(steps, spriteStepWaitHealthy, spriteStepAgentInstance)
|
||||
if !reconnect {
|
||||
steps = append(steps, spriteStepApplyNetworkPolicy)
|
||||
func newSpritesProgressPlan(reconnect bool) *spritesProgressPlan {
|
||||
if reconnect {
|
||||
return buildPlan([]spritesStepKey{
|
||||
spriteStepCreateSprite,
|
||||
spriteStepWaitHealthy,
|
||||
spriteStepAgentInstance,
|
||||
})
|
||||
}
|
||||
return buildPlan([]spritesStepKey{
|
||||
spriteStepCreateSprite,
|
||||
spriteStepUploadAgentctl,
|
||||
spriteStepUploadCredentials,
|
||||
spriteStepRunPrepareScript,
|
||||
spriteStepWaitHealthy,
|
||||
spriteStepAgentInstance,
|
||||
spriteStepApplyNetworkPolicy,
|
||||
})
|
||||
}
|
||||
|
||||
func buildPlan(steps []spritesStepKey) *spritesProgressPlan {
|
||||
indexes := make(map[spritesStepKey]int, len(steps))
|
||||
for i, key := range steps {
|
||||
indexes[key] = i
|
||||
}
|
||||
return spritesProgressPlan{steps: steps, indexes: indexes}
|
||||
return &spritesProgressPlan{steps: steps, indexes: indexes}
|
||||
}
|
||||
|
||||
func (p spritesProgressPlan) total() int {
|
||||
func (p *spritesProgressPlan) total() int {
|
||||
if p == nil {
|
||||
return 0
|
||||
}
|
||||
return len(p.steps)
|
||||
}
|
||||
|
||||
func (p spritesProgressPlan) has(key spritesStepKey) bool {
|
||||
func (p *spritesProgressPlan) has(key spritesStepKey) bool {
|
||||
if p == nil {
|
||||
return false
|
||||
}
|
||||
_, ok := p.indexes[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (p spritesProgressPlan) index(key spritesStepKey) int {
|
||||
func (p *spritesProgressPlan) index(key spritesStepKey) int {
|
||||
if p == nil {
|
||||
return -1
|
||||
}
|
||||
idx, ok := p.indexes[key]
|
||||
if !ok {
|
||||
return -1
|
||||
@@ -174,8 +193,25 @@ func (p spritesProgressPlan) index(key spritesStepKey) int {
|
||||
return idx
|
||||
}
|
||||
|
||||
// newSpritesStepReporter creates a reporting function that calls OnProgress if non-nil.
|
||||
func newSpritesStepReporter(onProgress PrepareProgressCallback, plan spritesProgressPlan) func(spritesStepKey, PrepareStep) {
|
||||
// replacePlan swaps the active plan in place (used by the reconnect-fell-back-
|
||||
// to-fresh path). The reporter dispatches off the live indexes, so existing
|
||||
// closures keep working after replacement.
|
||||
func (p *spritesProgressPlan) replacePlan(steps []spritesStepKey) {
|
||||
if p == nil {
|
||||
return
|
||||
}
|
||||
indexes := make(map[spritesStepKey]int, len(steps))
|
||||
for i, key := range steps {
|
||||
indexes[key] = i
|
||||
}
|
||||
p.steps = steps
|
||||
p.indexes = indexes
|
||||
}
|
||||
|
||||
// newSpritesStepReporter creates a reporting function that calls OnProgress if
|
||||
// non-nil. Index/total are resolved against the live plan on each call so
|
||||
// replacePlan is reflected in subsequent reports.
|
||||
func newSpritesStepReporter(onProgress PrepareProgressCallback, plan *spritesProgressPlan) func(spritesStepKey, PrepareStep) {
|
||||
return func(key spritesStepKey, step PrepareStep) {
|
||||
if onProgress == nil {
|
||||
return
|
||||
|
||||
@@ -18,7 +18,7 @@ func (r *SpritesExecutor) reconnectSprite(ctx context.Context, client *sprites.C
|
||||
defer cancel()
|
||||
sprite, err := client.GetSprite(stepCtx, name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to reconnect sprite %q: %w", name, err)
|
||||
return nil, fmt.Errorf("failed to reconnect sprite %q: %w", name, spritesutil.WrapNotFound(err))
|
||||
}
|
||||
return sprite, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package sprites
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrSpriteNotFound is returned when an upstream Sprites API call indicates the
|
||||
// sandbox does not exist. The upstream SDK currently returns plain errors with
|
||||
// a "sprite not found: <name>" message for 404 responses; this sentinel lets
|
||||
// callers branch on that case without ad-hoc string matching.
|
||||
var ErrSpriteNotFound = errors.New("sprite not found")
|
||||
|
||||
// WrapNotFound wraps err with ErrSpriteNotFound when err looks like an upstream
|
||||
// "sprite not found" / 404 response. Other errors are returned unchanged. Pass
|
||||
// the returned error to errors.Is(err, ErrSpriteNotFound) to detect the case.
|
||||
func WrapNotFound(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, ErrSpriteNotFound) {
|
||||
return err
|
||||
}
|
||||
if !looksLikeNotFound(err.Error()) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("%w: %w", ErrSpriteNotFound, err)
|
||||
}
|
||||
|
||||
// IsNotFound reports whether err originated from an upstream "not found"
|
||||
// response, even when WrapNotFound has not been applied. Useful for callers
|
||||
// that receive a raw upstream error.
|
||||
func IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, ErrSpriteNotFound) {
|
||||
return true
|
||||
}
|
||||
return looksLikeNotFound(err.Error())
|
||||
}
|
||||
|
||||
func looksLikeNotFound(msg string) bool {
|
||||
lower := strings.ToLower(msg)
|
||||
return strings.Contains(lower, "sprite not found") ||
|
||||
strings.Contains(lower, "not found")
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package sprites
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestWrapNotFoundWraps404Errors(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{name: "sprite not found exact", err: fmt.Errorf("sprite not found: kandev-abc"), want: true},
|
||||
{name: "uppercase variant", err: fmt.Errorf("Sprite Not Found: kandev-abc"), want: true},
|
||||
{name: "wrapped not found", err: fmt.Errorf("get sprite: %w", fmt.Errorf("sprite not found: kandev-abc")), want: true},
|
||||
{name: "generic 404 mention", err: fmt.Errorf("HTTP 404: not found"), want: true},
|
||||
{name: "auth failure", err: fmt.Errorf("401 unauthorized"), want: false},
|
||||
{name: "transient 500", err: fmt.Errorf("internal server error"), want: false},
|
||||
{name: "nil", err: nil, want: false},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
wrapped := WrapNotFound(tc.err)
|
||||
got := errors.Is(wrapped, ErrSpriteNotFound)
|
||||
if got != tc.want {
|
||||
t.Fatalf("errors.Is(WrapNotFound(%v), ErrSpriteNotFound) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
if got != IsNotFound(tc.err) {
|
||||
t.Fatalf("IsNotFound(%v) disagrees with errors.Is on wrapped form", tc.err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapNotFoundPreservesOriginalError(t *testing.T) {
|
||||
orig := fmt.Errorf("sprite not found: kandev-abc")
|
||||
wrapped := WrapNotFound(orig)
|
||||
|
||||
if !errors.Is(wrapped, orig) {
|
||||
t.Fatalf("WrapNotFound dropped the original error from the chain")
|
||||
}
|
||||
if !errors.Is(wrapped, ErrSpriteNotFound) {
|
||||
t.Fatalf("WrapNotFound dropped the sentinel from the chain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapNotFoundIsIdempotent(t *testing.T) {
|
||||
once := WrapNotFound(fmt.Errorf("sprite not found: kandev-abc"))
|
||||
twice := WrapNotFound(once)
|
||||
if once.Error() != twice.Error() {
|
||||
t.Fatalf("WrapNotFound is not idempotent: once=%q, twice=%q", once, twice)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWrapNotFoundNilPassthrough(t *testing.T) {
|
||||
if WrapNotFound(nil) != nil {
|
||||
t.Fatalf("WrapNotFound(nil) should return nil")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user