fix(storage): repair Go-cache quarantine lifecycle (#2602)

This commit is contained in:
Carlos Florêncio
2026-08-13 11:07:46 +01:00
committed by GitHub
parent e377809d78
commit 23328edad5
12 changed files with 830 additions and 80 deletions
@@ -504,10 +504,11 @@ func (c *workspaceQuarantineController) Purge(
continue
}
var deleted storagepkg.QuarantineEntry
var payloadRemoved bool
if force {
deleted, err = c.PermanentDeleteForce(ctx, entry.ID, confirmation)
deleted, payloadRemoved, err = c.permanentDeleteWithPayload(ctx, entry.ID, confirmation, true)
} else {
deleted, err = c.PermanentDelete(ctx, entry.ID, storagepkg.QuarantineConfirmationDelete)
deleted, payloadRemoved, err = c.permanentDeleteWithPayload(ctx, entry.ID, storagepkg.QuarantineConfirmationDelete, false)
}
if err != nil {
result.Failed++
@@ -517,7 +518,9 @@ func (c *workspaceQuarantineController) Purge(
continue
}
result.Deleted++
result.DeletedBytes += deleted.SizeBytes
if payloadRemoved {
result.DeletedBytes += deleted.SizeBytes
}
}
return result, errors.Join(purgeErrs...)
}
@@ -591,36 +594,52 @@ func (c *workspaceQuarantineController) permanentDelete(
confirmation string,
force bool,
) (storagepkg.QuarantineEntry, error) {
deleted, _, err := c.permanentDeleteWithPayload(ctx, id, confirmation, force)
return deleted, err
}
func (c *workspaceQuarantineController) permanentDeleteWithPayload(
ctx context.Context,
id string,
confirmation string,
force bool,
) (storagepkg.QuarantineEntry, bool, error) {
entry, err := c.store.GetQuarantineEntry(ctx, id)
if err != nil {
return storagepkg.QuarantineEntry{}, err
return storagepkg.QuarantineEntry{}, false, err
}
if entry.ResourceType == storagepkg.ResourceTypeGoCache {
if force {
return c.deleteGoCacheForce(ctx, entry, confirmation)
return c.deleteGoCacheWithRetention(ctx, entry, confirmation, true)
}
return c.deleteGoCache(ctx, entry, confirmation)
return c.deleteGoCacheWithRetention(ctx, entry, confirmation, false)
}
if entry.ResourceType == storagepkg.ResourceTypeTemporaryArtifact {
if c.temporary == nil {
return storagepkg.QuarantineEntry{}, errors.New("temporary artifact provider is unavailable")
return storagepkg.QuarantineEntry{}, false, errors.New("temporary artifact provider is unavailable")
}
var deleted storagepkg.QuarantineEntry
if force {
return c.temporary.PermanentDeleteForce(ctx, id, confirmation)
deleted, err = c.temporary.PermanentDeleteForce(ctx, id, confirmation)
} else {
deleted, err = c.temporary.PermanentDelete(ctx, id, confirmation)
}
return c.temporary.PermanentDelete(ctx, id, confirmation)
return deleted, true, err
}
if entry.ResourceType != storagepkg.ResourceTypeTaskWorkspace {
return storagepkg.QuarantineEntry{}, fmt.Errorf("%w: unsupported quarantine resource %q", storagepkg.ErrValidation, entry.ResourceType)
return storagepkg.QuarantineEntry{}, false, fmt.Errorf("%w: unsupported quarantine resource %q", storagepkg.ErrValidation, entry.ResourceType)
}
settings, err := c.settings.GetSettings(ctx)
if err != nil {
return storagepkg.QuarantineEntry{}, err
return storagepkg.QuarantineEntry{}, false, err
}
var deleted storagepkg.QuarantineEntry
if force {
return c.factory(settings).PermanentDeleteForce(ctx, id, confirmation)
deleted, err = c.factory(settings).PermanentDeleteForce(ctx, id, confirmation)
} else {
deleted, err = c.factory(settings).PermanentDelete(ctx, id, confirmation)
}
return c.factory(settings).PermanentDelete(ctx, id, confirmation)
return deleted, true, err
}
func (c *workspaceQuarantineController) restoreGoCache(
@@ -720,47 +739,56 @@ func (c *workspaceQuarantineController) acquireGoCacheMaintenance(
return lease, err
}
func (c *workspaceQuarantineController) deleteGoCache(
ctx context.Context,
entry storagepkg.QuarantineEntry,
confirmation string,
) (storagepkg.QuarantineEntry, error) {
return c.deleteGoCacheWithRetention(ctx, entry, confirmation, false)
}
func (c *workspaceQuarantineController) deleteGoCacheForce(
ctx context.Context,
entry storagepkg.QuarantineEntry,
confirmation string,
) (storagepkg.QuarantineEntry, error) {
return c.deleteGoCacheWithRetention(ctx, entry, confirmation, true)
}
func (c *workspaceQuarantineController) deleteGoCacheWithRetention(
ctx context.Context,
entry storagepkg.QuarantineEntry,
confirmation string,
force bool,
) (storagepkg.QuarantineEntry, error) {
) (storagepkg.QuarantineEntry, bool, error) {
if force {
if confirmation != storagepkg.QuarantineConfirmationForce {
return storagepkg.QuarantineEntry{}, storagepkg.ErrForceDeleteConfirmation
return storagepkg.QuarantineEntry{}, false, storagepkg.ErrForceDeleteConfirmation
}
} else if confirmation != storagepkg.QuarantineConfirmationDelete {
return storagepkg.QuarantineEntry{}, fmt.Errorf("%w: quarantine deletion requires DELETE confirmation", storagepkg.ErrValidation)
return storagepkg.QuarantineEntry{}, false, fmt.Errorf("%w: quarantine deletion requires DELETE confirmation", storagepkg.ErrValidation)
}
if err := c.validateGoCacheEntry(ctx, entry); err != nil {
return storagepkg.QuarantineEntry{}, err
return storagepkg.QuarantineEntry{}, false, err
}
if !force && time.Now().UTC().Before(entry.DeleteAfter) {
return storagepkg.QuarantineEntry{}, fmt.Errorf("%w: quarantine retention deadline has not elapsed", storagepkg.ErrConflict)
return storagepkg.QuarantineEntry{}, false, fmt.Errorf("%w: quarantine retention deadline has not elapsed", storagepkg.ErrConflict)
}
payloadPresent, err := goCacheQuarantinePayloadPresent(entry)
if err != nil {
return storagepkg.QuarantineEntry{}, false, err
}
if !payloadPresent {
deleted, err := c.persistGoCacheDeletion(ctx, entry)
return deleted, false, err
}
if err := c.rejectAmbiguousMissingGoCachePayload(entry); err != nil {
return storagepkg.QuarantineEntry{}, err
return storagepkg.QuarantineEntry{}, false, err
}
if err := os.RemoveAll(entry.QuarantinePath); err != nil {
return storagepkg.QuarantineEntry{}, fmt.Errorf("delete quarantined Go cache: %w", err)
return storagepkg.QuarantineEntry{}, false, fmt.Errorf("delete quarantined Go cache: %w", err)
}
deleted, err := c.persistGoCacheDeletion(ctx, entry)
return deleted, true, err
}
func goCacheQuarantinePayloadPresent(entry storagepkg.QuarantineEntry) (bool, error) {
if _, err := os.Lstat(entry.QuarantinePath); errors.Is(err, os.ErrNotExist) {
return false, nil
} else if err != nil {
return false, fmt.Errorf("inspect Go-cache quarantine payload: %w", err)
}
return true, nil
}
func (c *workspaceQuarantineController) persistGoCacheDeletion(
ctx context.Context,
entry storagepkg.QuarantineEntry,
) (storagepkg.QuarantineEntry, error) {
deleted, err := c.store.TransitionQuarantineEntry(
context.WithoutCancel(ctx), entry.ID, storagepkg.QuarantineStateDeleted, "",
)
@@ -693,48 +693,41 @@ func TestQuarantineControllerDoesNotTreatPopulatedReplacementAsRestoredPayload(t
storagepkg.QuarantineStateFailed,
}
for _, state := range states {
for _, operation := range []string{"restore", "delete"} {
t.Run(string(state)+"_"+operation, func(t *testing.T) {
home := t.TempDir()
original := filepath.Join(home, "cache", "go-build")
quarantined := filepath.Join(home, "trash", "go-cache", "entry-cache")
if err := os.MkdirAll(original, 0o700); err != nil {
t.Fatal(err)
}
artifact := filepath.Join(original, "replacement-artifact")
if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil {
t.Fatal(err)
}
settings, store := newStorageMaintenanceStores(t)
entry := createGoCacheQuarantineEntry(
t, store, original, quarantined, time.Now().UTC().Add(-time.Hour),
)
if state == storagepkg.QuarantineStateFailed {
markGoCacheQuarantineFailed(t, store, entry.ID)
}
controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home}
t.Run(string(state), func(t *testing.T) {
home := t.TempDir()
original := filepath.Join(home, "cache", "go-build")
quarantined := filepath.Join(home, "trash", "go-cache", "entry-cache")
if err := os.MkdirAll(original, 0o700); err != nil {
t.Fatal(err)
}
artifact := filepath.Join(original, "replacement-artifact")
if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil {
t.Fatal(err)
}
settings, store := newStorageMaintenanceStores(t)
entry := createGoCacheQuarantineEntry(
t, store, original, quarantined, time.Now().UTC().Add(-time.Hour),
)
if state == storagepkg.QuarantineStateFailed {
markGoCacheQuarantineFailed(t, store, entry.ID)
}
controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home}
var err error
if operation == "delete" {
_, err = controller.PermanentDelete(context.Background(), entry.ID, "DELETE")
} else {
_, err = controller.Restore(context.Background(), entry.ID)
}
if !errors.Is(err, storagepkg.ErrConflict) {
t.Fatalf("%s error = %v, want storage ErrConflict", operation, err)
}
stored, err := store.GetQuarantineEntry(context.Background(), entry.ID)
if err != nil {
t.Fatal(err)
}
if stored.State != state {
t.Fatalf("state = %q, want unchanged %q", stored.State, state)
}
if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" {
t.Fatalf("replacement cache changed: data=%q err=%v", data, err)
}
})
}
_, err := controller.Restore(context.Background(), entry.ID)
if !errors.Is(err, storagepkg.ErrConflict) {
t.Fatalf("Restore error = %v, want storage ErrConflict", err)
}
stored, err := store.GetQuarantineEntry(context.Background(), entry.ID)
if err != nil {
t.Fatal(err)
}
if stored.State != state {
t.Fatalf("state = %q, want unchanged %q", stored.State, state)
}
if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" {
t.Fatalf("replacement cache changed: data=%q err=%v", data, err)
}
})
}
}
@@ -827,6 +820,43 @@ type failingTransitionQuarantineStore struct {
err error
}
type removePayloadOnGetStore struct {
delegate *storagepkg.Store
path string
removed bool
}
func (s *removePayloadOnGetStore) GetQuarantineEntry(
ctx context.Context,
id string,
) (storagepkg.QuarantineEntry, error) {
entry, err := s.delegate.GetQuarantineEntry(ctx, id)
if err != nil || s.removed {
return entry, err
}
if err := os.RemoveAll(s.path); err != nil {
return storagepkg.QuarantineEntry{}, err
}
s.removed = true
return entry, nil
}
func (s *removePayloadOnGetStore) ListQuarantineEntries(
ctx context.Context,
includeTerminal bool,
) ([]storagepkg.QuarantineEntry, error) {
return s.delegate.ListQuarantineEntries(ctx, includeTerminal)
}
func (s *removePayloadOnGetStore) TransitionQuarantineEntry(
ctx context.Context,
id string,
next storagepkg.QuarantineState,
lastError string,
) (storagepkg.QuarantineEntry, error) {
return s.delegate.TransitionQuarantineEntry(ctx, id, next, lastError)
}
func (s *failingTransitionQuarantineStore) GetQuarantineEntry(
context.Context, string,
) (storagepkg.QuarantineEntry, error) {
@@ -915,6 +945,167 @@ func TestQuarantineControllerPermanentlyDeletesGoCache(t *testing.T) {
}
}
func TestQuarantineControllerPermanentlyDeletesMissingGoCachePayload(t *testing.T) {
for _, test := range []struct {
name string
force bool
}{
{name: "eligible"},
{name: "forced", force: true},
} {
t.Run(test.name, func(t *testing.T) {
home := t.TempDir()
original := filepath.Join(home, "cache", "go-build")
quarantined := filepath.Join(home, "trash", "go-cache", "entry-cache")
if err := os.MkdirAll(original, 0o700); err != nil {
t.Fatal(err)
}
artifact := filepath.Join(original, "replacement-artifact")
if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil {
t.Fatal(err)
}
settings, store := newStorageMaintenanceStores(t)
deleteAfter := time.Now().UTC().Add(-time.Hour)
if test.force {
deleteAfter = time.Now().UTC().Add(time.Hour)
}
entry := createGoCacheQuarantineEntry(t, store, original, quarantined, deleteAfter)
controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home}
var deleted storagepkg.QuarantineEntry
var err error
if test.force {
deleted, err = controller.PermanentDeleteForce(
context.Background(), entry.ID, storagepkg.QuarantineConfirmationForce,
)
} else {
deleted, err = controller.PermanentDelete(
context.Background(), entry.ID, storagepkg.QuarantineConfirmationDelete,
)
}
if err != nil {
t.Fatalf("delete missing payload: %v", err)
}
if deleted.State != storagepkg.QuarantineStateDeleted {
t.Fatalf("state = %q, want deleted", deleted.State)
}
if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" {
t.Fatalf("replacement cache changed: data=%q err=%v", data, err)
}
if _, err := os.Stat(quarantined); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("missing quarantine payload became present: %v", err)
}
stored, err := store.GetQuarantineEntry(context.Background(), entry.ID)
if err != nil {
t.Fatal(err)
}
if stored.State != storagepkg.QuarantineStateDeleted {
t.Fatalf("stored state = %q, want deleted", stored.State)
}
})
}
}
func TestQuarantineControllerPurgeReportsZeroBytesForMissingGoCachePayload(t *testing.T) {
home := t.TempDir()
original := filepath.Join(home, "cache", "go-build")
quarantined := filepath.Join(home, "trash", "go-cache", "missing-payload")
if err := os.MkdirAll(original, 0o700); err != nil {
t.Fatal(err)
}
artifact := filepath.Join(original, "replacement-artifact")
if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil {
t.Fatal(err)
}
settings, store := newStorageMaintenanceStores(t)
entry := storagepkg.QuarantineEntry{
ID: "missing-payload",
ResourceType: storagepkg.ResourceTypeGoCache,
OriginalPath: original,
QuarantinePath: quarantined,
SizeBytes: 42,
State: storagepkg.QuarantineStateQuarantined,
QuarantinedAt: time.Now().UTC().Add(-2 * time.Hour),
DeleteAfter: time.Now().UTC().Add(-time.Hour),
Metadata: json.RawMessage(`{"ownership":"managed"}`),
}
if err := store.CreateQuarantineEntry(context.Background(), &entry); err != nil {
t.Fatalf("create missing-payload entry: %v", err)
}
controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home}
result, err := controller.Purge(
context.Background(), storagepkg.QuarantinePurgeScopeEligible,
storagepkg.QuarantineConfirmationEligible,
)
if err != nil {
t.Fatalf("Purge: %v", err)
}
if result.Deleted != 1 || result.Failed != 0 || result.DeletedBytes != 0 {
t.Fatalf("purge result = %#v, want one deleted entry, zero failures, zero deleted bytes", result)
}
if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" {
t.Fatalf("replacement cache changed: data=%q err=%v", data, err)
}
stored, err := store.GetQuarantineEntry(context.Background(), entry.ID)
if err != nil {
t.Fatal(err)
}
if stored.State != storagepkg.QuarantineStateDeleted {
t.Fatalf("stored state = %q, want deleted", stored.State)
}
}
func TestQuarantineControllerPurgeReportsBytesFromDeletionOutcome(t *testing.T) {
home := t.TempDir()
original := filepath.Join(home, "cache", "go-build")
quarantined := filepath.Join(home, "trash", "go-cache", "removed-before-delete")
if err := os.MkdirAll(original, 0o700); err != nil {
t.Fatal(err)
}
liveArtifact := filepath.Join(original, "replacement-artifact")
if err := os.WriteFile(liveArtifact, []byte("active cache"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.MkdirAll(quarantined, 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(quarantined, "old-artifact"), []byte("old cache"), 0o600); err != nil {
t.Fatal(err)
}
settings, store := newStorageMaintenanceStores(t)
entry := storagepkg.QuarantineEntry{
ID: "removed-before-delete",
ResourceType: storagepkg.ResourceTypeGoCache,
OriginalPath: original,
QuarantinePath: quarantined,
SizeBytes: 42,
State: storagepkg.QuarantineStateQuarantined,
QuarantinedAt: time.Now().UTC().Add(-2 * time.Hour),
DeleteAfter: time.Now().UTC().Add(-time.Hour),
Metadata: json.RawMessage(`{"ownership":"managed"}`),
}
if err := store.CreateQuarantineEntry(context.Background(), &entry); err != nil {
t.Fatalf("create entry: %v", err)
}
deletingStore := &removePayloadOnGetStore{delegate: store, path: quarantined}
controller := &workspaceQuarantineController{settings: settings, store: deletingStore, homeDir: home}
result, err := controller.Purge(
context.Background(), storagepkg.QuarantinePurgeScopeEligible,
storagepkg.QuarantineConfirmationEligible,
)
if err != nil {
t.Fatalf("Purge: %v", err)
}
if result.Deleted != 1 || result.Failed != 0 || result.DeletedBytes != 0 {
t.Fatalf("purge result = %#v, want one deleted entry and zero deleted bytes", result)
}
if data, err := os.ReadFile(liveArtifact); err != nil || string(data) != "active cache" {
t.Fatalf("live cache changed: data=%q err=%v", data, err)
}
}
func TestQuarantineControllerDeletesHistoricalAdoptedGoCacheAfterReadoption(t *testing.T) {
root := t.TempDir()
home := filepath.Join(root, "home")
@@ -64,6 +64,8 @@ type Analysis struct {
// CleanupResult describes one cache rotation.
type CleanupResult struct {
Path string `json:"path"`
Skipped bool `json:"skipped"`
Reason string `json:"reason,omitempty"`
BytesBefore int64 `json:"bytes_before"`
BytesAfter int64 `json:"bytes_after"`
ReclaimedBytes int64 `json:"reclaimed_bytes"`
@@ -187,6 +189,13 @@ func (p *Provider) cleanup(ctx context.Context, explicit bool) (CleanupResult, e
}
entry, err := p.rotate(ctx, cachePath, result.BytesBefore, settings.QuarantineRetentionHours, adopted)
if err != nil {
var activeErr *storage.ActiveQuarantineIntentError
if errors.As(err, &activeErr) {
result.BytesAfter = result.BytesBefore
result.Skipped = true
result.Reason = "active_quarantine"
return result, nil
}
return result, err
}
result.QuarantineEntry = entry
@@ -8,6 +8,7 @@ import (
"path/filepath"
"sort"
"testing"
"time"
"github.com/kandev/kandev/internal/system/storage"
)
@@ -33,7 +34,7 @@ func TestCleanupResultJSONUsesStorageAPISnakeCase(t *testing.T) {
if err != nil {
t.Fatalf("Marshal CleanupResult: %v", err)
}
want := `{"path":"/cache","bytes_before":100,"bytes_after":20,"reclaimed_bytes":80,"quarantine_entry":null}`
want := `{"path":"/cache","skipped":false,"bytes_before":100,"bytes_after":20,"reclaimed_bytes":80,"quarantine_entry":null}`
if string(encoded) != want {
t.Fatalf("CleanupResult JSON = %s, want %s", encoded, want)
}
@@ -220,6 +221,69 @@ func TestCleanupRotatesOwnedCacheAboveThreshold(t *testing.T) {
}
}
func TestCleanupDefersWhenGoCacheQuarantineIsActive(t *testing.T) {
home := t.TempDir()
settings := storage.DefaultSettings()
settings.GoCache.Enabled = true
settings.GoCache.MaxBytes = 1
store := &recordingStore{}
provider := New(Config{
HomeDir: home,
TrashDir: filepath.Join(home, "trash"),
Settings: staticSettings{settings: settings},
Store: store,
})
env, err := provider.ExecutionEnvironment(context.Background())
if err != nil {
t.Fatalf("ExecutionEnvironment() error = %v", err)
}
cachePath := env["GOCACHE"]
if err := os.WriteFile(filepath.Join(cachePath, "replacement-artifact"), []byte("live"), 0o600); err != nil {
t.Fatalf("seed replacement cache: %v", err)
}
retainedPath := filepath.Join(home, "trash", "go-cache", "retained")
if err := os.MkdirAll(retainedPath, 0o700); err != nil {
t.Fatalf("create retained cache: %v", err)
}
if err := os.WriteFile(filepath.Join(retainedPath, "retained-artifact"), []byte("old"), 0o600); err != nil {
t.Fatalf("seed retained cache: %v", err)
}
active := &storage.QuarantineEntry{
ID: "retained",
ResourceType: storage.ResourceTypeGoCache,
OriginalPath: cachePath,
QuarantinePath: retainedPath,
SizeBytes: 3,
State: storage.QuarantineStateQuarantined,
QuarantinedAt: time.Now().UTC().Add(-time.Hour),
DeleteAfter: time.Now().UTC().Add(time.Hour),
Metadata: json.RawMessage(`{"ownership":"managed"}`),
}
if err := store.CreateQuarantineEntry(context.Background(), active); err != nil {
t.Fatalf("create active quarantine entry: %v", err)
}
result, err := provider.Cleanup(context.Background())
if err != nil {
t.Fatalf("Cleanup() error = %v", err)
}
if result.BytesBefore != 4 || result.BytesAfter != 4 || result.ReclaimedBytes != 0 {
t.Fatalf("cleanup result = %#v, want unchanged four-byte cache and no reclamation", result)
}
if !result.Skipped || result.Reason != "active_quarantine" || result.QuarantineEntry != nil {
t.Fatalf("cleanup deferral fields = %#v, want active_quarantine without a new entry", result)
}
if _, err := os.Stat(filepath.Join(cachePath, "replacement-artifact")); err != nil {
t.Fatalf("replacement cache changed: %v", err)
}
if _, err := os.Stat(filepath.Join(retainedPath, "retained-artifact")); err != nil {
t.Fatalf("retained cache changed: %v", err)
}
if len(store.entries) != 1 || store.entries[active.ID].State != storage.QuarantineStateQuarantined {
t.Fatalf("active quarantine entries = %#v, want unchanged retained entry", store.entries)
}
}
func TestCleanupNeverClaimsUnmarkedManagedPath(t *testing.T) {
home := t.TempDir()
cachePath := filepath.Join(home, "cache", "go-build")
@@ -20,6 +20,21 @@ type quarantineRetryStore interface {
) (QuarantineEntry, error)
}
// ActiveQuarantineIntentError reports that a retained quarantine entry still
// owns the original path. It remains compatible with ErrConflict so callers
// that treat the condition as an error keep their existing behavior.
type ActiveQuarantineIntentError struct {
OriginalPath string
}
func (e *ActiveQuarantineIntentError) Error() string {
return fmt.Sprintf("%s: quarantine intent for %s is still active", ErrConflict, e.OriginalPath)
}
func (e *ActiveQuarantineIntentError) Unwrap() error {
return ErrConflict
}
// ReleaseFailedQuarantineIntent releases a failed intent only when filesystem
// state proves its move did not occur. Ambiguous states remain active for
// recovery or reconciliation.
@@ -39,7 +54,7 @@ func ReleaseFailedQuarantineIntent(
continue
}
if entry.State == QuarantineStateQuarantined {
return false, fmt.Errorf("%w: quarantine intent for %s is still active", ErrConflict, originalPath)
return false, &ActiveQuarantineIntentError{OriginalPath: originalPath}
}
if entry.State == QuarantineStateFailed {
return releaseFailedIntentIfUnmoved(ctx, store, entry)
@@ -329,6 +329,33 @@ func TestReleaseFailedQuarantineIntentRequiresProvenUnmovedState(t *testing.T) {
}
}
func TestReleaseFailedQuarantineIntentClassifiesActiveIntent(t *testing.T) {
conn := newSQLite(t)
store := newStorageStore(t, db.NewPool(conn, conn))
original := filepath.Join(t.TempDir(), "original")
entry := testQuarantineEntry("active-entry")
entry.ResourceType = ResourceTypeGoCache
entry.OriginalPath = original
entry.State = QuarantineStateQuarantined
if err := store.CreateQuarantineEntry(context.Background(), &entry); err != nil {
t.Fatalf("create active intent: %v", err)
}
_, err := ReleaseFailedQuarantineIntent(
context.Background(), store, ResourceTypeGoCache, original,
)
var activeErr *ActiveQuarantineIntentError
if !errors.As(err, &activeErr) {
t.Fatalf("ReleaseFailedQuarantineIntent error = %v, want ActiveQuarantineIntentError", err)
}
if !errors.Is(err, ErrConflict) {
t.Fatalf("ReleaseFailedQuarantineIntent error = %v, want ErrConflict", err)
}
if activeErr.OriginalPath != original {
t.Fatalf("active intent original path = %q, want %q", activeErr.OriginalPath, original)
}
}
func TestStoreAllowsEveryQuarantineStateTransition(t *testing.T) {
tests := []struct {
name string
@@ -0,0 +1,159 @@
---
spec: docs/specs/system-page/storage-maintenance.md
created: 2026-08-12
status: done
---
# Implementation Plan: Go-cache Quarantine Lifecycle Repair
## Overview
An adopted Go cache can exceed its threshold while an older generation remains in quarantine.
The current provider treats that normal retention state as a conflict and fails each maintenance run.
It also cannot close an entry when its recorded payload is already absent.
The repair first makes the retained-generation condition a typed, successful deferral. It then makes
permanent deletion idempotent for a missing Go-cache payload without changing the live cache. The
final task updates the operator guide and records focused validation results.
Confirmed root cause: `gocache.Provider.rotate` calls `ReleaseFailedQuarantineIntent` before each
rotation. That helper returns `ErrConflict` for every active `quarantined` entry. The provider
propagates the error even though retention intentionally keeps the entry active. The live incident
also has a missing `quarantine_path`, but `deleteGoCacheWithRetention` rejects that state before it
can make the durable entry terminal.
---
## Backend
### Typed cleanup deferral
- Add a typed active-intent classification in
`apps/backend/internal/system/storage/quarantine_retry.go`. Preserve `errors.Is(err,
storage.ErrConflict)` and the current human-readable path context for callers that still treat the
condition as an error.
- Extend `gocache.CleanupResult` in
`apps/backend/internal/system/storage/gocache/provider.go` with `skipped` and `reason` fields.
- In `gocache.Provider.cleanup`, convert only the typed active-intent condition into a successful
result with `bytes_after == bytes_before`, zero reclaimed bytes, `skipped: true`, and
`reason: "active_quarantine"`.
- Keep failed or ambiguous quarantine intents on the existing fail-closed path. Do not weaken the
active-original unique index or allow more than one retained generation per cache path.
### Missing-payload permanent deletion
- Refactor Go-cache deletion in
`apps/backend/internal/backendapp/storage_maintenance.go` to distinguish an absent quarantine
payload from an unsafe or unreadable path.
- After the existing confirmation, retention, entry-state, ownership, and containment checks pass,
transition an absent payload to `deleted`. Do not inspect, remove, rename, or replace the original
cache path in this branch.
- Keep `restoreGoCache` on the existing ambiguous-state guard so that restore still fails closed when
the payload is absent.
- In `workspaceQuarantineController.Purge`, measure payload presence before deletion for result
accounting. Add zero to `deleted_bytes` when a successfully closed entry had no payload. Continue
to count the row in `deleted`.
- Do not add a schema migration or a new quarantine state. The existing `deleted` state accurately
records that no quarantined payload remains.
---
## Frontend
No frontend code change is required. `CleanupResult.Skipped` and `CleanupResult.Reason` extend the
`MaintenanceRun.Result` payload under `go_cache.result`; the existing run-result rendering accepts
those fields. Route shapes, UI actions, and confirmation flows remain unchanged, and the existing
**Delete**, **Clear eligible**, and **Force clear all** actions use the repaired backend paths.
---
## Public Documentation
- Update `docs/public/operations.md` in its how-to-oriented Storage maintenance section.
- Explain that one restorable Go-cache generation can defer later rotations without failing the run.
- Explain that permanent deletion can close a missing Go-cache payload without changing the live
replacement cache. Restore remains unavailable for missing payloads.
---
## Tests
- **What:** an above-threshold replacement cache with an active retained generation produces a
successful deferral and preserves both paths and the durable entry.
**File:** `apps/backend/internal/system/storage/gocache/provider_test.go`.
**How:** use the real provider with temporary cache and trash paths plus a recording quarantine
store. First prove that the regression test fails because cleanup returns `ErrConflict`. Then make
it pass with the exact result and filesystem assertions.
- **What:** the typed active-intent error remains compatible with `ErrConflict`, while failed-intent
retry and ambiguous-state behavior remain unchanged.
**Files:** `apps/backend/internal/system/storage/store_test.go` and
`apps/backend/internal/system/storage/gocache/provider_test.go`.
**How:** extend the existing table-driven retry tests and assert both classifications.
- **What:** eligible and forced permanent deletion close a missing Go-cache payload, preserve a
populated original cache, and report zero deleted bytes.
**File:** `apps/backend/internal/backendapp/storage_maintenance_test.go`.
**How:** use the production controller with a real temporary SQLite store and real filesystem
paths. Cover individual deletion and bulk purge.
- **What:** restore of the same missing-payload state still fails closed.
**File:** `apps/backend/internal/backendapp/storage_maintenance_test.go`.
**How:** retain and narrow the existing missing-payload restore regression test.
- **Integration boundary:** the backendapp tests exercise the production quarantine controller,
real `storage.Store`, SQLite state transitions, and filesystem effects. The existing
`MaintenanceRun.Result` envelope carries the result-field extension without route or UI action
changes, so a handler or frontend test is not required.
---
## E2E Tests
No E2E test is required. The repair changes backend lifecycle semantics without changing the UI,
route shapes, confirmation flows, or user interaction sequence.
---
## Verification Results
- Task 01: `cd apps/backend && go test ./internal/system/storage ./internal/system/storage/gocache`
passed (123 tests).
- Task 02: `cd apps/backend && go test ./internal/backendapp` passed (316 tests).
- Task 03: `node --test scripts/validate-public-docs.test.mjs && node scripts/validate-public-docs.mjs
&& git diff --check` passed (60 validation tests and 41 published pages).
---
## Implementation Waves And Parallel Candidates
The default execution order is sequential in the primary conversation. These waves do not
authorize subagents.
Wave 1:
- [x] [Task 01: Defer retained Go-cache rotation](task-01-defer-retained-rotation.md)
Wave 2:
- [x] [Task 02: Close missing Go-cache payloads](task-02-close-missing-payload.md)
Wave 3:
- [x] [Task 03: Update operator documentation](task-03-update-operator-docs.md)
Task 02 follows Task 01 so the primary conversation can integrate and record one lifecycle change
at a time. Task 03 documents both backend outcomes. No task is marked parallel-safe.
## Risks
- A broad conversion of `ErrConflict` into a skip can hide failed or ambiguous rename states. The
implementation must match only the new typed active-intent classification.
- Missing-payload deletion must never use the original cache as a fallback deletion target.
- Bulk result accounting must not claim that historical `size_bytes` were removed when the payload
was already absent.
- Restore must remain fail-closed because Kandev cannot prove that a populated original path is the
quarantined generation.
## Out of Scope
- Multiple simultaneous restorable generations for one Go-cache path.
- A new database state or migration for externally missing payloads.
- A new repair button, API route, or automatic deletion before the configured retention deadline.
- Changes to workspace or temporary-artifact restore semantics.
@@ -0,0 +1,79 @@
---
id: "01-defer-retained-rotation"
title: "Defer retained Go-cache rotation"
status: done
wave: 1
depends_on: []
plan: "plan.md"
spec: "../../specs/system-page/storage-maintenance.md"
---
# Task 01: Defer Retained Go-cache Rotation
## Intent
Make a retained Go-cache generation a successful cleanup deferral instead of a maintenance error.
Keep all failed and ambiguous intent states fail-closed.
## Acceptance
- An active `quarantined` entry for the same Go-cache path returns a typed active-intent
classification that remains compatible with `storage.ErrConflict`.
- Cleanup returns `skipped: true`, `reason: "active_quarantine"`, unchanged before/after bytes, and
zero reclaimed bytes without changing either cache path or the active entry.
- Failed-intent retry and ambiguous filesystem behavior remain unchanged.
## Files Likely Touched
- `apps/backend/internal/system/storage/quarantine_retry.go`
- `apps/backend/internal/system/storage/store_test.go`
- `apps/backend/internal/system/storage/gocache/provider.go`
- `apps/backend/internal/system/storage/gocache/provider_test.go`
- `docs/plans/go-cache-quarantine-lifecycle/plan.md`
- `docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md`
## Dependencies
None.
## Parallelism
`sequential`.
## Inputs
- Spec sections: **Go build cache**, **Failure modes**, **Persistence guarantees**, and **Scenarios**.
- Plan section: **Typed cleanup deferral**.
- Existing patterns: `ReleaseFailedQuarantineIntent` and temporary-artifact `CleanupResult` skip
fields.
## TDD Sequence
1. Add the above-threshold retained-generation regression test.
2. Run `go test ./internal/system/storage/gocache -run
TestCleanupDefersWhenGoCacheQuarantineIsActive` and record the expected `ErrConflict` failure.
3. Add the typed classification and minimal cleanup-result conversion.
4. Run the targeted packages and record the passing result.
## Verification
```bash
cd apps/backend && go test ./internal/system/storage ./internal/system/storage/gocache
```
## Output Contract
Report the files changed, the red and green test commands, the exact outcomes, remaining risks, and
the synchronized task and plan status.
## Results
- Red: `cd apps/backend && go test ./internal/system/storage/gocache -run
TestCleanupDefersWhenGoCacheQuarantineIsActive` failed because cleanup returned the existing
`storage.ErrConflict` for the active entry.
- Green: `cd apps/backend && go test ./internal/system/storage ./internal/system/storage/gocache`
passed (123 tests).
- Added `storage.ActiveQuarantineIntentError`, which preserves `errors.Is(err, storage.ErrConflict)`
and the existing path context. Go-cache cleanup now reports a successful `active_quarantine`
skip with unchanged byte counts and no new intent. Failed and ambiguous intent states remain
errors.
@@ -0,0 +1,78 @@
---
id: "02-close-missing-payload"
title: "Close missing Go-cache payloads"
status: done
wave: 2
depends_on: ["01-defer-retained-rotation"]
plan: "plan.md"
spec: "../../specs/system-page/storage-maintenance.md"
---
# Task 02: Close Missing Go-cache Payloads
## Intent
Make permanent deletion idempotent when a Go-cache quarantine payload is already absent. Preserve
the live original path and keep restore fail-closed.
## Acceptance
- Eligible and forced permanent deletion transition a valid missing-payload entry to `deleted`.
- The deletion path does not inspect recursively, remove, rename, or replace a populated original
cache, and bulk purge reports zero deleted bytes for the absent payload.
- Restore of the same state returns `storage.ErrConflict` and leaves the entry active.
## Files Likely Touched
- `apps/backend/internal/backendapp/storage_maintenance.go`
- `apps/backend/internal/backendapp/storage_maintenance_test.go`
- `docs/plans/go-cache-quarantine-lifecycle/plan.md`
- `docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md`
## Dependencies
- Task 01: retained rotation behavior and result classification.
## Parallelism
`sequential`.
## Inputs
- Spec sections: **Go build cache**, **Failure modes**, and **Scenarios**.
- Plan section: **Missing-payload permanent deletion**.
- Existing tests:
`TestQuarantineControllerDoesNotTreatPopulatedReplacementAsRestoredPayload` and
`TestQuarantineControllerDoesNotMarkMissingPayloadPlaceholderRestored`.
## TDD Sequence
1. Split the existing combined restore/delete assertion and add eligible, forced, and bulk
missing-payload deletion cases.
2. Run `go test ./internal/backendapp -run 'TestQuarantineController.*MissingGoCachePayload'` and
record the expected conflict failures.
3. Refactor the delete path without changing restore behavior.
4. Run the backendapp package and record the passing result.
## Verification
```bash
cd apps/backend && go test ./internal/backendapp
```
## Output Contract
Report the files changed, the red and green test commands, the exact outcomes, filesystem and
database assertions, remaining risks, and the synchronized task and plan status.
## Results
- Red: `cd apps/backend && go test ./internal/backendapp -run
'TestQuarantineController.*MissingGoCachePayload'` failed for the eligible, forced, and bulk
missing-payload cases because deletion returned `storage.ErrConflict`.
- Green: the same focused command passed (4 tests), and `cd apps/backend && go test
./internal/backendapp` passed (316 tests).
- Permanent deletion now closes a validated missing Go-cache payload without reading or changing the
original cache path. Restore continues through the existing fail-closed missing-payload guard.
Purge uses the deletion-time payload outcome and reports zero deleted bytes when the durable entry
was already missing on disk, including a payload disappearing before the deletion check.
@@ -0,0 +1,66 @@
---
id: "03-update-operator-docs"
title: "Update operator documentation"
status: done
wave: 3
depends_on: ["01-defer-retained-rotation", "02-close-missing-payload"]
plan: "plan.md"
spec: "../../specs/system-page/storage-maintenance.md"
---
# Task 03: Update Operator Documentation
## Intent
Document the retained-generation and missing-payload behavior in the existing Storage maintenance
how-to guide.
## Acceptance
- The guide states that one active Go-cache quarantine generation defers another rotation without
failing maintenance.
- The guide states that permanent deletion can close an absent payload without changing the live
cache, while restore remains fail-closed.
- Public-document validation passes, and the plan records all task results.
## Files Likely Touched
- `docs/public/operations.md`
- `docs/plans/go-cache-quarantine-lifecycle/plan.md`
- `docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md`
- `docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md`
- `docs/plans/go-cache-quarantine-lifecycle/task-03-update-operator-docs.md`
## Dependencies
- Task 01: final deferral result fields and semantics.
- Task 02: final deletion and byte-accounting semantics.
## Parallelism
`sequential`.
## Inputs
- Spec sections: **Go build cache**, **Failure modes**, and **Scenarios**.
- Plan section: **Public Documentation**.
- Public page type: how-to guide.
## Verification
```bash
node --test scripts/validate-public-docs.test.mjs && node scripts/validate-public-docs.mjs && git diff --check
```
## Output Contract
Report the documentation changes, exact validation outcomes, public-doc type, cleanup evidence,
remaining risks, and synchronized task and plan status.
## Results
- Updated `docs/public/operations.md` with the one-active-generation deferral result and the
missing-payload deletion, zero-byte accounting, and fail-closed restore behavior.
- Public page type: how-to guide.
- `node --test scripts/validate-public-docs.test.mjs && node scripts/validate-public-docs.mjs &&
git diff --check` passed (60 validation tests and 41 published pages).
+11
View File
@@ -155,6 +155,17 @@ remove every active quarantine entry, discarding restore windows for entries tha
deleted. Safety-validation or deletion failures may leave entries visible and retryable. This
override bypasses only the retention timestamp; path, ownership, state, and filesystem safety
checks still apply.
Kandev keeps at most one restorable Go-cache generation for each original cache path. If that
generation is still active when the replacement cache exceeds its limit, the next rotation is
deferred. The maintenance run succeeds and reports `active_quarantine`; both the live cache and
the retained generation stay unchanged.
If a Go-cache quarantine payload is already missing, **Delete**, **Clear eligible**, or **Force
clear all** can close its durable entry without changing the live replacement cache. The purge
reports zero deleted bytes for that entry. **Restore** remains unavailable because Kandev cannot
prove which cache generation is currently live.
If scheduled cleanup is disabled, no independent quarantine sweeper runs: use a full **Run now** or
one of the quarantine actions when you want cleanup.
+24 -1
View File
@@ -1,7 +1,7 @@
---
status: building
created: 2026-07-14
updated: 2026-08-08
updated: 2026-08-12
owner: cfl
---
@@ -204,6 +204,14 @@ Retention override:
owned cache into Kandev trash and recreates an empty cache when its size is greater than the
configured maximum. The limit is a cleanup trigger, not a hard quota; the cache can temporarily
grow beyond it while tasks are active.
- Only one restorable Go-cache generation can remain active for an original path. If the replacement
cache exceeds its maximum before that generation becomes terminal, cleanup reports a successful
no-op with `skipped: true` and `reason: "active_quarantine"`. It does not create another intent,
change either cache path, or fail the complete maintenance run.
- Permanent deletion treats an absent Go-cache quarantine payload as already removed after all
confirmation, retention, ownership, containment, and state checks succeed. It marks the durable
entry `deleted`, does not change a populated replacement cache, and reports zero deleted bytes for
the absent payload. Restore continues to fail closed when the payload is absent.
- Disabling managed Go cache stops injecting `GOCACHE` into new executions. It does not delete the
previously managed cache. Scheduled cleanup and a global manual run with no resource selection
leave it untouched; only a manual run whose non-empty selection includes `go_cache` may rotate it.
@@ -600,6 +608,10 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore
warning, and does not run destructive maintenance.
- A managed Go-cache cleanup failure leaves either the original cache or its quarantined rename
intact; it never recursively deletes outside the configured owned path.
- An active Go-cache quarantine generation defers another rotation without failing maintenance.
The active entry remains available for restore or permanent deletion.
- An absent Go-cache quarantine payload cannot be restored. Permanent deletion can close its durable
entry without deleting or replacing data at the original cache path.
- A temporary-artifact registry read, marker read, lease/heartbeat check, or ownership/path
validation failure keeps that root in place and records a skipped warning. There is no fallback to
prefix, mtime, or process-name classification.
@@ -639,6 +651,8 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore
cleanup and permanent quarantine deletion do not cascade-delete that history.
- Quarantined data remains restorable until permanent deletion succeeds. A failed permanent delete
remains visible and retryable.
- A retained Go-cache generation keeps its unique active-path slot until restore or permanent
deletion makes the entry terminal. A deferred cleanup does not change that entry or its deadline.
- Temporary-artifact ownership rows and matching markers survive backend restarts. A producer's
normal close remains idempotent; an interrupted active row is reconciled and can become eligible
only after the stale interval.
@@ -736,6 +750,14 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore
- **GIVEN** managed Go cache is enabled and is 20 GiB with a 15 GiB threshold, **WHEN** an idle
cleanup runs, **THEN** Kandev rotates the owned cache to trash, recreates an empty cache, and
reports the reclaimed bytes.
- **GIVEN** an active Go-cache quarantine entry and a replacement cache above its threshold,
**WHEN** cleanup runs, **THEN** the Go-cache result reports `skipped: true` with
`reason: "active_quarantine"`, reclaims zero bytes, and the maintenance run does not fail.
- **GIVEN** an active Go-cache entry whose quarantine payload is absent and whose original path
contains a replacement cache, **WHEN** eligible or forced permanent deletion runs, **THEN** the
entry becomes `deleted`, the replacement cache remains unchanged, and deleted bytes are zero.
- **GIVEN** an active Go-cache entry whose quarantine payload is absent, **WHEN** restore runs,
**THEN** restore fails closed and the entry remains active.
- **GIVEN** `/root/.cache/go-build` was not explicitly adopted, **WHEN** storage cleanup runs,
**THEN** Kandev does not modify it.
- **GIVEN** an external Go cache was adopted successfully, **WHEN** the Storage page rerenders or is
@@ -819,3 +841,4 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore
- [Storage overview cache and settings follow-up](../../plans/storage-overview-cache/plan.md)
- [Quarantine lifecycle follow-up](../../plans/quarantine-lifecycle/plan.md)
- [Progressive Storage loading and totals](../../plans/storage-progressive-loading/plan.md)
- [Go-cache quarantine lifecycle repair](../../plans/go-cache-quarantine-lifecycle/plan.md)