diff --git a/apps/backend/internal/backendapp/storage_maintenance.go b/apps/backend/internal/backendapp/storage_maintenance.go index 4db294cb70..f8dc3c53f4 100644 --- a/apps/backend/internal/backendapp/storage_maintenance.go +++ b/apps/backend/internal/backendapp/storage_maintenance.go @@ -504,10 +504,11 @@ func (c *workspaceQuarantineController) Purge( continue } var deleted storagepkg.QuarantineEntry + var payloadRemoved bool if force { - deleted, err = c.PermanentDeleteForce(ctx, entry.ID, confirmation) + deleted, payloadRemoved, err = c.permanentDeleteWithPayload(ctx, entry.ID, confirmation, true) } else { - deleted, err = c.PermanentDelete(ctx, entry.ID, storagepkg.QuarantineConfirmationDelete) + deleted, payloadRemoved, err = c.permanentDeleteWithPayload(ctx, entry.ID, storagepkg.QuarantineConfirmationDelete, false) } if err != nil { result.Failed++ @@ -517,7 +518,9 @@ func (c *workspaceQuarantineController) Purge( continue } result.Deleted++ - result.DeletedBytes += deleted.SizeBytes + if payloadRemoved { + result.DeletedBytes += deleted.SizeBytes + } } return result, errors.Join(purgeErrs...) } @@ -591,36 +594,52 @@ func (c *workspaceQuarantineController) permanentDelete( confirmation string, force bool, ) (storagepkg.QuarantineEntry, error) { + deleted, _, err := c.permanentDeleteWithPayload(ctx, id, confirmation, force) + return deleted, err +} + +func (c *workspaceQuarantineController) permanentDeleteWithPayload( + ctx context.Context, + id string, + confirmation string, + force bool, +) (storagepkg.QuarantineEntry, bool, error) { entry, err := c.store.GetQuarantineEntry(ctx, id) if err != nil { - return storagepkg.QuarantineEntry{}, err + return storagepkg.QuarantineEntry{}, false, err } if entry.ResourceType == storagepkg.ResourceTypeGoCache { if force { - return c.deleteGoCacheForce(ctx, entry, confirmation) + return c.deleteGoCacheWithRetention(ctx, entry, confirmation, true) } - return c.deleteGoCache(ctx, entry, confirmation) + return c.deleteGoCacheWithRetention(ctx, entry, confirmation, false) } if entry.ResourceType == storagepkg.ResourceTypeTemporaryArtifact { if c.temporary == nil { - return storagepkg.QuarantineEntry{}, errors.New("temporary artifact provider is unavailable") + return storagepkg.QuarantineEntry{}, false, errors.New("temporary artifact provider is unavailable") } + var deleted storagepkg.QuarantineEntry if force { - return c.temporary.PermanentDeleteForce(ctx, id, confirmation) + deleted, err = c.temporary.PermanentDeleteForce(ctx, id, confirmation) + } else { + deleted, err = c.temporary.PermanentDelete(ctx, id, confirmation) } - return c.temporary.PermanentDelete(ctx, id, confirmation) + return deleted, true, err } if entry.ResourceType != storagepkg.ResourceTypeTaskWorkspace { - return storagepkg.QuarantineEntry{}, fmt.Errorf("%w: unsupported quarantine resource %q", storagepkg.ErrValidation, entry.ResourceType) + return storagepkg.QuarantineEntry{}, false, fmt.Errorf("%w: unsupported quarantine resource %q", storagepkg.ErrValidation, entry.ResourceType) } settings, err := c.settings.GetSettings(ctx) if err != nil { - return storagepkg.QuarantineEntry{}, err + return storagepkg.QuarantineEntry{}, false, err } + var deleted storagepkg.QuarantineEntry if force { - return c.factory(settings).PermanentDeleteForce(ctx, id, confirmation) + deleted, err = c.factory(settings).PermanentDeleteForce(ctx, id, confirmation) + } else { + deleted, err = c.factory(settings).PermanentDelete(ctx, id, confirmation) } - return c.factory(settings).PermanentDelete(ctx, id, confirmation) + return deleted, true, err } func (c *workspaceQuarantineController) restoreGoCache( @@ -720,47 +739,56 @@ func (c *workspaceQuarantineController) acquireGoCacheMaintenance( return lease, err } -func (c *workspaceQuarantineController) deleteGoCache( - ctx context.Context, - entry storagepkg.QuarantineEntry, - confirmation string, -) (storagepkg.QuarantineEntry, error) { - return c.deleteGoCacheWithRetention(ctx, entry, confirmation, false) -} - -func (c *workspaceQuarantineController) deleteGoCacheForce( - ctx context.Context, - entry storagepkg.QuarantineEntry, - confirmation string, -) (storagepkg.QuarantineEntry, error) { - return c.deleteGoCacheWithRetention(ctx, entry, confirmation, true) -} - func (c *workspaceQuarantineController) deleteGoCacheWithRetention( ctx context.Context, entry storagepkg.QuarantineEntry, confirmation string, force bool, -) (storagepkg.QuarantineEntry, error) { +) (storagepkg.QuarantineEntry, bool, error) { if force { if confirmation != storagepkg.QuarantineConfirmationForce { - return storagepkg.QuarantineEntry{}, storagepkg.ErrForceDeleteConfirmation + return storagepkg.QuarantineEntry{}, false, storagepkg.ErrForceDeleteConfirmation } } else if confirmation != storagepkg.QuarantineConfirmationDelete { - return storagepkg.QuarantineEntry{}, fmt.Errorf("%w: quarantine deletion requires DELETE confirmation", storagepkg.ErrValidation) + return storagepkg.QuarantineEntry{}, false, fmt.Errorf("%w: quarantine deletion requires DELETE confirmation", storagepkg.ErrValidation) } if err := c.validateGoCacheEntry(ctx, entry); err != nil { - return storagepkg.QuarantineEntry{}, err + return storagepkg.QuarantineEntry{}, false, err } if !force && time.Now().UTC().Before(entry.DeleteAfter) { - return storagepkg.QuarantineEntry{}, fmt.Errorf("%w: quarantine retention deadline has not elapsed", storagepkg.ErrConflict) + return storagepkg.QuarantineEntry{}, false, fmt.Errorf("%w: quarantine retention deadline has not elapsed", storagepkg.ErrConflict) + } + payloadPresent, err := goCacheQuarantinePayloadPresent(entry) + if err != nil { + return storagepkg.QuarantineEntry{}, false, err + } + if !payloadPresent { + deleted, err := c.persistGoCacheDeletion(ctx, entry) + return deleted, false, err } if err := c.rejectAmbiguousMissingGoCachePayload(entry); err != nil { - return storagepkg.QuarantineEntry{}, err + return storagepkg.QuarantineEntry{}, false, err } if err := os.RemoveAll(entry.QuarantinePath); err != nil { - return storagepkg.QuarantineEntry{}, fmt.Errorf("delete quarantined Go cache: %w", err) + return storagepkg.QuarantineEntry{}, false, fmt.Errorf("delete quarantined Go cache: %w", err) } + deleted, err := c.persistGoCacheDeletion(ctx, entry) + return deleted, true, err +} + +func goCacheQuarantinePayloadPresent(entry storagepkg.QuarantineEntry) (bool, error) { + if _, err := os.Lstat(entry.QuarantinePath); errors.Is(err, os.ErrNotExist) { + return false, nil + } else if err != nil { + return false, fmt.Errorf("inspect Go-cache quarantine payload: %w", err) + } + return true, nil +} + +func (c *workspaceQuarantineController) persistGoCacheDeletion( + ctx context.Context, + entry storagepkg.QuarantineEntry, +) (storagepkg.QuarantineEntry, error) { deleted, err := c.store.TransitionQuarantineEntry( context.WithoutCancel(ctx), entry.ID, storagepkg.QuarantineStateDeleted, "", ) diff --git a/apps/backend/internal/backendapp/storage_maintenance_test.go b/apps/backend/internal/backendapp/storage_maintenance_test.go index 3cb1dc8a88..b6d7fef133 100644 --- a/apps/backend/internal/backendapp/storage_maintenance_test.go +++ b/apps/backend/internal/backendapp/storage_maintenance_test.go @@ -693,48 +693,41 @@ func TestQuarantineControllerDoesNotTreatPopulatedReplacementAsRestoredPayload(t storagepkg.QuarantineStateFailed, } for _, state := range states { - for _, operation := range []string{"restore", "delete"} { - t.Run(string(state)+"_"+operation, func(t *testing.T) { - home := t.TempDir() - original := filepath.Join(home, "cache", "go-build") - quarantined := filepath.Join(home, "trash", "go-cache", "entry-cache") - if err := os.MkdirAll(original, 0o700); err != nil { - t.Fatal(err) - } - artifact := filepath.Join(original, "replacement-artifact") - if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil { - t.Fatal(err) - } - settings, store := newStorageMaintenanceStores(t) - entry := createGoCacheQuarantineEntry( - t, store, original, quarantined, time.Now().UTC().Add(-time.Hour), - ) - if state == storagepkg.QuarantineStateFailed { - markGoCacheQuarantineFailed(t, store, entry.ID) - } - controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home} + t.Run(string(state), func(t *testing.T) { + home := t.TempDir() + original := filepath.Join(home, "cache", "go-build") + quarantined := filepath.Join(home, "trash", "go-cache", "entry-cache") + if err := os.MkdirAll(original, 0o700); err != nil { + t.Fatal(err) + } + artifact := filepath.Join(original, "replacement-artifact") + if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil { + t.Fatal(err) + } + settings, store := newStorageMaintenanceStores(t) + entry := createGoCacheQuarantineEntry( + t, store, original, quarantined, time.Now().UTC().Add(-time.Hour), + ) + if state == storagepkg.QuarantineStateFailed { + markGoCacheQuarantineFailed(t, store, entry.ID) + } + controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home} - var err error - if operation == "delete" { - _, err = controller.PermanentDelete(context.Background(), entry.ID, "DELETE") - } else { - _, err = controller.Restore(context.Background(), entry.ID) - } - if !errors.Is(err, storagepkg.ErrConflict) { - t.Fatalf("%s error = %v, want storage ErrConflict", operation, err) - } - stored, err := store.GetQuarantineEntry(context.Background(), entry.ID) - if err != nil { - t.Fatal(err) - } - if stored.State != state { - t.Fatalf("state = %q, want unchanged %q", stored.State, state) - } - if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" { - t.Fatalf("replacement cache changed: data=%q err=%v", data, err) - } - }) - } + _, err := controller.Restore(context.Background(), entry.ID) + if !errors.Is(err, storagepkg.ErrConflict) { + t.Fatalf("Restore error = %v, want storage ErrConflict", err) + } + stored, err := store.GetQuarantineEntry(context.Background(), entry.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != state { + t.Fatalf("state = %q, want unchanged %q", stored.State, state) + } + if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" { + t.Fatalf("replacement cache changed: data=%q err=%v", data, err) + } + }) } } @@ -827,6 +820,43 @@ type failingTransitionQuarantineStore struct { err error } +type removePayloadOnGetStore struct { + delegate *storagepkg.Store + path string + removed bool +} + +func (s *removePayloadOnGetStore) GetQuarantineEntry( + ctx context.Context, + id string, +) (storagepkg.QuarantineEntry, error) { + entry, err := s.delegate.GetQuarantineEntry(ctx, id) + if err != nil || s.removed { + return entry, err + } + if err := os.RemoveAll(s.path); err != nil { + return storagepkg.QuarantineEntry{}, err + } + s.removed = true + return entry, nil +} + +func (s *removePayloadOnGetStore) ListQuarantineEntries( + ctx context.Context, + includeTerminal bool, +) ([]storagepkg.QuarantineEntry, error) { + return s.delegate.ListQuarantineEntries(ctx, includeTerminal) +} + +func (s *removePayloadOnGetStore) TransitionQuarantineEntry( + ctx context.Context, + id string, + next storagepkg.QuarantineState, + lastError string, +) (storagepkg.QuarantineEntry, error) { + return s.delegate.TransitionQuarantineEntry(ctx, id, next, lastError) +} + func (s *failingTransitionQuarantineStore) GetQuarantineEntry( context.Context, string, ) (storagepkg.QuarantineEntry, error) { @@ -915,6 +945,167 @@ func TestQuarantineControllerPermanentlyDeletesGoCache(t *testing.T) { } } +func TestQuarantineControllerPermanentlyDeletesMissingGoCachePayload(t *testing.T) { + for _, test := range []struct { + name string + force bool + }{ + {name: "eligible"}, + {name: "forced", force: true}, + } { + t.Run(test.name, func(t *testing.T) { + home := t.TempDir() + original := filepath.Join(home, "cache", "go-build") + quarantined := filepath.Join(home, "trash", "go-cache", "entry-cache") + if err := os.MkdirAll(original, 0o700); err != nil { + t.Fatal(err) + } + artifact := filepath.Join(original, "replacement-artifact") + if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil { + t.Fatal(err) + } + settings, store := newStorageMaintenanceStores(t) + deleteAfter := time.Now().UTC().Add(-time.Hour) + if test.force { + deleteAfter = time.Now().UTC().Add(time.Hour) + } + entry := createGoCacheQuarantineEntry(t, store, original, quarantined, deleteAfter) + controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home} + + var deleted storagepkg.QuarantineEntry + var err error + if test.force { + deleted, err = controller.PermanentDeleteForce( + context.Background(), entry.ID, storagepkg.QuarantineConfirmationForce, + ) + } else { + deleted, err = controller.PermanentDelete( + context.Background(), entry.ID, storagepkg.QuarantineConfirmationDelete, + ) + } + if err != nil { + t.Fatalf("delete missing payload: %v", err) + } + if deleted.State != storagepkg.QuarantineStateDeleted { + t.Fatalf("state = %q, want deleted", deleted.State) + } + if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" { + t.Fatalf("replacement cache changed: data=%q err=%v", data, err) + } + if _, err := os.Stat(quarantined); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("missing quarantine payload became present: %v", err) + } + stored, err := store.GetQuarantineEntry(context.Background(), entry.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != storagepkg.QuarantineStateDeleted { + t.Fatalf("stored state = %q, want deleted", stored.State) + } + }) + } +} + +func TestQuarantineControllerPurgeReportsZeroBytesForMissingGoCachePayload(t *testing.T) { + home := t.TempDir() + original := filepath.Join(home, "cache", "go-build") + quarantined := filepath.Join(home, "trash", "go-cache", "missing-payload") + if err := os.MkdirAll(original, 0o700); err != nil { + t.Fatal(err) + } + artifact := filepath.Join(original, "replacement-artifact") + if err := os.WriteFile(artifact, []byte("active cache"), 0o600); err != nil { + t.Fatal(err) + } + settings, store := newStorageMaintenanceStores(t) + entry := storagepkg.QuarantineEntry{ + ID: "missing-payload", + ResourceType: storagepkg.ResourceTypeGoCache, + OriginalPath: original, + QuarantinePath: quarantined, + SizeBytes: 42, + State: storagepkg.QuarantineStateQuarantined, + QuarantinedAt: time.Now().UTC().Add(-2 * time.Hour), + DeleteAfter: time.Now().UTC().Add(-time.Hour), + Metadata: json.RawMessage(`{"ownership":"managed"}`), + } + if err := store.CreateQuarantineEntry(context.Background(), &entry); err != nil { + t.Fatalf("create missing-payload entry: %v", err) + } + controller := &workspaceQuarantineController{settings: settings, store: store, homeDir: home} + + result, err := controller.Purge( + context.Background(), storagepkg.QuarantinePurgeScopeEligible, + storagepkg.QuarantineConfirmationEligible, + ) + if err != nil { + t.Fatalf("Purge: %v", err) + } + if result.Deleted != 1 || result.Failed != 0 || result.DeletedBytes != 0 { + t.Fatalf("purge result = %#v, want one deleted entry, zero failures, zero deleted bytes", result) + } + if data, err := os.ReadFile(artifact); err != nil || string(data) != "active cache" { + t.Fatalf("replacement cache changed: data=%q err=%v", data, err) + } + stored, err := store.GetQuarantineEntry(context.Background(), entry.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != storagepkg.QuarantineStateDeleted { + t.Fatalf("stored state = %q, want deleted", stored.State) + } +} + +func TestQuarantineControllerPurgeReportsBytesFromDeletionOutcome(t *testing.T) { + home := t.TempDir() + original := filepath.Join(home, "cache", "go-build") + quarantined := filepath.Join(home, "trash", "go-cache", "removed-before-delete") + if err := os.MkdirAll(original, 0o700); err != nil { + t.Fatal(err) + } + liveArtifact := filepath.Join(original, "replacement-artifact") + if err := os.WriteFile(liveArtifact, []byte("active cache"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(quarantined, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(quarantined, "old-artifact"), []byte("old cache"), 0o600); err != nil { + t.Fatal(err) + } + settings, store := newStorageMaintenanceStores(t) + entry := storagepkg.QuarantineEntry{ + ID: "removed-before-delete", + ResourceType: storagepkg.ResourceTypeGoCache, + OriginalPath: original, + QuarantinePath: quarantined, + SizeBytes: 42, + State: storagepkg.QuarantineStateQuarantined, + QuarantinedAt: time.Now().UTC().Add(-2 * time.Hour), + DeleteAfter: time.Now().UTC().Add(-time.Hour), + Metadata: json.RawMessage(`{"ownership":"managed"}`), + } + if err := store.CreateQuarantineEntry(context.Background(), &entry); err != nil { + t.Fatalf("create entry: %v", err) + } + deletingStore := &removePayloadOnGetStore{delegate: store, path: quarantined} + controller := &workspaceQuarantineController{settings: settings, store: deletingStore, homeDir: home} + + result, err := controller.Purge( + context.Background(), storagepkg.QuarantinePurgeScopeEligible, + storagepkg.QuarantineConfirmationEligible, + ) + if err != nil { + t.Fatalf("Purge: %v", err) + } + if result.Deleted != 1 || result.Failed != 0 || result.DeletedBytes != 0 { + t.Fatalf("purge result = %#v, want one deleted entry and zero deleted bytes", result) + } + if data, err := os.ReadFile(liveArtifact); err != nil || string(data) != "active cache" { + t.Fatalf("live cache changed: data=%q err=%v", data, err) + } +} + func TestQuarantineControllerDeletesHistoricalAdoptedGoCacheAfterReadoption(t *testing.T) { root := t.TempDir() home := filepath.Join(root, "home") diff --git a/apps/backend/internal/system/storage/gocache/provider.go b/apps/backend/internal/system/storage/gocache/provider.go index cc2eaecaeb..a24834d52b 100644 --- a/apps/backend/internal/system/storage/gocache/provider.go +++ b/apps/backend/internal/system/storage/gocache/provider.go @@ -64,6 +64,8 @@ type Analysis struct { // CleanupResult describes one cache rotation. type CleanupResult struct { Path string `json:"path"` + Skipped bool `json:"skipped"` + Reason string `json:"reason,omitempty"` BytesBefore int64 `json:"bytes_before"` BytesAfter int64 `json:"bytes_after"` ReclaimedBytes int64 `json:"reclaimed_bytes"` @@ -187,6 +189,13 @@ func (p *Provider) cleanup(ctx context.Context, explicit bool) (CleanupResult, e } entry, err := p.rotate(ctx, cachePath, result.BytesBefore, settings.QuarantineRetentionHours, adopted) if err != nil { + var activeErr *storage.ActiveQuarantineIntentError + if errors.As(err, &activeErr) { + result.BytesAfter = result.BytesBefore + result.Skipped = true + result.Reason = "active_quarantine" + return result, nil + } return result, err } result.QuarantineEntry = entry diff --git a/apps/backend/internal/system/storage/gocache/provider_test.go b/apps/backend/internal/system/storage/gocache/provider_test.go index b8c6427f1a..b494665263 100644 --- a/apps/backend/internal/system/storage/gocache/provider_test.go +++ b/apps/backend/internal/system/storage/gocache/provider_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sort" "testing" + "time" "github.com/kandev/kandev/internal/system/storage" ) @@ -33,7 +34,7 @@ func TestCleanupResultJSONUsesStorageAPISnakeCase(t *testing.T) { if err != nil { t.Fatalf("Marshal CleanupResult: %v", err) } - want := `{"path":"/cache","bytes_before":100,"bytes_after":20,"reclaimed_bytes":80,"quarantine_entry":null}` + want := `{"path":"/cache","skipped":false,"bytes_before":100,"bytes_after":20,"reclaimed_bytes":80,"quarantine_entry":null}` if string(encoded) != want { t.Fatalf("CleanupResult JSON = %s, want %s", encoded, want) } @@ -220,6 +221,69 @@ func TestCleanupRotatesOwnedCacheAboveThreshold(t *testing.T) { } } +func TestCleanupDefersWhenGoCacheQuarantineIsActive(t *testing.T) { + home := t.TempDir() + settings := storage.DefaultSettings() + settings.GoCache.Enabled = true + settings.GoCache.MaxBytes = 1 + store := &recordingStore{} + provider := New(Config{ + HomeDir: home, + TrashDir: filepath.Join(home, "trash"), + Settings: staticSettings{settings: settings}, + Store: store, + }) + env, err := provider.ExecutionEnvironment(context.Background()) + if err != nil { + t.Fatalf("ExecutionEnvironment() error = %v", err) + } + cachePath := env["GOCACHE"] + if err := os.WriteFile(filepath.Join(cachePath, "replacement-artifact"), []byte("live"), 0o600); err != nil { + t.Fatalf("seed replacement cache: %v", err) + } + retainedPath := filepath.Join(home, "trash", "go-cache", "retained") + if err := os.MkdirAll(retainedPath, 0o700); err != nil { + t.Fatalf("create retained cache: %v", err) + } + if err := os.WriteFile(filepath.Join(retainedPath, "retained-artifact"), []byte("old"), 0o600); err != nil { + t.Fatalf("seed retained cache: %v", err) + } + active := &storage.QuarantineEntry{ + ID: "retained", + ResourceType: storage.ResourceTypeGoCache, + OriginalPath: cachePath, + QuarantinePath: retainedPath, + SizeBytes: 3, + State: storage.QuarantineStateQuarantined, + QuarantinedAt: time.Now().UTC().Add(-time.Hour), + DeleteAfter: time.Now().UTC().Add(time.Hour), + Metadata: json.RawMessage(`{"ownership":"managed"}`), + } + if err := store.CreateQuarantineEntry(context.Background(), active); err != nil { + t.Fatalf("create active quarantine entry: %v", err) + } + + result, err := provider.Cleanup(context.Background()) + if err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + if result.BytesBefore != 4 || result.BytesAfter != 4 || result.ReclaimedBytes != 0 { + t.Fatalf("cleanup result = %#v, want unchanged four-byte cache and no reclamation", result) + } + if !result.Skipped || result.Reason != "active_quarantine" || result.QuarantineEntry != nil { + t.Fatalf("cleanup deferral fields = %#v, want active_quarantine without a new entry", result) + } + if _, err := os.Stat(filepath.Join(cachePath, "replacement-artifact")); err != nil { + t.Fatalf("replacement cache changed: %v", err) + } + if _, err := os.Stat(filepath.Join(retainedPath, "retained-artifact")); err != nil { + t.Fatalf("retained cache changed: %v", err) + } + if len(store.entries) != 1 || store.entries[active.ID].State != storage.QuarantineStateQuarantined { + t.Fatalf("active quarantine entries = %#v, want unchanged retained entry", store.entries) + } +} + func TestCleanupNeverClaimsUnmarkedManagedPath(t *testing.T) { home := t.TempDir() cachePath := filepath.Join(home, "cache", "go-build") diff --git a/apps/backend/internal/system/storage/quarantine_retry.go b/apps/backend/internal/system/storage/quarantine_retry.go index ae39f8379b..2abc07e803 100644 --- a/apps/backend/internal/system/storage/quarantine_retry.go +++ b/apps/backend/internal/system/storage/quarantine_retry.go @@ -20,6 +20,21 @@ type quarantineRetryStore interface { ) (QuarantineEntry, error) } +// ActiveQuarantineIntentError reports that a retained quarantine entry still +// owns the original path. It remains compatible with ErrConflict so callers +// that treat the condition as an error keep their existing behavior. +type ActiveQuarantineIntentError struct { + OriginalPath string +} + +func (e *ActiveQuarantineIntentError) Error() string { + return fmt.Sprintf("%s: quarantine intent for %s is still active", ErrConflict, e.OriginalPath) +} + +func (e *ActiveQuarantineIntentError) Unwrap() error { + return ErrConflict +} + // ReleaseFailedQuarantineIntent releases a failed intent only when filesystem // state proves its move did not occur. Ambiguous states remain active for // recovery or reconciliation. @@ -39,7 +54,7 @@ func ReleaseFailedQuarantineIntent( continue } if entry.State == QuarantineStateQuarantined { - return false, fmt.Errorf("%w: quarantine intent for %s is still active", ErrConflict, originalPath) + return false, &ActiveQuarantineIntentError{OriginalPath: originalPath} } if entry.State == QuarantineStateFailed { return releaseFailedIntentIfUnmoved(ctx, store, entry) diff --git a/apps/backend/internal/system/storage/store_test.go b/apps/backend/internal/system/storage/store_test.go index 9a70fb9fd0..9c017b7518 100644 --- a/apps/backend/internal/system/storage/store_test.go +++ b/apps/backend/internal/system/storage/store_test.go @@ -329,6 +329,33 @@ func TestReleaseFailedQuarantineIntentRequiresProvenUnmovedState(t *testing.T) { } } +func TestReleaseFailedQuarantineIntentClassifiesActiveIntent(t *testing.T) { + conn := newSQLite(t) + store := newStorageStore(t, db.NewPool(conn, conn)) + original := filepath.Join(t.TempDir(), "original") + entry := testQuarantineEntry("active-entry") + entry.ResourceType = ResourceTypeGoCache + entry.OriginalPath = original + entry.State = QuarantineStateQuarantined + if err := store.CreateQuarantineEntry(context.Background(), &entry); err != nil { + t.Fatalf("create active intent: %v", err) + } + + _, err := ReleaseFailedQuarantineIntent( + context.Background(), store, ResourceTypeGoCache, original, + ) + var activeErr *ActiveQuarantineIntentError + if !errors.As(err, &activeErr) { + t.Fatalf("ReleaseFailedQuarantineIntent error = %v, want ActiveQuarantineIntentError", err) + } + if !errors.Is(err, ErrConflict) { + t.Fatalf("ReleaseFailedQuarantineIntent error = %v, want ErrConflict", err) + } + if activeErr.OriginalPath != original { + t.Fatalf("active intent original path = %q, want %q", activeErr.OriginalPath, original) + } +} + func TestStoreAllowsEveryQuarantineStateTransition(t *testing.T) { tests := []struct { name string diff --git a/docs/plans/go-cache-quarantine-lifecycle/plan.md b/docs/plans/go-cache-quarantine-lifecycle/plan.md new file mode 100644 index 0000000000..ed933026b6 --- /dev/null +++ b/docs/plans/go-cache-quarantine-lifecycle/plan.md @@ -0,0 +1,159 @@ +--- +spec: docs/specs/system-page/storage-maintenance.md +created: 2026-08-12 +status: done +--- + +# Implementation Plan: Go-cache Quarantine Lifecycle Repair + +## Overview + +An adopted Go cache can exceed its threshold while an older generation remains in quarantine. +The current provider treats that normal retention state as a conflict and fails each maintenance run. +It also cannot close an entry when its recorded payload is already absent. + +The repair first makes the retained-generation condition a typed, successful deferral. It then makes +permanent deletion idempotent for a missing Go-cache payload without changing the live cache. The +final task updates the operator guide and records focused validation results. + +Confirmed root cause: `gocache.Provider.rotate` calls `ReleaseFailedQuarantineIntent` before each +rotation. That helper returns `ErrConflict` for every active `quarantined` entry. The provider +propagates the error even though retention intentionally keeps the entry active. The live incident +also has a missing `quarantine_path`, but `deleteGoCacheWithRetention` rejects that state before it +can make the durable entry terminal. + +--- + +## Backend + +### Typed cleanup deferral + +- Add a typed active-intent classification in + `apps/backend/internal/system/storage/quarantine_retry.go`. Preserve `errors.Is(err, + storage.ErrConflict)` and the current human-readable path context for callers that still treat the + condition as an error. +- Extend `gocache.CleanupResult` in + `apps/backend/internal/system/storage/gocache/provider.go` with `skipped` and `reason` fields. +- In `gocache.Provider.cleanup`, convert only the typed active-intent condition into a successful + result with `bytes_after == bytes_before`, zero reclaimed bytes, `skipped: true`, and + `reason: "active_quarantine"`. +- Keep failed or ambiguous quarantine intents on the existing fail-closed path. Do not weaken the + active-original unique index or allow more than one retained generation per cache path. + +### Missing-payload permanent deletion + +- Refactor Go-cache deletion in + `apps/backend/internal/backendapp/storage_maintenance.go` to distinguish an absent quarantine + payload from an unsafe or unreadable path. +- After the existing confirmation, retention, entry-state, ownership, and containment checks pass, + transition an absent payload to `deleted`. Do not inspect, remove, rename, or replace the original + cache path in this branch. +- Keep `restoreGoCache` on the existing ambiguous-state guard so that restore still fails closed when + the payload is absent. +- In `workspaceQuarantineController.Purge`, measure payload presence before deletion for result + accounting. Add zero to `deleted_bytes` when a successfully closed entry had no payload. Continue + to count the row in `deleted`. +- Do not add a schema migration or a new quarantine state. The existing `deleted` state accurately + records that no quarantined payload remains. + +--- + +## Frontend + +No frontend code change is required. `CleanupResult.Skipped` and `CleanupResult.Reason` extend the +`MaintenanceRun.Result` payload under `go_cache.result`; the existing run-result rendering accepts +those fields. Route shapes, UI actions, and confirmation flows remain unchanged, and the existing +**Delete**, **Clear eligible**, and **Force clear all** actions use the repaired backend paths. + +--- + +## Public Documentation + +- Update `docs/public/operations.md` in its how-to-oriented Storage maintenance section. +- Explain that one restorable Go-cache generation can defer later rotations without failing the run. +- Explain that permanent deletion can close a missing Go-cache payload without changing the live + replacement cache. Restore remains unavailable for missing payloads. + +--- + +## Tests + +- **What:** an above-threshold replacement cache with an active retained generation produces a + successful deferral and preserves both paths and the durable entry. + **File:** `apps/backend/internal/system/storage/gocache/provider_test.go`. + **How:** use the real provider with temporary cache and trash paths plus a recording quarantine + store. First prove that the regression test fails because cleanup returns `ErrConflict`. Then make + it pass with the exact result and filesystem assertions. +- **What:** the typed active-intent error remains compatible with `ErrConflict`, while failed-intent + retry and ambiguous-state behavior remain unchanged. + **Files:** `apps/backend/internal/system/storage/store_test.go` and + `apps/backend/internal/system/storage/gocache/provider_test.go`. + **How:** extend the existing table-driven retry tests and assert both classifications. +- **What:** eligible and forced permanent deletion close a missing Go-cache payload, preserve a + populated original cache, and report zero deleted bytes. + **File:** `apps/backend/internal/backendapp/storage_maintenance_test.go`. + **How:** use the production controller with a real temporary SQLite store and real filesystem + paths. Cover individual deletion and bulk purge. +- **What:** restore of the same missing-payload state still fails closed. + **File:** `apps/backend/internal/backendapp/storage_maintenance_test.go`. + **How:** retain and narrow the existing missing-payload restore regression test. +- **Integration boundary:** the backendapp tests exercise the production quarantine controller, + real `storage.Store`, SQLite state transitions, and filesystem effects. The existing + `MaintenanceRun.Result` envelope carries the result-field extension without route or UI action + changes, so a handler or frontend test is not required. + +--- + +## E2E Tests + +No E2E test is required. The repair changes backend lifecycle semantics without changing the UI, +route shapes, confirmation flows, or user interaction sequence. + +--- + +## Verification Results + +- Task 01: `cd apps/backend && go test ./internal/system/storage ./internal/system/storage/gocache` + passed (123 tests). +- Task 02: `cd apps/backend && go test ./internal/backendapp` passed (316 tests). +- Task 03: `node --test scripts/validate-public-docs.test.mjs && node scripts/validate-public-docs.mjs + && git diff --check` passed (60 validation tests and 41 published pages). + +--- + +## Implementation Waves And Parallel Candidates + +The default execution order is sequential in the primary conversation. These waves do not +authorize subagents. + +Wave 1: + +- [x] [Task 01: Defer retained Go-cache rotation](task-01-defer-retained-rotation.md) + +Wave 2: + +- [x] [Task 02: Close missing Go-cache payloads](task-02-close-missing-payload.md) + +Wave 3: + +- [x] [Task 03: Update operator documentation](task-03-update-operator-docs.md) + +Task 02 follows Task 01 so the primary conversation can integrate and record one lifecycle change +at a time. Task 03 documents both backend outcomes. No task is marked parallel-safe. + +## Risks + +- A broad conversion of `ErrConflict` into a skip can hide failed or ambiguous rename states. The + implementation must match only the new typed active-intent classification. +- Missing-payload deletion must never use the original cache as a fallback deletion target. +- Bulk result accounting must not claim that historical `size_bytes` were removed when the payload + was already absent. +- Restore must remain fail-closed because Kandev cannot prove that a populated original path is the + quarantined generation. + +## Out of Scope + +- Multiple simultaneous restorable generations for one Go-cache path. +- A new database state or migration for externally missing payloads. +- A new repair button, API route, or automatic deletion before the configured retention deadline. +- Changes to workspace or temporary-artifact restore semantics. diff --git a/docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md b/docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md new file mode 100644 index 0000000000..f2847554d4 --- /dev/null +++ b/docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md @@ -0,0 +1,79 @@ +--- +id: "01-defer-retained-rotation" +title: "Defer retained Go-cache rotation" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/system-page/storage-maintenance.md" +--- + +# Task 01: Defer Retained Go-cache Rotation + +## Intent + +Make a retained Go-cache generation a successful cleanup deferral instead of a maintenance error. +Keep all failed and ambiguous intent states fail-closed. + +## Acceptance + +- An active `quarantined` entry for the same Go-cache path returns a typed active-intent + classification that remains compatible with `storage.ErrConflict`. +- Cleanup returns `skipped: true`, `reason: "active_quarantine"`, unchanged before/after bytes, and + zero reclaimed bytes without changing either cache path or the active entry. +- Failed-intent retry and ambiguous filesystem behavior remain unchanged. + +## Files Likely Touched + +- `apps/backend/internal/system/storage/quarantine_retry.go` +- `apps/backend/internal/system/storage/store_test.go` +- `apps/backend/internal/system/storage/gocache/provider.go` +- `apps/backend/internal/system/storage/gocache/provider_test.go` +- `docs/plans/go-cache-quarantine-lifecycle/plan.md` +- `docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md` + +## Dependencies + +None. + +## Parallelism + +`sequential`. + +## Inputs + +- Spec sections: **Go build cache**, **Failure modes**, **Persistence guarantees**, and **Scenarios**. +- Plan section: **Typed cleanup deferral**. +- Existing patterns: `ReleaseFailedQuarantineIntent` and temporary-artifact `CleanupResult` skip + fields. + +## TDD Sequence + +1. Add the above-threshold retained-generation regression test. +2. Run `go test ./internal/system/storage/gocache -run + TestCleanupDefersWhenGoCacheQuarantineIsActive` and record the expected `ErrConflict` failure. +3. Add the typed classification and minimal cleanup-result conversion. +4. Run the targeted packages and record the passing result. + +## Verification + +```bash +cd apps/backend && go test ./internal/system/storage ./internal/system/storage/gocache +``` + +## Output Contract + +Report the files changed, the red and green test commands, the exact outcomes, remaining risks, and +the synchronized task and plan status. + +## Results + +- Red: `cd apps/backend && go test ./internal/system/storage/gocache -run + TestCleanupDefersWhenGoCacheQuarantineIsActive` failed because cleanup returned the existing + `storage.ErrConflict` for the active entry. +- Green: `cd apps/backend && go test ./internal/system/storage ./internal/system/storage/gocache` + passed (123 tests). +- Added `storage.ActiveQuarantineIntentError`, which preserves `errors.Is(err, storage.ErrConflict)` + and the existing path context. Go-cache cleanup now reports a successful `active_quarantine` + skip with unchanged byte counts and no new intent. Failed and ambiguous intent states remain + errors. diff --git a/docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md b/docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md new file mode 100644 index 0000000000..10f9d24630 --- /dev/null +++ b/docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md @@ -0,0 +1,78 @@ +--- +id: "02-close-missing-payload" +title: "Close missing Go-cache payloads" +status: done +wave: 2 +depends_on: ["01-defer-retained-rotation"] +plan: "plan.md" +spec: "../../specs/system-page/storage-maintenance.md" +--- + +# Task 02: Close Missing Go-cache Payloads + +## Intent + +Make permanent deletion idempotent when a Go-cache quarantine payload is already absent. Preserve +the live original path and keep restore fail-closed. + +## Acceptance + +- Eligible and forced permanent deletion transition a valid missing-payload entry to `deleted`. +- The deletion path does not inspect recursively, remove, rename, or replace a populated original + cache, and bulk purge reports zero deleted bytes for the absent payload. +- Restore of the same state returns `storage.ErrConflict` and leaves the entry active. + +## Files Likely Touched + +- `apps/backend/internal/backendapp/storage_maintenance.go` +- `apps/backend/internal/backendapp/storage_maintenance_test.go` +- `docs/plans/go-cache-quarantine-lifecycle/plan.md` +- `docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md` + +## Dependencies + +- Task 01: retained rotation behavior and result classification. + +## Parallelism + +`sequential`. + +## Inputs + +- Spec sections: **Go build cache**, **Failure modes**, and **Scenarios**. +- Plan section: **Missing-payload permanent deletion**. +- Existing tests: + `TestQuarantineControllerDoesNotTreatPopulatedReplacementAsRestoredPayload` and + `TestQuarantineControllerDoesNotMarkMissingPayloadPlaceholderRestored`. + +## TDD Sequence + +1. Split the existing combined restore/delete assertion and add eligible, forced, and bulk + missing-payload deletion cases. +2. Run `go test ./internal/backendapp -run 'TestQuarantineController.*MissingGoCachePayload'` and + record the expected conflict failures. +3. Refactor the delete path without changing restore behavior. +4. Run the backendapp package and record the passing result. + +## Verification + +```bash +cd apps/backend && go test ./internal/backendapp +``` + +## Output Contract + +Report the files changed, the red and green test commands, the exact outcomes, filesystem and +database assertions, remaining risks, and the synchronized task and plan status. + +## Results + +- Red: `cd apps/backend && go test ./internal/backendapp -run + 'TestQuarantineController.*MissingGoCachePayload'` failed for the eligible, forced, and bulk + missing-payload cases because deletion returned `storage.ErrConflict`. +- Green: the same focused command passed (4 tests), and `cd apps/backend && go test + ./internal/backendapp` passed (316 tests). +- Permanent deletion now closes a validated missing Go-cache payload without reading or changing the + original cache path. Restore continues through the existing fail-closed missing-payload guard. + Purge uses the deletion-time payload outcome and reports zero deleted bytes when the durable entry + was already missing on disk, including a payload disappearing before the deletion check. diff --git a/docs/plans/go-cache-quarantine-lifecycle/task-03-update-operator-docs.md b/docs/plans/go-cache-quarantine-lifecycle/task-03-update-operator-docs.md new file mode 100644 index 0000000000..2192fbc455 --- /dev/null +++ b/docs/plans/go-cache-quarantine-lifecycle/task-03-update-operator-docs.md @@ -0,0 +1,66 @@ +--- +id: "03-update-operator-docs" +title: "Update operator documentation" +status: done +wave: 3 +depends_on: ["01-defer-retained-rotation", "02-close-missing-payload"] +plan: "plan.md" +spec: "../../specs/system-page/storage-maintenance.md" +--- + +# Task 03: Update Operator Documentation + +## Intent + +Document the retained-generation and missing-payload behavior in the existing Storage maintenance +how-to guide. + +## Acceptance + +- The guide states that one active Go-cache quarantine generation defers another rotation without + failing maintenance. +- The guide states that permanent deletion can close an absent payload without changing the live + cache, while restore remains fail-closed. +- Public-document validation passes, and the plan records all task results. + +## Files Likely Touched + +- `docs/public/operations.md` +- `docs/plans/go-cache-quarantine-lifecycle/plan.md` +- `docs/plans/go-cache-quarantine-lifecycle/task-01-defer-retained-rotation.md` +- `docs/plans/go-cache-quarantine-lifecycle/task-02-close-missing-payload.md` +- `docs/plans/go-cache-quarantine-lifecycle/task-03-update-operator-docs.md` + +## Dependencies + +- Task 01: final deferral result fields and semantics. +- Task 02: final deletion and byte-accounting semantics. + +## Parallelism + +`sequential`. + +## Inputs + +- Spec sections: **Go build cache**, **Failure modes**, and **Scenarios**. +- Plan section: **Public Documentation**. +- Public page type: how-to guide. + +## Verification + +```bash +node --test scripts/validate-public-docs.test.mjs && node scripts/validate-public-docs.mjs && git diff --check +``` + +## Output Contract + +Report the documentation changes, exact validation outcomes, public-doc type, cleanup evidence, +remaining risks, and synchronized task and plan status. + +## Results + +- Updated `docs/public/operations.md` with the one-active-generation deferral result and the + missing-payload deletion, zero-byte accounting, and fail-closed restore behavior. +- Public page type: how-to guide. +- `node --test scripts/validate-public-docs.test.mjs && node scripts/validate-public-docs.mjs && + git diff --check` passed (60 validation tests and 41 published pages). diff --git a/docs/public/operations.md b/docs/public/operations.md index 78505ea5ab..ee82108756 100644 --- a/docs/public/operations.md +++ b/docs/public/operations.md @@ -155,6 +155,17 @@ remove every active quarantine entry, discarding restore windows for entries tha deleted. Safety-validation or deletion failures may leave entries visible and retryable. This override bypasses only the retention timestamp; path, ownership, state, and filesystem safety checks still apply. + +Kandev keeps at most one restorable Go-cache generation for each original cache path. If that +generation is still active when the replacement cache exceeds its limit, the next rotation is +deferred. The maintenance run succeeds and reports `active_quarantine`; both the live cache and +the retained generation stay unchanged. + +If a Go-cache quarantine payload is already missing, **Delete**, **Clear eligible**, or **Force +clear all** can close its durable entry without changing the live replacement cache. The purge +reports zero deleted bytes for that entry. **Restore** remains unavailable because Kandev cannot +prove which cache generation is currently live. + If scheduled cleanup is disabled, no independent quarantine sweeper runs: use a full **Run now** or one of the quarantine actions when you want cleanup. diff --git a/docs/specs/system-page/storage-maintenance.md b/docs/specs/system-page/storage-maintenance.md index 81ffcb24e7..4682b8e9b9 100644 --- a/docs/specs/system-page/storage-maintenance.md +++ b/docs/specs/system-page/storage-maintenance.md @@ -1,7 +1,7 @@ --- status: building created: 2026-07-14 -updated: 2026-08-08 +updated: 2026-08-12 owner: cfl --- @@ -204,6 +204,14 @@ Retention override: owned cache into Kandev trash and recreates an empty cache when its size is greater than the configured maximum. The limit is a cleanup trigger, not a hard quota; the cache can temporarily grow beyond it while tasks are active. +- Only one restorable Go-cache generation can remain active for an original path. If the replacement + cache exceeds its maximum before that generation becomes terminal, cleanup reports a successful + no-op with `skipped: true` and `reason: "active_quarantine"`. It does not create another intent, + change either cache path, or fail the complete maintenance run. +- Permanent deletion treats an absent Go-cache quarantine payload as already removed after all + confirmation, retention, ownership, containment, and state checks succeed. It marks the durable + entry `deleted`, does not change a populated replacement cache, and reports zero deleted bytes for + the absent payload. Restore continues to fail closed when the payload is absent. - Disabling managed Go cache stops injecting `GOCACHE` into new executions. It does not delete the previously managed cache. Scheduled cleanup and a global manual run with no resource selection leave it untouched; only a manual run whose non-empty selection includes `go_cache` may rotate it. @@ -600,6 +608,10 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore warning, and does not run destructive maintenance. - A managed Go-cache cleanup failure leaves either the original cache or its quarantined rename intact; it never recursively deletes outside the configured owned path. +- An active Go-cache quarantine generation defers another rotation without failing maintenance. + The active entry remains available for restore or permanent deletion. +- An absent Go-cache quarantine payload cannot be restored. Permanent deletion can close its durable + entry without deleting or replacing data at the original cache path. - A temporary-artifact registry read, marker read, lease/heartbeat check, or ownership/path validation failure keeps that root in place and records a skipped warning. There is no fallback to prefix, mtime, or process-name classification. @@ -639,6 +651,8 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore cleanup and permanent quarantine deletion do not cascade-delete that history. - Quarantined data remains restorable until permanent deletion succeeds. A failed permanent delete remains visible and retryable. +- A retained Go-cache generation keeps its unique active-path slot until restore or permanent + deletion makes the entry terminal. A deferred cleanup does not change that entry or its deadline. - Temporary-artifact ownership rows and matching markers survive backend restarts. A producer's normal close remains idempotent; an interrupted active row is reconciled and can become eligible only after the stale interval. @@ -736,6 +750,14 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore - **GIVEN** managed Go cache is enabled and is 20 GiB with a 15 GiB threshold, **WHEN** an idle cleanup runs, **THEN** Kandev rotates the owned cache to trash, recreates an empty cache, and reports the reclaimed bytes. +- **GIVEN** an active Go-cache quarantine entry and a replacement cache above its threshold, + **WHEN** cleanup runs, **THEN** the Go-cache result reports `skipped: true` with + `reason: "active_quarantine"`, reclaims zero bytes, and the maintenance run does not fail. +- **GIVEN** an active Go-cache entry whose quarantine payload is absent and whose original path + contains a replacement cache, **WHEN** eligible or forced permanent deletion runs, **THEN** the + entry becomes `deleted`, the replacement cache remains unchanged, and deleted bytes are zero. +- **GIVEN** an active Go-cache entry whose quarantine payload is absent, **WHEN** restore runs, + **THEN** restore fails closed and the entry remains active. - **GIVEN** `/root/.cache/go-build` was not explicitly adopted, **WHEN** storage cleanup runs, **THEN** Kandev does not modify it. - **GIVEN** an external Go cache was adopted successfully, **WHEN** the Storage page rerenders or is @@ -819,3 +841,4 @@ distinct `DELETE ALL NOW` confirmation because it removes the configured restore - [Storage overview cache and settings follow-up](../../plans/storage-overview-cache/plan.md) - [Quarantine lifecycle follow-up](../../plans/quarantine-lifecycle/plan.md) - [Progressive Storage loading and totals](../../plans/storage-progressive-loading/plan.md) +- [Go-cache quarantine lifecycle repair](../../plans/go-cache-quarantine-lifecycle/plan.md)