Files
kandev/apps/backend/internal/worktree/branch_cleanup_interruption_test.go
T
2026-09-16 23:20:37 +01:00

499 lines
20 KiB
Go

package worktree
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"github.com/kandev/kandev/internal/task/models"
)
type cancelAfterArchivedRecoveryStore struct {
*SQLiteStore
cancel context.CancelFunc
once sync.Once
}
type failOnceArchivedCompactionCompleteStore struct {
*SQLiteStore
failed bool
}
type failOnceBranchRecoveryRestoredStore struct {
*SQLiteStore
failed bool
}
type panicArchivedCandidateStore struct {
*SQLiteStore
}
func (s *panicArchivedCandidateStore) IsArchivedBranchCandidate(context.Context, string) (bool, error) {
panic("injected archived candidate panic")
}
func (s *failOnceArchivedCompactionCompleteStore) PersistArchivedBranchCompactionComplete(
ctx context.Context, worktreeID, expectedRecoveryHead string,
) (bool, error) {
if !s.failed {
s.failed = true
return false, errors.New("injected compaction completion failure")
}
return s.SQLiteStore.PersistArchivedBranchCompactionComplete(ctx, worktreeID, expectedRecoveryHead)
}
func (s *failOnceBranchRecoveryRestoredStore) PersistBranchRecoveryRestored(
ctx context.Context, worktreeID, expectedRecoveryHead string,
) (bool, error) {
if !s.failed {
s.failed = true
return false, errors.New("injected recovery finalization failure")
}
return s.SQLiteStore.PersistBranchRecoveryRestored(ctx, worktreeID, expectedRecoveryHead)
}
func (s *cancelAfterArchivedRecoveryStore) PersistArchivedBranchRecoveryHead(
ctx context.Context, worktreeID, expected, recoveryHead string,
) (bool, error) {
persisted, err := s.SQLiteStore.PersistArchivedBranchRecoveryHead(
ctx, worktreeID, expected, recoveryHead,
)
if err == nil && persisted {
s.once.Do(s.cancel)
}
return persisted, err
}
func TestMaintainArchivedBranches_RetriesAfterRecoveryPersistInterruption(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "interrupted")
interruptedCtx, cancel := context.WithCancel(context.Background())
mgr.store = &cancelAfterArchivedRecoveryStore{SQLiteStore: store, cancel: cancel}
firstReceipt, firstErr := mgr.MaintainArchivedBranches(interruptedCtx, 1)
if firstErr == nil {
t.Fatal("interrupted maintenance error = nil, want context cancellation")
}
if firstReceipt.Deleted != 0 {
t.Fatalf("interrupted maintenance receipt = %+v, want no deletion", firstReceipt)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", wt.Branch)); got != wantHead {
t.Fatalf("branch head after interruption = %q, want %q", got, wantHead)
}
mgr.store = store
secondReceipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil {
t.Fatalf("retry archived branch maintenance: %v", err)
}
if secondReceipt.Attempted != 1 || secondReceipt.Deleted != 1 {
t.Fatalf("retry receipt = %+v, want one attempted deletion", secondReceipt)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "branch", "--list", wt.Branch)); got != "" {
t.Fatalf("integrated archived branch remains after retry: %q", got)
}
thirdReceipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil {
t.Fatalf("maintenance after completed compaction: %v", err)
}
if thirdReceipt.Attempted != 0 {
t.Fatalf("completed branch was selected again: %+v", thirdReceipt)
}
}
func TestMaintainArchivedBranches_KeepsCompactedRecoveryReachableThroughGC(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "gc-recovery")
receipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil || receipt.Deleted != 1 {
t.Fatalf("compact archived branch: receipt=%+v err=%v", receipt, err)
}
runGit(t, wt.RepositoryPath, "update-ref", "refs/heads/main", strings.Repeat("0", 40))
runGit(t, wt.RepositoryPath, "reflog", "expire", "--expire=now", "--all")
runGit(t, wt.RepositoryPath, "gc", "--prune=now")
persisted, err := store.GetWorktreeByID(context.Background(), wt.ID)
if err != nil {
t.Fatalf("load compacted recovery metadata: %v", err)
}
if err := mgr.restoreManagedBranchFromRecoveryHeadLocked(context.Background(), persisted); err != nil {
t.Fatalf("restore after integration ref rewrite and gc: %v", err)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", "refs/heads/"+wt.Branch)); got != wantHead {
t.Fatalf("restored head = %q, want %q", got, wantHead)
}
if output, err := exec.Command("git", "-C", wt.RepositoryPath, "show-ref", "--verify", "--quiet", recoveryRefName(wt.ID)).CombinedOutput(); err == nil {
t.Fatalf("recovery ref remains after successful restoration: %s", output)
}
if persisted, err := store.GetWorktreeByID(context.Background(), wt.ID); err != nil || persisted.BranchCompactedAt != nil {
t.Fatalf("restored compaction state = %+v err=%v, want uncompacted", persisted, err)
}
}
func TestMaintainArchivedBranches_RearchivesRestoredBranchWithoutSessionLaunch(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "rearchive-restored")
first, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil || first.Deleted != 1 {
t.Fatalf("initial compaction: receipt=%+v err=%v", first, err)
}
persisted, err := store.GetWorktreeByID(context.Background(), wt.ID)
if err != nil {
t.Fatalf("load compacted branch: %v", err)
}
if _, err := store.db.ExecContext(context.Background(), `UPDATE tasks SET archived_at = NULL WHERE id = ?`, wt.TaskID); err != nil {
t.Fatalf("unarchive task: %v", err)
}
if status := mgr.RecoverBranchStatus(context.Background(), persisted); status != BranchStatusLocal {
t.Fatalf("unarchive recovery status = %q, want local", status)
}
persisted, err = store.GetWorktreeByID(context.Background(), wt.ID)
if err != nil || persisted.BranchCompactedAt != nil || persisted.RecoveryHeadSHA != "" {
t.Fatalf("restored metadata = %+v err=%v, want uncompacted", persisted, err)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", "refs/heads/"+wt.Branch)); got != wantHead {
t.Fatalf("restored branch head = %q, want %q", got, wantHead)
}
if _, err := store.db.ExecContext(context.Background(), `UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = ?`, wt.TaskID); err != nil {
t.Fatalf("rearchive task: %v", err)
}
second, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil || second.Attempted != 1 || second.Deleted != 1 {
t.Fatalf("rearchive maintenance: receipt=%+v err=%v", second, err)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "branch", "--list", wt.Branch)); got != "" {
t.Fatalf("rearchived integrated branch remains: %q", got)
}
}
func TestRecoverBranchStatus_FinalizesInterruptedPreDeleteRecovery(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "recover-before-delete")
ctx := context.Background()
if persisted, err := store.PersistArchivedBranchRecoveryHead(ctx, wt.ID, "", wantHead); err != nil || !persisted {
t.Fatalf("persist recovery head: persisted=%v err=%v", persisted, err)
}
if reason := mgr.createRecoveryRef(ctx, wt, wantHead); reason != "" {
t.Fatalf("create recovery ref: %s", reason)
}
if _, err := store.db.ExecContext(ctx, `UPDATE tasks SET archived_at = NULL WHERE id = ?`, wt.TaskID); err != nil {
t.Fatalf("unarchive task: %v", err)
}
persisted, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load interrupted recovery: %v", err)
}
if persisted.BranchCompactedAt != nil {
t.Fatalf("interrupted recovery has compaction marker: %v", persisted.BranchCompactedAt)
}
if status := mgr.RecoverBranchStatus(ctx, persisted); status != BranchStatusLocal {
t.Fatalf("recovery status = %q, want local", status)
}
restored, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load finalized recovery: %v", err)
}
if restored.RecoveryHeadSHA != "" || restored.BranchCompactedAt != nil {
t.Fatalf("finalized recovery metadata = head %q compacted %v, want cleared",
restored.RecoveryHeadSHA, restored.BranchCompactedAt)
}
if exists, err := mgr.localBranchRefExists(ctx, wt.RepositoryPath, recoveryRefName(wt.ID)); err != nil || exists {
t.Fatalf("recovery ref after finalization: exists=%v err=%v, want absent", exists, err)
}
}
func TestRecoverBranchStatus_RetainsMetadataWhenRecoveryRefDeleteFails(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "recover-delete-failure")
ctx := context.Background()
receipt, err := mgr.MaintainArchivedBranches(ctx, 1)
if err != nil || receipt.Deleted != 1 {
t.Fatalf("compact archived branch: receipt=%+v err=%v", receipt, err)
}
persisted, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load compacted branch: %v", err)
}
zeroOID := strings.Repeat("0", len(wantHead))
runGit(t, wt.RepositoryPath, "update-ref", "refs/heads/"+wt.Branch, wantHead, zeroOID)
rejectRecoveryRefDelete(t, wt)
if status := mgr.RecoverBranchStatus(ctx, persisted); status != BranchStatusMissing {
t.Fatalf("recovery status = %q, want missing when cleanup is incomplete", status)
}
retained, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load retained recovery metadata: %v", err)
}
if retained.RecoveryHeadSHA != wantHead || retained.BranchCompactedAt == nil {
t.Fatalf("retained recovery metadata = head %q compacted %v, want head %q and marker",
retained.RecoveryHeadSHA, retained.BranchCompactedAt, wantHead)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", recoveryRefName(wt.ID))); got != wantHead {
t.Fatalf("recovery ref head = %q, want %q", got, wantHead)
}
}
// Reviewer-requested contract coverage: a failure after the recovery ref is
// removed must leave the local branch and durable metadata retryable.
func TestRecoverBranchStatus_RetriesAfterRecoveryMetadataClearFails(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "recover-finalize-failure")
ctx := context.Background()
receipt, err := mgr.MaintainArchivedBranches(ctx, 1)
if err != nil || receipt.Deleted != 1 {
t.Fatalf("compact archived branch: receipt=%+v err=%v", receipt, err)
}
persisted, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load compacted branch: %v", err)
}
mgr.store = &failOnceBranchRecoveryRestoredStore{SQLiteStore: store}
if status := mgr.RecoverBranchStatus(ctx, persisted); status != BranchStatusLocal {
t.Fatalf("recovery status after metadata failure = %q, want local", status)
}
interrupted, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load interrupted recovery: %v", err)
}
if interrupted.RecoveryHeadSHA != wantHead || interrupted.BranchCompactedAt == nil {
t.Fatalf("interrupted recovery metadata = head %q compacted %v, want head %q and marker",
interrupted.RecoveryHeadSHA, interrupted.BranchCompactedAt, wantHead)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", "refs/heads/"+wt.Branch)); got != wantHead {
t.Fatalf("restored branch head = %q, want %q", got, wantHead)
}
if exists, err := mgr.localBranchRefExists(ctx, wt.RepositoryPath, recoveryRefName(wt.ID)); err != nil || exists {
t.Fatalf("recovery ref after interrupted finalization: exists=%v err=%v, want absent", exists, err)
}
mgr.store = store
if status := mgr.RecoverBranchStatus(ctx, interrupted); status != BranchStatusLocal {
t.Fatalf("recovery status after retry = %q, want local", status)
}
finalized, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load finalized recovery: %v", err)
}
if finalized.RecoveryHeadSHA != "" || finalized.BranchCompactedAt != nil {
t.Fatalf("finalized recovery metadata = head %q compacted %v, want cleared",
finalized.RecoveryHeadSHA, finalized.BranchCompactedAt)
}
}
// Reviewer-requested contract coverage: production already reconciles an
// absent ref whose recovery SHA persisted before its completion marker.
func TestMaintainArchivedBranches_FinalizesAbsentRefAfterCompletionPersistFailure(t *testing.T) {
mgr, store, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "completion-interrupted")
mgr.store = &failOnceArchivedCompactionCompleteStore{SQLiteStore: store}
firstReceipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil {
t.Fatalf("maintenance with completion persistence failure: %v", err)
}
if firstReceipt.Attempted != 1 || firstReceipt.Deleted != 1 {
t.Fatalf("first receipt = %+v, want completed ref deletion", firstReceipt)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "branch", "--list", wt.Branch)); got != "" {
t.Fatalf("branch remains after exact-ref deletion: %q", got)
}
persisted, err := store.GetWorktreeByID(context.Background(), wt.ID)
if err != nil {
t.Fatalf("load interrupted compaction: %v", err)
}
if persisted.RecoveryHeadSHA != wantHead || persisted.BranchCompactedAt != nil {
t.Fatalf("interrupted metadata = head %q compacted %v, want head %q without marker",
persisted.RecoveryHeadSHA, persisted.BranchCompactedAt, wantHead)
}
rejectBranchDeleteDuringRetry(t, wt)
mgr.store = store
secondReceipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil {
t.Fatalf("finalize interrupted compaction: %v", err)
}
if secondReceipt.Attempted != 1 || secondReceipt.Deleted != 1 {
t.Fatalf("finalization receipt = %+v, want one finalized candidate", secondReceipt)
}
persisted, err = store.GetWorktreeByID(context.Background(), wt.ID)
if err != nil {
t.Fatalf("load finalized compaction: %v", err)
}
if persisted.RecoveryHeadSHA != wantHead || persisted.BranchCompactedAt == nil {
t.Fatalf("finalized metadata = head %q compacted %v, want preserved head %q and marker",
persisted.RecoveryHeadSHA, persisted.BranchCompactedAt, wantHead)
}
thirdReceipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil {
t.Fatalf("maintenance after finalization: %v", err)
}
if thirdReceipt.Attempted != 0 {
t.Fatalf("finalized candidate selected again: %+v", thirdReceipt)
}
}
func TestMaintainArchivedBranches_RetainsAdvancedRefAfterInterruptedCompaction(t *testing.T) {
mgr, store, wt, recoveryHead := archivedIntegratedBranchForMaintenance(t, "advanced-ref")
ctx := context.Background()
if persisted, err := store.PersistArchivedBranchRecoveryHead(ctx, wt.ID, "", recoveryHead); err != nil || !persisted {
t.Fatalf("persist recovery head: persisted=%v err=%v", persisted, err)
}
runGit(t, wt.RepositoryPath, "checkout", "main")
runGit(t, wt.RepositoryPath, "commit", "--allow-empty", "-m", "advance integration")
advancedHead := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", "HEAD"))
runGit(t, wt.RepositoryPath, "update-ref", "refs/heads/"+wt.Branch, advancedHead)
receipt, err := mgr.MaintainArchivedBranches(ctx, 1)
if err != nil {
t.Fatalf("maintenance: %v", err)
}
if receipt.Deleted != 0 || receipt.RetainedReasons[RetainedHeadChanged] != 1 {
t.Fatalf("advanced-ref receipt = %+v", receipt)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", wt.Branch)); got != advancedHead {
t.Fatalf("advanced branch head = %q, want %q", got, advancedHead)
}
persisted, err := store.GetWorktreeByID(ctx, wt.ID)
if err != nil {
t.Fatalf("load recovery metadata: %v", err)
}
if persisted.RecoveryHeadSHA != recoveryHead {
t.Fatalf("recovery head overwritten = %q, want %q", persisted.RecoveryHeadSHA, recoveryHead)
}
}
func TestMaintainArchivedBranches_ReleasesRepoLockAfterPanic(t *testing.T) {
mgr, store, wt, _ := archivedIntegratedBranchForMaintenance(t, "panic-lock")
mgr.store = &panicArchivedCandidateStore{SQLiteStore: store}
var recovered any
func() {
defer func() { recovered = recover() }()
_, _ = mgr.MaintainArchivedBranches(context.Background(), 1)
}()
if recovered == nil {
t.Fatal("maintenance panic was not observed")
}
repoLock := mgr.getRepoLock(wt.RepositoryPath)
if !repoLock.TryLock() {
t.Fatal("repository lock remained held after maintenance panic")
}
repoLock.Unlock()
mgr.releaseRepoLock(wt.RepositoryPath)
}
func rejectBranchDeleteDuringRetry(t *testing.T, wt *Worktree) {
t.Helper()
realGit, err := exec.LookPath("git")
if err != nil {
t.Fatalf("find git: %v", err)
}
binDir := t.TempDir()
wrapper := filepath.Join(binDir, "git")
branchRef := "refs/heads/" + wt.Branch
script := "#!/bin/sh\n" +
"case \" $* \" in\n" +
" *\" update-ref -d " + branchRef + " \"*) exit 97 ;;\n" +
"esac\n" +
"exec \"" + realGit + "\" \"$@\"\n"
if err := os.WriteFile(wrapper, []byte(script), 0o755); err != nil {
t.Fatalf("write git wrapper: %v", err)
}
t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
}
func rejectRecoveryRefDelete(t *testing.T, wt *Worktree) {
t.Helper()
realGit, err := exec.LookPath("git")
if err != nil {
t.Fatalf("find git: %v", err)
}
binDir := t.TempDir()
wrapper := filepath.Join(binDir, "git")
recoveryRef := recoveryRefName(wt.ID)
script := "#!/bin/sh\n" +
"case \" $* \" in\n" +
" *\" update-ref -d " + recoveryRef + " \"*) exit 97 ;;\n" +
"esac\n" +
"exec \"" + realGit + "\" \"$@\"\n"
if err := os.WriteFile(wrapper, []byte(script), 0o755); err != nil {
t.Fatalf("write git wrapper: %v", err)
}
t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
}
func TestMaintainArchivedBranches_RestoresRefWhenBranchBecomesLiveDuringDelete(t *testing.T) {
mgr, _, wt, wantHead := archivedIntegratedBranchForMaintenance(t, "live-race")
realGit, err := exec.LookPath("git")
if err != nil {
t.Fatalf("find git: %v", err)
}
binDir := t.TempDir()
racePath := filepath.Join(t.TempDir(), "racing-worktree")
wrapper := filepath.Join(binDir, "git")
branchRef := "refs/heads/" + wt.Branch
script := "#!/bin/sh\n" +
"case \" $* \" in\n" +
" *\" update-ref -d " + branchRef + " \"*)\n" +
" \"" + realGit + "\" -C \"" + wt.RepositoryPath + "\" worktree add \"" + racePath + "\" \"" + wt.Branch + "\" >/dev/null\n" +
" ;;\n" +
"esac\n" +
"exec \"" + realGit + "\" \"$@\"\n"
if err := os.WriteFile(wrapper, []byte(script), 0o755); err != nil {
t.Fatalf("write git wrapper: %v", err)
}
t.Cleanup(func() {
_ = exec.Command(realGit, "-C", wt.RepositoryPath, "worktree", "remove", "--force", racePath).Run()
})
t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
receipt, err := mgr.MaintainArchivedBranches(context.Background(), 1)
if err != nil {
t.Fatalf("maintenance with liveness race: %v", err)
}
if receipt.Deleted != 0 || receipt.RetainedReasons[RetainedLiveWorktree] != 1 {
t.Fatalf("liveness-race receipt = %+v, want live-worktree retention", receipt)
}
if got := strings.TrimSpace(runGit(t, wt.RepositoryPath, "rev-parse", branchRef)); got != wantHead {
t.Fatalf("branch head after liveness race = %q, want restored %q", got, wantHead)
}
}
func archivedIntegratedBranchForMaintenance(
t *testing.T, suffix string,
) (*Manager, *SQLiteStore, *Worktree, string) {
t.Helper()
mgr, store := newReferenceCleanupTestManager(t)
ctx := context.Background()
taskID := "task-maintenance-" + suffix
sessionID := "session-maintenance-" + suffix
seedReferenceCleanupSession(t, store, taskID, sessionID, models.TaskSessionStateCompleted)
wt := createReferenceCleanupWorktree(t, mgr, taskID, sessionID)
if _, err := store.db.ExecContext(ctx, `
INSERT INTO repositories (id, workspace_id, name, local_path, created_at, updated_at)
VALUES (?, 'workspace', 'repository', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
`, wt.RepositoryID, wt.RepositoryPath); err != nil {
t.Fatalf("persist repository path: %v", err)
}
if _, err := store.db.ExecContext(ctx, `UPDATE task_sessions SET base_branch = ? WHERE id = ?`, wt.BaseBranch, wt.SessionID); err != nil {
t.Fatalf("persist session base branch: %v", err)
}
runGit(t, wt.Path, "commit", "--allow-empty", "-m", "archive before integration")
wantHead := strings.TrimSpace(runGit(t, wt.Path, "rev-parse", "HEAD"))
if _, err := store.db.ExecContext(ctx, `UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = ?`, wt.TaskID); err != nil {
t.Fatalf("archive task: %v", err)
}
archiveReceipt, err := mgr.CleanupWorktreesWithReceipt(ctx, []*Worktree{wt})
if err != nil {
t.Fatalf("archive cleanup: %v", err)
}
if archiveReceipt.RetainedReasons[RetainedNotIntegrated] != 1 {
t.Fatalf("archive receipt = %+v, want not-integrated retention", archiveReceipt)
}
runGit(t, wt.RepositoryPath, "update-ref", "refs/heads/main", wantHead)
return mgr, store, wt, wantHead
}