fix: stop workspace tracker after consecutive git failures (#514)

This commit is contained in:
Carlos Florêncio
2026-04-01 22:54:21 +01:00
committed by GitHub
parent 5fb1c12364
commit d7b8fdce60
4 changed files with 96 additions and 6 deletions
@@ -32,6 +32,8 @@ func (wt *WorkspaceTracker) pollGitChanges(ctx context.Context) {
zap.String("initial_head", wt.cachedHeadSHA),
zap.String("initial_branch", wt.cachedBranchName))
var consecutiveFailures int
for {
select {
case <-ctx.Done():
@@ -45,6 +47,25 @@ func (wt *WorkspaceTracker) pollGitChanges(ctx context.Context) {
zap.String("workDir", wt.workDir))
return
}
// Quick git health check before running full change detection.
// If git is broken (e.g., worktree .git reference points to deleted gitdir),
// stop after maxConsecutiveGitFailures to avoid wasting CPU.
probeCmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-dir")
probeCmd.Dir = wt.workDir
if err := probeCmd.Run(); err != nil {
consecutiveFailures++
if consecutiveFailures >= maxConsecutiveGitFailures {
wt.logger.Error("git not functional, stopping git polling",
zap.String("workDir", wt.workDir),
zap.Int("consecutiveFailures", consecutiveFailures),
zap.Error(err))
return
}
continue
}
consecutiveFailures = 0
wt.checkGitChanges(ctx)
}
}
@@ -1,6 +1,7 @@
package process
import (
"bytes"
"context"
"fmt"
"os"
@@ -22,12 +23,19 @@ const DefaultFilePollInterval = 2 * time.Second
// This prevents the monitor loop from hanging if git is blocked (e.g., index lock).
const gitCommandTimeout = 10 * time.Second
// maxConsecutiveGitFailures is the number of consecutive git command failures
// before the polling loop gives up and stops. At 2-second intervals, 5 failures
// means ~10 seconds — enough to rule out transient issues (index lock, brief GC race)
// while stopping quickly enough to avoid log spam when git is permanently broken
// (e.g., worktree .git reference points to a deleted gitdir after task archival).
const maxConsecutiveGitFailures = 5
// monitorLoop polls for file changes using file mtimes and quick git checks.
// This is more efficient than fsnotify on macOS and avoids file descriptor exhaustion.
func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) {
defer wt.wg.Done()
ticker := time.NewTicker(DefaultFilePollInterval)
ticker := time.NewTicker(wt.filePollInterval)
defer ticker.Stop()
// Initial update
@@ -37,7 +45,9 @@ func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) {
// Cache the last known state (ignore error on initial fetch)
lastState, _ := wt.getWorkspaceState(ctx)
wt.logger.Info("file polling started", zap.Duration("interval", DefaultFilePollInterval))
wt.logger.Info("file polling started", zap.Duration("interval", wt.filePollInterval))
var consecutiveFailures int
for {
select {
@@ -60,10 +70,21 @@ func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) {
// Git command failed for another reason - skip this cycle and retry on next tick.
// Don't update lastState so the change will be detected when git recovers.
consecutiveFailures++
if consecutiveFailures >= maxConsecutiveGitFailures {
wt.logger.Error("git commands failing repeatedly, stopping workspace monitor",
zap.String("workDir", wt.workDir),
zap.Int("consecutiveFailures", consecutiveFailures),
zap.Error(err))
return
}
wt.logger.Warn("failed to get workspace state, will retry next cycle",
zap.String("workDir", wt.workDir),
zap.Int("consecutiveFailures", consecutiveFailures),
zap.Error(err))
continue
}
consecutiveFailures = 0
if currentState.changed(lastState) {
lastState = currentState
wt.logger.Debug("workspace state changed, updating")
@@ -112,9 +133,12 @@ func (wt *WorkspaceTracker) getWorkspaceState(ctx context.Context) (workspaceSta
// to dirty files, we also include the mtime of each dirty file.
cmd := exec.CommandContext(gitCtx, "git", "diff-files", "--name-only")
cmd.Dir = wt.workDir
var stderrBuf bytes.Buffer
cmd.Stderr = &stderrBuf
out, err := cmd.Output()
if err != nil {
return state, fmt.Errorf("git diff-files: %w", err)
return state, fmt.Errorf("git diff-files in %s: %w (stderr: %s)",
wt.workDir, err, strings.TrimSpace(stderrBuf.String()))
}
state.diffFilesID = wt.buildDirtyFilesID(string(out))
@@ -163,9 +187,12 @@ func (wt *WorkspaceTracker) getUntrackedFilesID(ctx context.Context) (string, er
// Get list of untracked files (excluding ignored)
cmd := exec.CommandContext(ctx, "git", "ls-files", "--others", "--exclude-standard")
cmd.Dir = wt.workDir
var stderrBuf bytes.Buffer
cmd.Stderr = &stderrBuf
out, err := cmd.Output()
if err != nil {
return "", fmt.Errorf("git ls-files: %w", err)
return "", fmt.Errorf("git ls-files in %s: %w (stderr: %s)",
wt.workDir, err, strings.TrimSpace(stderrBuf.String()))
}
if len(out) == 0 {
return "", nil // No untracked files is not an error
@@ -3,6 +3,7 @@ package process
import (
"context"
"os"
"path/filepath"
"testing"
"time"
)
@@ -41,3 +42,42 @@ func TestWorkspaceTracker_StopsWhenWorkDirDeleted(t *testing.T) {
t.Fatal("workspace tracker goroutines did not stop after workdir was deleted")
}
}
func TestWorkspaceTracker_StopsWhenGitBroken(t *testing.T) {
isolateTestGitEnv(t)
repoDir, cleanup := setupTestRepo(t)
defer cleanup()
log := newTestLogger(t)
wt := NewWorkspaceTracker(repoDir, log)
// Use fast poll intervals so the test completes quickly
wt.filePollInterval = 50 * time.Millisecond
wt.gitPollInterval = 50 * time.Millisecond
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
wt.Start(ctx)
// Corrupt the git repository by removing .git/HEAD.
// The directory still exists, but git commands will fail with exit 128.
headPath := filepath.Join(repoDir, ".git", "HEAD")
if err := os.Remove(headPath); err != nil {
t.Fatalf("failed to remove .git/HEAD: %v", err)
}
// Both loops should stop after maxConsecutiveGitFailures iterations
done := make(chan struct{})
go func() {
wt.wg.Wait()
close(done)
}()
select {
case <-done:
// Both goroutines exited — success
case <-time.After(5 * time.Second):
t.Fatal("workspace tracker goroutines did not stop after git was broken")
}
}
@@ -46,8 +46,9 @@ type WorkspaceTracker struct {
workspaceStreamSubscribers map[types.WorkspaceStreamSubscriber]struct{}
workspaceSubMu sync.RWMutex
// Git polling interval
gitPollInterval time.Duration
// Polling intervals
filePollInterval time.Duration
gitPollInterval time.Duration
// Control
stopCh chan struct{}
@@ -72,6 +73,7 @@ func NewWorkspaceTracker(workDir string, log *logger.Logger) *WorkspaceTracker {
gitIndexPath: gitIndexPath,
logger: log.WithFields(zap.String("component", "workspace-tracker")),
workspaceStreamSubscribers: make(map[types.WorkspaceStreamSubscriber]struct{}),
filePollInterval: DefaultFilePollInterval,
gitPollInterval: DefaultGitPollInterval,
stopCh: make(chan struct{}),
cancelCtx: ctx,