fix: stop workspace tracker after consecutive git failures (#514)
This commit is contained in:
@@ -32,6 +32,8 @@ func (wt *WorkspaceTracker) pollGitChanges(ctx context.Context) {
|
||||
zap.String("initial_head", wt.cachedHeadSHA),
|
||||
zap.String("initial_branch", wt.cachedBranchName))
|
||||
|
||||
var consecutiveFailures int
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -45,6 +47,25 @@ func (wt *WorkspaceTracker) pollGitChanges(ctx context.Context) {
|
||||
zap.String("workDir", wt.workDir))
|
||||
return
|
||||
}
|
||||
|
||||
// Quick git health check before running full change detection.
|
||||
// If git is broken (e.g., worktree .git reference points to deleted gitdir),
|
||||
// stop after maxConsecutiveGitFailures to avoid wasting CPU.
|
||||
probeCmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-dir")
|
||||
probeCmd.Dir = wt.workDir
|
||||
if err := probeCmd.Run(); err != nil {
|
||||
consecutiveFailures++
|
||||
if consecutiveFailures >= maxConsecutiveGitFailures {
|
||||
wt.logger.Error("git not functional, stopping git polling",
|
||||
zap.String("workDir", wt.workDir),
|
||||
zap.Int("consecutiveFailures", consecutiveFailures),
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
consecutiveFailures = 0
|
||||
|
||||
wt.checkGitChanges(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package process
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
@@ -22,12 +23,19 @@ const DefaultFilePollInterval = 2 * time.Second
|
||||
// This prevents the monitor loop from hanging if git is blocked (e.g., index lock).
|
||||
const gitCommandTimeout = 10 * time.Second
|
||||
|
||||
// maxConsecutiveGitFailures is the number of consecutive git command failures
|
||||
// before the polling loop gives up and stops. At 2-second intervals, 5 failures
|
||||
// means ~10 seconds — enough to rule out transient issues (index lock, brief GC race)
|
||||
// while stopping quickly enough to avoid log spam when git is permanently broken
|
||||
// (e.g., worktree .git reference points to a deleted gitdir after task archival).
|
||||
const maxConsecutiveGitFailures = 5
|
||||
|
||||
// monitorLoop polls for file changes using file mtimes and quick git checks.
|
||||
// This is more efficient than fsnotify on macOS and avoids file descriptor exhaustion.
|
||||
func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) {
|
||||
defer wt.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(DefaultFilePollInterval)
|
||||
ticker := time.NewTicker(wt.filePollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// Initial update
|
||||
@@ -37,7 +45,9 @@ func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) {
|
||||
// Cache the last known state (ignore error on initial fetch)
|
||||
lastState, _ := wt.getWorkspaceState(ctx)
|
||||
|
||||
wt.logger.Info("file polling started", zap.Duration("interval", DefaultFilePollInterval))
|
||||
wt.logger.Info("file polling started", zap.Duration("interval", wt.filePollInterval))
|
||||
|
||||
var consecutiveFailures int
|
||||
|
||||
for {
|
||||
select {
|
||||
@@ -60,10 +70,21 @@ func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) {
|
||||
|
||||
// Git command failed for another reason - skip this cycle and retry on next tick.
|
||||
// Don't update lastState so the change will be detected when git recovers.
|
||||
consecutiveFailures++
|
||||
if consecutiveFailures >= maxConsecutiveGitFailures {
|
||||
wt.logger.Error("git commands failing repeatedly, stopping workspace monitor",
|
||||
zap.String("workDir", wt.workDir),
|
||||
zap.Int("consecutiveFailures", consecutiveFailures),
|
||||
zap.Error(err))
|
||||
return
|
||||
}
|
||||
wt.logger.Warn("failed to get workspace state, will retry next cycle",
|
||||
zap.String("workDir", wt.workDir),
|
||||
zap.Int("consecutiveFailures", consecutiveFailures),
|
||||
zap.Error(err))
|
||||
continue
|
||||
}
|
||||
consecutiveFailures = 0
|
||||
if currentState.changed(lastState) {
|
||||
lastState = currentState
|
||||
wt.logger.Debug("workspace state changed, updating")
|
||||
@@ -112,9 +133,12 @@ func (wt *WorkspaceTracker) getWorkspaceState(ctx context.Context) (workspaceSta
|
||||
// to dirty files, we also include the mtime of each dirty file.
|
||||
cmd := exec.CommandContext(gitCtx, "git", "diff-files", "--name-only")
|
||||
cmd.Dir = wt.workDir
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stderr = &stderrBuf
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return state, fmt.Errorf("git diff-files: %w", err)
|
||||
return state, fmt.Errorf("git diff-files in %s: %w (stderr: %s)",
|
||||
wt.workDir, err, strings.TrimSpace(stderrBuf.String()))
|
||||
}
|
||||
state.diffFilesID = wt.buildDirtyFilesID(string(out))
|
||||
|
||||
@@ -163,9 +187,12 @@ func (wt *WorkspaceTracker) getUntrackedFilesID(ctx context.Context) (string, er
|
||||
// Get list of untracked files (excluding ignored)
|
||||
cmd := exec.CommandContext(ctx, "git", "ls-files", "--others", "--exclude-standard")
|
||||
cmd.Dir = wt.workDir
|
||||
var stderrBuf bytes.Buffer
|
||||
cmd.Stderr = &stderrBuf
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("git ls-files: %w", err)
|
||||
return "", fmt.Errorf("git ls-files in %s: %w (stderr: %s)",
|
||||
wt.workDir, err, strings.TrimSpace(stderrBuf.String()))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return "", nil // No untracked files is not an error
|
||||
|
||||
@@ -3,6 +3,7 @@ package process
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -41,3 +42,42 @@ func TestWorkspaceTracker_StopsWhenWorkDirDeleted(t *testing.T) {
|
||||
t.Fatal("workspace tracker goroutines did not stop after workdir was deleted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceTracker_StopsWhenGitBroken(t *testing.T) {
|
||||
isolateTestGitEnv(t)
|
||||
|
||||
repoDir, cleanup := setupTestRepo(t)
|
||||
defer cleanup()
|
||||
|
||||
log := newTestLogger(t)
|
||||
wt := NewWorkspaceTracker(repoDir, log)
|
||||
// Use fast poll intervals so the test completes quickly
|
||||
wt.filePollInterval = 50 * time.Millisecond
|
||||
wt.gitPollInterval = 50 * time.Millisecond
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
wt.Start(ctx)
|
||||
|
||||
// Corrupt the git repository by removing .git/HEAD.
|
||||
// The directory still exists, but git commands will fail with exit 128.
|
||||
headPath := filepath.Join(repoDir, ".git", "HEAD")
|
||||
if err := os.Remove(headPath); err != nil {
|
||||
t.Fatalf("failed to remove .git/HEAD: %v", err)
|
||||
}
|
||||
|
||||
// Both loops should stop after maxConsecutiveGitFailures iterations
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wt.wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
// Both goroutines exited — success
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("workspace tracker goroutines did not stop after git was broken")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,8 +46,9 @@ type WorkspaceTracker struct {
|
||||
workspaceStreamSubscribers map[types.WorkspaceStreamSubscriber]struct{}
|
||||
workspaceSubMu sync.RWMutex
|
||||
|
||||
// Git polling interval
|
||||
gitPollInterval time.Duration
|
||||
// Polling intervals
|
||||
filePollInterval time.Duration
|
||||
gitPollInterval time.Duration
|
||||
|
||||
// Control
|
||||
stopCh chan struct{}
|
||||
@@ -72,6 +73,7 @@ func NewWorkspaceTracker(workDir string, log *logger.Logger) *WorkspaceTracker {
|
||||
gitIndexPath: gitIndexPath,
|
||||
logger: log.WithFields(zap.String("component", "workspace-tracker")),
|
||||
workspaceStreamSubscribers: make(map[types.WorkspaceStreamSubscriber]struct{}),
|
||||
filePollInterval: DefaultFilePollInterval,
|
||||
gitPollInterval: DefaultGitPollInterval,
|
||||
stopCh: make(chan struct{}),
|
||||
cancelCtx: ctx,
|
||||
|
||||
Reference in New Issue
Block a user