diff --git a/apps/backend/internal/agentctl/server/process/workspace_git_poll.go b/apps/backend/internal/agentctl/server/process/workspace_git_poll.go index e0be429f58..56c27f84e9 100644 --- a/apps/backend/internal/agentctl/server/process/workspace_git_poll.go +++ b/apps/backend/internal/agentctl/server/process/workspace_git_poll.go @@ -32,6 +32,8 @@ func (wt *WorkspaceTracker) pollGitChanges(ctx context.Context) { zap.String("initial_head", wt.cachedHeadSHA), zap.String("initial_branch", wt.cachedBranchName)) + var consecutiveFailures int + for { select { case <-ctx.Done(): @@ -45,6 +47,25 @@ func (wt *WorkspaceTracker) pollGitChanges(ctx context.Context) { zap.String("workDir", wt.workDir)) return } + + // Quick git health check before running full change detection. + // If git is broken (e.g., worktree .git reference points to deleted gitdir), + // stop after maxConsecutiveGitFailures to avoid wasting CPU. + probeCmd := exec.CommandContext(ctx, "git", "rev-parse", "--git-dir") + probeCmd.Dir = wt.workDir + if err := probeCmd.Run(); err != nil { + consecutiveFailures++ + if consecutiveFailures >= maxConsecutiveGitFailures { + wt.logger.Error("git not functional, stopping git polling", + zap.String("workDir", wt.workDir), + zap.Int("consecutiveFailures", consecutiveFailures), + zap.Error(err)) + return + } + continue + } + consecutiveFailures = 0 + wt.checkGitChanges(ctx) } } diff --git a/apps/backend/internal/agentctl/server/process/workspace_monitor.go b/apps/backend/internal/agentctl/server/process/workspace_monitor.go index c3dd707ce1..0e0d2ccf29 100644 --- a/apps/backend/internal/agentctl/server/process/workspace_monitor.go +++ b/apps/backend/internal/agentctl/server/process/workspace_monitor.go @@ -1,6 +1,7 @@ package process import ( + "bytes" "context" "fmt" "os" @@ -22,12 +23,19 @@ const DefaultFilePollInterval = 2 * time.Second // This prevents the monitor loop from hanging if git is blocked (e.g., index lock). const gitCommandTimeout = 10 * time.Second +// maxConsecutiveGitFailures is the number of consecutive git command failures +// before the polling loop gives up and stops. At 2-second intervals, 5 failures +// means ~10 seconds — enough to rule out transient issues (index lock, brief GC race) +// while stopping quickly enough to avoid log spam when git is permanently broken +// (e.g., worktree .git reference points to a deleted gitdir after task archival). +const maxConsecutiveGitFailures = 5 + // monitorLoop polls for file changes using file mtimes and quick git checks. // This is more efficient than fsnotify on macOS and avoids file descriptor exhaustion. func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) { defer wt.wg.Done() - ticker := time.NewTicker(DefaultFilePollInterval) + ticker := time.NewTicker(wt.filePollInterval) defer ticker.Stop() // Initial update @@ -37,7 +45,9 @@ func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) { // Cache the last known state (ignore error on initial fetch) lastState, _ := wt.getWorkspaceState(ctx) - wt.logger.Info("file polling started", zap.Duration("interval", DefaultFilePollInterval)) + wt.logger.Info("file polling started", zap.Duration("interval", wt.filePollInterval)) + + var consecutiveFailures int for { select { @@ -60,10 +70,21 @@ func (wt *WorkspaceTracker) monitorLoop(ctx context.Context) { // Git command failed for another reason - skip this cycle and retry on next tick. // Don't update lastState so the change will be detected when git recovers. + consecutiveFailures++ + if consecutiveFailures >= maxConsecutiveGitFailures { + wt.logger.Error("git commands failing repeatedly, stopping workspace monitor", + zap.String("workDir", wt.workDir), + zap.Int("consecutiveFailures", consecutiveFailures), + zap.Error(err)) + return + } wt.logger.Warn("failed to get workspace state, will retry next cycle", + zap.String("workDir", wt.workDir), + zap.Int("consecutiveFailures", consecutiveFailures), zap.Error(err)) continue } + consecutiveFailures = 0 if currentState.changed(lastState) { lastState = currentState wt.logger.Debug("workspace state changed, updating") @@ -112,9 +133,12 @@ func (wt *WorkspaceTracker) getWorkspaceState(ctx context.Context) (workspaceSta // to dirty files, we also include the mtime of each dirty file. cmd := exec.CommandContext(gitCtx, "git", "diff-files", "--name-only") cmd.Dir = wt.workDir + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf out, err := cmd.Output() if err != nil { - return state, fmt.Errorf("git diff-files: %w", err) + return state, fmt.Errorf("git diff-files in %s: %w (stderr: %s)", + wt.workDir, err, strings.TrimSpace(stderrBuf.String())) } state.diffFilesID = wt.buildDirtyFilesID(string(out)) @@ -163,9 +187,12 @@ func (wt *WorkspaceTracker) getUntrackedFilesID(ctx context.Context) (string, er // Get list of untracked files (excluding ignored) cmd := exec.CommandContext(ctx, "git", "ls-files", "--others", "--exclude-standard") cmd.Dir = wt.workDir + var stderrBuf bytes.Buffer + cmd.Stderr = &stderrBuf out, err := cmd.Output() if err != nil { - return "", fmt.Errorf("git ls-files: %w", err) + return "", fmt.Errorf("git ls-files in %s: %w (stderr: %s)", + wt.workDir, err, strings.TrimSpace(stderrBuf.String())) } if len(out) == 0 { return "", nil // No untracked files is not an error diff --git a/apps/backend/internal/agentctl/server/process/workspace_monitor_test.go b/apps/backend/internal/agentctl/server/process/workspace_monitor_test.go index a3b024c966..c53ce6d429 100644 --- a/apps/backend/internal/agentctl/server/process/workspace_monitor_test.go +++ b/apps/backend/internal/agentctl/server/process/workspace_monitor_test.go @@ -3,6 +3,7 @@ package process import ( "context" "os" + "path/filepath" "testing" "time" ) @@ -41,3 +42,42 @@ func TestWorkspaceTracker_StopsWhenWorkDirDeleted(t *testing.T) { t.Fatal("workspace tracker goroutines did not stop after workdir was deleted") } } + +func TestWorkspaceTracker_StopsWhenGitBroken(t *testing.T) { + isolateTestGitEnv(t) + + repoDir, cleanup := setupTestRepo(t) + defer cleanup() + + log := newTestLogger(t) + wt := NewWorkspaceTracker(repoDir, log) + // Use fast poll intervals so the test completes quickly + wt.filePollInterval = 50 * time.Millisecond + wt.gitPollInterval = 50 * time.Millisecond + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + wt.Start(ctx) + + // Corrupt the git repository by removing .git/HEAD. + // The directory still exists, but git commands will fail with exit 128. + headPath := filepath.Join(repoDir, ".git", "HEAD") + if err := os.Remove(headPath); err != nil { + t.Fatalf("failed to remove .git/HEAD: %v", err) + } + + // Both loops should stop after maxConsecutiveGitFailures iterations + done := make(chan struct{}) + go func() { + wt.wg.Wait() + close(done) + }() + + select { + case <-done: + // Both goroutines exited — success + case <-time.After(5 * time.Second): + t.Fatal("workspace tracker goroutines did not stop after git was broken") + } +} diff --git a/apps/backend/internal/agentctl/server/process/workspace_tracker.go b/apps/backend/internal/agentctl/server/process/workspace_tracker.go index cae8fcfa94..f012ac719c 100644 --- a/apps/backend/internal/agentctl/server/process/workspace_tracker.go +++ b/apps/backend/internal/agentctl/server/process/workspace_tracker.go @@ -46,8 +46,9 @@ type WorkspaceTracker struct { workspaceStreamSubscribers map[types.WorkspaceStreamSubscriber]struct{} workspaceSubMu sync.RWMutex - // Git polling interval - gitPollInterval time.Duration + // Polling intervals + filePollInterval time.Duration + gitPollInterval time.Duration // Control stopCh chan struct{} @@ -72,6 +73,7 @@ func NewWorkspaceTracker(workDir string, log *logger.Logger) *WorkspaceTracker { gitIndexPath: gitIndexPath, logger: log.WithFields(zap.String("component", "workspace-tracker")), workspaceStreamSubscribers: make(map[types.WorkspaceStreamSubscriber]struct{}), + filePollInterval: DefaultFilePollInterval, gitPollInterval: DefaultGitPollInterval, stopCh: make(chan struct{}), cancelCtx: ctx,