perf: optimize git clone/fetch for large repos with many tags (#533)

This commit is contained in:
Carlos Florêncio
2026-04-03 15:34:53 +01:00
committed by GitHub
parent 8c6f16337d
commit 653ec2a1df
3 changed files with 90 additions and 9 deletions
+7 -4
View File
@@ -16,7 +16,10 @@ import (
)
// ghCredentialHelper is the git credential helper command that delegates to gh CLI.
const ghCredentialHelper = "!gh auth git-credential"
const (
ghCredentialHelper = "!gh auth git-credential"
gitNoTags = "--no-tags"
)
// Config holds configuration for the repository cloner.
type Config struct {
@@ -122,7 +125,7 @@ func (c *Cloner) gitCmd(ctx context.Context, args ...string) *exec.Cmd {
func (c *Cloner) fetch(ctx context.Context, repoPath string) {
c.logger.Debug("repository already cloned, fetching", zap.String("path", repoPath))
cmd := c.gitCmd(ctx, "-C", repoPath, "fetch", "--all", "--prune", "--force")
cmd := c.gitCmd(ctx, "-C", repoPath, "fetch", "--all", "--prune", "--force", gitNoTags)
if out, err := cmd.CombinedOutput(); err != nil {
c.logger.Warn("git fetch failed (non-fatal)",
zap.String("path", repoPath),
@@ -153,7 +156,7 @@ func (c *Cloner) clone(ctx context.Context, cloneURL, targetPath, owner, name st
}
// Fallback: git clone with credential helper.
cmd := c.gitCmd(ctx, "clone", cloneURL, targetPath)
cmd := c.gitCmd(ctx, "clone", "--filter=blob:none", gitNoTags, cloneURL, targetPath)
if out, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("git clone failed: %s: %w", string(out), err)
}
@@ -164,7 +167,7 @@ func (c *Cloner) clone(ctx context.Context, cloneURL, targetPath, owner, name st
// automatically via the user's gh CLI session.
func (c *Cloner) ghClone(ctx context.Context, owner, name, targetPath string) error {
nwo := owner + "/" + name
cmd := exec.CommandContext(ctx, "gh", "repo", "clone", nwo, targetPath)
cmd := exec.CommandContext(ctx, "gh", "repo", "clone", nwo, targetPath, "--", "--filter=blob:none", gitNoTags)
cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
out, err := cmd.CombinedOutput()
if err != nil {
@@ -0,0 +1,77 @@
package repoclone
import (
"context"
"os"
"os/exec"
"path/filepath"
"testing"
"github.com/kandev/kandev/internal/common/logger"
)
func TestClone_PreservesNonDefaultRemoteBranches(t *testing.T) {
t.Parallel()
originPath := initBareRepoWithReleaseBranch(t)
targetPath := filepath.Join(t.TempDir(), "clone")
cloner := NewCloner(Config{}, ProtocolSSH, t.TempDir(), logger.Default())
if err := cloner.clone(context.Background(), originPath, targetPath, "", ""); err != nil {
t.Fatalf("clone() unexpected error: %v", err)
}
if !gitRefExists(t, targetPath, "refs/remotes/origin/release") {
t.Fatal("expected cloned repo to contain origin/release for downstream worktree base branches")
}
}
func initBareRepoWithReleaseBranch(t *testing.T) string {
t.Helper()
root := t.TempDir()
originPath := filepath.Join(root, "origin.git")
workPath := filepath.Join(root, "work")
runGit(t, root, "init", "--bare", "-b", "main", originPath)
runGit(t, root, "clone", originPath, workPath)
runGit(t, workPath, "config", "user.email", "test@example.com")
runGit(t, workPath, "config", "user.name", "Test User")
readmePath := filepath.Join(workPath, "README.md")
if err := os.WriteFile(readmePath, []byte("main\n"), 0o644); err != nil {
t.Fatalf("write README.md: %v", err)
}
runGit(t, workPath, "add", "README.md")
runGit(t, workPath, "commit", "-m", "main commit")
runGit(t, workPath, "push", "origin", "main")
runGit(t, workPath, "checkout", "-b", "release")
if err := os.WriteFile(readmePath, []byte("release\n"), 0o644); err != nil {
t.Fatalf("write release README.md: %v", err)
}
runGit(t, workPath, "commit", "-am", "release commit")
runGit(t, workPath, "push", "origin", "release")
return originPath
}
func gitRefExists(t *testing.T, repoPath, ref string) bool {
t.Helper()
cmd := exec.Command("git", "show-ref", "--verify", "--quiet", ref)
cmd.Dir = repoPath
return cmd.Run() == nil
}
func runGit(t *testing.T, repoPath string, args ...string) string {
t.Helper()
cmd := exec.Command("git", args...)
cmd.Dir = repoPath
output, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("git %v failed: %v\n%s", args, err, output)
}
return string(output)
}
+6 -5
View File
@@ -18,8 +18,9 @@ import (
)
const (
defaultGitFetchTimeout = 60 * time.Second
defaultGitFetchTimeout = 90 * time.Second
defaultGitPullTimeout = 60 * time.Second
gitNoTags = "--no-tags"
)
// repoLockEntry tracks a repository lock and its reference count.
@@ -443,10 +444,10 @@ func (m *Manager) fetchBranchToLocal(ctx context.Context, repoPath, branch strin
zap.String("repo_path", repoPath))
// Try to fetch from origin to get the latest version.
fetchCtx, cancelFetch := context.WithTimeout(ctx, 30*time.Second)
fetchCtx, cancelFetch := context.WithTimeout(ctx, m.fetchTimeout)
defer cancelFetch()
fetchCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", "origin", branch+":"+branch)
fetchCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", gitNoTags, "origin", branch+":"+branch)
if output, err := fetchCmd.CombinedOutput(); err != nil {
outputStr := string(output)
@@ -454,7 +455,7 @@ func (m *Manager) fetchBranchToLocal(ctx context.Context, repoPath, branch strin
// the local ref. Retry by fetching only the remote-tracking ref (origin/branch),
// which is always safe regardless of worktree state.
if isFetchRefusedCheckedOut(outputStr) {
retryCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", "origin", branch)
retryCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", gitNoTags, "origin", branch)
if _, retryErr := retryCmd.CombinedOutput(); retryErr == nil {
m.logger.Info("fetched via remote-tracking ref (branch checked out elsewhere)",
zap.String("branch", branch))
@@ -1272,7 +1273,7 @@ func (m *Manager) pullBaseBranch(ctx context.Context, repoPath, baseBranch strin
fetchCtx, cancelFetch := context.WithTimeout(ctx, m.fetchTimeout)
defer cancelFetch()
fetchArgs := []string{"fetch", "origin"}
fetchArgs := []string{"fetch", gitNoTags, "origin"}
if localBranch != "" {
fetchArgs = append(fetchArgs, localBranch)
}