perf: optimize git clone/fetch for large repos with many tags (#533)
This commit is contained in:
@@ -16,7 +16,10 @@ import (
|
||||
)
|
||||
|
||||
// ghCredentialHelper is the git credential helper command that delegates to gh CLI.
|
||||
const ghCredentialHelper = "!gh auth git-credential"
|
||||
const (
|
||||
ghCredentialHelper = "!gh auth git-credential"
|
||||
gitNoTags = "--no-tags"
|
||||
)
|
||||
|
||||
// Config holds configuration for the repository cloner.
|
||||
type Config struct {
|
||||
@@ -122,7 +125,7 @@ func (c *Cloner) gitCmd(ctx context.Context, args ...string) *exec.Cmd {
|
||||
|
||||
func (c *Cloner) fetch(ctx context.Context, repoPath string) {
|
||||
c.logger.Debug("repository already cloned, fetching", zap.String("path", repoPath))
|
||||
cmd := c.gitCmd(ctx, "-C", repoPath, "fetch", "--all", "--prune", "--force")
|
||||
cmd := c.gitCmd(ctx, "-C", repoPath, "fetch", "--all", "--prune", "--force", gitNoTags)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
c.logger.Warn("git fetch failed (non-fatal)",
|
||||
zap.String("path", repoPath),
|
||||
@@ -153,7 +156,7 @@ func (c *Cloner) clone(ctx context.Context, cloneURL, targetPath, owner, name st
|
||||
}
|
||||
|
||||
// Fallback: git clone with credential helper.
|
||||
cmd := c.gitCmd(ctx, "clone", cloneURL, targetPath)
|
||||
cmd := c.gitCmd(ctx, "clone", "--filter=blob:none", gitNoTags, cloneURL, targetPath)
|
||||
if out, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("git clone failed: %s: %w", string(out), err)
|
||||
}
|
||||
@@ -164,7 +167,7 @@ func (c *Cloner) clone(ctx context.Context, cloneURL, targetPath, owner, name st
|
||||
// automatically via the user's gh CLI session.
|
||||
func (c *Cloner) ghClone(ctx context.Context, owner, name, targetPath string) error {
|
||||
nwo := owner + "/" + name
|
||||
cmd := exec.CommandContext(ctx, "gh", "repo", "clone", nwo, targetPath)
|
||||
cmd := exec.CommandContext(ctx, "gh", "repo", "clone", nwo, targetPath, "--", "--filter=blob:none", gitNoTags)
|
||||
cmd.Env = append(os.Environ(), "GIT_TERMINAL_PROMPT=0")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package repoclone
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/common/logger"
|
||||
)
|
||||
|
||||
func TestClone_PreservesNonDefaultRemoteBranches(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
originPath := initBareRepoWithReleaseBranch(t)
|
||||
targetPath := filepath.Join(t.TempDir(), "clone")
|
||||
|
||||
cloner := NewCloner(Config{}, ProtocolSSH, t.TempDir(), logger.Default())
|
||||
if err := cloner.clone(context.Background(), originPath, targetPath, "", ""); err != nil {
|
||||
t.Fatalf("clone() unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if !gitRefExists(t, targetPath, "refs/remotes/origin/release") {
|
||||
t.Fatal("expected cloned repo to contain origin/release for downstream worktree base branches")
|
||||
}
|
||||
}
|
||||
|
||||
func initBareRepoWithReleaseBranch(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
root := t.TempDir()
|
||||
originPath := filepath.Join(root, "origin.git")
|
||||
workPath := filepath.Join(root, "work")
|
||||
|
||||
runGit(t, root, "init", "--bare", "-b", "main", originPath)
|
||||
runGit(t, root, "clone", originPath, workPath)
|
||||
runGit(t, workPath, "config", "user.email", "test@example.com")
|
||||
runGit(t, workPath, "config", "user.name", "Test User")
|
||||
|
||||
readmePath := filepath.Join(workPath, "README.md")
|
||||
if err := os.WriteFile(readmePath, []byte("main\n"), 0o644); err != nil {
|
||||
t.Fatalf("write README.md: %v", err)
|
||||
}
|
||||
runGit(t, workPath, "add", "README.md")
|
||||
runGit(t, workPath, "commit", "-m", "main commit")
|
||||
runGit(t, workPath, "push", "origin", "main")
|
||||
|
||||
runGit(t, workPath, "checkout", "-b", "release")
|
||||
if err := os.WriteFile(readmePath, []byte("release\n"), 0o644); err != nil {
|
||||
t.Fatalf("write release README.md: %v", err)
|
||||
}
|
||||
runGit(t, workPath, "commit", "-am", "release commit")
|
||||
runGit(t, workPath, "push", "origin", "release")
|
||||
|
||||
return originPath
|
||||
}
|
||||
|
||||
func gitRefExists(t *testing.T, repoPath, ref string) bool {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command("git", "show-ref", "--verify", "--quiet", ref)
|
||||
cmd.Dir = repoPath
|
||||
return cmd.Run() == nil
|
||||
}
|
||||
|
||||
func runGit(t *testing.T, repoPath string, args ...string) string {
|
||||
t.Helper()
|
||||
|
||||
cmd := exec.Command("git", args...)
|
||||
cmd.Dir = repoPath
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("git %v failed: %v\n%s", args, err, output)
|
||||
}
|
||||
return string(output)
|
||||
}
|
||||
@@ -18,8 +18,9 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
defaultGitFetchTimeout = 60 * time.Second
|
||||
defaultGitFetchTimeout = 90 * time.Second
|
||||
defaultGitPullTimeout = 60 * time.Second
|
||||
gitNoTags = "--no-tags"
|
||||
)
|
||||
|
||||
// repoLockEntry tracks a repository lock and its reference count.
|
||||
@@ -443,10 +444,10 @@ func (m *Manager) fetchBranchToLocal(ctx context.Context, repoPath, branch strin
|
||||
zap.String("repo_path", repoPath))
|
||||
|
||||
// Try to fetch from origin to get the latest version.
|
||||
fetchCtx, cancelFetch := context.WithTimeout(ctx, 30*time.Second)
|
||||
fetchCtx, cancelFetch := context.WithTimeout(ctx, m.fetchTimeout)
|
||||
defer cancelFetch()
|
||||
|
||||
fetchCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", "origin", branch+":"+branch)
|
||||
fetchCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", gitNoTags, "origin", branch+":"+branch)
|
||||
if output, err := fetchCmd.CombinedOutput(); err != nil {
|
||||
outputStr := string(output)
|
||||
|
||||
@@ -454,7 +455,7 @@ func (m *Manager) fetchBranchToLocal(ctx context.Context, repoPath, branch strin
|
||||
// the local ref. Retry by fetching only the remote-tracking ref (origin/branch),
|
||||
// which is always safe regardless of worktree state.
|
||||
if isFetchRefusedCheckedOut(outputStr) {
|
||||
retryCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", "origin", branch)
|
||||
retryCmd := m.newNonInteractiveGitCmd(fetchCtx, repoPath, "fetch", gitNoTags, "origin", branch)
|
||||
if _, retryErr := retryCmd.CombinedOutput(); retryErr == nil {
|
||||
m.logger.Info("fetched via remote-tracking ref (branch checked out elsewhere)",
|
||||
zap.String("branch", branch))
|
||||
@@ -1272,7 +1273,7 @@ func (m *Manager) pullBaseBranch(ctx context.Context, repoPath, baseBranch strin
|
||||
fetchCtx, cancelFetch := context.WithTimeout(ctx, m.fetchTimeout)
|
||||
defer cancelFetch()
|
||||
|
||||
fetchArgs := []string{"fetch", "origin"}
|
||||
fetchArgs := []string{"fetch", gitNoTags, "origin"}
|
||||
if localBranch != "" {
|
||||
fetchArgs = append(fetchArgs, localBranch)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user