874 lines
34 KiB
Go
874 lines
34 KiB
Go
package github
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
// PRWatchSyncResult is the per-watch outcome from SyncWatchesBatched.
|
|
// Callers (poller, on-demand sync) can post-process — e.g. publish
|
|
// PRFeedback events — without re-fetching from GitHub.
|
|
type PRWatchSyncResult struct {
|
|
Watch *PRWatch
|
|
Status *PRStatus // nil when no PR was found for a searching watch, or alias missing
|
|
Found bool // true when status applied (PR data exists)
|
|
Changed bool // numbered watches only: true if checks/review state moved
|
|
SyncFailed bool // true when SyncTaskPR returned an error — callers must NOT publish events
|
|
// DiscoveryResolved indicates that the provider lookup completed, even if
|
|
// a later database write failed. It keeps health admission focused on
|
|
// provider evidence rather than task persistence.
|
|
DiscoveryResolved bool
|
|
}
|
|
|
|
const prSyncExplicitRefreshKeySuffix = "|explicit-refresh"
|
|
|
|
// SyncWatchesBatched runs the batched GraphQL queries for the supplied
|
|
// watches and applies the resulting DB updates: timestamps, task PR sync,
|
|
// watch PR-number promotion on detection, watch reset on merge/close.
|
|
// Returns per-watch results so callers can post-process (event publishing).
|
|
//
|
|
// Returns an error when the batched fetch itself fails — the caller should
|
|
// fall back to per-watch checks rather than silently dropping a poll cycle.
|
|
// Errors from the per-watch DB applies are logged but do not abort the
|
|
// loop, matching the per-watch path's best-effort semantics.
|
|
//
|
|
// This is the single seam both the 1-minute poller and the on-demand
|
|
// TriggerPRSyncAll / ListWorkspaceTaskPRs background refresh share, so a
|
|
// 40-watch workspace fans out to ~2 gh subprocess calls instead of 40.
|
|
func (s *Service) SyncWatchesBatched(ctx context.Context, watches []*PRWatch) ([]PRWatchSyncResult, error) {
|
|
return s.syncWatchesBatchedWithClient(ctx, s.client, "legacy", "", 0, watches, false)
|
|
}
|
|
|
|
// SyncWorkspaceWatchesBatched resolves one automation credential and rejects
|
|
// mixed or missing workspace ownership before making a provider call.
|
|
func (s *Service) SyncWorkspaceWatchesBatched(
|
|
ctx context.Context, workspaceID string, watches []*PRWatch,
|
|
) ([]PRWatchSyncResult, error) {
|
|
return s.syncWorkspaceWatchesBatched(ctx, workspaceID, watches, false)
|
|
}
|
|
|
|
func (s *Service) syncWorkspaceWatchesBatched(
|
|
ctx context.Context, workspaceID string, watches []*PRWatch, explicitRefresh bool,
|
|
) ([]PRWatchSyncResult, error) {
|
|
if len(watches) == 0 {
|
|
return nil, nil
|
|
}
|
|
if strings.TrimSpace(workspaceID) == "" {
|
|
return nil, ErrGitHubWorkspaceRequired
|
|
}
|
|
for _, watch := range watches {
|
|
if watch == nil || watch.WorkspaceID != workspaceID {
|
|
return nil, ErrGitHubWorkspaceRequired
|
|
}
|
|
}
|
|
resolved, err := s.resolveAutomationClient(ctx, workspaceID, "", "")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
credentialGeneration := int64(0)
|
|
if resolved.credential != nil {
|
|
credentialGeneration = resolved.credential.CredentialGeneration
|
|
}
|
|
if explicitRefresh {
|
|
s.invalidateWorkflowAttentionForResolvedWatches(resolved, watches)
|
|
}
|
|
return s.syncWatchesBatchedWithClient(
|
|
ctx, resolved.Client, resolved.CacheScope, workspaceID, credentialGeneration, watches, explicitRefresh,
|
|
)
|
|
}
|
|
|
|
type prWatchBatchGroup struct {
|
|
key string
|
|
target prDiscoveryHealthTarget
|
|
representative *PRWatch
|
|
watches []*PRWatch
|
|
numbered bool
|
|
attempt prDiscoveryWatchAttempt
|
|
}
|
|
|
|
func (s *Service) syncWatchesBatchedWithClient(
|
|
ctx context.Context, client Client, cacheScope, workspaceID string, credentialGeneration int64,
|
|
watches []*PRWatch, explicitRefresh bool,
|
|
) ([]PRWatchSyncResult, error) {
|
|
if len(watches) == 0 {
|
|
return nil, nil
|
|
}
|
|
exec, err := graphQLExecutorFor(client)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
groupsByKey := make(map[string]*prWatchBatchGroup, len(watches))
|
|
groups := make([]*prWatchBatchGroup, 0, len(watches))
|
|
for _, watch := range watches {
|
|
if watch == nil {
|
|
continue
|
|
}
|
|
target := prDiscoveryTargetForWatch(watch)
|
|
key := target.key()
|
|
group := groupsByKey[key]
|
|
if group == nil {
|
|
group = &prWatchBatchGroup{
|
|
key: key,
|
|
target: target,
|
|
representative: watch,
|
|
numbered: target.PRNumber > 0,
|
|
}
|
|
groupsByKey[key] = group
|
|
groups = append(groups, group)
|
|
}
|
|
group.watches = append(group.watches, watch)
|
|
}
|
|
admittedGroups := make([]*prWatchBatchGroup, 0, len(groups))
|
|
leaderGroups := make([]*prWatchBatchGroup, 0, len(groups))
|
|
joinedGroups := make([]*prWatchBatchGroup, 0, len(groups))
|
|
for _, group := range groups {
|
|
// Register duplicate consumers before the representative performs the
|
|
// atomic admission decision. The representative is registered by
|
|
// beginPRDiscoveryWatch so another entry point cannot observe it as a
|
|
// consumer before it has joined an existing attempt.
|
|
for _, watch := range group.watches[1:] {
|
|
s.trackPRDiscoveryWatchConsumer(workspaceID, cacheScope, credentialGeneration, watch)
|
|
}
|
|
attempt, ok := s.beginPRDiscoveryWatchWithOptions(
|
|
workspaceID, cacheScope, credentialGeneration, group.representative, explicitRefresh,
|
|
)
|
|
if !ok {
|
|
continue
|
|
}
|
|
group.attempt = attempt
|
|
admittedGroups = append(admittedGroups, group)
|
|
if attempt.joined {
|
|
joinedGroups = append(joinedGroups, group)
|
|
} else {
|
|
leaderGroups = append(leaderGroups, group)
|
|
}
|
|
}
|
|
if len(admittedGroups) == 0 {
|
|
return noOpPRWatchSyncResults(watches), nil
|
|
}
|
|
admitted := make([]*PRWatch, 0, len(leaderGroups))
|
|
for _, group := range leaderGroups {
|
|
admitted = append(admitted, group.representative)
|
|
}
|
|
var statuses *batchedWatchStatuses
|
|
if len(leaderGroups) > 0 {
|
|
numbered, searching := splitPRWatches(admitted)
|
|
statuses, err = s.fetchBatchedWatchStatuses(
|
|
ctx, client, exec, cacheScope, numbered, searching, explicitRefresh,
|
|
)
|
|
if err != nil {
|
|
if s.handleBatchedDiscoveryFetchError(
|
|
workspaceID, cacheScope, credentialGeneration, leaderGroups, admitted, numbered, searching, err,
|
|
) {
|
|
// Schema and throttling failures are not eligible for the legacy
|
|
// per-watch fallback: it would repeat the same rejected provider
|
|
// request once for every watch. Keep those targets paused under the
|
|
// shared retry deadline.
|
|
return noOpPRWatchSyncResults(watches), nil
|
|
}
|
|
// Other provider failures retain the existing transport fallback. Do
|
|
// not leave the batch attempt active or impose a health deadline before
|
|
// the caller can try the per-watch path.
|
|
return nil, err
|
|
}
|
|
}
|
|
if statuses == nil {
|
|
statuses = &batchedWatchStatuses{}
|
|
}
|
|
|
|
resultsByWatch := make(map[*PRWatch]PRWatchSyncResult, len(watches))
|
|
now := time.Now().UTC()
|
|
for _, group := range leaderGroups {
|
|
if group.attempt.result.invalidated() {
|
|
continue
|
|
}
|
|
var representativeResult PRWatchSyncResult
|
|
for index, watch := range group.watches {
|
|
var result PRWatchSyncResult
|
|
if group.numbered {
|
|
result = s.applyBatchedNumberedWatch(ctx, cacheScope, watch, statuses.byKey, now)
|
|
} else {
|
|
result = s.applyBatchedSearchingWatch(ctx, client, cacheScope, watch, statuses, now, false, false, false)
|
|
}
|
|
resultsByWatch[watch] = result
|
|
if index == 0 {
|
|
representativeResult = result
|
|
}
|
|
}
|
|
if group.numbered {
|
|
s.completePRDiscoveryWatchAttempt(
|
|
group.attempt, representativeResult.Status, true, representativeResult.SyncFailed, nil, false,
|
|
)
|
|
if s.isRepoCachedAsMissingForScope(cacheScope, group.target.Owner, group.target.Repo) {
|
|
s.finishPRDiscoveryWatchFailureCategory(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt,
|
|
PRDiscoveryHealthUnavailable,
|
|
)
|
|
} else {
|
|
// A successful known-PR query can legitimately return no alias
|
|
// when the PR is no longer open. That is still a provider success.
|
|
s.finishPRDiscoveryWatchSuccess(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt,
|
|
)
|
|
}
|
|
s.forgetPRDiscoveryWatchAttempt(workspaceID, cacheScope, credentialGeneration, group.attempt)
|
|
continue
|
|
}
|
|
|
|
s.completePRDiscoveryWatchAttempt(
|
|
group.attempt, representativeResult.Status, representativeResult.DiscoveryResolved,
|
|
representativeResult.SyncFailed, nil, false,
|
|
)
|
|
if representativeResult.DiscoveryResolved {
|
|
s.finishPRDiscoveryWatchSuccess(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt,
|
|
)
|
|
for _, watch := range group.watches {
|
|
result := resultsByWatch[watch]
|
|
target := prDiscoveryTargetForWatch(watch)
|
|
if result.Status != nil && result.Status.PR != nil && !result.SyncFailed {
|
|
target = prDiscoveryHealthTarget{
|
|
Owner: result.Status.PR.RepoOwner,
|
|
Repo: result.Status.PR.RepoName,
|
|
Branch: watch.Branch,
|
|
PRNumber: result.Status.PR.Number,
|
|
}
|
|
}
|
|
s.trackPRDiscoveryWatchTarget(
|
|
workspaceID, cacheScope, credentialGeneration, watch, target,
|
|
)
|
|
}
|
|
s.forgetPRDiscoveryWatchAttempt(workspaceID, cacheScope, credentialGeneration, group.attempt)
|
|
continue
|
|
}
|
|
if representativeResult.SyncFailed {
|
|
// An unresolved alias whose fallback failed is an unavailable
|
|
// discovery result. If the fallback was not due yet, release the
|
|
// attempt and preserve any existing failure without inventing a new
|
|
// one.
|
|
s.finishPRDiscoveryWatchFailureCategory(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt,
|
|
PRDiscoveryHealthUnavailable,
|
|
)
|
|
s.forgetPRDiscoveryWatchAttempt(workspaceID, cacheScope, credentialGeneration, group.attempt)
|
|
continue
|
|
}
|
|
s.releasePRDiscoveryWatch(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt,
|
|
)
|
|
s.forgetPRDiscoveryWatchAttempt(workspaceID, cacheScope, credentialGeneration, group.attempt)
|
|
}
|
|
for _, group := range joinedGroups {
|
|
status, resolved, syncFailed, sharedErr, suppressFallback := group.attempt.result.wait(ctx)
|
|
if sharedErr != nil {
|
|
if suppressFallback {
|
|
continue
|
|
}
|
|
return nil, sharedErr
|
|
}
|
|
if status == nil && !resolved && syncFailed {
|
|
continue
|
|
}
|
|
if group.numbered {
|
|
byKey := make(map[string]*PRStatus, 1)
|
|
if status != nil {
|
|
byKey[prStatusCacheKey(group.target.Owner, group.target.Repo, group.target.PRNumber)] = status
|
|
}
|
|
for _, watch := range group.watches {
|
|
resultsByWatch[watch] = s.applyBatchedNumberedWatch(ctx, cacheScope, watch, byKey, now)
|
|
}
|
|
} else {
|
|
byKey := make(map[string]*PRStatus, 1)
|
|
if status != nil {
|
|
byKey[graphqlBranchKey(group.target.Owner, group.target.Repo, group.target.Branch)] = status
|
|
}
|
|
sharedStatuses := &batchedWatchStatuses{byKey: byKey}
|
|
for _, watch := range group.watches {
|
|
resultsByWatch[watch] = s.applyBatchedSearchingWatch(
|
|
ctx, client, cacheScope, watch, sharedStatuses, now, true, resolved, syncFailed,
|
|
)
|
|
}
|
|
}
|
|
}
|
|
results := make([]PRWatchSyncResult, 0, len(watches))
|
|
for _, watch := range watches {
|
|
if result, ok := resultsByWatch[watch]; ok {
|
|
results = append(results, result)
|
|
} else {
|
|
results = append(results, PRWatchSyncResult{Watch: watch})
|
|
}
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func noOpPRWatchSyncResults(watches []*PRWatch) []PRWatchSyncResult {
|
|
results := make([]PRWatchSyncResult, 0, len(watches))
|
|
for _, watch := range watches {
|
|
results = append(results, PRWatchSyncResult{Watch: watch})
|
|
}
|
|
return results
|
|
}
|
|
|
|
func (s *Service) handleBatchedDiscoveryFetchError(
|
|
workspaceID, cacheScope string,
|
|
credentialGeneration int64,
|
|
leaderGroups []*prWatchBatchGroup,
|
|
admitted, numbered, searching []*PRWatch,
|
|
err error,
|
|
) bool {
|
|
failed := admitted
|
|
if strings.Contains(err.Error(), "batched branch query") {
|
|
failed = searching
|
|
} else if strings.Contains(err.Error(), "batched PR query") {
|
|
failed = numbered
|
|
}
|
|
category := classifyPRDiscoveryError(err)
|
|
pauseFailedTargets := category == PRDiscoveryHealthInvalidQuery || category == PRDiscoveryHealthRateLimited
|
|
failedSet := make(map[string]struct{}, len(failed))
|
|
for _, watch := range failed {
|
|
failedSet[prDiscoveryTargetForWatch(watch).key()] = struct{}{}
|
|
}
|
|
for _, group := range leaderGroups {
|
|
if _, failed := failedSet[group.key]; failed && pauseFailedTargets {
|
|
s.finishPRDiscoveryWatchFailure(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt, err,
|
|
)
|
|
} else {
|
|
s.releasePRDiscoveryWatch(
|
|
workspaceID, cacheScope, credentialGeneration, group.attempt,
|
|
)
|
|
}
|
|
s.completePRDiscoveryWatchAttempt(group.attempt, nil, false, true, err, pauseFailedTargets)
|
|
s.forgetPRDiscoveryWatchAttempt(workspaceID, cacheScope, credentialGeneration, group.attempt)
|
|
}
|
|
return pauseFailedTargets
|
|
}
|
|
|
|
const (
|
|
workflowAttentionBatchConcurrency = 4
|
|
workflowAttentionBatchBudget = 5 * time.Second
|
|
)
|
|
|
|
type workflowAttentionStatusGroup struct {
|
|
owner, repo, headSHA string
|
|
statuses []*PRStatus
|
|
}
|
|
|
|
// enrichBatchedWorkflowAttention adds Actions evidence to GraphQL statuses.
|
|
// Workflow runs are fetched once per credential scope, target repository, and
|
|
// head SHA; job reads are shared within that group. The enrichment is best
|
|
// effort so missing Actions permission never discards review or check data.
|
|
func (s *Service) enrichBatchedWorkflowAttention(
|
|
ctx context.Context, client Client, cacheScope string, statuses map[string]*PRStatus,
|
|
) {
|
|
groups := groupBatchedWorkflowAttentionStatuses(cacheScope, statuses)
|
|
if len(groups) == 0 {
|
|
return
|
|
}
|
|
|
|
baseCtx, cancelBase := derivedFetchContext(ctx)
|
|
defer cancelBase()
|
|
fetchCtx, cancel := context.WithTimeout(baseCtx, workflowAttentionBatchBudget)
|
|
defer cancel()
|
|
sem := make(chan struct{}, workflowAttentionBatchConcurrency)
|
|
var wg sync.WaitGroup
|
|
for _, group := range groups {
|
|
group := group
|
|
wg.Add(1)
|
|
go s.enrichBatchedWorkflowAttentionGroup(fetchCtx, client, cacheScope, group, sem, &wg)
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func groupBatchedWorkflowAttentionStatuses(
|
|
cacheScope string, statuses map[string]*PRStatus,
|
|
) map[string]*workflowAttentionStatusGroup {
|
|
groups := make(map[string]*workflowAttentionStatusGroup)
|
|
for _, status := range statuses {
|
|
if status == nil || status.PR == nil {
|
|
continue
|
|
}
|
|
if isTerminalPR(status.PR) {
|
|
status.WorkflowAttention = workflowAttentionNone(status.PR.HeadSHA)
|
|
status.WorkflowAttentionPopulated = true
|
|
continue
|
|
}
|
|
if status.PR.HeadSHA == "" {
|
|
status.WorkflowAttention = workflowAttentionUnknown("")
|
|
status.WorkflowAttentionPopulated = true
|
|
continue
|
|
}
|
|
key := workflowAttentionBatchKey(cacheScope, status.PR.RepoOwner, status.PR.RepoName, status.PR.HeadSHA)
|
|
group := groups[key]
|
|
if group == nil {
|
|
group = &workflowAttentionStatusGroup{
|
|
owner: status.PR.RepoOwner, repo: status.PR.RepoName, headSHA: status.PR.HeadSHA,
|
|
}
|
|
groups[key] = group
|
|
}
|
|
group.statuses = append(group.statuses, status)
|
|
}
|
|
return groups
|
|
}
|
|
|
|
func acquireWorkflowAttentionBatchSlot(ctx context.Context, sem chan struct{}) bool {
|
|
select {
|
|
case sem <- struct{}{}:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
|
|
func markBatchedWorkflowAttentionUnknown(group *workflowAttentionStatusGroup) {
|
|
for _, status := range group.statuses {
|
|
status.WorkflowAttention = workflowAttentionUnknown(group.headSHA)
|
|
status.WorkflowAttentionPopulated = true
|
|
}
|
|
}
|
|
|
|
func (s *Service) enrichBatchedWorkflowAttentionGroup(
|
|
ctx context.Context,
|
|
client Client,
|
|
cacheScope string,
|
|
group *workflowAttentionStatusGroup,
|
|
sem chan struct{},
|
|
wg *sync.WaitGroup,
|
|
) {
|
|
defer wg.Done()
|
|
if !acquireWorkflowAttentionBatchSlot(ctx, sem) {
|
|
markBatchedWorkflowAttentionUnknown(group)
|
|
return
|
|
}
|
|
defer func() { <-sem }()
|
|
|
|
runs, err := s.cachedWorkflowRuns(ctx, client, cacheScope, group.owner, group.repo, group.headSHA)
|
|
if err != nil {
|
|
if s.logger != nil {
|
|
s.logger.Debug("workflow attention read unavailable",
|
|
zap.String("owner", group.owner), zap.String("repo", group.repo), zap.Error(err))
|
|
}
|
|
markBatchedWorkflowAttentionUnknown(group)
|
|
return
|
|
}
|
|
s.applyBatchedWorkflowAttention(ctx, client, cacheScope, group, runs)
|
|
}
|
|
|
|
func (s *Service) applyBatchedWorkflowAttention(
|
|
ctx context.Context, client Client, cacheScope string, group *workflowAttentionStatusGroup, runs []WorkflowRun,
|
|
) {
|
|
jobs := make(map[workflowJobKey]struct {
|
|
value []WorkflowJob
|
|
err error
|
|
})
|
|
readJobs := func(jobCtx context.Context, runID int64, attempt int) ([]WorkflowJob, error) {
|
|
jobKey := workflowJobKey{Owner: group.owner, Repo: group.repo, RunID: runID, Attempt: attempt}
|
|
if cached, found := jobs[jobKey]; found {
|
|
return cached.value, cached.err
|
|
}
|
|
value, readErr := s.cachedWorkflowJobs(
|
|
jobCtx, client, cacheScope, group.owner, group.repo, runID, attempt, true,
|
|
)
|
|
jobs[jobKey] = struct {
|
|
value []WorkflowJob
|
|
err error
|
|
}{value: value, err: readErr}
|
|
return value, readErr
|
|
}
|
|
for _, status := range group.statuses {
|
|
if status == nil || status.PR == nil {
|
|
continue
|
|
}
|
|
status.WorkflowAttention = classifyWorkflowAttentionWithJobs(
|
|
ctx, group.owner, group.repo, status.PR, runs, readJobs,
|
|
)
|
|
status.WorkflowAttentionPopulated = true
|
|
}
|
|
}
|
|
|
|
func workflowAttentionBatchKey(cacheScope, owner, repo, headSHA string) string {
|
|
return workflowRunsCacheKey(cacheScope, owner, repo, headSHA)
|
|
}
|
|
|
|
// batchedWatchStatuses is the shared, read-only result of one batched fetch.
|
|
//
|
|
// byKey holds every PR the batch resolved, keyed by prStatusCacheKey for
|
|
// numbered watches and graphqlBranchKey for searching ones. branchResolvedEmpty
|
|
// holds the branch keys the batch answered definitively as "no open PR" — see
|
|
// branchBatchResult for why that distinction matters.
|
|
type batchedWatchStatuses struct {
|
|
byKey map[string]*PRStatus
|
|
branchResolvedEmpty map[string]struct{}
|
|
}
|
|
|
|
// fetchBatchedWatchStatuses runs the numbered- and branch-keyed GraphQL
|
|
// queries and merges their results. Returns an error when either query
|
|
// fails so the caller can fall back to per-watch checks. Watches whose
|
|
// (owner, repo) is already cached as missing are filtered out before
|
|
// hitting GraphQL, and any newly-discovered missing repos in the
|
|
// response are fed into the negative cache so subsequent polls
|
|
// short-circuit before acquiring the gh throttle.
|
|
//
|
|
// The full fetch is gated by a service-level singleflight keyed by the
|
|
// sorted ref set. Without it, a burst of concurrent SyncWatchesBatched
|
|
// calls (poller racing the WS sync racing the workspace background
|
|
// refresh) all hit GraphQL during the window before the first caller
|
|
// seeds the negative cache — exactly the storm this change is trying to
|
|
// calm. The shared-result map is read-only at the call sites (only
|
|
// map lookups in applyBatched*), so it's safe to share.
|
|
//
|
|
// The upstream fetch detaches from the leader's cancellation via
|
|
// derivedFetchContext so one caller (WS) disconnecting mid-flight
|
|
// doesn't cascade context.Canceled to all co-waiters — same pattern as
|
|
// GetPRFeedback / GetPRStatus. The leader's deadline is preserved so
|
|
// the fetch can't outlive the request budget.
|
|
func (s *Service) fetchBatchedWatchStatuses(
|
|
ctx context.Context, client Client, exec GraphQLExecutor, cacheScope string,
|
|
numbered, searching []*PRWatch, explicitRefresh bool,
|
|
) (*batchedWatchStatuses, error) {
|
|
key := scopedCacheKey(cacheScope, batchedFetchSingleflightKey(numbered, searching))
|
|
if explicitRefresh {
|
|
key += prSyncExplicitRefreshKeySuffix
|
|
}
|
|
fetchCtx, cancelFetch := derivedFetchContext(ctx)
|
|
defer cancelFetch()
|
|
v, err, _ := s.syncGroup.Do(key, func() (interface{}, error) {
|
|
// Snapshot the negative-cache generation BEFORE the batched
|
|
// fetch so an eviction (relink / clear) firing while the
|
|
// GraphQL call is in flight wins over the post-fetch
|
|
// markRepoAsMissing writes — see Service.markRepoAsMissing.
|
|
repoErrGen := s.repoErrorGenSnapshot()
|
|
combined := &batchedWatchStatuses{
|
|
byKey: make(map[string]*PRStatus, len(numbered)+len(searching)),
|
|
branchResolvedEmpty: make(map[string]struct{}, len(searching)),
|
|
}
|
|
if err := s.fetchBatchedPRStatuses(fetchCtx, exec, cacheScope, numbered, combined.byKey, repoErrGen); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := s.fetchBatchedBranchStatuses(fetchCtx, exec, cacheScope, searching, combined, repoErrGen); err != nil {
|
|
return nil, err
|
|
}
|
|
s.enrichBatchedWorkflowAttention(fetchCtx, client, cacheScope, combined.byKey)
|
|
return combined, nil
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if v == nil {
|
|
return nil, nil
|
|
}
|
|
return v.(*batchedWatchStatuses), nil
|
|
}
|
|
|
|
// batchedFetchSingleflightKey builds a deterministic key from the sorted
|
|
// ref tuples in (numbered, searching). Sorting ensures ordering doesn't
|
|
// split equivalent calls into separate slots, and lowercasing matches
|
|
// repoErrorCacheKey so the in-flight key is stable regardless of caller
|
|
// casing (watches from the DB always have consistent casing in practice,
|
|
// but it costs nothing to make the key insensitive). Distinct "n:"/"b:"
|
|
// prefixes prevent a numbered ref colliding with a branch ref that
|
|
// happens to render the same string.
|
|
func batchedFetchSingleflightKey(numbered, searching []*PRWatch) string {
|
|
parts := make([]string, 0, len(numbered)+len(searching))
|
|
for _, w := range numbered {
|
|
parts = append(parts, fmt.Sprintf("n:%s/%s#%d",
|
|
strings.ToLower(w.Owner), strings.ToLower(w.Repo), w.PRNumber))
|
|
}
|
|
for _, w := range searching {
|
|
parts = append(parts, fmt.Sprintf("b:%s/%s@%s",
|
|
strings.ToLower(w.Owner), strings.ToLower(w.Repo), w.Branch))
|
|
}
|
|
sort.Strings(parts)
|
|
return "batched-fetch:" + strings.Join(parts, "|")
|
|
}
|
|
|
|
// fetchBatchedPRStatuses runs the numbered-watch (PR-number-keyed) batch.
|
|
// Watches whose repo is in the negative cache are dropped before building
|
|
// the GraphQL refs so the dead-repo storm doesn't burn gh throttle slots.
|
|
// Missing repos surfaced by the response are negative-cached for the next
|
|
// 10 minutes; partial results for the repos that did resolve are merged
|
|
// into `combined`.
|
|
func (s *Service) fetchBatchedPRStatuses(
|
|
ctx context.Context, exec GraphQLExecutor, cacheScope string, numbered []*PRWatch, combined map[string]*PRStatus,
|
|
repoErrGen uint64,
|
|
) error {
|
|
refs := make([]graphQLPRRef, 0, len(numbered))
|
|
for _, w := range numbered {
|
|
if s.isRepoCachedAsMissingForScope(cacheScope, w.Owner, w.Repo) {
|
|
continue
|
|
}
|
|
refs = append(refs, graphQLPRRef{Owner: w.Owner, Repo: w.Repo, Number: w.PRNumber})
|
|
}
|
|
if len(refs) == 0 {
|
|
return nil
|
|
}
|
|
out, err := runBatchedPRQuery(ctx, exec, refs)
|
|
out, err = s.absorbMissingReposErr(out, err, cacheScope, repoErrGen)
|
|
if err != nil {
|
|
return fmt.Errorf("batched PR query: %w", err)
|
|
}
|
|
for k, v := range out {
|
|
combined[k] = v
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// fetchBatchedBranchStatuses runs the branch-keyed batch with the same
|
|
// negative-cache filter and missing-repo absorption as the numbered path.
|
|
func (s *Service) fetchBatchedBranchStatuses(
|
|
ctx context.Context, exec GraphQLExecutor, cacheScope string, searching []*PRWatch, combined *batchedWatchStatuses,
|
|
repoErrGen uint64,
|
|
) error {
|
|
refs := make([]graphQLBranchRef, 0, len(searching))
|
|
for _, w := range searching {
|
|
if s.isRepoCachedAsMissingForScope(cacheScope, w.Owner, w.Repo) {
|
|
continue
|
|
}
|
|
refs = append(refs, graphQLBranchRef{Owner: w.Owner, Repo: w.Repo, Branch: w.Branch})
|
|
}
|
|
if len(refs) == 0 {
|
|
return nil
|
|
}
|
|
out, err := runBatchedBranchQuery(ctx, exec, refs)
|
|
statuses, err := s.absorbMissingReposErr(out.Statuses, err, cacheScope, repoErrGen)
|
|
if err != nil {
|
|
return fmt.Errorf("batched branch query: %w", err)
|
|
}
|
|
for k, v := range statuses {
|
|
combined.byKey[k] = v
|
|
}
|
|
// Only carry the definitive negatives that survived error absorption. A
|
|
// purely-missing-repos error keeps its partial decode; anything else made
|
|
// absorbMissingReposErr return early above.
|
|
if statuses != nil {
|
|
for k := range out.ResolvedEmpty {
|
|
combined.branchResolvedEmpty[k] = struct{}{}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// absorbMissingReposErr extracts the negative-cacheable repos from a
|
|
// runBatched* result and seeds the cache, then returns (partial,
|
|
// remaining-err). When the error was PURELY missing-repo errors, the
|
|
// partial decode is preserved and a nil error is returned so the caller
|
|
// can use the resolved repos' data. When other errors were also
|
|
// present, the inner error is returned unchanged so the caller falls
|
|
// back to per-watch checks. `repoErrGen` is the negative-cache generation
|
|
// snapshot taken BEFORE the fetch so a concurrent eviction wins.
|
|
func (s *Service) absorbMissingReposErr(
|
|
out map[string]*PRStatus, err error, cacheScope string, repoErrGen uint64,
|
|
) (map[string]*PRStatus, error) {
|
|
if err == nil {
|
|
return out, nil
|
|
}
|
|
var missingErr *batchedMissingReposErr
|
|
if !errors.As(err, &missingErr) {
|
|
return nil, err
|
|
}
|
|
for _, r := range missingErr.Repos {
|
|
s.markRepoAsMissingForScope(cacheScope, r.Owner, r.Repo, repoErrGen)
|
|
s.logger.Debug("repo not resolvable; negative-cached",
|
|
zap.String("owner", r.Owner), zap.String("repo", r.Repo))
|
|
}
|
|
if missingErr.Inner != nil {
|
|
return nil, missingErr.Inner
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// applyBatchedNumberedWatch mirrors Poller.applyPRStatus on the service
|
|
// side so on-demand callers reuse the same DB-write sequence.
|
|
func (s *Service) applyBatchedNumberedWatch(
|
|
ctx context.Context, cacheScope string, w *PRWatch, statusByKey map[string]*PRStatus, now time.Time,
|
|
) PRWatchSyncResult {
|
|
// Repo is in the 10-min negative cache — the fetch path didn't include
|
|
// it in the GraphQL batch, so the apply path can't probe upstream either.
|
|
// Bump last_checked_at to suppress retry storms and surface SyncFailed
|
|
// so the WS handler can flag the result as permanent for the frontend.
|
|
if s.isRepoCachedAsMissingForScope(cacheScope, w.Owner, w.Repo) {
|
|
_ = s.store.UpdatePRWatchTimestamps(ctx, w.ID, now, w.LastCommentAt, "", "")
|
|
return PRWatchSyncResult{Watch: w, SyncFailed: true}
|
|
}
|
|
status, ok := statusByKey[prStatusCacheKey(w.Owner, w.Repo, w.PRNumber)]
|
|
if !ok || status == nil {
|
|
// Alias missing — best-effort liveness bump so we don't immediately re-probe.
|
|
_ = s.store.UpdatePRWatchTimestamps(ctx, w.ID, now, w.LastCommentAt, "", "")
|
|
return PRWatchSyncResult{Watch: w}
|
|
}
|
|
changed := status.ChecksState != w.LastCheckStatus ||
|
|
status.ReviewState != w.LastReviewState ||
|
|
prWatchFeedbackUpdatedSinceWatch(w, status)
|
|
commentAt := prWatchFeedbackWatermark(w, status)
|
|
if err := s.store.UpdatePRWatchTimestamps(ctx, w.ID, now, commentAt, status.ChecksState, status.ReviewState); err != nil {
|
|
s.logSyncError("failed to update PR watch timestamps", err, zap.String("id", w.ID))
|
|
}
|
|
// A numbered watch found its PR before this fix existed (or before the
|
|
// discovering session's own group redirect took effect) keeps the
|
|
// observing member's task_id forever — watches are never re-pointed once
|
|
// they have a PR (see apps/backend/CLAUDE.md's "PR status sync coverage").
|
|
// Resolve the effective owner here, the same way associatePRWithTask does
|
|
// for the association write, so this loop keeps syncing the owner's
|
|
// github_task_prs row instead of silently updating nothing.
|
|
effectiveTaskID := s.reconcileTaskPROwnership(ctx, w.SessionID, w.TaskID, w.RepositoryID, w.PRNumber)
|
|
|
|
// Gap-fill: a numbered watch can exist even when its exact task_pr row was
|
|
// never created. This targeted read is unconditional because the common
|
|
// existing-row path is cheap, and the missing-row path must repair before
|
|
// SyncTaskPR. If AssociatePRWithTask creates the row, it publishes the
|
|
// creation event; the following SyncTaskPR may publish a second event when
|
|
// status fields changed. That double event is harmless because clients
|
|
// re-fetch the task PR state.
|
|
if existing, err := s.store.GetTaskPRByRepoAndNumber(ctx, effectiveTaskID, w.RepositoryID, w.PRNumber); err != nil {
|
|
s.logSyncError("failed to load exact task PR", err,
|
|
zap.String("task_id", effectiveTaskID), zap.String("repository_id", w.RepositoryID),
|
|
zap.Int("pr_number", w.PRNumber))
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, SyncFailed: true}
|
|
} else if existing == nil && status.PR != nil {
|
|
if _, assocErr := s.associatePRWithTaskForSession(
|
|
ctx, w.WorkspaceID, w.SessionID, w.TaskID, w.RepositoryID, status.PR,
|
|
false, false, TaskPRSourceWatch,
|
|
); assocErr != nil {
|
|
s.logSyncError("failed to associate numbered PR with task", assocErr,
|
|
zap.String("task_id", w.TaskID), zap.Int("pr_number", w.PRNumber))
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, SyncFailed: true}
|
|
}
|
|
}
|
|
if syncErr := s.SyncTaskPR(ctx, effectiveTaskID, status); syncErr != nil {
|
|
s.logSyncError("failed to sync task PR", syncErr, zap.String("task_id", effectiveTaskID))
|
|
// SyncFailed=true so poller skips publishing PR feedback while the
|
|
// task_pr row is still stale — old applyPRStatus path early-returned
|
|
// on this error for the same reason.
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, SyncFailed: true}
|
|
}
|
|
// Reset to "searching" when the PR is merged/closed so a follow-up PR on the
|
|
// same branch can be detected without manual intervention.
|
|
if status.PR != nil && (status.PR.State == prStateMerged || status.PR.State == prStateClosed) {
|
|
hold, holdErr := s.ShouldHoldTerminalPRWatch(
|
|
ctx, effectiveTaskID, w.RepositoryID, w.PRNumber, status.PR.State,
|
|
)
|
|
if holdErr != nil {
|
|
s.logSyncError("failed to check terminal PR automation", holdErr, zap.String("id", w.ID))
|
|
// Fail conservatively like stale task-PR state: suppress feedback
|
|
// until terminal lifecycle retention can be determined safely.
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, Changed: changed, SyncFailed: true}
|
|
}
|
|
if !hold {
|
|
if resetErr := s.store.UpdatePRWatchPRNumber(ctx, w.ID, 0); resetErr != nil {
|
|
s.logSyncError("failed to reset completed PR watch", resetErr, zap.String("id", w.ID))
|
|
}
|
|
}
|
|
}
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, Changed: changed}
|
|
}
|
|
|
|
// lookupSearchingWatchPR resolves the PR for a searching watch the batched
|
|
// query returned nothing for. Returns (nil, nil) when there is no PR to
|
|
// promote — including when the probe was skipped.
|
|
//
|
|
// The batched query already asked GitHub `pullRequests(states: OPEN,
|
|
// headRefName: <branch>)` for this exact repository and branch. When it came
|
|
// back resolved-and-empty, re-asking through client.FindPRByBranch spends one
|
|
// more API call per watch per cycle to receive the same answer — and since a
|
|
// branch without a PR keeps that shape indefinitely, that call repeats forever.
|
|
// That redundancy was the dominant GitHub GraphQL consumer in production: 42
|
|
// searching watches burned ~150 points/minute against a 5000/hour budget, which
|
|
// parked the poller on `github rate-limited` roughly 40 minutes into every
|
|
// reset window. So on a definitive negative only the fork PARENT — a different
|
|
// repository the batch never asked about — is still worth a look, and that
|
|
// lookup is itself cached per (scope, owner, repo).
|
|
//
|
|
// An alias that did not resolve, or a branch carrying several open PRs that
|
|
// selectBatchedBranchPRNode refused to guess between, is not a definitive
|
|
// negative and keeps the full fork-network fallback.
|
|
//
|
|
// Either way the probe is throttled on the watch's own last_checked_at exactly
|
|
// as triggerPRDetection's is, so a watch costs at most one upstream probe per
|
|
// freshness window no matter how many callers sync it — the frontend retries
|
|
// every 5s, and that throttle previously guarded only the legacy per-watch
|
|
// path this one replaced. The 60s poller always clears the window, so
|
|
// detection latency is unchanged.
|
|
func (s *Service) lookupSearchingWatchPR(
|
|
ctx context.Context, client Client, cacheScope string, w *PRWatch,
|
|
branchKey string, statuses *batchedWatchStatuses, lastChecked *time.Time,
|
|
) (*PR, bool, error) {
|
|
if lastChecked != nil && time.Since(*lastChecked) < PRSyncFreshnessWindow {
|
|
return nil, false, nil
|
|
}
|
|
if _, definitive := statuses.branchResolvedEmpty[branchKey]; definitive {
|
|
pr, err := s.findPRInForkParent(ctx, client, cacheScope, w.Owner, w.Repo, w.Branch)
|
|
return pr, true, err
|
|
}
|
|
pr, err := s.findPRByBranchInForkNetwork(ctx, client, cacheScope, w.Owner, w.Repo, w.Branch)
|
|
return pr, true, err
|
|
}
|
|
|
|
// applyBatchedSearchingWatch mirrors Poller.applyDetectedPR on the service
|
|
// side. A searching watch (pr_number=0) is promoted to a known PR when the
|
|
// branch lookup returns one; otherwise we just bump last_checked_at.
|
|
func (s *Service) applyBatchedSearchingWatch(
|
|
ctx context.Context, client Client, cacheScope string, w *PRWatch, statuses *batchedWatchStatuses, now time.Time,
|
|
skipFallback, sharedResolved, sharedSyncFailed bool,
|
|
) PRWatchSyncResult {
|
|
// Read the previous probe time before the stamp below overwrites it —
|
|
// lookupSearchingWatchPR throttles on it.
|
|
lastChecked := w.LastCheckedAt
|
|
// Timestamps get bumped on both the "no PR found" and "PR detected"
|
|
// paths, so hoist the single call above the branch to make that
|
|
// invariant obvious.
|
|
_ = s.store.UpdatePRWatchTimestamps(ctx, w.ID, now, nil, "", "")
|
|
if s.isRepoCachedAsMissingForScope(cacheScope, w.Owner, w.Repo) {
|
|
return PRWatchSyncResult{Watch: w, SyncFailed: true}
|
|
}
|
|
branchKey := graphqlBranchKey(w.Owner, w.Repo, w.Branch)
|
|
status, ok := statuses.byKey[branchKey]
|
|
discoveryResolved := ok && status != nil && status.PR != nil
|
|
if !ok || status == nil || status.PR == nil {
|
|
if skipFallback {
|
|
return PRWatchSyncResult{Watch: w, DiscoveryResolved: sharedResolved, SyncFailed: sharedSyncFailed}
|
|
}
|
|
pr, attempted, err := s.lookupSearchingWatchPR(ctx, client, cacheScope, w, branchKey, statuses, lastChecked)
|
|
if err != nil {
|
|
s.logger.Debug("failed to search parent repository for PR",
|
|
zap.String("watch_id", w.ID), zap.String("branch", w.Branch), zap.Error(err))
|
|
return PRWatchSyncResult{Watch: w, SyncFailed: true}
|
|
}
|
|
if !attempted {
|
|
return PRWatchSyncResult{Watch: w}
|
|
}
|
|
discoveryResolved = true
|
|
if pr == nil {
|
|
return PRWatchSyncResult{Watch: w, DiscoveryResolved: true}
|
|
}
|
|
status = &PRStatus{PR: pr}
|
|
}
|
|
if err := s.rebindPRWatchRepository(ctx, w, status.PR); err != nil {
|
|
s.logSyncError("failed to rebind PR watch to detected repository", err,
|
|
zap.String("watch_id", w.ID), zap.Int("pr_number", status.PR.Number))
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, SyncFailed: true, DiscoveryResolved: discoveryResolved}
|
|
}
|
|
if err := s.store.UpdatePRWatchPRNumber(ctx, w.ID, status.PR.Number); err != nil {
|
|
s.logSyncError("failed to update PR watch with detected PR", err,
|
|
zap.String("watch_id", w.ID), zap.Int("pr_number", status.PR.Number))
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, DiscoveryResolved: discoveryResolved}
|
|
}
|
|
if _, err := s.associatePRWithTaskForSession(
|
|
ctx, w.WorkspaceID, w.SessionID, w.TaskID, w.RepositoryID, status.PR,
|
|
false, false, TaskPRSourceWatch,
|
|
); err != nil {
|
|
s.logSyncError("failed to associate detected PR with task", err,
|
|
zap.String("task_id", w.TaskID), zap.Int("pr_number", status.PR.Number))
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, DiscoveryResolved: discoveryResolved}
|
|
}
|
|
s.logger.Info("detected PR for session branch (batched)",
|
|
zap.String("watch_id", w.ID), zap.String("branch", w.Branch), zap.Int("pr_number", status.PR.Number))
|
|
return PRWatchSyncResult{Watch: w, Status: status, Found: true, DiscoveryResolved: discoveryResolved}
|
|
}
|