From 9944354518fabc0d3d012f25bab59f74a7b64237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Wed, 23 Sep 2026 13:31:12 +0100 Subject: [PATCH] fix: reduce GitHub cleanup and watch polling load (#3866) --- .../internal/azuredevops/controller_test.go | 2 +- .../controller_watch_routes_test.go | 4 +- .../service_task_work_item_test.go | 2 +- .../azuredevops/store_postgres_test.go | 3 +- .../internal/azuredevops/store_test.go | 16 +- .../azuredevops/watch_cleanup_test.go | 104 ++++ .../watch_service_pull_request_test.go | 1 + .../azuredevops/watch_service_trigger_test.go | 1 + .../internal/azuredevops/watch_store.go | 18 +- .../azuredevops/watch_store_mutation_test.go | 2 + .../internal/azuredevops/watch_store_test.go | 137 +++++ .../github_task_activity_adapter.go | 35 ++ apps/backend/internal/backendapp/main.go | 2 + .../websocket/task_notifications_test.go | 2 +- apps/backend/internal/github/gh_client.go | 15 +- .../internal/github/gh_client_runtime_test.go | 16 + apps/backend/internal/github/handlers.go | 23 +- .../internal/github/load_validation_test.go | 3 + apps/backend/internal/github/poller.go | 396 +++++++++++-- .../github/poller_cleanup_rate_limit_test.go | 72 +++ .../internal/github/poller_schedule.go | 198 +++++++ .../internal/github/poller_schedule_test.go | 297 ++++++++++ apps/backend/internal/github/poller_test.go | 258 ++++++++- .../internal/github/review_pr_task_test.go | 1 + apps/backend/internal/github/service.go | 89 ++- .../internal/github/service_cleanup.go | 222 ++++++-- .../github/service_cleanup_policy_test.go | 29 + .../github/service_cleanup_rate_limit_test.go | 437 +++++++++++++++ .../internal/github/service_connections.go | 2 +- .../backend/internal/github/service_issues.go | 10 +- apps/backend/internal/github/service_pr.go | 15 + .../github/service_pr_discovery_attempt.go | 35 +- .../internal/github/service_pr_sync_test.go | 4 +- .../internal/github/service_pr_unwatched.go | 167 ++++-- .../github/service_pr_unwatched_test.go | 115 +++- .../internal/github/service_pr_watch.go | 522 +++++++++++++++--- .../github/service_pr_watch_batched.go | 52 +- .../github/service_pr_watch_fallback_test.go | 272 +++++++++ .../github/service_pr_watch_schedule_test.go | 337 +++++++++++ .../github/service_review_cleanup_test.go | 275 ++++++++- .../github/service_review_reason_test.go | 1 + .../internal/github/service_reviews.go | 10 +- .../internal/github/service_routing.go | 2 + .../github/service_search_cache_test.go | 2 + apps/backend/internal/github/service_test.go | 8 + apps/backend/internal/github/store.go | 34 +- .../github/store_task_cleanup_test.go | 91 +++ apps/backend/internal/github/ttl_cache.go | 49 +- .../internal/github/workflow_attention.go | 16 + .../github/workflow_attention_cache.go | 203 +++++++ .../github/workflow_attention_cache_test.go | 305 ++++++++++ .../gitlab/controller_watch_scope_test.go | 2 + .../gitlab/service_cleanup_reason_test.go | 2 + .../internal/gitlab/service_cleanup_test.go | 110 ++++ .../gitlab/store_task_cleanup_test.go | 135 +++++ apps/backend/internal/gitlab/store_watches.go | 46 +- .../internal/task/models/pr_watch_activity.go | 11 + .../internal/task/repository/interface.go | 6 + .../repository/sqlite/pr_watch_activity.go | 148 +++++ .../sqlite/pr_watch_activity_test.go | 130 +++++ .../web/app/tasks/rich-task-list-row.test.tsx | 33 +- apps/web/app/tasks/rich-task-list-row.tsx | 3 +- ...e-pr-sidebar-automation-indicators.spec.ts | 4 +- .../tests/system/settings-header-tabs.spec.ts | 2 + .../github/task-pr-sync-resource.test.ts | 20 +- .../domains/github/task-pr-sync-resource.ts | 44 +- .../domains/github/use-pr-ci-popover.test.ts | 7 +- .../hooks/domains/github/use-pr-ci-popover.ts | 9 +- docs/plans/watch-task-cleanup/plan.md | 138 +++++ .../task-01-github-candidates.md | 103 ++++ .../task-02-github-rate-limits.md | 111 ++++ .../task-03-sibling-candidates.md | 102 ++++ .../task-04-minimal-cleanup.md | 92 +++ .../task-05-bounded-fallback.md | 109 ++++ .../task-06-adaptive-discovery.md | 153 +++++ .../task-07-actions-cache.md | 137 +++++ docs/public/integrations.md | 24 + .../github-pr-discovery-health.md | 19 + .../requirements/github-workflow-attention.md | 11 +- .../requirements/watch-task-cleanup.md | 41 ++ .../github-pr-discovery-health.md | 85 +++ .../github-workflow-attention.md | 40 ++ .../system-design/watch-task-cleanup.md | 126 +++++ 83 files changed, 6567 insertions(+), 348 deletions(-) create mode 100644 apps/backend/internal/backendapp/github_task_activity_adapter.go create mode 100644 apps/backend/internal/github/poller_cleanup_rate_limit_test.go create mode 100644 apps/backend/internal/github/poller_schedule.go create mode 100644 apps/backend/internal/github/poller_schedule_test.go create mode 100644 apps/backend/internal/github/service_cleanup_rate_limit_test.go create mode 100644 apps/backend/internal/github/service_pr_watch_fallback_test.go create mode 100644 apps/backend/internal/github/service_pr_watch_schedule_test.go create mode 100644 apps/backend/internal/github/workflow_attention_cache.go create mode 100644 apps/backend/internal/github/workflow_attention_cache_test.go create mode 100644 apps/backend/internal/gitlab/service_cleanup_test.go create mode 100644 apps/backend/internal/task/models/pr_watch_activity.go create mode 100644 apps/backend/internal/task/repository/sqlite/pr_watch_activity.go create mode 100644 apps/backend/internal/task/repository/sqlite/pr_watch_activity_test.go create mode 100644 docs/plans/watch-task-cleanup/plan.md create mode 100644 docs/plans/watch-task-cleanup/task-01-github-candidates.md create mode 100644 docs/plans/watch-task-cleanup/task-02-github-rate-limits.md create mode 100644 docs/plans/watch-task-cleanup/task-03-sibling-candidates.md create mode 100644 docs/plans/watch-task-cleanup/task-04-minimal-cleanup.md create mode 100644 docs/plans/watch-task-cleanup/task-05-bounded-fallback.md create mode 100644 docs/plans/watch-task-cleanup/task-06-adaptive-discovery.md create mode 100644 docs/plans/watch-task-cleanup/task-07-actions-cache.md create mode 100644 docs/specs/integrations/requirements/watch-task-cleanup.md create mode 100644 docs/specs/integrations/system-design/watch-task-cleanup.md diff --git a/apps/backend/internal/azuredevops/controller_test.go b/apps/backend/internal/azuredevops/controller_test.go index e7767cf4c9..7bf005eaa1 100644 --- a/apps/backend/internal/azuredevops/controller_test.go +++ b/apps/backend/internal/azuredevops/controller_test.go @@ -346,7 +346,7 @@ func TestControllerTaskPullRequestAssociationRoutes(t *testing.T) { WorkspaceID: "ws-1", Provider: RepositoryProvider, ProviderOwner: "project-1", ProviderRepoID: "repo-1", }}) - if _, err := service.Store().db.Exec(`CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { + if _, err := service.Store().db.Exec(`CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { t.Fatalf("create tasks table: %v", err) } if _, err := service.Store().db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1')`); err != nil { diff --git a/apps/backend/internal/azuredevops/controller_watch_routes_test.go b/apps/backend/internal/azuredevops/controller_watch_routes_test.go index 966525dfda..a329d918ff 100644 --- a/apps/backend/internal/azuredevops/controller_watch_routes_test.go +++ b/apps/backend/internal/azuredevops/controller_watch_routes_test.go @@ -190,7 +190,7 @@ func TestControllerTaskWorkItemRoutes(t *testing.T) { }}}} router, service := newRouteFixture(t, client) seedRouteConfig(t, service) - if _, err := service.Store().db.Exec(`CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { + if _, err := service.Store().db.Exec(`CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { t.Fatalf("create tasks table: %v", err) } if _, err := service.Store().db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'), ('task-2', 'ws-other')`); err != nil { @@ -229,7 +229,7 @@ func TestControllerTaskWorkItemRouteRejectsInvalidAssociations(t *testing.T) { }} router, service := newRouteFixture(t, client) seedRouteConfig(t, service) - if _, err := service.Store().db.Exec(`CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { + if _, err := service.Store().db.Exec(`CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { t.Fatalf("create tasks table: %v", err) } if _, err := service.Store().db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'), ('task-2', 'ws-other')`); err != nil { diff --git a/apps/backend/internal/azuredevops/service_task_work_item_test.go b/apps/backend/internal/azuredevops/service_task_work_item_test.go index 1650c6eef2..520361dedf 100644 --- a/apps/backend/internal/azuredevops/service_task_work_item_test.go +++ b/apps/backend/internal/azuredevops/service_task_work_item_test.go @@ -10,7 +10,7 @@ type taskWorkItemClient struct{ fakeDetailClient } func TestTaskWorkItemAssociation(t *testing.T) { client := &taskWorkItemClient{fakeDetailClient: fakeDetailClient{detail: &WorkItemDetail{WorkItem: WorkItem{ID: 101, Title: "Ship it", Project: "project-1", WebURL: "https://dev.azure.com/acme/p/_workitems/edit/101"}}}} service, store, _ := newTestService(t, func(*Config, string) Client { return client }) - if _, err := store.db.Exec(`CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { + if _, err := store.db.Exec(`CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { t.Fatalf("create tasks table: %v", err) } if _, err := store.db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1')`); err != nil { diff --git a/apps/backend/internal/azuredevops/store_postgres_test.go b/apps/backend/internal/azuredevops/store_postgres_test.go index f61c841fa5..adfd45caf4 100644 --- a/apps/backend/internal/azuredevops/store_postgres_test.go +++ b/apps/backend/internal/azuredevops/store_postgres_test.go @@ -15,7 +15,8 @@ func TestPostgresStoreSchemaReplay(t *testing.T) { if _, err := database.Exec(` CREATE TABLE tasks ( id TEXT PRIMARY KEY, - workspace_id TEXT NOT NULL + workspace_id TEXT NOT NULL, + archived_at TIMESTAMP ); INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1')`); err != nil { t.Fatalf("create task prerequisite: %v", err) diff --git a/apps/backend/internal/azuredevops/store_test.go b/apps/backend/internal/azuredevops/store_test.go index 9bc7498365..4d109692d5 100644 --- a/apps/backend/internal/azuredevops/store_test.go +++ b/apps/backend/internal/azuredevops/store_test.go @@ -22,9 +22,23 @@ func newTestDB(t *testing.T) *sqlx.DB { raw.SetMaxIdleConns(1) db := sqlx.NewDb(raw, "sqlite3") t.Cleanup(func() { _ = db.Close() }) + if _, err := db.Exec(`CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + archived_at DATETIME + )`); err != nil { + t.Fatalf("create tasks table: %v", err) + } return db } +func seedAzureTask(t *testing.T, store *Store, taskID, workspaceID string) { + t.Helper() + if _, err := store.db.Exec(`INSERT OR IGNORE INTO tasks (id, workspace_id) VALUES (?, ?)`, taskID, workspaceID); err != nil { + t.Fatalf("seed task %s: %v", taskID, err) + } +} + func TestStoreConfigRoundTripAndHealth(t *testing.T) { db := newTestDB(t) store, err := NewStore(db, db) @@ -356,7 +370,7 @@ func TestStoreTaskPRConcurrentUpsertPreservesStableIdentity(t *testing.T) { func TestStoreTaskPRListsByTaskAndWorkspace(t *testing.T) { db := newTestDB(t) - if _, err := db.Exec(`CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS tasks (id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL)`); err != nil { t.Fatalf("create tasks table: %v", err) } if _, err := db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES ('task-a', 'ws-a'), ('task-b', 'ws-a'), ('task-c', 'ws-b')`); err != nil { diff --git a/apps/backend/internal/azuredevops/watch_cleanup_test.go b/apps/backend/internal/azuredevops/watch_cleanup_test.go index 7ae3a0b370..964c7585d2 100644 --- a/apps/backend/internal/azuredevops/watch_cleanup_test.go +++ b/apps/backend/internal/azuredevops/watch_cleanup_test.go @@ -13,6 +13,22 @@ type cleanupClient struct { pullRequest *PullRequest } +type countingCleanupClient struct { + *cleanupClient + workItemCalls int + pullRequestCalls int +} + +func (c *countingCleanupClient) GetWorkItem(ctx context.Context, projectID string, id int) (*WorkItem, error) { + c.workItemCalls++ + return c.cleanupClient.GetWorkItem(ctx, projectID, id) +} + +func (c *countingCleanupClient) GetPullRequest(ctx context.Context, projectID, repositoryID string, id int) (*PullRequest, error) { + c.pullRequestCalls++ + return c.cleanupClient.GetPullRequest(ctx, projectID, repositoryID, id) +} + func (c *cleanupClient) GetWorkItem(context.Context, string, int) (*WorkItem, error) { if c.workItem == nil { return nil, mockNotFound("work item", 0) @@ -55,6 +71,7 @@ func TestCleanupWorkItemWatchHonorsAutoEngagementAndTerminalState(t *testing.T) if _, err := service.SetConfigForWorkspace(t.Context(), "ws-1", &SetConfigRequest{OrganizationURL: "https://dev.azure.com/acme", PAT: "pat"}); err != nil { t.Fatalf("set config: %v", err) } + seedAzureTask(t, store, "task-101", "ws-1") watch := &WorkItemWatch{WorkspaceID: "ws-1", ProjectID: "project-1", WIQL: "SELECT", CleanupPolicy: CleanupPolicyAuto} if err := store.CreateWorkItemWatch(t.Context(), watch); err != nil { t.Fatalf("create watch: %v", err) @@ -110,3 +127,90 @@ func TestCleanupPullRequestWatchHonorsNeverPolicy(t *testing.T) { t.Fatalf("never policy deleted tasks: %v", deleter.taskIDs) } } + +func TestCleanupSkipsHistoricalTasks(t *testing.T) { + client := &countingCleanupClient{cleanupClient: &cleanupClient{ + workItem: &WorkItem{ID: 101, State: "Closed"}, + pullRequest: &PullRequest{ID: 201, Status: "completed"}, + }} + service, store, _ := newTestService(t, func(*Config, string) Client { return client }) + if _, err := service.SetConfigForWorkspace(t.Context(), "ws-1", &SetConfigRequest{OrganizationURL: "https://dev.azure.com/acme", PAT: "pat"}); err != nil { + t.Fatalf("set config: %v", err) + } + if _, err := store.db.Exec(`CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + archived_at DATETIME + )`); err != nil { + t.Fatalf("create tasks table: %v", err) + } + for _, taskID := range []string{"work-active", "work-archived", "pr-active", "pr-archived"} { + if _, err := store.db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES (?, ?)`, taskID, "ws-1"); err != nil { + t.Fatalf("seed task %s: %v", taskID, err) + } + } + for _, taskID := range []string{"work-archived", "pr-archived"} { + if _, err := store.db.Exec(`UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = ?`, taskID); err != nil { + t.Fatalf("archive task %s: %v", taskID, err) + } + } + + workWatch := &WorkItemWatch{ID: "work-cleanup-service", WorkspaceID: "ws-1", ProjectID: "project-1", WIQL: "SELECT", CleanupPolicy: CleanupPolicyAuto} + if err := store.CreateWorkItemWatch(t.Context(), workWatch); err != nil { + t.Fatalf("create work-item watch: %v", err) + } + pullRequestWatch := &PullRequestWatch{ID: "pr-cleanup-service", WorkspaceID: "ws-1", ProjectID: "project-1", AzureRepositoryID: "repo-1", CleanupPolicy: CleanupPolicyAuto} + if err := store.CreatePullRequestWatch(t.Context(), pullRequestWatch); err != nil { + t.Fatalf("create pull-request watch: %v", err) + } + reserveWork := func(itemID int, taskID string) { + t.Helper() + reserved, err := store.ReserveWorkItemWatchTask(t.Context(), workWatch.ID, workWatch.Generation, workWatch.ProjectID, itemID, "https://azure/item") + if err != nil || !reserved { + t.Fatalf("reserve work item %d: reserved=%v err=%v", itemID, reserved, err) + } + if taskID != "" { + if err := store.AssignWorkItemWatchTaskID(t.Context(), workWatch.ID, workWatch.Generation, workWatch.ProjectID, itemID, taskID); err != nil { + t.Fatalf("assign work item %d: %v", itemID, err) + } + } + } + reservePullRequest := func(id int, taskID string) { + t.Helper() + reserved, err := store.ReservePullRequestWatchTask(t.Context(), pullRequestWatch.ID, pullRequestWatch.Generation, pullRequestWatch.ProjectID, pullRequestWatch.AzureRepositoryID, id, "https://azure/pr") + if err != nil || !reserved { + t.Fatalf("reserve pull request %d: reserved=%v err=%v", id, reserved, err) + } + if taskID != "" { + if err := store.AssignPullRequestWatchTaskID(t.Context(), pullRequestWatch.ID, pullRequestWatch.Generation, pullRequestWatch.ProjectID, pullRequestWatch.AzureRepositoryID, id, taskID); err != nil { + t.Fatalf("assign pull request %d: %v", id, err) + } + } + } + reserveWork(101, "work-active") + reserveWork(102, "work-archived") + reserveWork(103, "work-missing") + reserveWork(104, "") + reservePullRequest(201, "pr-active") + reservePullRequest(202, "pr-archived") + reservePullRequest(203, "pr-missing") + reservePullRequest(204, "") + + deleter := &recordingCleanupDeleter{} + service.SetCascadeTaskDeleter(deleter) + if deleted, err := service.CleanupWorkItemWatch(t.Context(), "ws-1", workWatch.ID); err != nil || deleted != 1 { + t.Fatalf("work-item cleanup = %d, %v; want one deletion", deleted, err) + } + if deleted, err := service.CleanupPullRequestWatch(t.Context(), "ws-1", pullRequestWatch.ID); err != nil || deleted != 1 { + t.Fatalf("pull-request cleanup = %d, %v; want one deletion", deleted, err) + } + if client.workItemCalls != 1 { + t.Fatalf("work-item status calls = %d, want 1 for the active task", client.workItemCalls) + } + if client.pullRequestCalls != 1 { + t.Fatalf("pull-request status calls = %d, want 1 for the active task", client.pullRequestCalls) + } + if len(deleter.taskIDs) != 2 || deleter.taskIDs[0] != "work-active" || deleter.taskIDs[1] != "pr-active" { + t.Fatalf("deleted task IDs = %v, want active tasks only", deleter.taskIDs) + } +} diff --git a/apps/backend/internal/azuredevops/watch_service_pull_request_test.go b/apps/backend/internal/azuredevops/watch_service_pull_request_test.go index 67f63b1e0c..40b6164656 100644 --- a/apps/backend/internal/azuredevops/watch_service_pull_request_test.go +++ b/apps/backend/internal/azuredevops/watch_service_pull_request_test.go @@ -458,6 +458,7 @@ func seedPullRequestReservation(t *testing.T, store *Store, watch *PullRequestWa if taskID == "" { return } + seedAzureTask(t, store, taskID, watch.WorkspaceID) if err := store.AssignPullRequestWatchTaskID( t.Context(), watch.ID, watch.Generation, watch.ProjectID, watch.AzureRepositoryID, pullRequestID, taskID, diff --git a/apps/backend/internal/azuredevops/watch_service_trigger_test.go b/apps/backend/internal/azuredevops/watch_service_trigger_test.go index ea755ccf22..efd1b42d1a 100644 --- a/apps/backend/internal/azuredevops/watch_service_trigger_test.go +++ b/apps/backend/internal/azuredevops/watch_service_trigger_test.go @@ -293,6 +293,7 @@ func seedWorkItemReservation(t *testing.T, store *Store, watch *WorkItemWatch, w if taskID == "" { return } + seedAzureTask(t, store, taskID, watch.WorkspaceID) if err := store.AssignWorkItemWatchTaskID( t.Context(), watch.ID, watch.Generation, watch.ProjectID, workItemID, taskID, ); err != nil { diff --git a/apps/backend/internal/azuredevops/watch_store.go b/apps/backend/internal/azuredevops/watch_store.go index a3c3ca6b3b..bb0493dc3b 100644 --- a/apps/backend/internal/azuredevops/watch_store.go +++ b/apps/backend/internal/azuredevops/watch_store.go @@ -368,8 +368,13 @@ func (s *Store) ListWorkItemWatchTasks(ctx context.Context, watchID string, gene var rows []WorkItemWatchTask err := s.ro.SelectContext(ctx, &rows, s.ro.Rebind(`SELECT id, watch_id, project_id, work_item_id, work_item_url, task_id, generation, created_at - FROM azure_devops_work_item_watch_tasks - WHERE watch_id = ? AND generation = ? ORDER BY created_at`), watchID, generation) + FROM azure_devops_work_item_watch_tasks t + WHERE t.watch_id = ? AND t.generation = ? + AND (t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + )) + ORDER BY t.created_at`), watchID, generation) if err != nil { return nil, err } @@ -384,8 +389,13 @@ func (s *Store) ListPullRequestWatchTasks(ctx context.Context, watchID string, g var rows []PullRequestWatchTask err := s.ro.SelectContext(ctx, &rows, s.ro.Rebind(`SELECT id, watch_id, project_id, azure_repository_id, pull_request_id, pull_request_url, task_id, generation, - created_at FROM azure_devops_pull_request_watch_tasks - WHERE watch_id = ? AND generation = ? ORDER BY created_at`), watchID, generation) + created_at FROM azure_devops_pull_request_watch_tasks t + WHERE t.watch_id = ? AND t.generation = ? + AND (t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + )) + ORDER BY t.created_at`), watchID, generation) if err != nil { return nil, err } diff --git a/apps/backend/internal/azuredevops/watch_store_mutation_test.go b/apps/backend/internal/azuredevops/watch_store_mutation_test.go index c185121890..b43a906a41 100644 --- a/apps/backend/internal/azuredevops/watch_store_mutation_test.go +++ b/apps/backend/internal/azuredevops/watch_store_mutation_test.go @@ -359,6 +359,7 @@ func TestStoreAssignWatchTaskIDDistinguishesOwnershipLossFromMissingReservation( if ok, err := store.ReserveWorkItemWatchTask(t.Context(), work.ID, work.Generation, "project-1", 101, "https://azure/101"); err != nil || !ok { t.Fatalf("reserve work item: ok=%v err=%v", ok, err) } + seedAzureTask(t, store, "task-1", "ws-1") if err := store.AssignWorkItemWatchTaskID(t.Context(), work.ID, work.Generation, "project-1", 101, "task-1"); err != nil { t.Fatalf("assign task id: %v", err) } @@ -394,6 +395,7 @@ func TestStoreAssignPullRequestWatchTaskIDResolvesGenerationFromTheSecondTable(t if ok, err := store.ReservePullRequestWatchTask(t.Context(), pr.ID, pr.Generation, "project-1", "azure-repo-1", 42, "https://azure/pr/42"); err != nil || !ok { t.Fatalf("reserve pull request: ok=%v err=%v", ok, err) } + seedAzureTask(t, store, "task-42", "ws-1") if err := store.AssignPullRequestWatchTaskID(t.Context(), pr.ID, pr.Generation, "project-1", "azure-repo-1", 42, "task-42"); err != nil { t.Fatalf("assign task id: %v", err) } diff --git a/apps/backend/internal/azuredevops/watch_store_test.go b/apps/backend/internal/azuredevops/watch_store_test.go index c39e95bac2..9ab3444177 100644 --- a/apps/backend/internal/azuredevops/watch_store_test.go +++ b/apps/backend/internal/azuredevops/watch_store_test.go @@ -6,6 +6,142 @@ import ( "testing" ) +func TestCleanupCandidateListsExcludeHistoricalTasks(t *testing.T) { + store, err := NewStore(newTestDB(t), nil) + if err != nil { + t.Fatalf("new store: %v", err) + } + if _, err := store.db.Exec(`CREATE TABLE IF NOT EXISTS tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + archived_at DATETIME + )`); err != nil { + t.Fatalf("create tasks table: %v", err) + } + ctx := context.Background() + workWatch := &WorkItemWatch{ID: "work-cleanup", WorkspaceID: "ws-1", ProjectID: "project-1", WIQL: "SELECT"} + if err := store.CreateWorkItemWatch(ctx, workWatch); err != nil { + t.Fatalf("create work-item watch: %v", err) + } + otherWorkWatch := &WorkItemWatch{ID: "work-other", WorkspaceID: "ws-2", ProjectID: "project-2", WIQL: "SELECT"} + if err := store.CreateWorkItemWatch(ctx, otherWorkWatch); err != nil { + t.Fatalf("create other work-item watch: %v", err) + } + pullRequestWatch := &PullRequestWatch{ID: "pr-cleanup", WorkspaceID: "ws-1", ProjectID: "project-1", AzureRepositoryID: "repo-1"} + if err := store.CreatePullRequestWatch(ctx, pullRequestWatch); err != nil { + t.Fatalf("create pull-request watch: %v", err) + } + otherPullRequestWatch := &PullRequestWatch{ID: "pr-other", WorkspaceID: "ws-2", ProjectID: "project-2", AzureRepositoryID: "repo-2"} + if err := store.CreatePullRequestWatch(ctx, otherPullRequestWatch); err != nil { + t.Fatalf("create other pull-request watch: %v", err) + } + + for _, task := range []struct { + id string + workspace string + }{ + {id: "work-active", workspace: "ws-1"}, + {id: "work-archived", workspace: "ws-1"}, + {id: "pr-active", workspace: "ws-1"}, + {id: "pr-archived", workspace: "ws-1"}, + {id: "other-work-active", workspace: "ws-2"}, + {id: "other-pr-active", workspace: "ws-2"}, + } { + if _, err := store.db.Exec(`INSERT INTO tasks (id, workspace_id) VALUES (?, ?)`, task.id, task.workspace); err != nil { + t.Fatalf("seed task %s: %v", task.id, err) + } + } + for _, taskID := range []string{"work-archived", "pr-archived"} { + if _, err := store.db.Exec(`UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = ?`, taskID); err != nil { + t.Fatalf("archive task %s: %v", taskID, err) + } + } + + reserveWork := func(watchID string, generation int64, projectID string, itemID int, taskID string) { + t.Helper() + reserved, err := store.ReserveWorkItemWatchTask(ctx, watchID, generation, projectID, itemID, "https://azure/item") + if err != nil || !reserved { + t.Fatalf("reserve work item %d: reserved=%v err=%v", itemID, reserved, err) + } + if taskID != "" { + if err := store.AssignWorkItemWatchTaskID(ctx, watchID, generation, projectID, itemID, taskID); err != nil { + t.Fatalf("assign work item %d: %v", itemID, err) + } + } + } + reservePullRequest := func(watchID string, generation int64, projectID, repositoryID string, pullRequestID int, taskID string) { + t.Helper() + reserved, err := store.ReservePullRequestWatchTask(ctx, watchID, generation, projectID, repositoryID, pullRequestID, "https://azure/pr") + if err != nil || !reserved { + t.Fatalf("reserve pull request %d: reserved=%v err=%v", pullRequestID, reserved, err) + } + if taskID != "" { + if err := store.AssignPullRequestWatchTaskID(ctx, watchID, generation, projectID, repositoryID, pullRequestID, taskID); err != nil { + t.Fatalf("assign pull request %d: %v", pullRequestID, err) + } + } + } + reserveWork(workWatch.ID, workWatch.Generation, workWatch.ProjectID, 101, "work-active") + reserveWork(workWatch.ID, workWatch.Generation, workWatch.ProjectID, 102, "work-archived") + reserveWork(workWatch.ID, workWatch.Generation, workWatch.ProjectID, 103, "work-missing") + reserveWork(workWatch.ID, workWatch.Generation, workWatch.ProjectID, 104, "") + reserveWork(otherWorkWatch.ID, otherWorkWatch.Generation, otherWorkWatch.ProjectID, 105, "other-work-active") + reservePullRequest(pullRequestWatch.ID, pullRequestWatch.Generation, pullRequestWatch.ProjectID, pullRequestWatch.AzureRepositoryID, 201, "pr-active") + reservePullRequest(pullRequestWatch.ID, pullRequestWatch.Generation, pullRequestWatch.ProjectID, pullRequestWatch.AzureRepositoryID, 202, "pr-archived") + reservePullRequest(pullRequestWatch.ID, pullRequestWatch.Generation, pullRequestWatch.ProjectID, pullRequestWatch.AzureRepositoryID, 203, "pr-missing") + reservePullRequest(pullRequestWatch.ID, pullRequestWatch.Generation, pullRequestWatch.ProjectID, pullRequestWatch.AzureRepositoryID, 204, "") + reservePullRequest(otherPullRequestWatch.ID, otherPullRequestWatch.Generation, otherPullRequestWatch.ProjectID, otherPullRequestWatch.AzureRepositoryID, 205, "other-pr-active") + + assertWorkRows := func(name string, rows []*WorkItemWatchTask, want map[string]bool) { + t.Helper() + got := make(map[string]bool, len(rows)) + for _, row := range rows { + got[row.TaskID] = true + } + if len(rows) != len(want) { + t.Fatalf("%s rows = %d (%v), want %d (%v)", name, len(rows), got, len(want), want) + } + for taskID := range want { + if !got[taskID] { + t.Errorf("%s missing task %q in %v", name, taskID, got) + } + } + } + assertPullRequestRows := func(name string, rows []*PullRequestWatchTask, want map[string]bool) { + t.Helper() + got := make(map[string]bool, len(rows)) + for _, row := range rows { + got[row.TaskID] = true + } + if len(rows) != len(want) { + t.Fatalf("%s rows = %d (%v), want %d (%v)", name, len(rows), got, len(want), want) + } + for taskID := range want { + if !got[taskID] { + t.Errorf("%s missing task %q in %v", name, taskID, got) + } + } + } + + workRows, err := store.ListWorkItemWatchTasks(ctx, workWatch.ID, workWatch.Generation) + if err != nil { + t.Fatalf("list work-item cleanup rows: %v", err) + } + assertWorkRows("work-item current generation", workRows, map[string]bool{"work-active": true, "": true}) + wrongGenerationRows, err := store.ListWorkItemWatchTasks(ctx, workWatch.ID, workWatch.Generation+1) + if err != nil { + t.Fatalf("list work-item wrong generation: %v", err) + } + if len(wrongGenerationRows) != 0 { + t.Fatalf("work-item wrong generation rows = %v, want empty", wrongGenerationRows) + } + pullRequestRows, err := store.ListPullRequestWatchTasks(ctx, pullRequestWatch.ID, pullRequestWatch.Generation) + if err != nil { + t.Fatalf("list pull-request cleanup rows: %v", err) + } + assertPullRequestRows("pull-request current generation", pullRequestRows, map[string]bool{"pr-active": true, "": true}) +} + func TestAzureWatchStoreRoundTripAndGenerationReservations(t *testing.T) { store, err := NewStore(newTestDB(t), nil) if err != nil { @@ -50,6 +186,7 @@ func TestAzureWatchStoreRoundTripAndGenerationReservations(t *testing.T) { if err != nil || reserved { t.Fatalf("duplicate reservation: reserved=%v err=%v", reserved, err) } + seedAzureTask(t, store, "task-1", "ws-1") if err := store.AssignWorkItemWatchTaskID(context.Background(), watch.ID, watch.Generation, "project-1", 101, "task-1"); err != nil { t.Fatalf("attach task: %v", err) } diff --git a/apps/backend/internal/backendapp/github_task_activity_adapter.go b/apps/backend/internal/backendapp/github_task_activity_adapter.go new file mode 100644 index 0000000000..b0c1cf15c0 --- /dev/null +++ b/apps/backend/internal/backendapp/github_task_activity_adapter.go @@ -0,0 +1,35 @@ +package backendapp + +import ( + "context" + + "github.com/kandev/kandev/internal/github" + tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// githubTaskActivityAdapter keeps the GitHub scheduler dependent on its +// narrow integration projection while the task repository owns the query and +// its activity-source rules. +type githubTaskActivityAdapter struct { + repo *tasksqlite.Repository +} + +func (a *githubTaskActivityAdapter) LoadPRWatchTaskActivity( + ctx context.Context, taskIDs []string, +) (map[string]github.PRWatchTaskActivity, error) { + result := make(map[string]github.PRWatchTaskActivity, len(taskIDs)) + if a == nil || a.repo == nil || len(taskIDs) == 0 { + return result, nil + } + activity, err := a.repo.LoadPRWatchTaskActivity(ctx, taskIDs) + if err != nil { + return nil, err + } + for taskID, value := range activity { + result[taskID] = github.PRWatchTaskActivity{ + LastActivityAt: value.LastActivityAt, + Running: value.Running, + } + } + return result, nil +} diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 110d216ab8..698fc3f037 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -788,6 +788,7 @@ func startAgentInfrastructure( // Wire GitHub service into orchestrator for PR auto-detection on push if services.GitHub != nil { orchestratorSvc.SetGitHubService(services.GitHub) + services.GitHub.SetTaskActivityProvider(&githubTaskActivityAdapter{repo: repos.Task}) services.GitHub.SetTaskDeleter(&taskDeleterAdapter{svc: services.Task}) taskStoreAdapter := githubTaskIssueStoreAdapter{svc: services.Task} services.GitHub.SetTaskIssueStore(taskStoreAdapter) @@ -1150,6 +1151,7 @@ func startGatewayAndServe( } ghPoller := githubpkg.NewPoller(services.GitHub, eventBus, log) ghPoller.SetTaskBranchProvider(orchestratorSvc) + ghPoller.SetTaskActivityProvider(&githubTaskActivityAdapter{repo: repos.Task}) ghPoller.Start(ctx) addCleanup(func() error { ghPoller.Stop(); return nil }) log.Info("GitHub poller started") diff --git a/apps/backend/internal/gateway/websocket/task_notifications_test.go b/apps/backend/internal/gateway/websocket/task_notifications_test.go index b7c417dfc9..9713a838bd 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications_test.go +++ b/apps/backend/internal/gateway/websocket/task_notifications_test.go @@ -207,7 +207,7 @@ func TestTaskEventBroadcaster_OrdersTranscriptMutationsAcrossTransportSubjects(t // lifecycle state events intentionally sharing one ordered wildcard). // // The old code had a second subscription system (subscribeEventBusHandlers in -// cmd/kandev/helpers.go) that subscribed to the same four events, causing +// cmd/kandev/helpers.go) that subscribed to the same routed events, causing // duplicate broadcasts. This test counts the broadcaster's internal // subscriptions directly to guard against re-introducing duplicates. func TestTaskEventBroadcaster_NoDuplicateSubscriptions(t *testing.T) { diff --git a/apps/backend/internal/github/gh_client.go b/apps/backend/internal/github/gh_client.go index 9b25eff840..49d1c01c35 100644 --- a/apps/backend/internal/github/gh_client.go +++ b/apps/backend/internal/github/gh_client.go @@ -51,10 +51,16 @@ func (c *GHClient) WithRateTracker(t *RateTracker) *GHClient { return c } +// RateResource reports the bucket used by gh subcommands such as `gh pr view` +// and `gh issue view`. The CLI implements these reads through GraphQL. +func (c *GHClient) RateResource() Resource { + return ResourceGraphQL +} + // reGHRateLimit matches the prose gh prints when a request hit a primary or // secondary rate limit. The exact text varies by gh version and locale, but // "rate limit" / "API rate limit" appear consistently. -var ghRateLimitMarkers = []string{"rate limit", "abuse detection"} +var ghRateLimitMarkers = []string{"rate limit", "abuse detection", "too many requests", "secondary limit"} func ghStderrIndicatesRateLimit(stderr string) bool { if stderr == "" { @@ -1408,6 +1414,13 @@ func (c *GHClient) runGH(ctx context.Context, stdin []byte, args ...string) (str return stdout.String(), fmt.Errorf("gh %s: %w", firstArg(args), execCtxErr) } c.inspectRateStderr(args, stderr.String()) + if ghStderrIndicatesRateLimit(stderr.String()) { + return stdout.String(), fmt.Errorf("gh %s: %w", firstArg(args), &GitHubAPIError{ + StatusCode: http.StatusTooManyRequests, + Endpoint: "gh " + strings.Join(args, " "), + Body: strings.TrimSpace(stderr.String()), + }) + } return stdout.String(), fmt.Errorf("gh %s: %w: %s", firstArg(args), runErr, stderr.String()) } return stdout.String(), nil diff --git a/apps/backend/internal/github/gh_client_runtime_test.go b/apps/backend/internal/github/gh_client_runtime_test.go index 14d30639c4..1dae5b0110 100644 --- a/apps/backend/internal/github/gh_client_runtime_test.go +++ b/apps/backend/internal/github/gh_client_runtime_test.go @@ -301,6 +301,22 @@ func TestGHClient_GetPRStatus(t *testing.T) { } } +func TestGHClient_GetPR_NormalizesCLIRateLimitStderr(t *testing.T) { + for _, stderr := range []string{ + "GraphQL: API rate limit already exceeded for user", + "gh: HTTP 429: Too Many Requests", + } { + t.Run(stderr, func(t *testing.T) { + newFakeGH(t, ghResponse{Prefix: "pr view", Stderr: stderr, Exit: 1}) + _, err := NewGHClient().GetPR(context.Background(), "acme", "widget", 42) + var apiErr *GitHubAPIError + if !errors.As(err, &apiErr) || apiErr.StatusCode != http.StatusTooManyRequests { + t.Fatalf("err = %v, want a typed 429 rate-limit error", err) + } + }) + } +} + func TestGHClient_GetPRFeedback(t *testing.T) { responses := append(ghPRStatusResponses(), ghResponse{ diff --git a/apps/backend/internal/github/handlers.go b/apps/backend/internal/github/handlers.go index d9765aa7da..868c3e73d0 100644 --- a/apps/backend/internal/github/handlers.go +++ b/apps/backend/internal/github/handlers.go @@ -275,12 +275,25 @@ func wsGetTaskPR(svc *Service, _ *logger.Logger) func(ctx context.Context, msg * // dead repos every 5s for the lifetime of the task, which was the // dominant signal in the production SyncWatchesBatched storm. func wsSyncTaskPR(svc *Service, _ *logger.Logger) func(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - return wsWithField("task_id", func(ctx context.Context, taskID string) (interface{}, error) { - prs, permanent, err := svc.TriggerPRSyncAllPermanent(ctx, taskID) + return func(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var payload struct { + TaskID string `json:"task_id"` + ExplicitRefresh bool `json:"explicit_refresh"` + } + if err := msg.ParsePayload(&payload); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "invalid payload: "+err.Error(), nil) + } + if payload.TaskID == "" { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "task_id required", nil) + } + prs, permanent, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, payload.TaskID, payload.ExplicitRefresh) if err != nil { + if errors.Is(err, repoerrors.ErrWorkspaceNotFound) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "workspace not found", nil) + } var partial *PartialPRSyncError if !permanent && (!errors.As(err, &partial) || len(prs) == 0) { - return nil, err + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, err.Error(), nil) } } // Return an envelope so the frontend always gets a deterministic @@ -290,8 +303,8 @@ func wsSyncTaskPR(svc *Service, _ *logger.Logger) func(ctx context.Context, msg if prs == nil { prs = []*TaskPR{} } - return map[string]interface{}{"prs": prs, "permanent": permanent}, nil - }) + return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{"prs": prs, "permanent": permanent}) + } } func wsGetPRFeedback(svc *Service, _ *logger.Logger) func(ctx context.Context, msg *ws.Message) (*ws.Message, error) { diff --git a/apps/backend/internal/github/load_validation_test.go b/apps/backend/internal/github/load_validation_test.go index 15e2bc708a..18fcae0be4 100644 --- a/apps/backend/internal/github/load_validation_test.go +++ b/apps/backend/internal/github/load_validation_test.go @@ -93,8 +93,11 @@ func TestLoadValidation_TenSimulatedMinutes_NoWatchCreationLoopBoundedLookups(t durations := make([]time.Duration, 0, simulatedCycles) var firstCycleWatchIDs, lastCycleWatchIDs map[string]struct{} + pollStart := time.Now().UTC() for cycle := 0; cycle < simulatedCycles; cycle++ { + cycleNow := pollStart.Add(time.Duration(cycle)*defaultPRPollInterval + time.Second) + poller.SetClock(func() time.Time { return cycleNow }) start := time.Now() poller.checkPRWatches(ctx) durations = append(durations, time.Since(start)) diff --git a/apps/backend/internal/github/poller.go b/apps/backend/internal/github/poller.go index 1ccbd81cee..41d6ff8125 100644 --- a/apps/backend/internal/github/poller.go +++ b/apps/backend/internal/github/poller.go @@ -3,20 +3,25 @@ package github import ( "context" "errors" + "fmt" + "sort" "sync" "time" "go.uber.org/zap" + "github.com/kandev/kandev/internal/common/authcircuit" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" ) const ( - defaultPRPollInterval = 1 * time.Minute - defaultReviewPollInterval = 5 * time.Minute - defaultIssuePollInterval = 5 * time.Minute + defaultPRPollInterval = 1 * time.Minute + defaultReviewPollInterval = 5 * time.Minute + defaultIssuePollInterval = 5 * time.Minute + prWatchFallbackWorkspaceBudget = 5 + prWatchFallbackCycleBudget = 10 // rateLimitedSleepCap bounds the rate-limit sleep so a misreported reset // (e.g. far-future reset_at after a CLI failure) cannot wedge the loop. rateLimitedSleepCap = 10 * time.Minute @@ -98,15 +103,21 @@ type repositoryTaskBranchProvider interface { // Poller runs background loops for PR monitoring and review queue checking. type Poller struct { - service *Service - eventBus bus.EventBus - logger *logger.Logger - taskBranchProvider TaskBranchProvider + service *Service + eventBus bus.EventBus + logger *logger.Logger + taskBranchProvider TaskBranchProvider + taskActivityProvider TaskActivityProvider + clock func() time.Time // circuits tracks per-workspace auth/config backoff for the PR-monitor // loop only (see poller_circuit.go). Safe for concurrent use; never nil. circuits *pollerCircuits + fallbackMu sync.Mutex + fallbackWorkspaceCursor int + fallbackTargetCursors map[string]int + cancel context.CancelFunc wg sync.WaitGroup started bool @@ -115,10 +126,12 @@ type Poller struct { // NewPoller creates a new background poller. func NewPoller(svc *Service, eventBus bus.EventBus, log *logger.Logger) *Poller { return &Poller{ - service: svc, - eventBus: eventBus, - logger: log, - circuits: newPollerCircuits(), + service: svc, + eventBus: eventBus, + logger: log, + circuits: newPollerCircuits(), + fallbackTargetCursors: make(map[string]int), + clock: time.Now, } } @@ -198,18 +211,17 @@ func (p *Poller) checkPRWatches(ctx context.Context) { if len(watches) == 0 { return } - - // Try the batched GraphQL path first — collapses N HTTP requests into a - // handful of multi-aliased GraphQL calls. On error or unsupported client - // (e.g. NoopClient), fall back to per-watch checks so one bad poll cycle - // doesn't lose state. - if p.tryBatchedPRWatchCheck(ctx, watches) { + watches = p.selectDuePRWatches(ctx, watches) + if len(watches) == 0 { return } - watches = p.filterOpenCircuitWatches(ctx, watches) - for _, watch := range watches { - p.checkSinglePRWatch(ctx, watch) - } + + // Try the batched GraphQL path first. Each workspace reports whether its + // watches completed, were deferred by a provider gate, or may use the + // bounded REST fallback. A failed workspace must not replay successful + // batches from its siblings. + outcomes := p.tryBatchedPRWatchCheck(ctx, watches) + p.runPRWatchFallback(ctx, fallbackEligiblePRWatches(outcomes)) } // filterOpenCircuitWatches refreshes each present workspace's credential @@ -259,39 +271,90 @@ func (p *Poller) refreshWorkspaceCircuitFingerprint(ctx context.Context, workspa } } +type prWatchBatchOutcomeState string + +const ( + prWatchBatchCompleted prWatchBatchOutcomeState = "completed" + prWatchBatchDeferred prWatchBatchOutcomeState = "deferred" + prWatchBatchFallbackEligible prWatchBatchOutcomeState = "fallback_eligible" +) + +type prWatchWorkspaceBatchOutcome struct { + workspaceID string + watches []*PRWatch + state prWatchBatchOutcomeState + results []PRWatchSyncResult +} + // tryBatchedPRWatchCheck runs the batched GraphQL flow for the supplied -// watches via the shared Service.SyncWatchesBatched seam. Returns true -// when the path succeeded so the caller can skip the per-watch fallback. +// watches via the shared Service.SyncWatchesBatched seam. It returns one +// outcome per workspace so a fallback can be bounded and scoped. // The poller's only extra responsibility on top of the service apply is // publishing PRFeedback events for status changes and merge/close events // (the on-demand sync path intentionally doesn't publish these). -func (p *Poller) tryBatchedPRWatchCheck(ctx context.Context, watches []*PRWatch) bool { +func (p *Poller) tryBatchedPRWatchCheck(ctx context.Context, watches []*PRWatch) []prWatchWorkspaceBatchOutcome { byWorkspace := make(map[string][]*PRWatch) for _, watch := range watches { if watch == nil || watch.WorkspaceID == "" { p.logger.Warn("PR watch is missing workspace ownership", zap.String("watch_id", watchID(watch))) - return false + continue } byWorkspace[watch.WorkspaceID] = append(byWorkspace[watch.WorkspaceID], watch) } - results := make([]PRWatchSyncResult, 0, len(watches)) - for workspaceID, workspaceWatches := range byWorkspace { + workspaceIDs := make([]string, 0, len(byWorkspace)) + for workspaceID := range byWorkspace { + workspaceIDs = append(workspaceIDs, workspaceID) + } + sort.Strings(workspaceIDs) + outcomes := make([]prWatchWorkspaceBatchOutcome, 0, len(workspaceIDs)) + for _, workspaceID := range workspaceIDs { + workspaceWatches := byWorkspace[workspaceID] incCanonicalPollRequests(len(workspaceWatches)) workspaceResults, err := p.service.SyncWorkspaceWatchesBatched(ctx, workspaceID, workspaceWatches) // A client without GraphQL support deliberately falls back to the // per-watch REST path below. It is not an authentication failure, so // it must not open the workspace circuit and filter that fallback out. if errors.Is(err, errGraphQLUnsupported) { - return false + outcomes = append(outcomes, prWatchWorkspaceBatchOutcome{ + workspaceID: workspaceID, + watches: workspaceWatches, + state: prWatchBatchFallbackEligible, + }) + continue } p.circuits.recordOutcome(workspaceID, classifyPollErr(err), time.Now().UTC()) if err != nil { p.logger.Debug("batched PR watch check failed", zap.String("workspace_id", workspaceID), zap.Error(err)) - return false + state := prWatchBatchDeferred + if prWatchBatchFallbackAllowed(err) { + state = prWatchBatchFallbackEligible + } + outcomes = append(outcomes, prWatchWorkspaceBatchOutcome{ + workspaceID: workspaceID, + watches: workspaceWatches, + state: state, + }) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { + break + } + continue + } + outcomes = append(outcomes, prWatchWorkspaceBatchOutcome{ + workspaceID: workspaceID, + watches: workspaceWatches, + state: prWatchBatchCompleted, + results: workspaceResults, + }) + p.publishBatchedPRWatchResults(ctx, workspaceResults) + if ctx.Err() != nil { + break } - results = append(results, workspaceResults...) } + return outcomes +} + +func (p *Poller) publishBatchedPRWatchResults(ctx context.Context, results []PRWatchSyncResult) { for _, r := range results { if !r.Found || r.Status == nil { continue @@ -322,7 +385,171 @@ func (p *Poller) tryBatchedPRWatchCheck(ctx context.Context, watches []*PRWatch) p.publishPRStatusEvent(ctx, r.Watch, effectiveTaskID, r.Status) } } - return true +} + +func fallbackEligiblePRWatches(outcomes []prWatchWorkspaceBatchOutcome) []*PRWatch { + var watches []*PRWatch + for _, outcome := range outcomes { + if outcome.state == prWatchBatchFallbackEligible { + watches = append(watches, outcome.watches...) + } + } + return watches +} + +func prWatchBatchFallbackAllowed(err error) bool { + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + category := classifyPRDiscoveryError(err) + if category == PRDiscoveryHealthRateLimited || category == PRDiscoveryHealthInvalidQuery { + return false + } + switch classifyPollErr(err) { + case authcircuit.FailureClassAuth, authcircuit.FailureClassConfig: + return false + default: + return true + } +} + +func (p *Poller) runPRWatchFallback(ctx context.Context, watches []*PRWatch) { + selected := p.selectPRWatchFallbackTargets(watches) + groups := groupPRWatchFallbackTargets(watches) + stopped := make(map[string]bool) + for _, watch := range selected { + if ctx.Err() != nil { + return + } + if watch == nil || stopped[watch.WorkspaceID] { + continue + } + err := p.checkPRWatchGroupWithError(ctx, groups[prWatchFallbackTargetKey(watch)]) + if err == nil { + continue + } + if ctx.Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + if prWatchFallbackShouldStopWorkspace(err) { + stopped[watch.WorkspaceID] = true + } + } +} + +func groupPRWatchFallbackTargets(watches []*PRWatch) map[string][]*PRWatch { + groups := make(map[string][]*PRWatch, len(watches)) + for _, watch := range watches { + if watch == nil { + continue + } + key := prWatchFallbackTargetKey(watch) + groups[key] = append(groups[key], watch) + } + return groups +} + +func deduplicatePRWatchFallbackTargets(watches []*PRWatch) (map[string][]*PRWatch, []string) { + byWorkspace := make(map[string]map[string]*PRWatch) + for _, watch := range watches { + if watch != nil { + workspaceTargets := byWorkspace[watch.WorkspaceID] + if workspaceTargets == nil { + workspaceTargets = make(map[string]*PRWatch) + byWorkspace[watch.WorkspaceID] = workspaceTargets + } + key := prWatchFallbackTargetKey(watch) + if _, exists := workspaceTargets[key]; !exists { + workspaceTargets[key] = watch + } + } + } + workspaceIDs := make([]string, 0, len(byWorkspace)) + workspaceGroups := make(map[string][]*PRWatch, len(byWorkspace)) + for workspaceID, targetWatches := range byWorkspace { + workspaceWatches := make([]*PRWatch, 0, len(targetWatches)) + for _, watch := range targetWatches { + workspaceWatches = append(workspaceWatches, watch) + } + sort.SliceStable(workspaceWatches, func(i, j int) bool { + return prWatchFallbackTargetKey(workspaceWatches[i]) < prWatchFallbackTargetKey(workspaceWatches[j]) + }) + workspaceGroups[workspaceID] = workspaceWatches + workspaceIDs = append(workspaceIDs, workspaceID) + } + sort.Strings(workspaceIDs) + return workspaceGroups, workspaceIDs +} + +func (p *Poller) selectPRWatchFallbackTargets(watches []*PRWatch) []*PRWatch { + workspaceGroups, workspaceIDs := deduplicatePRWatchFallbackTargets(watches) + if len(workspaceIDs) == 0 { + return nil + } + + p.fallbackMu.Lock() + defer p.fallbackMu.Unlock() + if p.fallbackTargetCursors == nil { + p.fallbackTargetCursors = make(map[string]int) + } + start := p.fallbackWorkspaceCursor % len(workspaceIDs) + selected := make([]*PRWatch, 0, minInt(prWatchFallbackCycleBudget, len(watches))) + selectedByWorkspace := make(map[string]int, len(workspaceIDs)) + for len(selected) < prWatchFallbackCycleBudget { + added := false + for offset := 0; offset < len(workspaceIDs) && len(selected) < prWatchFallbackCycleBudget; offset++ { + workspaceID := workspaceIDs[(start+offset)%len(workspaceIDs)] + group := workspaceGroups[workspaceID] + count := selectedByWorkspace[workspaceID] + if count >= prWatchFallbackWorkspaceBudget || count >= len(group) { + continue + } + cursor := p.fallbackTargetCursors[workspaceID] % len(group) + selected = append(selected, group[(cursor+count)%len(group)]) + selectedByWorkspace[workspaceID] = count + 1 + added = true + } + if !added { + break + } + } + for workspaceID, count := range selectedByWorkspace { + group := workspaceGroups[workspaceID] + cursor := p.fallbackTargetCursors[workspaceID] % len(group) + p.fallbackTargetCursors[workspaceID] = (cursor + count) % len(group) + } + p.fallbackWorkspaceCursor = (start + 1) % len(workspaceIDs) + return selected +} + +func prWatchFallbackTargetKey(watch *PRWatch) string { + if watch == nil { + return "" + } + return watch.WorkspaceID + "\x00" + watch.Owner + "\x00" + watch.Repo + "\x00" + watch.Branch + "\x00" + fmt.Sprintf("%010d", watch.PRNumber) +} + +func prWatchFallbackShouldStopWorkspace(err error) bool { + if err == nil { + return false + } + category := classifyPRDiscoveryError(err) + if category == PRDiscoveryHealthRateLimited || category == PRDiscoveryHealthInvalidQuery { + return true + } + switch classifyPollErr(err) { + case authcircuit.FailureClassAuth, authcircuit.FailureClassConfig: + return true + default: + return false + } +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b } func watchID(watch *PRWatch) string { @@ -347,11 +574,14 @@ func splitPRWatches(watches []*PRWatch) (numbered, searching []*PRWatch) { } func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { + _ = p.checkSinglePRWatchWithError(ctx, watch) +} + +func (p *Poller) checkSinglePRWatchWithError(ctx context.Context, watch *PRWatch) error { incCanonicalPollRequests(1) // PRWatch with pr_number=0 means we're still searching for a PR on this branch. if watch.PRNumber == 0 { - p.detectPRForWatch(ctx, watch) - return + return p.detectPRForWatchWithError(ctx, watch) } status, hasNew, err := p.service.CheckPRWatchForWorkspace(ctx, watch) @@ -359,10 +589,19 @@ func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { if err != nil { p.logger.Debug("failed to check PR watch", zap.String("id", watch.ID), zap.Error(err)) - return + return err } if status == nil { - return + return nil + } + return p.applyPRStatusForWatch(ctx, watch, status, hasNew) +} + +func (p *Poller) applyPRStatusForWatch( + ctx context.Context, watch *PRWatch, status *PRStatus, hasNew bool, +) error { + if watch == nil || status == nil { + return nil } // A numbered watch found its PR before this fix existed (or before the @@ -380,7 +619,7 @@ func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { if syncErr := p.service.SyncTaskPR(ctx, effectiveTaskID, status); syncErr != nil { p.logger.Error("failed to sync task PR", zap.String("task_id", effectiveTaskID), zap.Error(syncErr)) - return // Keep watch so the next cycle can retry + return nil // Keep watch so the next cycle can retry } // When the tracked PR is merged or closed, reset the watch back to the // "searching" state (pr_number=0) rather than deleting it. This lets the @@ -395,10 +634,10 @@ func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { if holdErr != nil { p.logger.Warn("failed to check terminal PR automation", zap.String("task_id", effectiveTaskID), zap.Error(holdErr)) - return + return nil } if hold { - return + return nil } if resetErr := p.service.store.UpdatePRWatchPRNumber(ctx, watch.ID, 0); resetErr != nil { p.logger.Error("failed to reset completed PR watch", @@ -409,23 +648,70 @@ func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { zap.String("state", status.PR.State), zap.Int("pr_number", watch.PRNumber)) } - return + return nil } if !hasNew { - return + return nil } p.publishPRStatusEvent(ctx, watch, effectiveTaskID, status) + return nil +} + +func (p *Poller) checkPRWatchGroupWithError(ctx context.Context, watches []*PRWatch) error { + if len(watches) == 0 || watches[0] == nil { + return nil + } + representative := watches[0] + incCanonicalPollRequests(1) + if representative.PRNumber == 0 { + pr, err := p.service.findPRByBranchForWatch(ctx, representative) + p.circuits.recordOutcome(representative.WorkspaceID, classifyPollErr(err), time.Now().UTC()) + if err != nil { + p.logger.Debug("failed to search for shared PR watch target", + zap.String("watch_id", representative.ID), zap.Error(err)) + return err + } + for _, watch := range watches { + p.applyDetectedPR(ctx, watch, pr) + } + return nil + } + + status, hasNew, err := p.service.CheckPRWatchForWorkspace(ctx, representative) + p.circuits.recordOutcome(representative.WorkspaceID, classifyPollErr(err), time.Now().UTC()) + if err != nil { + p.logger.Debug("failed to check shared PR watch target", + zap.String("watch_id", representative.ID), zap.Error(err)) + return err + } + if status == nil { + return nil + } + for index, watch := range watches { + watchHasNew := hasNew + if index > 0 { + watchHasNew, _ = p.service.applyPRWatchStatus(ctx, watch, status) + } + if err := p.applyPRStatusForWatch(ctx, watch, status, watchHasNew); err != nil { + return err + } + } + return nil } // detectPRForWatch searches GitHub for a PR on the watch's branch. // If found, updates the watch with the PR number and creates the TaskPR association. func (p *Poller) detectPRForWatch(ctx context.Context, watch *PRWatch) { + _ = p.detectPRForWatchWithError(ctx, watch) +} + +func (p *Poller) detectPRForWatchWithError(ctx context.Context, watch *PRWatch) error { if watch == nil || watch.WorkspaceID == "" { p.logger.Warn("cannot detect PR for watch without workspace ownership", zap.String("watch_id", watchID(watch))) - return + return ErrGitHubWorkspaceRequired } pr, err := p.service.findPRByBranchForWatch(ctx, watch) @@ -435,9 +721,13 @@ func (p *Poller) detectPRForWatch(ctx context.Context, watch *PRWatch) { zap.String("watch_id", watch.ID), zap.String("branch", watch.Branch), zap.Error(err)) - return + return err } + p.applyDetectedPR(ctx, watch, pr) + return nil +} +func (p *Poller) applyDetectedPR(ctx context.Context, watch *PRWatch, pr *PR) { // Update last_checked_at regardless of result now := time.Now().UTC() _ = p.service.store.UpdatePRWatchTimestamps(ctx, watch.ID, now, nil, "", "") @@ -504,6 +794,22 @@ func (p *Poller) SetTaskBranchProvider(provider TaskBranchProvider) { p.taskBranchProvider = provider } +// SetTaskActivityProvider wires the persisted task activity projection used +// by adaptive searching-watch admission. +func (p *Poller) SetTaskActivityProvider(provider TaskActivityProvider) { + p.taskActivityProvider = provider +} + +// SetClock installs the scheduler clock. Production uses time.Now; tests can +// use a fixed clock to prove interval boundaries without sleeping. +func (p *Poller) SetClock(clock func() time.Time) { + if clock == nil { + p.clock = time.Now + return + } + p.clock = clock +} + // reconcileWatches ensures PR watches exist for all tasks that need them, // and refreshes stale branches on existing watches that haven't found a PR yet. func (p *Poller) reconcileWatches(ctx context.Context) { @@ -656,6 +962,9 @@ func (p *Poller) checkReviewWatches(ctx context.Context) { if cleaned, err := p.service.CleanupMergedReviewTasks(ctx, watch); err != nil { p.logCleanupError("failed to cleanup merged review tasks", err, zap.String("watch_id", watch.ID)) + if cleanupBatchShouldStop(err) { + return + } } else if cleaned > 0 { p.logger.Info("cleaned up merged review tasks", zap.String("watch_id", watch.ID), zap.Int("deleted", cleaned)) @@ -749,6 +1058,9 @@ func (p *Poller) checkIssueWatches(ctx context.Context) { if cleaned, err := p.service.CleanupClosedIssueTasks(ctx, watch); err != nil { p.logger.Warn("failed to cleanup closed issue tasks", zap.String("watch_id", watch.ID), zap.Error(err)) + if cleanupBatchShouldStop(err) { + return + } } else if cleaned > 0 { p.logger.Info("cleaned up closed issue tasks", zap.String("watch_id", watch.ID), zap.Int("deleted", cleaned)) diff --git a/apps/backend/internal/github/poller_cleanup_rate_limit_test.go b/apps/backend/internal/github/poller_cleanup_rate_limit_test.go new file mode 100644 index 0000000000..f26891f615 --- /dev/null +++ b/apps/backend/internal/github/poller_cleanup_rate_limit_test.go @@ -0,0 +1,72 @@ +package github + +import ( + "context" + "net/http" + "reflect" + "testing" +) + +func TestCleanupPollCycleSkipsRemainingCleanupAfterRateLimit(t *testing.T) { + t.Run("review per-watch and orphan cleanup", func(t *testing.T) { + poller, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{feedback: map[int]cleanupPRResponse{ + 1: {err: rateLimitAPIError(http.StatusTooManyRequests, "rate limit exceeded")}, + }} + configureCleanupRateLimitClient(t, svc, client, nil) + ctx := context.Background() + + first := &ReviewWatch{WorkspaceID: "ws-1", Enabled: true} + second := &ReviewWatch{WorkspaceID: "ws-1", Enabled: true} + orphan := &ReviewWatch{WorkspaceID: "ws-1", Enabled: false} + for _, watch := range []*ReviewWatch{first, second, orphan} { + if err := store.CreateReviewWatch(ctx, watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + } + createReviewCleanupRows(t, store, first, "review-first") + createReviewCleanupRows(t, store, second, "review-second") + createReviewCleanupRows(t, store, orphan, "review-orphan") + recorder := &recordingTaskDeleter{} + svc.SetTaskDeleter(recorder) + + poller.checkReviewWatches(ctx) + if !reflect.DeepEqual(client.prCalls, []int{1}) { + t.Fatalf("GetPR calls = %v, want only first watch", client.prCalls) + } + if len(recorder.calls) != 0 { + t.Fatalf("DeleteTask calls = %v, want none after rate-limit stop", recorder.calls) + } + }) + + t.Run("issue per-watch and orphan cleanup", func(t *testing.T) { + poller, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{issues: map[int]cleanupIssueResponse{ + 1: {err: rateLimitAPIError(http.StatusForbidden, "abuse detection")}, + }} + configureCleanupRateLimitClient(t, svc, client, nil) + ctx := context.Background() + + first := &IssueWatch{WorkspaceID: "ws-1", Enabled: true} + second := &IssueWatch{WorkspaceID: "ws-1", Enabled: true} + orphan := &IssueWatch{WorkspaceID: "ws-1", Enabled: false} + for _, watch := range []*IssueWatch{first, second, orphan} { + if err := store.CreateIssueWatch(ctx, watch); err != nil { + t.Fatalf("CreateIssueWatch: %v", err) + } + } + createIssueCleanupRows(t, store, first, "issue-first") + createIssueCleanupRows(t, store, second, "issue-second") + createIssueCleanupRows(t, store, orphan, "issue-orphan") + recorder := &recordingTaskDeleter{} + svc.SetTaskDeleter(recorder) + + poller.checkIssueWatches(ctx) + if !reflect.DeepEqual(client.issueCalls, []int{1}) { + t.Fatalf("issue calls = %v, want only first watch", client.issueCalls) + } + if len(recorder.calls) != 0 { + t.Fatalf("DeleteTask calls = %v, want none after rate-limit stop", recorder.calls) + } + }) +} diff --git a/apps/backend/internal/github/poller_schedule.go b/apps/backend/internal/github/poller_schedule.go new file mode 100644 index 0000000000..10ea30b9a8 --- /dev/null +++ b/apps/backend/internal/github/poller_schedule.go @@ -0,0 +1,198 @@ +package github + +import ( + "context" + "sort" + "strconv" + "time" + + "github.com/kandev/kandev/internal/common/logger" + "go.uber.org/zap" +) + +const ( + searchFastPollInterval = 1 * time.Minute + searchIdlePollInterval = 15 * time.Minute + searchVeryIdlePollInterval = 30 * time.Minute + searchIdleThreshold = 2 * time.Hour + searchVeryIdleThreshold = 24 * time.Hour +) + +// PRWatchTaskActivity is the narrow task-domain projection used to decide +// whether a searching PR watch should be checked. It intentionally excludes +// generic task/watch UpdatedAt values because provider synchronization can +// write those fields. +type PRWatchTaskActivity struct { + LastActivityAt time.Time + Running bool +} + +// TaskActivityProvider supplies persisted activity for a bulk set of tasks. +// Implementations must use task-local evidence only; the poller never asks a +// provider or remote Git host to classify idleness. +type TaskActivityProvider interface { + LoadPRWatchTaskActivity(ctx context.Context, taskIDs []string) (map[string]PRWatchTaskActivity, error) +} + +func (p *Poller) selectDuePRWatches(ctx context.Context, watches []*PRWatch) []*PRWatch { + return selectDuePRWatches(ctx, watches, p.taskActivityProvider, p.now(), p.logger) +} + +func selectDuePRWatches( + ctx context.Context, + watches []*PRWatch, + provider TaskActivityProvider, + now time.Time, + log *logger.Logger, +) []*PRWatch { + if len(watches) == 0 { + return nil + } + activityByTask := loadPRWatchTaskActivities(ctx, watches, provider, log) + groupDue := duePRWatchGroups(now, watches, activityByTask) + + due := make([]*PRWatch, 0, len(watches)) + for _, watch := range watches { + if watch == nil { + continue + } + if groupDue[prWatchDiscoveryTargetKey(watch)] { + due = append(due, watch) + } + } + return due +} + +func loadPRWatchTaskActivities( + ctx context.Context, + watches []*PRWatch, + provider TaskActivityProvider, + log *logger.Logger, +) map[string]PRWatchTaskActivity { + if provider == nil { + return nil + } + taskIDs := prWatchActivityTaskIDs(watches) + if len(taskIDs) == 0 { + return nil + } + loaded, err := provider.LoadPRWatchTaskActivity(ctx, taskIDs) + if err != nil { + if log != nil { + log.Debug("failed to load PR watch task activity", zap.Error(err)) + } + return nil + } + return loaded +} + +func prWatchActivityTaskIDs(watches []*PRWatch) []string { + seen := make(map[string]struct{}, len(watches)) + taskIDs := make([]string, 0, len(watches)) + for _, watch := range watches { + if watch == nil || watch.PRNumber != 0 || watch.TaskID == "" { + continue + } + if _, ok := seen[watch.TaskID]; ok { + continue + } + seen[watch.TaskID] = struct{}{} + taskIDs = append(taskIDs, watch.TaskID) + } + sort.Strings(taskIDs) + return taskIDs +} + +func duePRWatchGroups( + now time.Time, + watches []*PRWatch, + activityByTask map[string]PRWatchTaskActivity, +) map[string]bool { + groupDue := make(map[string]bool, len(watches)) + for _, watch := range watches { + markDuePRWatchGroup(groupDue, now, watch, activityByTask) + } + return groupDue +} + +func markDuePRWatchGroup( + groupDue map[string]bool, + now time.Time, + watch *PRWatch, + activityByTask map[string]PRWatchTaskActivity, +) { + if watch == nil { + return + } + key := prWatchDiscoveryTargetKey(watch) + if watch.PRNumber != 0 { + groupDue[key] = true + return + } + activity := activityByTask[watch.TaskID] + if prWatchIsDue(now, watch, activity) { + groupDue[key] = true + } +} + +func prWatchSearchInterval(now time.Time, watch *PRWatch, activity PRWatchTaskActivity) time.Duration { + if watch == nil || watch.PRNumber != 0 { + return defaultPRPollInterval + } + if activity.Running || activity.LastActivityAt.IsZero() { + return searchFastPollInterval + } + activityAt := activity.LastActivityAt + if watch != nil && watch.CreatedAt.After(activityAt) { + activityAt = watch.CreatedAt + } + idleFor := now.Sub(activityAt) + if idleFor < searchIdleThreshold { + return searchFastPollInterval + } + if idleFor < searchVeryIdleThreshold { + return searchIdlePollInterval + } + return searchVeryIdlePollInterval +} + +func prWatchIsDue(now time.Time, watch *PRWatch, activity PRWatchTaskActivity) bool { + if watch == nil { + return false + } + if watch.LastCheckedAt == nil { + return true + } + return now.Sub(*watch.LastCheckedAt) >= prWatchSearchInterval(now, watch, activity) +} + +func prWatchDiscoveryTargetKey(watch *PRWatch) string { + if watch == nil { + return "" + } + mode := "search" + if watch.PRNumber != 0 { + mode = "pr:" + strconv.Itoa(watch.PRNumber) + } + return watch.WorkspaceID + "\x00" + mode + "\x00" + watch.Owner + "\x00" + watch.Repo + "\x00" + watch.Branch +} + +func (p *Poller) now() time.Time { + if p != nil && p.clock != nil { + return p.clock().UTC() + } + return time.Now().UTC() +} + +func (s *Service) now() time.Time { + if s == nil { + return time.Now().UTC() + } + s.clockMu.RLock() + clock := s.clock + s.clockMu.RUnlock() + if clock != nil { + return clock().UTC() + } + return time.Now().UTC() +} diff --git a/apps/backend/internal/github/poller_schedule_test.go b/apps/backend/internal/github/poller_schedule_test.go new file mode 100644 index 0000000000..991da60897 --- /dev/null +++ b/apps/backend/internal/github/poller_schedule_test.go @@ -0,0 +1,297 @@ +package github + +import ( + "context" + "errors" + "testing" + "time" +) + +type scheduleActivityProvider struct { + activities map[string]PRWatchTaskActivity + err error + calls int +} + +func (p *scheduleActivityProvider) LoadPRWatchTaskActivity( + _ context.Context, taskIDs []string, +) (map[string]PRWatchTaskActivity, error) { + p.calls++ + if p.err != nil { + return nil, p.err + } + result := make(map[string]PRWatchTaskActivity, len(taskIDs)) + for _, taskID := range taskIDs { + if activity, ok := p.activities[taskID]; ok { + result[taskID] = activity + } + } + return result, nil +} + +func TestSearchingWatchAdaptiveSchedule(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + lastChecked := now.Add(-15 * time.Minute) + createdAt := now.Add(-48 * time.Hour) + + tests := []struct { + name string + watch *PRWatch + activity PRWatchTaskActivity + wantEvery time.Duration + wantDue bool + }{ + { + name: "running is fast", + watch: scheduleWatch("running", 0, timePtr(lastChecked), createdAt), + activity: PRWatchTaskActivity{Running: true, LastActivityAt: now.Add(-12 * time.Hour)}, + wantEvery: searchFastPollInterval, + wantDue: true, + }, + { + name: "below two hours is fast", + watch: scheduleWatch("recent", 0, timePtr(lastChecked), createdAt), + activity: PRWatchTaskActivity{LastActivityAt: now.Add(-2*time.Hour + time.Second)}, + wantEvery: searchFastPollInterval, + wantDue: true, + }, + { + name: "two hour boundary is fifteen minutes", + watch: scheduleWatch("two-hours", 0, timePtr(now.Add(-15*time.Minute)), createdAt), + activity: PRWatchTaskActivity{LastActivityAt: now.Add(-2 * time.Hour)}, + wantEvery: searchIdlePollInterval, + wantDue: true, + }, + { + name: "just below twenty four hours is fifteen minutes", + watch: scheduleWatch("day-minus", 0, timePtr(now.Add(-15*time.Minute)), createdAt), + activity: PRWatchTaskTaskActivityAt(now.Add(-24*time.Hour + time.Second)), + wantEvery: searchIdlePollInterval, + wantDue: true, + }, + { + name: "twenty four hour boundary is thirty minutes", + watch: scheduleWatch("day", 0, timePtr(now.Add(-30*time.Minute)), createdAt), + activity: PRWatchTaskTaskActivityAt(now.Add(-24 * time.Hour)), + wantEvery: searchVeryIdlePollInterval, + wantDue: true, + }, + { + name: "slow target waits for its interval", + watch: scheduleWatch("slow", 0, timePtr(now.Add(-10*time.Minute)), createdAt), + activity: PRWatchTaskTaskActivityAt(now.Add(-24 * time.Hour)), + wantEvery: searchVeryIdlePollInterval, + wantDue: false, + }, + { + name: "new watch is due", + watch: scheduleWatch("new", 0, nil, now), + activity: PRWatchTaskActivity{}, + wantEvery: searchFastPollInterval, + wantDue: true, + }, + { + name: "unknown activity stays fast", + watch: scheduleWatch("unknown", 0, timePtr(now.Add(-time.Minute)), createdAt), + activity: PRWatchTaskActivity{}, + wantEvery: searchFastPollInterval, + wantDue: true, + }, + { + name: "known PR stays fast", + watch: scheduleWatch("known", 42, timePtr(now.Add(-time.Minute)), createdAt), + activity: PRWatchTaskActivity{LastActivityAt: now.Add(-48 * time.Hour)}, + wantEvery: defaultPRPollInterval, + wantDue: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := prWatchSearchInterval(now, tt.watch, tt.activity); got != tt.wantEvery { + t.Fatalf("interval = %s, want %s", got, tt.wantEvery) + } + if got := prWatchIsDue(now, tt.watch, tt.activity); got != tt.wantDue { + t.Fatalf("due = %t, want %t", got, tt.wantDue) + } + }) + } +} + +func TestSearchingWatchCreationKeepsNewWatchFast(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + watch := scheduleWatch("old-task-new-watch", 0, timePtr(now.Add(-2*time.Minute)), now.Add(-30*time.Minute)) + activity := PRWatchTaskActivity{LastActivityAt: now.Add(-48 * time.Hour)} + + if got := prWatchSearchInterval(now, watch, activity); got != searchFastPollInterval { + t.Fatalf("new watch interval = %s, want %s", got, searchFastPollInterval) + } + if !prWatchIsDue(now, watch, activity) { + t.Fatal("new watch after its first completed check is not due on the fast tier") + } +} + +func TestSearchingWatchAdaptiveScheduleSharedTargets(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + provider := &scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{ + "fast-task": {LastActivityAt: now.Add(-time.Hour)}, + "slow-task": {LastActivityAt: now.Add(-24 * time.Hour)}, + }} + poller := &Poller{taskActivityProvider: provider, clock: func() time.Time { return now }} + + sharedFast := scheduleWatch("fast-task", 0, timePtr(now.Add(-time.Minute)), now.Add(-time.Hour)) + sharedFast.Owner = "owner" + sharedFast.Repo = "repo" + sharedFast.Branch = "shared" + sharedSlow := scheduleWatch("slow-task", 0, timePtr(now.Add(-time.Minute)), now.Add(-24*time.Hour)) + sharedSlow.Owner = "owner" + sharedSlow.Repo = "repo" + sharedSlow.Branch = "shared" + allIdle := scheduleWatch("slow-task", 0, timePtr(now.Add(-time.Minute)), now.Add(-24*time.Hour)) + allIdle.Owner = "owner" + allIdle.Repo = "repo" + allIdle.Branch = "idle" + + got := poller.selectDuePRWatches(context.Background(), []*PRWatch{sharedSlow, sharedFast, allIdle}) + if len(got) != 2 { + t.Fatalf("due watches = %d, want shared target only (2)", len(got)) + } + if got[0] != sharedSlow || got[1] != sharedFast { + t.Fatalf("due watches = [%p, %p], want shared watches [%p, %p]", got[0], got[1], sharedSlow, sharedFast) + } + if provider.calls != 1 { + t.Fatalf("activity provider calls = %d, want 1 bulk call", provider.calls) + } +} + +func TestSearchingWatchAdaptiveScheduleSeparatesNumberedTarget(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + provider := &scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{ + "searching-task": {LastActivityAt: now.Add(-24 * time.Hour)}, + }} + poller := &Poller{taskActivityProvider: provider, clock: func() time.Time { return now }} + + numbered := scheduleWatch("numbered-task", 42, timePtr(now.Add(-time.Minute)), now.Add(-time.Hour)) + numbered.Owner, numbered.Repo, numbered.Branch = "owner", "repo", "shared" + searching := scheduleWatch("searching-task", 0, timePtr(now.Add(-time.Minute)), now.Add(-24*time.Hour)) + searching.Owner, searching.Repo, searching.Branch = "owner", "repo", "shared" + + got := poller.selectDuePRWatches(context.Background(), []*PRWatch{searching, numbered}) + if len(got) != 1 || got[0] != numbered { + t.Fatalf("due watches = %#v, want only numbered watch", got) + } +} + +func TestSearchingWatchAdaptiveScheduleActivityAndRestart(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + watch := scheduleWatch("task-1", 0, timePtr(now.Add(-10*time.Minute)), now.Add(-48*time.Hour)) + activity := PRWatchTaskActivity{LastActivityAt: now.Add(-24 * time.Hour)} + provider := &scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{"task-1": activity}} + + first := &Poller{taskActivityProvider: provider, clock: func() time.Time { return now }} + if got := first.selectDuePRWatches(context.Background(), []*PRWatch{watch}); len(got) != 0 { + t.Fatalf("slow watch due before interval = %d, want 0", len(got)) + } + + // A fresh activity observation makes the same persisted last check due on + // the next scheduler tick, even though the previous cycle used the slow tier. + provider.activities["task-1"] = PRWatchTaskActivity{LastActivityAt: now.Add(-time.Minute)} + if got := first.selectDuePRWatches(context.Background(), []*PRWatch{watch}); len(got) != 1 { + t.Fatalf("watch after fresh activity due = %d, want 1", len(got)) + } + + // A new poller sees the same persisted timestamp and activity projection. + provider.activities["task-1"] = activity + restarted := &Poller{taskActivityProvider: provider, clock: func() time.Time { return now.Add(20 * time.Minute) }} + if got := restarted.selectDuePRWatches(context.Background(), []*PRWatch{watch}); len(got) != 1 { + t.Fatalf("watch after restart due = %d, want 1", len(got)) + } +} + +func TestSearchingWatchAdaptiveScheduleProviderFailureFailsOpen(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + watch := scheduleWatch("task-1", 0, timePtr(now.Add(-time.Minute)), now.Add(-48*time.Hour)) + poller := &Poller{ + taskActivityProvider: &scheduleActivityProvider{err: errors.New("activity unavailable")}, + clock: func() time.Time { return now }, + } + if got := poller.selectDuePRWatches(context.Background(), []*PRWatch{watch}); len(got) != 1 { + t.Fatalf("watch after activity read failure due = %d, want 1", len(got)) + } +} + +func TestSearchingWatchAdaptiveScheduleKeepsExternalPRDiscoverable(t *testing.T) { + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + watch := scheduleWatch("task-external", 0, timePtr(now.Add(-30*time.Minute)), now.Add(-48*time.Hour)) + activity := PRWatchTaskActivity{LastActivityAt: now.Add(-48 * time.Hour)} + poller := &Poller{ + taskActivityProvider: &scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{ + watch.TaskID: activity, + }}, + clock: func() time.Time { return now }, + } + if got := poller.selectDuePRWatches(context.Background(), []*PRWatch{watch}); len(got) != 1 { + t.Fatalf("external PR first slow-tier check = %d, want 1", len(got)) + } + // No local commit or task activity is added. The 30-minute tier remains a + // periodic search, so a PR opened externally remains discoverable. + poller.clock = func() time.Time { return now.Add(30 * time.Minute) } + if got := poller.selectDuePRWatches(context.Background(), []*PRWatch{watch}); len(got) != 1 { + t.Fatalf("external PR later slow-tier check = %d, want 1", len(got)) + } +} + +func TestSearchingWatchAdaptiveScheduleExcludesArchivedAndDeletedTasks(t *testing.T) { + poller, _, _, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-schedule-active", false) + seedTask(t, store, "task-schedule-archived", true) + for _, taskID := range []string{"task-schedule-active", "task-schedule-archived"} { + if err := store.CreatePRWatch(ctx, withTestWorkspace(&PRWatch{ + SessionID: "session-" + taskID, TaskID: taskID, Owner: "owner", Repo: "repo", Branch: taskID, + })); err != nil { + t.Fatalf("create watch for %s: %v", taskID, err) + } + } + + active, err := store.ListActivePRWatches(ctx) + if err != nil { + t.Fatalf("list active watches: %v", err) + } + if len(active) != 1 || active[0].TaskID != "task-schedule-active" { + t.Fatalf("active watches = %+v, want active task only", active) + } + if _, err := store.db.Exec(`DELETE FROM tasks WHERE id = ?`, "task-schedule-active"); err != nil { + t.Fatalf("delete task: %v", err) + } + active, err = store.ListActivePRWatches(ctx) + if err != nil { + t.Fatalf("list watches after delete: %v", err) + } + if len(active) != 0 { + t.Fatalf("watches after task delete = %+v, want none", active) + } + _ = poller +} + +func scheduleWatch(taskID string, prNumber int, lastChecked *time.Time, createdAt time.Time) *PRWatch { + return &PRWatch{ + ID: taskID + "-watch", + TaskID: taskID, + WorkspaceID: "workspace-1", + Owner: "owner-" + taskID, + Repo: "repo-" + taskID, + Branch: "branch-" + taskID, + PRNumber: prNumber, + LastCheckedAt: lastChecked, + CreatedAt: createdAt, + } +} + +func timePtr(value time.Time) *time.Time { + return &value +} + +func PRWatchTaskTaskActivityAt(at time.Time) PRWatchTaskActivity { + return PRWatchTaskActivity{LastActivityAt: at} +} diff --git a/apps/backend/internal/github/poller_test.go b/apps/backend/internal/github/poller_test.go index bae3db4fb5..de7431bb91 100644 --- a/apps/backend/internal/github/poller_test.go +++ b/apps/backend/internal/github/poller_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "runtime" "strings" "sync" @@ -742,6 +743,7 @@ func TestRefreshStaleBranches_SkipsWhenResolverReturnsEmpty(t *testing.T) { // exercise the batched poll path without hitting the network. type graphQLMockClient struct { *MockClient + mu sync.Mutex prResponses []string // FIFO; one entry consumed per ExecuteGraphQL call carrying "Batch" branchResponses []string // FIFO; consumed for "Branches" queries prErr error // returned for the next "Batch" call @@ -751,13 +753,21 @@ type graphQLMockClient struct { // onExecute, if set, is called at the top of every ExecuteGraphQL before // canned responses are consumed. Used by the singleflight coalescing // test to block the first in-flight call while a second one races. - onExecute func() + onExecute func() + statusCalls int +} + +func (m *graphQLMockClient) GetPRStatus(ctx context.Context, owner, repo string, number int) (*PRStatus, error) { + m.statusCalls++ + return m.MockClient.GetPRStatus(ctx, owner, repo, number) } func (m *graphQLMockClient) ExecuteGraphQL(_ context.Context, query string, _ map[string]any, out any) error { if m.onExecute != nil { m.onExecute() } + m.mu.Lock() + defer m.mu.Unlock() if strings.Contains(query, "query Branches") { m.branchQueries = append(m.branchQueries, query) if m.branchErr != nil { @@ -827,7 +837,7 @@ func TestTryBatchedPRWatchCheck_NumberedWatch_AppliesStatus(t *testing.T) { t.Fatalf("create task PR: %v", err) } - if !poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if !batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Fatalf("expected batched path to succeed") } @@ -1095,7 +1105,7 @@ func TestPollerFallbackAfterUnavailableBatchDoesNotPauseDiscovery(t *testing.T) URL: "https://x/52", BaseBranch: "main", }) client.branchErr = errors.New("graphql transport unavailable") - if poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Fatal("unavailable batch unexpectedly succeeded") } // The poller's per-watch fallback must be able to acquire the released @@ -1186,7 +1196,7 @@ func TestTryBatchedPRWatchCheck_PersistsExactReviewThreadCount(t *testing.T) { t.Fatalf("create task PR: %v", err) } - if !poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if !batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Fatal("expected batched path to succeed") } @@ -1225,7 +1235,7 @@ func TestTryBatchedPRWatchCheck_PreservesReviewThreadCountOnPaginationFailure(t t.Fatalf("create task PR: %v", err) } - if poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Fatal("expected batched path to fail") } updated, err := store.GetTaskPR(ctx, "t1") @@ -1293,7 +1303,7 @@ func TestTryBatchedPRWatchCheck_PublishesOnPRFeedbackWatermarkChange(t *testing. t.Fatalf("create task PR: %v", err) } - if !poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if !batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Fatalf("expected batched path to succeed") } select { @@ -1448,7 +1458,7 @@ func TestTryBatchedPRWatchCheck_SearchingWatch_DetectsPR(t *testing.T) { t.Fatalf("create PR watch: %v", err) } - if !poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if !batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Fatalf("expected batched path to succeed") } @@ -1471,11 +1481,239 @@ func TestTryBatchedPRWatchCheck_FallsBackOnUnsupportedClient(t *testing.T) { if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { t.Fatalf("create PR watch: %v", err) } - if poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch}) { + if batchedPRWatchCompleted(poller.tryBatchedPRWatchCheck(ctx, []*PRWatch{watch})) { t.Errorf("expected false when client does not implement GraphQLExecutor") } } +func batchedPRWatchCompleted(outcomes []prWatchWorkspaceBatchOutcome) bool { + if len(outcomes) == 0 { + return false + } + for _, outcome := range outcomes { + if outcome.state != prWatchBatchCompleted { + return false + } + } + return true +} + +type fallbackProbeClient struct { + Client + mu sync.Mutex + findErr error + branches []string +} + +func (c *fallbackProbeClient) FindPRByBranch(ctx context.Context, owner, repo, branch string) (*PR, error) { + c.mu.Lock() + c.branches = append(c.branches, branch) + err := c.findErr + c.mu.Unlock() + if err != nil { + return nil, err + } + return c.Client.FindPRByBranch(ctx, owner, repo, branch) +} + +func (c *fallbackProbeClient) fallbackBranches() []string { + c.mu.Lock() + defer c.mu.Unlock() + return append([]string(nil), c.branches...) +} + +func TestPRWatchFallbackBudgetAndFairness(t *testing.T) { + poller, service, _, store := setupPollerTest(t) + ctx := context.Background() + + unsupported := &fallbackProbeClient{Client: NewMockClient()} + transientGraphQL := &graphQLMockClient{ + MockClient: NewMockClient(), + branchErr: errors.New("graphql transport unavailable"), + } + transient := &fallbackProbeClient{Client: transientGraphQL} + successful := &graphQLMockClient{ + MockClient: NewMockClient(), + prResponses: []string{ + `{"data":{"repo0":{"pr0":{"state":"OPEN","title":"batched","url":"https://x/42","isDraft":false,"mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","headRefName":"feat","baseRefName":"main","headRefOid":"abc","author":{"login":"alice"},"createdAt":"2026-01-01T00:00:00Z","updatedAt":"2026-01-02T00:00:00Z","reviews":{"nodes":[]},"reviewRequests":{"totalCount":0},"commits":{"nodes":[{"commit":{"statusCheckRollup":{"state":"SUCCESS"}}}]}}}}}`, + `{"data":{"repo0":{"pr0":{"state":"OPEN","title":"batched","url":"https://x/42","isDraft":false,"mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","headRefName":"feat","baseRefName":"main","headRefOid":"abc","author":{"login":"alice"},"createdAt":"2026-01-01T00:00:00Z","updatedAt":"2026-01-02T00:00:00Z","reviews":{"nodes":[]},"reviewRequests":{"totalCount":0},"commits":{"nodes":[{"commit":{"statusCheckRollup":{"state":"SUCCESS"}}}]}}}}}`, + }, + } + + service.resolver = NewCredentialResolver(testConnectionReader{workspaces: map[string]*WorkspaceConnection{ + "workspace-a": {WorkspaceID: "workspace-a", Source: ConnectionSourcePAT, GitHubHost: defaultGitHubHost, Login: "a", Status: ConnectionStatusActive, CredentialGeneration: 1}, + "workspace-b": {WorkspaceID: "workspace-b", Source: ConnectionSourcePAT, GitHubHost: defaultGitHubHost, Login: "b", Status: ConnectionStatusActive, CredentialGeneration: 1}, + "workspace-c": {WorkspaceID: "workspace-c", Source: ConnectionSourcePAT, GitHubHost: defaultGitHubHost, Login: "c", Status: ConnectionStatusActive, CredentialGeneration: 1}, + }}, nil) + service.resolver.SetAutomationProvider(workspaceAutomationCredentialProvider{clients: map[string]Client{ + "workspace-a": unsupported, + "workspace-b": transient, + "workspace-c": successful, + }}) + + for _, workspaceID := range []string{"workspace-a", "workspace-b"} { + for index := 0; index < 6; index++ { + taskID := fmt.Sprintf("%s-task-%d", workspaceID, index) + seedTask(t, store, taskID, false) + if err := store.CreatePRWatch(ctx, &PRWatch{ + WorkspaceID: workspaceID, + SessionID: taskID + "-session", + TaskID: taskID, + Owner: "o", + Repo: "r", + Branch: fmt.Sprintf("%s-branch-%d", workspaceID, index), + }); err != nil { + t.Fatalf("create %s watch %d: %v", workspaceID, index, err) + } + } + } + seedTask(t, store, "workspace-c-task", false) + if err := store.CreatePRWatch(ctx, &PRWatch{ + WorkspaceID: "workspace-c", SessionID: "workspace-c-session", TaskID: "workspace-c-task", + Owner: "o", Repo: "r", PRNumber: 42, Branch: "feat", LastCheckStatus: "pending", + }); err != nil { + t.Fatalf("create successful workspace watch: %v", err) + } + if err := store.CreateTaskPR(ctx, &TaskPR{ + TaskID: "workspace-c-task", Owner: "o", Repo: "r", PRNumber: 42, State: "open", + }); err != nil { + t.Fatalf("create successful workspace task PR: %v", err) + } + + eventsSeen := 0 + if _, err := poller.eventBus.Subscribe(events.GitHubPRFeedback, func(context.Context, *bus.Event) error { + eventsSeen++ + return nil + }); err != nil { + t.Fatalf("subscribe PR feedback: %v", err) + } + + poller.checkPRWatches(ctx) + firstUnsupported := unsupported.fallbackBranches() + firstTransient := transient.fallbackBranches() + if len(firstUnsupported) > 5 || len(firstTransient) > 5 || len(firstUnsupported)+len(firstTransient) > 10 { + t.Fatalf("first fallback cycle exceeded budget: unsupported=%d transient=%d", len(firstUnsupported), len(firstTransient)) + } + if len(firstUnsupported) == 0 || len(firstTransient) == 0 { + t.Fatalf("first fallback cycle starved a workspace: unsupported=%d transient=%d", len(firstUnsupported), len(firstTransient)) + } + if successful.statusCalls != 0 { + t.Fatalf("successful batched workspace repeated through fallback: status calls=%d", successful.statusCalls) + } + if eventsSeen != 1 { + t.Fatalf("successful batched workspace events=%d, want 1", eventsSeen) + } + + poller.checkPRWatches(ctx) + secondUnsupported := unsupported.fallbackBranches() + secondTransient := transient.fallbackBranches() + if len(secondUnsupported) > 10 || len(secondTransient) > 10 || len(secondUnsupported)+len(secondTransient) > 20 { + t.Fatalf("second fallback cycles exceeded cumulative budget: unsupported=%d transient=%d", len(secondUnsupported), len(secondTransient)) + } + if !containsFallbackBranch(secondUnsupported, "workspace-a-branch-5") || !containsFallbackBranch(secondTransient, "workspace-b-branch-5") { + t.Fatalf("fallback target rotation starved the last target: unsupported=%v transient=%v", secondUnsupported, secondTransient) + } + if successful.statusCalls != 0 { + t.Fatalf("successful batched workspace repeated through fallback on second cycle: status calls=%d", successful.statusCalls) + } + if eventsSeen != 1 { + t.Fatalf("successful batched workspace events=%d after second cycle, want the original event only", eventsSeen) + } +} + +func TestPRWatchFallbackDeduplicatesSharedTargetsAndFansOut(t *testing.T) { + poller, service, _, store := setupPollerTest(t) + client := &fallbackProbeClient{Client: NewMockClient()} + configureTestWorkspaceAuth(t, service, client, testWorkspaceID) + watches := seedSearchingFallbackWatches(t, store, 5) + watches[1].Branch = watches[0].Branch + + poller.runPRWatchFallback(context.Background(), watches) + branches := client.fallbackBranches() + if len(branches) != 4 { + t.Fatalf("fallback calls = %d, want one per unique target", len(branches)) + } + sharedCalls := 0 + for _, branch := range branches { + if branch == watches[0].Branch { + sharedCalls++ + } + } + if sharedCalls != 1 { + t.Fatalf("shared target calls = %d, want one", sharedCalls) + } + for _, watch := range watches { + updated, err := store.GetPRWatch(context.Background(), watch.ID) + if err != nil { + t.Fatalf("get watch %q: %v", watch.ID, err) + } + if updated == nil || updated.LastCheckedAt == nil { + t.Fatalf("watch %q did not receive the shared fallback result", watch.ID) + } + } +} + +func containsFallbackBranch(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func seedSearchingFallbackWatches(t *testing.T, store *Store, count int) []*PRWatch { + t.Helper() + ctx := context.Background() + watches := make([]*PRWatch, 0, count) + for index := 0; index < count; index++ { + taskID := fmt.Sprintf("fallback-task-%d", index) + seedTask(t, store, taskID, false) + watch := withTestWorkspace(&PRWatch{ + SessionID: fmt.Sprintf("fallback-session-%d", index), + TaskID: taskID, + Owner: "o", + Repo: "r", + Branch: fmt.Sprintf("fallback-branch-%d", index), + }) + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create fallback watch %d: %v", index, err) + } + watches = append(watches, watch) + } + return watches +} + +func TestPRWatchFallbackStopsOnRateLimitAndCancellation(t *testing.T) { + t.Run("rate limit stops workspace", func(t *testing.T) { + poller, service, _, store := setupPollerTest(t) + rateClient := &fallbackProbeClient{ + Client: NewMockClient(), + findErr: rateLimitAPIError(429, "secondary limit"), + } + configureTestWorkspaceAuth(t, service, rateClient, testWorkspaceID) + + poller.runPRWatchFallback(context.Background(), seedSearchingFallbackWatches(t, store, 3)) + if got := len(rateClient.fallbackBranches()); got != 1 { + t.Fatalf("rate-limited fallback calls = %d, want one affected workspace probe", got) + } + }) + + t.Run("cancellation stops the cycle", func(t *testing.T) { + poller, service, _, store := setupPollerTest(t) + cancelClient := &fallbackProbeClient{ + Client: NewMockClient(), + findErr: context.Canceled, + } + configureTestWorkspaceAuth(t, service, cancelClient, testWorkspaceID) + + poller.runPRWatchFallback(context.Background(), seedSearchingFallbackWatches(t, store, 3)) + if got := len(cancelClient.fallbackBranches()); got != 1 { + t.Fatalf("cancelled fallback calls = %d, want one probe before cycle stop", got) + } + }) +} + // TestSyncWatchesBatched_ManyWatches_TwoGraphQLCalls is the regression test // for the OS-freeze caused by per-watch gh subprocess fan-out. A workspace // with N numbered + M searching watches must fire exactly TWO GraphQL @@ -1681,6 +1919,10 @@ func TestRefreshStaleWorkspaceWatches_CoalescesConcurrentCalls(t *testing.T) { } // A fresh refresh must start (key was cleared in the defer). + // The adaptive selector still honors the one-minute fast tier after the + // first empty search, so move the injected service clock past that due + // boundary before asserting a later refresh can run. + svc.SetClock(func() time.Time { return time.Now().UTC().Add(searchFastPollInterval) }) release2 := make(chan struct{}) gh.onExecute = func() { started <- struct{}{} diff --git a/apps/backend/internal/github/review_pr_task_test.go b/apps/backend/internal/github/review_pr_task_test.go index 61cf4f6275..c5b798f280 100644 --- a/apps/backend/internal/github/review_pr_task_test.go +++ b/apps/backend/internal/github/review_pr_task_test.go @@ -162,6 +162,7 @@ func TestAssignReviewPRTaskID_UpdatesTaskID(t *testing.T) { ); err != nil { t.Fatalf("Reserve: %v", err) } + seedTask(t, store, "task-xyz", false) if err := store.AssignReviewPRTaskID(ctx, watch.ID, "acme", "widget", 42, "task-xyz"); err != nil { t.Fatalf("AssignReviewPRTaskID: %v", err) diff --git a/apps/backend/internal/github/service.go b/apps/backend/internal/github/service.go index 9df010f7a8..e913c63b37 100644 --- a/apps/backend/internal/github/service.go +++ b/apps/backend/internal/github/service.go @@ -141,6 +141,9 @@ type Service struct { comparisonTargetObserver ComparisonTargetObserver taskIssueStore TaskIssueStore workspaceGroupOwnerResolver WorkspaceGroupOwnerResolver + taskActivityProvider TaskActivityProvider + clockMu sync.RWMutex + clock func() time.Time // cascadeTaskDeleter is the cascade-delete entry point used by the // watch reset flow. It is distinct from taskDeleter (which only deletes // a single task by ID) because reset must walk the task tree and clean @@ -153,6 +156,8 @@ type Service struct { searchCache *ttlCache prStatusCache *ttlCache prFeedbackCache *ttlCache + workflowRunsCache *ttlCache + workflowJobsCache *ttlCache mergeMethodsCache *ttlCache accessibleReposCache *ttlCache repoErrorCache *ttlCache @@ -185,6 +190,21 @@ type Service struct { // LoadOrStore guard, not iteration. inflightWorkspaceRefreshes sync.Map + // passiveFallback* limits the legacy per-watch fallback used by passive + // workspace refreshes. The admission window is shared across workspaces so + // a batch outage cannot turn a single background tick into an unbounded CLI + // fan-out. Admissions are retained as timestamps to enforce a rolling + // minute instead of allowing a wall-clock boundary burst. + passiveFallbackMu sync.Mutex + passiveFallbackAdmissions []passiveFallbackAdmission + passiveFallbackTargetCursors map[string]int + + // passiveWorkspaceRefreshAt suppresses repeated workspace reads that find + // no due watch. Stale-task refreshes bypass this admission and remain + // independent of the passive cooldown. + passiveWorkspaceRefreshMu sync.Mutex + passiveWorkspaceRefreshAt map[string]time.Time + // stopCtx / stopCancel / bgWG own the lifecycle of background goroutines // the service spawns lazily (currently refreshStaleWorkspaceWatches). // Stop() cancels stopCtx and waits for bgWG to drain, satisfying the @@ -217,29 +237,34 @@ func (s *Service) authorizeWorkspaceAccess(ctx context.Context, workspaceID stri func NewService(client Client, authMethod string, secrets SecretProvider, store *Store, eventBus bus.EventBus, log *logger.Logger) *Service { stopCtx, stopCancel := context.WithCancel(context.Background()) service := &Service{ - client: client, - authMethod: authMethod, - secrets: secrets, - store: store, - eventBus: eventBus, - logger: log, - searchCache: newTTLCache(), - prStatusCache: newTTLCache(), - prFeedbackCache: newPRFeedbackCache(), - mergeMethodsCache: newMergeMethodsCache(), - accessibleReposCache: newAccessibleReposCache(), - repoErrorCache: newRepoErrorCache(), - forkParentCache: newForkParentCache(), - protectionCache: newBranchProtectionCache(), - rateTracker: NewRateTracker(eventBus, log), - prDiscoveryHealth: newPRDiscoveryHealth(eventBus, log), - prDiscoveryAttempts: make(map[string]*prDiscoveryAttemptResult), - tokenClientFactory: func(token string) Client { return NewPATClient(token) }, - ghAccountLister: ListGHAccounts, - cleanupFailureCounts: make(map[string]int), - appRegistrationRuntimes: make(map[string]*githubAppRuntime), - stopCtx: stopCtx, - stopCancel: stopCancel, + client: client, + authMethod: authMethod, + secrets: secrets, + store: store, + eventBus: eventBus, + logger: log, + searchCache: newTTLCache(), + prStatusCache: newTTLCache(), + prFeedbackCache: newPRFeedbackCache(), + workflowRunsCache: newWorkflowAttentionCache(), + workflowJobsCache: newWorkflowAttentionCache(), + mergeMethodsCache: newMergeMethodsCache(), + accessibleReposCache: newAccessibleReposCache(), + repoErrorCache: newRepoErrorCache(), + forkParentCache: newForkParentCache(), + protectionCache: newBranchProtectionCache(), + rateTracker: NewRateTracker(eventBus, log), + prDiscoveryHealth: newPRDiscoveryHealth(eventBus, log), + prDiscoveryAttempts: make(map[string]*prDiscoveryAttemptResult), + tokenClientFactory: func(token string) Client { return NewPATClient(token) }, + ghAccountLister: ListGHAccounts, + cleanupFailureCounts: make(map[string]int), + passiveFallbackTargetCursors: make(map[string]int), + passiveWorkspaceRefreshAt: make(map[string]time.Time), + appRegistrationRuntimes: make(map[string]*githubAppRuntime), + stopCtx: stopCtx, + stopCancel: stopCancel, + clock: time.Now, } if store != nil { service.resolver = NewCredentialResolver(store, secrets) @@ -340,6 +365,24 @@ func (s *Service) SetCascadeTaskDeleter(d watchreset.TaskDeleter) { // SetTaskSessionChecker sets the session checker for cleanup operations. func (s *Service) SetTaskSessionChecker(c TaskSessionChecker) { s.taskSessionChecker = c } +// SetTaskActivityProvider wires the persisted task activity projection used +// by passive adaptive searching-watch refreshes. +func (s *Service) SetTaskActivityProvider(provider TaskActivityProvider) { + s.taskActivityProvider = provider +} + +// SetClock installs the service clock used by passive refresh admission. +// Production uses time.Now; tests can use a fixed clock. +func (s *Service) SetClock(clock func() time.Time) { + s.clockMu.Lock() + defer s.clockMu.Unlock() + if clock == nil { + s.clock = time.Now + return + } + s.clock = clock +} + // SetSecretManager sets the secret manager for token configuration operations. func (s *Service) SetSecretManager(m SecretManager) { s.secretManager = m } diff --git a/apps/backend/internal/github/service_cleanup.go b/apps/backend/internal/github/service_cleanup.go index 3662483276..82d77a8093 100644 --- a/apps/backend/internal/github/service_cleanup.go +++ b/apps/backend/internal/github/service_cleanup.go @@ -2,11 +2,61 @@ package github import ( "context" + "errors" "fmt" + "net/http" "go.uber.org/zap" ) +func cleanupBatchAdmission(ctx context.Context, client Client, tracker *RateTracker, policy string) error { + if err := ctx.Err(); err != nil { + return err + } + if policy == CleanupPolicyNever || tracker == nil { + return nil + } + resource := cleanupRateResource(client) + if tracker.WaitDuration(resource) <= 0 { + return nil + } + return &GitHubAPIError{ + StatusCode: http.StatusTooManyRequests, + Endpoint: "cleanup", + Body: fmt.Sprintf("GitHub %s rate limit exhausted", resource), + } +} + +type cleanupRateResourceReporter interface { + RateResource() Resource +} + +func cleanupRateResource(client Client) Resource { + if reporter, ok := client.(cleanupRateResourceReporter); ok { + if resource := reporter.RateResource(); resource != "" { + return resource + } + } + return ResourceCore +} + +func cleanupBatchShouldStop(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + var apiErr *GitHubAPIError + if errors.As(err, &apiErr) && isGitHubRateLimitAPIError(apiErr) { + return true + } + // Legacy GHClient calls return the CLI's stderr wrapped in an exec error. + // Keep the established marker classifier at this boundary so cleanup stops + // after the first affected row even when no typed API response exists. + return ghStderrIndicatesRateLimit(err.Error()) +} + // --- Review-task cleanup --- // CleanupMergedReviewTasks checks PRs tracked by a review watch and deletes @@ -27,7 +77,8 @@ func (s *Service) CleanupMergedReviewTasks(ctx context.Context, watch *ReviewWat return 0, fmt.Errorf("list review PR tasks: %w", err) } policy := NormalizeCleanupPolicy(watch.CleanupPolicy) - return s.cleanupReviewPRTaskBatch(ctx, resolved.Client, prTasks, func(_ *ReviewPRTask) string { return policy }), nil + return s.cleanupReviewPRTaskBatch(ctx, resolved.Client, resolved.RateTracker, prTasks, + func(_ *ReviewPRTask) string { return policy }) } // CleanupAllOrphanedReviewTasks sweeps dedup rows whose watch is deleted or @@ -81,9 +132,9 @@ func (s *Service) CleanupReviewTasksForWorkspace(ctx context.Context, workspaceI candidates = append(candidates, rpt) } } - return s.cleanupReviewPRTaskBatch(ctx, resolved.Client, candidates, func(rpt *ReviewPRTask) string { + return s.cleanupReviewPRTaskBatch(ctx, resolved.Client, resolved.RateTracker, candidates, func(rpt *ReviewPRTask) string { return watchPolicies[rpt.ReviewWatchID] - }), nil + }) } // cleanupAllReviewTasks is the shared body. When orphansOnly is true, rows @@ -133,9 +184,13 @@ func (s *Service) cleanupAllReviewTasks(ctx context.Context, orphansOnly bool) ( zap.String("workspace_id", workspaceID), zap.Error(resolveErr)) continue } - deleted += s.cleanupReviewPRTaskBatch(ctx, resolved.Client, candidates, func(rpt *ReviewPRTask) string { + batchDeleted, batchErr := s.cleanupReviewPRTaskBatch(ctx, resolved.Client, resolved.RateTracker, candidates, func(rpt *ReviewPRTask) string { return policyCache[rpt.ReviewWatchID] }) + deleted += batchDeleted + if batchErr != nil { + return deleted, batchErr + } } return deleted, nil } @@ -197,11 +252,15 @@ func (s *Service) deleteTaskWithReason(ctx context.Context, taskID, reason strin // //nolint:dupl // mirrors cleanupIssueTaskBatch — different types, same structure func (s *Service) cleanupReviewPRTaskBatch( - ctx context.Context, client Client, prTasks []*ReviewPRTask, resolvePolicy func(*ReviewPRTask) string, -) int { + ctx context.Context, client Client, tracker *RateTracker, prTasks []*ReviewPRTask, + resolvePolicy func(*ReviewPRTask) string, +) (int, error) { deleted := 0 for _, rpt := range prTasks { policy := resolvePolicy(rpt) + if err := cleanupBatchAdmission(ctx, client, tracker, policy); err != nil { + return deleted, err + } // Orphan reservation: process was killed after ReserveReviewPRTask // succeeded but before AssignReviewPRTaskID ran, so task_id is empty // and there is no task to delete. Clean up the dedup row once the PR @@ -212,7 +271,14 @@ func (s *Service) cleanupReviewPRTaskBatch( // increment `deleted`: the count is reported back to the // settings-page toast as "Deleted N tasks", and these rows // never had an associated task. - if should, _ := s.shouldDeleteReviewTaskWithClient(ctx, client, rpt, policy); should { + should, _, err := s.shouldDeleteReviewTaskWithClient(ctx, client, rpt, policy) + if err != nil { + if cleanupBatchShouldStop(err) { + return deleted, err + } + continue + } + if should { if err := s.store.DeleteReviewPRTask(ctx, rpt.ID); err != nil { s.logger.Warn("failed to delete orphan reservation row", zap.String("dedup_id", rpt.ID), zap.Error(err)) @@ -220,7 +286,13 @@ func (s *Service) cleanupReviewPRTaskBatch( } continue } - shouldDelete, reason := s.shouldDeleteReviewTaskWithClient(ctx, client, rpt, policy) + shouldDelete, reason, err := s.shouldDeleteReviewTaskWithClient(ctx, client, rpt, policy) + if err != nil { + if cleanupBatchShouldStop(err) { + return deleted, err + } + continue + } if !shouldDelete { continue } @@ -256,7 +328,7 @@ func (s *Service) cleanupReviewPRTaskBatch( zap.String("repo", rpt.RepoOwner+"/"+rpt.RepoName)) deleted++ } - return deleted + return deleted, nil } // shouldDeleteReviewTask checks whether a review PR task is eligible for @@ -268,68 +340,95 @@ func (s *Service) cleanupReviewPRTaskBatch( // Terminal state covers: PR merged/closed, OR the authenticated user already // approved the PR on GitHub (so it's effectively done from their POV). func (s *Service) shouldDeleteReviewTask(ctx context.Context, rpt *ReviewPRTask, policy string) (bool, string) { - return s.shouldDeleteReviewTaskWithClient(ctx, s.client, rpt, policy) + should, reason, _ := s.shouldDeleteReviewTaskWithClient(ctx, s.client, rpt, policy) + return should, reason } func (s *Service) shouldDeleteReviewTaskWithClient( ctx context.Context, client Client, rpt *ReviewPRTask, policy string, -) (bool, string) { +) (bool, string, error) { if policy == CleanupPolicyNever { - return false, "" + return false, "", nil } failureKey := reviewFailureKey(rpt) - feedback, err := client.GetPRFeedback(ctx, rpt.RepoOwner, rpt.RepoName, rpt.PRNumber) + pr, err := client.GetPR(ctx, rpt.RepoOwner, rpt.RepoName, rpt.PRNumber) if err != nil { s.trackCleanupFailure(failureKey, "review", rpt.RepoOwner+"/"+rpt.RepoName, rpt.PRNumber, err) - return false, "" + return false, "", err } s.resetCleanupFailure(failureKey) - if feedback.PR == nil { - return false, "" + if pr == nil { + return false, "", nil } var reason string - if feedback.PR.State == prStateMerged || feedback.PR.State == prStateClosed { + if pr.State == prStateMerged || pr.State == prStateClosed { reason = "pr_merged_or_closed" //nolint:goconst // also referenced by string in service_cleanup_policy_test.go; introducing a constant would require coupling the test } else { - // Check if the authenticated user already approved the PR on GitHub. - user, _ := client.GetAuthenticatedUser(ctx) - for _, review := range feedback.Reviews { - if review.State == "APPROVED" && review.Author == user { - reason = "pr_approved_by_user" - break - } + reason, err = s.reviewApprovalCleanupReason(ctx, client, rpt) + if err != nil { + s.trackCleanupFailure(failureKey, "review", rpt.RepoOwner+"/"+rpt.RepoName, rpt.PRNumber, err) + return false, "", err } } if reason == "" { - return false, "" + return false, "", nil } if policy == CleanupPolicyAlways || rpt.TaskID == "" { - return true, reason + return true, reason, nil } if s.store != nil { enabled, err := s.HasEnabledTaskPRAgentPrompts(ctx, rpt.TaskID) if err != nil { s.logger.Debug("failed to check task PR agent prompt options", zap.String("task_id", rpt.TaskID), zap.Error(err)) - return false, "" + return false, "", nil } if enabled { - return false, "" + return false, "", nil } } if s.taskSessionChecker == nil { - return true, reason + return true, reason, nil } hasUserMsg, err := s.taskSessionChecker.HasUserAuthoredMessage(ctx, rpt.TaskID) if err != nil { s.logger.Debug("failed to check task user messages", zap.String("task_id", rpt.TaskID), zap.Error(err)) - return false, "" + return false, "", nil } if hasUserMsg { - return false, "" + return false, "", nil } - return true, reason + return true, reason, nil +} + +func (s *Service) reviewApprovalCleanupReason( + ctx context.Context, client Client, rpt *ReviewPRTask, +) (string, error) { + reviews, err := client.ListPRReviews(ctx, rpt.RepoOwner, rpt.RepoName, rpt.PRNumber) + if err != nil { + return "", err + } + approved := false + for _, review := range reviews { + if review.State == reviewStateApproved { + approved = true + break + } + } + if !approved { + return "", nil + } + user, userErr := client.GetAuthenticatedUser(ctx) + if userErr != nil { + return "", userErr + } + for _, review := range reviews { + if review.State == reviewStateApproved && review.Author == user { + return "pr_approved_by_user", nil + } + } + return "", nil } // reviewFailureKey builds the stable per-row identifier used for failure @@ -396,7 +495,8 @@ func (s *Service) CleanupClosedIssueTasks(ctx context.Context, watch *IssueWatch return 0, fmt.Errorf("list issue watch tasks: %w", err) } policy := NormalizeCleanupPolicy(watch.CleanupPolicy) - return s.cleanupIssueTaskBatch(ctx, resolved.Client, issueTasks, func(_ *IssueWatchTask) string { return policy }), nil + return s.cleanupIssueTaskBatch(ctx, resolved.Client, resolved.RateTracker, issueTasks, + func(_ *IssueWatchTask) string { return policy }) } // CleanupAllOrphanedIssueTasks sweeps dedup rows whose watch is deleted or @@ -444,9 +544,9 @@ func (s *Service) CleanupIssueTasksForWorkspace(ctx context.Context, workspaceID candidates = append(candidates, it) } } - return s.cleanupIssueTaskBatch(ctx, resolved.Client, candidates, func(it *IssueWatchTask) string { + return s.cleanupIssueTaskBatch(ctx, resolved.Client, resolved.RateTracker, candidates, func(it *IssueWatchTask) string { return watchPolicies[it.IssueWatchID] - }), nil + }) } //nolint:dupl // mirrors cleanupAllReviewTasks — different types, same orchestration @@ -484,9 +584,13 @@ func (s *Service) cleanupAllIssueTasks(ctx context.Context, orphansOnly bool) (i zap.String("workspace_id", workspaceID), zap.Error(resolveErr)) continue } - deleted += s.cleanupIssueTaskBatch(ctx, resolved.Client, candidates, func(it *IssueWatchTask) string { + batchDeleted, batchErr := s.cleanupIssueTaskBatch(ctx, resolved.Client, resolved.RateTracker, candidates, func(it *IssueWatchTask) string { return policyCache[it.IssueWatchID] }) + deleted += batchDeleted + if batchErr != nil { + return deleted, batchErr + } } return deleted, nil } @@ -523,15 +627,26 @@ func (s *Service) buildIssueWatchCaches( //nolint:dupl // mirrors cleanupReviewPRTaskBatch — different types, same structure func (s *Service) cleanupIssueTaskBatch( - ctx context.Context, client Client, issueTasks []*IssueWatchTask, resolvePolicy func(*IssueWatchTask) string, -) int { + ctx context.Context, client Client, tracker *RateTracker, issueTasks []*IssueWatchTask, + resolvePolicy func(*IssueWatchTask) string, +) (int, error) { deleted := 0 for _, it := range issueTasks { policy := resolvePolicy(it) + if err := cleanupBatchAdmission(ctx, client, tracker, policy); err != nil { + return deleted, err + } if it.TaskID == "" { // Orphan reservation row — no task was created. Clean it up // but don't count it as a deleted task. - if should, _ := s.shouldDeleteIssueTaskWithClient(ctx, client, it, policy); should { + should, _, err := s.shouldDeleteIssueTaskWithClient(ctx, client, it, policy) + if err != nil { + if cleanupBatchShouldStop(err) { + return deleted, err + } + continue + } + if should { if err := s.store.DeleteIssueWatchTask(ctx, it.ID); err != nil { s.logger.Warn("failed to delete orphan reservation row", zap.String("dedup_id", it.ID), zap.Error(err)) @@ -539,7 +654,13 @@ func (s *Service) cleanupIssueTaskBatch( } continue } - shouldDelete, reason := s.shouldDeleteIssueTaskWithClient(ctx, client, it, policy) + shouldDelete, reason, err := s.shouldDeleteIssueTaskWithClient(ctx, client, it, policy) + if err != nil { + if cleanupBatchShouldStop(err) { + return deleted, err + } + continue + } if !shouldDelete { continue } @@ -571,45 +692,46 @@ func (s *Service) cleanupIssueTaskBatch( zap.String("repo", it.RepoOwner+"/"+it.RepoName)) deleted++ } - return deleted + return deleted, nil } // shouldDeleteIssueTask checks whether an issue task is eligible for cleanup. // Policy gating mirrors shouldDeleteReviewTask. func (s *Service) shouldDeleteIssueTask(ctx context.Context, it *IssueWatchTask, policy string) (bool, string) { - return s.shouldDeleteIssueTaskWithClient(ctx, s.client, it, policy) + should, reason, _ := s.shouldDeleteIssueTaskWithClient(ctx, s.client, it, policy) + return should, reason } func (s *Service) shouldDeleteIssueTaskWithClient( ctx context.Context, client Client, it *IssueWatchTask, policy string, -) (bool, string) { +) (bool, string, error) { if policy == CleanupPolicyNever { - return false, "" + return false, "", nil } failureKey := issueFailureKey(it) state, err := client.GetIssueState(ctx, it.RepoOwner, it.RepoName, it.IssueNumber) if err != nil { s.trackCleanupFailure(failureKey, "issue", it.RepoOwner+"/"+it.RepoName, it.IssueNumber, err) - return false, "" + return false, "", err } s.resetCleanupFailure(failureKey) if state != "closed" { - return false, "" + return false, "", nil } reason := "issue_closed" if policy == CleanupPolicyAlways || it.TaskID == "" || s.taskSessionChecker == nil { - return true, reason + return true, reason, nil } hasUserMsg, err := s.taskSessionChecker.HasUserAuthoredMessage(ctx, it.TaskID) if err != nil { s.logger.Debug("failed to check task user messages", zap.String("task_id", it.TaskID), zap.Error(err)) - return false, "" + return false, "", nil } if hasUserMsg { - return false, "" + return false, "", nil } - return true, reason + return true, reason, nil } func issueFailureKey(it *IssueWatchTask) string { diff --git a/apps/backend/internal/github/service_cleanup_policy_test.go b/apps/backend/internal/github/service_cleanup_policy_test.go index effc62222a..fec591b52b 100644 --- a/apps/backend/internal/github/service_cleanup_policy_test.go +++ b/apps/backend/internal/github/service_cleanup_policy_test.go @@ -46,6 +46,17 @@ func (c *prFeedbackStub) GetPRFeedback(_ context.Context, _, _ string, _ int) (* return &PRFeedback{PR: &PR{State: c.state}}, nil } +func (c *prFeedbackStub) GetPR(_ context.Context, owner, repo string, number int) (*PR, error) { + if c.err != nil { + return nil, c.err + } + return &PR{Number: number, State: c.state, RepoOwner: owner, RepoName: repo}, nil +} + +func (c *prFeedbackStub) ListPRReviews(context.Context, string, string, int) ([]PRReview, error) { + return nil, c.err +} + func reviewTaskFixture() *ReviewPRTask { return &ReviewPRTask{ ID: "rpt-1", @@ -258,6 +269,7 @@ func TestCleanupAllOrphanedReviewTasks_DisabledWatch_StillCleansUp(t *testing.T) if err := store.CreateReviewWatch(ctx, watch); err != nil { t.Fatalf("CreateReviewWatch: %v", err) } + seedTask(t, store, "task-88", false) rpt := &ReviewPRTask{ ReviewWatchID: watch.ID, RepoOwner: "acme", @@ -343,6 +355,7 @@ func TestCleanupAllOrphanedReviewTasks_SkipsEnabledWatchRows(t *testing.T) { if err := store.CreateReviewWatch(ctx, watch); err != nil { t.Fatalf("CreateReviewWatch: %v", err) } + seedTask(t, store, "task-111", false) rpt := &ReviewPRTask{ ReviewWatchID: watch.ID, RepoOwner: "acme", @@ -432,6 +445,7 @@ func TestCheckReviewWatches_RunsOrphanSweepWhenNoEnabledWatches(t *testing.T) { if err := store.CreateReviewWatch(ctx, watch); err != nil { t.Fatalf("CreateReviewWatch: %v", err) } + seedTask(t, store, "task-9001", false) rpt := &ReviewPRTask{ ReviewWatchID: watch.ID, RepoOwner: "acme", @@ -466,6 +480,7 @@ func TestCheckIssueWatches_RunsOrphanSweepWhenNoEnabledWatches(t *testing.T) { if err := store.CreateIssueWatch(ctx, watch); err != nil { t.Fatalf("CreateIssueWatch: %v", err) } + seedTask(t, store, "task-9002", false) _, _ = store.ReserveIssueWatchTask(ctx, watch.ID, "acme", "widget", 9002, "https://example/9002") if err := store.AssignIssueWatchTaskID(ctx, watch.ID, "acme", "widget", 9002, "task-9002"); err != nil { t.Fatalf("AssignIssueWatchTaskID: %v", err) @@ -556,6 +571,7 @@ func TestCleanupAllOrphanedIssueTasks_DisabledWatch_StillCleansUp(t *testing.T) if err := store.CreateIssueWatch(ctx, watch); err != nil { t.Fatalf("CreateIssueWatch: %v", err) } + seedTask(t, store, "task-88", false) _, _ = store.ReserveIssueWatchTask(ctx, watch.ID, "acme", "widget", 88, "https://example/88") if err := store.AssignIssueWatchTaskID(ctx, watch.ID, "acme", "widget", 88, "task-88"); err != nil { t.Fatalf("AssignIssueWatchTaskID: %v", err) @@ -622,6 +638,7 @@ func TestCleanupAllOrphanedIssueTasks_SkipsEnabledWatchRows(t *testing.T) { if err := store.CreateIssueWatch(ctx, watch); err != nil { t.Fatalf("CreateIssueWatch: %v", err) } + seedTask(t, store, "task-111", false) _, _ = store.ReserveIssueWatchTask(ctx, watch.ID, "acme", "widget", 111, "https://example/111") if err := store.AssignIssueWatchTaskID(ctx, watch.ID, "acme", "widget", 111, "task-111"); err != nil { t.Fatalf("AssignIssueWatchTaskID: %v", err) @@ -759,6 +776,7 @@ func TestDeleteReviewWatchesByWorkspace(t *testing.T) { if err := store.CreateReviewWatch(ctx, w); err != nil { t.Fatalf("CreateReviewWatch: %v", err) } + seedTask(t, store, "task-"+w.ID, false) rpt := &ReviewPRTask{ ReviewWatchID: w.ID, RepoOwner: "acme", @@ -868,3 +886,14 @@ func (c *switchingFeedbackClient) GetPRFeedback(_ context.Context, _, _ string, } return &PRFeedback{PR: &PR{State: c.state}}, nil } + +func (c *switchingFeedbackClient) GetPR(_ context.Context, owner, repo string, number int) (*PR, error) { + if c.err != nil { + return nil, c.err + } + return &PR{Number: number, State: c.state, RepoOwner: owner, RepoName: repo}, nil +} + +func (c *switchingFeedbackClient) ListPRReviews(context.Context, string, string, int) ([]PRReview, error) { + return nil, c.err +} diff --git a/apps/backend/internal/github/service_cleanup_rate_limit_test.go b/apps/backend/internal/github/service_cleanup_rate_limit_test.go new file mode 100644 index 0000000000..6b59facdeb --- /dev/null +++ b/apps/backend/internal/github/service_cleanup_rate_limit_test.go @@ -0,0 +1,437 @@ +package github + +import ( + "context" + "errors" + "fmt" + "net/http" + "reflect" + "testing" + "time" +) + +type cleanupRateLimitClient struct { + NoopClient + feedback map[int]cleanupPRResponse + issues map[int]cleanupIssueResponse + prCalls []int + reviewCalls []int + issueCalls []int + user string + userErr error +} + +type cleanupGHCLIClient struct { + *GHClient + err error + calls []int +} + +func (c *cleanupGHCLIClient) GetPR(_ context.Context, _, _ string, number int) (*PR, error) { + c.calls = append(c.calls, number) + if c.err != nil { + return nil, c.err + } + return &PR{Number: number, State: prStateMerged}, nil +} + +type cleanupPRResponse struct { + state string + reviews []PRReview + err error +} + +type cleanupIssueResponse struct { + state string + err error +} + +func (c *cleanupRateLimitClient) GetPR(_ context.Context, owner, repo string, number int) (*PR, error) { + c.prCalls = append(c.prCalls, number) + response := c.feedback[number] + if response.err != nil { + return nil, response.err + } + state := response.state + if state == "" { + state = prStateMerged + } + return &PR{Number: number, State: state, RepoOwner: owner, RepoName: repo}, nil +} + +func (c *cleanupRateLimitClient) ListPRReviews(_ context.Context, _, _ string, number int) ([]PRReview, error) { + c.reviewCalls = append(c.reviewCalls, number) + response := c.feedback[number] + if response.err != nil { + return nil, response.err + } + return response.reviews, nil +} + +func (c *cleanupRateLimitClient) GetIssueState(_ context.Context, _, _ string, number int) (string, error) { + c.issueCalls = append(c.issueCalls, number) + response := c.issues[number] + if response.err != nil { + return "", response.err + } + if response.state == "" { + return "closed", nil + } + return response.state, nil +} + +func (c *cleanupRateLimitClient) GetAuthenticatedUser(context.Context) (string, error) { + if c.userErr != nil { + return "", c.userErr + } + if c.user == "" { + return mockDefaultUser, nil + } + return c.user, nil +} + +type cleanupAutomationProvider struct { + client Client + tracker *RateTracker +} + +func (p cleanupAutomationProvider) ResolveAutomation( + _ context.Context, + connection *WorkspaceConnection, + _ ResolveCredentialRequest, +) (*ResolvedCredential, error) { + return &ResolvedCredential{ + Client: p.client, + Capabilities: allTokenCapabilities(), + RateTracker: p.tracker, + Principal: AuthPrincipal{ + Kind: AuthPrincipalHuman, + Source: ConnectionSourcePAT, + Login: connection.Login, + }, + }, nil +} + +func configureCleanupRateLimitClient(t *testing.T, svc *Service, client Client, tracker *RateTracker) { + t.Helper() + configureTestWorkspaceAuth(t, svc, client, "ws-1") + if tracker != nil { + svc.resolver.SetAutomationProvider(cleanupAutomationProvider{client: client, tracker: tracker}) + } +} + +func createReviewCleanupRows(t *testing.T, store *Store, watch *ReviewWatch, taskIDs ...string) { + t.Helper() + ctx := context.Background() + for i, taskID := range taskIDs { + if taskID != "" { + seedTask(t, store, taskID, false) + } + if err := store.CreateReviewPRTask(ctx, &ReviewPRTask{ + ReviewWatchID: watch.ID, + RepoOwner: "acme", + RepoName: "widget", + PRNumber: i + 1, + TaskID: taskID, + }); err != nil { + t.Fatalf("CreateReviewPRTask(%d): %v", i+1, err) + } + } +} + +func createIssueCleanupRows(t *testing.T, store *Store, watch *IssueWatch, taskIDs ...string) { + t.Helper() + ctx := context.Background() + for i, taskID := range taskIDs { + if taskID != "" { + seedTask(t, store, taskID, false) + } + issueNumber := i + 1 + reserved, err := store.ReserveIssueWatchTask( + ctx, watch.ID, "acme", "widget", issueNumber, + fmt.Sprintf("https://github.com/acme/widget/issues/%d", issueNumber), + ) + if err != nil || !reserved { + t.Fatalf("ReserveIssueWatchTask(%d): reserved=%v err=%v", issueNumber, reserved, err) + } + if taskID != "" { + if err := store.AssignIssueWatchTaskID(ctx, watch.ID, "acme", "widget", issueNumber, taskID); err != nil { + t.Fatalf("AssignIssueWatchTaskID(%d): %v", issueNumber, err) + } + } + } +} + +func rateLimitAPIError(status int, body string) error { + return fmt.Errorf("cleanup fetch: %w", &GitHubAPIError{StatusCode: status, Body: body}) +} + +func assertCleanupRateLimitError(t *testing.T, err error) { + t.Helper() + if err == nil { + t.Fatal("expected cleanup to return a rate-limit error") + } + var apiErr *GitHubAPIError + if !errors.As(err, &apiErr) || !isGitHubRateLimitAPIError(apiErr) { + t.Fatalf("cleanup error = %v, want a classified rate-limit API error", err) + } +} + +func TestCleanupBatchStopsOnRateLimit(t *testing.T) { + t.Run("review partial count and retry", func(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{feedback: map[int]cleanupPRResponse{ + 2: {err: rateLimitAPIError(http.StatusTooManyRequests, "secondary limit")}, + }} + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "review-one", "review-two", "review-three") + recorder := &recordingTaskDeleter{} + svc.SetTaskDeleter(recorder) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + assertCleanupRateLimitError(t, err) + if deleted != 1 { + t.Fatalf("deleted = %d, want partial count 1", deleted) + } + if !reflect.DeepEqual(client.prCalls, []int{1, 2}) { + t.Fatalf("GetPR calls = %v, want [1 2]", client.prCalls) + } + + client.feedback[2] = cleanupPRResponse{} + deleted, err = svc.CleanupMergedReviewTasks(context.Background(), watch) + if err != nil || deleted != 2 { + t.Fatalf("retry deleted=%d err=%v, want 2 and nil", deleted, err) + } + }) + + t.Run("issue wrapped forbidden rate limit", func(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{issues: map[int]cleanupIssueResponse{ + 1: {err: rateLimitAPIError(http.StatusForbidden, "abuse detection mechanism triggered")}, + }} + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &IssueWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateIssueWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateIssueWatch: %v", err) + } + createIssueCleanupRows(t, store, watch, "issue-one", "issue-two") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + + deleted, err := svc.CleanupClosedIssueTasks(context.Background(), watch) + assertCleanupRateLimitError(t, err) + if deleted != 0 || !reflect.DeepEqual(client.issueCalls, []int{1}) { + t.Fatalf("deleted=%d issue calls=%v, want 0 and [1]", deleted, client.issueCalls) + } + }) + + t.Run("empty reservation is admitted and can stop the batch", func(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{feedback: map[int]cleanupPRResponse{ + 1: {err: rateLimitAPIError(http.StatusForbidden, "API rate limit exceeded")}, + }} + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "", "review-after-limit") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + assertCleanupRateLimitError(t, err) + if deleted != 0 || !reflect.DeepEqual(client.prCalls, []int{1}) { + t.Fatalf("deleted=%d GetPR calls=%v, want 0 and [1]", deleted, client.prCalls) + } + }) +} + +func TestCleanupBatchPreservesOrdinaryErrorsAndStopsOnCancellation(t *testing.T) { + t.Run("unrelated forbidden continues", func(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{feedback: map[int]cleanupPRResponse{ + 1: {err: rateLimitAPIError(http.StatusForbidden, "resource forbidden")}, + }} + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "forbidden", "healthy") + recorder := &recordingTaskDeleter{} + svc.SetTaskDeleter(recorder) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + if err != nil || deleted != 1 { + t.Fatalf("deleted=%d err=%v, want 1 and nil", deleted, err) + } + if !reflect.DeepEqual(client.prCalls, []int{1, 2}) || !reflect.DeepEqual(recorder.calls, []string{"healthy"}) { + t.Fatalf("GetPR calls=%v deletes=%v, want [1 2] and [healthy]", client.prCalls, recorder.calls) + } + }) + + t.Run("authenticated user rate limit propagates", func(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{ + feedback: map[int]cleanupPRResponse{1: { + state: "open", + reviews: []PRReview{{State: "APPROVED", Author: mockDefaultUser}}, + }}, + userErr: rateLimitAPIError(http.StatusTooManyRequests, "rate limit exceeded"), + } + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "approved") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + assertCleanupRateLimitError(t, err) + if deleted != 0 { + t.Fatalf("deleted = %d, want 0", deleted) + } + }) + + t.Run("cancellation stops before provider call", func(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{} + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "cancelled") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + deleted, err := svc.CleanupMergedReviewTasks(ctx, watch) + if !errors.Is(err, context.Canceled) || deleted != 0 || len(client.prCalls) != 0 { + t.Fatalf("deleted=%d err=%v calls=%v, want cancellation before provider call", deleted, err, client.prCalls) + } + }) +} + +func TestCleanupTracksNonRateAuthenticatedUserErrors(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{ + feedback: map[int]cleanupPRResponse{ + 1: { + state: prStateOpen, + reviews: []PRReview{{State: reviewStateApproved, Author: mockDefaultUser}}, + }, + }, + userErr: errors.New("temporary authenticated-user lookup failure"), + } + configureCleanupRateLimitClient(t, svc, client, nil) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "user-lookup-failure") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + if err != nil || deleted != 0 { + t.Fatalf("cleanup deleted=%d err=%v, want ordinary failure to be retained", deleted, err) + } + + key := reviewFailureKey(&ReviewPRTask{ + ReviewWatchID: watch.ID, + RepoOwner: "acme", + RepoName: "widget", + PRNumber: 1, + }) + svc.cleanupFailureMu.Lock() + failures := svc.cleanupFailureCounts[key] + svc.cleanupFailureMu.Unlock() + if failures != 1 { + t.Fatalf("authenticated-user failure count = %d, want 1", failures) + } +} + +func TestCleanupBatchStopsWhenWorkspaceCoreTrackerIsExhausted(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupRateLimitClient{} + tracker := NewRateTracker(nil, nil) + tracker.Record(RateSnapshot{ + Resource: ResourceCore, Remaining: 0, Limit: 5000, + ResetAt: time.Now().Add(time.Hour), UpdatedAt: time.Now().UTC(), + }) + configureCleanupRateLimitClient(t, svc, client, tracker) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "tracker-exhausted") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + assertCleanupRateLimitError(t, err) + if deleted != 0 || len(client.prCalls) != 0 { + t.Fatalf("deleted=%d GetPR calls=%v, want 0 and no provider call", deleted, client.prCalls) + } +} + +func TestCleanupBatchAdmissionReopensAfterRateLimitReset(t *testing.T) { + tracker := NewRateTracker(nil, nil) + tracker.Record(RateSnapshot{ + Resource: ResourceCore, Remaining: 0, Limit: 5000, + ResetAt: time.Now().Add(time.Hour), UpdatedAt: time.Now().UTC(), + }) + tracker.mu.Lock() + snapshot := tracker.snapshots[ResourceCore] + snapshot.ResetAt = time.Now().Add(-time.Second) + tracker.snapshots[ResourceCore] = snapshot + tracker.mu.Unlock() + + if err := cleanupBatchAdmission(context.Background(), NewMockClient(), tracker, CleanupPolicyAlways); err != nil { + t.Fatalf("cleanup admission after reset = %v, want nil", err) + } +} + +func TestCleanupBatchStopsForGHCLIWrappedGraphQLRateLimit(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + client := &cleanupGHCLIClient{ + GHClient: NewGHClient(), + err: fmt.Errorf("get PR #1: gh pr: exit status 1: GraphQL: API rate limit already exceeded for user"), + } + tracker := NewRateTracker(nil, nil) + tracker.Record(RateSnapshot{ + Resource: ResourceCore, Remaining: 0, Limit: 5000, + ResetAt: time.Now().Add(time.Hour), UpdatedAt: time.Now().UTC(), + }) + configureCleanupRateLimitClient(t, svc, client, tracker) + watch := &ReviewWatch{WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(context.Background(), watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + createReviewCleanupRows(t, store, watch, "cli-rate", "cli-after") + svc.SetTaskDeleter(&recordingTaskDeleter{}) + + deleted, err := svc.CleanupMergedReviewTasks(context.Background(), watch) + if err == nil || !cleanupBatchShouldStop(err) { + t.Fatalf("cleanup error = %v, want a rate-limit stop", err) + } + if deleted != 0 || !reflect.DeepEqual(client.calls, []int{1}) { + t.Fatalf("deleted=%d GetPR calls=%v, want 0 and [1]", deleted, client.calls) + } + + // Core exhaustion belongs to an unrelated REST bucket and must not block + // a GH CLI GraphQL cleanup before its provider call. + client.err = nil + tracker.Record(RateSnapshot{ + Resource: ResourceGraphQL, Remaining: 0, Limit: 5000, + ResetAt: time.Now().Add(time.Hour), UpdatedAt: time.Now().UTC(), + }) + deleted, err = svc.CleanupMergedReviewTasks(context.Background(), watch) + assertCleanupRateLimitError(t, err) + if deleted != 0 || !reflect.DeepEqual(client.calls, []int{1}) { + t.Fatalf("graphql admission deleted=%d GetPR calls=%v, want 0 and [1]", deleted, client.calls) + } +} diff --git a/apps/backend/internal/github/service_connections.go b/apps/backend/internal/github/service_connections.go index 5ec5e29426..81ed2240bc 100644 --- a/apps/backend/internal/github/service_connections.go +++ b/apps/backend/internal/github/service_connections.go @@ -421,7 +421,7 @@ func (s *Service) invalidateWorkspaceCredential(workspaceID string) { func (s *Service) clearAuthCaches() { for _, cache := range []*ttlCache{ s.searchCache, s.prStatusCache, s.prFeedbackCache, s.mergeMethodsCache, - s.accessibleReposCache, s.repoErrorCache, + s.accessibleReposCache, s.repoErrorCache, s.workflowRunsCache, s.workflowJobsCache, } { if cache != nil { cache.clear() diff --git a/apps/backend/internal/github/service_issues.go b/apps/backend/internal/github/service_issues.go index 91bb17b475..c4610f6286 100644 --- a/apps/backend/internal/github/service_issues.go +++ b/apps/backend/internal/github/service_issues.go @@ -210,20 +210,20 @@ func (s *Service) DeleteIssueWatch(ctx context.Context, id string) error { } } if s.taskDeleter != nil { - issueTasks, err := s.store.ListIssueWatchTasksByWatch(ctx, id) + taskIDs, err := s.store.ListIssueWatchTaskIDsByWatch(ctx, id) if err != nil { s.logger.Warn("failed to list issue tasks for pre-delete sweep", zap.String("watch_id", id), zap.Error(err)) } else { - for _, it := range issueTasks { - if it.TaskID == "" { + for _, taskID := range taskIDs { + if taskID == "" { continue } - if err := s.taskDeleter.DeleteTask(ctx, it.TaskID); err != nil && + if err := s.taskDeleter.DeleteTask(ctx, taskID); err != nil && !isTaskNotFound(err) { s.logger.Warn("failed to delete issue task during watch cleanup", zap.String("watch_id", id), - zap.String("task_id", it.TaskID), + zap.String("task_id", taskID), zap.Error(err)) } } diff --git a/apps/backend/internal/github/service_pr.go b/apps/backend/internal/github/service_pr.go index 7a570b7341..e588d5c687 100644 --- a/apps/backend/internal/github/service_pr.go +++ b/apps/backend/internal/github/service_pr.go @@ -116,6 +116,7 @@ func (s *Service) requestReviewersWithClient( key := scopedCacheKey(cacheScope, prStatusCacheKey(owner, repo, number)) s.prFeedbackCache.invalidateKey(key) s.prStatusCache.invalidateKey(key) + s.invalidateWorkflowAttentionForPR(cacheScope, owner, repo, number, "") return nil } @@ -356,6 +357,13 @@ func (s *Service) getPRFeedback( // consumed past the deadline. fetchCtx, cancelFetch := derivedFetchContext(ctx) defer cancelFetch() + fetchCtx = withWorkflowAttentionCollector(fetchCtx, func( + collectorCtx context.Context, collectorClient Client, collectorOwner, collectorRepo string, pr *PR, + ) (*WorkflowAttention, error) { + return s.collectWorkflowAttention( + collectorCtx, collectorClient, cacheScope, collectorOwner, collectorRepo, pr, + ) + }) key := scopedCacheKey(cacheScope, prStatusCacheKey(owner, repo, number)) v, err := s.prFeedbackCache.doOrFetch(key, func() (any, error) { feedback, fetchErr := client.GetPRFeedback(fetchCtx, owner, repo, number) @@ -407,6 +415,13 @@ func (s *Service) getPRStatus( // GetPRFeedback for the cascading-cancel + deadline-preserve rationale. fetchCtx, cancelFetch := derivedFetchContext(ctx) defer cancelFetch() + fetchCtx = withWorkflowAttentionCollector(fetchCtx, func( + collectorCtx context.Context, collectorClient Client, collectorOwner, collectorRepo string, pr *PR, + ) (*WorkflowAttention, error) { + return s.collectWorkflowAttention( + collectorCtx, collectorClient, cacheScope, collectorOwner, collectorRepo, pr, + ) + }) key := scopedCacheKey(cacheScope, prStatusCacheKey(owner, repo, number)) v, err := s.prStatusCache.doOrFetch(key, func() (any, error) { return client.GetPRStatus(fetchCtx, owner, repo, number) diff --git a/apps/backend/internal/github/service_pr_discovery_attempt.go b/apps/backend/internal/github/service_pr_discovery_attempt.go index ce00903ed9..dc5d42a2af 100644 --- a/apps/backend/internal/github/service_pr_discovery_attempt.go +++ b/apps/backend/internal/github/service_pr_discovery_attempt.go @@ -156,6 +156,14 @@ func prDiscoveryTargetForWatch(watch *PRWatch) prDiscoveryHealthTarget { func (s *Service) beginPRDiscoveryWatch( workspaceID, cacheScope string, credentialGeneration int64, watch *PRWatch, +) (prDiscoveryWatchAttempt, bool) { + return s.beginPRDiscoveryWatchWithOptions( + workspaceID, cacheScope, credentialGeneration, watch, false, + ) +} + +func (s *Service) beginPRDiscoveryWatchWithOptions( + workspaceID, cacheScope string, credentialGeneration int64, watch *PRWatch, explicitRefresh bool, ) (prDiscoveryWatchAttempt, bool) { target := prDiscoveryTargetForWatch(watch) attempt := prDiscoveryWatchAttempt{target: target} @@ -164,8 +172,8 @@ func (s *Service) beginPRDiscoveryWatch( return attempt, true } s.prDiscoveryAttemptsMu.Lock() - invalidatedTargets, number, admitted, joined, eventHealth := health.trackConsumerAndBegin( - workspaceID, cacheScope, credentialGeneration, prDiscoveryWatchID(watch), target, + invalidatedTargets, number, admitted, joined, eventHealth := health.trackConsumerAndBeginWithOptions( + workspaceID, cacheScope, credentialGeneration, prDiscoveryWatchID(watch), target, explicitRefresh, ) s.invalidatePRDiscoveryAttemptsLocked(workspaceID, invalidatedTargets) if s.prDiscoveryAttempts == nil { @@ -178,6 +186,10 @@ func (s *Service) beginPRDiscoveryWatch( if joined && result != nil { attempt.joined = true } else { + if explicitRefresh && result != nil { + result.complete(nil, false, true, nil, true) + delete(s.prDiscoveryAttempts, key) + } result = newPRDiscoveryAttemptResult(workspaceID, target.key()) s.prDiscoveryAttempts[key] = result } @@ -201,11 +213,11 @@ func (h *prDiscoveryHealthStore) beginWithJoin( scopeKey := prDiscoveryScopeKey(workspaceID, cacheScope) delete(h.retired, scopeKey) scope := h.scopeLocked(workspaceID, cacheScope, credentialGeneration) - return h.beginWithJoinScopeLocked(scope, target) + return h.beginWithJoinScopeLocked(scope, target, false) } func (h *prDiscoveryHealthStore) beginWithJoinScopeLocked( - scope *prDiscoveryHealthScope, target prDiscoveryHealthTarget, + scope *prDiscoveryHealthScope, target prDiscoveryHealthTarget, explicitRefresh bool, ) (uint64, bool, bool) { if scope == nil { return 0, false, false @@ -214,10 +226,10 @@ func (h *prDiscoveryHealthStore) beginWithJoinScopeLocked( state := scope.targets[key] now := h.nowLocked() if state != nil { - if state.active { + if state.active && !explicitRefresh { return state.attempt, true, true } - if state.retryAt != nil && now.Before(*state.retryAt) { + if !state.active && state.retryAt != nil && now.Before(*state.retryAt) { return 0, false, false } } @@ -236,6 +248,15 @@ func (h *prDiscoveryHealthStore) beginWithJoinScopeLocked( func (h *prDiscoveryHealthStore) trackConsumerAndBegin( workspaceID, cacheScope string, credentialGeneration int64, watchID string, target prDiscoveryHealthTarget, +) ([]string, uint64, bool, bool, *PRDiscoveryHealth) { + return h.trackConsumerAndBeginWithOptions( + workspaceID, cacheScope, credentialGeneration, watchID, target, false, + ) +} + +func (h *prDiscoveryHealthStore) trackConsumerAndBeginWithOptions( + workspaceID, cacheScope string, credentialGeneration int64, watchID string, + target prDiscoveryHealthTarget, explicitRefresh bool, ) ([]string, uint64, bool, bool, *PRDiscoveryHealth) { if h == nil || workspaceID == "" || cacheScope == "" || watchID == "" { return nil, 0, true, false, nil @@ -276,7 +297,7 @@ func (h *prDiscoveryHealthStore) trackConsumerAndBegin( if changed { scope.revision++ } - number, admitted, joined := h.beginWithJoinScopeLocked(scope, target) + number, admitted, joined := h.beginWithJoinScopeLocked(scope, target, explicitRefresh) var eventHealth *PRDiscoveryHealth if changed { snapshot := h.snapshotLocked(scope) diff --git a/apps/backend/internal/github/service_pr_sync_test.go b/apps/backend/internal/github/service_pr_sync_test.go index 2c4f271467..91a4baa9f4 100644 --- a/apps/backend/internal/github/service_pr_sync_test.go +++ b/apps/backend/internal/github/service_pr_sync_test.go @@ -137,7 +137,7 @@ func TestTriggerPRSyncAll_ThrottlesDetectionProbe(t *testing.T) { t.Fatal(err) } - if _, err := svc.TriggerPRSyncAll(ctx, "t1"); err != nil { + if _, _, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, "t1", false); err != nil { t.Fatalf("first TriggerPRSyncAll: %v", err) } if got := mockClient.FindPRByBranchCallCount(); got != 1 { @@ -145,7 +145,7 @@ func TestTriggerPRSyncAll_ThrottlesDetectionProbe(t *testing.T) { } // Second sync immediately after — well within PRSyncFreshnessWindow. - if _, err := svc.TriggerPRSyncAll(ctx, "t1"); err != nil { + if _, _, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, "t1", false); err != nil { t.Fatalf("second TriggerPRSyncAll: %v", err) } if got := mockClient.FindPRByBranchCallCount(); got != 1 { diff --git a/apps/backend/internal/github/service_pr_unwatched.go b/apps/backend/internal/github/service_pr_unwatched.go index d366c43e95..9918b060f4 100644 --- a/apps/backend/internal/github/service_pr_unwatched.go +++ b/apps/backend/internal/github/service_pr_unwatched.go @@ -23,10 +23,13 @@ import ( // // Terminal rows (merged / closed) and detached rows are excluded — their // state can no longer change, so re-fetching them would grow every batch -// without bound as a task accumulates PRs. Rows synced inside -// PRSyncFreshnessWindow are excluded too, mirroring triggerPRStatusSync's own -// freshness short-circuit so a burst of syncs costs one upstream call. -func unwatchedTaskPRs(rows []*TaskPR, watches []*PRWatch, now time.Time) []*TaskPR { +// without bound as a task accumulates PRs. Passive reads also exclude rows +// synced inside PRSyncFreshnessWindow, mirroring triggerPRStatusSync's own +// freshness short-circuit. An explicit refresh bypasses only that freshness +// check; it keeps the terminal, detached, and malformed-row guards. +func unwatchedTaskPRs( + rows []*TaskPR, watches []*PRWatch, now time.Time, explicitRefresh bool, +) []*TaskPR { watched := make(map[string]struct{}, len(watches)) for _, w := range watches { if w == nil || w.PRNumber == 0 { @@ -36,7 +39,10 @@ func unwatchedTaskPRs(rows []*TaskPR, watches []*PRWatch, now time.Time) []*Task } pending := make([]*TaskPR, 0, len(rows)) for _, tp := range rows { - if !taskPRNeedsUnwatchedSync(tp, now) { + if !taskPREligibleForUnwatchedSync(tp) { + continue + } + if !explicitRefresh && !taskPRNeedsUnwatchedSync(tp, now) { continue } if _, ok := watched[prStatusCacheKey(tp.Owner, tp.Repo, tp.PRNumber)]; ok { @@ -47,11 +53,13 @@ func unwatchedTaskPRs(rows []*TaskPR, watches []*PRWatch, now time.Time) []*Task return pending } +func taskPREligibleForUnwatchedSync(tp *TaskPR) bool { + return tp != nil && tp.TaskID != "" && tp.PRNumber != 0 && tp.DetachedAt == nil && + tp.State != prStateMerged && tp.State != prStateClosed +} + func taskPRNeedsUnwatchedSync(tp *TaskPR, now time.Time) bool { - if tp == nil || tp.TaskID == "" || tp.PRNumber == 0 || tp.DetachedAt != nil { - return false - } - if tp.State == prStateMerged || tp.State == prStateClosed { + if !taskPREligibleForUnwatchedSync(tp) { return false } return tp.LastSyncedAt == nil || now.Sub(*tp.LastSyncedAt) >= PRSyncFreshnessWindow @@ -63,15 +71,17 @@ func taskPRNeedsUnwatchedSync(tp *TaskPR, now time.Time) bool { // reconciliation paths — a dead repo must not fail the caller's sync. // // Only lifecycle and head-scoped workflow-attention fields are written (see -// reconcileTaskPRLifecycle). Check and review aggregates deliberately stay -// untouched: they belong to the active, watch-covered PR, and a row nobody -// watches only needs to learn that it reached a terminal state or that its -// current head needs provider attention. +// reconcileTaskPRLifecycle). Passive reads use the cheap PR query. An explicit +// refresh may also collect fresh Actions evidence for a row nobody watches. +// Check and review aggregates deliberately stay untouched: they belong to the +// active, watch-covered PR. // // fallbackWorkspaceID covers legacy rows written before task_prs carried // workspace ownership; rows with neither are skipped because there is no // credential to resolve. -func (s *Service) syncUnwatchedTaskPRs(ctx context.Context, pending []*TaskPR, fallbackWorkspaceID string) { +func (s *Service) syncUnwatchedTaskPRs( + ctx context.Context, pending []*TaskPR, fallbackWorkspaceID string, explicitRefresh bool, +) { byWorkspace := make(map[string][]*TaskPR, 1) for _, tp := range pending { workspaceID := tp.WorkspaceID @@ -86,18 +96,30 @@ func (s *Service) syncUnwatchedTaskPRs(ctx context.Context, pending []*TaskPR, f byWorkspace[workspaceID] = append(byWorkspace[workspaceID], tp) } for workspaceID, group := range byWorkspace { - s.syncUnwatchedTaskPRGroup(ctx, workspaceID, group) + s.syncUnwatchedTaskPRGroup(ctx, workspaceID, group, explicitRefresh) } } -func (s *Service) syncUnwatchedTaskPRGroup(ctx context.Context, workspaceID string, group []*TaskPR) { +func (s *Service) syncUnwatchedTaskPRGroup( + ctx context.Context, workspaceID string, group []*TaskPR, explicitRefresh bool, +) { resolved, err := s.resolveAutomationClient(ctx, workspaceID, "", "") if err != nil { s.logger.Debug("resolve client for unwatched task PR sync failed", zap.String("workspace_id", workspaceID), zap.Error(err)) return } - live := s.fetchUnwatchedTaskPRs(ctx, resolved, group) + if explicitRefresh { + for _, tp := range group { + if tp == nil { + continue + } + s.invalidateWorkflowAttentionForResolvedPR( + resolved, tp.Owner, tp.Repo, tp.PRNumber, tp.HeadSHA, + ) + } + } + live := s.fetchUnwatchedTaskPRs(ctx, resolved, group, explicitRefresh) seen := make(map[string]struct{}, len(group)) for _, tp := range group { if tp == nil || tp.TaskID == "" { @@ -182,7 +204,7 @@ func (s *Service) reconcileTaskPRLifecycle(ctx context.Context, tp *TaskPR, stat tp.AutoMergeObservedAt = autoMergeObservedAt tp.WorkflowAttention = nextWorkflowAttention tp.WorkflowAttentionJSON = marshalWorkflowAttention(nextWorkflowAttention) - now := time.Now().UTC() + now := s.now() tp.LastSyncedAt = &now // AC-18: publish must reflect the row as stored, not this call's @@ -202,7 +224,7 @@ func (s *Service) reconcileTaskPRLifecycle(ctx context.Context, tp *TaskPR, stat // already computed rather than the bare PR (AC-08/AC-10/AC-14 apply here // exactly as they do to the watch-driven batched query). func (s *Service) fetchUnwatchedTaskPRs( - ctx context.Context, resolved *resolvedServiceClient, group []*TaskPR, + ctx context.Context, resolved *resolvedServiceClient, group []*TaskPR, explicitRefresh bool, ) map[string]*PRStatus { refs := make([]graphQLPRRef, 0, len(group)) for _, tp := range group { @@ -217,13 +239,13 @@ func (s *Service) fetchUnwatchedTaskPRs( return nil } if exec, execErr := graphQLExecutorFor(resolved.Client); execErr == nil { - out, err := s.batchedUnwatchedFetch(ctx, exec, resolved.CacheScope, refs) + out, err := s.batchedUnwatchedFetch(ctx, exec, resolved.Client, resolved.CacheScope, refs, explicitRefresh) if err == nil { return out } s.logger.Debug("batched unwatched task PR query failed; falling back per PR", zap.Error(err)) } - return s.fetchUnwatchedTaskPRsPerPR(ctx, resolved, refs) + return s.fetchUnwatchedTaskPRsPerPR(ctx, resolved, refs, explicitRefresh) } // batchedUnwatchedFetch runs the batched query under the same service-level @@ -237,9 +259,13 @@ func (s *Service) fetchUnwatchedTaskPRs( // derivedFetchContext so one caller disconnecting mid-flight doesn't cascade // context.Canceled to its co-waiters, while keeping the leader's deadline. func (s *Service) batchedUnwatchedFetch( - ctx context.Context, exec GraphQLExecutor, cacheScope string, refs []graphQLPRRef, + ctx context.Context, exec GraphQLExecutor, client Client, cacheScope string, + refs []graphQLPRRef, explicitRefresh bool, ) (map[string]*PRStatus, error) { key := scopedCacheKey(cacheScope, "unwatched:"+batchedRefsKey(refs)) + if explicitRefresh { + key += prSyncExplicitRefreshKeySuffix + } fetchCtx, cancelFetch := derivedFetchContext(ctx) defer cancelFetch() v, err, _ := s.syncGroup.Do(key, func() (interface{}, error) { @@ -248,6 +274,9 @@ func (s *Service) batchedUnwatchedFetch( repoErrGen := s.repoErrorGenSnapshot() out, queryErr := runBatchedPRQuery(fetchCtx, exec, refs) out, queryErr = s.absorbMissingReposErr(out, queryErr, cacheScope, repoErrGen) + if queryErr == nil && explicitRefresh { + s.enrichBatchedWorkflowAttention(fetchCtx, client, cacheScope, out) + } return out, queryErr }) if err != nil { @@ -273,29 +302,50 @@ func batchedRefsKey(refs []graphQLPRRef) string { return strings.Join(parts, "|") } -// fetchUnwatchedTaskPRsPerPR reads one PR at a time. GetPR is deliberate: the -// full status helper also lists reviews and check runs (three calls per PR) to -// compute aggregates this path does not write. GetPR is still a full -// single-pull-request fetch, so is_draft/changed_files/merged_by_login are -// real observations here too (AC-10); ClosureAttributionPopulated stays -// false since neither REST nor the gh CLI can see the closing actor (AC-15). +// fetchUnwatchedTaskPRsPerPR reads one PR at a time. Passive reads use GetPR: +// the full status helper also lists reviews and check runs (three calls per PR) +// to compute aggregates this path does not write. Explicit reads use the full +// status helper so they can refresh Actions attention as part of a user +// refresh. GetPR is still a full single-pull-request fetch, so +// is_draft/changed_files/merged_by_login are real observations here too +// (AC-10); ClosureAttributionPopulated stays false since neither REST nor the +// gh CLI can see the closing actor (AC-15). func (s *Service) fetchUnwatchedTaskPRsPerPR( - ctx context.Context, resolved *resolvedServiceClient, refs []graphQLPRRef, + ctx context.Context, resolved *resolvedServiceClient, refs []graphQLPRRef, explicitRefresh bool, ) map[string]*PRStatus { out := make(map[string]*PRStatus, len(refs)) + statusCtx := ctx + if explicitRefresh { + statusCtx = withWorkflowAttentionCollector(ctx, func( + collectorCtx context.Context, collectorClient Client, owner, repo string, pr *PR, + ) (*WorkflowAttention, error) { + return s.collectWorkflowAttention( + collectorCtx, collectorClient, resolved.CacheScope, owner, repo, pr, + ) + }) + } for _, ref := range refs { - pr, err := resolved.Client.GetPR(ctx, ref.Owner, ref.Repo, ref.Number) + var ( + status *PRStatus + err error + ) + if explicitRefresh { + status, err = resolved.Client.GetPRStatus(statusCtx, ref.Owner, ref.Repo, ref.Number) + } else { + var pr *PR + pr, err = resolved.Client.GetPR(ctx, ref.Owner, ref.Repo, ref.Number) + if pr != nil { + status = &PRStatus{PR: pr, OutcomeFieldsPopulated: true} + } + } if err != nil { s.logger.Debug("per-PR unwatched task PR read failed", zap.String("owner", ref.Owner), zap.String("repo", ref.Repo), zap.Int("pr_number", ref.Number), zap.Error(err)) continue } - if pr != nil { - out[prStatusCacheKey(ref.Owner, ref.Repo, ref.Number)] = &PRStatus{ - PR: pr, - OutcomeFieldsPopulated: true, - } + if status != nil && status.PR != nil { + out[prStatusCacheKey(ref.Owner, ref.Repo, ref.Number)] = status } } return out @@ -304,15 +354,15 @@ func (s *Service) fetchUnwatchedTaskPRsPerPR( // reconcileTaskUnwatchedPRs refreshes the task's unwatched rows and returns // the reloaded row set, so the WS caller hands the frontend the merged state // rather than the pre-sync snapshot. Returns the rows unchanged when nothing -// needed reconciling. +// needs reconciling. func (s *Service) reconcileTaskUnwatchedPRs( - ctx context.Context, taskID string, watches []*PRWatch, + ctx context.Context, taskID string, watches []*PRWatch, explicitRefresh bool, ) ([]*TaskPR, error) { rows, err := s.store.ListTaskPRsByTask(ctx, taskID) if err != nil { return nil, fmt.Errorf("list task PRs: %w", err) } - pending := unwatchedTaskPRs(rows, watches, time.Now().UTC()) + pending := unwatchedTaskPRs(rows, watches, time.Now().UTC(), explicitRefresh) if len(pending) == 0 { return rows, nil } @@ -320,7 +370,7 @@ func (s *Service) reconcileTaskUnwatchedPRs( if len(watches) > 0 && watches[0] != nil { fallbackWorkspaceID = watches[0].WorkspaceID } - s.syncUnwatchedTaskPRs(ctx, pending, fallbackWorkspaceID) + s.syncUnwatchedTaskPRs(ctx, pending, fallbackWorkspaceID, explicitRefresh) return s.store.ListTaskPRsByTask(ctx, taskID) } @@ -328,11 +378,26 @@ func (s *Service) reconcileTaskUnwatchedPRs( // non-terminal PR rows for every stale task in one pass, so the background // workspace refresh can fan both into a single batched call each. func (s *Service) collectStaleWorkspaceSyncTargets( - ctx context.Context, staleTasks map[string]struct{}, + ctx context.Context, workspaceID string, staleTasks map[string]struct{}, ) ([]*PRWatch, []*TaskPR) { var allWatches []*PRWatch var pending []*TaskPR - now := time.Now().UTC() + now := s.now() + seenWatches := make(map[string]struct{}) + appendWatch := func(watch *PRWatch) { + if watch == nil { + return + } + key := watch.ID + if key == "" { + key = watch.WorkspaceID + "\x00" + watch.TaskID + "\x00" + watch.RepositoryID + "\x00" + watch.Owner + "\x00" + watch.Repo + "\x00" + watch.Branch + "\x00" + fmt.Sprint(watch.PRNumber) + } + if _, ok := seenWatches[key]; ok { + return + } + seenWatches[key] = struct{}{} + allWatches = append(allWatches, watch) + } for taskID := range staleTasks { watches, err := s.store.ListPRWatchesByTask(ctx, taskID) if err != nil { @@ -340,14 +405,30 @@ func (s *Service) collectStaleWorkspaceSyncTargets( zap.String("task_id", taskID), zap.Error(err)) continue } - allWatches = append(allWatches, watches...) + for _, watch := range watches { + appendWatch(watch) + } rows, err := s.store.ListTaskPRsByTask(ctx, taskID) if err != nil { s.logger.Debug("list task PRs for refresh failed", zap.String("task_id", taskID), zap.Error(err)) continue } - pending = append(pending, unwatchedTaskPRs(rows, watches, now)...) + pending = append(pending, unwatchedTaskPRs(rows, watches, now, false)...) + } + // Searching watches can be absent from the task-PR projection or can have + // a fresh cached row while still needing their adaptive discovery check. + // Load them from the active workspace inventory so passive page refreshes + // cannot bypass the schedule through the PR freshness window. + activeWatches, err := s.store.ListActivePRWatchesForWorkspace(ctx, workspaceID) + if err != nil { + s.logger.Debug("list active PR watches for adaptive refresh failed", zap.Error(err)) + } else { + for _, watch := range activeWatches { + if watch != nil && watch.PRNumber == 0 { + appendWatch(watch) + } + } } return allWatches, pending } diff --git a/apps/backend/internal/github/service_pr_unwatched_test.go b/apps/backend/internal/github/service_pr_unwatched_test.go index c49fe9a56d..5f62a5f1ee 100644 --- a/apps/backend/internal/github/service_pr_unwatched_test.go +++ b/apps/backend/internal/github/service_pr_unwatched_test.go @@ -137,6 +137,105 @@ func TestTriggerPRSyncAll_RefreshesTaskPRWithNoWatchAtAll(t *testing.T) { } } +func TestTriggerPRSyncAllExplicitRefreshesUnwatchedWorkflowAttention(t *testing.T) { + _, svc, mockClient, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-explicit-unwatched", false) + seedUnwatchedTaskPR(t, store, "task-explicit-unwatched", 143, "feature/approval") + + mockClient.AddPR(&PR{ + Number: 143, + Title: "Workflow approval attention", + State: prStateOpen, + RepoOwner: "owner", + RepoName: "repo", + HeadSHA: "workflow-head-143", + HeadBranch: "feature/approval", + HeadRepoOwner: "contributor", + HeadRepoName: "repo-fork", + BaseBranch: "main", + }) + mockClient.ReplaceWorkflowRuns("owner", "repo", "workflow-head-143", []WorkflowRun{{ + ID: 34494307522, RunAttempt: 1, WorkflowID: 93444456632, + Name: "Run tests", Event: workflowEventPullRequest, Status: workflowStatusCompleted, + Conclusion: workflowConclusionActionRequired, HeadSHA: "workflow-head-143", + HeadBranch: "feature/approval", HeadRepoOwner: "contributor", HeadRepoName: "repo-fork", + PullRequests: []WorkflowRunPullRequest{{ + Number: 143, HeadSHA: "workflow-head-143", HeadBranch: "feature/approval", + HeadRepoOwner: "contributor", HeadRepoName: "repo-fork", + }}, + }}) + + if _, _, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, "task-explicit-unwatched", true); err != nil { + t.Fatalf("initial explicit refresh: %v", err) + } + got, err := store.GetTaskPR(ctx, "task-explicit-unwatched") + if err != nil || got == nil || got.WorkflowAttention == nil { + t.Fatalf("load initial workflow attention: err=%v row=%+v", err, got) + } + if got.WorkflowAttention.State != WorkflowAttentionApprovalRequired { + t.Fatalf("initial workflow attention = %q, want approval_required", got.WorkflowAttention.State) + } + + mockClient.ReplaceWorkflowRuns("owner", "repo", "workflow-head-143", nil) + if _, _, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, "task-explicit-unwatched", true); err != nil { + t.Fatalf("follow-up explicit refresh: %v", err) + } + got, err = store.GetTaskPR(ctx, "task-explicit-unwatched") + if err != nil || got == nil || got.WorkflowAttention == nil { + t.Fatalf("load refreshed workflow attention: err=%v row=%+v", err, got) + } + if got.WorkflowAttention.State != WorkflowAttentionNone { + t.Fatalf("refreshed workflow attention = %q, want none", got.WorkflowAttention.State) + } +} + +func TestTriggerPRSyncAllExplicitRefreshesUnwatchedWorkflowAttentionBatch(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-explicit-unwatched-batch", false) + seedUnwatchedTaskPR(t, store, "task-explicit-unwatched-batch", 144, "feature/approval") + + const headSHA = "workflow-head-144" + gh.prResponses = []string{ + batchedOpenPRResponse("feature/approval", headSHA), + batchedOpenPRResponse("feature/approval", headSHA), + } + gh.ReplaceWorkflowRuns("owner", "repo", headSHA, []WorkflowRun{{ + ID: 34494307523, RunAttempt: 1, WorkflowID: 93444456633, + Name: "Run tests", Event: workflowEventPullRequest, Status: workflowStatusCompleted, + Conclusion: workflowConclusionActionRequired, HeadSHA: headSHA, + HeadBranch: "feature/approval", HeadRepoOwner: "contributor", HeadRepoName: "repo-fork", + PullRequests: []WorkflowRunPullRequest{{ + Number: 144, HeadSHA: headSHA, HeadBranch: "feature/approval", + HeadRepoOwner: "contributor", HeadRepoName: "repo-fork", + }}, + }}) + + if _, _, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, "task-explicit-unwatched-batch", true); err != nil { + t.Fatalf("initial explicit batch refresh: %v", err) + } + got, err := store.GetTaskPR(ctx, "task-explicit-unwatched-batch") + if err != nil || got == nil || got.WorkflowAttention == nil { + t.Fatalf("load initial batch workflow attention: err=%v row=%+v", err, got) + } + if got.WorkflowAttention.State != WorkflowAttentionApprovalRequired { + t.Fatalf("initial batch workflow attention = %q, want approval_required", got.WorkflowAttention.State) + } + + gh.ReplaceWorkflowRuns("owner", "repo", headSHA, nil) + if _, _, err := svc.TriggerPRSyncAllPermanentWithOptions(ctx, "task-explicit-unwatched-batch", true); err != nil { + t.Fatalf("follow-up explicit batch refresh: %v", err) + } + got, err = store.GetTaskPR(ctx, "task-explicit-unwatched-batch") + if err != nil || got == nil || got.WorkflowAttention == nil { + t.Fatalf("load refreshed batch workflow attention: err=%v row=%+v", err, got) + } + if got.WorkflowAttention.State != WorkflowAttentionNone { + t.Fatalf("refreshed batch workflow attention = %q, want none", got.WorkflowAttention.State) + } +} + // TestRefreshStaleWorkspaceWatches_HealsUnwatchedRow covers the surface the // kanban board and the topbar aggregate read from: the workspace-wide // background refresh (driven by ListWorkspaceTaskPRs) fans in watches only, @@ -302,6 +401,20 @@ func batchedMergedPRResponse(headBranch, mergedAt string) string { }}}}` } +func batchedOpenPRResponse(headBranch, headSHA string) string { + return `{"data":{"repo0":{"pr0":{ + "state": "OPEN", "title": "PR", "url": "https://x/1", + "isDraft": false, "mergeable": "UNKNOWN", "mergeStateStatus": "UNKNOWN", + "headRefName": "` + headBranch + `", "baseRefName": "main", "headRefOid": "` + headSHA + `", + "headRepository": {"id": "1", "name": "repo-fork", "nameWithOwner": "contributor/repo-fork", "url": "https://github.com/contributor/repo-fork"}, + "headRepositoryOwner": {"login": "contributor"}, + "author": {"login": "alice"}, "mergedAt": null, "closedAt": null, + "reviews": {"nodes": []}, "reviewRequests": {"totalCount": 0}, + "reviewThreads": {"totalCount": 0, "nodes": [], "pageInfo": {"hasNextPage": false, "endCursor": null}}, + "commits": {"nodes": [{"commit": {"statusCheckRollup": {"state": "SUCCESS"}}}]} + }}}}` +} + // subscribeTaskPRUpdated returns a channel fed by github.task_pr.updated // events. Channel synchronization rather than a polling sleep: the background // refresh does no subprocess work, so its write is observable through the @@ -490,7 +603,7 @@ func TestUnwatchedTaskPRs_Selection(t *testing.T) { nil, } - got := unwatchedTaskPRs(rows, watches, now) + got := unwatchedTaskPRs(rows, watches, now, false) if len(got) != 2 { t.Fatalf("expected 2 selected rows, got %d (%+v)", len(got), got) } diff --git a/apps/backend/internal/github/service_pr_watch.go b/apps/backend/internal/github/service_pr_watch.go index 9cc785482c..b82d890608 100644 --- a/apps/backend/internal/github/service_pr_watch.go +++ b/apps/backend/internal/github/service_pr_watch.go @@ -1,9 +1,12 @@ +//revive:disable:file-length-limit // PR watch orchestration shares one provider-state boundary. + package github import ( "context" "errors" "fmt" + "sort" "strconv" "strings" "sync" @@ -74,6 +77,7 @@ func (s *Service) createPRWatch( if err := s.store.CreatePRWatch(ctx, w); err != nil { return nil, fmt.Errorf("create PR watch: %w", err) } + s.resetPassiveWorkspaceRefreshAdmission(workspaceID) s.logger.Info("created PR watch", zap.String("session_id", sessionID), zap.String("repository_id", repositoryID), @@ -178,6 +182,9 @@ func (s *Service) UpdatePRWatchBranchIfSearching(ctx context.Context, id, branch if err := s.store.UpdatePRWatchBranchIfSearching(ctx, id, branch); err != nil { return err } + if watch != nil { + s.resetPassiveWorkspaceRefreshAdmission(watch.WorkspaceID) + } s.removePRDiscoveryWatchConsumer(watch) return nil } @@ -219,6 +226,9 @@ func (s *Service) ResetPRWatch(ctx context.Context, id, branch string) error { if err := s.store.ResetPRWatch(ctx, id, branch); err != nil { return err } + if watch != nil { + s.resetPassiveWorkspaceRefreshAdmission(watch.WorkspaceID) + } s.removePRDiscoveryWatchConsumer(watch) return nil } @@ -228,7 +238,7 @@ func (s *Service) CheckPRWatch(ctx context.Context, watch *PRWatch) (*PRStatus, if s.client == nil { return nil, false, fmt.Errorf("github client not available") } - return s.checkPRWatchWithClient(ctx, s.client, watch) + return s.checkPRWatchWithClient(ctx, s.client, "legacy", watch) } func (s *Service) CheckPRWatchForWorkspace(ctx context.Context, watch *PRWatch) (*PRStatus, bool, error) { @@ -239,31 +249,44 @@ func (s *Service) CheckPRWatchForWorkspace(ctx context.Context, watch *PRWatch) if err != nil { return nil, false, err } - return s.checkPRWatchWithClient(ctx, resolved.Client, watch) + return s.checkPRWatchWithClient(ctx, resolved.Client, resolved.CacheScope, watch) } func (s *Service) checkPRWatchWithClient( - ctx context.Context, client Client, watch *PRWatch, + ctx context.Context, client Client, cacheScope string, watch *PRWatch, ) (*PRStatus, bool, error) { + ctx = withWorkflowAttentionCollector(ctx, func( + collectorCtx context.Context, collectorClient Client, owner, repo string, pr *PR, + ) (*WorkflowAttention, error) { + return s.collectWorkflowAttention(collectorCtx, collectorClient, cacheScope, owner, repo, pr) + }) status, err := client.GetPRStatus(ctx, watch.Owner, watch.Repo, watch.PRNumber) if err != nil { return nil, false, err } - // Check for check status or review state changes - hasNew := status.ChecksState != watch.LastCheckStatus || status.ReviewState != watch.LastReviewState - commentAt := prWatchFeedbackWatermark(watch, status) - hasNew = hasNew || prWatchFeedbackUpdatedSinceWatch(watch, status) - - // Update watch timestamps - now := time.Now().UTC() - if err := s.store.UpdatePRWatchTimestamps(ctx, watch.ID, now, commentAt, status.ChecksState, status.ReviewState); err != nil { - s.logSyncError("failed to update PR watch timestamps", err, zap.String("id", watch.ID)) + hasNew, updateErr := s.applyPRWatchStatus(ctx, watch, status) + if updateErr != nil { + s.logSyncError("failed to update PR watch timestamps", updateErr, zap.String("id", watch.ID)) } return status, hasNew, nil } +func (s *Service) applyPRWatchStatus( + ctx context.Context, watch *PRWatch, status *PRStatus, +) (bool, error) { + if watch == nil || status == nil { + return false, nil + } + hasNew := status.ChecksState != watch.LastCheckStatus || status.ReviewState != watch.LastReviewState + commentAt := prWatchFeedbackWatermark(watch, status) + hasNew = hasNew || prWatchFeedbackUpdatedSinceWatch(watch, status) + now := time.Now().UTC() + err := s.store.UpdatePRWatchTimestamps(ctx, watch.ID, now, commentAt, status.ChecksState, status.ReviewState) + return hasNew, err +} + // logSyncError logs a PR-watch synchronization failure at ERROR, except when // the error is a context cancellation. That happens when the poll/sync cycle is // interrupted by backend shutdown, which is expected teardown rather than a @@ -358,6 +381,7 @@ func (s *Service) ensurePRWatch( if err := s.store.CreatePRWatch(ctx, w); err != nil { return nil, fmt.Errorf("ensure PR watch: %w", err) } + s.resetPassiveWorkspaceRefreshAdmission(workspaceID) s.logger.Info("created PR watch for task (will search for PR)", zap.String("session_id", sessionID), zap.String("task_id", taskID), @@ -877,18 +901,57 @@ func (s *Service) ListWorkspaceTaskPRs(ctx context.Context, workspaceID string) staleTasks := make(map[string]struct{}) for taskID, prs := range result { for _, tp := range prs { - if tp.LastSyncedAt == nil || time.Since(*tp.LastSyncedAt) >= PRSyncFreshnessWindow { + if tp.LastSyncedAt == nil || s.now().Sub(*tp.LastSyncedAt) >= PRSyncFreshnessWindow { staleTasks[taskID] = struct{}{} break } } } - if len(staleTasks) > 0 { + refreshNeeded := len(staleTasks) > 0 + if refreshNeeded || s.admitPassiveWorkspaceRefresh(workspaceID, s.now()) { + if activeWatches, watchErr := s.store.ListActivePRWatchesForWorkspace(ctx, workspaceID); watchErr != nil { + s.logger.Debug("list active PR watches for passive refresh failed", zap.Error(watchErr)) + } else { + for _, watch := range activeWatches { + if watch != nil && watch.PRNumber == 0 { + refreshNeeded = true + break + } + } + } + } + if refreshNeeded { s.refreshStaleWorkspaceWatches(workspaceID, staleTasks) } return result, nil } +func (s *Service) admitPassiveWorkspaceRefresh(workspaceID string, now time.Time) bool { + if s == nil || strings.TrimSpace(workspaceID) == "" { + return false + } + s.passiveWorkspaceRefreshMu.Lock() + defer s.passiveWorkspaceRefreshMu.Unlock() + if last, ok := s.passiveWorkspaceRefreshAt[workspaceID]; ok && !now.Before(last) && + now.Sub(last) < searchFastPollInterval { + return false + } + if s.passiveWorkspaceRefreshAt == nil { + s.passiveWorkspaceRefreshAt = make(map[string]time.Time) + } + s.passiveWorkspaceRefreshAt[workspaceID] = now + return true +} + +func (s *Service) resetPassiveWorkspaceRefreshAdmission(workspaceID string) { + if s == nil || strings.TrimSpace(workspaceID) == "" { + return + } + s.passiveWorkspaceRefreshMu.Lock() + delete(s.passiveWorkspaceRefreshAt, workspaceID) + s.passiveWorkspaceRefreshMu.Unlock() +} + // refreshStaleWorkspaceWatches fans in all stale watches across the stale // tasks and runs ONE batched GraphQL sync, so a 40-watch workspace fires // ~2 gh subprocess calls instead of 40 (one per_task * five concurrent). @@ -916,24 +979,34 @@ func (s *Service) refreshStaleWorkspaceWatches(workspaceID string, staleTasks ma defer s.inflightWorkspaceRefreshes.Delete(workspaceID) syncCtx, cancel := context.WithTimeout(s.stopCtx, 60*time.Second) defer cancel() - allWatches, unwatched := s.collectStaleWorkspaceSyncTargets(syncCtx, staleTasks) + allWatches, unwatched := s.collectStaleWorkspaceSyncTargets(syncCtx, workspaceID, staleTasks) + allWatches = selectDuePRWatches(syncCtx, allWatches, s.taskActivityProvider, s.now(), s.logger) // PRs left behind by a branch handover have no watch to fan in, so // they would stay stale on every workspace load. Reconcile them in // one batched call for the whole workspace; once a row reaches a // terminal state it drops out of this set permanently. if len(unwatched) > 0 { - s.syncUnwatchedTaskPRs(syncCtx, unwatched, workspaceID) + s.syncUnwatchedTaskPRs(syncCtx, unwatched, workspaceID, false) } if len(allWatches) == 0 { return } if _, err := s.SyncWorkspaceWatchesBatched(syncCtx, workspaceID, allWatches); err != nil { // Batched fetch failed (noop client, auth blip, GraphQL error). - // Fall back to per-task sync with bounded concurrency so we - // don't spawn one gh per watch in lockstep. + // Classify the failure before fallback. Authentication and rate + // failures are workspace-wide, so another provider call cannot + // improve the result and would consume the same exhausted budget. s.logger.Debug("batched workspace PR sync failed; falling back per-task", zap.Int("watches", len(allWatches)), zap.Error(err)) - s.refreshStaleTasksPerTask(syncCtx, staleTasks) + if !prWatchBatchFallbackAllowed(err) { + s.logger.Debug("skipping workspace PR fallback after provider admission failure", + zap.String("workspace_id", workspaceID), zap.Error(err)) + return + } + if fallbackErr := s.refreshWatchesPerWatch(syncCtx, workspaceID, allWatches); fallbackErr != nil { + s.logger.Debug("workspace PR fallback stopped after provider admission failure", + zap.String("workspace_id", workspaceID), zap.Error(fallbackErr)) + } } }() } @@ -963,7 +1036,7 @@ func (s *Service) refreshStaleTasksPerTask(ctx context.Context, staleTasks map[s go func(id string) { defer wg.Done() defer func() { <-sem }() - if _, syncErr := s.TriggerPRSyncAll(ctx, id); syncErr != nil { + if _, _, syncErr := s.TriggerPRSyncAllPermanentWithOptions(ctx, id, false); syncErr != nil { s.logger.Debug("background PR sync failed", zap.String("task_id", id), zap.Error(syncErr)) } @@ -972,6 +1045,238 @@ func (s *Service) refreshStaleTasksPerTask(ctx context.Context, staleTasks map[s wg.Wait() } +// refreshWatchesPerWatch is the passive fallback when a workspace batch is +// unavailable. The caller has already applied adaptive due admission, so this +// path checks only the selected watches and cannot turn a slow searching watch +// into an immediate provider call through TriggerPRSyncAll. +func (s *Service) refreshWatchesPerWatch(ctx context.Context, workspaceID string, watches []*PRWatch) error { + selected := s.selectPassiveFallbackTargets(workspaceID, watches) + groups := groupPRWatchFallbackTargets(watches) + // The target budget is deliberately admitted before this loop. Running the + // small set serially makes the stop rule precise: a rate/auth failure never + // has more than the current provider call in flight when the remaining + // targets are abandoned. + for _, watch := range selected { + if err := ctx.Err(); err != nil { + return err + } + err := s.refreshPassiveFallbackTarget(ctx, groups[prWatchFallbackTargetKey(watch)]) + if err == nil { + continue + } + s.logger.Debug("background PR watch fallback failed", + zap.String("watch_id", watch.ID), zap.Error(err)) + if prWatchFallbackShouldStopWorkspace(err) { + return err + } + } + return nil +} + +func (s *Service) refreshPassiveFallbackTarget(ctx context.Context, watches []*PRWatch) error { + if len(watches) == 0 || watches[0] == nil { + return nil + } + representative := watches[0] + var snapshot *TaskPR + var err error + if representative.PRNumber == 0 { + snapshot, err = s.triggerPRDetection(ctx, representative, representative.TaskID) + } else { + snapshot, err = s.triggerPRStatusSync(ctx, representative, representative.TaskID) + } + if err != nil { + return err + } + for _, watch := range watches[1:] { + if err := s.applyPassiveFallbackSnapshot(ctx, watch, snapshot); err != nil { + s.logger.Debug("background PR watch fallback apply failed", + zap.String("watch_id", watch.ID), zap.Error(err)) + } + } + return nil +} + +func (s *Service) applyPassiveFallbackSnapshot(ctx context.Context, watch *PRWatch, snapshot *TaskPR) error { + if watch == nil { + return nil + } + if snapshot == nil { + return s.store.UpdatePRWatchTimestamps(ctx, watch.ID, time.Now().UTC(), nil, "", "") + } + status := prStatusFromTaskPRSnapshot(snapshot) + effectiveTaskID := s.reconcileTaskPROwnership(ctx, watch.SessionID, watch.TaskID, watch.RepositoryID, watch.PRNumber) + if watch.PRNumber == 0 { + if err := s.rebindPRWatchRepository(ctx, watch, status.PR); err != nil { + return err + } + watch.PRNumber = status.PR.Number + if err := s.store.UpdatePRWatchPRNumber(ctx, watch.ID, status.PR.Number); err != nil { + return err + } + } + existing, err := s.loadTaskPRForWatch(ctx, effectiveTaskID, watch) + if err != nil { + return err + } + if existing == nil { + if _, err := s.associatePRWithTaskForSession( + ctx, watch.WorkspaceID, watch.SessionID, watch.TaskID, watch.RepositoryID, status.PR, + false, false, TaskPRSourceWatch, + ); err != nil { + return err + } + } + if _, err := s.applyPRWatchStatus(ctx, watch, status); err != nil { + return err + } + if err := s.SyncTaskPR(ctx, effectiveTaskID, status); err != nil { + return err + } + if status.PR.State == prStateMerged || status.PR.State == prStateClosed { + hold, err := s.ShouldHoldTerminalPRWatch(ctx, effectiveTaskID, watch.RepositoryID, watch.PRNumber, status.PR.State) + if err != nil { + return err + } + if !hold { + return s.store.UpdatePRWatchPRNumber(ctx, watch.ID, 0) + } + } + return nil +} + +func prStatusFromTaskPRSnapshot(snapshot *TaskPR) *PRStatus { + pr := &PR{ + Number: snapshot.PRNumber, + Title: snapshot.PRTitle, + URL: snapshot.PRURL, + State: snapshot.State, + HeadBranch: snapshot.HeadBranch, + HeadSHA: snapshot.HeadSHA, + BaseBranch: snapshot.BaseBranch, + AuthorLogin: snapshot.AuthorLogin, + RepoOwner: snapshot.Owner, + RepoName: snapshot.Repo, + MergeableState: snapshot.MergeableState, + MergeQueueState: snapshot.MergeQueueState, + MergeQueuePosition: snapshot.MergeQueuePosition, + MergeQueueEntryID: snapshot.MergeQueueEntryID, + MergeQueueEntryHeadSHA: snapshot.MergeQueueEntryHeadSHA, + MergeQueueEstimatedTimeToMergeSeconds: snapshot.MergeQueueEstimatedTimeToMergeSeconds, + MergeQueueLastRemovalID: snapshot.MergeQueueLastRemovalID, + MergeQueueLastRemovedAt: snapshot.MergeQueueLastRemovedAt, + MergeQueueLastRemovalReason: snapshot.MergeQueueLastRemovalReason, + MergeQueueLastRemovalBeforeSHA: snapshot.MergeQueueLastRemovalBeforeSHA, + Additions: snapshot.Additions, + Deletions: snapshot.Deletions, + CreatedAt: snapshot.CreatedAt, + MergedAt: snapshot.MergedAt, + ClosedAt: snapshot.ClosedAt, + } + if snapshot.IsDraft != nil { + pr.Draft = *snapshot.IsDraft + pr.IsDraftObserved = true + } + return &PRStatus{ + PR: pr, + WorkflowAttention: snapshot.WorkflowAttention, + ReviewState: snapshot.ReviewState, + ChecksState: snapshot.ChecksState, + MergeableState: snapshot.MergeableState, + MergeQueueState: snapshot.MergeQueueState, + MergeQueuePosition: snapshot.MergeQueuePosition, + MergeQueueEntryID: snapshot.MergeQueueEntryID, + MergeQueueEntryHeadSHA: snapshot.MergeQueueEntryHeadSHA, + MergeQueueEstimatedTimeToMergeSeconds: snapshot.MergeQueueEstimatedTimeToMergeSeconds, + MergeQueueLastRemovalID: snapshot.MergeQueueLastRemovalID, + MergeQueueLastRemovedAt: snapshot.MergeQueueLastRemovedAt, + MergeQueueLastRemovalReason: snapshot.MergeQueueLastRemovalReason, + MergeQueueLastRemovalBeforeSHA: snapshot.MergeQueueLastRemovalBeforeSHA, + ReviewCount: snapshot.ReviewCount, + PendingReviewCount: snapshot.PendingReviewCount, + RequiredReviews: snapshot.RequiredReviews, + ChecksTotal: snapshot.ChecksTotal, + ChecksPassing: snapshot.ChecksPassing, + ChecksPopulated: true, + ReviewCountsPopulated: true, + UnresolvedReviewThreads: snapshot.UnresolvedReviewThreads, + UnresolvedReviewThreadsPopulated: true, + OutcomeFieldsPopulated: snapshot.IsDraft != nil || snapshot.ChangedFiles != nil || snapshot.MergedByLogin != nil, + WorkflowAttentionPopulated: snapshot.WorkflowAttention != nil, + } +} + +type passiveFallbackAdmission struct { + at time.Time + workspaceID string +} + +func (s *Service) selectPassiveFallbackTargets(workspaceID string, watches []*PRWatch) []*PRWatch { + if len(watches) == 0 || workspaceID == "" { + return nil + } + byTarget := make(map[string]*PRWatch, len(watches)) + for _, watch := range watches { + if watch != nil { + key := prWatchFallbackTargetKey(watch) + if _, exists := byTarget[key]; !exists { + byTarget[key] = watch + } + } + } + ordered := make([]*PRWatch, 0, len(byTarget)) + for _, watch := range byTarget { + ordered = append(ordered, watch) + } + if len(ordered) == 0 { + return nil + } + sort.SliceStable(ordered, func(i, j int) bool { + return prWatchFallbackTargetKey(ordered[i]) < prWatchFallbackTargetKey(ordered[j]) + }) + + now := s.now() + s.passiveFallbackMu.Lock() + defer s.passiveFallbackMu.Unlock() + if s.passiveFallbackTargetCursors == nil { + s.passiveFallbackTargetCursors = make(map[string]int) + } + cutoff := now.Add(-time.Minute) + retained := s.passiveFallbackAdmissions[:0] + workspaceUsed := 0 + for _, admission := range s.passiveFallbackAdmissions { + if !admission.at.After(cutoff) { + continue + } + retained = append(retained, admission) + if admission.workspaceID == workspaceID { + workspaceUsed++ + } + } + s.passiveFallbackAdmissions = retained + remaining := minInt( + prWatchFallbackWorkspaceBudget-workspaceUsed, + prWatchFallbackCycleBudget-len(s.passiveFallbackAdmissions), + ) + if remaining <= 0 { + return nil + } + if remaining > len(ordered) { + remaining = len(ordered) + } + cursor := s.passiveFallbackTargetCursors[workspaceID] % len(ordered) + selected := make([]*PRWatch, 0, remaining) + for offset := 0; offset < remaining; offset++ { + selected = append(selected, ordered[(cursor+offset)%len(ordered)]) + s.passiveFallbackAdmissions = append(s.passiveFallbackAdmissions, passiveFallbackAdmission{ + at: now, + workspaceID: workspaceID, + }) + } + s.passiveFallbackTargetCursors[workspaceID] = (cursor + remaining) % len(ordered) + return selected +} + // findTaskPRForStatus locates the TaskPR row matching the (task, owner, repo, // pr_number) tuple from a poll result. Multi-repo tasks can have multiple // rows for the same task — narrowing by (owner, repo, pr_number) ensures the @@ -1451,7 +1756,7 @@ func (s *Service) TriggerPRSync(ctx context.Context, taskID string) (*TaskPR, er return s.triggerPRDetection(ctx, watch, taskID) } - return s.triggerPRStatusSync(ctx, watch, taskID) + return s.triggerPRStatusSyncWithOptions(ctx, watch, taskID, true) } // TriggerPRSyncAll performs an immediate PR status sync for every PR watch @@ -1467,7 +1772,7 @@ func (s *Service) TriggerPRSync(ctx context.Context, taskID string) (*TaskPR, er // when the batched fetch itself fails; in those cases we fan out one // subprocess per watch as before. func (s *Service) TriggerPRSyncAll(ctx context.Context, taskID string) ([]*TaskPR, error) { - prs, _, err := s.triggerPRSyncAllPermanent(ctx, taskID) + prs, _, err := s.triggerPRSyncAllPermanent(ctx, taskID, true) return prs, err } @@ -1479,10 +1784,23 @@ func (s *Service) TriggerPRSyncAll(ctx context.Context, taskID string) ([]*TaskP // pointing at a deleted repo doesn't keep hammering the gh throttle for // the lifetime of the task. func (s *Service) TriggerPRSyncAllPermanent(ctx context.Context, taskID string) ([]*TaskPR, bool, error) { - return s.triggerPRSyncAllPermanent(ctx, taskID) + return s.triggerPRSyncAllPermanent(ctx, taskID, true) } -func (s *Service) triggerPRSyncAllPermanent(ctx context.Context, taskID string) ([]*TaskPR, bool, error) { +// TriggerPRSyncAllPermanentWithOptions runs the task PR sync with an explicit +// refresh decision supplied by the request boundary. Automatic task views use +// passive admission so an idle searching watch does not become a provider +// request; a user refresh can bypass that admission and invalidate cached +// observations. +func (s *Service) TriggerPRSyncAllPermanentWithOptions( + ctx context.Context, taskID string, explicitRefresh bool, +) ([]*TaskPR, bool, error) { + return s.triggerPRSyncAllPermanent(ctx, taskID, explicitRefresh) +} + +func (s *Service) triggerPRSyncAllPermanent( + ctx context.Context, taskID string, explicitRefresh bool, +) ([]*TaskPR, bool, error) { watches, err := s.store.ListPRWatchesByTask(ctx, taskID) if err != nil { return nil, false, fmt.Errorf("list PR watches: %w", err) @@ -1493,35 +1811,45 @@ func (s *Service) triggerPRSyncAllPermanent(ctx context.Context, taskID string) // sync path at all, so reconcile them here rather than returning the // stored snapshot verbatim. Empty slice if none. permanent=false (the // task can still acquire a watch later via push detection). - existing, listErr := s.reconcileTaskUnwatchedPRs(ctx, taskID, nil) + existing, listErr := s.reconcileTaskUnwatchedPRs(ctx, taskID, nil, explicitRefresh) if listErr != nil { return nil, false, listErr } return existing, false, nil } - prs, syncErr := s.runBatchedOrPerWatchSync(ctx, taskID, watches) + allWatches := watches + watchesToSync := watches + if !explicitRefresh { + watchesToSync = selectDuePRWatches(ctx, watches, s.taskActivityProvider, s.now(), s.logger) + } + prs, syncErr := s.runBatchedOrPerWatchSync(ctx, taskID, watchesToSync, explicitRefresh) // PRs the task linked from an earlier branch are no longer covered by any // watch — the loop above cannot see them, and without this they keep their // last-observed state (open, green, mergeable) forever. - if reconciled, listErr := s.reconcileTaskUnwatchedPRs(ctx, taskID, watches); listErr != nil { + if reconciled, listErr := s.reconcileTaskUnwatchedPRs(ctx, taskID, allWatches, explicitRefresh); listErr != nil { s.logger.Debug("unwatched task PR reconciliation failed", zap.String("task_id", taskID), zap.Error(listErr)) } else { prs = reconciled } - return prs, s.areAllWatchesPermanentlyMissing(ctx, watches), syncErr + return prs, s.areAllWatchesPermanentlyMissing(ctx, allWatches), syncErr } // runBatchedOrPerWatchSync is the shared "try batched, fall back to // per-watch" body of triggerPRSyncAllPermanent. Split out so the // permanent-flag computation can wrap the result without duplicating // the batched / fallback branches inline. -func (s *Service) runBatchedOrPerWatchSync(ctx context.Context, taskID string, watches []*PRWatch) ([]*TaskPR, error) { +func (s *Service) runBatchedOrPerWatchSync( + ctx context.Context, taskID string, watches []*PRWatch, explicitRefresh bool, +) ([]*TaskPR, error) { + if len(watches) == 0 { + return s.store.ListTaskPRsByTask(ctx, taskID) + } workspaceID := watches[0].WorkspaceID - if _, batchErr := s.SyncWorkspaceWatchesBatched(ctx, workspaceID, watches); batchErr != nil { + if _, batchErr := s.syncWorkspaceWatchesBatched(ctx, workspaceID, watches, explicitRefresh); batchErr != nil { s.logger.Debug("batched PR sync failed; falling back to per-watch", zap.String("task_id", taskID), zap.Error(batchErr)) - return s.triggerPRSyncAllPerWatch(ctx, taskID, watches) + return s.triggerPRSyncAllPerWatch(ctx, taskID, watches, explicitRefresh) } // Batched path applied all DB updates inline; reload so the WS caller // sees the freshest TaskPR rows. @@ -1551,16 +1879,18 @@ func (s *Service) areAllWatchesPermanentlyMissing(ctx context.Context, watches [ // per watch. Kept as a fallback for the NoopClient and for the rare case // where the batched GraphQL call fails (auth glitch, network blip) so a // single bad cycle doesn't leave the UI staring at stale data. -func (s *Service) triggerPRSyncAllPerWatch(ctx context.Context, taskID string, watches []*PRWatch) ([]*TaskPR, error) { +func (s *Service) triggerPRSyncAllPerWatch( + ctx context.Context, taskID string, watches []*PRWatch, explicitRefresh bool, +) ([]*TaskPR, error) { results := make([]*TaskPR, 0, len(watches)) var syncErrs []error for _, w := range watches { var tp *TaskPR var syncErr error if w.PRNumber == 0 { - tp, syncErr = s.triggerPRDetection(ctx, w, taskID) + tp, syncErr = s.triggerPRDetectionWithOptions(ctx, w, taskID, explicitRefresh) } else { - tp, syncErr = s.triggerPRStatusSync(ctx, w, taskID) + tp, syncErr = s.triggerPRStatusSyncWithOptions(ctx, w, taskID, explicitRefresh) } if syncErr != nil { // Debug, not Warn: this is best-effort background reconciliation and @@ -1611,6 +1941,12 @@ func (e *PartialPRSyncError) Unwrap() error { } func (s *Service) triggerPRDetection(ctx context.Context, watch *PRWatch, taskID string) (*TaskPR, error) { + return s.triggerPRDetectionWithOptions(ctx, watch, taskID, false) +} + +func (s *Service) triggerPRDetectionWithOptions( + ctx context.Context, watch *PRWatch, taskID string, explicitRefresh bool, +) (*TaskPR, error) { if watch == nil || strings.TrimSpace(watch.WorkspaceID) == "" { return nil, ErrGitHubWorkspaceRequired } @@ -1626,8 +1962,11 @@ func (s *Service) triggerPRDetection(ctx context.Context, watch *PRWatch, taskID // two never collide. singleflight only blocks same-key calls, so the nested // triggerPRStatusSync (different key) below cannot deadlock. key := scopedCacheKey(resolved.CacheScope, "pr-detect:"+watch.ID) + if explicitRefresh { + key += prSyncExplicitRefreshKeySuffix + } v, err, _ := s.syncGroup.Do(key, func() (interface{}, error) { - return s.detectPRForWatchOnce(ctx, resolved, watch, taskID) + return s.detectPRForWatchOnce(ctx, resolved, watch, taskID, explicitRefresh) }) if err != nil { return nil, err @@ -1643,6 +1982,7 @@ func (s *Service) triggerPRDetection(ctx context.Context, watch *PRWatch, taskID // singleflight so only one probe per watch is in flight at a time. func (s *Service) detectPRForWatchOnce( ctx context.Context, resolved *resolvedServiceClient, watch *PRWatch, taskID string, + explicitRefresh bool, ) (*TaskPR, error) { // Short-circuit when this repo is already in the 10-min negative // cache. Without this, an unresolvable repo gets a fresh per-watch @@ -1657,7 +1997,7 @@ func (s *Service) detectPRForWatchOnce( // Without this, a branch whose PR never appears (e.g. an unresolvable repo) // re-hits `gh` on every on-demand sync, and the frontend re-syncs every 5s // while no PR is found — flooding the logs with identical failures. - if watch.LastCheckedAt != nil && time.Since(*watch.LastCheckedAt) < PRSyncFreshnessWindow { + if !explicitRefresh && watch.LastCheckedAt != nil && time.Since(*watch.LastCheckedAt) < PRSyncFreshnessWindow { effectiveTaskID := s.reconcileTaskPROwnership(ctx, watch.SessionID, watch.TaskID, watch.RepositoryID, watch.PRNumber) return s.store.GetTaskPRByRepository(ctx, effectiveTaskID, watch.RepositoryID) } @@ -1671,8 +2011,8 @@ func (s *Service) detectPRForWatchOnce( if resolved.credential != nil { credentialGeneration = resolved.credential.CredentialGeneration } - attempt, admitted := s.beginPRDiscoveryWatch( - watch.WorkspaceID, resolved.CacheScope, credentialGeneration, watch, + attempt, admitted := s.beginPRDiscoveryWatchWithOptions( + watch.WorkspaceID, resolved.CacheScope, credentialGeneration, watch, explicitRefresh, ) if !admitted { effectiveTaskID := s.reconcileTaskPROwnership(ctx, watch.SessionID, watch.TaskID, watch.RepositoryID, watch.PRNumber) @@ -1758,10 +2098,16 @@ func (s *Service) detectPRForWatchOnce( ) // Also fetch status so the first response includes review/check state watch.PRNumber = pr.Number - return s.triggerPRStatusSync(ctx, watch, taskID) + return s.triggerPRStatusSyncWithOptions(ctx, watch, taskID, explicitRefresh) } func (s *Service) triggerPRStatusSync(ctx context.Context, watch *PRWatch, taskID string) (*TaskPR, error) { + return s.triggerPRStatusSyncWithOptions(ctx, watch, taskID, false) +} + +func (s *Service) triggerPRStatusSyncWithOptions( + ctx context.Context, watch *PRWatch, taskID string, explicitRefresh bool, +) (*TaskPR, error) { if watch == nil || strings.TrimSpace(watch.WorkspaceID) == "" { return nil, ErrGitHubWorkspaceRequired } @@ -1781,27 +2127,16 @@ func (s *Service) triggerPRStatusSync(ctx context.Context, watch *PRWatch, taskI if err != nil { return nil, err } - // Freshness check: skip GitHub API if this exact PR row was recently - // synced. Multi-branch tasks can have multiple PRs in the same repo, so a - // repo-only lookup would let a fresh sibling suppress this PR's sync. - loadTaskPR := func(c context.Context) (*TaskPR, error) { - tp, err := s.store.GetTaskPRByRepoAndNumber(c, taskID, watch.RepositoryID, watch.PRNumber) - if err != nil { - return nil, err - } - if tp != nil { - return tp, nil - } - // Fall back to the legacy untagged row for single-repo tasks that - // haven't been re-associated under the multi-repo schema yet. - if watch.RepositoryID != "" { - return nil, nil - } - return s.store.GetTaskPR(c, taskID) + if explicitRefresh { + s.invalidateWorkflowAttentionForResolvedPR( + resolved, watch.Owner, watch.Repo, watch.PRNumber, "", + ) } - if tp, _ := loadTaskPR(ctx); tp != nil && tp.LastSyncedAt != nil { - if time.Since(*tp.LastSyncedAt) < PRSyncFreshnessWindow { - return tp, nil + if !explicitRefresh { + if tp, _ := s.loadTaskPRForWatch(ctx, taskID, watch); tp != nil && tp.LastSyncedAt != nil { + if time.Since(*tp.LastSyncedAt) < PRSyncFreshnessWindow { + return tp, nil + } } } @@ -1811,16 +2146,39 @@ func (s *Service) triggerPRStatusSync(ctx context.Context, watch *PRWatch, taskI if s.isRepoCachedAsMissingForScope(resolved.CacheScope, watch.Owner, watch.Repo) { return nil, ErrRepoNotResolvable } + return s.runPRStatusSync(ctx, resolved, watch, taskID, rawTaskID, explicitRefresh) +} - // Coalesce concurrent syncs for the same PR +// loadTaskPRForWatch reads the exact repository row first, then preserves the +// legacy single-repository fallback for older associations. +func (s *Service) loadTaskPRForWatch( + ctx context.Context, taskID string, watch *PRWatch, +) (*TaskPR, error) { + tp, err := s.store.GetTaskPRByRepoAndNumber(ctx, taskID, watch.RepositoryID, watch.PRNumber) + if err != nil || tp != nil || watch.RepositoryID != "" { + return tp, err + } + return s.store.GetTaskPR(ctx, taskID) +} + +func (s *Service) runPRStatusSync( + ctx context.Context, + resolved *resolvedServiceClient, + watch *PRWatch, + taskID, rawTaskID string, + explicitRefresh bool, +) (*TaskPR, error) { key := scopedCacheKey(resolved.CacheScope, fmt.Sprintf("%s/%s/%d", watch.Owner, watch.Repo, watch.PRNumber)) + if explicitRefresh { + key += prSyncExplicitRefreshKeySuffix + } v, err, _ := s.syncGroup.Do(key, func() (interface{}, error) { bgCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() // Snapshot cache generation BEFORE the per-watch probe so a // concurrent eviction wins; see Service.markRepoAsMissing. repoErrGen := s.repoErrorGenSnapshot() - status, _, checkErr := s.checkPRWatchWithClient(bgCtx, resolved.Client, watch) + status, _, checkErr := s.checkPRWatchWithClient(bgCtx, resolved.Client, resolved.CacheScope, watch) if checkErr != nil { if isRepoNotResolvableErr(checkErr) { s.markRepoAsMissingForScope(resolved.CacheScope, watch.Owner, watch.Repo, repoErrGen) @@ -1829,26 +2187,19 @@ func (s *Service) triggerPRStatusSync(ctx context.Context, watch *PRWatch, taskI return nil, checkErr } if status == nil { - return loadTaskPR(bgCtx) + return s.loadTaskPRForWatch(bgCtx, taskID, watch) } - if existing, loadErr := loadTaskPR(bgCtx); loadErr != nil { + existing, loadErr := s.loadTaskPRForWatch(bgCtx, taskID, watch) + if loadErr != nil { return nil, loadErr - } else if existing == nil && status.PR != nil { - // Gap-fill a numbered watch whose exact task_pr row was never - // created. AssociatePRWithTask publishes the creation event; the - // following SyncTaskPR may publish again if status fields changed. - // That double event is harmless because clients re-fetch state. - if _, assocErr := s.associatePRWithTaskForSession( - bgCtx, watch.WorkspaceID, watch.SessionID, rawTaskID, watch.RepositoryID, status.PR, - false, false, TaskPRSourceWatch, - ); assocErr != nil { - return nil, assocErr - } + } + if err := s.ensurePRStatusAssociation(bgCtx, watch, rawTaskID, existing, status); err != nil { + return nil, err } if syncErr := s.SyncTaskPR(bgCtx, taskID, status); syncErr != nil { return nil, syncErr } - return loadTaskPR(bgCtx) + return s.loadTaskPRForWatch(bgCtx, taskID, watch) }) if err != nil { return nil, err @@ -1858,3 +2209,24 @@ func (s *Service) triggerPRStatusSync(ctx context.Context, watch *PRWatch, taskI } return v.(*TaskPR), nil } + +func (s *Service) ensurePRStatusAssociation( + ctx context.Context, + watch *PRWatch, + rawTaskID string, + existing *TaskPR, + status *PRStatus, +) error { + if existing != nil || status.PR == nil { + return nil + } + // Gap-fill a numbered watch whose exact task_pr row was never created. + // AssociatePRWithTask publishes the creation event; the following + // SyncTaskPR may publish again if status fields changed. That double event + // is harmless because clients re-fetch state. + _, assocErr := s.associatePRWithTaskForSession( + ctx, watch.WorkspaceID, watch.SessionID, rawTaskID, watch.RepositoryID, status.PR, + false, false, TaskPRSourceWatch, + ) + return assocErr +} diff --git a/apps/backend/internal/github/service_pr_watch_batched.go b/apps/backend/internal/github/service_pr_watch_batched.go index 7cb8fce268..2d0c9b02c6 100644 --- a/apps/backend/internal/github/service_pr_watch_batched.go +++ b/apps/backend/internal/github/service_pr_watch_batched.go @@ -27,6 +27,8 @@ type PRWatchSyncResult struct { DiscoveryResolved bool } +const prSyncExplicitRefreshKeySuffix = "|explicit-refresh" + // SyncWatchesBatched runs the batched GraphQL queries for the supplied // watches and applies the resulting DB updates: timestamps, task PR sync, // watch PR-number promotion on detection, watch reset on merge/close. @@ -41,13 +43,19 @@ type PRWatchSyncResult struct { // TriggerPRSyncAll / ListWorkspaceTaskPRs background refresh share, so a // 40-watch workspace fans out to ~2 gh subprocess calls instead of 40. func (s *Service) SyncWatchesBatched(ctx context.Context, watches []*PRWatch) ([]PRWatchSyncResult, error) { - return s.syncWatchesBatchedWithClient(ctx, s.client, "legacy", "", 0, watches) + return s.syncWatchesBatchedWithClient(ctx, s.client, "legacy", "", 0, watches, false) } // SyncWorkspaceWatchesBatched resolves one automation credential and rejects // mixed or missing workspace ownership before making a provider call. func (s *Service) SyncWorkspaceWatchesBatched( ctx context.Context, workspaceID string, watches []*PRWatch, +) ([]PRWatchSyncResult, error) { + return s.syncWorkspaceWatchesBatched(ctx, workspaceID, watches, false) +} + +func (s *Service) syncWorkspaceWatchesBatched( + ctx context.Context, workspaceID string, watches []*PRWatch, explicitRefresh bool, ) ([]PRWatchSyncResult, error) { if len(watches) == 0 { return nil, nil @@ -68,8 +76,11 @@ func (s *Service) SyncWorkspaceWatchesBatched( if resolved.credential != nil { credentialGeneration = resolved.credential.CredentialGeneration } + if explicitRefresh { + s.invalidateWorkflowAttentionForResolvedWatches(resolved, watches) + } return s.syncWatchesBatchedWithClient( - ctx, resolved.Client, resolved.CacheScope, workspaceID, credentialGeneration, watches, + ctx, resolved.Client, resolved.CacheScope, workspaceID, credentialGeneration, watches, explicitRefresh, ) } @@ -83,7 +94,8 @@ type prWatchBatchGroup struct { } func (s *Service) syncWatchesBatchedWithClient( - ctx context.Context, client Client, cacheScope, workspaceID string, credentialGeneration int64, watches []*PRWatch, + ctx context.Context, client Client, cacheScope, workspaceID string, credentialGeneration int64, + watches []*PRWatch, explicitRefresh bool, ) ([]PRWatchSyncResult, error) { if len(watches) == 0 { return nil, nil @@ -124,8 +136,8 @@ func (s *Service) syncWatchesBatchedWithClient( for _, watch := range group.watches[1:] { s.trackPRDiscoveryWatchConsumer(workspaceID, cacheScope, credentialGeneration, watch) } - attempt, ok := s.beginPRDiscoveryWatch( - workspaceID, cacheScope, credentialGeneration, group.representative, + attempt, ok := s.beginPRDiscoveryWatchWithOptions( + workspaceID, cacheScope, credentialGeneration, group.representative, explicitRefresh, ) if !ok { continue @@ -148,7 +160,9 @@ func (s *Service) syncWatchesBatchedWithClient( var statuses *batchedWatchStatuses if len(leaderGroups) > 0 { numbered, searching := splitPRWatches(admitted) - statuses, err = s.fetchBatchedWatchStatuses(ctx, client, exec, cacheScope, numbered, searching) + statuses, err = s.fetchBatchedWatchStatuses( + ctx, client, exec, cacheScope, numbered, searching, explicitRefresh, + ) if err != nil { if s.handleBatchedDiscoveryFetchError( workspaceID, cacheScope, credentialGeneration, leaderGroups, admitted, numbered, searching, err, @@ -435,10 +449,7 @@ func (s *Service) enrichBatchedWorkflowAttentionGroup( } defer func() { <-sem }() - key := workflowAttentionBatchKey(cacheScope, group.owner, group.repo, group.headSHA) - value, err, _ := s.syncGroup.Do(key, func() (interface{}, error) { - return client.ListWorkflowRuns(ctx, group.owner, group.repo, group.headSHA) - }) + runs, err := s.cachedWorkflowRuns(ctx, client, cacheScope, group.owner, group.repo, group.headSHA) if err != nil { if s.logger != nil { s.logger.Debug("workflow attention read unavailable", @@ -447,16 +458,11 @@ func (s *Service) enrichBatchedWorkflowAttentionGroup( markBatchedWorkflowAttentionUnknown(group) return } - runs, ok := value.([]WorkflowRun) - if !ok { - markBatchedWorkflowAttentionUnknown(group) - return - } - s.applyBatchedWorkflowAttention(ctx, client, group, runs) + s.applyBatchedWorkflowAttention(ctx, client, cacheScope, group, runs) } func (s *Service) applyBatchedWorkflowAttention( - ctx context.Context, client Client, group *workflowAttentionStatusGroup, runs []WorkflowRun, + ctx context.Context, client Client, cacheScope string, group *workflowAttentionStatusGroup, runs []WorkflowRun, ) { jobs := make(map[workflowJobKey]struct { value []WorkflowJob @@ -467,7 +473,9 @@ func (s *Service) applyBatchedWorkflowAttention( if cached, found := jobs[jobKey]; found { return cached.value, cached.err } - value, readErr := client.ListWorkflowRunJobs(jobCtx, group.owner, group.repo, runID, attempt) + value, readErr := s.cachedWorkflowJobs( + jobCtx, client, cacheScope, group.owner, group.repo, runID, attempt, true, + ) jobs[jobKey] = struct { value []WorkflowJob err error @@ -486,7 +494,7 @@ func (s *Service) applyBatchedWorkflowAttention( } func workflowAttentionBatchKey(cacheScope, owner, repo, headSHA string) string { - return scopedCacheKey(cacheScope, fmt.Sprintf("workflow-attention:%s/%s@%s", strings.ToLower(owner), strings.ToLower(repo), headSHA)) + return workflowRunsCacheKey(cacheScope, owner, repo, headSHA) } // batchedWatchStatuses is the shared, read-only result of one batched fetch. @@ -522,9 +530,13 @@ type batchedWatchStatuses struct { // GetPRFeedback / GetPRStatus. The leader's deadline is preserved so // the fetch can't outlive the request budget. func (s *Service) fetchBatchedWatchStatuses( - ctx context.Context, client Client, exec GraphQLExecutor, cacheScope string, numbered, searching []*PRWatch, + ctx context.Context, client Client, exec GraphQLExecutor, cacheScope string, + numbered, searching []*PRWatch, explicitRefresh bool, ) (*batchedWatchStatuses, error) { key := scopedCacheKey(cacheScope, batchedFetchSingleflightKey(numbered, searching)) + if explicitRefresh { + key += prSyncExplicitRefreshKeySuffix + } fetchCtx, cancelFetch := derivedFetchContext(ctx) defer cancelFetch() v, err, _ := s.syncGroup.Do(key, func() (interface{}, error) { diff --git a/apps/backend/internal/github/service_pr_watch_fallback_test.go b/apps/backend/internal/github/service_pr_watch_fallback_test.go new file mode 100644 index 0000000000..6b13583ab1 --- /dev/null +++ b/apps/backend/internal/github/service_pr_watch_fallback_test.go @@ -0,0 +1,272 @@ +package github + +import ( + "context" + "errors" + "sync" + "testing" + "time" +) + +type passiveFallbackGraphQLClient struct { + *graphQLMockClient + mu sync.Mutex + findErr error + findCalls int +} + +func (c *passiveFallbackGraphQLClient) FindPRByBranch( + context.Context, string, string, string, +) (*PR, error) { + c.mu.Lock() + c.findCalls++ + err := c.findErr + c.mu.Unlock() + return nil, err +} + +func (c *passiveFallbackGraphQLClient) findPRByBranchCallCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.findCalls +} + +func TestPassiveFallbackAdmissionBoundsWorkspaceAndGlobalWindow(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + clock := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + svc.SetClock(func() time.Time { return clock }) + watches := seedSearchingFallbackWatches(t, store, 7) + + first := svc.selectPassiveFallbackTargets(testWorkspaceID, watches) + if len(first) != prWatchFallbackWorkspaceBudget { + t.Fatalf("first workspace fallback targets = %d, want %d", len(first), prWatchFallbackWorkspaceBudget) + } + + otherWorkspace := make([]*PRWatch, 7) + for index := range otherWorkspace { + otherWorkspace[index] = &PRWatch{ + ID: "other-" + watches[index].ID, + WorkspaceID: "workspace-other", + Owner: "o", + Repo: "r", + Branch: watches[index].Branch, + } + } + second := svc.selectPassiveFallbackTargets("workspace-other", otherWorkspace) + if len(second) != prWatchFallbackWorkspaceBudget { + t.Fatalf("second workspace fallback targets = %d, want %d", len(second), prWatchFallbackWorkspaceBudget) + } + if third := svc.selectPassiveFallbackTargets(testWorkspaceID, watches); len(third) != 0 { + t.Fatalf("same-window workspace fallback targets = %d, want 0", len(third)) + } + + clock = clock.Add(time.Minute) + rotated := svc.selectPassiveFallbackTargets(testWorkspaceID, watches) + if len(rotated) != prWatchFallbackWorkspaceBudget { + t.Fatalf("next-window fallback targets = %d, want %d", len(rotated), prWatchFallbackWorkspaceBudget) + } + if rotated[0].ID == first[0].ID { + t.Fatalf("fallback target rotation restarted at %q", rotated[0].ID) + } +} + +func TestPassiveFallbackAdmissionUsesRollingMinute(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + start := time.Date(2026, time.September, 22, 12, 0, 59, 0, time.UTC) + clock := start + svc.SetClock(func() time.Time { return clock }) + firstWorkspace := seedSearchingFallbackWatches(t, store, 7) + secondWorkspace := make([]*PRWatch, 7) + for index := range secondWorkspace { + secondWorkspace[index] = &PRWatch{ + ID: "rolling-other-" + firstWorkspace[index].ID, + WorkspaceID: "workspace-rolling-other", + Owner: "o", + Repo: "r", + Branch: firstWorkspace[index].Branch, + } + } + + if got := len(svc.selectPassiveFallbackTargets(testWorkspaceID, firstWorkspace)); got != prWatchFallbackWorkspaceBudget { + t.Fatalf("first workspace admissions = %d, want %d", got, prWatchFallbackWorkspaceBudget) + } + if got := len(svc.selectPassiveFallbackTargets("workspace-rolling-other", secondWorkspace)); got != prWatchFallbackWorkspaceBudget { + t.Fatalf("second workspace admissions = %d, want %d", got, prWatchFallbackWorkspaceBudget) + } + + // A wall-clock window would reset at 12:01:00 and admit another burst. + // The rolling window keeps the ten admissions active until each is a full + // minute old. + clock = start.Add(2 * time.Second) + if got := len(svc.selectPassiveFallbackTargets(testWorkspaceID, firstWorkspace)); got != 0 { + t.Fatalf("boundary burst admitted %d targets, want 0", got) + } + + clock = start.Add(time.Minute) + if got := len(svc.selectPassiveFallbackTargets(testWorkspaceID, firstWorkspace)); got != prWatchFallbackWorkspaceBudget { + t.Fatalf("expired rolling admissions = %d, want %d", got, prWatchFallbackWorkspaceBudget) + } +} + +func TestPassiveFallbackStopsAfterFirstRateLimit(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + rateClient := &fallbackProbeClient{ + Client: NewMockClient(), + findErr: rateLimitAPIError(429, "secondary limit"), + } + configureTestWorkspaceAuth(t, svc, rateClient, testWorkspaceID) + watches := seedSearchingFallbackWatches(t, store, 7) + + err := svc.refreshWatchesPerWatch(context.Background(), testWorkspaceID, watches) + if err == nil || !prWatchFallbackShouldStopWorkspace(err) { + t.Fatalf("fallback error = %v, want a workspace stop", err) + } + if got := len(rateClient.fallbackBranches()); got != 1 { + t.Fatalf("rate-limited passive fallback calls = %d, want one", got) + } +} + +func TestPassiveFallbackSkipsRateLimitedBatch(t *testing.T) { + _, svc, client, store := setupBatchedPollerTest(t) + rateErr := rateLimitAPIError(429, "secondary limit") + client.branchErr = rateErr + watches := seedSearchingFallbackWatches(t, store, 7) + staleTasks := make(map[string]struct{}, len(watches)) + for _, watch := range watches { + staleTasks[watch.TaskID] = struct{}{} + } + + svc.refreshStaleWorkspaceWatches(testWorkspaceID, staleTasks) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, ok := svc.inflightWorkspaceRefreshes.Load(testWorkspaceID); !ok { + break + } + time.Sleep(5 * time.Millisecond) + } + if _, ok := svc.inflightWorkspaceRefreshes.Load(testWorkspaceID); ok { + t.Fatal("workspace refresh did not finish") + } + if got := len(client.branchQueries); got != 1 { + t.Fatalf("batch branch queries = %d, want one", got) + } + // The GraphQL error is rate limited, so the passive legacy fallback must + // not issue a second provider request for any of the seven targets. + if got := client.FindPRByBranchCallCount(); got != 0 { + t.Fatalf("per-watch fallback calls = %d, want 0", got) + } +} + +func TestPassiveFallbackOrdinaryErrorContinuesWithinBudget(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ordinary := &fallbackProbeClient{Client: NewMockClient(), findErr: errors.New("temporary transport failure")} + configureTestWorkspaceAuth(t, svc, ordinary, testWorkspaceID) + watches := seedSearchingFallbackWatches(t, store, 7) + + if err := svc.refreshWatchesPerWatch(context.Background(), testWorkspaceID, watches); err != nil { + t.Fatalf("ordinary fallback error = %v, want best-effort completion", err) + } + if got := len(ordinary.fallbackBranches()); got != prWatchFallbackWorkspaceBudget { + t.Fatalf("ordinary passive fallback calls = %d, want %d", got, prWatchFallbackWorkspaceBudget) + } +} + +func TestPassiveFallbackDeduplicatesSharedTargetsAndFansOut(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ctx := context.Background() + client := &fallbackProbeClient{Client: NewMockClient()} + configureTestWorkspaceAuth(t, svc, client, testWorkspaceID) + watches := seedSearchingFallbackWatches(t, store, 5) + watches[1].Branch = watches[0].Branch + + if err := svc.refreshWatchesPerWatch(context.Background(), testWorkspaceID, watches); err != nil { + t.Fatalf("passive fallback error = %v", err) + } + branches := client.fallbackBranches() + if len(branches) != 4 { + t.Fatalf("passive fallback calls = %d, want one per unique target", len(branches)) + } + sharedCalls := 0 + for _, branch := range branches { + if branch == watches[0].Branch { + sharedCalls++ + } + } + if sharedCalls != 1 { + t.Fatalf("passive shared target calls = %d, want one", sharedCalls) + } + for _, watch := range watches { + updated, err := store.GetPRWatch(ctx, watch.ID) + if err != nil { + t.Fatalf("get watch %q: %v", watch.ID, err) + } + if updated == nil || updated.LastCheckedAt == nil { + t.Fatalf("watch %q did not receive the shared fallback result", watch.ID) + } + } +} + +func TestPassiveWorkspaceFallbackBoundsTargetsAndStopsAfterFirstRateLimit(t *testing.T) { + _, svc, _, store := setupBatchedPollerTest(t) + client := &passiveFallbackGraphQLClient{ + graphQLMockClient: &graphQLMockClient{ + MockClient: NewMockClient(), + branchErr: errors.New("temporary graphql transport failure"), + }, + findErr: rateLimitAPIError(429, "secondary limit"), + } + configureTestWorkspaceAuth(t, svc, client, testWorkspaceID) + watches := seedSearchingFallbackWatches(t, store, 7) + staleTasks := make(map[string]struct{}, len(watches)) + for _, watch := range watches { + staleTasks[watch.TaskID] = struct{}{} + } + + svc.refreshStaleWorkspaceWatches(testWorkspaceID, staleTasks) + waitForPassiveWorkspaceRefresh(t, svc, testWorkspaceID) + + if got := client.findPRByBranchCallCount(); got != 1 { + t.Fatalf("workspace fallback calls after first rate limit = %d, want 1", got) + } + if got := len(client.branchQueries); got != 1 { + t.Fatalf("workspace batch queries = %d, want 1", got) + } + svc.Stop() +} + +func TestPassiveWorkspaceFallbackUsesFiveTargetBudget(t *testing.T) { + _, svc, _, store := setupBatchedPollerTest(t) + client := &passiveFallbackGraphQLClient{ + graphQLMockClient: &graphQLMockClient{ + MockClient: NewMockClient(), + branchErr: errors.New("temporary graphql transport failure"), + }, + findErr: errors.New("temporary REST transport failure"), + } + configureTestWorkspaceAuth(t, svc, client, testWorkspaceID) + watches := seedSearchingFallbackWatches(t, store, 7) + staleTasks := make(map[string]struct{}, len(watches)) + for _, watch := range watches { + staleTasks[watch.TaskID] = struct{}{} + } + + svc.refreshStaleWorkspaceWatches(testWorkspaceID, staleTasks) + waitForPassiveWorkspaceRefresh(t, svc, testWorkspaceID) + + if got := client.findPRByBranchCallCount(); got != prWatchFallbackWorkspaceBudget { + t.Fatalf("workspace fallback calls = %d, want %d", got, prWatchFallbackWorkspaceBudget) + } + svc.Stop() +} + +func waitForPassiveWorkspaceRefresh(t *testing.T, svc *Service, workspaceID string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if _, ok := svc.inflightWorkspaceRefreshes.Load(workspaceID); !ok { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("workspace refresh did not finish") +} diff --git a/apps/backend/internal/github/service_pr_watch_schedule_test.go b/apps/backend/internal/github/service_pr_watch_schedule_test.go new file mode 100644 index 0000000000..0670cf73b9 --- /dev/null +++ b/apps/backend/internal/github/service_pr_watch_schedule_test.go @@ -0,0 +1,337 @@ +package github + +import ( + "context" + "sync/atomic" + "testing" + "time" + + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestPassiveWorkspaceRefreshHonorsAdaptiveSchedule(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + // Keep the persisted watch older than the injected scheduler clock. This + // isolates task inactivity from the new-watch fast baseline. + now := time.Now().UTC().Add(48 * time.Hour) + svc.SetClock(func() time.Time { return now }) + provider := &scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{ + "task-passive-slow": {LastActivityAt: now.Add(-24 * time.Hour)}, + }} + svc.SetTaskActivityProvider(provider) + seedTask(t, store, "task-passive-slow", false) + watch, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-passive", "task-passive-slow", "repo-1", "owner", "repo", 0, "feature/passive") + if err != nil { + t.Fatalf("create passive watch: %v", err) + } + lastChecked := now.Add(-10 * time.Minute) + if err := store.UpdatePRWatchTimestamps(ctx, watch.ID, lastChecked, nil, "", ""); err != nil { + t.Fatalf("stamp passive watch: %v", err) + } + + if _, err := svc.ListWorkspaceTaskPRs(ctx, testWorkspaceID); err != nil { + t.Fatalf("passive workspace read: %v", err) + } + time.Sleep(100 * time.Millisecond) + if len(gh.branchQueries) != 0 { + t.Fatal("passive refresh checked a slow searching watch before its due interval") + } + + gh.branchResponses = []string{`{"data":{"b0":{"pullRequests":{"nodes":[]}}}}`} + started := make(chan struct{}, 1) + gh.onExecute = func() { started <- struct{}{} } + svc.SetClock(func() time.Time { return now.Add(30 * time.Minute) }) + if _, err := svc.ListWorkspaceTaskPRs(ctx, testWorkspaceID); err != nil { + t.Fatalf("due passive workspace read: %v", err) + } + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("passive refresh did not check the slow searching watch at its due interval") + } + svc.Stop() +} + +func TestPassiveWorkspaceRefreshCooldownSkipsRepeatedWorkspaceReads(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + now := time.Now().UTC() + svc.SetClock(func() time.Time { return now }) + seedTask(t, store, "task-passive-cooldown", false) + if _, err := svc.CreatePRWatchForWorkspace( + ctx, testWorkspaceID, "session-passive-cooldown", "task-passive-cooldown", "repo-1", + "owner", "repo", 0, "feature/passive-cooldown", + ); err != nil { + t.Fatalf("create passive watch: %v", err) + } + gh.branchResponses = []string{ + `{"data":{"b0":{"pullRequests":{"nodes":[]}}}}`, + `{"data":{"b0":{"pullRequests":{"nodes":[]}}}}`, + } + + if _, err := svc.ListWorkspaceTaskPRs(ctx, testWorkspaceID); err != nil { + t.Fatalf("first passive workspace read: %v", err) + } + waitForPassiveWorkspaceRefresh(t, svc, testWorkspaceID) + if got := len(gh.branchQueries); got != 1 { + t.Fatalf("first passive refresh queries = %d, want 1", got) + } + + if _, err := svc.ListWorkspaceTaskPRs(ctx, testWorkspaceID); err != nil { + t.Fatalf("second passive workspace read: %v", err) + } + time.Sleep(100 * time.Millisecond) + if got := len(gh.branchQueries); got != 1 { + t.Fatalf("cooldown allowed repeated passive refresh queries = %d, want 1", got) + } + + svc.SetClock(func() time.Time { return now.Add(2 * searchFastPollInterval) }) + if _, err := svc.ListWorkspaceTaskPRs(ctx, testWorkspaceID); err != nil { + t.Fatalf("due passive workspace read: %v", err) + } + waitForPassiveWorkspaceRefresh(t, svc, testWorkspaceID) + if got := len(gh.branchQueries); got != 2 { + t.Fatalf("passive refresh after cooldown queries = %d, want 2", got) + } + svc.Stop() +} + +func TestExplicitPRRefreshBypassesAdaptiveIdleSchedule(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + now := time.Date(2026, time.September, 22, 12, 0, 0, 0, time.UTC) + svc.SetClock(func() time.Time { return now }) + svc.SetTaskActivityProvider(&scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{ + "task-explicit-slow": {LastActivityAt: now.Add(-24 * time.Hour)}, + }}) + seedTask(t, store, "task-explicit-slow", false) + watch, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-explicit", "task-explicit-slow", "repo-1", "owner", "repo", 0, "feature/explicit") + if err != nil { + t.Fatalf("create explicit watch: %v", err) + } + if err := store.UpdatePRWatchTimestamps(ctx, watch.ID, now.Add(-10*time.Minute), nil, "", ""); err != nil { + t.Fatalf("stamp explicit watch: %v", err) + } + gh.branchResponses = []string{`{"data":{"b0":{"pullRequests":{"nodes":[]}}}}`} + if _, err := svc.TriggerPRSyncAll(ctx, "task-explicit-slow"); err != nil { + t.Fatalf("explicit PR refresh: %v", err) + } + if len(gh.branchQueries) != 1 { + t.Fatalf("explicit refresh branch queries = %d, want 1", len(gh.branchQueries)) + } +} + +func TestWSSyncTaskPRUsesPassiveAdmissionAndExplicitRefresh(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + now := time.Now().UTC().Add(48 * time.Hour) + svc.SetClock(func() time.Time { return now }) + svc.SetTaskActivityProvider(&scheduleActivityProvider{activities: map[string]PRWatchTaskActivity{ + "task-ws-passive": {LastActivityAt: now.Add(-24 * time.Hour)}, + }}) + seedTask(t, store, "task-ws-passive", false) + watch, err := svc.CreatePRWatchForWorkspace( + ctx, testWorkspaceID, "session-ws-passive", "task-ws-passive", "repo-1", + "owner", "repo", 0, "feature/ws-passive", + ) + if err != nil { + t.Fatalf("create watch: %v", err) + } + if err := store.UpdatePRWatchTimestamps(ctx, watch.ID, now.Add(-10*time.Minute), nil, "", ""); err != nil { + t.Fatalf("stamp watch: %v", err) + } + + passive, err := ws.NewRequest("passive", ws.ActionGitHubTaskPRSync, map[string]any{ + "task_id": "task-ws-passive", + }) + if err != nil { + t.Fatalf("build passive request: %v", err) + } + if _, err := wsSyncTaskPR(svc, nil)(ctx, passive); err != nil { + t.Fatalf("passive sync request: %v", err) + } + if len(gh.branchQueries) != 0 { + t.Fatalf("passive idle branch queries = %d, want 0", len(gh.branchQueries)) + } + + gh.branchResponses = []string{`{"data":{"b0":{"pullRequests":{"nodes":[]}}}}`} + explicit, err := ws.NewRequest("explicit", ws.ActionGitHubTaskPRSync, map[string]any{ + "task_id": "task-ws-passive", + "explicit_refresh": true, + }) + if err != nil { + t.Fatalf("build explicit request: %v", err) + } + if _, err := wsSyncTaskPR(svc, nil)(ctx, explicit); err != nil { + t.Fatalf("explicit sync request: %v", err) + } + if len(gh.branchQueries) != 1 { + t.Fatalf("explicit branch queries = %d, want 1", len(gh.branchQueries)) + } + svc.Stop() +} + +func TestWSSyncTaskPRExplicitRefreshSupersedesInFlightPassiveBatch(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + defer svc.Stop() + ctx := context.Background() + seedTask(t, store, "task-ws-inflight", false) + _, err := svc.CreatePRWatchForWorkspace( + ctx, testWorkspaceID, "session-ws-inflight", "task-ws-inflight", "repo-1", + "owner", "repo", 42, "feature/ws-inflight", + ) + if err != nil { + t.Fatalf("create watch: %v", err) + } + if err := store.CreateTaskPR(ctx, &TaskPR{ + TaskID: "task-ws-inflight", Owner: "owner", Repo: "repo", PRNumber: 42, + PRURL: "https://github.com/owner/repo/pull/42", PRTitle: "PR 42", + HeadBranch: "feature/ws-inflight", BaseBranch: "main", State: "open", HeadSHA: "initial", + }); err != nil { + t.Fatalf("create task PR: %v", err) + } + gh.prResponses = []string{ + `{"data":{"repo0":{"pr0":{"state":"OPEN","title":"PR 42","url":"https://github.com/owner/repo/pull/42","isDraft":false,"mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","headRefName":"feature/ws-inflight","baseRefName":"main","headRefOid":"passive-head","author":{"login":"alice"},"createdAt":"2026-01-01T00:00:00Z","updatedAt":"2026-01-02T00:00:00Z","reviews":{"nodes":[]},"reviewRequests":{"totalCount":0},"commits":{"nodes":[]}}}}}`, + `{"data":{"repo0":{"pr0":{"state":"OPEN","title":"PR 42","url":"https://github.com/owner/repo/pull/42","isDraft":false,"mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","headRefName":"feature/ws-inflight","baseRefName":"main","headRefOid":"explicit-head","author":{"login":"alice"},"createdAt":"2026-01-01T00:00:00Z","updatedAt":"2026-01-02T00:00:00Z","reviews":{"nodes":[]},"reviewRequests":{"totalCount":0},"commits":{"nodes":[]}}}}}`, + } + started := make(chan struct{}) + secondStarted := make(chan struct{}, 1) + releaseFirst := make(chan struct{}) + var executeCount atomic.Int32 + gh.onExecute = func() { + switch executeCount.Add(1) { + case 1: + close(started) + <-releaseFirst + case 2: + secondStarted <- struct{}{} + } + } + + passive, err := ws.NewRequest("passive-inflight", ws.ActionGitHubTaskPRSync, map[string]any{ + "task_id": "task-ws-inflight", + }) + if err != nil { + t.Fatalf("build passive request: %v", err) + } + passiveDone := make(chan error, 1) + go func() { + _, requestErr := wsSyncTaskPR(svc, nil)(ctx, passive) + passiveDone <- requestErr + }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("passive sync did not enter the provider call") + } + + explicit, err := ws.NewRequest("explicit-inflight", ws.ActionGitHubTaskPRSync, map[string]any{ + "task_id": "task-ws-inflight", + "explicit_refresh": true, + }) + if err != nil { + t.Fatalf("build explicit request: %v", err) + } + explicitDone := make(chan error, 1) + go func() { + _, requestErr := wsSyncTaskPR(svc, nil)(ctx, explicit) + explicitDone <- requestErr + }() + select { + case <-secondStarted: + case <-time.After(2 * time.Second): + close(releaseFirst) + <-passiveDone + <-explicitDone + t.Fatal("explicit refresh joined the in-flight passive provider call") + } + close(releaseFirst) + select { + case err := <-passiveDone: + if err != nil { + t.Fatalf("passive sync: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("passive sync did not finish") + } + select { + case err := <-explicitDone: + if err != nil { + t.Fatalf("explicit sync: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("explicit sync did not finish") + } + if got := len(gh.prQueries); got != 2 { + t.Fatalf("provider batch calls = %d, want two independent reads", got) + } +} + +func TestWSSyncTaskPRPassiveKeepsActionsCache(t *testing.T) { + _, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-ws-actions", false) + _, err := svc.CreatePRWatchForWorkspace( + ctx, testWorkspaceID, "session-ws-actions", "task-ws-actions", "repo-1", + "owner", "repo", 42, "feature/ws-actions", + ) + if err != nil { + t.Fatalf("create numbered watch: %v", err) + } + scope := testAutomationScope(t, svc, testWorkspaceID) + key := workflowRunsCacheKey(scope, "owner", "repo", "head-ws-actions") + resolved, err := svc.resolveAutomationClient(ctx, testWorkspaceID, "", "") + if err != nil { + t.Fatalf("resolve workspace client: %v", err) + } + personalScope := resolved.cacheScopeForPurpose(CredentialPurposePersonalRead) + personalKey := workflowRunsCacheKey(personalScope, "owner", "repo", "head-ws-actions") + svc.workflowRunsCache.setWithTTL(key, []WorkflowRun{{ + ID: 1, RunAttempt: 1, Status: workflowStatusCompleted, Conclusion: "success", + }}, workflowAttentionLongTTL) + svc.workflowRunsCache.setWithTTL(personalKey, []WorkflowRun{{ + ID: 1, RunAttempt: 1, Status: workflowStatusCompleted, Conclusion: "success", + }}, workflowAttentionLongTTL) + gh.ReplaceWorkflowRuns("owner", "repo", "head-ws-actions", []WorkflowRun{{ + ID: 2, RunAttempt: 1, Status: workflowStatusCompleted, Conclusion: "success", + }}) + response := `{"data":{"repo0":{"pr0":{"state":"OPEN","title":"PR","url":"https://x/42","isDraft":false,"mergeable":"MERGEABLE","mergeStateStatus":"CLEAN","headRefName":"feature/ws-actions","baseRefName":"main","headRefOid":"head-ws-actions","author":{"login":"alice"},"createdAt":"2026-01-01T00:00:00Z","updatedAt":"2026-01-02T00:00:00Z","reviews":{"nodes":[]},"reviewRequests":{"totalCount":0},"commits":{"nodes":[{"commit":{"statusCheckRollup":{"state":"SUCCESS"}}}]}}}}}` + gh.prResponses = []string{response, response} + + passive, err := ws.NewRequest("passive-actions", ws.ActionGitHubTaskPRSync, map[string]any{ + "task_id": "task-ws-actions", + }) + if err != nil { + t.Fatalf("build passive actions request: %v", err) + } + if _, err := wsSyncTaskPR(svc, nil)(ctx, passive); err != nil { + t.Fatalf("passive actions sync: %v", err) + } + passiveValue, ok := svc.workflowRunsCache.get(key) + if !ok || passiveValue.([]WorkflowRun)[0].ID != 1 { + t.Fatal("passive task sync invalidated completed Actions cache") + } + if personalValue, ok := svc.workflowRunsCache.get(personalKey); !ok || personalValue.([]WorkflowRun)[0].ID != 1 { + t.Fatal("passive task sync invalidated the personal-read Actions cache") + } + + explicit, err := ws.NewRequest("explicit-actions", ws.ActionGitHubTaskPRSync, map[string]any{ + "task_id": "task-ws-actions", + "explicit_refresh": true, + }) + if err != nil { + t.Fatalf("build explicit actions request: %v", err) + } + if _, err := wsSyncTaskPR(svc, nil)(ctx, explicit); err != nil { + t.Fatalf("explicit actions sync: %v", err) + } + explicitValue, ok := svc.workflowRunsCache.get(key) + if !ok || explicitValue.([]WorkflowRun)[0].ID != 2 { + t.Fatalf("explicit task refresh did not replace the Actions cache: %#v", explicitValue) + } + if _, ok := svc.workflowRunsCache.get(personalKey); ok { + t.Fatal("explicit task refresh left the personal-read Actions cache populated") + } + svc.Stop() +} diff --git a/apps/backend/internal/github/service_review_cleanup_test.go b/apps/backend/internal/github/service_review_cleanup_test.go index 1464aab55f..e7e22ed609 100644 --- a/apps/backend/internal/github/service_review_cleanup_test.go +++ b/apps/backend/internal/github/service_review_cleanup_test.go @@ -2,10 +2,215 @@ package github import ( "context" + "errors" "fmt" "testing" + "time" ) +type countingCleanupClient struct { + *MockClient + prCalls int + reviewCalls int +} + +func (c *countingCleanupClient) GetPRFeedback(ctx context.Context, owner, repo string, number int) (*PRFeedback, error) { + return c.MockClient.GetPRFeedback(ctx, owner, repo, number) +} + +func (c *countingCleanupClient) GetPR(ctx context.Context, owner, repo string, number int) (*PR, error) { + c.prCalls++ + return c.MockClient.GetPR(ctx, owner, repo, number) +} + +func (c *countingCleanupClient) ListPRReviews(ctx context.Context, owner, repo string, number int) ([]PRReview, error) { + c.reviewCalls++ + return c.MockClient.ListPRReviews(ctx, owner, repo, number) +} + +func TestCleanupMergedReviewTasksSkipsHistoricalTasks(t *testing.T) { + _, svc, mockClient, store := setupPollerTest(t) + ctx := context.Background() + + client := &countingCleanupClient{MockClient: mockClient} + configureTestWorkspaceAuth(t, svc, client, "ws-1") + + watch := &ReviewWatch{WorkspaceID: "ws-1", Enabled: true} + if err := store.CreateReviewWatch(ctx, watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + seedTask(t, store, "active-review", false) + seedTask(t, store, "archived-review", true) + + rows := []*ReviewPRTask{ + {ReviewWatchID: watch.ID, RepoOwner: "acme", RepoName: "widget", PRNumber: 1, TaskID: "active-review"}, + {ReviewWatchID: watch.ID, RepoOwner: "acme", RepoName: "widget", PRNumber: 2, TaskID: "archived-review"}, + {ReviewWatchID: watch.ID, RepoOwner: "acme", RepoName: "widget", PRNumber: 3, TaskID: "missing-review"}, + {ReviewWatchID: watch.ID, RepoOwner: "acme", RepoName: "widget", PRNumber: 4}, + {ReviewWatchID: watch.ID, RepoOwner: "acme", RepoName: "widget", PRNumber: 5}, + } + for _, row := range rows { + if err := store.CreateReviewPRTask(ctx, row); err != nil { + t.Fatalf("CreateReviewPRTask #%d: %v", row.PRNumber, err) + } + } + mockClient.AddPR(&PR{Number: 1, State: prStateMerged, RepoOwner: "acme", RepoName: "widget"}) + mockClient.AddPR(&PR{Number: 2, State: prStateMerged, RepoOwner: "acme", RepoName: "widget"}) + mockClient.AddPR(&PR{Number: 3, State: prStateMerged, RepoOwner: "acme", RepoName: "widget"}) + mockClient.AddPR(&PR{Number: 4, State: prStateMerged, RepoOwner: "acme", RepoName: "widget"}) + mockClient.AddPR(&PR{Number: 5, State: prStateOpen, RepoOwner: "acme", RepoName: "widget"}) + + recorder := &recordingTaskDeleter{} + svc.SetTaskDeleter(recorder) + + deleted, err := svc.CleanupMergedReviewTasks(ctx, watch) + if err != nil { + t.Fatalf("CleanupMergedReviewTasks: %v", err) + } + if deleted != 1 { + t.Fatalf("deleted = %d, want 1 active task", deleted) + } + if len(recorder.calls) != 1 || recorder.calls[0] != "active-review" { + t.Fatalf("DeleteTask calls = %v, want [active-review]", recorder.calls) + } + if client.prCalls != 3 { + t.Fatalf("GetPR calls = %d, want 3 eligible rows only", client.prCalls) + } + if client.reviewCalls != 1 { + t.Fatalf("ListPRReviews calls = %d, want 1 for the open eligible row", client.reviewCalls) + } + + remaining, err := store.ListReviewPRTaskIDsByWatch(ctx, watch.ID) + if err != nil { + t.Fatalf("ListReviewPRTaskIDsByWatch: %v", err) + } + got := make(map[string]bool, len(remaining)) + for _, taskID := range remaining { + got[taskID] = true + } + want := map[string]bool{"archived-review": true, "missing-review": true, "": true} + if len(got) != len(want) || !got["archived-review"] || !got["missing-review"] || !got[""] { + t.Fatalf("remaining task IDs = %#v, want historical rows and open empty reservation", got) + } +} + +type minimalCleanupClient struct { + NoopClient + prs map[int]*PR + reviews map[int][]PRReview + user string + getPRCalls int + listReviewCalls int + userCalls int + feedbackCalls int + commentCalls int + checkCalls int + workflowRunCalls int + workflowJobCalls int +} + +func (c *minimalCleanupClient) GetPR(_ context.Context, owner, repo string, number int) (*PR, error) { + c.getPRCalls++ + pr, ok := c.prs[number] + if !ok { + return nil, errors.New("unexpected PR lookup") + } + copy := *pr + copy.RepoOwner = owner + copy.RepoName = repo + return ©, nil +} + +func (c *minimalCleanupClient) ListPRReviews(_ context.Context, _, _ string, number int) ([]PRReview, error) { + c.listReviewCalls++ + return append([]PRReview(nil), c.reviews[number]...), nil +} + +func (c *minimalCleanupClient) GetAuthenticatedUser(context.Context) (string, error) { + c.userCalls++ + return c.user, nil +} + +func (c *minimalCleanupClient) GetPRFeedback(context.Context, string, string, int) (*PRFeedback, error) { + c.feedbackCalls++ + return nil, errors.New("cleanup must not fetch PR feedback") +} + +func (c *minimalCleanupClient) ListPRComments(context.Context, string, string, int, *time.Time) ([]PRComment, error) { + c.commentCalls++ + return nil, errors.New("cleanup must not fetch PR comments") +} + +func (c *minimalCleanupClient) ListCheckRuns(context.Context, string, string, string) ([]CheckRun, error) { + c.checkCalls++ + return nil, errors.New("cleanup must not fetch check runs") +} + +func (c *minimalCleanupClient) ListWorkflowRuns(context.Context, string, string, string) ([]WorkflowRun, error) { + c.workflowRunCalls++ + return nil, errors.New("cleanup must not fetch workflow runs") +} + +func (c *minimalCleanupClient) ListWorkflowRunJobs(context.Context, string, string, int64, int) ([]WorkflowJob, error) { + c.workflowJobCalls++ + return nil, errors.New("cleanup must not fetch workflow jobs") +} + +func TestCleanupUsesMinimalPRReads(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ctx := context.Background() + client := &minimalCleanupClient{ + prs: map[int]*PR{ + 1: {Number: 1, State: prStateMerged}, + 2: {Number: 2, State: prStateClosed}, + 3: {Number: 3, State: prStateOpen}, + 4: {Number: 4, State: prStateOpen}, + }, + reviews: map[int][]PRReview{ + 3: {{State: "APPROVED", Author: "alice"}}, + 4: {{State: "COMMENTED", Author: "reviewer"}}, + }, + user: "alice", + } + configureTestWorkspaceAuth(t, svc, client, "ws-1") + watch := &ReviewWatch{WorkspaceID: "ws-1", Enabled: true, CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(ctx, watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + for number := 1; number <= 4; number++ { + taskID := fmt.Sprintf("cleanup-task-%d", number) + seedTask(t, store, taskID, false) + if err := store.CreateReviewPRTask(ctx, &ReviewPRTask{ + ReviewWatchID: watch.ID, + RepoOwner: "acme", + RepoName: "widget", + PRNumber: number, + TaskID: taskID, + }); err != nil { + t.Fatalf("CreateReviewPRTask #%d: %v", number, err) + } + } + recorder := &recordingTaskDeleter{} + svc.SetTaskDeleter(recorder) + + deleted, err := svc.CleanupMergedReviewTasks(ctx, watch) + if err != nil || deleted != 3 { + t.Fatalf("cleanup = %d, %v; want three terminal or approved tasks", deleted, err) + } + if client.getPRCalls != 4 { + t.Fatalf("GetPR calls = %d, want 4", client.getPRCalls) + } + if client.listReviewCalls != 2 { + t.Fatalf("ListPRReviews calls = %d, want 2 for open PRs", client.listReviewCalls) + } + if client.userCalls != 1 { + t.Fatalf("GetAuthenticatedUser calls = %d, want 1 for the approval candidate", client.userCalls) + } + if client.feedbackCalls != 0 || client.commentCalls != 0 || client.checkCalls != 0 || client.workflowRunCalls != 0 || client.workflowJobCalls != 0 { + t.Fatalf("unexpected cleanup reads: feedback=%d comments=%d checks=%d workflow_runs=%d workflow_jobs=%d", client.feedbackCalls, client.commentCalls, client.checkCalls, client.workflowRunCalls, client.workflowJobCalls) + } +} + // stubTaskDeleter implements TaskDeleter for testing. type stubTaskDeleter struct { err error @@ -15,9 +220,8 @@ func (s *stubTaskDeleter) DeleteTask(_ context.Context, _ string) error { return s.err } -// TestCleanupMergedReviewTasks_TaskAlreadyDeleted verifies that when DeleteTask -// returns "not found" the orphaned dedup record is still removed, preventing the -// 5-minute poller from logging the same warning indefinitely. +// TestCleanupMergedReviewTasks_TaskAlreadyDeleted verifies that a dedup row for +// a missing task is ignored without consuming provider quota. func TestCleanupMergedReviewTasks_TaskAlreadyDeleted(t *testing.T) { _, svc, mockClient, store := setupPollerTest(t) ctx := context.Background() @@ -50,8 +254,8 @@ func TestCleanupMergedReviewTasks_TaskAlreadyDeleted(t *testing.T) { RepoName: "widget", }) - // Stub: DeleteTask returns the sentinel-wrapped not-found error as the - // real adapter (see cmd/kandev/turn_adapters.go's taskDeleterAdapter) does. + // A missing task is not a cleanup candidate. The task deleter remains wired + // to prove the cleanup path does not try to delete it. svc.SetTaskDeleter(&stubTaskDeleter{ err: fmt.Errorf("%w: %s", ErrTaskNotFound, taskID), }) @@ -60,17 +264,66 @@ func TestCleanupMergedReviewTasks_TaskAlreadyDeleted(t *testing.T) { if err != nil { t.Fatalf("CleanupMergedReviewTasks returned error: %v", err) } - if deleted != 1 { - t.Errorf("expected 1 deleted, got %d", deleted) + if deleted != 0 { + t.Errorf("expected 0 deleted for a missing task, got %d", deleted) } - // The orphaned dedup record must be gone. - remaining, err := store.ListReviewPRTasksByWatch(ctx, watch.ID) + // The historical dedup record remains available for deduplication, but is + // excluded from cleanup candidate queries. + remaining, err := store.ListReviewPRTaskIDsByWatch(ctx, watch.ID) if err != nil { - t.Fatalf("ListReviewPRTasksByWatch: %v", err) + t.Fatalf("ListReviewPRTaskIDsByWatch: %v", err) + } + if len(remaining) != 1 || remaining[0] != taskID { + t.Errorf("remaining task IDs = %v, want [%s]", remaining, taskID) + } +} + +type callbackTaskDeleter func(context.Context, string) error + +func (f callbackTaskDeleter) DeleteTask(ctx context.Context, taskID string) error { + return f(ctx, taskID) +} + +func TestCleanupMergedReviewTasks_TaskDeletedAfterSelection(t *testing.T) { + _, svc, mockClient, store := setupPollerTest(t) + ctx := context.Background() + + watch := &ReviewWatch{WorkspaceID: "ws-1", Enabled: true} + if err := store.CreateReviewWatch(ctx, watch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + seedTask(t, store, "task-race", false) + if err := store.CreateReviewPRTask(ctx, &ReviewPRTask{ + ReviewWatchID: watch.ID, + RepoOwner: "acme", + RepoName: "widget", + PRNumber: 43, + TaskID: "task-race", + }); err != nil { + t.Fatalf("CreateReviewPRTask: %v", err) + } + mockClient.AddPR(&PR{Number: 43, State: prStateMerged, RepoOwner: "acme", RepoName: "widget"}) + svc.SetTaskDeleter(callbackTaskDeleter(func(ctx context.Context, taskID string) error { + if _, err := store.db.ExecContext(ctx, `DELETE FROM tasks WHERE id = ?`, taskID); err != nil { + t.Fatalf("delete selected task: %v", err) + } + return fmt.Errorf("%w: %s", ErrTaskNotFound, taskID) + })) + + deleted, err := svc.CleanupMergedReviewTasks(ctx, watch) + if err != nil { + t.Fatalf("CleanupMergedReviewTasks: %v", err) + } + if deleted != 1 { + t.Fatalf("deleted = %d, want 1 after selected task disappears", deleted) + } + remaining, err := store.ListReviewPRTaskIDsByWatch(ctx, watch.ID) + if err != nil { + t.Fatalf("ListReviewPRTaskIDsByWatch: %v", err) } if len(remaining) != 0 { - t.Errorf("expected 0 remaining dedup records, got %d", len(remaining)) + t.Fatalf("remaining task IDs = %v, want none", remaining) } } diff --git a/apps/backend/internal/github/service_review_reason_test.go b/apps/backend/internal/github/service_review_reason_test.go index 605c33d9af..6a965635b4 100644 --- a/apps/backend/internal/github/service_review_reason_test.go +++ b/apps/backend/internal/github/service_review_reason_test.go @@ -35,6 +35,7 @@ func TestCleanupMergedReviewTasks_ThreadsApprovedReason(t *testing.T) { if err := store.CreateReviewWatch(ctx, watch); err != nil { t.Fatalf("CreateReviewWatch: %v", err) } + seedTask(t, store, "task-approved", false) rpt := &ReviewPRTask{ ReviewWatchID: watch.ID, diff --git a/apps/backend/internal/github/service_reviews.go b/apps/backend/internal/github/service_reviews.go index c5be9665ba..45999d9056 100644 --- a/apps/backend/internal/github/service_reviews.go +++ b/apps/backend/internal/github/service_reviews.go @@ -215,20 +215,20 @@ func (s *Service) DeleteReviewWatch(ctx context.Context, id string) error { } } if s.taskDeleter != nil { - prTasks, err := s.store.ListReviewPRTasksByWatch(ctx, id) + taskIDs, err := s.store.ListReviewPRTaskIDsByWatch(ctx, id) if err != nil { s.logger.Warn("failed to list review PR tasks for pre-delete sweep", zap.String("watch_id", id), zap.Error(err)) } else { - for _, rpt := range prTasks { - if rpt.TaskID == "" { + for _, taskID := range taskIDs { + if taskID == "" { continue } - if err := s.taskDeleter.DeleteTask(ctx, rpt.TaskID); err != nil && + if err := s.taskDeleter.DeleteTask(ctx, taskID); err != nil && !isTaskNotFound(err) { s.logger.Warn("failed to delete review task during watch cleanup", zap.String("watch_id", id), - zap.String("task_id", rpt.TaskID), + zap.String("task_id", taskID), zap.Error(err)) } } diff --git a/apps/backend/internal/github/service_routing.go b/apps/backend/internal/github/service_routing.go index bf7de65ec3..e3ab531d08 100644 --- a/apps/backend/internal/github/service_routing.go +++ b/apps/backend/internal/github/service_routing.go @@ -16,6 +16,7 @@ type resolvedServiceClient struct { Principal AuthPrincipal CacheScope string Capabilities map[GitHubAppCapability]bool + RateTracker *RateTracker credential *ResolvedCredential } @@ -77,6 +78,7 @@ func (s *Service) resolveServiceClient( Principal: resolved.Principal, CacheScope: credentialCacheScope(resolved, purpose), Capabilities: resolved.Capabilities, + RateTracker: resolved.RateTracker, credential: resolved, }, nil } diff --git a/apps/backend/internal/github/service_search_cache_test.go b/apps/backend/internal/github/service_search_cache_test.go index 81124ad89f..7e8f4351f3 100644 --- a/apps/backend/internal/github/service_search_cache_test.go +++ b/apps/backend/internal/github/service_search_cache_test.go @@ -32,6 +32,8 @@ func newTestService(client Client) *Service { searchCache: newTTLCache(), prStatusCache: newTTLCache(), prFeedbackCache: newPRFeedbackCache(), + workflowRunsCache: newWorkflowAttentionCache(), + workflowJobsCache: newWorkflowAttentionCache(), accessibleReposCache: newAccessibleReposCache(), repoErrorCache: newRepoErrorCache(), } diff --git a/apps/backend/internal/github/service_test.go b/apps/backend/internal/github/service_test.go index 19d207b59c..de24249a1b 100644 --- a/apps/backend/internal/github/service_test.go +++ b/apps/backend/internal/github/service_test.go @@ -815,6 +815,14 @@ func setupSyncTest(t *testing.T) (*Service, *Store, *mockEventBus) { rawDB.SetMaxIdleConns(1) sqlxDB := sqlx.NewDb(rawDB, "sqlite3") t.Cleanup(func() { _ = sqlxDB.Close() }) + if _, err := sqlxDB.Exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + workspace_id TEXT, + archived_at DATETIME + )`); err != nil { + t.Fatalf("create tasks table: %v", err) + } store, err := NewStore(sqlxDB, sqlxDB) if err != nil { diff --git a/apps/backend/internal/github/store.go b/apps/backend/internal/github/store.go index c67faa48bf..a5003404d2 100644 --- a/apps/backend/internal/github/store.go +++ b/apps/backend/internal/github/store.go @@ -4912,8 +4912,13 @@ func (s *Store) ReleaseReviewPRTask(ctx context.Context, reviewWatchID, repoOwne func (s *Store) ListReviewPRTasksByWatch(ctx context.Context, watchID string) ([]*ReviewPRTask, error) { var tasks []*ReviewPRTask err := s.ro.SelectContext(ctx, &tasks, s.ro.Rebind( - `SELECT id, review_watch_id, repo_owner, repo_name, pr_number, pr_url, task_id, created_at - FROM github_review_pr_tasks WHERE review_watch_id = ?`), watchID) + `SELECT d.id, d.review_watch_id, d.repo_owner, d.repo_name, d.pr_number, d.pr_url, d.task_id, d.created_at + FROM github_review_pr_tasks d + WHERE d.review_watch_id = ? + AND (d.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks t + WHERE t.id = d.task_id AND t.archived_at IS NULL + ))`), watchID) return tasks, err } @@ -4923,8 +4928,12 @@ func (s *Store) ListReviewPRTasksByWatch(ctx context.Context, watchID string) ([ func (s *Store) ListAllReviewPRTasks(ctx context.Context) ([]*ReviewPRTask, error) { var tasks []*ReviewPRTask err := s.ro.SelectContext(ctx, &tasks, - `SELECT id, review_watch_id, repo_owner, repo_name, pr_number, pr_url, task_id, created_at - FROM github_review_pr_tasks`) + `SELECT d.id, d.review_watch_id, d.repo_owner, d.repo_name, d.pr_number, d.pr_url, d.task_id, d.created_at + FROM github_review_pr_tasks d + WHERE d.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks t + WHERE t.id = d.task_id AND t.archived_at IS NULL + )`) return tasks, err } @@ -5307,8 +5316,13 @@ func (s *Store) HasIssueWatchTask(ctx context.Context, issueWatchID, repoOwner, func (s *Store) ListIssueWatchTasksByWatch(ctx context.Context, watchID string) ([]*IssueWatchTask, error) { var tasks []*IssueWatchTask err := s.ro.SelectContext(ctx, &tasks, s.ro.Rebind( - `SELECT id, issue_watch_id, repo_owner, repo_name, issue_number, issue_url, task_id, created_at - FROM github_issue_watch_tasks WHERE issue_watch_id = ?`), watchID) + `SELECT d.id, d.issue_watch_id, d.repo_owner, d.repo_name, d.issue_number, d.issue_url, d.task_id, d.created_at + FROM github_issue_watch_tasks d + WHERE d.issue_watch_id = ? + AND (d.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks t + WHERE t.id = d.task_id AND t.archived_at IS NULL + ))`), watchID) return tasks, err } @@ -5317,8 +5331,12 @@ func (s *Store) ListIssueWatchTasksByWatch(ctx context.Context, watchID string) func (s *Store) ListAllIssueWatchTasks(ctx context.Context) ([]*IssueWatchTask, error) { var tasks []*IssueWatchTask err := s.ro.SelectContext(ctx, &tasks, - `SELECT id, issue_watch_id, repo_owner, repo_name, issue_number, issue_url, task_id, created_at - FROM github_issue_watch_tasks`) + `SELECT d.id, d.issue_watch_id, d.repo_owner, d.repo_name, d.issue_number, d.issue_url, d.task_id, d.created_at + FROM github_issue_watch_tasks d + WHERE d.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks t + WHERE t.id = d.task_id AND t.archived_at IS NULL + )`) return tasks, err } diff --git a/apps/backend/internal/github/store_task_cleanup_test.go b/apps/backend/internal/github/store_task_cleanup_test.go index 5cf6d20afd..1cb64a2981 100644 --- a/apps/backend/internal/github/store_task_cleanup_test.go +++ b/apps/backend/internal/github/store_task_cleanup_test.go @@ -2,6 +2,7 @@ package github import ( "context" + "fmt" "testing" ) @@ -34,6 +35,96 @@ func TestDeleteTaskPRsByTaskIDRemovesOnlyTargetRows(t *testing.T) { } } +func TestCleanupCandidateListsExcludeHistoricalTasks(t *testing.T) { + _, _, _, store := setupPollerTest(t) + ctx := context.Background() + + reviewWatch := &ReviewWatch{WorkspaceID: testWorkspaceID, Enabled: true} + if err := store.CreateReviewWatch(ctx, reviewWatch); err != nil { + t.Fatalf("CreateReviewWatch: %v", err) + } + issueWatch := &IssueWatch{WorkspaceID: testWorkspaceID, Enabled: true} + if err := store.CreateIssueWatch(ctx, issueWatch); err != nil { + t.Fatalf("CreateIssueWatch: %v", err) + } + + seedTask(t, store, "active-review", false) + seedTask(t, store, "archived-review", true) + seedTask(t, store, "active-issue", false) + seedTask(t, store, "archived-issue", true) + + for _, taskID := range []string{"active-review", "archived-review", "missing-review", ""} { + if err := store.CreateReviewPRTask(ctx, &ReviewPRTask{ + ReviewWatchID: reviewWatch.ID, + RepoOwner: "acme", + RepoName: "widget", + PRNumber: len(taskID) + 1, + TaskID: taskID, + }); err != nil { + t.Fatalf("CreateReviewPRTask(%q): %v", taskID, err) + } + } + for _, taskID := range []string{"active-issue", "archived-issue", "missing-issue", ""} { + issueNumber := len(taskID) + 1 + reserved, err := store.ReserveIssueWatchTask( + ctx, issueWatch.ID, "acme", "widget", issueNumber, + fmt.Sprintf("https://github.com/acme/widget/issues/%d", issueNumber), + ) + if err != nil || !reserved { + t.Fatalf("ReserveIssueWatchTask(%q): reserved=%v err=%v", taskID, reserved, err) + } + if taskID != "" { + if err := store.AssignIssueWatchTaskID(ctx, issueWatch.ID, "acme", "widget", issueNumber, taskID); err != nil { + t.Fatalf("AssignIssueWatchTaskID(%q): %v", taskID, err) + } + } + } + + assertReviewCandidates := func(name string, candidates []*ReviewPRTask) { + t.Helper() + got := make(map[string]bool, len(candidates)) + for _, candidate := range candidates { + got[candidate.TaskID] = true + } + want := map[string]bool{"active-review": true, "": true} + if len(got) != len(want) || !got["active-review"] || !got[""] { + t.Errorf("%s task IDs = %#v, want active-review and empty reservation", name, got) + } + } + assertIssueCandidates := func(name string, candidates []*IssueWatchTask) { + t.Helper() + got := make(map[string]bool, len(candidates)) + for _, candidate := range candidates { + got[candidate.TaskID] = true + } + if len(got) != 2 || !got["active-issue"] || !got[""] { + t.Errorf("%s task IDs = %#v, want active-issue and empty reservation", name, got) + } + } + + byWatchReview, err := store.ListReviewPRTasksByWatch(ctx, reviewWatch.ID) + if err != nil { + t.Fatalf("ListReviewPRTasksByWatch: %v", err) + } + assertReviewCandidates("review by-watch", byWatchReview) + allReview, err := store.ListAllReviewPRTasks(ctx) + if err != nil { + t.Fatalf("ListAllReviewPRTasks: %v", err) + } + assertReviewCandidates("review global", allReview) + + byWatchIssue, err := store.ListIssueWatchTasksByWatch(ctx, issueWatch.ID) + if err != nil { + t.Fatalf("ListIssueWatchTasksByWatch: %v", err) + } + assertIssueCandidates("issue by-watch", byWatchIssue) + allIssue, err := store.ListAllIssueWatchTasks(ctx) + if err != nil { + t.Fatalf("ListAllIssueWatchTasks: %v", err) + } + assertIssueCandidates("issue global", allIssue) +} + // TestHealTaskOwnedOrphansOnStoreInit covers AC5/AC6: store // initialization deletes github_task_prs/github_pr_watches rows whose task_id // has no matching tasks.id, preserves rows for active and archived tasks, and diff --git a/apps/backend/internal/github/ttl_cache.go b/apps/backend/internal/github/ttl_cache.go index b5fdddf397..7eddde83ab 100644 --- a/apps/backend/internal/github/ttl_cache.go +++ b/apps/backend/internal/github/ttl_cache.go @@ -133,12 +133,16 @@ func (c *ttlCache) generation() uint64 { } func (c *ttlCache) set(key string, value any) { + c.setWithTTL(key, value, c.ttl) +} + +func (c *ttlCache) setWithTTL(key string, value any, ttl time.Duration) { c.mu.Lock() defer c.mu.Unlock() if len(c.entries) >= c.maxSize { c.evictLocked() } - c.entries[key] = ttlEntry{value: value, expiresAt: c.now().Add(c.ttl)} + c.entries[key] = ttlEntry{value: value, expiresAt: c.now().Add(ttl)} } // setIfCurrentGeneration writes `value` for `key` only if the cache's @@ -185,6 +189,12 @@ func (c *ttlCache) endFill(key string, gen uint64) { } func (c *ttlCache) setIfCurrentVersions(key string, value any, gen, keyEpoch uint64) { + c.setIfCurrentVersionsWithTTL(key, value, gen, keyEpoch, c.ttl) +} + +func (c *ttlCache) setIfCurrentVersionsWithTTL( + key string, value any, gen, keyEpoch uint64, ttl time.Duration, +) { c.mu.Lock() defer c.mu.Unlock() state, ok := c.keyEpochs[key] @@ -194,7 +204,7 @@ func (c *ttlCache) setIfCurrentVersions(key string, value any, gen, keyEpoch uin if len(c.entries) >= c.maxSize { c.evictLocked() } - c.entries[key] = ttlEntry{value: value, expiresAt: c.now().Add(c.ttl)} + c.entries[key] = ttlEntry{value: value, expiresAt: c.now().Add(ttl)} } // evictLocked first drops expired entries; if still over the cap, drops the @@ -258,6 +268,25 @@ func (c *ttlCache) invalidateKey(key string) { c.keyEpochs[key] = state } +// invalidatePrefix removes a bounded family of entries while preserving +// unrelated fills. It is used when a refresh knows the repository and +// credential scope but not the head SHA or run IDs that are currently cached. +func (c *ttlCache) invalidatePrefix(prefix string) { + c.mu.Lock() + defer c.mu.Unlock() + for key := range c.entries { + if strings.HasPrefix(key, prefix) { + delete(c.entries, key) + } + } + for key, state := range c.keyEpochs { + if strings.HasPrefix(key, prefix) { + state.epoch++ + c.keyEpochs[key] = state + } + } +} + // cachedErr wraps an error stored in a ttlCache so that a deterministic // upstream failure (e.g. ErrRepoNotResolvable) can be negative-cached // alongside the cache's normal success-value entries. Currently used by @@ -274,6 +303,16 @@ type cachedErr struct{ err error } // caller still receives the freshly-fetched value but the cache stays empty // (the next ensure() will re-fetch under the new generation). func (c *ttlCache) doOrFetch(key string, fetch func() (any, error)) (any, error) { + return c.doOrFetchWithTTL(key, fetch, func(any) time.Duration { return c.ttl }) +} + +// doOrFetchWithTTL is doOrFetch with a result-dependent expiry. The fetch is +// still singleflighted and errors are still omitted from the cache. The TTL +// selector runs only for a successful result, so a provider failure cannot +// accidentally become a short-lived cached absence. +func (c *ttlCache) doOrFetchWithTTL( + key string, fetch func() (any, error), ttlFor func(any) time.Duration, +) (any, error) { if v, ok := c.get(key); ok { return v, nil } @@ -291,7 +330,11 @@ func (c *ttlCache) doOrFetch(key string, fetch func() (any, error)) (any, error) if err != nil { return nil, err } - c.setIfCurrentVersions(key, v, gen, keyEpoch) + ttl := c.ttl + if ttlFor != nil { + ttl = ttlFor(v) + } + c.setIfCurrentVersionsWithTTL(key, v, gen, keyEpoch, ttl) return v, nil }) return v, err diff --git a/apps/backend/internal/github/workflow_attention.go b/apps/backend/internal/github/workflow_attention.go index 1382f73609..b1fff0033e 100644 --- a/apps/backend/internal/github/workflow_attention.go +++ b/apps/backend/internal/github/workflow_attention.go @@ -30,6 +30,19 @@ type workflowJobKey struct { Attempt int } +type workflowAttentionCollector func(context.Context, Client, string, string, *PR) (*WorkflowAttention, error) + +type workflowAttentionCollectorContextKey struct{} + +func withWorkflowAttentionCollector(ctx context.Context, collector workflowAttentionCollector) context.Context { + return context.WithValue(ctx, workflowAttentionCollectorContextKey{}, collector) +} + +func workflowAttentionCollectorFromContext(ctx context.Context) workflowAttentionCollector { + collector, _ := ctx.Value(workflowAttentionCollectorContextKey{}).(workflowAttentionCollector) + return collector +} + // ghWorkflowRun and its nested values are shared by the gh CLI and PAT // clients because both consume the same Actions REST response shape. type ghWorkflowRun struct { @@ -177,6 +190,9 @@ func workflowAttentionUnknown(headSHA string) *WorkflowAttention { func collectWorkflowAttention( ctx context.Context, client Client, owner, repo string, pr *PR, ) (*WorkflowAttention, error) { + if collector := workflowAttentionCollectorFromContext(ctx); collector != nil { + return collector(ctx, client, owner, repo, pr) + } if pr == nil { return workflowAttentionUnknown(""), nil } diff --git a/apps/backend/internal/github/workflow_attention_cache.go b/apps/backend/internal/github/workflow_attention_cache.go new file mode 100644 index 0000000000..1ecdfe8f73 --- /dev/null +++ b/apps/backend/internal/github/workflow_attention_cache.go @@ -0,0 +1,203 @@ +package github + +import ( + "context" + "fmt" + "strings" + "time" +) + +const ( + workflowAttentionCacheMaxEntries = 512 + workflowAttentionShortTTL = 30 * time.Second + workflowAttentionLongTTL = 5 * time.Minute +) + +func newWorkflowAttentionCache() *ttlCache { + c := newTTLCache() + c.ttl = workflowAttentionShortTTL + c.maxSize = workflowAttentionCacheMaxEntries + return c +} + +func workflowRunsCacheKey(cacheScope, owner, repo, headSHA string) string { + return scopedCacheKey(cacheScope, fmt.Sprintf( + "workflow-runs:%s/%s@%s", strings.ToLower(owner), strings.ToLower(repo), strings.ToLower(headSHA), + )) +} + +func workflowJobsCacheKey(cacheScope, owner, repo string, runID int64, attempt int) string { + return scopedCacheKey(cacheScope, fmt.Sprintf( + "workflow-jobs:%s/%s#%d/%d", strings.ToLower(owner), strings.ToLower(repo), runID, attempt, + )) +} + +func workflowRunsCachePrefix(cacheScope, owner, repo string) string { + return scopedCacheKey(cacheScope, fmt.Sprintf( + "workflow-runs:%s/%s@", strings.ToLower(owner), strings.ToLower(repo), + )) +} + +func workflowJobsCachePrefix(cacheScope, owner, repo string) string { + return scopedCacheKey(cacheScope, fmt.Sprintf( + "workflow-jobs:%s/%s#", strings.ToLower(owner), strings.ToLower(repo), + )) +} + +func workflowRunsCacheTTL(value any) time.Duration { + runs, ok := value.([]WorkflowRun) + if !ok || len(runs) == 0 { + return workflowAttentionShortTTL + } + for _, run := range runs { + if !strings.EqualFold(run.Status, workflowStatusCompleted) || + run.Conclusion == "" || strings.EqualFold(run.Conclusion, workflowConclusionActionRequired) { + return workflowAttentionShortTTL + } + } + return workflowAttentionLongTTL +} + +func workflowJobsCacheTTL(value any) time.Duration { + jobs, ok := value.([]WorkflowJob) + if !ok || len(jobs) == 0 { + return workflowAttentionShortTTL + } + for _, job := range jobs { + if !strings.EqualFold(job.Status, workflowStatusCompleted) || job.Conclusion == "" { + return workflowAttentionShortTTL + } + } + return workflowAttentionLongTTL +} + +func (s *Service) cachedWorkflowRuns( + ctx context.Context, client Client, cacheScope, owner, repo, headSHA string, +) ([]WorkflowRun, error) { + if s == nil || s.workflowRunsCache == nil { + return client.ListWorkflowRuns(ctx, owner, repo, headSHA) + } + key := workflowRunsCacheKey(cacheScope, owner, repo, headSHA) + value, err := s.workflowRunsCache.doOrFetchWithTTL(key, func() (any, error) { + return client.ListWorkflowRuns(ctx, owner, repo, headSHA) + }, workflowRunsCacheTTL) + if err != nil { + return nil, err + } + runs, ok := value.([]WorkflowRun) + if !ok { + return nil, fmt.Errorf("invalid cached workflow runs value") + } + return runs, nil +} + +func (s *Service) cachedWorkflowJobs( + ctx context.Context, + client Client, + cacheScope, owner, repo string, + runID int64, attempt int, shortLived bool, +) ([]WorkflowJob, error) { + if s == nil || s.workflowJobsCache == nil { + return client.ListWorkflowRunJobs(ctx, owner, repo, runID, attempt) + } + key := workflowJobsCacheKey(cacheScope, owner, repo, runID, attempt) + ttlFor := workflowJobsCacheTTL + if shortLived { + ttlFor = func(any) time.Duration { return workflowAttentionShortTTL } + } + value, err := s.workflowJobsCache.doOrFetchWithTTL(key, func() (any, error) { + return client.ListWorkflowRunJobs(ctx, owner, repo, runID, attempt) + }, ttlFor) + if err != nil { + return nil, err + } + jobs, ok := value.([]WorkflowJob) + if !ok { + return nil, fmt.Errorf("invalid cached workflow jobs value") + } + return jobs, nil +} + +func (s *Service) collectWorkflowAttention( + ctx context.Context, client Client, cacheScope, owner, repo string, pr *PR, +) (*WorkflowAttention, error) { + if pr == nil { + return workflowAttentionUnknown(""), nil + } + if isTerminalPR(pr) { + return workflowAttentionNone(pr.HeadSHA), nil + } + if pr.HeadSHA == "" { + return workflowAttentionUnknown(""), nil + } + + runs, err := s.cachedWorkflowRuns(ctx, client, cacheScope, owner, repo, pr.HeadSHA) + if err != nil { + return workflowAttentionUnknown(pr.HeadSHA), err + } + return classifyWorkflowAttentionWithJobs(ctx, owner, repo, pr, runs, + func(jobCtx context.Context, runID int64, attempt int) ([]WorkflowJob, error) { + return s.cachedWorkflowJobs(jobCtx, client, cacheScope, owner, repo, runID, attempt, true) + }, + ), nil +} + +func (s *Service) invalidateWorkflowAttentionForPR( + cacheScope, owner, repo string, number int, headSHA string, +) { + if s == nil { + return + } + key := scopedCacheKey(cacheScope, prStatusCacheKey(owner, repo, number)) + if number > 0 { + if s.prStatusCache != nil { + s.prStatusCache.invalidateKey(key) + } + if s.prFeedbackCache != nil { + s.prFeedbackCache.invalidateKey(key) + } + } + if s.workflowRunsCache != nil { + if headSHA == "" { + s.workflowRunsCache.invalidatePrefix(workflowRunsCachePrefix(cacheScope, owner, repo)) + } else { + s.workflowRunsCache.invalidateKey(workflowRunsCacheKey(cacheScope, owner, repo, headSHA)) + } + } + if s.workflowJobsCache != nil { + s.workflowJobsCache.invalidatePrefix(workflowJobsCachePrefix(cacheScope, owner, repo)) + } +} + +// invalidateWorkflowAttentionForResolvedPR clears the automation namespace +// and the personal-read namespace for the same resolved credential. The +// status synchronizer uses the former, while the feedback endpoint uses the +// latter, so an explicit refresh must invalidate both views together. +func (s *Service) invalidateWorkflowAttentionForResolvedPR( + resolved *resolvedServiceClient, owner, repo string, number int, headSHA string, +) { + if resolved == nil { + return + } + s.invalidateWorkflowAttentionForPR(resolved.CacheScope, owner, repo, number, headSHA) + personalScope := resolved.cacheScopeForPurpose(CredentialPurposePersonalRead) + if personalScope != "" && personalScope != resolved.CacheScope { + s.invalidateWorkflowAttentionForPR(personalScope, owner, repo, number, headSHA) + } +} + +func (s *Service) invalidateWorkflowAttentionForResolvedWatches( + resolved *resolvedServiceClient, watches []*PRWatch, +) { + if resolved == nil { + return + } + for _, watch := range watches { + if watch == nil { + continue + } + s.invalidateWorkflowAttentionForResolvedPR( + resolved, watch.Owner, watch.Repo, watch.PRNumber, "", + ) + } +} diff --git a/apps/backend/internal/github/workflow_attention_cache_test.go b/apps/backend/internal/github/workflow_attention_cache_test.go new file mode 100644 index 0000000000..d770ff67db --- /dev/null +++ b/apps/backend/internal/github/workflow_attention_cache_test.go @@ -0,0 +1,305 @@ +package github + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" +) + +type workflowAttentionCacheClient struct { + *MockClient + runCalls atomic.Int32 + jobCalls atomic.Int32 + runsErr error + jobsErr error + started chan struct{} + release chan struct{} +} + +func (c *workflowAttentionCacheClient) ListWorkflowRuns( + ctx context.Context, owner, repo, headSHA string, +) ([]WorkflowRun, error) { + c.runCalls.Add(1) + if c.started != nil { + select { + case c.started <- struct{}{}: + default: + } + } + if c.release != nil { + <-c.release + } + if c.runsErr != nil { + return nil, c.runsErr + } + return c.MockClient.ListWorkflowRuns(ctx, owner, repo, headSHA) +} + +func (c *workflowAttentionCacheClient) ListWorkflowRunJobs( + ctx context.Context, owner, repo string, runID int64, attempt int, +) ([]WorkflowJob, error) { + c.jobCalls.Add(1) + if c.jobsErr != nil { + return nil, c.jobsErr + } + return c.MockClient.ListWorkflowRunJobs(ctx, owner, repo, runID, attempt) +} + +func (c *workflowAttentionCacheClient) GetPRStatus( + ctx context.Context, owner, repo string, number int, +) (*PRStatus, error) { + return getPRStatus(ctx, c, owner, repo, number) +} + +func (c *workflowAttentionCacheClient) GetPRFeedback( + ctx context.Context, owner, repo string, number int, +) (*PRFeedback, error) { + return getPRFeedback(ctx, c, owner, repo, number) +} + +func TestWorkflowAttentionCacheExpiryAndInvalidation(t *testing.T) { + ctx := context.Background() + clock := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) + client := &workflowAttentionCacheClient{MockClient: NewMockClient()} + svc := newTestService(client) + setWorkflowAttentionCacheClock(svc, func() time.Time { return clock }) + + prOne := &PR{ + Number: 1, State: "open", HeadSHA: "same-sha", HeadBranch: "feature", + RepoOwner: "acme", RepoName: "widget", HeadRepoOwner: "acme", HeadRepoName: "widget", + } + prTwo := *prOne + prTwo.Number = 2 + client.AddPR(prOne) + client.AddPR(&prTwo) + client.ReplaceWorkflowRuns("acme", "widget", "same-sha", []WorkflowRun{{ + ID: 11, RunAttempt: 1, WorkflowID: 7, Name: "CI", Event: "pull_request", + Status: workflowStatusCompleted, Conclusion: "success", HeadSHA: "same-sha", + HeadBranch: "feature", HeadRepoOwner: "acme", HeadRepoName: "widget", + CreatedAt: clock, UpdatedAt: clock, + }}) + + if _, err := svc.collectWorkflowAttention(ctx, client, "scope-a", "acme", "widget", prOne); err != nil { + t.Fatalf("first workflow read: %v", err) + } + if _, err := svc.collectWorkflowAttention(ctx, client, "scope-a", "acme", "widget", &prTwo); err != nil { + t.Fatalf("same-SHA second PR workflow read: %v", err) + } + if got := client.runCalls.Load(); got != 1 { + t.Fatalf("same-SHA PRs used %d run reads, want 1", got) + } + + // Service status reads use the same Actions observation as the batch path. + svc.prStatusCache.clear() + if _, err := svc.getPRStatus(ctx, client, "scope-a", "acme", "widget", prOne.Number); err != nil { + t.Fatalf("service status read: %v", err) + } + if got := client.runCalls.Load(); got != 1 { + t.Fatalf("service status reused no Actions cache: run reads = %d, want 1", got) + } + svc.prFeedbackCache.clear() + if _, err := svc.getPRFeedback(ctx, client, "scope-a", "", "acme", "widget", prOne.Number); err != nil { + t.Fatalf("service feedback read: %v", err) + } + if got := client.runCalls.Load(); got != 1 { + t.Fatalf("service feedback reused no Actions cache: run reads = %d, want 1", got) + } + + // A completed collection is retained for five minutes. A same-SHA rerun is + // visible after that expiry and the next successful read. + clock = clock.Add(5*time.Minute + time.Nanosecond) + client.ReplaceWorkflowRuns("acme", "widget", "same-sha", []WorkflowRun{{ + ID: 12, RunAttempt: 2, WorkflowID: 7, Name: "CI", Event: "pull_request", + Status: workflowStatusCompleted, Conclusion: "success", HeadSHA: "same-sha", + HeadBranch: "feature", HeadRepoOwner: "acme", HeadRepoName: "widget", + CreatedAt: clock, UpdatedAt: clock, + }}) + svc.prStatusCache.clear() + if _, err := svc.getPRStatus(ctx, client, "scope-a", "acme", "widget", prOne.Number); err != nil { + t.Fatalf("same-SHA rerun read: %v", err) + } + if got := client.runCalls.Load(); got != 2 { + t.Fatalf("same-SHA rerun remained cached: run reads = %d, want 2", got) + } + + // Pending observations use the short expiry. Replace the data after the + // short window and verify the completed result is read. + clock = clock.Add(time.Second) + client.ReplaceWorkflowRuns("acme", "widget", "same-sha", []WorkflowRun{{ + ID: 13, RunAttempt: 1, WorkflowID: 7, Name: "CI", Event: "pull_request", + Status: "in_progress", HeadSHA: "same-sha", HeadBranch: "feature", + HeadRepoOwner: "acme", HeadRepoName: "widget", CreatedAt: clock, UpdatedAt: clock, + }}) + svc.invalidateWorkflowAttentionForPR("scope-a", "acme", "widget", prOne.Number, "same-sha") + svc.prStatusCache.clear() + if _, err := svc.getPRStatus(ctx, client, "scope-a", "acme", "widget", prOne.Number); err != nil { + t.Fatalf("pending workflow read: %v", err) + } + clock = clock.Add(30*time.Second + time.Nanosecond) + client.ReplaceWorkflowRuns("acme", "widget", "same-sha", []WorkflowRun{{ + ID: 14, RunAttempt: 1, WorkflowID: 7, Name: "CI", Event: "pull_request", + Status: workflowStatusCompleted, Conclusion: "failure", HeadSHA: "same-sha", + HeadBranch: "feature", HeadRepoOwner: "acme", HeadRepoName: "widget", + CreatedAt: clock, UpdatedAt: clock, + }}) + svc.prStatusCache.clear() + if _, err := svc.getPRStatus(ctx, client, "scope-a", "acme", "widget", prOne.Number); err != nil { + t.Fatalf("expired pending workflow read: %v", err) + } + if got := client.runCalls.Load(); got != 4 { + t.Fatalf("short workflow expiry did not refetch: run reads = %d, want 4", got) + } + + // Jobs include the run attempt in their key. A later attempt cannot reuse + // an earlier attempt's observation. + client.ReplaceWorkflowRunJobs("acme", "widget", 99, 1, []WorkflowJob{{ + ID: 991, Status: workflowStatusCompleted, Conclusion: "success", + }}) + client.ReplaceWorkflowRunJobs("acme", "widget", 99, 2, []WorkflowJob{{ + ID: 992, Status: workflowStatusCompleted, Conclusion: "failure", + }}) + if _, err := svc.cachedWorkflowJobs(ctx, client, "scope-a", "acme", "widget", 99, 1, false); err != nil { + t.Fatalf("first job attempt: %v", err) + } + if _, err := svc.cachedWorkflowJobs(ctx, client, "scope-a", "acme", "widget", 99, 1, false); err != nil { + t.Fatalf("cached first job attempt: %v", err) + } + if _, err := svc.cachedWorkflowJobs(ctx, client, "scope-a", "acme", "widget", 99, 2, false); err != nil { + t.Fatalf("second job attempt: %v", err) + } + if got := client.jobCalls.Load(); got != 2 { + t.Fatalf("job attempts were not isolated: job reads = %d, want 2", got) + } + + // Errors are returned but never cached as an empty observation. + readErr := errors.New("actions unavailable") + client.runsErr = readErr + if _, err := svc.cachedWorkflowRuns(ctx, client, "scope-a", "acme", "widget", "error-sha"); !errors.Is(err, readErr) { + t.Fatalf("workflow read error = %v, want %v", err, readErr) + } + client.runsErr = nil + if _, err := svc.cachedWorkflowRuns(ctx, client, "scope-a", "acme", "widget", "error-sha"); err != nil { + t.Fatalf("workflow read after error: %v", err) + } + if got := client.runCalls.Load(); got != 6 { + t.Fatalf("workflow error was cached: run reads = %d, want 6", got) + } + + // Different credential generations use different cache namespaces. + if _, err := svc.cachedWorkflowRuns(ctx, client, "scope-b", "acme", "widget", "error-sha"); err != nil { + t.Fatalf("second credential scope: %v", err) + } + if got := client.runCalls.Load(); got != 7 { + t.Fatalf("credential scopes shared Actions data: run reads = %d, want 7", got) + } + + // Explicit refresh invalidates both the outer PR response and the Actions + // observations. The next status read therefore sees a fresh provider value. + outerKey := scopedCacheKey("scope-a", prStatusCacheKey("acme", "widget", prOne.Number)) + svc.prStatusCache.set(outerKey, "stale") + svc.invalidateWorkflowAttentionForPR("scope-a", "acme", "widget", prOne.Number, "error-sha") + if _, ok := svc.prStatusCache.get(outerKey); ok { + t.Fatal("explicit refresh left the outer status cache populated") + } + if _, err := svc.cachedWorkflowRuns(ctx, client, "scope-a", "acme", "widget", "error-sha"); err != nil { + t.Fatalf("workflow read after explicit refresh: %v", err) + } + if got := client.runCalls.Load(); got != 8 { + t.Fatalf("explicit refresh reused Actions data: run reads = %d, want 8", got) + } + + // Concurrent misses for one key coalesce, even when they arrive through + // separate service callers. + concurrent := newTestService(client) + setWorkflowAttentionCacheClock(concurrent, func() time.Time { return clock }) + client.started = make(chan struct{}, 1) + client.release = make(chan struct{}) + const callers = 12 + var ready sync.WaitGroup + var done sync.WaitGroup + ready.Add(callers) + done.Add(callers) + start := make(chan struct{}) + for range callers { + go func() { + defer done.Done() + ready.Done() + <-start + if _, err := concurrent.cachedWorkflowRuns(ctx, client, "scope-c", "acme", "widget", "concurrent-sha"); err != nil { + t.Errorf("concurrent workflow read: %v", err) + } + }() + } + ready.Wait() + close(start) + <-client.started + close(client.release) + done.Wait() + if got := client.runCalls.Load(); got != 9 { + t.Fatalf("concurrent misses did not coalesce: run reads = %d, want 9", got) + } + + // The run cache is bounded. Stagger fake time so eviction order is + // deterministic, then ensure the oldest entry is fetched again. + eviction := newTestService(client) + setWorkflowAttentionCacheClock(eviction, func() time.Time { return clock }) + for index := 0; index < workflowAttentionCacheMaxEntries; index++ { + if _, err := eviction.cachedWorkflowRuns(ctx, client, "scope-evict", "acme", "widget", "evict-"+itoa(index)); err != nil { + t.Fatalf("eviction fill %d: %v", index, err) + } + clock = clock.Add(time.Nanosecond) + } + before := client.runCalls.Load() + if _, err := eviction.cachedWorkflowRuns(ctx, client, "scope-evict", "acme", "widget", "evict-overflow"); err != nil { + t.Fatalf("eviction overflow: %v", err) + } + if _, err := eviction.cachedWorkflowRuns(ctx, client, "scope-evict", "acme", "widget", "evict-0"); err != nil { + t.Fatalf("evicted entry refill: %v", err) + } + if got := client.runCalls.Load(); got != before+2 { + t.Fatalf("bounded eviction kept oldest entry: reads before=%d after=%d", before, got) + } +} + +func TestWorkflowAttentionCacheInvalidationDropsInFlightObservation(t *testing.T) { + client := &workflowAttentionCacheClient{ + MockClient: NewMockClient(), + started: make(chan struct{}, 1), + release: make(chan struct{}), + } + svc := newTestService(client) + ctx := context.Background() + + var firstErr error + done := make(chan struct{}) + go func() { + _, firstErr = svc.cachedWorkflowRuns(ctx, client, "scope", "acme", "widget", "head") + close(done) + }() + <-client.started + svc.invalidateWorkflowAttentionForPR("scope", "acme", "widget", 1, "head") + close(client.release) + <-done + if firstErr != nil { + t.Fatalf("in-flight workflow read: %v", firstErr) + } + if _, err := svc.cachedWorkflowRuns(ctx, client, "scope", "acme", "widget", "head"); err != nil { + t.Fatalf("workflow read after invalidation: %v", err) + } + if got := client.runCalls.Load(); got != 2 { + t.Fatalf("in-flight result repopulated invalidated cache: run reads = %d, want 2", got) + } +} + +func setWorkflowAttentionCacheClock(svc *Service, now func() time.Time) { + svc.workflowRunsCache.now = now + svc.workflowJobsCache.now = now +} + +func itoa(value int) string { + return fmt.Sprintf("%d", value) +} diff --git a/apps/backend/internal/gitlab/controller_watch_scope_test.go b/apps/backend/internal/gitlab/controller_watch_scope_test.go index 22a4162027..9c1c36a350 100644 --- a/apps/backend/internal/gitlab/controller_watch_scope_test.go +++ b/apps/backend/internal/gitlab/controller_watch_scope_test.go @@ -160,6 +160,7 @@ func TestCleanupWatchRoutesOnlyDeleteRequestedWorkspaceTasks(t *testing.T) { iid int task string }{{reviewA, "a/project", 1, "review-a"}, {reviewB, "b/project", 2, "review-b"}} { + seedTask(t, store, row.task, row.watch.WorkspaceID) if _, err := store.ReserveReviewMRTask(t.Context(), row.watch.ID, row.path, row.iid, "url"); err != nil { t.Fatal(err) } @@ -173,6 +174,7 @@ func TestCleanupWatchRoutesOnlyDeleteRequestedWorkspaceTasks(t *testing.T) { iid int task string }{{issueA, "a/project", 3, "issue-a"}, {issueB, "b/project", 4, "issue-b"}} { + seedTask(t, store, row.task, row.watch.WorkspaceID) if _, err := store.ReserveIssueWatchTask(t.Context(), row.watch.ID, row.path, row.iid, "url"); err != nil { t.Fatal(err) } diff --git a/apps/backend/internal/gitlab/service_cleanup_reason_test.go b/apps/backend/internal/gitlab/service_cleanup_reason_test.go index 42871f9340..e773e4e743 100644 --- a/apps/backend/internal/gitlab/service_cleanup_reason_test.go +++ b/apps/backend/internal/gitlab/service_cleanup_reason_test.go @@ -109,6 +109,7 @@ func TestCleanupAllReviewTasks_ThreadsReason(t *testing.T) { if err := store.CreateReviewWatch(ctx, watch); err != nil { t.Fatalf("CreateReviewWatch: %v", err) } + seedTask(t, store, "task-merged", "ws-1") if _, err := store.ReserveReviewMRTask(ctx, watch.ID, project, iid, "url"); err != nil { t.Fatalf("ReserveReviewMRTask: %v", err) } @@ -147,6 +148,7 @@ func TestCleanupAllIssueTasks_ThreadsReason(t *testing.T) { if err := store.CreateIssueWatch(ctx, watch); err != nil { t.Fatalf("CreateIssueWatch: %v", err) } + seedTask(t, store, "task-closed", "ws-1") if _, err := store.ReserveIssueWatchTask(ctx, watch.ID, project, iid, "url"); err != nil { t.Fatalf("ReserveIssueWatchTask: %v", err) } diff --git a/apps/backend/internal/gitlab/service_cleanup_test.go b/apps/backend/internal/gitlab/service_cleanup_test.go new file mode 100644 index 0000000000..df009244e5 --- /dev/null +++ b/apps/backend/internal/gitlab/service_cleanup_test.go @@ -0,0 +1,110 @@ +package gitlab + +import ( + "context" + "testing" +) + +type countingGitLabCleanupClient struct { + *MockClient + mrStatusCalls int + issueStateCalls int +} + +func (c *countingGitLabCleanupClient) GetMRStatus(ctx context.Context, projectPath string, iid int) (*MRStatus, error) { + c.mrStatusCalls++ + return c.MockClient.GetMRStatus(ctx, projectPath, iid) +} + +func (c *countingGitLabCleanupClient) GetIssueState(ctx context.Context, projectPath string, iid int) (string, error) { + c.issueStateCalls++ + return c.MockClient.GetIssueState(ctx, projectPath, iid) +} + +type cleanupTaskRecorder struct { + taskIDs []string +} + +func (r *cleanupTaskRecorder) DeleteTask(_ context.Context, taskID string) error { + r.taskIDs = append(r.taskIDs, taskID) + return nil +} + +func TestCleanupSkipsHistoricalTasks(t *testing.T) { + store := newTestStore(t) + seedWorkspace(t, store, "ws-1") + + client := &countingGitLabCleanupClient{MockClient: NewMockClient(DefaultHost)} + client.SeedMR("team/review", &MR{IID: 1, State: gitlabStateMerged}) + client.SeedIssue("team/issue", &Issue{IID: 1, State: gitlabStateClosed}) + service := NewService(DefaultHost, client, AuthMethodNone, nil, newTestLogger(t)) + service.SetStore(store) + recorder := &cleanupTaskRecorder{} + service.SetTaskDeleter(recorder) + + reviewWatch := &ReviewWatch{ID: "review-cleanup-service", WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateReviewWatch(t.Context(), reviewWatch); err != nil { + t.Fatalf("create review watch: %v", err) + } + issueWatch := &IssueWatch{ID: "issue-cleanup-service", WorkspaceID: "ws-1", CleanupPolicy: CleanupPolicyAlways} + if err := store.CreateIssueWatch(t.Context(), issueWatch); err != nil { + t.Fatalf("create issue watch: %v", err) + } + for _, taskID := range []string{"review-active", "review-archived", "issue-active", "issue-archived"} { + seedTask(t, store, taskID, "ws-1") + } + for _, taskID := range []string{"review-archived", "issue-archived"} { + if _, err := store.db.Exec(`UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = ?`, taskID); err != nil { + t.Fatalf("archive task %s: %v", taskID, err) + } + } + + reserveReview := func(iid int, taskID string) { + t.Helper() + reserved, err := store.ReserveReviewMRTask(t.Context(), reviewWatch.ID, "team/review", iid, "https://gitlab.example/mr") + if err != nil || !reserved { + t.Fatalf("reserve review %d: reserved=%v err=%v", iid, reserved, err) + } + if taskID != "" { + if err := store.AssignReviewMRTaskID(t.Context(), reviewWatch.ID, "team/review", iid, taskID); err != nil { + t.Fatalf("assign review %d: %v", iid, err) + } + } + } + reserveIssue := func(iid int, taskID string) { + t.Helper() + reserved, err := store.ReserveIssueWatchTask(t.Context(), issueWatch.ID, "team/issue", iid, "https://gitlab.example/issue") + if err != nil || !reserved { + t.Fatalf("reserve issue %d: reserved=%v err=%v", iid, reserved, err) + } + if taskID != "" { + if err := store.AssignIssueWatchTaskID(t.Context(), issueWatch.ID, "team/issue", iid, taskID); err != nil { + t.Fatalf("assign issue %d: %v", iid, err) + } + } + } + reserveReview(1, "review-active") + reserveReview(2, "review-archived") + reserveReview(3, "review-missing") + reserveReview(4, "") + reserveIssue(1, "issue-active") + reserveIssue(2, "issue-archived") + reserveIssue(3, "issue-missing") + reserveIssue(4, "") + + if deleted, err := service.CleanupAllReviewTasks(t.Context()); err != nil || deleted != 1 { + t.Fatalf("review cleanup = %d, %v; want one deletion", deleted, err) + } + if deleted, err := service.CleanupAllIssueTasks(t.Context()); err != nil || deleted != 1 { + t.Fatalf("issue cleanup = %d, %v; want one deletion", deleted, err) + } + if client.mrStatusCalls != 1 { + t.Fatalf("review status calls = %d, want 1 for the active task", client.mrStatusCalls) + } + if client.issueStateCalls != 1 { + t.Fatalf("issue status calls = %d, want 1 for the active task", client.issueStateCalls) + } + if len(recorder.taskIDs) != 2 || recorder.taskIDs[0] != "review-active" || recorder.taskIDs[1] != "issue-active" { + t.Fatalf("deleted task IDs = %v, want active tasks only", recorder.taskIDs) + } +} diff --git a/apps/backend/internal/gitlab/store_task_cleanup_test.go b/apps/backend/internal/gitlab/store_task_cleanup_test.go index 46b0527911..bf4143a915 100644 --- a/apps/backend/internal/gitlab/store_task_cleanup_test.go +++ b/apps/backend/internal/gitlab/store_task_cleanup_test.go @@ -5,6 +5,141 @@ import ( "testing" ) +func TestCleanupCandidateListsExcludeHistoricalTasks(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + for _, workspaceID := range []string{"ws-1", "ws-2"} { + seedWorkspace(t, store, workspaceID) + } + + reviewWatch := &ReviewWatch{ID: "review-cleanup", WorkspaceID: "ws-1"} + if err := store.CreateReviewWatch(ctx, reviewWatch); err != nil { + t.Fatalf("create review watch: %v", err) + } + otherReviewWatch := &ReviewWatch{ID: "review-other", WorkspaceID: "ws-2"} + if err := store.CreateReviewWatch(ctx, otherReviewWatch); err != nil { + t.Fatalf("create other review watch: %v", err) + } + issueWatch := &IssueWatch{ID: "issue-cleanup", WorkspaceID: "ws-1"} + if err := store.CreateIssueWatch(ctx, issueWatch); err != nil { + t.Fatalf("create issue watch: %v", err) + } + otherIssueWatch := &IssueWatch{ID: "issue-other", WorkspaceID: "ws-2"} + if err := store.CreateIssueWatch(ctx, otherIssueWatch); err != nil { + t.Fatalf("create other issue watch: %v", err) + } + + for _, taskID := range []string{"review-active", "review-archived", "issue-active", "issue-archived", "other-review-active", "other-issue-active"} { + workspaceID := "ws-1" + if taskID == "other-review-active" || taskID == "other-issue-active" { + workspaceID = "ws-2" + } + seedTask(t, store, taskID, workspaceID) + } + for _, taskID := range []string{"review-archived", "issue-archived"} { + if _, err := store.db.Exec(`UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = ?`, taskID); err != nil { + t.Fatalf("archive task %s: %v", taskID, err) + } + } + + reserveReview := func(watchID, project string, iid int, taskID string) { + t.Helper() + reserved, err := store.ReserveReviewMRTask(ctx, watchID, project, iid, "https://gitlab.example/mr") + if err != nil || !reserved { + t.Fatalf("reserve review %s/%d: reserved=%v err=%v", project, iid, reserved, err) + } + if taskID != "" { + if err := store.AssignReviewMRTaskID(ctx, watchID, project, iid, taskID); err != nil { + t.Fatalf("assign review %s/%d: %v", project, iid, err) + } + } + } + reserveIssue := func(watchID, project string, iid int, taskID string) { + t.Helper() + reserved, err := store.ReserveIssueWatchTask(ctx, watchID, project, iid, "https://gitlab.example/issue") + if err != nil || !reserved { + t.Fatalf("reserve issue %s/%d: reserved=%v err=%v", project, iid, reserved, err) + } + if taskID != "" { + if err := store.AssignIssueWatchTaskID(ctx, watchID, project, iid, taskID); err != nil { + t.Fatalf("assign issue %s/%d: %v", project, iid, err) + } + } + } + reserveReview(reviewWatch.ID, "team/review", 1, "review-active") + reserveReview(reviewWatch.ID, "team/review", 2, "review-archived") + reserveReview(reviewWatch.ID, "team/review", 3, "review-missing") + reserveReview(reviewWatch.ID, "team/review", 4, "") + reserveReview(otherReviewWatch.ID, "team/other-review", 5, "other-review-active") + reserveIssue(issueWatch.ID, "team/issue", 1, "issue-active") + reserveIssue(issueWatch.ID, "team/issue", 2, "issue-archived") + reserveIssue(issueWatch.ID, "team/issue", 3, "issue-missing") + reserveIssue(issueWatch.ID, "team/issue", 4, "") + reserveIssue(otherIssueWatch.ID, "team/other-issue", 5, "other-issue-active") + + assertReviewTasks := func(name string, rows []*ReviewMRTask, want map[string]bool) { + t.Helper() + got := make(map[string]bool, len(rows)) + for _, row := range rows { + got[row.TaskID] = true + } + if len(rows) != len(want) { + t.Fatalf("%s rows = %d (%v), want %d (%v)", name, len(rows), got, len(want), want) + } + for taskID := range want { + if !got[taskID] { + t.Errorf("%s missing task %q in %v", name, taskID, got) + } + } + } + assertIssueTasks := func(name string, rows []*IssueWatchTask, want map[string]bool) { + t.Helper() + got := make(map[string]bool, len(rows)) + for _, row := range rows { + got[row.TaskID] = true + } + if len(rows) != len(want) { + t.Fatalf("%s rows = %d (%v), want %d (%v)", name, len(rows), got, len(want), want) + } + for taskID := range want { + if !got[taskID] { + t.Errorf("%s missing task %q in %v", name, taskID, got) + } + } + } + + reviewByWatch, err := store.ListReviewMRTasksByWatch(ctx, reviewWatch.ID) + if err != nil { + t.Fatalf("list review rows by watch: %v", err) + } + reviewWorkspace, err := store.ListReviewMRTasksForWorkspace(ctx, "ws-1") + if err != nil { + t.Fatalf("list review rows for workspace: %v", err) + } + reviewAll, err := store.ListAllReviewMRTasks(ctx) + if err != nil { + t.Fatalf("list all review rows: %v", err) + } + issueByWatch, err := store.ListIssueWatchTasksByWatch(ctx, issueWatch.ID) + if err != nil { + t.Fatalf("list issue rows by watch: %v", err) + } + issueWorkspace, err := store.ListIssueWatchTasksForWorkspace(ctx, "ws-1") + if err != nil { + t.Fatalf("list issue rows for workspace: %v", err) + } + issueAll, err := store.ListAllIssueWatchTasks(ctx) + if err != nil { + t.Fatalf("list all issue rows: %v", err) + } + assertReviewTasks("review by watch", reviewByWatch, map[string]bool{"review-active": true, "": true}) + assertReviewTasks("review workspace", reviewWorkspace, map[string]bool{"review-active": true, "": true}) + assertReviewTasks("review all", reviewAll, map[string]bool{"review-active": true, "": true, "other-review-active": true}) + assertIssueTasks("issue by watch", issueByWatch, map[string]bool{"issue-active": true, "": true}) + assertIssueTasks("issue workspace", issueWorkspace, map[string]bool{"issue-active": true, "": true}) + assertIssueTasks("issue all", issueAll, map[string]bool{"issue-active": true, "": true, "other-issue-active": true}) +} + func TestDeleteTaskMRsByTaskIDRemovesOnlyTargetRows(t *testing.T) { store := newTestStore(t) ctx := context.Background() diff --git a/apps/backend/internal/gitlab/store_watches.go b/apps/backend/internal/gitlab/store_watches.go index 41258d5e06..01da3f731b 100644 --- a/apps/backend/internal/gitlab/store_watches.go +++ b/apps/backend/internal/gitlab/store_watches.go @@ -518,8 +518,13 @@ func (s *Store) reviewWatchGeneration(ctx context.Context, watchID string) (int6 func (s *Store) ListReviewMRTasksByWatch(ctx context.Context, watchID string) ([]*ReviewMRTask, error) { var rows []ReviewMRTask if err := s.ro.SelectContext(ctx, &rows, s.ro.Rebind( - `SELECT `+reviewMRTaskSelectCols+` FROM gitlab_review_mr_tasks - WHERE review_watch_id = ? ORDER BY created_at ASC`), watchID); err != nil { + `SELECT `+reviewMRTaskSelectCols+` FROM gitlab_review_mr_tasks t + WHERE t.review_watch_id = ? + AND (t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + )) + ORDER BY t.created_at ASC`), watchID); err != nil { return nil, err } out := make([]*ReviewMRTask, 0, len(rows)) @@ -533,7 +538,12 @@ func (s *Store) ListReviewMRTasksByWatch(ctx context.Context, watchID string) ([ func (s *Store) ListAllReviewMRTasks(ctx context.Context) ([]*ReviewMRTask, error) { var rows []ReviewMRTask if err := s.ro.SelectContext(ctx, &rows, s.ro.Rebind( - `SELECT `+reviewMRTaskSelectCols+` FROM gitlab_review_mr_tasks ORDER BY created_at ASC`)); err != nil { + `SELECT `+reviewMRTaskSelectCols+` FROM gitlab_review_mr_tasks t + WHERE t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + ) + ORDER BY t.created_at ASC`)); err != nil { return nil, err } out := make([]*ReviewMRTask, 0, len(rows)) @@ -550,7 +560,12 @@ func (s *Store) ListReviewMRTasksForWorkspace(ctx context.Context, workspaceID s t.mr_url, t.task_id, t.generation, t.created_at FROM gitlab_review_mr_tasks t JOIN gitlab_review_watches w ON w.id = t.review_watch_id - WHERE w.workspace_id = ? ORDER BY t.created_at ASC`), workspaceID); err != nil { + WHERE w.workspace_id = ? + AND (t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + )) + ORDER BY t.created_at ASC`), workspaceID); err != nil { return nil, err } out := make([]*ReviewMRTask, 0, len(rows)) @@ -882,8 +897,13 @@ func (s *Store) issueWatchGeneration(ctx context.Context, watchID string) (int64 func (s *Store) ListIssueWatchTasksByWatch(ctx context.Context, watchID string) ([]*IssueWatchTask, error) { var rows []IssueWatchTask if err := s.ro.SelectContext(ctx, &rows, s.ro.Rebind( - `SELECT `+issueWatchTaskSelectCols+` FROM gitlab_issue_watch_tasks - WHERE issue_watch_id = ? ORDER BY created_at ASC`), watchID); err != nil { + `SELECT `+issueWatchTaskSelectCols+` FROM gitlab_issue_watch_tasks t + WHERE t.issue_watch_id = ? + AND (t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + )) + ORDER BY t.created_at ASC`), watchID); err != nil { return nil, err } out := make([]*IssueWatchTask, 0, len(rows)) @@ -897,7 +917,12 @@ func (s *Store) ListIssueWatchTasksByWatch(ctx context.Context, watchID string) func (s *Store) ListAllIssueWatchTasks(ctx context.Context) ([]*IssueWatchTask, error) { var rows []IssueWatchTask if err := s.ro.SelectContext(ctx, &rows, s.ro.Rebind( - `SELECT `+issueWatchTaskSelectCols+` FROM gitlab_issue_watch_tasks ORDER BY created_at ASC`)); err != nil { + `SELECT `+issueWatchTaskSelectCols+` FROM gitlab_issue_watch_tasks t + WHERE t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + ) + ORDER BY t.created_at ASC`)); err != nil { return nil, err } out := make([]*IssueWatchTask, 0, len(rows)) @@ -914,7 +939,12 @@ func (s *Store) ListIssueWatchTasksForWorkspace(ctx context.Context, workspaceID t.issue_url, t.task_id, t.generation, t.created_at FROM gitlab_issue_watch_tasks t JOIN gitlab_issue_watches w ON w.id = t.issue_watch_id - WHERE w.workspace_id = ? ORDER BY t.created_at ASC`), workspaceID); err != nil { + WHERE w.workspace_id = ? + AND (t.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks task_row + WHERE task_row.id = t.task_id AND task_row.archived_at IS NULL + )) + ORDER BY t.created_at ASC`), workspaceID); err != nil { return nil, err } out := make([]*IssueWatchTask, 0, len(rows)) diff --git a/apps/backend/internal/task/models/pr_watch_activity.go b/apps/backend/internal/task/models/pr_watch_activity.go new file mode 100644 index 0000000000..2515e7bda8 --- /dev/null +++ b/apps/backend/internal/task/models/pr_watch_activity.go @@ -0,0 +1,11 @@ +package models + +import "time" + +// PRWatchTaskActivity is the task-domain activity projection used by +// integration polling. It excludes generic task UpdatedAt values, which +// provider synchronization may update itself. +type PRWatchTaskActivity struct { + LastActivityAt time.Time + Running bool +} diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index 3c6f59eef3..6ad0dd5fcc 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -249,6 +249,12 @@ type TaskActivityRepository interface { LoadTaskLastActivity(ctx context.Context, taskIDs []string) (map[string]time.Time, error) } +// PRWatchTaskActivityRepository loads the bounded activity projection for a +// bulk set of task IDs. +type PRWatchTaskActivityRepository interface { + LoadPRWatchTaskActivity(ctx context.Context, taskIDs []string) (map[string]models.PRWatchTaskActivity, error) +} + // TaskRepoRepository handles the task↔repository junction table (models.TaskRepository rows). // Named TaskRepoRepository to reduce reader confusion with the TaskRepository sub-interface above. type TaskRepoRepository interface { diff --git a/apps/backend/internal/task/repository/sqlite/pr_watch_activity.go b/apps/backend/internal/task/repository/sqlite/pr_watch_activity.go new file mode 100644 index 0000000000..d450e43c29 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/pr_watch_activity.go @@ -0,0 +1,148 @@ +package sqlite + +import ( + "context" + "database/sql" + "fmt" + + "github.com/kandev/kandev/internal/task/models" +) + +// LoadPRWatchTaskActivity returns the bounded, task-local activity projection +// used by GitHub searching-watch admission. Provider and watch bookkeeping is +// deliberately absent from this query: those writers can update generic +// UpdatedAt columns without representing work on the branch. +func (r *Repository) LoadPRWatchTaskActivity( + ctx context.Context, taskIDs []string, +) (map[string]models.PRWatchTaskActivity, error) { + result := make(map[string]models.PRWatchTaskActivity, len(taskIDs)) + if len(taskIDs) == 0 { + return result, nil + } + + for _, chunk := range chunkIDs(taskIDs, sqliteMaxHostParams/7) { + placeholders, ids := buildInPlaceholders(chunk) + if err := r.loadPRWatchTaskActivityChunk(ctx, placeholders, ids, result); err != nil { + return nil, err + } + } + return result, nil +} + +func (r *Repository) loadPRWatchTaskActivityChunk( + ctx context.Context, + placeholders string, + ids []interface{}, + result map[string]models.PRWatchTaskActivity, +) error { + query := r.prWatchActivityQuery(placeholders) + args := make([]interface{}, 0, len(ids)*7+1) + for range 7 { + args = append(args, ids...) + } + args = append(args, TriggeredByLiveMonitor) + rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(query), args...) + if err != nil { + return fmt.Errorf("load PR watch task activity: %w", err) + } + if err := scanPRWatchActivityRows(rows, result); err != nil { + return err + } + + runningRows, err := r.ro.QueryContext(ctx, r.ro.Rebind(` + SELECT task_id + FROM task_sessions + WHERE task_id IN (`+placeholders+`) + AND state IN ('STARTING', 'RUNNING') + GROUP BY task_id`), ids...) + if err != nil { + return fmt.Errorf("load running PR watch tasks: %w", err) + } + defer func() { _ = runningRows.Close() }() + for runningRows.Next() { + var taskID string + if err := runningRows.Scan(&taskID); err != nil { + return fmt.Errorf("scan running PR watch task: %w", err) + } + activity := result[taskID] + activity.Running = true + result[taskID] = activity + } + if err := runningRows.Err(); err != nil { + return fmt.Errorf("read running PR watch tasks: %w", err) + } + return nil +} + +func (r *Repository) prWatchActivityQuery(placeholders string) string { + lifecycleOnlyPredicate := turnLifecycleOnlyPredicate(r.ro.DriverName(), "turn") + return ` + WITH activity AS ( + SELECT id AS task_id, created_at AS activity_at + FROM tasks + WHERE id IN (` + placeholders + `) + UNION ALL + SELECT task_id, created_at AS activity_at + FROM task_session_messages + WHERE task_id IN (` + placeholders + `) + AND author_type = 'user' + UNION ALL + SELECT task_id, queued_at AS activity_at + FROM queued_messages + WHERE task_id IN (` + placeholders + `) + AND queued_by NOT IN ('agent', 'workflow', 'server') + UNION ALL + SELECT task_id, started_at AS activity_at + FROM task_session_turns turn + WHERE task_id IN (` + placeholders + `) + AND started_at IS NOT NULL + AND NOT (` + lifecycleOnlyPredicate + `) + UNION ALL + SELECT task_id, completed_at AS activity_at + FROM task_session_turns turn + WHERE task_id IN (` + placeholders + `) + AND completed_at IS NOT NULL + AND NOT (` + lifecycleOnlyPredicate + `) + UNION ALL + SELECT sess.task_id, c.committed_at AS activity_at + FROM task_session_commits c + INNER JOIN task_sessions sess ON sess.id = c.session_id + WHERE sess.task_id IN (` + placeholders + `) + UNION ALL + SELECT sess.task_id, snap.created_at AS activity_at + FROM task_session_git_snapshots snap + INNER JOIN task_sessions sess ON sess.id = snap.session_id + WHERE sess.task_id IN (` + placeholders + `) + AND snap.triggered_by <> ? + AND (snap.head_commit <> '' OR snap.remote_branch <> '') + ) + SELECT task_id, MAX(activity_at) AS activity_at + FROM activity + GROUP BY task_id` + +} + +func scanPRWatchActivityRows( + rows *sql.Rows, + result map[string]models.PRWatchTaskActivity, +) error { + defer func() { _ = rows.Close() }() + for rows.Next() { + var ( + taskID string + rawActivityAt interface{} + ) + if err := rows.Scan(&taskID, &rawActivityAt); err != nil { + return fmt.Errorf("scan PR watch task activity: %w", err) + } + activityAt, err := parseTaskActivityTime(rawActivityAt) + if err != nil { + return fmt.Errorf("parse PR watch task activity %q: %w", taskID, err) + } + result[taskID] = models.PRWatchTaskActivity{LastActivityAt: activityAt} + } + if err := rows.Err(); err != nil { + return fmt.Errorf("read PR watch task activity: %w", err) + } + return nil +} diff --git a/apps/backend/internal/task/repository/sqlite/pr_watch_activity_test.go b/apps/backend/internal/task/repository/sqlite/pr_watch_activity_test.go new file mode 100644 index 0000000000..3985faa3ab --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/pr_watch_activity_test.go @@ -0,0 +1,130 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/orchestrator/messagequeue" +) + +func TestLoadPRWatchTaskActivityUsesAuthoritativeSources(t *testing.T) { + repo, db := newTaskStatusSummaryTestRepo(t) + queueRepo, err := messagequeue.NewSQLiteRepository(db, db) + if err != nil { + t.Fatalf("initialize queue schema: %v", err) + } + ctx := context.Background() + base := time.Date(2026, time.September, 1, 10, 0, 0, 0, time.UTC) + + seedTask := func(taskID string, createdAt, updatedAt time.Time) { + t.Helper() + if _, err := db.Exec(db.Rebind(` + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), taskID, "workspace-watch-activity", taskID, createdAt, updatedAt); err != nil { + t.Fatalf("seed task %q: %v", taskID, err) + } + } + seedTask("task-watch-activity", base, base.Add(48*time.Hour)) + seedTask("task-watch-created-only", base.Add(time.Hour), base.Add(72*time.Hour)) + + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_sessions (id, task_id, state, started_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), "session-watch-activity", "task-watch-activity", "RUNNING", base, base); err != nil { + t.Fatalf("seed running session: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, completed_at, metadata, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `), "turn-watch-lifecycle", "session-watch-activity", "task-watch-activity", base.Add(2*time.Hour), base.Add(3*time.Hour), `{"lifecycle_only":true}`, base.Add(2*time.Hour), base.Add(3*time.Hour)); err != nil { + t.Fatalf("seed lifecycle turn: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, completed_at, metadata, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `), "turn-watch-agent", "session-watch-activity", "task-watch-activity", base.Add(4*time.Hour), base.Add(5*time.Hour), `{}`, base.Add(4*time.Hour), base.Add(5*time.Hour)); err != nil { + t.Fatalf("seed agent turn: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_messages (id, task_session_id, task_id, turn_id, author_type, content, type, metadata, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `), "message-watch-user", "session-watch-activity", "task-watch-activity", "turn-watch-agent", "user", "prompt", "message", `{}`, base.Add(6*time.Hour)); err != nil { + t.Fatalf("seed user message: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_messages (id, task_session_id, task_id, turn_id, author_type, content, type, metadata, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `), "message-watch-agent", "session-watch-activity", "task-watch-activity", "turn-watch-agent", "agent", "response", "message", `{}`, base.Add(24*time.Hour)); err != nil { + t.Fatalf("seed agent message: %v", err) + } + if err := queueRepo.Insert(ctx, &messagequeue.QueuedMessage{ + ID: "queued-watch-user", SessionID: "session-watch-activity", TaskID: "task-watch-activity", + Content: "queued prompt", QueuedAt: base.Add(7 * time.Hour), QueuedBy: "user-1", + }, 10); err != nil { + t.Fatalf("seed user queue message: %v", err) + } + if err := queueRepo.Insert(ctx, &messagequeue.QueuedMessage{ + ID: "queued-watch-agent", SessionID: "session-watch-activity", TaskID: "task-watch-activity", + Content: "queued lifecycle", QueuedAt: base.Add(36 * time.Hour), QueuedBy: messagequeue.QueuedByAgent, + }, 10); err != nil { + t.Fatalf("seed agent queue message: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_commits (id, session_id, commit_sha, committed_at, created_at) + VALUES (?, ?, ?, ?, ?) + `), "commit-watch", "session-watch-activity", "sha-watch", base.Add(8*time.Hour), base.Add(8*time.Hour)); err != nil { + t.Fatalf("seed session commit: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_environments (id, task_id, created_at, updated_at) + VALUES (?, ?, ?, ?) + `), "environment-watch", "task-watch-activity", base, base); err != nil { + t.Fatalf("seed task environment: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_git_snapshots ( + id, task_environment_id, session_id, snapshot_type, branch, head_commit, triggered_by, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `), "snapshot-watch-live", "environment-watch", "session-watch-activity", "status", "main", "live-sha", TriggeredByLiveMonitor, base.Add(30*time.Hour)); err != nil { + t.Fatalf("seed live snapshot: %v", err) + } + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_git_snapshots ( + id, task_environment_id, session_id, snapshot_type, branch, head_commit, triggered_by, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `), "snapshot-watch-push", "environment-watch", "session-watch-activity", "status", "main", "pushed-sha", "agent_completed", base.Add(9*time.Hour)); err != nil { + t.Fatalf("seed observed push snapshot: %v", err) + } + + got, err := repo.LoadPRWatchTaskActivity(ctx, []string{ + "task-watch-activity", "task-watch-created-only", "missing-task", + }) + if err != nil { + t.Fatalf("load PR watch activity: %v", err) + } + activity, ok := got["task-watch-activity"] + if !ok { + t.Fatal("activity projection omitted task-watch-activity") + } + if !activity.Running { + t.Fatal("running session was not reported") + } + if want := base.Add(9 * time.Hour); !activity.LastActivityAt.Equal(want) { + t.Fatalf("activity timestamp = %s, want %s", activity.LastActivityAt, want) + } + createdOnly, ok := got["task-watch-created-only"] + if !ok || !createdOnly.LastActivityAt.Equal(base.Add(time.Hour)) { + t.Fatalf("created-only activity = %+v, want task creation timestamp", createdOnly) + } + if _, ok := got["missing-task"]; ok { + t.Fatal("missing task must not appear in activity projection") + } + + // A task row update is provider bookkeeping and must not make an idle + // searching watch look active. + if createdOnly.LastActivityAt.Equal(base.Add(72 * time.Hour)) { + t.Fatal("generic task updated_at leaked into activity projection") + } +} diff --git a/apps/web/app/tasks/rich-task-list-row.test.tsx b/apps/web/app/tasks/rich-task-list-row.test.tsx index 41edf88420..d2c28c5e81 100644 --- a/apps/web/app/tasks/rich-task-list-row.test.tsx +++ b/apps/web/app/tasks/rich-task-list-row.test.tsx @@ -88,12 +88,16 @@ function makeMR(overrides: Partial = {}): TaskMR { }; } -function renderRow(showTaskDetails: boolean, initialState: HydrationState = {}) { +function renderRow( + showTaskDetails: boolean, + initialState: HydrationState = {}, + taskOverrides: Partial = {}, +) { return render( { + renderRow( + true, + { + taskPRs: { byTaskId: { "task-1": [makePR()] } }, + workspaces: { items: [], activeId: "ws1" }, + }, + { + status_summary: { + revision: 1, + updated_at: "2026-09-23T00:00:00Z", + pull_request: { + number: 1, + state: "open", + auto_fix_enabled: true, + auto_merge_enabled: true, + }, + }, + }, + ); + + expect(screen.getByTestId("pr-task-automation-auto-fix")).not.toBeNull(); + expect(screen.getByTestId("pr-task-automation-auto-merge")).not.toBeNull(); + }); }); describe("TaskListRowPrimaryContent, contributions hidden (AC10)", () => { diff --git a/apps/web/app/tasks/rich-task-list-row.tsx b/apps/web/app/tasks/rich-task-list-row.tsx index 8f0fbcb42a..2055a74bf4 100644 --- a/apps/web/app/tasks/rich-task-list-row.tsx +++ b/apps/web/app/tasks/rich-task-list-row.tsx @@ -6,6 +6,7 @@ import { PRTaskIcon } from "@/components/github/pr-task-icon"; import { RegisteredChangeRequestTaskIcon } from "@/components/integrations/registered-change-request-task-icon"; import { TaskRowMetadata } from "@/components/task/task-row-plugin-slots"; import { MRTaskIcon } from "@/components/gitlab/mr-task-icon"; +import { taskPRInfoFromSummary } from "@/lib/task-pr-info"; import { useTaskPendingInput, type PendingInput } from "@/hooks/use-task-pending-input"; import { getTaskStateIcon } from "@/lib/ui/state-icons"; import type { Repository, Task } from "@/lib/types/http"; @@ -57,7 +58,7 @@ function PrimaryTaskLine({ onClick={(event) => event.stopPropagation()} onPointerDown={(event) => event.stopPropagation()} > - + )} diff --git a/apps/web/e2e/tests/pr/mobile-pr-sidebar-automation-indicators.spec.ts b/apps/web/e2e/tests/pr/mobile-pr-sidebar-automation-indicators.spec.ts index dacf4370f7..108e198766 100644 --- a/apps/web/e2e/tests/pr/mobile-pr-sidebar-automation-indicators.spec.ts +++ b/apps/web/e2e/tests/pr/mobile-pr-sidebar-automation-indicators.spec.ts @@ -41,7 +41,9 @@ async function seedSidebarAutomation( state: "open", review_state: "approved", checks_state: "success", - mergeable_state: "clean", + // Keep auto-merge enabled for the indicator, but leave the PR blocked so + // the background CI automation cannot merge it before the picker opens. + mergeable_state: "blocked", }); await apiClient.updateTaskCIAutomationOptions(targetTask.task_id, { repository_id: seedData.repositoryId, diff --git a/apps/web/e2e/tests/system/settings-header-tabs.spec.ts b/apps/web/e2e/tests/system/settings-header-tabs.spec.ts index a1b54f0916..e34d567e95 100644 --- a/apps/web/e2e/tests/system/settings-header-tabs.spec.ts +++ b/apps/web/e2e/tests/system/settings-header-tabs.spec.ts @@ -27,7 +27,9 @@ test.describe("Settings header tabs", () => { expect(tabListBox!.x).toBeGreaterThan(titleBox!.x + titleBox!.width); await logsTab.focus(); + await expect(logsTab).toBeFocused(); await testPage.keyboard.press("ArrowLeft"); + await expect(databaseTab).toBeFocused(); await expect(databaseTab).toHaveAttribute("aria-selected", "true"); await expect(logsTab).toHaveAttribute("aria-selected", "false"); await testPage.keyboard.press("ArrowRight"); diff --git a/apps/web/hooks/domains/github/task-pr-sync-resource.test.ts b/apps/web/hooks/domains/github/task-pr-sync-resource.test.ts index 8f87c4eff8..3c56a639f2 100644 --- a/apps/web/hooks/domains/github/task-pr-sync-resource.test.ts +++ b/apps/web/hooks/domains/github/task-pr-sync-resource.test.ts @@ -84,9 +84,9 @@ describe("task PR sync resource request ownership", () => { const firstListener = vi.fn(); const secondListener = vi.fn(); + const firstRefresh = resource.refresh(scope); const releaseFirst = resource.subscribe(scope, firstListener); const releaseSecond = resource.subscribe(scope, secondListener); - const firstRefresh = resource.refresh(scope); const secondRefresh = resource.refresh(scope); expect(requester).toHaveBeenCalledTimes(1); @@ -102,6 +102,20 @@ describe("task PR sync resource request ownership", () => { releaseSecond(); }); + it("marks only a user refresh as explicit", async () => { + const store = createStore(); + const requester = vi.fn().mockResolvedValue({ prs: [] }); + const resource = createTaskPRSyncResource(store, requester, { retryDelayMs: 100 }); + const release = resource.subscribe(scope, vi.fn()); + + await Promise.resolve(); + expect(requester.mock.calls[0]?.[1]).toEqual({ explicitRefresh: false }); + + await resource.refresh(scope); + expect(requester.mock.calls[1]?.[1]).toEqual({ explicitRefresh: true }); + release(); + }); + // @covers AC-INTEGRATIONS-GITHUB-TASK-PR-SYNC-COORDINATION-001.4 it("keeps retries after one release and stops them after the final release", async () => { const store = createStore(); @@ -362,7 +376,9 @@ describe("task PR sync resource recovery", () => { const reacquired = resource.subscribe(scope, vi.fn()); await Promise.resolve(); - expect(requester).toHaveBeenCalledTimes(2); + // The explicit refresh starts a replacement request even though the + // original passive request was released while it was in flight. + expect(requester).toHaveBeenCalledTimes(3); reacquired(); }); }); diff --git a/apps/web/hooks/domains/github/task-pr-sync-resource.ts b/apps/web/hooks/domains/github/task-pr-sync-resource.ts index 1255fe1c37..9859dad51a 100644 --- a/apps/web/hooks/domains/github/task-pr-sync-resource.ts +++ b/apps/web/hooks/domains/github/task-pr-sync-resource.ts @@ -15,7 +15,14 @@ export type TaskPRSyncResponse = | null | undefined; -export type TaskPRSyncRequester = (scope: TaskPRSyncScope) => Promise; +export type TaskPRSyncRequestOptions = { + explicitRefresh?: boolean; +}; + +export type TaskPRSyncRequester = ( + scope: TaskPRSyncScope, + options?: TaskPRSyncRequestOptions, +) => Promise; export type TaskPRSyncResourceOptions = { retryDelayMs?: number; @@ -46,6 +53,7 @@ type ResourceEntry = { retryTimer: ReturnType | null; stopStoreSubscription: (() => void) | null; connectionStatus: AppState["connection"]["status"]; + requestExplicitRefresh: boolean; }; type ResourceStore = { @@ -56,10 +64,16 @@ type ResourceStore = { maxRetries: number; }; -async function requestTaskPRSync(scope: TaskPRSyncScope): Promise { +async function requestTaskPRSync( + scope: TaskPRSyncScope, + options: TaskPRSyncRequestOptions = {}, +): Promise { const client = getWebSocketClient(); if (!client) throw NO_WEBSOCKET_CLIENT; - return client.request("github.task_pr.sync", { task_id: scope.taskId }); + return client.request("github.task_pr.sync", { + task_id: scope.taskId, + ...(options.explicitRefresh ? { explicit_refresh: true } : {}), + }); } function scopeKey(scope: TaskPRSyncScope): string { @@ -82,6 +96,7 @@ function createEntry(scope: TaskPRSyncScope): ResourceEntry { retryTimer: null, stopStoreSubscription: null, connectionStatus: "disconnected", + requestExplicitRefresh: false, }; } @@ -231,10 +246,11 @@ async function runRequest( store: ResourceStore, entry: ResourceEntry, requestGeneration: number, + explicitRefresh: boolean, ): Promise { let result: TaskPRSyncResponse; try { - result = await store.requester(entry.scope); + result = await store.requester(entry.scope, { explicitRefresh }); } catch (error) { if (error !== NO_WEBSOCKET_CLIENT) publishFailure(store, entry, requestGeneration); return; @@ -247,6 +263,7 @@ function settleRequest(store: ResourceStore, entry: ResourceEntry, promise: Prom const requestStartedAt = entry.requestStartedAt; entry.promise = null; entry.requestStartedAt = null; + entry.requestExplicitRefresh = false; if (!isCurrentEntry(store, entry)) return; if (entry.listeners.size === 0) { store.entries.delete(entry.key); @@ -263,11 +280,22 @@ function settleRequest(store: ResourceStore, entry: ResourceEntry, promise: Prom } } -function startRequest(store: ResourceStore, entry: ResourceEntry): Promise { - if (entry.promise) return entry.promise; +function startRequest( + store: ResourceStore, + entry: ResourceEntry, + explicitRefresh = false, +): Promise { + if (entry.promise) { + if (!explicitRefresh || entry.requestExplicitRefresh) return entry.promise; + entry.requestGeneration += 1; + entry.promise = null; + entry.requestStartedAt = null; + entry.requestExplicitRefresh = false; + } const requestGeneration = ++entry.requestGeneration; entry.requestStartedAt = Date.now(); - const promise = runRequest(store, entry, requestGeneration); + entry.requestExplicitRefresh = explicitRefresh; + const promise = runRequest(store, entry, requestGeneration, explicitRefresh); entry.promise = promise; void promise.then( () => settleRequest(store, entry, promise), @@ -343,7 +371,7 @@ function createResource( return { getSnapshot: (scope) => resourceStore.entries.get(scopeKey(scope))?.loaded ?? false, invalidate: (scope) => invalidate(resourceStore, scope), - refresh: (scope) => startRequest(resourceStore, entryFor(resourceStore, scope)), + refresh: (scope) => startRequest(resourceStore, entryFor(resourceStore, scope), true), subscribe: (scope, listener) => subscribe(resourceStore, scope, listener), }; } diff --git a/apps/web/hooks/domains/github/use-pr-ci-popover.test.ts b/apps/web/hooks/domains/github/use-pr-ci-popover.test.ts index f9d234d7e8..c83e72f76f 100644 --- a/apps/web/hooks/domains/github/use-pr-ci-popover.test.ts +++ b/apps/web/hooks/domains/github/use-pr-ci-popover.test.ts @@ -114,8 +114,9 @@ describe("usePRCIPopover refresh indicator", () => { rerender({ enabled: true }); await flushOpen(); - // The feedback fetch has already resolved; the summary sync has not. - await waitFor(() => expect(result.current.isFetching).toBe(false)); + // The feedback read waits for the summary sync so an explicit refresh + // cannot race with a stale Actions response. + expect(getPRFeedbackMock).not.toHaveBeenCalled(); expect(result.current.isRefreshing).toBe(true); await act(async () => { @@ -123,6 +124,8 @@ describe("usePRCIPopover refresh indicator", () => { await sync.promise; }); + await waitFor(() => expect(getPRFeedbackMock).toHaveBeenCalledTimes(1)); + await waitFor(() => expect(result.current.isFetching).toBe(false)); await waitFor(() => expect(result.current.isRefreshing).toBe(false)); }); diff --git a/apps/web/hooks/domains/github/use-pr-ci-popover.ts b/apps/web/hooks/domains/github/use-pr-ci-popover.ts index d9ecd4a44f..2cc43ac7de 100644 --- a/apps/web/hooks/domains/github/use-pr-ci-popover.ts +++ b/apps/web/hooks/domains/github/use-pr-ci-popover.ts @@ -113,8 +113,13 @@ export function usePRCIPopover( wasEnabledRef.current = enabled; if (!opened) return; queueMicrotask(() => { - refetch(); - trackSync(refreshTaskPR?.()); + const sync = refreshTaskPR?.(); + trackSync(sync); + if (sync) { + void sync.then(refetch, refetch); + } else { + refetch(); + } }); }, [enabled, refetch, refreshTaskPR, trackSync]); diff --git a/docs/plans/watch-task-cleanup/plan.md b/docs/plans/watch-task-cleanup/plan.md new file mode 100644 index 0000000000..ed5794e3e2 --- /dev/null +++ b/docs/plans/watch-task-cleanup/plan.md @@ -0,0 +1,138 @@ +--- +created: 2026-09-22 +status: complete +requirements: + - REQ-INTEGRATIONS-WATCH-CLEANUP-001 + - REQ-INTEGRATIONS-WATCH-CLEANUP-002 + - REQ-INTEGRATIONS-GITHUB-PR-POLLING-001 + - REQ-INTEGRATIONS-GITHUB-PR-POLLING-002 + - REQ-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002 +system_design: + - ../../specs/integrations/system-design/watch-task-cleanup.md + - ../../specs/integrations/system-design/github-pr-discovery-health.md + - ../../specs/integrations/system-design/github-workflow-attention.md +legacy_specs: [] +--- + +# Implementation plan: Watch task cleanup + +## Overview + +Prevent historical tasks from consuming provider quota. Implement GitHub +eligibility first, GitHub batch termination second, and sibling filters third. +Then reduce cleanup reads, bound fallback, add adaptive discovery, and cache +Actions observations. All seven sequential work orders are complete. + +## Source documents + +- [Requirements](../../specs/integrations/requirements/watch-task-cleanup.md) +- [System design](../../specs/integrations/system-design/watch-task-cleanup.md) +- [Discovery requirements](../../specs/integrations/requirements/github-pr-discovery-health.md) +- [Discovery design](../../specs/integrations/system-design/github-pr-discovery-health.md) +- [Actions requirements](../../specs/integrations/requirements/github-workflow-attention.md) +- [Actions design](../../specs/integrations/system-design/github-workflow-attention.md) + +## Confirmed root cause and reproduction + +GitHub cleanup list queries read historical deduplication rows without task +eligibility checks. Service gates fetch remote state before engagement checks. +Fetch errors become false eligibility results, so the batch continues. + +A temporary in-memory SQLite reproduction executed the actual SQL extracted +from both GitHub review list methods. Both returned task IDs `active`, +`archived`, `deleted`, and an empty reservation. Only the first and last should +be cleanup candidates. No production database or provider API was accessed. + +GitLab review and issue queries also lack task eligibility filters. Azure +DevOps review and work-item queries filter watch generation only. Both +providers fetch upstream state before checking task engagement. + +The current GitHub feedback helper requests PR data, reviews, comments, checks, +and optional workflow attention. The exact request count differs from the +reported older implementation, but historical-row amplification remains. + +## Scope and compatibility + +Preserve cleanup policies, lifecycle-prompt protection, empty GitHub +reservations, explicit watch deletion, and reset behavior. Ignore historical +records without changing deduplication. No UI or schema changes are required. +Unarchived Done tasks remain eligible. Archive races after query selection +remain outside the atomicity guarantee. + +## Work orders + +- [x] [Task 01: Filter GitHub cleanup candidates](task-01-github-candidates.md) +- [x] [Task 02: Stop GitHub cleanup on rate limits](task-02-github-rate-limits.md) +- [x] [Task 03: Filter sibling provider cleanup candidates](task-03-sibling-candidates.md) + +- [x] [Task 04: Use minimal PR reads for cleanup](task-04-minimal-cleanup.md) +- [x] [Task 05: Bound per-workspace PR fallback](task-05-bounded-fallback.md) +- [x] [Task 06: Back off idle searching watches](task-06-adaptive-discovery.md) +- [x] [Task 07: Cache Actions observations with bounded expiry](task-07-actions-cache.md) + +## Polling and cache values + +| Function | Shipped behavior | Bound/details | +| --- | --- | --- | +| PR poller scheduler tick | 1 minute | Fixed scheduler tick | +| Searching, running or idle below 2 hours | 1 minute | Fast discovery | +| Searching, idle 2 to below 24 hours | 15 minutes | Activity restores next-tick eligibility | +| Searching, idle at least 24 hours | 30 minutes | Never permanently stops | +| Known PR background checks | 1 minute | Fixed cadence | +| Review-watch discovery and cleanup | 5 minutes | Eligible tasks only | +| Issue-watch discovery and cleanup | 5 minutes | Eligible tasks only | +| Status/search response TTL | 30 seconds | Outer response cache | +| Explicit feedback response TTL | 8 seconds | Explicit refresh invalidates | +| PR sync freshness window | 30 seconds | Passive status freshness | +| Actions changing, empty, or attention-required | 30 seconds | Run and job observations | +| Actions all complete, no attention required | 5 minutes | Same-SHA reruns appear after expiry | +| Actions runs/jobs capacity | 512 entries each | In-memory bounded caches | +| Actions batch concurrency / total budget | 4 / 5 seconds | Preserved | +| Fallback target budget | 5 per workspace, 10 globally | Rotating per cycle | + +Activity restores next-tick eligibility. Explicit refresh bypasses idle and +Actions cache delays but never quota/auth admission. Poll ticks can add up to +1 minute to the interval. Missing activity information selects fast polling. +A completed-SHA external rerun may take up to 6 minutes to appear automatically. + +Retain discovery rate-limit retry delays of 1, 2, 4, 8, then 15 minutes when +no provider retry time is available. Existing provider retry deadlines are +capped at 15 minutes by discovery health. Auth/config circuit backoff starts +at 2 minutes, caps at 6 hours, and uses 25 percent jitter. The poller rate-limit +sleep cap remains 10 minutes per wait; it does not clear the exhaustion state. + +## Tests + +Each work order lists new regression names, existing test locations, and exact +commands. Use store-backed service tests to prove candidate selection reaches +provider-call behavior. Browser tests add no evidence for this backend-only +change. Run each regression before production edits and record its expected +failure. Then run the affected package suites. + +## Verification results + +- `python3 scripts/list-docs.py validate`: passed, 299 decisions and 1110 specifications. +- `python3 scripts/lint-spec-files.py --all`: passed. +- `python3 scripts/lint-spec-files.test.py`: passed, 36 tests. +- `git diff --check`: passed. + +Implementation is complete. Production, permanent test, specification, plan, +and public documentation changes are present in the working tree and remain +uncommitted. + +## Risks + +- Existing tests often attach fictitious task IDs without task rows. Update + active fixtures rather than weakening eligibility. +- GitHub by-watch queries also support explicit watch deletion. Preserve its + unfiltered inventory through existing task-ID methods. +- A batch can stop only after an error becomes visible. Concurrent requests + within its current feedback fetch may already be running. +- Ignored historical deduplication rows remain stored. This fix bounds polling, + not storage retention or quota consumed by other active work. + +## Documentation impact + +Internal requirements, designs, and work orders describe the expanded scope. +No controls change. The public integration guide documents delayed idle +discovery and bounded Actions freshness as shipped behavior. diff --git a/docs/plans/watch-task-cleanup/task-01-github-candidates.md b/docs/plans/watch-task-cleanup/task-01-github-candidates.md new file mode 100644 index 0000000000..3861ede528 --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-01-github-candidates.md @@ -0,0 +1,103 @@ +--- +id: "01-github-candidates" +title: "Filter GitHub cleanup candidates" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-WATCH-CLEANUP-001 +acceptance_criteria: + - AC-INTEGRATIONS-WATCH-CLEANUP-001.1 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.2 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.3 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.4 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.5 +system_design: + - ../../specs/integrations/system-design/watch-task-cleanup.md +--- + +# Task 01: Filter GitHub cleanup candidates + +## Summary + +Filter GitHub cleanup candidates. Implement with TDD and record the regression results. + +## In scope + +Add mixed-state regressions in `store_task_cleanup_test.go` and +`service_review_cleanup_test.go`. Name the new tests +`TestCleanupCandidateListsExcludeHistoricalTasks` and +`TestCleanupMergedReviewTasksSkipsHistoricalTasks`. +Cover by-watch, global, orphan, and workspace cleanup for reviews and issues. +Assert zero upstream calls for archived and missing tasks. Cover terminal and +nonterminal empty reservations, all policies, engagement, and lifecycle prompts. + +Apply the candidate predicate to all four GitHub list methods. Preserve +explicit watch deletion with unfiltered task-ID queries in `service_reviews.go` +and `service_issues.go`. Add archived-child deletion assertions to existing +watch deletion tests. Assert deduplication remains present through unfiltered +lookups, not filtered cleanup queries. + +Update obsolete missing-task cleanup expectations to zero provider calls and +zero deletions. Keep task-not-found race coverage by deleting an initially +eligible task after selection. + +## Out of scope + +New policies, UI, schema migrations, and unrelated provider changes. + +## Acceptance + +- Mixed candidate lists and store-backed cleanup satisfy AC 001.1 through 001.4. +- Explicit watch deletion, reset, scope, and deduplication satisfy AC 001.5. +- Regression tests fail before the query changes and pass afterward. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/github -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/github/store.go` +- `apps/backend/internal/github/service_reviews.go` +- `apps/backend/internal/github/service_issues.go` +- `apps/backend/internal/github/store_task_cleanup_test.go` +- `apps/backend/internal/github/service_review_cleanup_test.go` +- `apps/backend/internal/github/service_cleanup_policy_test.go` +- `apps/backend/internal/github/store_watch_reset_test.go` +- `apps/backend/internal/github/store_watch_disable_test.go` + +## Dependencies + +None. + +## Risks + +Preserve task inventory outside cleanup. Seed actual active tasks in fixtures. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/watch-task-cleanup.md) +- [Design](../../specs/integrations/system-design/watch-task-cleanup.md) +- Existing source and tests listed above. + +## Results + +Implemented the task eligibility predicate for all four GitHub cleanup list +queries. Explicit watch deletion keeps its unfiltered task-ID inventory, so +archived and missing task rows are preserved for deduplication while active +tasks and empty reservations remain cleanup candidates. Added mixed-state +store and service regressions, updated active-task fixtures, and retained a +post-selection task-not-found race regression. + +Verification: `go test ./internal/github -count=1` passed. diff --git a/docs/plans/watch-task-cleanup/task-02-github-rate-limits.md b/docs/plans/watch-task-cleanup/task-02-github-rate-limits.md new file mode 100644 index 0000000000..a0562ed3b2 --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-02-github-rate-limits.md @@ -0,0 +1,111 @@ +--- +id: "02-github-rate-limits" +title: "Stop GitHub cleanup on rate limits" +status: done +wave: 2 +depends_on: ["01-github-candidates"] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-WATCH-CLEANUP-002 +acceptance_criteria: + - AC-INTEGRATIONS-WATCH-CLEANUP-002.1 + - AC-INTEGRATIONS-WATCH-CLEANUP-002.2 + - AC-INTEGRATIONS-WATCH-CLEANUP-002.3 + - AC-INTEGRATIONS-WATCH-CLEANUP-002.4 +system_design: + - ../../specs/integrations/system-design/watch-task-cleanup.md +--- + +# Task 02: Stop GitHub cleanup on rate limits + +## Summary + +Stop GitHub cleanup on rate limits. Implement with TDD and record the regression results. + +## In scope + +Add `TestCleanupBatchStopsOnRateLimit` in +`service_cleanup_policy_test.go`, with review and issue cases. Include an empty +reservation first, partial deletion counts, wrapped 403/429 errors, unrelated +403 errors, authenticated-user errors, context cancellation, and later retries. +Add `TestCleanupPollCycleSkipsRemainingCleanupAfterRateLimit` in +`poller_test.go`. Cover per-watch and final orphan calls for both poll loops. + +Propagate errors through the private deletion gates, batch helpers, and public +cleanup methods in `service_cleanup.go`. Use the existing typed classifier, +including normalization of GitHub CLI stderr rate-limit failures at the +`GHClient` boundary. Thread the resolved automation tracker into batch +admission and stop against the resource used by the resolved client (including +GraphQL for CLI-backed PR and issue reads). Include a stub that exhausts the +tracker while returning successful feedback. Preserve scope: do not consult +another workspace's tracker. + +Preserve ordinary row-error continuation and completed counts. In `poller.go`, +stop further cleanup calls for the current cycle after a rate-limit error. +Keep scheduled recovery and discovery behavior intact. + +## Out of scope + +New policies, UI, schema migrations, and unrelated provider changes. + +## Acceptance + +- Review and issue batches stop on errors and tracker exhaustion. +- Public cleanup methods preserve partial counts and expose the stop error. +- Poller regressions prove remaining cleanup and orphan calls are suppressed. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/github -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/github/service_cleanup.go` +- `apps/backend/internal/github/poller.go` +- `apps/backend/internal/github/service_cleanup_policy_test.go` +- `apps/backend/internal/github/service_review_cleanup_test.go` +- `apps/backend/internal/github/poller_test.go` +- `apps/backend/internal/github/poller_cleanup_log_test.go` + +## Dependencies + +01-github-candidates. + +## Risks + +Preserve task inventory outside cleanup. Seed actual active tasks in fixtures. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/watch-task-cleanup.md) +- [Design](../../specs/integrations/system-design/watch-task-cleanup.md) +- Existing source and tests listed above. + +## Results + +Implemented rate-limit and cancellation stop propagation through review and +issue cleanup batches. Batches preserve completed deletion counts, continue +after ordinary row errors, honor the resolved workspace tracker (Core for REST +clients and GraphQL for GHClient), and return recognized 403/429 errors. +GHClient now normalizes CLI rate-limit +stderr into the typed error path, and cleanup admission checks the resolved +client's resource bucket so GraphQL-backed CLI cleanup does not continue after +its quota is exhausted. Poller cycles stop before later watches and the final +orphan sweep after a stop-worthy cleanup error. Added service and poller +regressions for wrapped errors, tracker exhaustion, CLI-shaped rate limits, +cancellation, partial counts, ordinary 403 continuation, and later retry +recovery. +Cleanup admission also reopens automatically once the tracked resource reset +deadline has passed, even when no intervening provider call refreshed the +tracker. + +Verification: `go test ./internal/github -count=1` passed. diff --git a/docs/plans/watch-task-cleanup/task-03-sibling-candidates.md b/docs/plans/watch-task-cleanup/task-03-sibling-candidates.md new file mode 100644 index 0000000000..ce5cf286bf --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-03-sibling-candidates.md @@ -0,0 +1,102 @@ +--- +id: "03-sibling-candidates" +title: "Filter sibling provider cleanup candidates" +status: done +wave: 3 +depends_on: ["02-github-rate-limits"] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-WATCH-CLEANUP-001 +acceptance_criteria: + - AC-INTEGRATIONS-WATCH-CLEANUP-001.1 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.2 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.4 + - AC-INTEGRATIONS-WATCH-CLEANUP-001.5 +system_design: + - ../../specs/integrations/system-design/watch-task-cleanup.md +--- + +# Task 03: Filter sibling provider cleanup candidates + +## Summary + +Filter sibling provider cleanup candidates. Implement with TDD and record the regression results. + +## In scope + +Add `TestCleanupCandidateListsExcludeHistoricalTasks` and +`TestCleanupSkipsHistoricalTasks` to the GitLab and Azure DevOps packages. +Create `gitlab/service_cleanup_test.go` beside the existing +`service_cleanup_reason_test.go` and `service_cleanup_mr_automation_test.go`. +Use their existing store and cleanup test fixtures. Cover active, archived, +missing, and empty-task rows together. Assert no remote status calls for +historical tasks and preserve active-task policies. + +Filter GitLab review and issue queries in `store_watches.go`, including global, +workspace, and by-watch methods. Filter Azure DevOps work-item and PR list +queries in `watch_store.go`. Preserve workspace and generation constraints. +GitLab and Azure DevOps continue to skip empty reservations in cleanup. +Do not change sibling rate-limit handling or task deletion adapters. + +## Out of scope + +New policies, UI, schema migrations, and unrelated provider changes. + +## Acceptance + +- Both provider packages exclude historical tasks from upstream cleanup calls. +- Active policy behavior and workspace/generation boundaries remain intact. +- Empty reservations and reset inventory preserve existing semantics. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/gitlab ./internal/azuredevops -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/gitlab/store_watches.go` +- `apps/backend/internal/gitlab/store_task_cleanup_test.go` +- `apps/backend/internal/gitlab/service_cleanup_test.go` +- `apps/backend/internal/azuredevops/watch_store.go` +- `apps/backend/internal/azuredevops/watch_store_test.go` +- `apps/backend/internal/azuredevops/watch_cleanup_test.go` + +## Dependencies + +02-github-rate-limits. + +## Risks + +Preserve task inventory outside cleanup. Seed actual active tasks in fixtures. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/watch-task-cleanup.md) +- [Design](../../specs/integrations/system-design/watch-task-cleanup.md) +- Existing source and tests listed above. + +## Results + +Implemented the shared task eligibility predicate for GitLab review and issue +candidate queries, including global, workspace, and by-watch reads. Azure +DevOps work-item and pull-request reads now apply the same archived-or-missing +task filter while preserving watch generation scoping. Empty reservations stay +visible to cleanup, which continues to skip them before provider status calls. + +Added mixed-state store regressions and instrumented cleanup tests for both +providers. Active task policies still delete terminal records, while archived +and missing task IDs generate zero provider calls. Updated SQLite fixtures to +provide the central task table and seed active task rows where cleanup reads +task eligibility. + +Verification: `go test ./internal/gitlab ./internal/azuredevops -count=1` +passed. diff --git a/docs/plans/watch-task-cleanup/task-04-minimal-cleanup.md b/docs/plans/watch-task-cleanup/task-04-minimal-cleanup.md new file mode 100644 index 0000000000..f19b5f4c1f --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-04-minimal-cleanup.md @@ -0,0 +1,92 @@ +--- +id: "04-minimal-cleanup" +title: "Use minimal PR reads for cleanup" +status: done +wave: 4 +depends_on: ["03-sibling-candidates"] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-WATCH-CLEANUP-001 +acceptance_criteria: + - AC-INTEGRATIONS-WATCH-CLEANUP-001.6 +system_design: + - ../../specs/integrations/system-design/watch-task-cleanup.md +--- + +# Task 04: Use minimal PR reads for cleanup + +## Summary + +Use minimal PR reads for cleanup. Implement with TDD and preserve workspace identity. + +## In scope + +Replace cleanup-only GetPRFeedback calls with GetPR and conditional review +reads. Fetch viewer identity only when an approval candidate requires it. +Preserve all cleanup policies and approval matching. Propagate each fetch error +to the batch handling from Task 02. Update existing cleanup stubs and tests. + +Add TestCleanupUsesMinimalPRReads to service_review_cleanup_test.go. Assert +zero comments, checks, workflow-run, and job reads. Terminal PRs require only +GetPR. Open PRs retain approval-based cleanup. Include rate limits at each +required fetch and preserve partial counts. + +## Out of scope + +New UI controls, credential policies, global schedulers, and unrelated providers. + +## Acceptance + +- Meet every linked acceptance criterion with deterministic regressions. +- Preserve existing policy, scope, and error-handling contracts. +- Record the expected red test result and passing package results. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/github -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/github/service_cleanup.go` +- `apps/backend/internal/github/service_review_cleanup_test.go` +- `apps/backend/internal/github/service_cleanup_policy_test.go` + +## Dependencies + +03-sibling-candidates. + +## Risks + +Mocks must implement the smaller read path. Preserve existing review approval semantics. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/watch-task-cleanup.md) +- [Design](../../specs/integrations/system-design/watch-task-cleanup.md) +- Source files and existing adjacent tests listed above. New helper/test files are created as needed. + +## Results + +Implemented the minimal GitHub cleanup read path. Cleanup now fetches PR state +with `GetPR`, fetches reviews only for open PRs, and requests the authenticated +user only when at least one approval review exists. Cleanup no longer requests +comments, check runs, workflow runs, or workflow jobs through the feedback +helper. Fetch errors use the Task 02 batch handling, including rate-limit stop +and partial-count behavior. + +Added deterministic coverage for terminal and open PR reads, approval-based +cleanup, historical-row cleanup, and rate limits. The focused red tests failed +before the production change as expected. Verification passed: + +```bash +(cd apps/backend && go test ./internal/github -count=1) +``` diff --git a/docs/plans/watch-task-cleanup/task-05-bounded-fallback.md b/docs/plans/watch-task-cleanup/task-05-bounded-fallback.md new file mode 100644 index 0000000000..7685c95855 --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-05-bounded-fallback.md @@ -0,0 +1,109 @@ +--- +id: "05-bounded-fallback" +title: "Bound per-workspace PR fallback" +status: done +wave: 5 +depends_on: ["04-minimal-cleanup"] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-GITHUB-PR-POLLING-002 +acceptance_criteria: + - AC-INTEGRATIONS-GITHUB-PR-POLLING-002.1 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-002.2 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-002.3 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-002.4 +system_design: + - ../../specs/integrations/system-design/github-pr-discovery-health.md +--- + +# Task 05: Bound per-workspace PR fallback + +## Summary + +Bound per-workspace PR fallback. Implement with TDD and preserve workspace identity. + +## In scope + +Replace the all-workspace boolean result from tryBatchedPRWatchCheck with +per-workspace outcomes. Retain auth, rate-limit, and invalid-query suppression. +Publish successful workspace results even when another workspace fails. + +Permit at most 5 fallback targets per workspace and 10 per cycle globally. +Rotate both workspaces and targets across cycles. Unsupported GraphQL clients +use the same budget. Stop the affected workspace on auth or rate-limit failure. +Do not mark deferred work as checked or consume admission permits indefinitely. + +Add TestPRWatchFallbackBudgetAndFairness to poller_test.go. Cover multiple +workspaces, successful earlier batches, unsupported clients, transient errors, +rate limits during fallback, cancellation, event publication, and fair rotation. +Retain TestCheckPRWatches_BatchedAuthFailureSkipsPerWatchFallback. + +## Out of scope + +New UI controls, credential policies, global schedulers, and unrelated providers. + +## Acceptance + +- Meet every linked acceptance criterion with deterministic regressions. +- Preserve existing policy, scope, and error-handling contracts. +- Record the expected red test result and passing package results. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/github -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/github/poller.go` +- `apps/backend/internal/github/poller_test.go` +- `apps/backend/internal/github/poller_circuit_test.go` +- `apps/backend/internal/github/service_pr_watch_batched.go` + +## Dependencies + +04-minimal-cleanup. + +## Risks + +A target can cost several HTTP requests. The budget limits targets, not pagination. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/github-pr-discovery-health.md) +- [Design](../../specs/integrations/system-design/github-pr-discovery-health.md) +- Source files and existing adjacent tests listed above. New helper/test files are created as needed. + +## Results + +Implemented per-workspace PR batch outcomes and bounded REST fallback. A +successful workspace publishes its results immediately and is never replayed +through REST when another workspace fails. Authentication, rate-limit, and +invalid-query outcomes remain deferred; transient failures and clients without +GraphQL support use a rotating round-robin fallback with a five-target +workspace budget and ten-target cycle budget. Fallback stops the affected +workspace on rate-limit or authentication errors and stops the cycle on +cancellation. The passive workspace refresh path applies the same five-target +workspace and ten-target rolling one-minute budgets with fair target rotation; +it classifies batch failures before fallback and runs fallback checks +sequentially so an authentication or rate-limit error leaves no unscheduled +tail of calls for that workspace. +Equivalent workspace targets are deduplicated before workspace and global +budgets are admitted, and one fallback result is applied to every watch in the +target group. + +Added deterministic coverage for mixed workspaces, successful event +publication, unsupported and transient clients, target rotation, fallback +budgets, rate-limit stopping, and cancellation. Verification passed: + +```bash +(cd apps/backend && go test ./internal/github -count=1) +``` diff --git a/docs/plans/watch-task-cleanup/task-06-adaptive-discovery.md b/docs/plans/watch-task-cleanup/task-06-adaptive-discovery.md new file mode 100644 index 0000000000..0182d124ad --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-06-adaptive-discovery.md @@ -0,0 +1,153 @@ +--- +id: "06-adaptive-discovery" +title: "Back off idle searching watches" +status: done +wave: 6 +depends_on: ["05-bounded-fallback"] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-GITHUB-PR-POLLING-001 +acceptance_criteria: + - AC-INTEGRATIONS-GITHUB-PR-POLLING-001.1 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-001.2 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-001.3 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-001.4 + - AC-INTEGRATIONS-GITHUB-PR-POLLING-001.5 +system_design: + - ../../specs/integrations/system-design/github-pr-discovery-health.md +--- + +# Task 06: Back off idle searching watches + +## Summary + +Back off idle searching watches. Implement with TDD and preserve workspace identity. + +## In scope + +Add a due-target selector with an injected clock. Keep the scheduler tick at +1 minute. Searching intervals are 1 minute below 2 hours idle or while running, +15 minutes from 2 to below 24 hours, and 30 minutes thereafter. +New watches receive a next-tick attempt. Known PRs remain at 1 minute. + +Load task activity in bulk through a narrow adapter. Reuse task-domain query +patterns without treating generic UpdatedAt or provider notifications as activity. +Include session state, human conversation, agent execution, and observed branch +commit/push activity. Missing evidence retains fast discovery. No provider reads +or remote Git scans are allowed for activity classification. + +Use the fastest eligible member for shared discovery targets. Preserve due +state across reconciliation and restart using existing persisted activity and +LastCheckedAt. Passive sync must honor scheduling; explicit refresh bypasses +idleness only. Keep quota/auth admission intact. + +Add TestSearchingWatchAdaptiveSchedule in a new poller_schedule_test.go. +Cover exact boundaries, restart, activity after a slow check, unknown activity, +all-idle and mixed shared targets, archive/delete, and external PR discovery +without a new commit. Add service-level passive/explicit refresh regressions. +Inspect LoadTaskLastActivity in task/repository/sqlite/task_status_summary.go +as a query pattern, not as an assumed clean activity signal. + +Update `docs/public/integrations.md` during implementation to describe the +shipped freshness bounds. Keep the public guide unchanged during planning. +Run these additional checks after that documentation edit: + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` + +## Out of scope + +New UI controls, credential policies, global schedulers, and unrelated providers. + +## Acceptance + +- Meet every linked acceptance criterion with deterministic regressions. +- Preserve existing policy, scope, and error-handling contracts. +- Record the expected red test result and passing package results. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/github -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/github/poller.go` +- `apps/backend/internal/github/poller_schedule.go` +- `apps/backend/internal/github/poller_schedule_test.go` +- `apps/backend/internal/github/store.go` +- `apps/backend/internal/github/models.go` +- `apps/backend/internal/github/service_pr_watch.go` +- `apps/backend/internal/github/service_pr_watch_batched.go` +- `apps/backend/internal/task/repository/interface.go` (only if the activity adapter needs a new task-domain query) +- `apps/backend/internal/task/repository/sqlite/task_status_summary.go` and adjacent tests + +If the task activity query changes, also run: + +```bash +(cd apps/backend && go test ./internal/task/repository/sqlite -run "Activity|Watch" -count=1) +``` + +- `docs/public/integrations.md` + +## Dependencies + +05-bounded-fallback. + +## Risks + +Provider writes can contaminate generic timestamps. Passive UI requests can defeat backoff. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/github-pr-discovery-health.md) +- [Design](../../specs/integrations/system-design/github-pr-discovery-health.md) +- Source files and existing adjacent tests listed above. New helper/test files are created as needed. + +## Results + +Implemented adaptive passive discovery scheduling with a one-minute scheduler +tick. Searching watches check every minute while running or recently active, +then back off to 15 minutes after two hours of inactivity and 30 minutes after +24 hours. A newly created watch uses the newer of its creation time and the +authoritative task activity timestamp, so an old task does not make a fresh +watch enter a slow tier after its first check. Known pull requests remain on +the one-minute cadence, and activity, restart, archive, and deletion behavior +retain due state correctly. + +Added a bulk task-activity projection and backend adapter. The projection uses +human conversation, agent execution, session state, and observed branch +activity, while excluding generic task timestamps, lifecycle messages, and +live-monitor snapshots. Shared discovery targets use the fastest eligible +watch, while numbered and searching watches sharing a branch keep separate due +admission. Passive refresh and its bounded per-watch fallback honor due admission; +explicit refresh remains immediate. + +Passive workspace reads also use a one-minute per-workspace admission cooldown +when no stale task requires refresh. New or reset watches clear that cooldown, +and stale-task refreshes bypass it. + +The task WebSocket sync boundary now carries an explicit refresh bit. Automatic +subscription, reconnect, and retry requests use passive due admission, while a +user refresh bypasses idle admission and still obeys quota and authentication +gates. An explicit refresh supersedes an active passive provider attempt rather +than reusing its result; concurrent explicit refreshes still share provider +work, and discovery retry deadlines remain authoritative. Added deterministic scheduler, repository-query, production-shaped +handler, and passive/explicit refresh regressions. Verification passed: + +```bash +(cd apps/backend && go test ./internal/github -count=1) +(cd apps/backend && go test ./internal/task/repository/sqlite -run "Activity|Watch" -count=1) +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` diff --git a/docs/plans/watch-task-cleanup/task-07-actions-cache.md b/docs/plans/watch-task-cleanup/task-07-actions-cache.md new file mode 100644 index 0000000000..80cd912c4a --- /dev/null +++ b/docs/plans/watch-task-cleanup/task-07-actions-cache.md @@ -0,0 +1,137 @@ +--- +id: "07-actions-cache" +title: "Cache Actions observations with bounded expiry" +status: done +wave: 7 +depends_on: ["06-adaptive-discovery"] +plan: "plan.md" +requirements: + - REQ-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002 +acceptance_criteria: + - AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.1 + - AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.2 + - AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.3 + - AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.4 +system_design: + - ../../specs/integrations/system-design/github-workflow-attention.md +--- + +# Task 07: Cache Actions observations with bounded expiry + +## Summary + +Cache Actions observations with bounded expiry. Implement with TDD and preserve workspace identity. + +## In scope + +Share workflow-run and job observations across batch enrichment and service +status/feedback reads. Scope by credential generation and repository; runs also +use head SHA, and jobs use run ID plus attempt. Classify each PR separately. + +Use 30 seconds for empty, pending, running, unknown, or attention-required data. +Use 5 minutes only for nonempty completed data without attention requirements. +Keep 512 entries each for runs and jobs, and coalesce concurrent misses. Do not +cache errors or change merge/check eligibility. Preserve the existing 4-worker, +5-second batch enrichment budget. + +Explicit refresh invalidates relevant outer and Actions cache entries. Prevent +in-flight old observations from repopulating after invalidation. Head and +credential changes must select fresh entries. Future mutation hooks can reuse +this invalidation seam; no approval or rerun UI is added. + +Add TestWorkflowAttentionCacheExpiryAndInvalidation to a new +workflow_attention_cache_test.go. Cover same-SHA reruns, new run attempts, +empty/approval/error results, concurrent callers, credential isolation, different +PRs sharing one SHA, explicit refresh, and bounded eviction. Use fake time. +Retain the existing workflow attention classification suite. + +Update `docs/public/integrations.md` during implementation to describe the +shipped freshness bounds. Keep the public guide unchanged during planning. +Run these additional checks after that documentation edit: + +```bash +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` + +## Out of scope + +New UI controls, credential policies, global schedulers, and unrelated providers. + +## Acceptance + +- Meet every linked acceptance criterion with deterministic regressions. +- Preserve existing policy, scope, and error-handling contracts. +- Record the expected red test result and passing package results. + +## Verification + +Run from the repository root. + +```bash +(cd apps/backend && go test ./internal/github -count=1) +git diff --check +``` + +## Files likely touched + +- `apps/backend/internal/github/service_pr_watch_batched.go` +- `apps/backend/internal/github/service_pr.go` +- `apps/backend/internal/github/service.go` +- `apps/backend/internal/github/ttl_cache.go` +- `apps/backend/internal/github/workflow_attention.go` +- `apps/backend/internal/github/workflow_attention_cache.go` +- `apps/backend/internal/github/workflow_attention_cache_test.go` +- `apps/backend/internal/github/workflow_attention_test.go` + +- `docs/public/integrations.md` + +## Dependencies + +06-adaptive-discovery. + +## Risks + +Same-SHA results can change. Never freeze a completed workflow collection indefinitely. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/integrations/requirements/github-workflow-attention.md) +- [Design](../../specs/integrations/system-design/github-workflow-attention.md) +- Source files and existing adjacent tests listed above. New helper/test files are created as needed. + +## Results + +Implemented bounded Actions observation caches shared by batched workflow +enrichment and service status, feedback, and watch reads. Run entries are +scoped by credential, repository, and head SHA. Job entries are scoped by +credential, repository, run ID, and attempt. Completed nonempty observations +without attention use a five-minute TTL; empty, changing, unknown, and +attention-required observations use 30 seconds. Each cache has 512 entries, +singleflight coalescing, and provider-error omission. + +Explicit PR refresh invalidates the affected outer status or feedback entry +and repository Actions entries. Automatic task subscriptions, reconnects, and +retries use passive sync and preserve a completed Actions observation; only a +user refresh carries explicit refresh intent and invalidates the relevant +entries. Per-key epochs prevent an older in-flight read from repopulating an +invalidated cache. Credential changes clear the caches and credential +generations remain part of every workspace cache key. Existing workflow +classification, fork identity, check state, merge eligibility, and the +four-worker, five-second batch budget remain unchanged. + +Added `TestWorkflowAttentionCacheExpiryAndInvalidation` and an in-flight +invalidation regression covering same-SHA reruns, attempts, empty and +attention data, provider errors, concurrent callers, credential isolation, +shared SHAs, explicit refresh, and bounded eviction. The focused test failed +before the cache seams existed as expected. Verification passed: + +```bash +(cd apps/backend && go test ./internal/github -count=1) +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +``` diff --git a/docs/public/integrations.md b/docs/public/integrations.md index f2de93aa02..a41747e0be 100644 --- a/docs/public/integrations.md +++ b/docs/public/integrations.md @@ -109,6 +109,19 @@ The **PR discovery failed** warning is separate from the quota values: When discovery succeeds, the warning clears and the status revision advances. If the warning remains after a quota refresh, wait for the retry time or verify the selected connection and repository scope. +#### Pull request watch freshness + +Background checks for a known pull request run at least once per minute. A +watch that is still searching for a pull request also checks once per minute +while its task is running or has activity within the last two hours. Searching +checks use a 15-minute interval after two hours of inactivity and a 30-minute +interval after 24 hours. The checks continue at the 30-minute interval, so a +pull request opened later on an unchanged branch remains discoverable. + +Passive workspace refreshes follow these intervals. An explicit pull request +refresh checks immediately, subject to the workspace's authentication and +GitHub rate limits. + ### Automation and personal identity PAT and CLI connections are human identities. They provide both workspace automation and the fallback identity for **My GitHub** views and user-triggered actions. Settings show this shared identity inside **Workspace GitHub access** instead of repeating it as a separate **My GitHub identity** section. @@ -341,6 +354,17 @@ For a task with linked GitHub pull requests, open the PR status control above th If a current pull-request head has a GitHub Actions workflow that requires maintainer approval, the PR status control shows **Awaiting maintainer approval**, even when GitHub reports no checks. Detailed desktop and mobile views show the workflow name, the reason, and a link to GitHub. Approval-only workflow attention is not a failed check, does not start **Auto-fix CI & address comments**, and does not make the pull request ready for **Auto-merge or requeue when ready**. If GitHub does not provide enough evidence, Kandev keeps the workflow state unavailable or marks the last same-head observation as stale instead of claiming approval. +Kandev refreshes GitHub Actions observations for empty, running, changing, or +attention-required results within 30 seconds. Nonempty completed results with +no attention requirement remain cached for up to five minutes. The cache is +Run observations are scoped to the selected credential, repository, and head +SHA. Job observations are scoped to the selected credential, repository, run +ID, and workflow attempt. +An explicit pull request refresh clears the relevant observation immediately. +Reruns on the same SHA appear after the completed-result cache expires or after +an explicit refresh. Provider errors are retried and are not stored as an +empty workflow result. + This is a GitHub-only lifecycle feature. Kandev reuses the existing lightweight task PR poller, which checks watched linked PRs roughly once per minute; it does not add a separate scheduler. Saving enabled options also evaluates the task's current linked PRs without waiting for the next poll. Auto-fix waits for the pull request checks to settle, then sends one repair round for newly failed checks, review comments, an ordinary merge conflict, or an actionable merge-queue removal. It snapshots each check, comment, conflict, and removal state, so repeated observations do not create duplicate rounds. An authoritative conflict resolution clears its checkpoint without using a round; an unknown mergeability state preserves the checkpoint. Updating an already queued auto-fix message does not use another round, and auto-fix pauses after 10 rounds unless the user disables and re-enables it. diff --git a/docs/specs/integrations/requirements/github-pr-discovery-health.md b/docs/specs/integrations/requirements/github-pr-discovery-health.md index 36384d3f94..d216f5910a 100644 --- a/docs/specs/integrations/requirements/github-pr-discovery-health.md +++ b/docs/specs/integrations/requirements/github-pr-discovery-health.md @@ -72,6 +72,21 @@ and provider status. Task and UI systems consume its results. immediate fallback using the same invalid query or an automatic retry more often than once per minute for that discovery target. +### REQ-INTEGRATIONS-GITHUB-PR-POLLING-001: Adaptive background discovery + +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-001.1:** Searching watches shall use a 1-minute interval while their task runs or has less than 2 hours of inactivity. They shall use 15 minutes from 2 hours to less than 24 hours, then 30 minutes. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-001.2:** Task execution, task conversation activity, or an observed branch commit or push shall restore fast discovery. A new watch shall receive an initial attempt on the next poll tick. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-001.3:** Idle discovery shall not stop permanently. A PR opened externally on an unchanged branch shall remain discoverable. Known PRs shall retain their 1-minute background cadence. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-001.4:** Explicit PR refresh shall bypass the idle schedule but obey existing rate-limit and authentication admission. Passive frontend refreshes shall obey the idle schedule for searching targets. An explicit refresh shall not reuse the result of an active passive provider attempt. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-001.5:** Poll attempts and provider status writes shall not count as task activity. Shared targets shall use the fastest eligible member's schedule. Archived and deleted members shall not accelerate it. + +### REQ-INTEGRATIONS-GITHUB-PR-POLLING-002: Bounded batch fallback + +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-002.1:** Rate-limit, authentication, and invalid-query batch failures shall not trigger per-watch fallback for the affected workspace. Successful workspace batches shall not be repeated because another workspace failed. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-002.2:** Transient failures shall permit at most 5 fallback target checks per workspace and 10 across one poll cycle. Deferred targets shall retain their state and receive a fair opportunity on later cycles. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-002.3:** A rate-limit or authentication error during fallback shall stop remaining checks for that workspace. Cancellation shall stop the cycle. Existing retry deadlines remain authoritative. +- **AC-INTEGRATIONS-GITHUB-PR-POLLING-002.4:** Clients without GraphQL support shall retain REST support under the same bounded schedule. Deferral shall not imply a successful provider observation or discard pending status events. + ## Exclusions No new credentials, automatic PR creation, inferred PR links from chat, changed @@ -86,3 +101,7 @@ runtime status, not a permanent incident archive. - [System design](../system-design/github-pr-discovery-health.md) - [Implementation package](../../../plans/github-pr-discovery-health/plan.md) - [Watch reconciliation repair](../../../plans/github-pr-watch-reconciliation/plan.md) + +## Polling efficiency package + +- [Implementation plan](../../../plans/watch-task-cleanup/plan.md) diff --git a/docs/specs/integrations/requirements/github-workflow-attention.md b/docs/specs/integrations/requirements/github-workflow-attention.md index 07b010bd2d..0e9766f8f3 100644 --- a/docs/specs/integrations/requirements/github-workflow-attention.md +++ b/docs/specs/integrations/requirements/github-workflow-attention.md @@ -26,11 +26,18 @@ The shared [task summary](../../ui/requirements/pr-task-status-summary.md) owns - **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.2:** Approval-only workflows shall not count as failed, passed, or running checks. They shall not trigger CI repair rounds or enable automatic merging. - **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.3:** Desktop task summaries and the existing phone PR drawer shall show the same localized reason. Detailed surfaces shall link to GitHub. - **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.4:** When approval and failed checks coexist, Kandev shall show both. Each linked PR shall retain its own status. -- **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.5:** After approval, cancellation, rerun, or a head change, the next successful refresh shall reconcile the reason. Reloads shall retain the latest stored observation. +- **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.5:** After approval, cancellation, rerun, or a head change, the next successful provider refresh shall reconcile the reason. Explicit refresh shall bypass cached Actions observations. Reloads shall retain the latest stored observation. - **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.6:** Missing checks or unstable mergeability alone shall not imply approval. Unavailable provider evidence shall remain distinct from an observed absence of approval gates. - **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.7:** When GitHub requires an action but the reason is ambiguous, Kandev shall show "Workflow needs attention" instead of an approval claim. - **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001.8:** An approval reason shall replace unexplained "unstable" copy in the affected summary. Independent conflicts, reviews, queue state, and actual failures shall remain visible. +### REQ-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002: Bounded Actions caching + +- **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.1:** Equivalent Actions reads shall share cached results and concurrent fetches within one credential scope, repository, and head SHA. Different credentials shall never share results. +- **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.2:** Background observations shall expire after 30 seconds for empty, pending, running, or attention-required results. Nonempty completed results without attention requirements shall expire after 5 minutes, including success and failure. +- **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.3:** A head change, credential change, or explicit refresh shall prevent reuse of the affected stale observation. Same-SHA reruns shall become visible after expiry and the next successful poll. +- **AC-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002.4:** Provider errors shall not become cached absence. Cached observations shall not become merge eligibility evidence. Classification shall remain specific to each PR, including fork identity. + ## Out of scope - Approving, rerunning, cancelling, or merging GitHub workflows or pull requests. @@ -41,3 +48,5 @@ The shared [task summary](../../ui/requirements/pr-task-status-summary.md) owns ## Implementation plans - [Workflow attention](../../../plans/github-workflow-attention/plan.md) + +- [Polling efficiency](../../../plans/watch-task-cleanup/plan.md) diff --git a/docs/specs/integrations/requirements/watch-task-cleanup.md b/docs/specs/integrations/requirements/watch-task-cleanup.md new file mode 100644 index 0000000000..6deb39dbeb --- /dev/null +++ b/docs/specs/integrations/requirements/watch-task-cleanup.md @@ -0,0 +1,41 @@ +--- +status: draft +system: integrations +created: 2026-09-22 +owners: + - kandev +--- + +# Watch task cleanup + +## Overview + +Integration cleanup must preserve provider capacity for active work. Historical +tasks must not generate repeated provider requests. Integrations own this +contract because they select cleanup candidates and request provider state. + +## Requirements + +### REQ-INTEGRATIONS-WATCH-CLEANUP-001: Cleanup eligibility + +- **AC-INTEGRATIONS-WATCH-CLEANUP-001.1:** Every archived or deleted task shall generate zero provider state requests during a cleanup sweep. This applies to GitHub reviews and issues, GitLab reviews and issues, and Azure DevOps reviews and work items. +- **AC-INTEGRATIONS-WATCH-CLEANUP-001.2:** In a collection containing active, archived, deleted, and unassigned records, each existing unarchived task shall remain eligible independently. Eligibility shall not depend on another record's state. +- **AC-INTEGRATIONS-WATCH-CLEANUP-001.3:** GitHub reservations without an assigned task shall remain eligible. Terminal reservations shall be removed under `auto` and `always`, without increasing the deleted-task count. `never` shall preserve them without provider requests. +- **AC-INTEGRATIONS-WATCH-CLEANUP-001.4:** Existing policies shall remain effective for active tasks. `never` shall make no cleanup state requests. `always` shall delete terminal tasks. `auto` shall preserve user-engaged tasks and existing lifecycle-prompt protections. +- **AC-INTEGRATIONS-WATCH-CLEANUP-001.5:** Background and manual cleanup shall use the same task eligibility rules. Watch reset, explicit watch deletion, workspace authorization, and deduplication shall retain their existing behavior. + +- **AC-INTEGRATIONS-WATCH-CLEANUP-001.6:** Review cleanup shall request only PR state and, for open PRs, review and viewer identity evidence. It shall not request comments, checks, workflow runs, or workflow jobs. + +### REQ-INTEGRATIONS-WATCH-CLEANUP-002: GitHub rate-limit termination + +- **AC-INTEGRATIONS-WATCH-CLEANUP-002.1:** After a GitHub cleanup batch receives a recognized rate-limit error, it shall start no further row checks. Completed deletions shall remain counted. Unprocessed tasks and reservations shall remain intact. +- **AC-INTEGRATIONS-WATCH-CLEANUP-002.2:** Rate-limit recognition shall include HTTP 429 and HTTP 403 responses indicating rate limits or abuse detection. An unrelated HTTP 403 shall not be classified as a rate limit. +- **AC-INTEGRATIONS-WATCH-CLEANUP-002.3:** A cleanup sweep shall return its rate-limit error to its caller. That poll cycle shall skip its remaining cleanup calls after this error. Ordinary row errors shall preserve the row and retain existing continuation behavior. +- **AC-INTEGRATIONS-WATCH-CLEANUP-002.4:** A later scheduled sweep shall remain able to retry. Cancellation shall stop remaining batch work. Requests already running within one feedback fetch may finish or be cancelled. + +## Exclusions + +No cleanup policy, task retention setting, UI control, or provider credential +contract is added. Unarchived tasks on a Done step remain active candidates. +This contract does not guarantee spare quota against unrelated API consumers. +GitLab and Azure DevOps reservation semantics remain unchanged. diff --git a/docs/specs/integrations/system-design/github-pr-discovery-health.md b/docs/specs/integrations/system-design/github-pr-discovery-health.md index 204a817199..abbdd4c843 100644 --- a/docs/specs/integrations/system-design/github-pr-discovery-health.md +++ b/docs/specs/integrations/system-design/github-pr-discovery-health.md @@ -2,6 +2,8 @@ status: draft system: integrations requirements: + - REQ-INTEGRATIONS-GITHUB-PR-POLLING-001 + - REQ-INTEGRATIONS-GITHUB-PR-POLLING-002 - REQ-INTEGRATIONS-GITHUB-PR-DISCOVERY-001 - REQ-INTEGRATIONS-GITHUB-PR-DISCOVERY-HEALTH-001 created: 2026-09-11 @@ -187,3 +189,86 @@ historical scope and results. - [Sync coordination](github-task-pr-sync-coordination.md) - [Authentication ownership ADR](../../../decisions/0047-github-authentication-ownership.md) - [Implementation package](../../../plans/github-pr-discovery-health/plan.md) + +## Adaptive discovery schedule + +`REQ-INTEGRATIONS-GITHUB-PR-POLLING-001` maps to this section. +Keep `defaultPRPollInterval` at 1 minute. Select due searching targets before +batch construction. Known numbered watches keep the current cadence. + +| Searching task state | Minimum interval | +| --- | --- | +| Running or activity age below 2 hours | 1 minute | +| Activity age from 2 hours to below 24 hours | 15 minutes | +| Activity age at least 24 hours | 30 minutes | + +Use injected time and `PRWatch.LastCheckedAt` for due checks. Round execution +to the next 1-minute tick, so actual delay may include one extra tick. +Use watch creation as an activity baseline for the first 2 hours. +A never-checked watch is immediately eligible. Missing activity evidence +fails open to fast polling. No new random delay is required in this package. + +Add a narrow task-activity projection, loaded in bulk for eligible watch tasks. +Reuse the bounded query pattern in +`task/repository/sqlite/task_status_summary.go:LoadTaskLastActivity`, but not +its generic task-update source. Preserve that existing UI query unchanged. +Use persisted session execution state and conversation timestamps, plus observed +commit/push activity. Do not use generic task or watch `UpdatedAt`: GitHub sync +can update those fields itself. Exclude automatic provider notifications from +conversation activity. Include human messages and agent execution activity. +Do not scan remote branches or invoke GitHub to calculate idleness. +Record observed branch activity from existing local Git event paths; absence of +such an event is not proof that no remote push occurred. + +An activity transition restores next-tick eligibility even after a slow poll. +Persisted task activity and last-check timestamps survive restart. A runtime +branch-activity marker may reset on restart without stopping periodic discovery. +Expire runtime markers after 24 hours and remove them with their watch. +Reconciliation of an unchanged watch must preserve its age and last check. + +Group equivalent discovery targets before admission. Any active member selects +the 1-minute schedule for that target; do not duplicate the provider query. +Numbered targets and searching targets remain separate groups even when they +share a repository and branch, because their polling cadences differ. +Passive task/page refresh must use the same admission rule or it defeats the +background savings. Explicit user refresh bypasses only idle admission, never +quota or auth gates. Preserve `PRSyncFreshnessWindow` at 30 seconds for existing +sync consumers and keep the distinction between automatic and explicit reads. +If a passive provider attempt is already active, an explicit refresh starts a +new admitted attempt instead of joining the passive result. The superseded +attempt cannot publish its result, and concurrent explicit refreshes share the +explicit batch singleflight key. A retry deadline from an authentication, +rate-limit, or invalid-query failure remains authoritative. +When a passive workspace read finds no stale task, admit workspace discovery +reads at most once per minute. A newly created or reset watch clears that +cooldown, and stale-task refreshes bypass it. + +## Bounded fallback + +`REQ-INTEGRATIONS-GITHUB-PR-POLLING-002` maps to this section. +Replace the poller's all-or-nothing boolean batch outcome with per-workspace +outcomes: completed, deferred, fallback-eligible. Publish successful results +without waiting for every workspace to succeed. Preserve existing shared +health admission for rate limits and invalid queries, and auth circuits. + +Allow 5 fallback target checks per workspace, capped at 10 per cycle globally. +Apply this limit to unsupported-GraphQL clients too. Rotate workspace and target +selection between cycles so fixed ordering cannot starve later targets. +These are target-check limits, not HTTP-request limits: a check may paginate +or fetch several resources. Stop a workspace on the first authentication, +configuration, rate-limit, or invalid-query error and check cancellation before +each target. Never retry successful batches via REST. +Do not mark deferred targets as successfully checked. +Deduplicate equivalent watches before consuming either budget and apply one +fallback result to every watch in the selected target group. + +Retain existing failure schedules: discovery retry starts at 1 minute and caps +at 15 minutes; auth/config circuits start at 2 minutes and cap at 6 hours with +25 percent jitter. Provider reset deadlines and credential changes retain +existing handling. This package adds no independent retry loop. + +## Polling efficiency delivery + +See [the implementation plan](../../../plans/watch-task-cleanup/plan.md). +Use deterministic clock tests for exact 2-hour and 24-hour boundaries, +restart, mixed groups, passive reads, explicit refresh, and fallback fairness. diff --git a/docs/specs/integrations/system-design/github-workflow-attention.md b/docs/specs/integrations/system-design/github-workflow-attention.md index af14f3935d..75144127ea 100644 --- a/docs/specs/integrations/system-design/github-workflow-attention.md +++ b/docs/specs/integrations/system-design/github-workflow-attention.md @@ -2,6 +2,7 @@ status: draft system: integrations requirements: + - REQ-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002 - REQ-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-001 --- @@ -144,3 +145,42 @@ Desktop and mobile Playwright tests verify the existing entry points and the pro - [Requirements](../requirements/github-workflow-attention.md) - [Shared task summary design](../../ui/system-design/pr-task-status-summary.md) - [Implementation plan](../../../plans/github-workflow-attention/plan.md) + +## Expiring Actions observations + +`REQ-INTEGRATIONS-GITHUB-WORKFLOW-ATTENTION-002` maps to this section. +Share raw workflow-run observations across batched enrichment and service +feedback/status reads. Keep per-PR classification outside the shared cache. +Key entries by credential scope and generation, normalized repository identity, +and head SHA. Reuse the bounded `ttlCache` and its singleflight/invalidation +patterns; add per-entry expiry support if needed. Keep at most 512 run entries +and 512 job entries. Errors are not cached. + +| Observation | TTL | +| --- | --- | +| Empty list or any pending/running/unknown status | 30 seconds | +| Any approval/action-required or ambiguous attention result | 30 seconds | +| Nonempty, all completed, no attention requirement | 5 minutes | + +A SHA does not freeze the workflow run collection. Reruns and new runs can +arrive for the same SHA. Never use an infinite TTL. Cache job observations by +credential scope, repository, run ID, and attempt, with the same expiry classes. +A run-attempt change must not reuse jobs from an earlier attempt. + +Keep existing batched enrichment limits: concurrency 4 and total budget +5 seconds. Preserve the outer status/search cache TTL of 30 seconds and the +feedback TTL of 8 seconds. Long Actions entries only affect workflow attention, +not checks, review status, or auto-merge decisions. + +Explicit refresh invalidates the relevant Actions and outer response entries, +then coalesces concurrent refreshes. Generation checks prevent an older +in-flight request from repopulating invalidated data. Passive page polling +uses the cache. Route future approval/rerun mutations through this invalidation +seam; adding those mutations is outside this package. + +External reruns on a completed SHA can take up to 5 minutes plus the next +1-minute poll tick to appear. Attention and running results use the short TTL. +Provider errors retain unknown/stale-positive semantics and existing retry +admission. Terminal PRs continue to skip Actions reads altogether. + +See [the implementation plan](../../../plans/watch-task-cleanup/plan.md). diff --git a/docs/specs/integrations/system-design/watch-task-cleanup.md b/docs/specs/integrations/system-design/watch-task-cleanup.md new file mode 100644 index 0000000000..7a63508069 --- /dev/null +++ b/docs/specs/integrations/system-design/watch-task-cleanup.md @@ -0,0 +1,126 @@ +--- +status: draft +system: integrations +requirements: + - REQ-INTEGRATIONS-WATCH-CLEANUP-001 + - REQ-INTEGRATIONS-WATCH-CLEANUP-002 +created: 2026-09-22 +owners: + - kandev +--- + +# Watch task cleanup design + +## Ownership and mapping + +Integration stores select candidates. Integration services preserve cleanup +policies and call the existing task deletion adapters. + +| Requirement | Design section | +| --- | --- | +| REQ-INTEGRATIONS-WATCH-CLEANUP-001 | Candidate selection and compatibility | +| REQ-INTEGRATIONS-WATCH-CLEANUP-002 | GitHub batch termination | + +## Candidate selection and compatibility + +Cleanup queries use this predicate for each deduplication row `d`: + +```sql +d.task_id = '' OR EXISTS ( + SELECT 1 FROM tasks t + WHERE t.id = d.task_id AND t.archived_at IS NULL +) +``` + +Keep watch, workspace, and generation predicates outside this parenthesized +expression. A plain inner join loses reservations. A left join with only an +archive-null check incorrectly includes deleted tasks. + +Apply this filter to GitHub `ListReviewPRTasksByWatch`, `ListAllReviewPRTasks`, +`ListIssueWatchTasksByWatch`, and `ListAllIssueWatchTasks` in `store.go`. +Workspace cleanup already uses the all-record queries and watch ownership. +There are no GitHub workspace-specific list methods in this checkout. + +`DeleteReviewWatch` and `DeleteIssueWatch` also consume the by-watch methods. +Move their explicit deletion inventory to the existing unfiltered +`ListReviewPRTaskIDsByWatch` and `ListIssueWatchTaskIDsByWatch` methods. +Preserve their authorization, empty-ID handling, and best-effort deletion. +Reset and deduplication lookups remain unfiltered. + +GitLab applies the predicate to all, workspace, and by-watch review and issue +queries in `store_watches.go`. Preserve workspace joins, generation values, +and ordering. Its cleanup currently skips empty reservations and retains that +behavior. + +Azure DevOps applies the predicate to `ListWorkItemWatchTasks` and +`ListPullRequestWatchTasks` in `watch_store.go`. Preserve watch generation +scoping. Its cleanup also continues to skip empty reservations. + +Archived and missing-task records are ignored, not purged by this change. +Retaining them avoids changing deduplication and allows restored tasks to +become eligible again. Existing task-deletion handlers may still remove them. +No schema migration or index is needed: the existence query uses the task key. + +Eligibility reflects committed task state at query time. This change does not +make remote requests atomic with a concurrent archive or deletion. + +## GitHub batch termination + +Extend the private deletion gates to return errors alongside eligibility and +reason. Extend both batch helpers to return a partial count and error. +Use `errors.As` with `GitHubAPIError` and the existing +`isGitHubRateLimitAPIError` classifier in `errors.go`. + +Check cancellation before each row. On a rate-limit error, return immediately, +including on the empty-reservation path. Ordinary fetch errors retain existing +per-row failure tracking and continuation. Do not classify every 403 as a +rate limit. Propagate rate-limit errors from `GetAuthenticatedUser` as well. + +Per-watch, global, orphan, and workspace cleanup entry points propagate the +partial count and error. Global sweeps stop before the next workspace batch. +In each review or issue poll cycle, a rate-limit error suppresses subsequent +cleanup calls, including its final orphan sweep. Discovery remains subject to +its existing search-bucket rules. This is a cycle-local stop, not a new global +credential circuit or scheduler. + +Keep existing client rate tracking and reset behavior. A later poll retries +normally. Emit one batch-level error through existing caller logging instead +of one new failure for each unvisited row. Unvisited rows do not increment +failure counters. +Cleanup admission evaluates the current reset deadline, so an exhausted +resource becomes eligible again after reset even if no provider call refreshes +the tracker in between sweeps. + +## Minimal review cleanup lookup + +Replace the cleanup gate's `GetPRFeedback` call with `GetPR`. For merged or +closed PRs, apply the existing policy without additional provider reads. +For open PRs, fetch `ListPRReviews` and the authenticated user only when needed +for the existing approval rule. Preserve approval matching and policy behavior. +Never fetch comments, check runs, workflow runs, or workflow jobs for cleanup. +Reuse existing authenticated-user caching. Do not introduce another cleanup TTL. +Propagate rate-limit errors from each required read to the batch stop path. +Thread the resolved automation `RateTracker` into batch admission and honor +core exhaustion before each row. Never substitute another workspace's tracker. + +## Test boundaries + +Use real store fixtures and instrumented clients. Cover mixed collections, +all cleanup entry points, policy matrices, empty reservations, preserved +deduplication, and explicit watch deletion. Fixtures representing active tasks +must insert actual task rows. Missing-task fixtures must assert zero provider +calls rather than treating an absent task as active. + +Rate tests cover wrapped typed errors, ordinary 403 responses, cancellation, +partial counts, an error on the first reservation, tracker exhaustion, and +later-cycle recovery. Poller tests prove the final orphan sweep does not restart +cleanup after a rate-limit stop. + +## Related contracts + +- [GitHub identity ownership](../../../decisions/0047-github-authentication-ownership.md) +- [PR lifecycle prompt protection](../../../decisions/0051-pr-agent-notifications-extend-task-pr-automation.md) +- [Requirements](../requirements/watch-task-cleanup.md) + +Existing authorization and lifecycle protections remain authoritative. This +local candidate filter needs no new architecture decision.