From f5a505c46873b3fd249f55de94cef96bbbf8e34a Mon Sep 17 00:00:00 2001 From: Alassane <79718216+yattdev@users.noreply.github.com> Date: Tue, 22 Sep 2026 05:31:53 -0400 Subject: [PATCH] fix: eliminate session-scoped PR-watch amplification and bound SQLite growth (#3319) Co-authored-by: ayattara Co-authored-by: Carlos Florencio --- .../runtime/lifecycle/execution_store.go | 2 + .../runtime/lifecycle/execution_store_test.go | 24 + .../agent/runtime/lifecycle/metrics_vars.go | 7 + .../acp_command_prefix_allowlist_test.go | 33 + .../agentctl/server/utility/acp_executor.go | 15 +- apps/backend/internal/backendapp/helpers.go | 31 + .../internal/common/authcircuit/circuit.go | 175 ++++ .../common/authcircuit/circuit_test.go | 179 ++++ .../internal/db/sqlguard/exemptions.json | 5 + apps/backend/internal/db/sqlite.go | 6 +- apps/backend/internal/github/errors.go | 54 ++ .../internal/github/load_validation_test.go | 156 ++++ apps/backend/internal/github/metrics_vars.go | 40 +- .../internal/github/metrics_vars_test.go | 69 ++ apps/backend/internal/github/poller.go | 113 ++- .../backend/internal/github/poller_circuit.go | 91 ++ .../internal/github/poller_circuit_test.go | 202 +++++ apps/backend/internal/github/poller_test.go | 254 +++++- .../internal/github/service_connections.go | 23 + .../service_pr_discovery_health_test.go | 9 +- .../internal/github/service_pr_watch.go | 50 +- .../service_pr_watch_multi_branch_test.go | 154 +++- ..._pr_watch_workspace_group_redirect_test.go | 31 + apps/backend/internal/github/store.go | 809 +++++++++++++++++- .../github/store_pr_watch_migration_test.go | 527 ++++++++++++ .../internal/github/store_watch_reset_test.go | 258 ++++++ apps/backend/internal/health/checks.go | 55 ++ apps/backend/internal/health/checks_test.go | 53 ++ apps/backend/internal/health/types.go | 20 + apps/backend/internal/launcher/launcher.go | 3 + .../internal/launcher/maintenance_database.go | 234 +++++ .../launcher/maintenance_database_test.go | 270 ++++++ apps/backend/internal/maintenance/backup.go | 70 ++ apps/backend/internal/maintenance/compact.go | 197 +++++ apps/backend/internal/maintenance/execute.go | 106 +++ .../internal/maintenance/fixture_test.go | 214 +++++ .../internal/maintenance/maintenance.go | 55 ++ apps/backend/internal/maintenance/report.go | 193 +++++ apps/backend/internal/maintenance/run.go | 212 +++++ apps/backend/internal/maintenance/run_test.go | 448 ++++++++++ .../orchestrator/event_handlers_github.go | 138 ++- .../event_handlers_github_test.go | 151 ++++ .../messagequeue/count_pending_test.go | 52 ++ .../orchestrator/messagequeue/metrics_vars.go | 43 + .../messagequeue/repository_memory.go | 14 + .../messagequeue/repository_sqlite.go | 28 + .../orchestrator/messagequeue/service.go | 3 + .../storeconformance/owner_actions.go | 3 +- .../internal/system/database/metrics_vars.go | 21 + .../backend/internal/system/database/stats.go | 72 +- .../internal/system/database/stats_test.go | 51 ++ .../task/handlers/message_handlers.go | 40 +- .../handlers/message_list_handlers_test.go | 35 +- .../task/handlers/process_handlers_test.go | 6 + apps/backend/internal/task/models/git.go | 10 + .../task/models/message_shell_output.go | 36 + apps/backend/internal/task/models/models.go | 14 + .../internal/task/repository/interface.go | 6 + .../internal/task/repository/sqlite/base.go | 13 + .../task/repository/sqlite/base_migrations.go | 34 + .../task/repository/sqlite/base_queries.go | 42 +- .../repository/sqlite/git_snapshot_digest.go | 140 +++ .../sqlite/git_snapshot_digest_test.go | 170 ++++ .../task/repository/sqlite/git_snapshots.go | 37 +- .../task/repository/sqlite/message.go | 30 +- .../sqlite/message_history_load_test.go | 166 ++++ .../task/repository/sqlite/message_payload.go | 279 ++++++ .../sqlite/message_payload_replay.go | 12 +- .../sqlite/message_payload_retention_test.go | 95 ++ .../repository/sqlite/message_payload_test.go | 466 ++++++++++ .../task/repository/sqlite/metrics_vars.go | 29 + .../repository/sqlite/metrics_vars_test.go | 44 + .../internal/task/repository/sqlite/plan.go | 73 ++ .../repository/sqlite/plan_retention_test.go | 176 ++++ .../internal/task/service/constants.go | 7 +- .../internal/task/service/service_messages.go | 10 + .../task/statussummary/metrics_vars.go | 15 + .../task/statussummary/metrics_vars_test.go | 34 + .../internal/task/statussummary/projector.go | 90 +- .../projector_contention_test.go | 249 ++++++ .../task/statussummary/projector_load_test.go | 186 ++++ .../projector_pending_authority_test.go | 16 + .../internal/workflowsync/circuit_test.go | 191 +++++ apps/backend/internal/workflowsync/failure.go | 69 ++ .../internal/workflowsync/failure_test.go | 95 ++ .../internal/workflowsync/metrics_vars.go | 51 ++ apps/backend/internal/workflowsync/models.go | 51 ++ apps/backend/internal/workflowsync/service.go | 172 +++- apps/backend/internal/workflowsync/store.go | 97 ++- .../workflowsync/store_postgres_test.go | 3 +- .../internal/workflowsync/store_test.go | 98 ++- apps/web/e2e/helpers/api-client.ts | 26 +- apps/web/e2e/tests/pr/pr-detection.spec.ts | 4 +- .../session/use-shell-command-output.test.ts | 128 ++- .../session/use-shell-command-output.ts | 138 ++- ...2026-08-31-task-owned-pr-watch-identity.md | 32 + .../plans/pr-watch-and-storage-bounds/plan.md | 125 +++ .../task-01-canonical-watch-migration.md | 100 +++ .../task-02-idempotent-polling-events.md | 146 ++++ .../task-03-contention-safe-projection.md | 133 +++ ...sk-04-github-auth-backoff-observability.md | 192 +++++ .../task-05-bounded-history-storage.md | 248 ++++++ .../task-06-database-maintenance-command.md | 153 ++++ .../task-07-operator-docs-load-validation.md | 176 ++++ docs/public/cli.md | 88 ++ docs/public/operations.md | 42 + docs/specs/platform/README.md | 2 + .../platform/pr-watch-and-storage-bounds.md | 76 ++ .../pr-watch-and-bounded-storage.md | 214 +++++ .../pr-watch-and-bounded-storage.md | 134 +++ docs/specs/tasks/README.md | 2 + 111 files changed, 11348 insertions(+), 221 deletions(-) create mode 100644 apps/backend/internal/agent/runtime/lifecycle/metrics_vars.go create mode 100644 apps/backend/internal/agentctl/server/utility/acp_command_prefix_allowlist_test.go create mode 100644 apps/backend/internal/common/authcircuit/circuit.go create mode 100644 apps/backend/internal/common/authcircuit/circuit_test.go create mode 100644 apps/backend/internal/github/load_validation_test.go create mode 100644 apps/backend/internal/github/poller_circuit.go create mode 100644 apps/backend/internal/github/poller_circuit_test.go create mode 100644 apps/backend/internal/github/store_pr_watch_migration_test.go create mode 100644 apps/backend/internal/launcher/maintenance_database.go create mode 100644 apps/backend/internal/launcher/maintenance_database_test.go create mode 100644 apps/backend/internal/maintenance/backup.go create mode 100644 apps/backend/internal/maintenance/compact.go create mode 100644 apps/backend/internal/maintenance/execute.go create mode 100644 apps/backend/internal/maintenance/fixture_test.go create mode 100644 apps/backend/internal/maintenance/maintenance.go create mode 100644 apps/backend/internal/maintenance/report.go create mode 100644 apps/backend/internal/maintenance/run.go create mode 100644 apps/backend/internal/maintenance/run_test.go create mode 100644 apps/backend/internal/orchestrator/messagequeue/metrics_vars.go create mode 100644 apps/backend/internal/system/database/metrics_vars.go create mode 100644 apps/backend/internal/task/repository/sqlite/git_snapshot_digest.go create mode 100644 apps/backend/internal/task/repository/sqlite/git_snapshot_digest_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/message_history_load_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/message_payload.go create mode 100644 apps/backend/internal/task/repository/sqlite/message_payload_retention_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/message_payload_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/metrics_vars.go create mode 100644 apps/backend/internal/task/repository/sqlite/metrics_vars_test.go create mode 100644 apps/backend/internal/task/repository/sqlite/plan_retention_test.go create mode 100644 apps/backend/internal/task/statussummary/metrics_vars.go create mode 100644 apps/backend/internal/task/statussummary/metrics_vars_test.go create mode 100644 apps/backend/internal/task/statussummary/projector_contention_test.go create mode 100644 apps/backend/internal/task/statussummary/projector_load_test.go create mode 100644 apps/backend/internal/workflowsync/circuit_test.go create mode 100644 apps/backend/internal/workflowsync/failure.go create mode 100644 apps/backend/internal/workflowsync/failure_test.go create mode 100644 apps/backend/internal/workflowsync/metrics_vars.go create mode 100644 docs/decisions/2026-08-31-task-owned-pr-watch-identity.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/plan.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-01-canonical-watch-migration.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-02-idempotent-polling-events.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-03-contention-safe-projection.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-04-github-auth-backoff-observability.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-05-bounded-history-storage.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-06-database-maintenance-command.md create mode 100644 docs/plans/pr-watch-and-storage-bounds/task-07-operator-docs-load-validation.md create mode 100644 docs/specs/platform/pr-watch-and-storage-bounds.md create mode 100644 docs/specs/platform/requirements/pr-watch-and-bounded-storage.md create mode 100644 docs/specs/platform/system-design/pr-watch-and-bounded-storage.md diff --git a/apps/backend/internal/agent/runtime/lifecycle/execution_store.go b/apps/backend/internal/agent/runtime/lifecycle/execution_store.go index 2041fdc4b6..89cc9d6b53 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/execution_store.go +++ b/apps/backend/internal/agent/runtime/lifecycle/execution_store.go @@ -90,6 +90,7 @@ func (s *ExecutionStore) Add(execution *AgentExecution) error { if execution.ContainerID != "" { s.byContainer[execution.ContainerID] = execution.ID } + agentActiveRuntimes.Set(int64(len(s.executions))) return nil } @@ -114,6 +115,7 @@ func (s *ExecutionStore) Remove(executionID string) { // Remove from primary map delete(s.executions, executionID) + agentActiveRuntimes.Set(int64(len(s.executions))) } // Get returns an agent execution by its ID. diff --git a/apps/backend/internal/agent/runtime/lifecycle/execution_store_test.go b/apps/backend/internal/agent/runtime/lifecycle/execution_store_test.go index 4136e208a2..7a650cf023 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/execution_store_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/execution_store_test.go @@ -2,11 +2,35 @@ package lifecycle import ( "errors" + "expvar" "testing" v1 "github.com/kandev/kandev/pkg/api/v1" ) +func TestActiveRuntimeGaugePublished(t *testing.T) { + if expvar.Get("agent_active_runtimes") == nil { + t.Fatal("agent_active_runtimes expvar is not published") + } +} + +func TestActiveRuntimeGaugeTracksExecutionOwnership(t *testing.T) { + store := NewExecutionStore() + if err := store.Add(&AgentExecution{ID: "runtime-metric-1"}); err != nil { + t.Fatalf("Add(first): %v", err) + } + if err := store.Add(&AgentExecution{ID: "runtime-metric-2"}); err != nil { + t.Fatalf("Add(second): %v", err) + } + if got := agentActiveRuntimes.Value(); got != 2 { + t.Fatalf("active runtime gauge = %d, want 2", got) + } + store.Remove("runtime-metric-1") + if got := agentActiveRuntimes.Value(); got != 1 { + t.Fatalf("active runtime gauge after remove = %d, want 1", got) + } +} + // TestExecutionStore_AddRejectsDuplicateSession is the regression test for the // process-leak bug where two paths created executions for the same session // concurrently and Add silently overwrote the bySession index, orphaning the diff --git a/apps/backend/internal/agent/runtime/lifecycle/metrics_vars.go b/apps/backend/internal/agent/runtime/lifecycle/metrics_vars.go new file mode 100644 index 0000000000..6279426d8b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/metrics_vars.go @@ -0,0 +1,7 @@ +package lifecycle + +import "expvar" + +// agentActiveRuntimes is a process-wide gauge of executions currently owned +// by the lifecycle runtime. It carries no task/session/execution labels. +var agentActiveRuntimes = expvar.NewInt("agent_active_runtimes") diff --git a/apps/backend/internal/agentctl/server/utility/acp_command_prefix_allowlist_test.go b/apps/backend/internal/agentctl/server/utility/acp_command_prefix_allowlist_test.go new file mode 100644 index 0000000000..dd695caa8f --- /dev/null +++ b/apps/backend/internal/agentctl/server/utility/acp_command_prefix_allowlist_test.go @@ -0,0 +1,33 @@ +package utility + +import "testing" + +func TestResolveACPCommandPrefixAllowsFixedKandevGuard(t *testing.T) { + t.Parallel() + + if got := resolveACPCommandPrefix(kandevAgentGuardPath); got != kandevAgentGuardPath { + t.Fatalf("resolveACPCommandPrefix(%q) = %q, want fixed guard path", kandevAgentGuardPath, got) + } +} + +func TestResolveACPCommandPrefixRejectsGuardLookalikes(t *testing.T) { + t.Parallel() + + for _, name := range []string{ + "kandev-agent-guard", + "/tmp/kandev-agent-guard", + "/usr/local/bin/kandev-agent-guard-copy", + } { + if got := resolveACPCommandPrefix(name); got != "" { + t.Errorf("resolveACPCommandPrefix(%q) = %q, want empty", name, got) + } + } +} + +func TestResolveACPCommandPrefixRetainsExistingAllowlist(t *testing.T) { + t.Parallel() + + if got := resolveACPCommandPrefix("npx"); got != "npx" { + t.Fatalf("resolveACPCommandPrefix(npx) = %q, want npx", got) + } +} diff --git a/apps/backend/internal/agentctl/server/utility/acp_executor.go b/apps/backend/internal/agentctl/server/utility/acp_executor.go index a04b6b3bef..0fe94c5b47 100644 --- a/apps/backend/internal/agentctl/server/utility/acp_executor.go +++ b/apps/backend/internal/agentctl/server/utility/acp_executor.go @@ -32,6 +32,7 @@ import ( const ( openCodeCommand = "opencode" openCodeACPSubcommand = "acp" + kandevAgentGuardPath = "/usr/local/bin/kandev-agent-guard" acpCommandTerminateGrace = 250 * time.Millisecond acpCommandForceKillGrace = 500 * time.Millisecond @@ -88,7 +89,7 @@ func (e *ACPInferenceExecutor) Execute(ctx context.Context, req *PromptRequest) cmdArgs := args[1:] if len(cfg.CommandPrefix) > 0 { args = append(append([]string{}, cfg.CommandPrefix...), args...) - resolvedCmd = resolveProbeCommand(args[0]) + resolvedCmd = resolveACPCommandPrefix(args[0]) if resolvedCmd == "" { return &PromptResponse{Success: false, Error: fmt.Sprintf("command prefix %q is not an allowed ACP command", args[0])}, nil } @@ -1268,6 +1269,18 @@ func resolveProbeCommand(name string) string { return "" } +// resolveACPCommandPrefix validates the optional launcher that wraps an +// already allow-listed ACP agent command. The deployment guard is accepted +// only at its fixed image path: accepting a matching basename from another +// directory would let a caller substitute an untrusted wrapper. Existing +// allow-listed agent commands retain their prior prefix behavior. +func resolveACPCommandPrefix(name string) string { + if name == kandevAgentGuardPath { + return kandevAgentGuardPath + } + return resolveProbeCommand(name) +} + // stderrTailLimit bounds how much of a spawned agent's stderr reaches the log: // enough for an npm failure or the head of a panic, small enough that a chatty // agent cannot flood it. diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index 5a498515ea..db08368361 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -1821,9 +1821,14 @@ func registerHealthRoutes(p routeParams) { oslimits.NewOSLimitsChecker(oslimits.NewInotifyProbe()), 5*time.Minute, ) + var workflowSyncProvider health.WorkflowSyncStatusProvider + if p.services.WorkflowSync != nil { + workflowSyncProvider = workflowSyncHealthAdapter{svc: p.services.WorkflowSync} + } checkers := []health.Checker{ health.NewGitExecutableChecker(), githubChecker, + health.NewWorkflowSyncChecker(workflowSyncProvider), health.NewAgentChecker(p.agentSettingsController), osLimitsChecker, } @@ -1837,6 +1842,32 @@ func registerHealthRoutes(p routeParams) { health.RegisterRoutes(p.router, healthSvc, p.log) } +// workflowSyncHealthAdapter bridges workflowsync.Service's own circuit +// summary type to the structural shape consumed by the health package +// without importing health into workflowsync (cycle), matching +// githubWorkspaceHealthAdapter below. +type workflowSyncHealthAdapter struct { + svc *workflowsync.Service +} + +func (a workflowSyncHealthAdapter) WorkflowSyncCircuitSummary( + ctx context.Context, +) (health.WorkflowSyncCircuitSummary, error) { + if a.svc == nil { + return health.WorkflowSyncCircuitSummary{}, nil + } + summary, err := a.svc.WorkflowSyncCircuitSummary(ctx) + if err != nil { + return health.WorkflowSyncCircuitSummary{}, err + } + return health.WorkflowSyncCircuitSummary{ + Total: summary.Total, + OpenAuth: summary.OpenAuth, + OpenConfig: summary.OpenConfig, + OpenTransient: summary.OpenTransient, + }, nil +} + type githubWorkspaceHealthAdapter struct { svc *github.Service } diff --git a/apps/backend/internal/common/authcircuit/circuit.go b/apps/backend/internal/common/authcircuit/circuit.go new file mode 100644 index 0000000000..633180b8f3 --- /dev/null +++ b/apps/backend/internal/common/authcircuit/circuit.go @@ -0,0 +1,175 @@ +// Package authcircuit provides a small, storage-agnostic circuit-breaker +// value type shared by integrations (GitHub PR-watch polling, workflow-sync) +// that must stop calling a third-party API once its credential or +// configuration is permanently broken, and resume promptly once the +// credential/configuration changes rather than waiting out a fixed timer. +// +// A permanent (auth/config) failure is fundamentally different from a +// transient one (network blip, 5xx, rate limiting): retrying a transient +// failure sooner might succeed, but retrying an auth/config failure will +// keep failing identically until a human fixes the credential or config — +// continuing to call the provider on the old schedule only wastes quota and +// produces log noise. State captures just enough to make that distinction +// durable (when embedded in a caller's own persisted row) without owning +// any I/O itself. +package authcircuit + +import ( + "math" + "math/rand" + "time" +) + +// FailureClass categorizes why an integration call failed, so the caller can +// decide whether continued retries are worthwhile. +type FailureClass string + +const ( + // FailureClassNone means the last attempt succeeded (or none has run + // yet). No backoff is scheduled. + FailureClassNone FailureClass = "" + // FailureClassTransient marks a failure that may resolve on its own — + // network errors, 5xx responses, secondary rate limiting. Retried on a + // short, bounded schedule. + FailureClassTransient FailureClass = "transient" + // FailureClassAuth marks an authentication failure (expired, revoked, or + // invalid credential). Only a credential change should be able to + // shorten the backoff; the fingerprint-reset path exists for exactly + // this. + FailureClassAuth FailureClass = "auth" + // FailureClassConfig marks a caller-configuration failure (bad + // repository/branch/path, integration not configured). Only a + // configuration change should shorten the backoff. + FailureClassConfig FailureClass = "config" +) + +// Backoff bounds the exponential delay schedule used between retries for one +// failure class. +type Backoff struct { + // Base is the delay after the first failure. + Base time.Duration + // Max caps the delay regardless of how many consecutive failures have + // occurred, so a long-broken integration still gets probed periodically + // (bounded by JitterFraction) rather than backing off forever. + Max time.Duration + // JitterFraction adds up to this fraction of the computed delay as + // additional random delay, so many circuits opened at the same moment + // (e.g. a credential rotation that breaks several workspaces at once) + // don't all retry in lockstep. + JitterFraction float64 +} + +// delay computes the backoff for the given 1-indexed consecutive-failure +// count. rng must return a value in [0, 1). +func (b Backoff) delay(consecutiveFailures int, rng func() float64) time.Duration { + if consecutiveFailures < 1 { + consecutiveFailures = 1 + } + // consecutiveFailures-1 so the first failure uses Base exactly. + shift := consecutiveFailures - 1 + if shift > 32 { // guard against overflow on a pathologically long streak + shift = 32 + } + scaled := float64(b.Base) * math.Pow(2, float64(shift)) + if scaled <= 0 || scaled > float64(b.Max) { + scaled = float64(b.Max) + } + if b.JitterFraction > 0 { + scaled += scaled * b.JitterFraction * rng() + } + d := time.Duration(scaled) + if d > b.Max { + d = b.Max + } + if d < b.Base { + d = b.Base + } + return d +} + +// TransientBackoff is the default schedule for transient failures: short +// enough that a real blip recovers quickly, capped so an unattended outage +// doesn't spin ever-longer. +var TransientBackoff = Backoff{Base: 30 * time.Second, Max: 10 * time.Minute, JitterFraction: 0.25} + +// PermanentBackoff is the default schedule for auth/config failures. Its +// cap is intentionally much longer than TransientBackoff's — nothing about +// waiting longer fixes a bad credential — but it still periodically probes +// (bounded by Max) in case an external fix (e.g. GitHub-side token renewal) +// happened without Kandev observing a fingerprint change. +var PermanentBackoff = Backoff{Base: 2 * time.Minute, Max: 6 * time.Hour, JitterFraction: 0.25} + +// BackoffFor returns the schedule to use for the given failure class. +func BackoffFor(class FailureClass) Backoff { + if class == FailureClassAuth || class == FailureClassConfig { + return PermanentBackoff + } + return TransientBackoff +} + +// State is the persisted circuit-breaker state for one integration target +// (e.g. one workspace's GitHub connection, or one workflow-sync config). +// Callers embed this by value in their own row/config struct and own its +// persistence; this package never performs I/O. +type State struct { + FailureClass FailureClass `json:"failure_class,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures,omitempty"` + NextRetryAt *time.Time `json:"next_retry_at,omitempty"` + // Fingerprint is the last-observed opaque credential/config fingerprint. + // Never contains secrets — callers must derive it from generation + // counters, status enums, or content hashes only. + Fingerprint string `json:"-"` +} + +// Open reports whether the circuit is currently open (skip calling the +// provider) at the given time. +func (s State) Open(now time.Time) bool { + return s.NextRetryAt != nil && now.Before(*s.NextRetryAt) +} + +// RecordFailure classifies the failure, increments the consecutive-failure +// count, and schedules NextRetryAt using the bounded exponential+jitter +// schedule for that class. rng is called for jitter; pass nil in production +// to use math/rand, or a deterministic stub in tests. +func (s *State) RecordFailure(now time.Time, class FailureClass, rng func() float64) { + if rng == nil { + rng = rand.Float64 + } + s.ConsecutiveFailures++ + s.FailureClass = class + delay := BackoffFor(class).delay(s.ConsecutiveFailures, rng) + next := now.Add(delay) + s.NextRetryAt = &next +} + +// RecordSuccess clears all failure/backoff state. Fingerprint is left +// untouched — it is only ever advanced by ResetIfFingerprintChanged or by +// the caller directly, since it tracks identity, not outcome. +func (s *State) RecordSuccess() { + s.FailureClass = FailureClassNone + s.ConsecutiveFailures = 0 + s.NextRetryAt = nil +} + +// ResetIfFingerprintChanged clears failure/backoff state and reports true +// when newFingerprint differs from the previously observed one. An empty +// newFingerprint is treated as "unknown" and never triggers a reset (a +// caller that cannot currently compute a fingerprint must not accidentally +// clear an open circuit). The very first observed fingerprint (transition +// from "" to non-empty) updates Fingerprint but is not itself treated as a +// change, since there is no prior value to have differed from. +func (s *State) ResetIfFingerprintChanged(newFingerprint string) bool { + if newFingerprint == "" { + return false + } + if newFingerprint == s.Fingerprint { + return false + } + firstObservation := s.Fingerprint == "" + s.Fingerprint = newFingerprint + if firstObservation { + return false + } + s.RecordSuccess() + return true +} diff --git a/apps/backend/internal/common/authcircuit/circuit_test.go b/apps/backend/internal/common/authcircuit/circuit_test.go new file mode 100644 index 0000000000..c9f2f17db6 --- /dev/null +++ b/apps/backend/internal/common/authcircuit/circuit_test.go @@ -0,0 +1,179 @@ +package authcircuit + +import ( + "testing" + "time" +) + +func noJitter() float64 { return 0 } + +func TestBackoffDelayGrowsExponentiallyAndCaps(t *testing.T) { + b := Backoff{Base: 1 * time.Second, Max: 10 * time.Second, JitterFraction: 0} + got := []time.Duration{ + b.delay(1, noJitter), + b.delay(2, noJitter), + b.delay(3, noJitter), + b.delay(4, noJitter), // 8s, still under cap + b.delay(5, noJitter), // would be 16s, capped to 10s + b.delay(100, noJitter), + } + want := []time.Duration{ + 1 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, + 10 * time.Second, 10 * time.Second, + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("delay(%d) = %v, want %v", i+1, got[i], want[i]) + } + } +} + +func TestBackoffDelayNeverBelowBase(t *testing.T) { + b := Backoff{Base: 5 * time.Second, Max: 60 * time.Second, JitterFraction: 0} + if got := b.delay(0, noJitter); got != b.Base { + t.Fatalf("delay(0) = %v, want base %v (non-positive attempt clamps to 1)", got, b.Base) + } + if got := b.delay(-3, noJitter); got != b.Base { + t.Fatalf("delay(-3) = %v, want base %v", got, b.Base) + } +} + +func TestBackoffJitterStaysWithinBounds(t *testing.T) { + b := Backoff{Base: 10 * time.Second, Max: 100 * time.Second, JitterFraction: 0.25} + for _, rngVal := range []float64{0, 0.5, 0.999} { + got := b.delay(1, func() float64 { return rngVal }) + min := b.Base + max := b.Base + time.Duration(float64(b.Base)*0.25) + if got < min || got > max { + t.Fatalf("delay with rng=%v = %v, want in [%v, %v]", rngVal, got, min, max) + } + } +} + +func TestBackoffForSelectsScheduleByClass(t *testing.T) { + if BackoffFor(FailureClassTransient) != TransientBackoff { + t.Fatal("transient class should use TransientBackoff") + } + if BackoffFor(FailureClassNone) != TransientBackoff { + t.Fatal("none class should default to TransientBackoff") + } + if BackoffFor(FailureClassAuth) != PermanentBackoff { + t.Fatal("auth class should use PermanentBackoff") + } + if BackoffFor(FailureClassConfig) != PermanentBackoff { + t.Fatal("config class should use PermanentBackoff") + } +} + +func TestStateOpenReflectsNextRetryAt(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + var s State + if s.Open(now) { + t.Fatal("zero-value state must not be open") + } + future := now.Add(time.Minute) + s.NextRetryAt = &future + if !s.Open(now) { + t.Fatal("state with future NextRetryAt must be open") + } + if s.Open(future.Add(time.Second)) { + t.Fatal("state must not be open once now has passed NextRetryAt") + } +} + +func TestRecordFailureIncrementsAndSchedulesBackoff(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + var s State + s.RecordFailure(now, FailureClassAuth, noJitter) + if s.ConsecutiveFailures != 1 { + t.Fatalf("ConsecutiveFailures = %d, want 1", s.ConsecutiveFailures) + } + if s.FailureClass != FailureClassAuth { + t.Fatalf("FailureClass = %q, want auth", s.FailureClass) + } + if s.NextRetryAt == nil || !s.NextRetryAt.Equal(now.Add(PermanentBackoff.Base)) { + t.Fatalf("NextRetryAt = %v, want %v", s.NextRetryAt, now.Add(PermanentBackoff.Base)) + } + if !s.Open(now) { + t.Fatal("state must be open immediately after RecordFailure") + } + + s.RecordFailure(now, FailureClassAuth, noJitter) + if s.ConsecutiveFailures != 2 { + t.Fatalf("ConsecutiveFailures = %d, want 2", s.ConsecutiveFailures) + } + wantSecond := now.Add(PermanentBackoff.Base * 2) + if !s.NextRetryAt.Equal(wantSecond) { + t.Fatalf("second NextRetryAt = %v, want %v", s.NextRetryAt, wantSecond) + } +} + +func TestRecordSuccessClearsFailureState(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := State{Fingerprint: "keep-me"} + s.RecordFailure(now, FailureClassTransient, noJitter) + s.RecordSuccess() + if s.FailureClass != FailureClassNone || s.ConsecutiveFailures != 0 || s.NextRetryAt != nil { + t.Fatalf("RecordSuccess left state = %+v, want all cleared", s) + } + if s.Fingerprint != "keep-me" { + t.Fatal("RecordSuccess must not touch Fingerprint") + } +} + +func TestResetIfFingerprintChanged(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + t.Run("empty fingerprint never resets", func(t *testing.T) { + s := State{} + s.RecordFailure(now, FailureClassAuth, noJitter) + if changed := s.ResetIfFingerprintChanged(""); changed { + t.Fatal("empty fingerprint must not report a change") + } + if !s.Open(now) { + t.Fatal("circuit must remain open when fingerprint is unknown") + } + }) + + t.Run("first observation records but does not reset", func(t *testing.T) { + s := State{} + s.RecordFailure(now, FailureClassAuth, noJitter) + if changed := s.ResetIfFingerprintChanged("gen-1"); changed { + t.Fatal("first observed fingerprint must not itself report a change") + } + if s.Fingerprint != "gen-1" { + t.Fatalf("Fingerprint = %q, want gen-1", s.Fingerprint) + } + if !s.Open(now) { + t.Fatal("first observation must not clear an already-open circuit") + } + }) + + t.Run("unchanged fingerprint does not reset", func(t *testing.T) { + s := State{Fingerprint: "gen-1"} + s.RecordFailure(now, FailureClassAuth, noJitter) + if changed := s.ResetIfFingerprintChanged("gen-1"); changed { + t.Fatal("identical fingerprint must not report a change") + } + if !s.Open(now) { + t.Fatal("circuit must remain open when the fingerprint has not changed") + } + }) + + t.Run("changed fingerprint resets the circuit", func(t *testing.T) { + s := State{Fingerprint: "gen-1"} + s.RecordFailure(now, FailureClassAuth, noJitter) + if !s.Open(now) { + t.Fatal("precondition: circuit should be open before the reset") + } + if changed := s.ResetIfFingerprintChanged("gen-2"); !changed { + t.Fatal("changed fingerprint must report a change") + } + if s.Fingerprint != "gen-2" { + t.Fatalf("Fingerprint = %q, want gen-2", s.Fingerprint) + } + if s.Open(now) { + t.Fatal("circuit must be closed immediately after a fingerprint change") + } + }) +} diff --git a/apps/backend/internal/db/sqlguard/exemptions.json b/apps/backend/internal/db/sqlguard/exemptions.json index 39b6e3f2bf..2aa69b989a 100644 --- a/apps/backend/internal/db/sqlguard/exemptions.json +++ b/apps/backend/internal/db/sqlguard/exemptions.json @@ -32,8 +32,11 @@ {"file": "internal/github/store.go", "symbol": "addTaskCIRoundColumns", "rule": "datetime-type", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, {"file": "internal/github/store.go", "symbol": "addPRScopeMigrationColumn", "rule": "datetime-type", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, {"file": "internal/github/store.go", "symbol": "migratePRTablesForMultiRepo", "rule": "datetime-type", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, + {"file": "internal/github/store.go", "symbol": "rebuildPRWatchesTableForTaskOwnership", "rule": "datetime-type", "reason": "This SQLite table rebuild is skipped for PostgreSQL"}, {"file": "internal/github/store.go", "symbol": "rebuildIfHasLegacyConstraint", "rule": "sqlite-catalog", "reason": "SQLite cannot alter this legacy constraint in place"}, {"file": "internal/github/store.go", "symbol": "rebuildIfHasLegacyConstraint", "rule": "raw-placeholder", "reason": "SQLite-only table rebuild migration"}, + {"file": "internal/github/store.go", "symbol": "tableSQLContains", "rule": "raw-placeholder", "reason": "This SQLite schema probe is skipped for PostgreSQL"}, + {"file": "internal/github/store.go", "symbol": "tableSQLContains", "rule": "sqlite-catalog", "reason": "This SQLite schema probe is skipped for PostgreSQL"}, {"file": "internal/gitlab/store.go", "symbol": "createTablesSQL", "rule": "boolean-integer", "reason": "The provider schema template is rendered by dialect.MustRenderSchema"}, {"file": "internal/gitlab/store.go", "symbol": "createTablesSQL", "rule": "datetime-type", "reason": "The provider schema template is rendered by dialect.MustRenderSchema"}, {"file": "internal/gitlab/store.go", "symbol": "migrateMRWatchUniqueKey", "rule": "datetime-type", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, @@ -60,6 +63,7 @@ {"file": "internal/linear/store.go", "symbol": "addIssueWatchLastErrorColumns", "rule": "datetime-type", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, {"file": "internal/linear/store.go", "symbol": "migrateLegacySingletonTable", "rule": "datetime-type", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, {"file": "internal/linear/store.go", "symbol": "migrateLegacySingletonTable", "rule": "raw-placeholder", "reason": "This legacy SQLite migration is skipped for PostgreSQL"}, + {"file": "internal/maintenance/compact.go", "symbol": "verifySQLiteIntegrity", "rule": "sqlite-catalog", "reason": "The maintenance command opens a verified SQLite snapshot through the SQLite-only reader"}, {"file": "internal/notifications/store/sqlite.go", "symbol": "migrateDeliveries", "rule": "sqlite-catalog", "reason": "SQLite-only notification migration"}, {"file": "internal/office/repository/sqlite/agent_summary.go", "symbol": "RunCountsByDayForAgent", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"}, {"file": "internal/office/repository/sqlite/agent_summary.go", "symbol": "TasksByPriorityByDayForAgent", "rule": "sqlite-date-function", "reason": "The Office repository is explicitly SQLite-only"}, @@ -110,6 +114,7 @@ {"file": "internal/task/repository/sqlite/worktree_ownership_targets.go", "symbol": "tableColumns", "rule": "raw-placeholder", "reason": "SQLite-only worktree ownership migration"}, {"file": "internal/workflow/repository/phase2_sqlite.go", "symbol": "backfillParticipantsCreatedAtFromRowid", "rule": "sqlite-date-function", "reason": "Guarded by dialect.IsPostgres and returns before this statement on every non-SQLite driver"}, {"file": "internal/workflow/repository/sqlite.go", "symbol": "DeleteStep", "rule": "sqlite-catalog", "reason": "The workflow SQLite repository owns this catalog probe"}, + {"file": "internal/workflowsync/store.go", "symbol": "addCircuitColumns", "rule": "datetime-type", "reason": "The migration is rendered by schemaSQLForDriver before execution"}, {"file": "internal/workflowsync/store.go", "symbol": "createTablesSQL", "rule": "datetime-type", "reason": "The provider schema template is rendered by dialect.MustRenderSchema"}, {"file": "internal/agent/settings/store/sqlite.go", "symbol": "UpdateAgentProfileEnabled", "rule": "boolean-integer", "reason": "The agent settings schema stores user_modified as an INTEGER flag"}, {"file": "internal/office/configsync/store.go", "symbol": "UpsertConfigForWorkspace", "rule": "boolean-integer", "reason": "The Office config-sync schema stores last_ok as an INTEGER flag"}, diff --git a/apps/backend/internal/db/sqlite.go b/apps/backend/internal/db/sqlite.go index b4154073e1..225c2448f6 100644 --- a/apps/backend/internal/db/sqlite.go +++ b/apps/backend/internal/db/sqlite.go @@ -58,10 +58,12 @@ func OpenSQLite(dbPath string) (*sql.DB, error) { func OpenSQLiteReader(dbPath string) (*sql.DB, error) { normalizedPath := normalizeSQLitePath(dbPath) - // Reader DSN: read-only mode, FK enforcement, shared cache. + // Reader DSN: read-only mode and FK enforcement. + // Do not enable SQLite shared-cache mode: an active reader in that cache + // can make a later reader observe its stale snapshot after a writer commits. // journal_mode and synchronous are database-level (set by the writer). dsn := fmt.Sprintf( - "file:%s?_foreign_keys=on&_mode=ro&_busy_timeout=%d&_cache=shared", + "file:%s?_foreign_keys=on&mode=ro&_busy_timeout=%d", normalizedPath, int(defaultBusyTimeout/time.Millisecond), ) diff --git a/apps/backend/internal/github/errors.go b/apps/backend/internal/github/errors.go index 22d7266ac7..5c7e46d346 100644 --- a/apps/backend/internal/github/errors.go +++ b/apps/backend/internal/github/errors.go @@ -2,7 +2,10 @@ package github import ( "errors" + "net/http" "strings" + + "github.com/kandev/kandev/internal/common/authcircuit" ) // ErrInvalidPRURL signals that a caller-supplied PR URL could not be parsed. @@ -117,3 +120,54 @@ func isRepoNotResolvableErr(err error) bool { } return false } + +// classifyPollErr maps a background PR-watch poll failure to an +// authcircuit.FailureClass, for Poller's per-workspace in-memory circuit +// (see poller_circuit.go). This is deliberately a coarser, workspace-level +// signal than the existing repoErrorCache (which negative-caches a single +// repo for 10 minutes): an auth failure means every watch in the workspace +// is broken, not just one repo, so it uses the longer permanent-backoff +// schedule and only resets on a credential change +// (Service.WorkspaceConnectionFingerprint), while a repo-not-resolvable +// signal is treated as a configuration problem here too — both caches agree +// "stop retrying this" and neither depends on the other. +func classifyPollErr(err error) authcircuit.FailureClass { + if err == nil { + return authcircuit.FailureClassNone + } + if errors.Is(err, ErrGitHubConnectionInvalid) || errors.Is(err, ErrGitHubNotConfigured) { + return authcircuit.FailureClassAuth + } + if isRepoNotResolvableErr(err) { + return authcircuit.FailureClassConfig + } + var apiErr *GitHubAPIError + if errors.As(err, &apiErr) { + if isGitHubRateLimitAPIError(apiErr) { + return authcircuit.FailureClassTransient + } + switch apiErr.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + return authcircuit.FailureClassAuth + case http.StatusNotFound, http.StatusUnprocessableEntity, http.StatusBadRequest: + return authcircuit.FailureClassConfig + default: + return authcircuit.FailureClassTransient + } + } + return authcircuit.FailureClassTransient +} + +func isGitHubRateLimitAPIError(err *GitHubAPIError) bool { + if err == nil { + return false + } + if err.StatusCode == http.StatusTooManyRequests { + return true + } + if err.StatusCode != http.StatusForbidden { + return false + } + body := strings.ToLower(err.Body) + return strings.Contains(body, "rate limit") || strings.Contains(body, "abuse detection") +} diff --git a/apps/backend/internal/github/load_validation_test.go b/apps/backend/internal/github/load_validation_test.go new file mode 100644 index 0000000000..15e2bc708a --- /dev/null +++ b/apps/backend/internal/github/load_validation_test.go @@ -0,0 +1,156 @@ +package github + +import ( + "context" + "fmt" + "strings" + "testing" + "time" +) + +// TestLoadValidation_TenSimulatedMinutes_NoWatchCreationLoopBoundedLookups is +// the Task 07 deterministic load-validation fixture (plan.md Task 07, +// "Document operations and validate sustained load"). It reproduces the +// original amplification scenario at scale — one task resumed across 50 +// historical sessions (mirroring AC1's 50-session fixture) alongside 24 +// unrelated concurrently-active tasks, standing in for many simultaneous +// task/coordinator boards — and drives ten simulated poll cycles at the +// poller's real one-minute cadence (defaultPRPollInterval) without sleeping +// wall-clock time between them. +// +// It asserts the three AC1/AC3-shaped invariants a production amplification +// regression would violate: +// 1. No watch-creation loop: the exact set of watch IDs after the first +// converged cycle is byte-for-byte identical after the tenth cycle, and +// the canonical watch count equals the number of distinct +// task/repository/branch identities (25), never the 74 total +// historical-session entries the provider reports. +// 2. Bounded lookups: each cycle performs exactly one batched GraphQL +// branch-search call for all 25 still-searching canonical watches +// (below graphQLBatchChunkSize), independent of how many sessions ever +// existed for any one task. +// 3. Stable latency: no cycle's wall-clock duration exceeds a generous +// absolute bound, ruling out the quadratic-cost-per-cycle shape a +// duplicate-row leak would produce. +func TestLoadValidation_TenSimulatedMinutes_NoWatchCreationLoopBoundedLookups(t *testing.T) { + poller, svc, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + + const ( + historicalSessions = 50 + otherTasks = 24 + canonicalWatches = otherTasks + 1 // + the one resumed historical task + simulatedCycles = 10 // ten simulated minutes at defaultPRPollInterval + ) + + // Seed the task rows ListActivePRWatches joins against. + seedTask(t, store, "task-hist", false) + for i := 0; i < otherTasks; i++ { + seedTask(t, store, fmt.Sprintf("task-%02d", i), false) + } + + // The resumed task: 50 historical sessions, all resolving to the same + // task/repository/branch — the exact amplification shape from AC1. + var provided []TaskBranchInfo + for i := 0; i < historicalSessions; i++ { + provided = append(provided, TaskBranchInfo{ + WorkspaceID: testWorkspaceID, + TaskID: "task-hist", + SessionID: fmt.Sprintf("session-hist-%03d", i), + RepositoryID: "repo-hist", + Owner: "acme", + Repo: "hist-repo", + Branch: "feature-hist", + }) + } + // 24 unrelated concurrently-active tasks/boards, each its own canonical + // identity, standing in for many simultaneous coordinator-driven tasks. + for i := 0; i < otherTasks; i++ { + provided = append(provided, TaskBranchInfo{ + WorkspaceID: testWorkspaceID, + TaskID: fmt.Sprintf("task-%02d", i), + SessionID: fmt.Sprintf("session-%02d", i), + RepositoryID: fmt.Sprintf("repo-%02d", i), + Owner: "acme", + Repo: fmt.Sprintf("repo-%02d", i), + Branch: fmt.Sprintf("feature-%02d", i), + }) + } + poller.SetTaskBranchProvider(&mockTaskBranchProvider{tasks: provided}) + + // One canned branch-search response per simulated cycle: every canonical + // watch resolves to zero open PRs, so all 25 remain "searching" and + // generate a fresh lookup on the following cycle too (worst case for + // lookup-count amplification). + emptyAliases := make([]string, canonicalWatches) + for i := range emptyAliases { + emptyAliases[i] = fmt.Sprintf(`"b%d":{"pullRequests":{"nodes":[]}}`, i) + } + branchResp := fmt.Sprintf(`{"data":{%s}}`, strings.Join(emptyAliases, ",")) + for i := 0; i < simulatedCycles; i++ { + gh.branchResponses = append(gh.branchResponses, branchResp) + } + + durations := make([]time.Duration, 0, simulatedCycles) + var firstCycleWatchIDs, lastCycleWatchIDs map[string]struct{} + + for cycle := 0; cycle < simulatedCycles; cycle++ { + start := time.Now() + poller.checkPRWatches(ctx) + durations = append(durations, time.Since(start)) + + watches, err := svc.ListActivePRWatches(ctx) + if err != nil { + t.Fatalf("cycle %d: ListActivePRWatches: %v", cycle, err) + } + if got := len(watches); got != canonicalWatches { + t.Fatalf("cycle %d: watch count = %d, want %d canonical watches (not %d historical entries)", + cycle, got, canonicalWatches, historicalSessions+otherTasks) + } + ids := make(map[string]struct{}, len(watches)) + for _, w := range watches { + ids[w.ID] = struct{}{} + } + if cycle == 0 { + firstCycleWatchIDs = ids + } + lastCycleWatchIDs = ids + } + + // Invariant 1: zero net inserts/deletes across the nine cycles following + // convergence — the exact watch-ID set is unchanged. + if len(firstCycleWatchIDs) != len(lastCycleWatchIDs) { + t.Fatalf("watch set size changed across cycles: first=%d last=%d", + len(firstCycleWatchIDs), len(lastCycleWatchIDs)) + } + for id := range firstCycleWatchIDs { + if _, ok := lastCycleWatchIDs[id]; !ok { + t.Fatalf("watch %s present after cycle 0 but missing after cycle %d (creation loop / churn)", + id, simulatedCycles-1) + } + } + + // Invariant 2: exactly one batched branch lookup per cycle — 25 canonical + // watches collapse into one GraphQL call, not 74 (or 25 * N) calls. + if got := len(gh.branchQueries); got != simulatedCycles { + t.Fatalf("branch GraphQL calls = %d, want %d (one batched lookup per cycle)", got, simulatedCycles) + } + if got := len(gh.prQueries); got != 0 { + t.Fatalf("numbered PR GraphQL calls = %d, want 0 (no watch ever resolved a PR number)", got) + } + + // Invariant 3: stable per-cycle latency; a duplicate-row leak would make + // later cycles measurably and increasingly slower than the first. + const maxCycleLatency = 2 * time.Second + var total time.Duration + for i, d := range durations { + total += d + if d > maxCycleLatency { + t.Fatalf("cycle %d latency = %v, want <= %v", i, d, maxCycleLatency) + } + } + t.Logf("load validation: %d cycles, %d canonical watches (from %d historical-session entries), "+ + "%d branch GraphQL calls, total cycle time %v, average cycle time %v", + simulatedCycles, canonicalWatches, historicalSessions+otherTasks, + len(gh.branchQueries), total, total/time.Duration(simulatedCycles)) +} diff --git a/apps/backend/internal/github/metrics_vars.go b/apps/backend/internal/github/metrics_vars.go index 57870cbf1b..ce4117d141 100644 --- a/apps/backend/internal/github/metrics_vars.go +++ b/apps/backend/internal/github/metrics_vars.go @@ -10,7 +10,22 @@ import ( // durable, snapshot-checkable signals are the writer-health invariants // AC-36/AC-37, not these counters. Mirrors the label idiom in // internal/office/scheduler/metrics_vars.go. -var taskPROutcomeSyncsTotal = expvar.NewMap("github_task_pr_outcome_syncs_total") +var ( + taskPROutcomeSyncsTotal = expvar.NewMap("github_task_pr_outcome_syncs_total") + // authCircuitSkipsTotal and authCircuitResetsTotal are unlabeled counts + // (no per-workspace label — workspace IDs must never be metric labels). + // They bound the PR-monitor loop's auth/config circuit-breaker activity + // (see poller_circuit.go): skips confirm a broken credential/config + // stopped costing a GitHub call every cycle; resets confirm a + // credential change (rotate/reconnect) is promptly detected. + authCircuitSkipsTotal = expvar.NewInt("github_pr_watch_auth_circuit_skips_total") + authCircuitResetsTotal = expvar.NewInt("github_pr_watch_auth_circuit_resets_total") + prWatchActive = expvar.NewInt("github_pr_watch_active") + prWatchSearching = expvar.NewInt("github_pr_watch_searching") + prWatchDuplicates = expvar.NewInt("github_pr_watch_duplicates") + prWatchOrphans = expvar.NewInt("github_pr_watch_orphans") + canonicalPollRequests = expvar.NewInt("github_pr_watch_canonical_poll_requests_total") +) // outcomeMetricLabel builds a "k1=v1;k2=v2;..." label string for an expvar // map key, matching the idiom in internal/office/scheduler/metrics_vars.go. @@ -38,3 +53,26 @@ func boolLabel(b bool) string { } return "false" } + +// incAuthCircuitSkip records that a poll cycle skipped a workspace's PR +// watches entirely because its auth/config circuit was still open. +func incAuthCircuitSkip() { + authCircuitSkipsTotal.Add(1) +} + +// incAuthCircuitReset records a credential-fingerprint-triggered circuit +// reset (rotate/reconnect/revoke-then-reconfigure detected). +func incAuthCircuitReset() { + authCircuitResetsTotal.Add(1) +} + +func recordPRWatchCardinality(cardinality PRWatchCardinality) { + prWatchActive.Set(cardinality.Active) + prWatchSearching.Set(cardinality.Searching) + prWatchDuplicates.Set(cardinality.Duplicates) + prWatchOrphans.Set(cardinality.Orphans) +} + +func incCanonicalPollRequests(count int) { + canonicalPollRequests.Add(int64(count)) +} diff --git a/apps/backend/internal/github/metrics_vars_test.go b/apps/backend/internal/github/metrics_vars_test.go index fe6d1977f4..8be25172f7 100644 --- a/apps/backend/internal/github/metrics_vars_test.go +++ b/apps/backend/internal/github/metrics_vars_test.go @@ -1,6 +1,7 @@ package github import ( + "context" "expvar" "strconv" "strings" @@ -27,6 +28,47 @@ func readOutcomeCounter(t *testing.T, m *expvar.Map, prefix string) int64 { return total } +func TestPRWatchCardinalityCountsCurrentCanonicalState(t *testing.T) { + _, _, _, store := setupPollerTest(t) + ctx := context.Background() + if _, err := store.db.Exec(`CREATE TABLE task_repositories ( + id TEXT PRIMARY KEY, task_id TEXT NOT NULL, repository_id TEXT NOT NULL + )`); err != nil { + t.Fatalf("create task_repositories: %v", err) + } + if _, err := store.db.Exec(`INSERT INTO task_repositories (id, task_id, repository_id) + VALUES ('binding-active', 'task-active', 'repo-attached')`); err != nil { + t.Fatalf("seed task_repositories: %v", err) + } + seedTask(t, store, "task-active", false) + seedTask(t, store, "task-archived", true) + + watches := []*PRWatch{ + withTestWorkspace(&PRWatch{ID: "watch-searching", TaskID: "task-active", Branch: "feature/a"}), + withTestWorkspace(&PRWatch{ID: "watch-discovered", TaskID: "task-active", Branch: "feature/b", PRNumber: 42}), + withTestWorkspace(&PRWatch{ID: "watch-archived", TaskID: "task-archived", Branch: "feature/c"}), + withTestWorkspace(&PRWatch{ID: "watch-orphan", TaskID: "task-missing", Branch: "feature/d"}), + withTestWorkspace(&PRWatch{ID: "watch-detached", TaskID: "task-active", RepositoryID: "repo-detached", Branch: "feature/e"}), + withTestWorkspace(&PRWatch{ID: "watch-attached", TaskID: "task-active", RepositoryID: "repo-attached", Branch: "feature/f"}), + } + for _, watch := range watches { + watch.Owner = "owner" + watch.Repo = "repo" + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("CreatePRWatch(%s): %v", watch.ID, err) + } + } + + got, err := store.PRWatchCardinality(ctx) + if err != nil { + t.Fatalf("PRWatchCardinality: %v", err) + } + want := (PRWatchCardinality{Active: 4, Searching: 3, Duplicates: 0, Orphans: 2}) + if got != want { + t.Fatalf("PRWatchCardinality = %+v, want %+v", got, want) + } +} + func TestOutcomeMetricLabel(t *testing.T) { cases := []struct { name string @@ -83,6 +125,11 @@ func TestIncTaskPROutcomeSyncRecordsOnExpvarMap(t *testing.T) { func TestOutcomeExpvarMapsPublishedAtKnownNames(t *testing.T) { expected := []string{ "github_task_pr_outcome_syncs_total", + "github_pr_watch_active", + "github_pr_watch_searching", + "github_pr_watch_duplicates", + "github_pr_watch_orphans", + "github_pr_watch_canonical_poll_requests_total", } for _, name := range expected { if expvar.Get(name) == nil { @@ -90,3 +137,25 @@ func TestOutcomeExpvarMapsPublishedAtKnownNames(t *testing.T) { } } } + +func TestPRWatchOperationalMetricsRecordAggregateValues(t *testing.T) { + recordPRWatchCardinality(PRWatchCardinality{Active: 7, Searching: 3, Duplicates: 1, Orphans: 2}) + if got := prWatchActive.Value(); got != 7 { + t.Fatalf("active watch gauge = %d, want 7", got) + } + if got := prWatchSearching.Value(); got != 3 { + t.Fatalf("searching watch gauge = %d, want 3", got) + } + if got := prWatchDuplicates.Value(); got != 1 { + t.Fatalf("duplicate watch gauge = %d, want 1", got) + } + if got := prWatchOrphans.Value(); got != 2 { + t.Fatalf("orphan watch gauge = %d, want 2", got) + } + + before := canonicalPollRequests.Value() + incCanonicalPollRequests(4) + if got := canonicalPollRequests.Value() - before; got != 4 { + t.Fatalf("canonical poll request delta = %d, want 4", got) + } +} diff --git a/apps/backend/internal/github/poller.go b/apps/backend/internal/github/poller.go index 46239c0034..1ccbd81cee 100644 --- a/apps/backend/internal/github/poller.go +++ b/apps/backend/internal/github/poller.go @@ -88,6 +88,14 @@ type TaskBranchProvider interface { ResolveBranchForWatch(ctx context.Context, watch *PRWatch) string } +type multiBranchTaskBranchProvider interface { + ResolveBranchesForRepository(ctx context.Context, taskID, repositoryID string) []string +} + +type repositoryTaskBranchProvider interface { + ResolveBranchForRepository(ctx context.Context, taskID, repositoryID string) string +} + // Poller runs background loops for PR monitoring and review queue checking. type Poller struct { service *Service @@ -95,6 +103,10 @@ type Poller struct { logger *logger.Logger taskBranchProvider TaskBranchProvider + // circuits tracks per-workspace auth/config backoff for the PR-monitor + // loop only (see poller_circuit.go). Safe for concurrent use; never nil. + circuits *pollerCircuits + cancel context.CancelFunc wg sync.WaitGroup started bool @@ -106,6 +118,7 @@ func NewPoller(svc *Service, eventBus bus.EventBus, log *logger.Logger) *Poller service: svc, eventBus: eventBus, logger: log, + circuits: newPollerCircuits(), } } @@ -172,6 +185,16 @@ func (p *Poller) checkPRWatches(ctx context.Context) { p.logger.Error("failed to list PR watches", zap.Error(err)) return } + if cardinality, metricErr := p.service.PRWatchCardinality(ctx); metricErr != nil { + p.logger.Debug("failed to observe PR watch cardinality", zap.Error(metricErr)) + } else { + recordPRWatchCardinality(cardinality) + } + if len(watches) == 0 { + return + } + + watches = p.filterOpenCircuitWatches(ctx, watches) if len(watches) == 0 { return } @@ -183,11 +206,59 @@ func (p *Poller) checkPRWatches(ctx context.Context) { if p.tryBatchedPRWatchCheck(ctx, watches) { return } + watches = p.filterOpenCircuitWatches(ctx, watches) for _, watch := range watches { p.checkSinglePRWatch(ctx, watch) } } +// filterOpenCircuitWatches refreshes each present workspace's credential +// fingerprint at most once per cycle (resetting an open circuit on +// rotate/reconnect/re-auth) and excludes every watch belonging to a +// workspace whose circuit is still open afterward — so a broken automation +// credential does not cost a GitHub call every poll cycle, only one +// exploratory probe per backoff window (see authcircuit.PermanentBackoff). +func (p *Poller) filterOpenCircuitWatches(ctx context.Context, watches []*PRWatch) []*PRWatch { + now := time.Now().UTC() + refreshed := make(map[string]bool, len(watches)) + filtered := make([]*PRWatch, 0, len(watches)) + for _, watch := range watches { + if watch == nil || watch.WorkspaceID == "" { + filtered = append(filtered, watch) + continue + } + if !refreshed[watch.WorkspaceID] { + refreshed[watch.WorkspaceID] = true + p.refreshWorkspaceCircuitFingerprint(ctx, watch.WorkspaceID) + } + if p.circuits.open(watch.WorkspaceID, now) { + incAuthCircuitSkip() + continue + } + filtered = append(filtered, watch) + } + return filtered +} + +// refreshWorkspaceCircuitFingerprint resets workspaceID's circuit when its +// GitHub connection's non-secret fingerprint ("status:generation") has +// changed since last observed. A lookup error or empty fingerprint (no +// connection, or the connection was just deleted) is treated as "unknown" +// and never resets an open circuit — see +// authcircuit.State.ResetIfFingerprintChanged. +func (p *Poller) refreshWorkspaceCircuitFingerprint(ctx context.Context, workspaceID string) { + if p.service == nil { + return + } + fingerprint, err := p.service.WorkspaceConnectionFingerprint(ctx, workspaceID) + if err != nil || fingerprint == "" { + return + } + if p.circuits.resetIfFingerprintChanged(workspaceID, fingerprint) { + incAuthCircuitReset() + } +} + // tryBatchedPRWatchCheck runs the batched GraphQL flow for the supplied // watches via the shared Service.SyncWatchesBatched seam. Returns true // when the path succeeded so the caller can skip the per-watch fallback. @@ -205,7 +276,15 @@ func (p *Poller) tryBatchedPRWatchCheck(ctx context.Context, watches []*PRWatch) } results := make([]PRWatchSyncResult, 0, len(watches)) for workspaceID, workspaceWatches := range byWorkspace { + incCanonicalPollRequests(len(workspaceWatches)) workspaceResults, err := p.service.SyncWorkspaceWatchesBatched(ctx, workspaceID, workspaceWatches) + // A client without GraphQL support deliberately falls back to the + // per-watch REST path below. It is not an authentication failure, so + // it must not open the workspace circuit and filter that fallback out. + if errors.Is(err, errGraphQLUnsupported) { + return false + } + p.circuits.recordOutcome(workspaceID, classifyPollErr(err), time.Now().UTC()) if err != nil { p.logger.Debug("batched PR watch check failed", zap.String("workspace_id", workspaceID), zap.Error(err)) @@ -268,6 +347,7 @@ func splitPRWatches(watches []*PRWatch) (numbered, searching []*PRWatch) { } func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { + incCanonicalPollRequests(1) // PRWatch with pr_number=0 means we're still searching for a PR on this branch. if watch.PRNumber == 0 { p.detectPRForWatch(ctx, watch) @@ -275,6 +355,7 @@ func (p *Poller) checkSinglePRWatch(ctx context.Context, watch *PRWatch) { } status, hasNew, err := p.service.CheckPRWatchForWorkspace(ctx, watch) + p.circuits.recordOutcome(watch.WorkspaceID, classifyPollErr(err), time.Now().UTC()) if err != nil { p.logger.Debug("failed to check PR watch", zap.String("id", watch.ID), zap.Error(err)) @@ -348,6 +429,7 @@ func (p *Poller) detectPRForWatch(ctx context.Context, watch *PRWatch) { } pr, err := p.service.findPRByBranchForWatch(ctx, watch) + p.circuits.recordOutcome(watch.WorkspaceID, classifyPollErr(err), time.Now().UTC()) if err != nil { p.logger.Debug("failed to search for PR by branch", zap.String("watch_id", watch.ID), @@ -460,7 +542,15 @@ func (p *Poller) refreshStaleBranches(ctx context.Context) { if watch.PRNumber != 0 { continue // already found a PR, branch is correct } - currentBranch := p.taskBranchProvider.ResolveBranchForWatch(ctx, watch) + if branchSetPreservesWatch(ctx, p.taskBranchProvider, watch) { + continue + } + currentBranch := "" + if repositoryProvider, ok := p.taskBranchProvider.(repositoryTaskBranchProvider); ok { + currentBranch = repositoryProvider.ResolveBranchForRepository(ctx, watch.TaskID, watch.RepositoryID) + } else { + currentBranch = p.taskBranchProvider.ResolveBranchForWatch(ctx, watch) + } if currentBranch == "" || currentBranch == watch.Branch { continue } @@ -475,6 +565,27 @@ func (p *Poller) refreshStaleBranches(ctx context.Context) { } } +func branchSetPreservesWatch(ctx context.Context, provider TaskBranchProvider, watch *PRWatch) bool { + multiBranchProvider, ok := provider.(multiBranchTaskBranchProvider) + if !ok { + return false + } + branches := multiBranchProvider.ResolveBranchesForRepository(ctx, watch.TaskID, watch.RepositoryID) + if len(branches) == 0 { + return false + } + return branchListContains(branches, watch.Branch) +} + +func branchListContains(branches []string, branch string) bool { + for _, candidate := range branches { + if candidate == branch { + return true + } + } + return false +} + // reviewQueueLoop polls review watches for new PRs. func (p *Poller) reviewQueueLoop(ctx context.Context) { defer p.wg.Done() diff --git a/apps/backend/internal/github/poller_circuit.go b/apps/backend/internal/github/poller_circuit.go new file mode 100644 index 0000000000..090069c6d2 --- /dev/null +++ b/apps/backend/internal/github/poller_circuit.go @@ -0,0 +1,91 @@ +package github + +import ( + "sync" + "time" + + "github.com/kandev/kandev/internal/common/authcircuit" +) + +// pollerCircuits tracks a per-workspace in-memory auth/config circuit for +// the PR-monitor loop (Poller.checkPRWatches). It is in-memory only, never +// persisted — mirroring the existing repoErrorCache precedent (see +// errors.go): a backend restart clears it, costing at most one wasted probe +// per workspace on the next cycle rather than resuming a sustained +// hammering pattern. review/issue watch loops are not covered by this wave +// (see task-04 results). +type pollerCircuits struct { + mu sync.Mutex + byWorkspace map[string]authcircuit.State +} + +func newPollerCircuits() *pollerCircuits { + return &pollerCircuits{byWorkspace: make(map[string]authcircuit.State)} +} + +// open reports whether workspaceID's circuit currently forbids a poll. +func (c *pollerCircuits) open(workspaceID string, now time.Time) bool { + c.mu.Lock() + defer c.mu.Unlock() + return c.byWorkspace[workspaceID].Open(now) +} + +// recordOutcome updates workspaceID's circuit after a poll attempt. +// FailureClassNone (nil error) clears any prior failure state. +func (c *pollerCircuits) recordOutcome(workspaceID string, class authcircuit.FailureClass, now time.Time) { + if workspaceID == "" { + return + } + c.mu.Lock() + defer c.mu.Unlock() + state := c.byWorkspace[workspaceID] + if class == authcircuit.FailureClassNone { + state.RecordSuccess() + delete(c.byWorkspace, workspaceID) + return + } else { + state.RecordFailure(now, class, nil) + } + c.byWorkspace[workspaceID] = state +} + +// resetIfFingerprintChanged resets workspaceID's circuit when its +// credential fingerprint (Service.WorkspaceConnectionFingerprint) has +// changed since last observed — a rotate/reconnect/revoke-then-reconfigure +// resumes polling on the very next cycle instead of waiting out the +// remaining backoff. Returns true when a reset happened. +func (c *pollerCircuits) resetIfFingerprintChanged(workspaceID, fingerprint string) bool { + if workspaceID == "" { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + state := c.byWorkspace[workspaceID] + changed := state.ResetIfFingerprintChanged(fingerprint) + if !changed && !state.Open(time.Now().UTC()) { + return false + } + c.byWorkspace[workspaceID] = state + return changed +} + +// summary aggregates open-circuit counts by failure class, for bounded +// health/metrics reporting. Never returns workspace IDs. +func (c *pollerCircuits) summary(now time.Time) (openAuth, openConfig, openTransient int) { + c.mu.Lock() + defer c.mu.Unlock() + for _, state := range c.byWorkspace { + if !state.Open(now) { + continue + } + switch state.FailureClass { + case authcircuit.FailureClassAuth: + openAuth++ + case authcircuit.FailureClassConfig: + openConfig++ + default: + openTransient++ + } + } + return openAuth, openConfig, openTransient +} diff --git a/apps/backend/internal/github/poller_circuit_test.go b/apps/backend/internal/github/poller_circuit_test.go new file mode 100644 index 0000000000..d5bfcce647 --- /dev/null +++ b/apps/backend/internal/github/poller_circuit_test.go @@ -0,0 +1,202 @@ +package github + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/authcircuit" +) + +// TestPollerCircuits_RecordFailureThenOpen confirms a classified failure +// opens the circuit for the backoff window, and a subsequent success clears +// it — the two building blocks checkPRWatches/tryBatchedPRWatchCheck rely on. +func TestPollerCircuits_RecordFailureThenOpen(t *testing.T) { + c := newPollerCircuits() + now := time.Now().UTC() + + if c.open("ws-1", now) { + t.Fatalf("a workspace with no recorded outcome must not be open") + } + + c.recordOutcome("ws-1", authcircuit.FailureClassAuth, now) + if !c.open("ws-1", now) { + t.Fatalf("expected circuit to be open immediately after an auth failure") + } + if c.open("ws-2", now) { + t.Fatalf("recording a failure for ws-1 must not open ws-2's circuit") + } + + c.recordOutcome("ws-1", authcircuit.FailureClassNone, now) + if c.open("ws-1", now) { + t.Fatalf("a success must clear the open circuit") + } +} + +// TestPollerCircuits_ResetIfFingerprintChanged confirms the credential +// fingerprint reset path: unchanged/empty fingerprints never reset, but a +// genuine change does and reports true exactly once. +func TestPollerCircuits_ResetIfFingerprintChanged(t *testing.T) { + c := newPollerCircuits() + now := time.Now().UTC() + c.recordOutcome("ws-1", authcircuit.FailureClassAuth, now) + + if c.resetIfFingerprintChanged("ws-1", "") { + t.Fatalf("an empty fingerprint must never reset an open circuit") + } + if !c.open("ws-1", now) { + t.Fatalf("circuit should still be open after an empty-fingerprint no-op") + } + + // First observation of a fingerprint is not itself a "change". + if c.resetIfFingerprintChanged("ws-1", "active:1") { + t.Fatalf("the first observed fingerprint must not itself count as a change") + } + if !c.open("ws-1", now) { + t.Fatalf("circuit should still be open after first fingerprint observation") + } + + if !c.resetIfFingerprintChanged("ws-1", "active:2") { + t.Fatalf("a changed fingerprint must reset the circuit") + } + if c.open("ws-1", now) { + t.Fatalf("circuit must be closed immediately after a fingerprint-triggered reset") + } +} + +// TestPollerCircuits_Summary confirms bounded, class-grouped aggregation +// with no per-workspace identifiers. +func TestPollerCircuits_Summary(t *testing.T) { + c := newPollerCircuits() + now := time.Now().UTC() + c.recordOutcome("ws-auth", authcircuit.FailureClassAuth, now) + c.recordOutcome("ws-config", authcircuit.FailureClassConfig, now) + c.recordOutcome("ws-transient", authcircuit.FailureClassTransient, now) + c.recordOutcome("ws-ok", authcircuit.FailureClassNone, now) + + auth, config, transient := c.summary(now) + if auth != 1 || config != 1 || transient != 1 { + t.Fatalf("summary = (auth=%d, config=%d, transient=%d), want (1, 1, 1)", auth, config, transient) + } +} + +func TestClassifyPollErr_NilIsNone(t *testing.T) { + if got := classifyPollErr(nil); got != authcircuit.FailureClassNone { + t.Fatalf("classifyPollErr(nil) = %q, want none", got) + } +} + +func TestClassifyPollErr_ConnectionSentinelsAreAuth(t *testing.T) { + for _, err := range []error{ErrGitHubConnectionInvalid, ErrGitHubNotConfigured} { + if got := classifyPollErr(err); got != authcircuit.FailureClassAuth { + t.Fatalf("classifyPollErr(%v) = %q, want auth", err, got) + } + } +} + +func TestClassifyPollErr_RepoNotResolvableIsConfig(t *testing.T) { + if got := classifyPollErr(ErrRepoNotResolvable); got != authcircuit.FailureClassConfig { + t.Fatalf("classifyPollErr(ErrRepoNotResolvable) = %q, want config", got) + } + wrapped := errors.New("Could not resolve to a Repository with the name 'x'") + if got := classifyPollErr(wrapped); got != authcircuit.FailureClassConfig { + t.Fatalf("classifyPollErr(graphql-not-found) = %q, want config", got) + } +} + +func TestClassifyPollErr_GitHubAPIErrorByStatus(t *testing.T) { + tests := []struct { + status int + want authcircuit.FailureClass + }{ + {http.StatusUnauthorized, authcircuit.FailureClassAuth}, + {http.StatusForbidden, authcircuit.FailureClassAuth}, + {http.StatusTooManyRequests, authcircuit.FailureClassTransient}, + {http.StatusNotFound, authcircuit.FailureClassConfig}, + {http.StatusUnprocessableEntity, authcircuit.FailureClassConfig}, + {http.StatusBadRequest, authcircuit.FailureClassConfig}, + {http.StatusInternalServerError, authcircuit.FailureClassTransient}, + } + for _, tt := range tests { + err := &GitHubAPIError{StatusCode: tt.status, Endpoint: "/x"} + if got := classifyPollErr(err); got != tt.want { + t.Fatalf("classifyPollErr(status=%d) = %q, want %q", tt.status, got, tt.want) + } + } +} + +func TestClassifyPollErr_GitHubAPIForbiddenRateLimitIsTransient(t *testing.T) { + err := &GitHubAPIError{StatusCode: http.StatusForbidden, Endpoint: "/x", Body: "API rate limit exceeded"} + if got := classifyPollErr(err); got != authcircuit.FailureClassTransient { + t.Fatalf("classifyPollErr(403 rate limit) = %q, want transient", got) + } +} + +func TestClassifyPollErr_UnrecognizedIsTransient(t *testing.T) { + if got := classifyPollErr(errors.New("connection reset")); got != authcircuit.FailureClassTransient { + t.Fatalf("classifyPollErr(unrecognized) = %q, want transient", got) + } +} + +// TestCheckPRWatches_OpenCircuitSkipsSearchEntirely confirms checkPRWatches +// never calls FindPRByBranch for a workspace whose circuit is already open +// — the end-to-end behavior AC-9 requires for the PR-monitor loop. +func TestCheckPRWatches_OpenCircuitSkipsSearchEntirely(t *testing.T) { + poller, _, mockClient, store := setupPollerTest(t) + ctx := context.Background() + + seedTask(t, store, "t1", false) + watch := withTestWorkspace(&PRWatch{TaskID: "t1", Owner: "o", Repo: "r", Branch: "feat"}) + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create PR watch: %v", err) + } + + poller.circuits.recordOutcome(testWorkspaceID, authcircuit.FailureClassAuth, time.Now().UTC()) + + poller.checkPRWatches(ctx) + + if got := mockClient.FindPRByBranchCallCount(); got != 0 { + t.Fatalf("FindPRByBranch calls = %d, want 0 while the workspace circuit is open", got) + } +} + +// TestCheckPRWatches_ClosedCircuitStillSearches is the control for the test +// above: with no recorded failures, the same watch is searched normally. +func TestCheckPRWatches_ClosedCircuitStillSearches(t *testing.T) { + poller, _, mockClient, store := setupPollerTest(t) + ctx := context.Background() + + seedTask(t, store, "t1", false) + watch := withTestWorkspace(&PRWatch{TaskID: "t1", Owner: "o", Repo: "r", Branch: "feat"}) + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create PR watch: %v", err) + } + + poller.checkSinglePRWatch(ctx, watch) + + if got := mockClient.FindPRByBranchCallCount(); got != 1 { + t.Fatalf("FindPRByBranch calls = %d, want 1 with a closed circuit", got) + } +} + +func TestCheckPRWatches_BatchedAuthFailureSkipsPerWatchFallback(t *testing.T) { + poller, _, gh, store := setupBatchedPollerTest(t) + ctx := context.Background() + + seedTask(t, store, "t1", false) + watch := withTestWorkspace(&PRWatch{ + TaskID: "t1", Owner: "o", Repo: "r", Branch: "feat", + }) + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create PR watch: %v", err) + } + gh.branchErr = ErrGitHubConnectionInvalid + + poller.checkPRWatches(ctx) + + if got := gh.FindPRByBranchCallCount(); got != 0 { + t.Fatalf("FindPRByBranch calls = %d, want 0 after batched auth failure opens the workspace circuit", got) + } +} diff --git a/apps/backend/internal/github/poller_test.go b/apps/backend/internal/github/poller_test.go index caf995fd84..bae3db4fb5 100644 --- a/apps/backend/internal/github/poller_test.go +++ b/apps/backend/internal/github/poller_test.go @@ -156,6 +156,54 @@ func TestCheckSinglePRWatch_MergedPR_SyncsThenResets(t *testing.T) { } } +func TestCheckPRWatches_MergedPR_SyncsThenResets(t *testing.T) { + poller, _, mockClient, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-1", false) + + now := time.Now().UTC() + mergedAt := now.Add(-time.Hour) + mockClient.AddPR(&PR{ + Number: 42, + Title: "Feature PR", + State: prStateMerged, + HeadSHA: "abc123", + HeadBranch: "feature-branch", + RepoOwner: "owner", + RepoName: "repo", + MergedAt: &mergedAt, + }) + + watch := withTestWorkspace(&PRWatch{ + SessionID: "sess-1", + TaskID: "task-1", + Owner: "owner", + Repo: "repo", + PRNumber: 42, + Branch: "feature-branch", + }) + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create PR watch: %v", err) + } + if err := store.CreateTaskPR(ctx, &TaskPR{ + TaskID: "task-1", Owner: "owner", Repo: "repo", PRNumber: 42, + PRURL: "https://github.com/owner/repo/pull/42", PRTitle: "Feature PR", + HeadBranch: "feature-branch", BaseBranch: "main", State: prStateOpen, + }); err != nil { + t.Fatalf("create task PR: %v", err) + } + + poller.checkPRWatches(ctx) + + updated, err := store.GetTaskPR(ctx, "task-1") + if err != nil { + t.Fatalf("get task PR: %v", err) + } + if updated == nil || updated.State != prStateMerged { + t.Fatalf("expected merged task PR after poll, got %#v", updated) + } +} + func TestCheckSinglePRWatch_OpenPR_SyncsOnChange(t *testing.T) { poller, _, mockClient, store := setupPollerTest(t) ctx := context.Background() @@ -239,15 +287,23 @@ func TestCheckSinglePRWatch_OpenPR_SyncsOnChange(t *testing.T) { // mockTaskBranchProvider implements TaskBranchProvider for testing. type mockTaskBranchProvider struct { - tasks []TaskBranchInfo - err error - branches map[string]string // sessionID -> branch + tasks []TaskBranchInfo + err error + branches map[string]string // repositoryID -> branch + branchSets map[string][]string // repositoryID -> branches } func (m *mockTaskBranchProvider) ListTasksNeedingPRWatch(_ context.Context) ([]TaskBranchInfo, error) { return m.tasks, m.err } +func (m *mockTaskBranchProvider) ResolveBranchForRepository(_ context.Context, _, repositoryID string) string { + if m.branches != nil { + return m.branches[repositoryID] + } + return "" +} + func (m *mockTaskBranchProvider) ResolveBranchForWatch(_ context.Context, watch *PRWatch) string { if m.branches != nil { return m.branches[watch.SessionID] @@ -255,6 +311,13 @@ func (m *mockTaskBranchProvider) ResolveBranchForWatch(_ context.Context, watch return "" } +func (m *mockTaskBranchProvider) ResolveBranchesForRepository(_ context.Context, _, repositoryID string) []string { + if m.branchSets != nil { + return m.branchSets[repositoryID] + } + return nil +} + func TestReconcileWatches_CreatesWatchesForTasks(t *testing.T) { poller, _, _, store := setupPollerTest(t) ctx := context.Background() @@ -415,21 +478,23 @@ func TestRefreshStaleBranches_UpdatesBranchWhenChanged(t *testing.T) { // Create a watch with pr_number=0 on old branch. seedTask(t, store, "t1", false) watch := &PRWatch{ - SessionID: "s1", - TaskID: "t1", - Owner: "myorg", - Repo: "myrepo", - PRNumber: 0, - Branch: "old-branch", + SessionID: "s1", + TaskID: "t1", + RepositoryID: "repo-1", + Owner: "myorg", + Repo: "myrepo", + PRNumber: 0, + Branch: "old-branch", } if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { t.Fatalf("create PR watch: %v", err) } - // Provider resolves a different branch for this session. + // Provider resolves a different branch for this repository, independent + // of session identity. prov := &mockTaskBranchProvider{ branches: map[string]string{ - "s1": "new-branch", + "repo-1": "new-branch", }, } poller.SetTaskBranchProvider(prov) @@ -449,18 +514,143 @@ func TestRefreshStaleBranches_UpdatesBranchWhenChanged(t *testing.T) { } } +// TestRefreshStaleBranches_ResolvesByRepositoryNotSession locks in the fix for +// a real multi-repo correctness bug: the branch resolver must be scoped to +// watch.RepositoryID, not resolved once per session and reused for every +// repository. Two still-searching watches on the SAME session but DIFFERENT +// repositories must each pick up their own repository's branch. +func TestRefreshStaleBranches_ResolvesByRepositoryNotSession(t *testing.T) { + poller, _, _, store := setupPollerTest(t) + ctx := context.Background() + + seedTask(t, store, "t1", false) + watchA := &PRWatch{ + SessionID: "s1", + TaskID: "t1", + RepositoryID: "repo-a", + Owner: "myorg", + Repo: "repo-a-name", + PRNumber: 0, + Branch: "old-a", + } + watchB := &PRWatch{ + SessionID: "s1", + TaskID: "t1", + RepositoryID: "repo-b", + Owner: "myorg", + Repo: "repo-b-name", + PRNumber: 0, + Branch: "old-b", + } + if err := store.CreatePRWatch(ctx, withTestWorkspace(watchA)); err != nil { + t.Fatalf("create PR watch A: %v", err) + } + if err := store.CreatePRWatch(ctx, withTestWorkspace(watchB)); err != nil { + t.Fatalf("create PR watch B: %v", err) + } + + prov := &mockTaskBranchProvider{ + branches: map[string]string{ + "repo-a": "new-a", + "repo-b": "new-b", + }, + } + poller.SetTaskBranchProvider(prov) + + poller.refreshStaleBranches(ctx) + + updatedA, err := store.GetPRWatchBySessionAndRepo(ctx, "s1", "repo-a") + if err != nil { + t.Fatalf("get watch A: %v", err) + } + if updatedA.Branch != "new-a" { + t.Errorf("expected repo-a branch %q, got %q", "new-a", updatedA.Branch) + } + updatedB, err := store.GetPRWatchBySessionAndRepo(ctx, "s1", "repo-b") + if err != nil { + t.Fatalf("get watch B: %v", err) + } + if updatedB.Branch != "new-b" { + t.Errorf("expected repo-b branch %q, got %q", "new-b", updatedB.Branch) + } +} + +func TestRefreshStaleBranches_PreservesSameRepositoryMultiBranchWatches(t *testing.T) { + poller, _, _, store := setupPollerTest(t) + ctx := context.Background() + + seedTask(t, store, "t1", false) + watchA := &PRWatch{ + SessionID: "s1", TaskID: "t1", RepositoryID: "repo-1", + Owner: "myorg", Repo: "myrepo", PRNumber: 0, Branch: "feature-a", + } + watchB := &PRWatch{ + SessionID: "s2", TaskID: "t1", RepositoryID: "repo-1", + Owner: "myorg", Repo: "myrepo", PRNumber: 0, Branch: "feature-b", + } + for _, watch := range []*PRWatch{watchA, watchB} { + if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { + t.Fatalf("create PR watch %s: %v", watch.Branch, err) + } + } + + prov := &mockTaskBranchProvider{ + branches: map[string]string{"repo-1": "feature-a"}, + branchSets: map[string][]string{"repo-1": []string{"feature-a", "feature-b"}}, + } + poller.SetTaskBranchProvider(prov) + + poller.refreshStaleBranches(ctx) + + watches, err := store.ListPRWatchesByTask(ctx, "t1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + got := map[string]bool{} + for _, watch := range watches { + got[watch.Branch] = true + } + if !got["feature-a"] || !got["feature-b"] || len(watches) != 2 { + t.Fatalf("watches after refresh = %+v, want both same-repository branches preserved", watches) + } +} + +func TestRefreshStaleBranches_RemovesSearchingWatchMissingFromMultiBranchSet(t *testing.T) { + poller, _, _, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "t1", false) + for _, watch := range []*PRWatch{ + {SessionID: "s1", TaskID: "t1", RepositoryID: "repo-1", Owner: "myorg", Repo: "myrepo", Branch: "feature-a"}, + {SessionID: "s2", TaskID: "t1", RepositoryID: "repo-1", Owner: "myorg", Repo: "myrepo", Branch: "feature-b"}, + } { + if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { + t.Fatalf("create PR watch %s: %v", watch.Branch, err) + } + } + poller.SetTaskBranchProvider(&mockTaskBranchProvider{branches: map[string]string{"repo-1": "feature-a"}, branchSets: map[string][]string{"repo-1": {"feature-a", "feature-c"}}}) + poller.refreshStaleBranches(ctx) + watches, err := store.ListPRWatchesByTask(ctx, "t1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(watches) != 1 || watches[0].Branch != "feature-a" { + t.Fatalf("watches after removing obsolete branch = %+v, want only feature-a", watches) + } +} + func TestRefreshStaleBranches_SkipsWhenBranchUnchanged(t *testing.T) { poller, _, _, store := setupPollerTest(t) ctx := context.Background() seedTask(t, store, "t1", false) watch := &PRWatch{ - SessionID: "s1", - TaskID: "t1", - Owner: "myorg", - Repo: "myrepo", - PRNumber: 0, - Branch: "same-branch", + SessionID: "s1", + TaskID: "t1", + RepositoryID: "repo-1", + Owner: "myorg", + Repo: "myrepo", + PRNumber: 0, + Branch: "same-branch", } if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { t.Fatalf("create PR watch: %v", err) @@ -468,7 +658,7 @@ func TestRefreshStaleBranches_SkipsWhenBranchUnchanged(t *testing.T) { prov := &mockTaskBranchProvider{ branches: map[string]string{ - "s1": "same-branch", + "repo-1": "same-branch", }, } poller.SetTaskBranchProvider(prov) @@ -488,12 +678,13 @@ func TestRefreshStaleBranches_SkipsWatchesWithPR(t *testing.T) { // Watch that already found a PR (pr_number > 0). seedTask(t, store, "t1", false) watch := &PRWatch{ - SessionID: "s1", - TaskID: "t1", - Owner: "myorg", - Repo: "myrepo", - PRNumber: 42, - Branch: "old-branch", + SessionID: "s1", + TaskID: "t1", + RepositoryID: "repo-1", + Owner: "myorg", + Repo: "myrepo", + PRNumber: 42, + Branch: "old-branch", } if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { t.Fatalf("create PR watch: %v", err) @@ -501,7 +692,7 @@ func TestRefreshStaleBranches_SkipsWatchesWithPR(t *testing.T) { prov := &mockTaskBranchProvider{ branches: map[string]string{ - "s1": "new-branch", + "repo-1": "new-branch", }, } poller.SetTaskBranchProvider(prov) @@ -521,12 +712,13 @@ func TestRefreshStaleBranches_SkipsWhenResolverReturnsEmpty(t *testing.T) { seedTask(t, store, "t1", false) watch := &PRWatch{ - SessionID: "s1", - TaskID: "t1", - Owner: "myorg", - Repo: "myrepo", - PRNumber: 0, - Branch: "old-branch", + SessionID: "s1", + TaskID: "t1", + RepositoryID: "repo-1", + Owner: "myorg", + Repo: "myrepo", + PRNumber: 0, + Branch: "old-branch", } if err := store.CreatePRWatch(ctx, withTestWorkspace(watch)); err != nil { t.Fatalf("create PR watch: %v", err) diff --git a/apps/backend/internal/github/service_connections.go b/apps/backend/internal/github/service_connections.go index 52e0b3416e..5ec5e29426 100644 --- a/apps/backend/internal/github/service_connections.go +++ b/apps/backend/internal/github/service_connections.go @@ -37,6 +37,29 @@ func (s *Service) GetWorkspaceConnectionHealth(ctx context.Context) (WorkspaceCo return s.store.GetWorkspaceConnectionHealth(ctx) } +// WorkspaceConnectionFingerprint returns an opaque, non-secret string that +// changes whenever the workspace's automation connection is replaced, +// reconnected, or has its credential rotated/revoked (status or credential +// generation changes). Callers (e.g. workflow-sync's and the PR-watch +// poller's auth circuit breakers) use this to detect "the credential +// changed" and reset an open backoff circuit promptly instead of waiting out +// the schedule. Returns "" when no connection exists — an empty fingerprint +// must never be treated as "changed" by callers, since there was nothing to +// compare against. +func (s *Service) WorkspaceConnectionFingerprint(ctx context.Context, workspaceID string) (string, error) { + if s == nil || s.store == nil { + return "", nil + } + connection, err := s.store.GetWorkspaceConnection(ctx, workspaceID) + if err != nil { + return "", err + } + if connection == nil { + return "", nil + } + return fmt.Sprintf("%s:%d", connection.Status, connection.CredentialGeneration), nil +} + func (s *Service) SetWorkspaceConnection( ctx context.Context, workspaceID string, diff --git a/apps/backend/internal/github/service_pr_discovery_health_test.go b/apps/backend/internal/github/service_pr_discovery_health_test.go index 6d3089a7d4..a1bd73b500 100644 --- a/apps/backend/internal/github/service_pr_discovery_health_test.go +++ b/apps/backend/internal/github/service_pr_discovery_health_test.go @@ -246,13 +246,16 @@ func TestPRDiscoveryHealth_CapacityReportsDegradedWithoutEvictingLiveScopes(t *t func TestPRDiscoveryHealth_RemovesFailedTargetAfterLastWatchConsumer(t *testing.T) { _, service, _, store := setupBatchedPollerTest(t) ctx := context.Background() - seedTask(t, store, "task-health-consumers", false) + seedTask(t, store, "task-health-consumers-a", false) + seedTask(t, store, "task-health-consumers-b", false) + // Both watches share one failed-target consumer set, but task-owned + // watch uniqueness requires one task per (repository, branch) target. watchA := withTestWorkspace(&PRWatch{ - SessionID: "session-health-consumer-a", TaskID: "task-health-consumers", Owner: "o", Repo: "r", + SessionID: "session-health-consumer-a", TaskID: "task-health-consumers-a", Owner: "o", Repo: "r", Branch: "feature/health-consumers", }) watchB := withTestWorkspace(&PRWatch{ - SessionID: "session-health-consumer-b", TaskID: "task-health-consumers", Owner: "o", Repo: "r", + SessionID: "session-health-consumer-b", TaskID: "task-health-consumers-b", Owner: "o", Repo: "r", Branch: "feature/health-consumers", }) for _, watch := range []*PRWatch{watchA, watchB} { diff --git a/apps/backend/internal/github/service_pr_watch.go b/apps/backend/internal/github/service_pr_watch.go index d30577447e..9cc785482c 100644 --- a/apps/backend/internal/github/service_pr_watch.go +++ b/apps/backend/internal/github/service_pr_watch.go @@ -47,14 +47,20 @@ func (s *Service) createPRWatch( // from a prior auth failure should be re-probed immediately rather // than held for the rest of the 10-min TTL. s.evictRepoNegative(owner, repo) - existing, err := s.store.GetPRWatchBySessionRepoAndBranch(ctx, sessionID, repositoryID, branch) + // Task-owned identity (ADR 2026-08-31-task-owned-pr-watch-identity): + // look up the canonical watch by task/repository, not by session, so a + // resumed or concurrent session on the same task/branch reuses the + // existing watch instead of creating a duplicate. A non-zero prNumber + // (e.g. AssociatePRByURL, which already knows the PR) checks discovered + // identity; otherwise it checks searching identity. + taskID = s.resolveEffectiveAssociationTaskIDForSession(ctx, sessionID, taskID, repositoryID) + existing, err := s.getExistingCanonicalPRWatch(ctx, taskID, repositoryID, prNumber, branch) if err != nil { return nil, err } if existing != nil { - return existing, nil // already watching this (session, repo, branch) + return existing, nil // already watching this (task, repo, branch/PR) } - taskID = s.resolveEffectiveAssociationTaskIDForSession(ctx, sessionID, taskID, repositoryID) w := &PRWatch{ WorkspaceID: workspaceID, SessionID: sessionID, @@ -76,6 +82,21 @@ func (s *Service) createPRWatch( return w, nil } +// getExistingCanonicalPRWatch resolves the task-owned canonical watch for a +// prospective (task, repository, branch/PR) target: discovered identity +// (task_id, repository_id, pr_number) when prNumber is known, otherwise +// searching identity (task_id, repository_id, branch). Shared by +// createPRWatch and ensurePRWatch so both dedupe against the same canonical +// row regardless of which session is asking. +func (s *Service) getExistingCanonicalPRWatch( + ctx context.Context, taskID, repositoryID string, prNumber int, branch string, +) (*PRWatch, error) { + if prNumber != 0 { + return s.store.GetPRWatchByTaskRepoPRNumber(ctx, taskID, repositoryID, prNumber) + } + return s.store.GetPRWatchByTaskRepoBranch(ctx, taskID, repositoryID, branch) +} + // GetPRWatch returns one PR watch after authorizing its stored workspace. func (s *Service) GetPRWatch(ctx context.Context, id string) (*PRWatch, error) { watch, err := s.store.GetPRWatch(ctx, id) @@ -115,6 +136,12 @@ func (s *Service) ListActivePRWatches(ctx context.Context) ([]*PRWatch, error) { return s.store.ListActivePRWatches(ctx) } +// PRWatchCardinality returns aggregate watch-health gauges without exposing +// watch identities. +func (s *Service) PRWatchCardinality(ctx context.Context) (PRWatchCardinality, error) { + return s.store.PRWatchCardinality(ctx) +} + // ListActivePRWatchesForWorkspace returns active PR watches for one authorized // workspace. The all-workspaces variant is reserved for identity-less pollers. func (s *Service) ListActivePRWatchesForWorkspace(ctx context.Context, workspaceID string) ([]*PRWatch, error) { @@ -300,14 +327,24 @@ func (s *Service) ensurePRWatch( // rationale. The existing-watch early-return path must not inherit a // stale "missing" verdict from a prior incarnation. s.evictRepoNegative(owner, repo) - existing, err := s.store.GetPRWatchBySessionRepoAndBranch(ctx, sessionID, repositoryID, branch) + // Task-owned identity — see createPRWatch's getExistingCanonicalPRWatch + // comment. Reusing the canonical searching watch here is what stops a + // resumed/concurrent session from multiplying watches for the same + // task/repository/branch (ADR 2026-08-31-task-owned-pr-watch-identity). + taskID = s.resolveEffectiveAssociationTaskIDForSession(ctx, sessionID, taskID, repositoryID) + existing, err := s.store.GetPRWatchByTaskRepoBranch(ctx, taskID, repositoryID, branch) if err != nil { return nil, err } + if existing == nil { + existing, err = s.store.GetPRWatchByTaskRepoBranchAny(ctx, taskID, repositoryID, branch) + if err != nil { + return nil, err + } + } if existing != nil { return existing, nil } - taskID = s.resolveEffectiveAssociationTaskIDForSession(ctx, sessionID, taskID, repositoryID) w := &PRWatch{ WorkspaceID: workspaceID, SessionID: sessionID, @@ -321,8 +358,9 @@ func (s *Service) ensurePRWatch( if err := s.store.CreatePRWatch(ctx, w); err != nil { return nil, fmt.Errorf("ensure PR watch: %w", err) } - s.logger.Info("created PR watch for session (will search for PR)", + s.logger.Info("created PR watch for task (will search for PR)", zap.String("session_id", sessionID), + zap.String("task_id", taskID), zap.String("repository_id", repositoryID), zap.String("branch", branch)) return w, nil diff --git a/apps/backend/internal/github/service_pr_watch_multi_branch_test.go b/apps/backend/internal/github/service_pr_watch_multi_branch_test.go index 7e8a4231d9..4e4a677de2 100644 --- a/apps/backend/internal/github/service_pr_watch_multi_branch_test.go +++ b/apps/backend/internal/github/service_pr_watch_multi_branch_test.go @@ -2,6 +2,7 @@ package github import ( "context" + "sync" "testing" "time" ) @@ -140,7 +141,7 @@ func TestUpdatePRWatchBranchIfSearching_NoCollision_UpdatesBranch(t *testing.T) // TestUpdatePRWatchBranchIfSearching_CollidesWithSibling_DropsSource locks // the fix for the UNIQUE-constraint collision: when a sibling watch already -// owns the destination (session, repo, branch) triple, the source row is +// owns the destination (task, repo, branch) triple, even from another session, the source row is // deleted instead of triggering a UNIQUE constraint error. func TestUpdatePRWatchBranchIfSearching_CollidesWithSibling_DropsSource(t *testing.T) { _, svc, _, store := setupPollerTest(t) @@ -151,7 +152,7 @@ func TestUpdatePRWatchBranchIfSearching_CollidesWithSibling_DropsSource(t *testi if err != nil { t.Fatalf("create source watch: %v", err) } - sibling, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-1", "task-1", "repo-1", "owner", "repo", 0, "feature/B") + sibling, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-2", "task-1", "repo-1", "owner", "repo", 0, "feature/B") if err != nil { t.Fatalf("create sibling watch: %v", err) } @@ -160,7 +161,7 @@ func TestUpdatePRWatchBranchIfSearching_CollidesWithSibling_DropsSource(t *testi t.Fatalf("UpdatePRWatchBranchIfSearching must not error on sibling collision: %v", err) } - all, err := store.ListPRWatchesBySession(ctx, "session-1") + all, err := store.ListPRWatchesByTask(ctx, "task-1") if err != nil { t.Fatalf("list watches: %v", err) } @@ -219,6 +220,153 @@ func TestUpdatePRWatchBranchIfSearching_CollidesWithSiblingHasPR_DropsSource(t * } } +func TestResetPRWatch_CollidesWithCanonicalSibling_DropsSource(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-1", false) + + source, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-1", "task-1", "repo-1", "owner", "repo", 0, "feature/A") + if err != nil { + t.Fatalf("create source watch: %v", err) + } + sibling, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-2", "task-1", "repo-1", "owner", "repo", 0, "feature/B") + if err != nil { + t.Fatalf("create sibling watch: %v", err) + } + + if err := svc.ResetPRWatch(ctx, source.ID, "feature/B"); err != nil { + t.Fatalf("ResetPRWatch must merge task-owned branch collisions: %v", err) + } + + all, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(all) != 1 { + t.Fatalf("expected source watch dropped, got %d remaining", len(all)) + } + if all[0].ID != sibling.ID { + t.Fatalf("expected sibling watch %q to survive, got %q", sibling.ID, all[0].ID) + } + if all[0].Branch != "feature/B" || all[0].PRNumber != 0 { + t.Fatalf("sibling watch changed unexpectedly: %+v", all[0]) + } +} + +func TestResetPRWatch_CollidesWithDiscoveredSibling_DropsSource(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-1", false) + + source, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-1", "task-1", "repo-1", "owner", "repo", 0, "feature/A") + if err != nil { + t.Fatalf("create source watch: %v", err) + } + sibling, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-2", "task-1", "repo-1", "owner", "repo", 99, "feature/B") + if err != nil { + t.Fatalf("create sibling watch: %v", err) + } + + if err := svc.ResetPRWatch(ctx, source.ID, "feature/B"); err != nil { + t.Fatalf("ResetPRWatch must merge with a task-owned discovered branch sibling: %v", err) + } + + all, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(all) != 1 { + t.Fatalf("expected source watch dropped, got %d remaining", len(all)) + } + if all[0].ID != sibling.ID { + t.Fatalf("expected sibling watch %q to survive, got %q", sibling.ID, all[0].ID) + } + if all[0].Branch != "feature/B" || all[0].PRNumber != 99 { + t.Fatalf("discovered sibling watch changed unexpectedly: %+v", all[0]) + } +} + +func TestResetPRWatch_ConcurrentSessionsCoalesceDestination(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-1", false) + + first, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-A", "task-1", "repo-1", "owner", "repo", 0, "feature/A") + if err != nil { + t.Fatalf("create session A watch: %v", err) + } + second, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-B", "task-1", "repo-1", "owner", "repo", 0, "feature/B") + if err != nil { + t.Fatalf("create session B watch: %v", err) + } + + start := make(chan struct{}) + var wg sync.WaitGroup + errs := make(chan error, 2) + for _, id := range []string{first.ID, second.ID} { + wg.Add(1) + go func(watchID string) { + defer wg.Done() + <-start + errs <- svc.ResetPRWatch(ctx, watchID, "feature/B") + }(id) + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent ResetPRWatch: %v", err) + } + } + + watches, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(watches) != 1 { + t.Fatalf("expected one coalesced destination watch, got %d", len(watches)) + } + if watches[0].Branch != "feature/B" || watches[0].PRNumber != 0 { + t.Fatalf("unexpected coalesced watch: %+v", watches[0]) + } +} + +func TestResetPRWatch_CollisionDoesNotSwitchBackToSource(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + ctx := context.Background() + seedTask(t, store, "task-1", false) + + source, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-A", "task-1", "repo-1", "owner", "repo", 0, "feature/A") + if err != nil { + t.Fatalf("create source watch: %v", err) + } + sibling, err := svc.CreatePRWatchForWorkspace(ctx, testWorkspaceID, "session-B", "task-1", "repo-1", "owner", "repo", 0, "feature/B") + if err != nil { + t.Fatalf("create sibling watch: %v", err) + } + + if err := svc.ResetPRWatch(ctx, source.ID, sibling.Branch); err != nil { + t.Fatalf("ResetPRWatch must coalesce the source into the canonical branch: %v", err) + } + // A delayed reset from the source session must be a no-op after its row + // was merged, rather than recreating or switching the canonical watch. + if err := svc.ResetPRWatch(ctx, source.ID, source.Branch); err != nil { + t.Fatalf("stale source reset must remain a no-op: %v", err) + } + + watches, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(watches) != 1 || watches[0].ID != sibling.ID { + t.Fatalf("expected only canonical sibling after stale reset, got %+v", watches) + } + if watches[0].Branch != sibling.Branch || watches[0].PRNumber != 0 { + t.Fatalf("canonical sibling switched unexpectedly: %+v", watches[0]) + } +} + // TestUpdatePRWatchBranchIfSearching_PRAlreadyFound_NoOp preserves the // existing "searching" guard: a watch that already found its PR // (pr_number != 0) must not have its branch overwritten. diff --git a/apps/backend/internal/github/service_pr_watch_workspace_group_redirect_test.go b/apps/backend/internal/github/service_pr_watch_workspace_group_redirect_test.go index cdc8b9653a..c39c750247 100644 --- a/apps/backend/internal/github/service_pr_watch_workspace_group_redirect_test.go +++ b/apps/backend/internal/github/service_pr_watch_workspace_group_redirect_test.go @@ -312,6 +312,37 @@ func TestAssociatePRByURL_WatchAndAssociationAgreeOnRedirectedOwner(t *testing.T } } +func TestCreatePRWatch_RedirectedMemberReusesOwnerCanonicalWatch(t *testing.T) { + _, svc, _, store := setupPollerTest(t) + + issueStore := newMultiTaskIssueStore() + issueStore.addTask(&taskmodels.Task{ID: "member1"}, "repo-canonical") + issueStore.addTask(&taskmodels.Task{ID: "owner1"}, "repo-canonical") + svc.SetTaskIssueStore(issueStore) + svc.SetWorkspaceGroupOwnerResolver(&fakeWorkspaceGroupOwnerResolver{ownerTaskID: "owner1"}) + + ctx := context.Background() + existing, err := svc.CreatePRWatch(ctx, "owner-session", "owner1", "repo-canonical", "org", "repo", 0, "feature-x") + if err != nil { + t.Fatalf("create owner watch: %v", err) + } + + got, err := svc.CreatePRWatch(ctx, "member-session", "member1", "repo-canonical", "org", "repo", 0, "feature-x") + if err != nil { + t.Fatalf("create redirected member watch: %v", err) + } + if got.ID != existing.ID { + t.Fatalf("watch ID = %q, want existing owner watch %q", got.ID, existing.ID) + } + watches, err := store.ListPRWatchesByTask(ctx, "owner1") + if err != nil { + t.Fatalf("list owner watches: %v", err) + } + if len(watches) != 1 { + t.Fatalf("owner watches = %+v, want exactly one canonical watch", watches) + } +} + // TestCheckSinglePRWatch_LegacyMemberWatchSyncsOwnersTaskPR covers a watch // that already carries a member task's ID — either written before this fix // existed, or (per apps/backend/CLAUDE.md's "PR status sync coverage") any diff --git a/apps/backend/internal/github/store.go b/apps/backend/internal/github/store.go index 4806277697..c67faa48bf 100644 --- a/apps/backend/internal/github/store.go +++ b/apps/backend/internal/github/store.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgconn" "github.com/jmoiron/sqlx" dbutil "github.com/kandev/kandev/internal/db" @@ -27,6 +28,7 @@ type Store struct { deploymentAppPersistenceMu sync.Mutex appLifecycleLocksMu sync.Mutex appLifecycleLocks map[string]*appRegistrationLifecycleLock + prWatchMigration *PRWatchMigrationStats } type appRegistrationLifecycleLock struct { @@ -93,10 +95,17 @@ func (s *Store) lockAppRegistrationLifecycle(registrationID string) func() { // installs migrated from the single-repo schema get the column dropped to // `”` (empty) and the constraints rebuilt by `migratePRTablesForMultiRepo`. const createTablesSQL = ` + -- github_pr_watches is task-owned (ADR 2026-08-31-task-owned-pr-watch-identity): + -- a searching watch (pr_number = 0) is canonical by (task_id, repository_id, + -- branch) and a discovered watch (pr_number != 0) is canonical by + -- (task_id, repository_id, pr_number). Both are enforced by the partial + -- unique indexes in applyIdempotentSchemaIndexes, not an inline UNIQUE + -- constraint here (SQLite can't express a partial UNIQUE inline). + -- session_id is optional provenance only - never a uniqueness component. CREATE TABLE IF NOT EXISTS github_pr_watches ( id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL DEFAULT '', - session_id TEXT NOT NULL, + session_id TEXT NOT NULL DEFAULT '', task_id TEXT NOT NULL, repository_id TEXT NOT NULL DEFAULT '', owner TEXT NOT NULL, @@ -108,8 +117,7 @@ const createTablesSQL = ` last_check_status TEXT DEFAULT '', last_review_state TEXT DEFAULT '', created_at DATETIME NOT NULL, - updated_at DATETIME NOT NULL, - UNIQUE(session_id, repository_id, branch) + updated_at DATETIME NOT NULL ); CREATE TABLE IF NOT EXISTS github_task_prs ( @@ -832,6 +840,9 @@ func (s *Store) initSchemaData(legacyUpgrade bool) error { if err := s.healTaskOwnedOrphans(); err != nil { return err } + if err := s.migratePRWatchesToTaskOwnership(); err != nil { + return fmt.Errorf("migrate PR watches to task ownership: %w", err) + } if err := s.migrateTaskCIOptionsToPRScope(); err != nil { return fmt.Errorf("migrate task CI options to PR scope: %w", err) } @@ -951,6 +962,28 @@ func (s *Store) applyIdempotentSchemaIndexes() error { )); err != nil { return fmt.Errorf("create idx_github_task_ci_pr_state_task: %w", err) } + // Canonical PR-watch identity (ADR 2026-08-31-task-owned-pr-watch-identity): + // a searching watch is unique per (task, repository, branch); a discovered + // watch is unique per (task, repository, pr_number). session_id never + // participates. Partial indexes because SQLite can't express a + // conditional UNIQUE inline in CREATE TABLE. Applied unconditionally on + // every boot (fresh installs and post-migration upgrades alike) - once + // migratePRWatchesToTaskOwnership has deduplicated any legacy rows, both + // creations are no-ops that just confirm the invariant holds. + if _, err := s.db.Exec(schemaSQLForDriver( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_github_pr_watches_searching + ON github_pr_watches (task_id, repository_id, branch) WHERE pr_number = 0`, + s.db.DriverName(), + )); err != nil { + return fmt.Errorf("create idx_github_pr_watches_searching: %w", err) + } + if _, err := s.db.Exec(schemaSQLForDriver( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_github_pr_watches_discovered + ON github_pr_watches (task_id, repository_id, pr_number) WHERE pr_number != 0`, + s.db.DriverName(), + )); err != nil { + return fmt.Errorf("create idx_github_pr_watches_discovered: %w", err) + } return nil } @@ -1735,6 +1768,399 @@ func (s *Store) rebuildIfHasLegacyConstraint(table, legacyConstraint, createNew, return tx.Commit() } +// PRWatchMigrationStats reports the outcome of migratePRWatchesToTaskOwnership. +// Populated only when the legacy session-scoped schema was detected; nil on a +// fresh install or a database that has already been migrated. Callers (tests, +// future health/metrics wiring) read this via Store.PRWatchMigrationStats +// instead of relying on log scraping. +type PRWatchMigrationStats struct { + RowsBefore int + RowsAfter int + OrphansRemoved int + DuplicatesRemoved int +} + +// PRWatchMigrationStats returns the aggregate counts from the most recent +// canonical PR-watch migration performed during store initialization, or nil +// when no legacy schema was found (fresh install / already migrated). +func (s *Store) PRWatchMigrationStats() *PRWatchMigrationStats { + return s.prWatchMigration +} + +// legacyPRWatchesUniqueConstraint is the pre-ADR-2026-08-31 constraint that +// scoped watch identity to session_id instead of task_id. Its presence in +// sqlite_master.sql is the idempotency trigger for +// migratePRWatchesToTaskOwnership: a fresh install's createTablesSQL no +// longer includes it, and a database that has already been migrated has had +// it rebuilt away, so both cases skip the migration entirely. +const legacyPRWatchesUniqueConstraint = "UNIQUE(session_id, repository_id, branch)" + +// migratePRWatchesToTaskOwnership makes github_pr_watches task-owned per +// ADR-2026-08-31-task-owned-pr-watch-identity: a searching watch (pr_number = +// 0) becomes canonical by (task_id, repository_id, branch) and a discovered +// watch (pr_number != 0) becomes canonical by (task_id, repository_id, +// pr_number), instead of the legacy (session_id, repository_id, branch). +// +// Idempotent and transactional: it only runs when the legacy UNIQUE +// constraint is still present (checked via sqlite_master.sql), and either +// fully commits or leaves every row untouched. It must run after this +// process's pre-migration SnapshotSQLite backup (persistence.Provide runs +// that backup before any repository store, including this one, opens its +// schema - see internal/persistence/provider.go) so a failure here can +// always be rolled back from the snapshot. +// +// Steps, in order: (1) clear session_id provenance for sessions that no +// longer exist, (2) remove watches for tasks that no longer exist or whose +// repository has been detached from the task (never polled - see +// ListActivePRWatches - so safe to drop), (3) deduplicate remaining rows by +// (task_id, repository_id, branch), preferring any discovered row and +// merging the newest check/comment/review watermarks into the survivor, (4) +// a second pass deduplicates by (task_id, repository_id, pr_number) in case +// two different branches independently resolved to the same PR, and (5) the +// table is rebuilt without the legacy constraint so the caller can install +// the new partial unique indexes (applyIdempotentSchemaIndexes) without a +// uniqueness violation. +func (s *Store) migratePRWatchesToTaskOwnership() error { + if dialect.IsPostgres(s.db.DriverName()) { + return s.migratePRWatchesToTaskOwnershipRows(false) + } + hasLegacy, err := s.tableSQLContains("github_pr_watches", legacyPRWatchesUniqueConstraint) + if err != nil { + return fmt.Errorf("inspect github_pr_watches schema: %w", err) + } + if !hasLegacy { + return nil + } + return s.migratePRWatchesToTaskOwnershipRows(true) +} + +func (s *Store) migratePRWatchesToTaskOwnershipRows(rebuild bool) error { + // Resolve table existence up front, on the shared pool, before opening + // the transaction below: the SQLite writer pool is capped to a single + // connection, so calling s.tableExists (which borrows a pool connection + // via s.db) once a transaction already holds that one connection + // deadlocks forever waiting for a connection that will never free up. + hasSessions := s.tableExists("sessions") + hasTasks := s.tableExists("tasks") + hasTaskRepositories := s.tableExists("task_repositories") + + tx, err := s.db.Beginx() + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + stats := &PRWatchMigrationStats{} + if err := tx.Get(&stats.RowsBefore, `SELECT COUNT(*) FROM github_pr_watches`); err != nil { + return fmt.Errorf("count PR watches before migration: %w", err) + } + + if err := clearStalePRWatchSessionProvenance(tx, hasSessions); err != nil { + return err + } + + orphansRemoved, err := removeOrphanedPRWatches(tx, hasTasks, hasTaskRepositories) + if err != nil { + return err + } + stats.OrphansRemoved = int(orphansRemoved) + + var all []*PRWatch + if err := tx.Select(&all, `SELECT * FROM github_pr_watches`); err != nil { + return fmt.Errorf("load PR watches for dedup: %w", err) + } + toDelete, survivors := dedupPRWatchesForTaskOwnership(all) + + if err := applyPRWatchDedup(tx, toDelete, survivors); err != nil { + return err + } + stats.DuplicatesRemoved = len(toDelete) + + if rebuild { + if err := rebuildPRWatchesTableForTaskOwnership(tx); err != nil { + return err + } + } + + noop, err := prWatchMigrationNoop(tx, rebuild, stats) + if err != nil { + return err + } + if noop { + return nil + } + + if err := tx.Get(&stats.RowsAfter, `SELECT COUNT(*) FROM github_pr_watches`); err != nil { + return fmt.Errorf("count PR watches after migration: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit PR watch task-ownership migration: %w", err) + } + s.prWatchMigration = stats + return nil +} + +func prWatchMigrationNoop(tx *sqlx.Tx, rebuild bool, stats *PRWatchMigrationStats) (bool, error) { + if rebuild || stats.DuplicatesRemoved != 0 || stats.OrphansRemoved != 0 { + return false, nil + } + if err := tx.Get(&stats.RowsAfter, `SELECT COUNT(*) FROM github_pr_watches`); err != nil { + return false, fmt.Errorf("count PR watches after migration: %w", err) + } + if stats.RowsBefore != stats.RowsAfter { + return false, nil + } + if err := tx.Commit(); err != nil { + return false, fmt.Errorf("commit PR watch task-ownership migration: %w", err) + } + return true, nil +} + +// clearStalePRWatchSessionProvenance clears session_id on rows whose session +// no longer exists. session_id is provenance-only (ADR +// 2026-08-31-task-owned-pr-watch-identity) so this never removes a row, it +// only stops an already-gone session from being reported as the owner. +func clearStalePRWatchSessionProvenance(tx *sqlx.Tx, hasSessions bool) error { + if !hasSessions { + return nil + } + if _, err := tx.Exec(` + UPDATE github_pr_watches SET session_id = '' + WHERE session_id <> '' AND NOT EXISTS ( + SELECT 1 FROM sessions WHERE sessions.id = github_pr_watches.session_id + )`); err != nil { + return fmt.Errorf("clear stale PR watch session provenance: %w", err) + } + return nil +} + +// removeOrphanedPRWatches deletes rows whose task no longer exists and rows +// whose repository has been detached from their task, returning the total +// number of rows removed. Either table may be absent in unit tests that +// don't bring up the task package's schema, in which case that half of the +// sweep is skipped. +func removeOrphanedPRWatches(tx *sqlx.Tx, hasTasks, hasTaskRepositories bool) (int64, error) { + var removed int64 + if hasTasks { + res, err := tx.Exec(` + DELETE FROM github_pr_watches + WHERE NOT EXISTS (SELECT 1 FROM tasks WHERE tasks.id = github_pr_watches.task_id)`) + if err != nil { + return 0, fmt.Errorf("remove orphaned-task PR watches: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("count orphaned-task PR watches removed: %w", err) + } + removed += n + } + if hasTaskRepositories { + res, err := tx.Exec(` + DELETE FROM github_pr_watches + WHERE repository_id <> '' AND NOT EXISTS ( + SELECT 1 FROM task_repositories tr + WHERE tr.task_id = github_pr_watches.task_id AND tr.repository_id = github_pr_watches.repository_id + )`) + if err != nil { + return 0, fmt.Errorf("remove detached-repository PR watches: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("count detached-repository PR watches removed: %w", err) + } + removed += n + } + return removed, nil +} + +// dedupPRWatchesForTaskOwnership groups the given rows into their canonical +// searching/discovered identities and decides which row in each group +// survives. Pass 1 dedups by tracked branch (preferring a discovered row +// over a still-searching one so a stale duplicate that never got promoted +// does not outlive the row that actually found the PR). Pass 2 then +// guarantees discovered uniqueness across branch groups - e.g. two +// different branches (a rename) independently resolving to the same PR +// number. +func dedupPRWatchesForTaskOwnership(all []*PRWatch) (toDelete map[string]bool, survivors map[string]*PRWatch) { + toDelete = map[string]bool{} + survivors = map[string]*PRWatch{} + + type branchKey struct{ taskID, repositoryID, branch string } + branchGroups := map[branchKey][]*PRWatch{} + for _, w := range all { + if w.PRNumber != 0 { + survivors[w.ID] = w + continue + } + k := branchKey{w.TaskID, w.RepositoryID, w.Branch} + branchGroups[k] = append(branchGroups[k], w) + } + for _, group := range branchGroups { + mergePRWatchGroup(group, toDelete, survivors) + } + + type prKey struct { + taskID, repositoryID string + prNumber int + } + prGroups := map[prKey][]*PRWatch{} + for id, w := range survivors { + if toDelete[id] || w.PRNumber == 0 { + continue + } + k := prKey{w.TaskID, w.RepositoryID, w.PRNumber} + prGroups[k] = append(prGroups[k], w) + } + for _, group := range prGroups { + if len(group) < 2 { + continue + } + mergePRWatchGroup(group, toDelete, survivors) + } + + return toDelete, survivors +} + +// mergePRWatchGroup picks the survivor for one duplicate group (preferring a +// discovered row when present), merges the group's watermarks onto it, and +// marks every other member of the group for deletion. +func mergePRWatchGroup(group []*PRWatch, toDelete map[string]bool, survivors map[string]*PRWatch) { + candidates := discoveredWatches(group) + if len(candidates) == 0 { + candidates = group + } + survivor := pickPRWatchSurvivor(candidates) + mergePRWatchWatermarks(survivor, group) + survivors[survivor.ID] = survivor + for _, w := range group { + if w.ID != survivor.ID { + toDelete[w.ID] = true + } + } +} + +// applyPRWatchDedup deletes the losing rows from a dedup pass and persists +// the winning rows' merged watermarks. +func applyPRWatchDedup(tx *sqlx.Tx, toDelete map[string]bool, survivors map[string]*PRWatch) error { + for id := range toDelete { + if _, err := tx.Exec(tx.Rebind(`DELETE FROM github_pr_watches WHERE id = ?`), id); err != nil { + return fmt.Errorf("remove duplicate PR watch %s: %w", id, err) + } + } + for id, w := range survivors { + if toDelete[id] { + continue + } + if _, err := tx.Exec(tx.Rebind(` + UPDATE github_pr_watches + SET last_checked_at = ?, last_comment_at = ?, last_check_status = ?, last_review_state = ? + WHERE id = ?`), + w.LastCheckedAt, w.LastCommentAt, w.LastCheckStatus, w.LastReviewState, id); err != nil { + return fmt.Errorf("merge PR watch watermark %s: %w", id, err) + } + } + return nil +} + +// rebuildPRWatchesTableForTaskOwnership rebuilds github_pr_watches without +// the legacy UNIQUE(session_id, repository_id, branch) constraint. SQLite +// can't ALTER TABLE DROP CONSTRAINT, so this uses the standard +// create-copy-drop-rename sequence (mirrors migratePRTablesForMultiRepo). +func rebuildPRWatchesTableForTaskOwnership(tx *sqlx.Tx) error { + for _, stmt := range []string{ + `CREATE TABLE github_pr_watches_new ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + session_id TEXT NOT NULL DEFAULT '', + task_id TEXT NOT NULL, + repository_id TEXT NOT NULL DEFAULT '', + owner TEXT NOT NULL, + repo TEXT NOT NULL, + pr_number INTEGER NOT NULL, + branch TEXT NOT NULL, + last_checked_at DATETIME, + last_comment_at DATETIME, + last_check_status TEXT DEFAULT '', + last_review_state TEXT DEFAULT '', + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL + )`, + `INSERT INTO github_pr_watches_new ( + id, workspace_id, session_id, task_id, repository_id, owner, repo, pr_number, branch, + last_checked_at, last_comment_at, last_check_status, last_review_state, created_at, updated_at + ) SELECT + id, COALESCE(workspace_id, ''), COALESCE(session_id, ''), task_id, COALESCE(repository_id, ''), + owner, repo, pr_number, branch, + last_checked_at, last_comment_at, last_check_status, last_review_state, created_at, updated_at + FROM github_pr_watches`, + `DROP TABLE github_pr_watches`, + `ALTER TABLE github_pr_watches_new RENAME TO github_pr_watches`, + } { + if _, err := tx.Exec(stmt); err != nil { + return fmt.Errorf("rebuild github_pr_watches for task ownership: %w", err) + } + } + return nil +} + +// tableSQLContains reports whether `table`'s stored CREATE statement in +// sqlite_master contains the literal substring. Returns false (not an error) +// when the table doesn't exist, matching rebuildIfHasLegacyConstraint's +// tolerance for unexpected schema drift in tests. +func (s *Store) tableSQLContains(table, substr string) (bool, error) { + var existingSQL string + err := s.db.QueryRow(`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`, table).Scan(&existingSQL) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + return strings.Contains(existingSQL, substr), nil +} + +// discoveredWatches returns the subset of watches that have found a PR +// (pr_number != 0). +func discoveredWatches(group []*PRWatch) []*PRWatch { + var out []*PRWatch + for _, w := range group { + if w.PRNumber != 0 { + out = append(out, w) + } + } + return out +} + +// pickPRWatchSurvivor deterministically picks the most-recently-updated +// watch from a duplicate group as the canonical row to keep. +func pickPRWatchSurvivor(group []*PRWatch) *PRWatch { + best := group[0] + for _, w := range group[1:] { + if w.UpdatedAt.After(best.UpdatedAt) { + best = w + } + } + return best +} + +// mergePRWatchWatermarks folds the newest last_comment_at across the group +// into survivor, and the newest last_checked_at together with its paired +// last_check_status/last_review_state (both observed at that same check) so +// a duplicate that happened to poll more recently isn't silently discarded. +func mergePRWatchWatermarks(survivor *PRWatch, group []*PRWatch) { + for _, w := range group { + if w.LastCommentAt != nil && (survivor.LastCommentAt == nil || w.LastCommentAt.After(*survivor.LastCommentAt)) { + survivor.LastCommentAt = w.LastCommentAt + } + if w.LastCheckedAt != nil && (survivor.LastCheckedAt == nil || w.LastCheckedAt.After(*survivor.LastCheckedAt)) { + survivor.LastCheckedAt = w.LastCheckedAt + survivor.LastCheckStatus = w.LastCheckStatus + survivor.LastReviewState = w.LastReviewState + } + } +} + // --- PR Watch operations --- // CreatePRWatch creates a new PR watch. @@ -1812,6 +2238,57 @@ func (s *Store) GetPRWatchBySessionRepoAndBranch(ctx context.Context, sessionID, return &w, err } +// GetPRWatchByTaskRepoBranch returns the canonical searching watch (pr_number +// = 0) for a (task, repository, branch) triple, or nil. This is the task-owned +// identity lookup (ADR 2026-08-31-task-owned-pr-watch-identity): unlike +// GetPRWatchBySessionRepoAndBranch, session identity plays no part, so it +// finds an existing watch regardless of which session originally created it. +func (s *Store) GetPRWatchByTaskRepoBranch(ctx context.Context, taskID, repositoryID, branch string) (*PRWatch, error) { + var w PRWatch + err := s.ro.GetContext(ctx, &w, + s.ro.Rebind(`SELECT * FROM github_pr_watches + WHERE task_id = ? AND repository_id = ? AND branch = ? AND pr_number = 0 LIMIT 1`, + ), + taskID, repositoryID, branch) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return &w, err +} + +// GetPRWatchByTaskRepoBranchAny returns the canonical watch for a task, +// repository, and branch regardless of whether PR discovery has completed. +// A discovered watch must prevent reconciliation from creating a second +// searching row for the same branch. +func (s *Store) GetPRWatchByTaskRepoBranchAny(ctx context.Context, taskID, repositoryID, branch string) (*PRWatch, error) { + var w PRWatch + err := s.ro.GetContext(ctx, &w, + s.ro.Rebind(`SELECT * FROM github_pr_watches + WHERE task_id = ? AND repository_id = ? AND branch = ? + ORDER BY pr_number DESC, updated_at DESC LIMIT 1`, + ), taskID, repositoryID, branch) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return &w, err +} + +// GetPRWatchByTaskRepoPRNumber returns the canonical discovered watch for a +// (task, repository, pr_number) triple, or nil. Task-owned counterpart to +// GetPRWatchByTaskRepoBranch for watches that have already found their PR. +func (s *Store) GetPRWatchByTaskRepoPRNumber(ctx context.Context, taskID, repositoryID string, prNumber int) (*PRWatch, error) { + var w PRWatch + err := s.ro.GetContext(ctx, &w, + s.ro.Rebind(`SELECT * FROM github_pr_watches + WHERE task_id = ? AND repository_id = ? AND pr_number = ? LIMIT 1`, + ), + taskID, repositoryID, prNumber) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return &w, err +} + // ListPRWatchesBySession returns every PR watch for a session (one per repo // in multi-repo workspaces). Empty slice when no watches exist. func (s *Store) ListPRWatchesBySession(ctx context.Context, sessionID string) ([]*PRWatch, error) { @@ -1856,6 +2333,63 @@ func (s *Store) ListActivePRWatches(ctx context.Context) ([]*PRWatch, error) { return watches, err } +// PRWatchCardinality is the aggregate, identity-free state exported for +// operational metrics. Counts never contain task, repository, branch, or +// workspace labels. +type PRWatchCardinality struct { + Active int64 + Searching int64 + Duplicates int64 + Orphans int64 +} + +// PRWatchCardinality returns current watch health across the installation. +// The duplicate count is the number of rows beyond one canonical survivor; +// unique indexes normally keep it at zero, making a non-zero value a direct +// corruption/migration signal. +func (s *Store) PRWatchCardinality(ctx context.Context) (PRWatchCardinality, error) { + var out PRWatchCardinality + activeQuery := ` + SELECT COUNT(*), COALESCE(SUM(CASE WHEN w.pr_number = 0 THEN 1 ELSE 0 END), 0) + FROM github_pr_watches w + INNER JOIN tasks t ON t.id = w.task_id + WHERE t.archived_at IS NULL` + if err := s.ro.QueryRowContext(ctx, activeQuery).Scan(&out.Active, &out.Searching); err != nil { + return PRWatchCardinality{}, fmt.Errorf("count active PR watches: %w", err) + } + + duplicateQueries := []string{ + `SELECT COALESCE(SUM(group_count - 1), 0) FROM ( + SELECT COUNT(*) AS group_count FROM github_pr_watches + WHERE pr_number = 0 GROUP BY task_id, repository_id, branch HAVING COUNT(*) > 1 + ) duplicate_groups`, + `SELECT COALESCE(SUM(group_count - 1), 0) FROM ( + SELECT COUNT(*) AS group_count FROM github_pr_watches + WHERE pr_number <> 0 GROUP BY task_id, repository_id, pr_number HAVING COUNT(*) > 1 + ) duplicate_groups`, + } + for _, query := range duplicateQueries { + var duplicates int64 + if err := s.ro.QueryRowContext(ctx, query).Scan(&duplicates); err != nil { + return PRWatchCardinality{}, fmt.Errorf("count duplicate PR watches: %w", err) + } + out.Duplicates += duplicates + } + + orphanQuery := ` + SELECT COUNT(*) FROM github_pr_watches w + LEFT JOIN tasks t ON t.id = w.task_id + LEFT JOIN task_repositories tr ON tr.repository_id = w.repository_id AND tr.task_id = w.task_id + WHERE t.id IS NULL OR (w.repository_id <> '' AND tr.repository_id IS NULL)` + if err := s.ro.QueryRowContext(ctx, orphanQuery).Scan(&out.Orphans); err != nil { + fallback := `SELECT COUNT(*) FROM github_pr_watches w LEFT JOIN tasks t ON t.id = w.task_id WHERE t.id IS NULL` + if fallbackErr := s.ro.QueryRowContext(ctx, fallback).Scan(&out.Orphans); fallbackErr != nil { + return PRWatchCardinality{}, fmt.Errorf("count orphaned PR watches: %w", err) + } + } + return out, nil +} + // ListActivePRWatchesForWorkspace returns active watches only for one workspace. func (s *Store) ListActivePRWatchesForWorkspace(ctx context.Context, workspaceID string) ([]*PRWatch, error) { var watches []*PRWatch @@ -1896,12 +2430,67 @@ func (s *Store) DeletePRWatchesByTaskID(ctx context.Context, taskID string) (int return n, nil } -// UpdatePRWatchPRNumber updates a PR watch's PR number after discovery. +// UpdatePRWatchPRNumber updates a PR watch's PR number after discovery (or +// resets it to 0 to re-search). Promoting to a non-zero PR number could +// collide with the canonical discovered-watch identity (task_id, +// repository_id, pr_number) if a sibling watch already tracks that PR - e.g. +// two branches (a rename) independently resolving to the same PR. In that +// case the source watch is redundant (the sibling already owns the PR's +// state) and is dropped rather than left to trip the unique index. func (s *Store) UpdatePRWatchPRNumber(ctx context.Context, id string, prNumber int) error { - _, err := s.db.ExecContext(ctx, s.db.Rebind( - `UPDATE github_pr_watches SET pr_number = ?, updated_at = ? WHERE id = ?`), - prNumber, time.Now().UTC(), id) - return err + if prNumber == 0 { + _, err := s.db.ExecContext(ctx, + s.db.Rebind(`UPDATE github_pr_watches SET pr_number = 0, updated_at = ? WHERE id = ?`), + time.Now().UTC(), id) + return err + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + var taskID, repositoryID string + err = tx.QueryRowContext(ctx, + s.db.Rebind(`SELECT task_id, repository_id FROM github_pr_watches WHERE id = ?`), id). + Scan(&taskID, &repositoryID) + if errors.Is(err, sql.ErrNoRows) { + return tx.Commit() + } + if err != nil { + return err + } + + var probe int // existence probe only; value unused + err = tx.QueryRowContext(ctx, + s.db.Rebind(`SELECT 1 FROM github_pr_watches + WHERE task_id = ? AND repository_id = ? AND pr_number = ? AND id <> ?`, + ), + taskID, repositoryID, prNumber, id).Scan(&probe) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if err == nil { + if _, err := tx.ExecContext(ctx, s.db.Rebind(`DELETE FROM github_pr_watches WHERE id = ?`), id); err != nil { + return err + } + return tx.Commit() + } + + const savepoint = "update_pr_watch_pr_number" + if _, err := tx.ExecContext(ctx, s.db.Rebind("SAVEPOINT "+savepoint)); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, + s.db.Rebind(`UPDATE github_pr_watches SET pr_number = ?, updated_at = ? WHERE id = ?`), + prNumber, time.Now().UTC(), id); err != nil { + return recoverPRWatchDiscoveredUpdate(ctx, tx, s.db.Rebind, savepoint, id, err, tx.Commit) + } + if _, err := tx.ExecContext(ctx, s.db.Rebind("RELEASE SAVEPOINT "+savepoint)); err != nil { + return err + } + return tx.Commit() } // UpdatePRWatchRepository repairs the provider repository identity after PR @@ -1920,21 +2509,164 @@ func (s *Store) UpdatePRWatchRepository(ctx context.Context, id, owner, repo str // for a PR on the new branch without leaving an inconsistent intermediate // state. func (s *Store) ResetPRWatch(ctx context.Context, id, branch string) error { - _, err := s.db.ExecContext(ctx, s.db.Rebind( - `UPDATE github_pr_watches SET branch = ?, pr_number = 0, updated_at = ? WHERE id = ?`), - branch, time.Now().UTC(), id) - return err + // SQLite's deferred transactions let two sessions both probe an empty + // destination before either writer acquires the lock. BEGIN IMMEDIATE + // serializes the probe and the coalescing update as one operation. + var tx prWatchTx + var commit func() error + var rollback func() + if dialect.IsPostgres(s.db.DriverName()) { + pgTx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + tx = pgTx + commit = pgTx.Commit + rollback = func() { _ = pgTx.Rollback() } + } else { + conn, err := s.db.Conn(ctx) + if err != nil { + return err + } + if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + _ = conn.Close() + return err + } + tx = conn + commit = func() error { + _, err := conn.ExecContext(ctx, "COMMIT") + return err + } + rollback = func() { + _, _ = conn.ExecContext(context.WithoutCancel(ctx), "ROLLBACK") + _ = conn.Close() + } + } + defer rollback() + + var taskID, repositoryID string + err := tx.QueryRowContext(ctx, s.db.Rebind( + `SELECT task_id, repository_id FROM github_pr_watches WHERE id = ?`), id). + Scan(&taskID, &repositoryID) + if errors.Is(err, sql.ErrNoRows) { + return commit() + } + if err != nil { + return err + } + + var probe int + err = tx.QueryRowContext(ctx, s.db.Rebind( + `SELECT 1 FROM github_pr_watches + WHERE task_id = ? AND repository_id = ? AND branch = ? AND id <> ?`, + ), taskID, repositoryID, branch, id).Scan(&probe) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return err + } + if err == nil { + if _, err := tx.ExecContext(ctx, s.db.Rebind(`DELETE FROM github_pr_watches WHERE id = ?`), id); err != nil { + return err + } + return commit() + } + + const savepoint = "reset_pr_watch_update" + if _, err := tx.ExecContext(ctx, s.db.Rebind("SAVEPOINT "+savepoint)); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, s.db.Rebind( + `UPDATE github_pr_watches SET branch = ?, pr_number = 0, updated_at = ? WHERE id = ?`, + ), branch, time.Now().UTC(), id); err != nil { + return recoverPRWatchUniqueUpdate(ctx, tx, s.db.Rebind, savepoint, id, err, commit) + } + if _, err := tx.ExecContext(ctx, s.db.Rebind("RELEASE SAVEPOINT "+savepoint)); err != nil { + return err + } + return commit() +} + +const prWatchSearchingIndexName = "idx_github_pr_watches_searching" +const prWatchDiscoveredIndexName = "idx_github_pr_watches_discovered" + +func isPRWatchUniqueViolation(err error) bool { + if err == nil { + return false + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code == "23505" && pgErr.ConstraintName == prWatchSearchingIndexName + } + return strings.Contains(err.Error(), "UNIQUE constraint failed") +} + +func isPRWatchDiscoveredUniqueViolation(err error) bool { + if err == nil { + return false + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code == "23505" && pgErr.ConstraintName == prWatchDiscoveredIndexName + } + return strings.Contains(err.Error(), "UNIQUE constraint failed") +} + +func recoverPRWatchUniqueUpdate( + ctx context.Context, + tx prWatchTx, + rebind func(string) string, + savepoint, id string, + updateErr error, + commit func() error, +) error { + if _, err := tx.ExecContext(ctx, rebind("ROLLBACK TO SAVEPOINT "+savepoint)); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, rebind("RELEASE SAVEPOINT "+savepoint)); err != nil { + return err + } + if !isPRWatchUniqueViolation(updateErr) { + return updateErr + } + if _, err := tx.ExecContext(ctx, rebind(`DELETE FROM github_pr_watches WHERE id = ?`), id); err != nil { + return err + } + return commit() +} + +func recoverPRWatchDiscoveredUpdate( + ctx context.Context, + tx prWatchTx, + rebind func(string) string, + savepoint, id string, + updateErr error, + commit func() error, +) error { + if _, err := tx.ExecContext(ctx, rebind("ROLLBACK TO SAVEPOINT "+savepoint)); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, rebind("RELEASE SAVEPOINT "+savepoint)); err != nil { + return err + } + // An external writer can still race between the preflight probe and the + // UPDATE. Treat that discovered-index collision like the probe-found path. + if !isPRWatchDiscoveredUniqueViolation(updateErr) { + return updateErr + } + if _, err := tx.ExecContext(ctx, rebind(`DELETE FROM github_pr_watches WHERE id = ?`), id); err != nil { + return err + } + return commit() } // UpdatePRWatchBranchIfSearching atomically updates branch only when pr_number = 0, // preventing races with concurrent PR association. // // Collision semantics: a sibling watch may already own the destination -// (session_id, repository_id, branch) triple — e.g. multi-branch task where -// the agent's live branch collapsed onto a peer watch's branch. In that -// case the raw UPDATE would trip the UNIQUE constraint. We instead drop the -// source row (which is still searching, pr_number=0, so it owns no PR -// state) and let the sibling continue to track the branch. +// (task_id, repository_id, branch) triple — e.g. multi-branch task where the +// agent's live branch collapsed onto a peer watch's branch. In that case the +// raw UPDATE would trip the UNIQUE constraint. We instead drop the source row +// (which is still searching, pr_number=0, so it owns no PR state) and let the +// sibling continue to track the branch. func (s *Store) UpdatePRWatchBranchIfSearching(ctx context.Context, id, branch string) error { tx, err := s.db.BeginTxx(ctx, nil) if err != nil { @@ -1942,11 +2674,11 @@ func (s *Store) UpdatePRWatchBranchIfSearching(ctx context.Context, id, branch s } defer func() { _ = tx.Rollback() }() - var sessionID, repositoryID string + var taskID, repositoryID string var prNumber int - err = tx.QueryRowxContext(ctx, tx.Rebind( - `SELECT session_id, repository_id, pr_number FROM github_pr_watches WHERE id = ?`), id). - Scan(&sessionID, &repositoryID, &prNumber) + err = tx.QueryRowContext(ctx, + s.db.Rebind(`SELECT task_id, repository_id, pr_number FROM github_pr_watches WHERE id = ?`), id). + Scan(&taskID, &repositoryID, &prNumber) if errors.Is(err, sql.ErrNoRows) { return tx.Commit() } @@ -1958,10 +2690,10 @@ func (s *Store) UpdatePRWatchBranchIfSearching(ctx context.Context, id, branch s } var probe int // existence probe only; value unused - err = tx.QueryRowxContext(ctx, tx.Rebind( - `SELECT 1 FROM github_pr_watches - WHERE session_id = ? AND repository_id = ? AND branch = ? AND id <> ?`, - ), sessionID, repositoryID, branch, id).Scan(&probe) + err = tx.QueryRowContext(ctx, + s.db.Rebind(`SELECT 1 FROM github_pr_watches + WHERE task_id = ? AND repository_id = ? AND branch = ? AND id <> ?`), + taskID, repositoryID, branch, id).Scan(&probe) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } @@ -1969,23 +2701,26 @@ func (s *Store) UpdatePRWatchBranchIfSearching(ctx context.Context, id, branch s return dropSourceAndCommit(ctx, tx, id) } - if _, err := tx.ExecContext(ctx, tx.Rebind( - `UPDATE github_pr_watches SET branch = ?, updated_at = ? WHERE id = ? AND pr_number = 0`), - branch, time.Now().UTC(), id); err != nil { - // Defensive belt-and-suspenders: the SQLite writer pool is - // SetMaxOpenConns(1), so an in-process CreatePRWatch cannot - // commit a sibling row between our probe and this UPDATE. But - // an external writer (separate process touching the same file, - // future pool reshuffle) could; if the UPDATE still trips - // UNIQUE, treat it identically to the probe-found path. - if strings.Contains(err.Error(), "UNIQUE constraint failed") || strings.Contains(err.Error(), "duplicate key") { - return dropSourceAndCommit(ctx, tx, id) - } + const savepoint = "update_pr_watch_branch" + if _, err := tx.ExecContext(ctx, s.db.Rebind("SAVEPOINT "+savepoint)); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, s.db.Rebind( + `UPDATE github_pr_watches SET branch = ?, updated_at = ? WHERE id = ? AND pr_number = 0`, + ), branch, time.Now().UTC(), id); err != nil { + return recoverPRWatchUniqueUpdate(ctx, tx, s.db.Rebind, savepoint, id, err, tx.Commit) + } + if _, err := tx.ExecContext(ctx, s.db.Rebind("RELEASE SAVEPOINT "+savepoint)); err != nil { return err } return tx.Commit() } +type prWatchTx interface { + ExecContext(context.Context, string, ...any) (sql.Result, error) + QueryRowContext(context.Context, string, ...any) *sql.Row +} + // dropSourceAndCommit removes a still-searching source watch (pr_number=0) // whose destination branch is already owned by a sibling row, then commits. func dropSourceAndCommit(ctx context.Context, tx *sqlx.Tx, id string) error { diff --git a/apps/backend/internal/github/store_pr_watch_migration_test.go b/apps/backend/internal/github/store_pr_watch_migration_test.go new file mode 100644 index 0000000000..ac6e76bef5 --- /dev/null +++ b/apps/backend/internal/github/store_pr_watch_migration_test.go @@ -0,0 +1,527 @@ +package github + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// seedLegacyPRWatch inserts a row directly into the legacy +// UNIQUE(session_id, repository_id, branch) schema produced by +// openLegacyGitHubDB, bypassing the (already task-owned) Store API so tests +// can reproduce the pre-migration on-disk shape exactly. +func seedLegacyPRWatch(t *testing.T, db *sqlx.DB, w *PRWatch) { + t.Helper() + if w.ID == "" { + t.Fatalf("seedLegacyPRWatch: id required") + } + _, err := db.Exec(` + INSERT INTO github_pr_watches ( + id, session_id, task_id, repository_id, owner, repo, pr_number, branch, + last_checked_at, last_comment_at, last_check_status, last_review_state, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + w.ID, w.SessionID, w.TaskID, w.RepositoryID, w.Owner, w.Repo, w.PRNumber, w.Branch, + w.LastCheckedAt, w.LastCommentAt, w.LastCheckStatus, w.LastReviewState, + w.CreatedAt, w.UpdatedAt) + if err != nil { + t.Fatalf("seed legacy PR watch %s: %v", w.ID, err) + } +} + +// TestPRWatchMigration_DedupsFiftyResumedSessionWatches is acceptance +// criterion 1: fifty historical sessions resuming the same task/repository/ +// branch must collapse to exactly one canonical searching watch, and the +// migration's aggregate counters must reflect that collapse. +func TestPRWatchMigration_DedupsFiftyResumedSessionWatches(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'); + `); err != nil { + t.Fatalf("seed workspace/task: %v", err) + } + + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + const sessions = 50 + for i := 0; i < sessions; i++ { + ts := base.Add(time.Duration(i) * time.Minute) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: fmt.Sprintf("watch-%03d", i), + SessionID: fmt.Sprintf("session-%03d", i), + TaskID: "task-1", + Owner: "acme", + Repo: "repo", + PRNumber: 0, + Branch: "feature/x", + CreatedAt: ts, + UpdatedAt: ts, + }) + } + + store, err := NewStore(db, db) + if err != nil { + t.Fatalf("migrate legacy store: %v", err) + } + + stats := store.PRWatchMigrationStats() + if stats == nil { + t.Fatalf("expected migration stats to be populated") + } + if stats.RowsBefore != sessions { + t.Fatalf("RowsBefore = %d, want %d", stats.RowsBefore, sessions) + } + if stats.RowsAfter != 1 { + t.Fatalf("RowsAfter = %d, want 1", stats.RowsAfter) + } + if stats.DuplicatesRemoved != sessions-1 { + t.Fatalf("DuplicatesRemoved = %d, want %d", stats.DuplicatesRemoved, sessions-1) + } + + survivor, err := store.GetPRWatchByTaskRepoBranch(ctx, "task-1", "", "feature/x") + if err != nil { + t.Fatalf("get canonical watch: %v", err) + } + if survivor == nil { + t.Fatalf("expected exactly one canonical searching watch to survive") + } + + all, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches by task: %v", err) + } + if len(all) != 1 { + t.Fatalf("watches remaining for task-1 = %d, want 1", len(all)) + } + + // A second boot against the already-migrated schema must be a pure + // no-op: no further rows removed, and the migration is skipped because + // the legacy UNIQUE constraint no longer exists. + store2, err := NewStore(db, db) + if err != nil { + t.Fatalf("second migration pass: %v", err) + } + if stats2 := store2.PRWatchMigrationStats(); stats2 != nil { + t.Fatalf("expected no-op second migration, got stats %+v", stats2) + } + all2, err := store2.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches by task after second pass: %v", err) + } + if len(all2) != 1 { + t.Fatalf("watches remaining after second pass = %d, want 1", len(all2)) + } +} + +func TestPRWatchMigration_SkipsSQLiteTableRebuildForPostgres(t *testing.T) { + db := openLegacyGitHubDB(t) + store := &Store{db: sqlx.NewDb(db.DB, "pgx"), ro: sqlx.NewDb(db.DB, "pgx")} + + require.NoError(t, store.migratePRWatchesToTaskOwnership()) + if store.PRWatchMigrationStats() != nil { + t.Fatal("PostgreSQL store recorded SQLite legacy-migration statistics") + } + + var tableSQL string + require.NoError(t, db.Get(&tableSQL, "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'github_pr_watches'")) + assert.Contains(t, tableSQL, legacyPRWatchesUniqueConstraint) +} + +func TestPRWatchMigration_PostgresDriverDeduplicatesRowsWithoutTableRebuild(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-a", SessionID: "session-a", TaskID: "task-1", + Owner: "acme", Repo: "repo", PRNumber: 0, Branch: "feature/x", + CreatedAt: base, UpdatedAt: base, + }) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-b", SessionID: "session-b", TaskID: "task-1", + Owner: "acme", Repo: "repo", PRNumber: 0, Branch: "feature/x", + CreatedAt: base.Add(time.Minute), UpdatedAt: base.Add(time.Minute), + }) + store := &Store{db: sqlx.NewDb(db.DB, "pgx"), ro: sqlx.NewDb(db.DB, "pgx")} + + require.NoError(t, store.migratePRWatchesToTaskOwnership()) + stats := store.PRWatchMigrationStats() + require.NotNil(t, stats) + assert.Equal(t, 2, stats.RowsBefore) + assert.Equal(t, 1, stats.RowsAfter) + assert.Equal(t, 1, stats.DuplicatesRemoved) + + var count int + require.NoError(t, db.GetContext(ctx, &count, `SELECT COUNT(*) FROM github_pr_watches WHERE task_id = 'task-1'`)) + assert.Equal(t, 1, count) + var tableSQL string + require.NoError(t, db.Get(&tableSQL, "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'github_pr_watches'")) + assert.Contains(t, tableSQL, legacyPRWatchesUniqueConstraint) +} + +// TestPRWatchMigration_PrefersDiscoveredOverSearching is acceptance +// criterion 8: when a duplicate group contains both a still-searching row +// and a row that already discovered its PR, the discovered row must survive +// so in-flight PR state (checks/review watermarks) is not thrown away. +func TestPRWatchMigration_PrefersDiscoveredOverSearching(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'); + `); err != nil { + t.Fatalf("seed workspace/task: %v", err) + } + + older := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := older.Add(time.Hour) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-searching", SessionID: "session-a", TaskID: "task-1", + Owner: "acme", Repo: "repo", PRNumber: 0, Branch: "feature/x", + CreatedAt: older, UpdatedAt: newer, // newer UpdatedAt, but still searching + }) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-discovered", SessionID: "session-b", TaskID: "task-1", + Owner: "acme", Repo: "repo", PRNumber: 42, Branch: "feature/x", + LastCheckStatus: "success", LastReviewState: "approved", + CreatedAt: older, UpdatedAt: older, // older UpdatedAt, but already discovered + }) + + store, err := NewStore(db, db) + if err != nil { + t.Fatalf("migrate legacy store: %v", err) + } + + survivor, err := store.GetPRWatchByTaskRepoPRNumber(ctx, "task-1", "", 42) + if err != nil { + t.Fatalf("get discovered watch: %v", err) + } + if survivor == nil { + t.Fatalf("expected the discovered watch to survive migration") + } + if survivor.ID != "watch-discovered" { + t.Fatalf("survivor ID = %q, want watch-discovered", survivor.ID) + } + if survivor.LastCheckStatus != "success" || survivor.LastReviewState != "approved" { + t.Fatalf("survivor watermarks = %+v, want preserved success/approved", survivor) + } + + stillSearching, err := store.GetPRWatchByTaskRepoBranch(ctx, "task-1", "", "feature/x") + if err != nil { + t.Fatalf("get searching watch: %v", err) + } + if stillSearching == nil || stillSearching.ID != "watch-searching" { + t.Fatalf("searching watch = %+v, want independent searching identity", stillSearching) + } +} + +// TestPRWatchMigration_MergesCrossBranchDiscoveredCollision covers the pass-2 +// case where two different branches (e.g. before/after a rename) each +// independently discovered the SAME pull request. Migration must collapse +// them to one discovered row keyed by (task_id, repository_id, pr_number), +// preserving the newest watermarks. +func TestPRWatchMigration_MergesCrossBranchDiscoveredCollision(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'); + `); err != nil { + t.Fatalf("seed workspace/task: %v", err) + } + + older := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + newer := older.Add(time.Hour) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-old-branch", SessionID: "session-a", TaskID: "task-1", + Owner: "acme", Repo: "repo", PRNumber: 7, Branch: "feature/old-name", + LastCheckStatus: "pending", + CreatedAt: older, UpdatedAt: older, + }) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-new-branch", SessionID: "session-b", TaskID: "task-1", + Owner: "acme", Repo: "repo", PRNumber: 7, Branch: "feature/new-name", + LastCheckStatus: "success", + CreatedAt: newer, UpdatedAt: newer, + }) + + store, err := NewStore(db, db) + if err != nil { + t.Fatalf("migrate legacy store: %v", err) + } + + all, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(all) != 1 { + t.Fatalf("watches remaining = %d, want 1 (cross-branch PR collision must merge)", len(all)) + } + if all[0].LastCheckStatus != "success" { + t.Fatalf("survivor LastCheckStatus = %q, want newest (success)", all[0].LastCheckStatus) + } +} + +func TestPRWatchMigration_PreservesDistinctDiscoveredWatchesOnSameBranch(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + if _, err := db.ExecContext(ctx, "INSERT INTO workspaces (id) VALUES ('ws-1'); INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1')"); err != nil { + t.Fatalf("seed workspace/task: %v", err) + } + now := time.Now().UTC() + for _, watch := range []*PRWatch{ + {ID: "watch-pr-1", SessionID: "session-a", TaskID: "task-1", Owner: "acme", Repo: "repo", PRNumber: 1, Branch: "feature/x", CreatedAt: now, UpdatedAt: now}, + {ID: "watch-pr-2", SessionID: "session-b", TaskID: "task-1", Owner: "acme", Repo: "repo", PRNumber: 2, Branch: "feature/x", CreatedAt: now, UpdatedAt: now}, + } { + seedLegacyPRWatch(t, db, watch) + } + store, err := NewStore(db, db) + if err != nil { + t.Fatalf("migrate legacy store: %v", err) + } + watches, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(watches) != 2 { + t.Fatalf("watches remaining = %d, want 2 distinct discovered PR identities", len(watches)) + } +} + +// TestPRWatchMigration_RemovesDetachedRepositoryOrphans covers the +// repository-detachment half of the orphan sweep: a watch whose repository +// is no longer attached to its task must be removed even though the task +// itself still exists. +func TestPRWatchMigration_RemovesDetachedRepositoryOrphans(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + CREATE TABLE task_repositories (task_id TEXT NOT NULL, repository_id TEXT NOT NULL, position INTEGER NOT NULL DEFAULT 0); + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'); + INSERT INTO task_repositories (task_id, repository_id, position) VALUES ('task-1', 'repo-attached', 0); + `); err != nil { + t.Fatalf("seed workspace/task/repositories: %v", err) + } + + now := time.Now().UTC() + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-attached", SessionID: "session-a", TaskID: "task-1", + RepositoryID: "repo-attached", Owner: "acme", Repo: "repo", Branch: "main", + CreatedAt: now, UpdatedAt: now, + }) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-detached", SessionID: "session-b", TaskID: "task-1", + RepositoryID: "repo-detached", Owner: "acme", Repo: "repo", Branch: "main", + CreatedAt: now, UpdatedAt: now, + }) + + store, err := NewStore(db, db) + if err != nil { + t.Fatalf("migrate legacy store: %v", err) + } + + stats := store.PRWatchMigrationStats() + if stats == nil || stats.OrphansRemoved != 1 { + t.Fatalf("stats = %+v, want OrphansRemoved = 1", stats) + } + + if got, err := store.GetPRWatch(ctx, "watch-detached"); err != nil || got != nil { + t.Fatalf("expected detached-repository watch to be removed, got %+v (err %v)", got, err) + } + if got, err := store.GetPRWatch(ctx, "watch-attached"); err != nil || got == nil { + t.Fatalf("expected attached-repository watch to survive, got %+v (err %v)", got, err) + } +} + +// TestPRWatchMigration_ClearsProvenanceForMissingSessions covers the +// session_id optionality change: a row whose session no longer exists must +// survive with its session_id cleared, not be deleted or treated as +// ineligible for polling. +func TestPRWatchMigration_ClearsProvenanceForMissingSessions(t *testing.T) { + db := openLegacyGitHubDB(t) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + CREATE TABLE sessions (id TEXT PRIMARY KEY); + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1'); + INSERT INTO sessions (id) VALUES ('session-alive'); + `); err != nil { + t.Fatalf("seed workspace/task/sessions: %v", err) + } + + now := time.Now().UTC() + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-alive-session", SessionID: "session-alive", TaskID: "task-1", + Owner: "acme", Repo: "repo", Branch: "main", CreatedAt: now, UpdatedAt: now, + }) + seedLegacyPRWatch(t, db, &PRWatch{ + ID: "watch-dead-session", SessionID: "session-completed-and-gone", TaskID: "task-1", + Owner: "acme", Repo: "repo", Branch: "other", CreatedAt: now, UpdatedAt: now, + }) + + store, err := NewStore(db, db) + if err != nil { + t.Fatalf("migrate legacy store: %v", err) + } + + got, err := store.GetPRWatch(ctx, "watch-dead-session") + if err != nil { + t.Fatalf("get watch with dead session: %v", err) + } + if got == nil { + t.Fatalf("expected watch with a missing session to survive (session_id is provenance only)") + } + if got.SessionID != "" { + t.Fatalf("SessionID = %q, want cleared to empty", got.SessionID) + } +} + +// TestCreatePRWatch_DedupsAcrossResumedSessions is acceptance criteria 1/5: +// two sessions calling the service layer for the same task/repository/branch +// must reuse the single canonical watch rather than creating a duplicate, +// regardless of session identity. +func TestCreatePRWatch_DedupsAcrossResumedSessions(t *testing.T) { + svc, store, _ := setupSyncTest(t) + ctx := context.Background() + + first, err := svc.CreatePRWatch(ctx, "session-1", "task-1", "repo-1", "acme", "repo", 0, "feature/x") + if err != nil { + t.Fatalf("create PR watch (session-1): %v", err) + } + second, err := svc.CreatePRWatch(ctx, "session-2", "task-1", "repo-1", "acme", "repo", 0, "feature/x") + if err != nil { + t.Fatalf("create PR watch (session-2): %v", err) + } + if first.ID != second.ID { + t.Fatalf("expected same canonical watch across sessions, got %s and %s", first.ID, second.ID) + } + + all, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(all) != 1 { + t.Fatalf("watches for task-1 = %d, want 1", len(all)) + } +} + +// TestEnsurePRWatch_DedupsAcrossResumedSessions mirrors the above for the +// EnsurePRWatch entry point used by push-detection / worktree resume paths. +func TestEnsurePRWatch_DedupsAcrossResumedSessions(t *testing.T) { + svc, store, _ := setupSyncTest(t) + ctx := context.Background() + + first, err := svc.EnsurePRWatch(ctx, "session-1", "task-1", "repo-1", "acme", "repo", "feature/x") + if err != nil { + t.Fatalf("ensure PR watch (session-1): %v", err) + } + second, err := svc.EnsurePRWatch(ctx, "session-2", "task-1", "repo-1", "acme", "repo", "feature/x") + if err != nil { + t.Fatalf("ensure PR watch (session-2): %v", err) + } + if first.ID != second.ID { + t.Fatalf("expected same canonical watch across sessions, got %s and %s", first.ID, second.ID) + } + + all, err := store.ListPRWatchesByTask(ctx, "task-1") + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(all) != 1 { + t.Fatalf("watches for task-1 = %d, want 1", len(all)) + } +} + +// TestListActivePRWatches_ReviewTaskSurvivesCompletedSession is acceptance +// criterion 2: a task in Review whose originating session has completed (or +// disappeared entirely) must remain in ListActivePRWatches as long as the +// task itself is not archived/deleted. Watch eligibility must not depend on +// session state; archiving the task is what finally stops polling. +func TestListActivePRWatches_ReviewTaskSurvivesCompletedSession(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + if _, err := store.db.ExecContext(ctx, + `INSERT INTO tasks (id, workspace_id, archived_at) VALUES ('task-in-review', 'ws-1', NULL)`); err != nil { + t.Fatalf("seed review task: %v", err) + } + + watch := &PRWatch{ + // SessionID intentionally references a session that has already + // completed and been cleaned up elsewhere; it must not gate + // eligibility here. + SessionID: "session-long-gone", TaskID: "task-in-review", + Owner: "acme", Repo: "repo", PRNumber: 99, Branch: "feature/x", + } + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create PR watch: %v", err) + } + + active, err := store.ListActivePRWatches(ctx) + if err != nil { + t.Fatalf("list active watches: %v", err) + } + if len(active) != 1 || active[0].ID != watch.ID { + t.Fatalf("active watches = %+v, want the Review task's watch to remain monitored", active) + } + + // Archiving the task (the actual eligibility boundary) is what removes + // it from polling — not session completion. + if _, err := store.db.ExecContext(ctx, + `UPDATE tasks SET archived_at = CURRENT_TIMESTAMP WHERE id = 'task-in-review'`); err != nil { + t.Fatalf("archive task: %v", err) + } + activeAfterArchive, err := store.ListActivePRWatches(ctx) + if err != nil { + t.Fatalf("list active watches after archive: %v", err) + } + if len(activeAfterArchive) != 0 { + t.Fatalf("active watches after archive = %+v, want none", activeAfterArchive) + } +} + +// TestPRWatchCanonicalIndexes_EnforceUniquenessAtDatabaseLevel is defense in +// depth for acceptance criterion 5: even if a caller bypassed the +// service-layer dedup in getExistingCanonicalPRWatch (e.g. a future +// regression, or a second backend process racing this one), the partial +// unique indexes installed by applyIdempotentSchemaIndexes must themselves +// reject a second searching row and a second discovered row for the same +// canonical key. +func TestPRWatchCanonicalIndexes_EnforceUniquenessAtDatabaseLevel(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + if err := store.CreatePRWatch(ctx, &PRWatch{ + ID: "watch-searching-1", TaskID: "task-1", RepositoryID: "repo-1", + Owner: "acme", Repo: "repo", PRNumber: 0, Branch: "feature/x", + }); err != nil { + t.Fatalf("create first searching watch: %v", err) + } + err := store.CreatePRWatch(ctx, &PRWatch{ + ID: "watch-searching-2", TaskID: "task-1", RepositoryID: "repo-1", + Owner: "acme", Repo: "repo", PRNumber: 0, Branch: "feature/x", + }) + if err == nil { + t.Fatalf("expected UNIQUE violation inserting a second searching watch for the same (task, repository, branch)") + } + + if err := store.CreatePRWatch(ctx, &PRWatch{ + ID: "watch-discovered-1", TaskID: "task-1", RepositoryID: "repo-1", + Owner: "acme", Repo: "repo", PRNumber: 42, Branch: "feature/y", + }); err != nil { + t.Fatalf("create first discovered watch: %v", err) + } + err = store.CreatePRWatch(ctx, &PRWatch{ + ID: "watch-discovered-2", TaskID: "task-1", RepositoryID: "repo-1", + Owner: "acme", Repo: "repo", PRNumber: 42, Branch: "feature/z", + }) + if err == nil { + t.Fatalf("expected UNIQUE violation inserting a second discovered watch for the same (task, repository, pr_number)") + } +} diff --git a/apps/backend/internal/github/store_watch_reset_test.go b/apps/backend/internal/github/store_watch_reset_test.go index 4af1b75ca4..5a3fce37f5 100644 --- a/apps/backend/internal/github/store_watch_reset_test.go +++ b/apps/backend/internal/github/store_watch_reset_test.go @@ -2,10 +2,268 @@ package github import ( "context" + "errors" "testing" "time" + + "github.com/jackc/pgx/v5/pgconn" + + "github.com/kandev/kandev/internal/testutil" ) +func TestIsPRWatchUniqueViolation_PostgresConstraint(t *testing.T) { + err := &pgconn.PgError{Code: "23505", ConstraintName: prWatchSearchingIndexName} + if !isPRWatchUniqueViolation(errors.Join(errors.New("wrapped"), err)) { + t.Fatal("expected PostgreSQL PR-watch unique violation to be recognized") + } +} + +func TestIsPRWatchDiscoveredUniqueViolation_PostgresConstraint(t *testing.T) { + err := &pgconn.PgError{Code: "23505", ConstraintName: prWatchDiscoveredIndexName} + if !isPRWatchDiscoveredUniqueViolation(errors.Join(errors.New("wrapped"), err)) { + t.Fatal("expected PostgreSQL PR-watch discovered unique violation to be recognized") + } +} + +func TestStore_UpdatePRWatchBranchIfSearching_PostgresDifferentSessionCollision(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT, archived_at TIMESTAMP); + CREATE TABLE workspaces (id TEXT PRIMARY KEY); + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1');`); err != nil { + t.Fatalf("create task fixtures: %v", err) + } + if _, err := db.ExecContext(ctx, ` + CREATE TABLE github_pr_watches ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + session_id TEXT NOT NULL DEFAULT '', + task_id TEXT NOT NULL, + repository_id TEXT NOT NULL DEFAULT '', + owner TEXT NOT NULL, + repo TEXT NOT NULL, + pr_number INTEGER NOT NULL, + branch TEXT NOT NULL, + last_checked_at TIMESTAMP, + last_comment_at TIMESTAMP, + last_check_status TEXT DEFAULT '', + last_review_state TEXT DEFAULT '', + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ); + CREATE UNIQUE INDEX idx_github_pr_watches_searching + ON github_pr_watches (task_id, repository_id, branch) WHERE pr_number = 0;`); err != nil { + t.Fatalf("create postgres PR-watch fixture: %v", err) + } + store := &Store{db: db, ro: db} + now := time.Now().UTC() + source := &PRWatch{ + ID: "watch-source-pg", WorkspaceID: "ws-1", SessionID: "session-1", TaskID: "task-1", + RepositoryID: "repo-1", Owner: "owner", Repo: "repo", Branch: "feature/A", + CreatedAt: now, UpdatedAt: now, + } + sibling := &PRWatch{ + ID: "watch-sibling-pg", WorkspaceID: "ws-1", SessionID: "session-2", TaskID: "task-1", + RepositoryID: "repo-1", Owner: "owner", Repo: "repo", Branch: "feature/B", + CreatedAt: now, UpdatedAt: now, + } + for _, watch := range []*PRWatch{source, sibling} { + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create watch %s: %v", watch.ID, err) + } + } + + if err := store.UpdatePRWatchBranchIfSearching(ctx, source.ID, sibling.Branch); err != nil { + t.Fatalf("different-session task-owned collision must coalesce: %v", err) + } + watches, err := store.ListPRWatchesByTask(ctx, source.TaskID) + if err != nil { + t.Fatalf("list watches: %v", err) + } + if len(watches) != 1 || watches[0].ID != sibling.ID { + t.Fatalf("expected sibling to survive alone, got %+v", watches) + } +} + +func TestStore_UpdatePRWatchPRNumber_PostgresDifferentSessionCollision(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT, archived_at TIMESTAMP); + CREATE TABLE workspaces (id TEXT PRIMARY KEY); + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1');`); err != nil { + t.Fatalf("create task fixtures: %v", err) + } + if _, err := db.ExecContext(ctx, ` + CREATE TABLE github_pr_watches ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + session_id TEXT NOT NULL DEFAULT '', + task_id TEXT NOT NULL, + repository_id TEXT NOT NULL DEFAULT '', + owner TEXT NOT NULL, + repo TEXT NOT NULL, + pr_number INTEGER NOT NULL, + branch TEXT NOT NULL, + last_checked_at TIMESTAMP, + last_comment_at TIMESTAMP, + last_check_status TEXT DEFAULT '', + last_review_state TEXT DEFAULT '', + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ); + CREATE UNIQUE INDEX idx_github_pr_watches_discovered + ON github_pr_watches (task_id, repository_id, pr_number) WHERE pr_number <> 0;`); err != nil { + t.Fatalf("create postgres PR-watch fixture: %v", err) + } + store := &Store{db: db, ro: db} + now := time.Now().UTC() + watches := []*PRWatch{ + {ID: "watch-source-pg", WorkspaceID: "ws-1", SessionID: "session-1", TaskID: "task-1", RepositoryID: "repo-1", Owner: "owner", Repo: "repo", Branch: "feature/A", CreatedAt: now, UpdatedAt: now}, + {ID: "watch-sibling-pg", WorkspaceID: "ws-1", SessionID: "session-2", TaskID: "task-1", RepositoryID: "repo-1", Owner: "owner", Repo: "repo", Branch: "feature/B", CreatedAt: now, UpdatedAt: now}, + } + for _, watch := range watches { + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create watch %s: %v", watch.ID, err) + } + } + + if err := store.UpdatePRWatchPRNumber(ctx, watches[1].ID, 99); err != nil { + t.Fatalf("promote sibling watch: %v", err) + } + if err := store.UpdatePRWatchPRNumber(ctx, watches[0].ID, 99); err != nil { + t.Fatalf("different-session discovered-watch collision must coalesce: %v", err) + } + + got, err := store.GetPRWatchByTaskRepoPRNumber(ctx, "task-1", "repo-1", 99) + if err != nil { + t.Fatalf("get discovered watch: %v", err) + } + if got == nil || got.ID != watches[1].ID { + t.Fatalf("expected sibling to survive as discovered watch, got %+v", got) + } + searching, err := store.GetPRWatchByTaskRepoBranch(ctx, "task-1", "repo-1", "feature/A") + if err != nil { + t.Fatalf("get source searching watch: %v", err) + } + if searching != nil { + t.Fatalf("expected source watch to be dropped, got %+v", searching) + } +} + +func TestStore_UpdatePRWatchPRNumber_PostgresUpdateCollisionKeepsTransactionUsable(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + ctx := context.Background() + if _, err := db.ExecContext(ctx, ` + CREATE TABLE tasks (id TEXT PRIMARY KEY, workspace_id TEXT, archived_at TIMESTAMP); + CREATE TABLE workspaces (id TEXT PRIMARY KEY); + INSERT INTO workspaces (id) VALUES ('ws-1'); + INSERT INTO tasks (id, workspace_id) VALUES ('task-1', 'ws-1');`); err != nil { + t.Fatalf("create task fixtures: %v", err) + } + if _, err := db.ExecContext(ctx, ` + CREATE TABLE github_pr_watches ( + id TEXT PRIMARY KEY, + workspace_id TEXT NOT NULL DEFAULT '', + session_id TEXT NOT NULL DEFAULT '', + task_id TEXT NOT NULL, + repository_id TEXT NOT NULL DEFAULT '', + owner TEXT NOT NULL, + repo TEXT NOT NULL, + pr_number INTEGER NOT NULL, + branch TEXT NOT NULL, + last_checked_at TIMESTAMP, + last_comment_at TIMESTAMP, + last_check_status TEXT DEFAULT '', + last_review_state TEXT DEFAULT '', + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL + ); + CREATE UNIQUE INDEX idx_github_pr_watches_discovered + ON github_pr_watches (task_id, repository_id, pr_number) WHERE pr_number <> 0; + CREATE OR REPLACE FUNCTION insert_colliding_pr_watch() + RETURNS trigger AS $$ + BEGIN + IF NEW.id = 'watch-source-pg' AND NEW.pr_number = 99 THEN + INSERT INTO github_pr_watches ( + id, workspace_id, session_id, task_id, repository_id, owner, repo, + pr_number, branch, created_at, updated_at + ) VALUES ( + 'watch-sibling-pg', NEW.workspace_id, 'session-2', NEW.task_id, + NEW.repository_id, NEW.owner, NEW.repo, NEW.pr_number, + 'feature/B', NEW.updated_at, NEW.updated_at + ); + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER insert_colliding_pr_watch_before_update + BEFORE UPDATE OF pr_number ON github_pr_watches + FOR EACH ROW EXECUTE FUNCTION insert_colliding_pr_watch();`); err != nil { + t.Fatalf("create postgres PR-watch fixture: %v", err) + } + store := &Store{db: db, ro: db} + now := time.Now().UTC() + source := &PRWatch{ + ID: "watch-source-pg", WorkspaceID: "ws-1", SessionID: "session-1", TaskID: "task-1", + RepositoryID: "repo-1", Owner: "owner", Repo: "repo", Branch: "feature/A", + CreatedAt: now, UpdatedAt: now, + } + if err := store.CreatePRWatch(ctx, source); err != nil { + t.Fatalf("create source watch: %v", err) + } + + if err := store.UpdatePRWatchPRNumber(ctx, source.ID, 99); err != nil { + t.Fatalf("update-time discovered-watch collision must keep transaction usable: %v", err) + } + + got, err := store.GetPRWatch(ctx, source.ID) + if err != nil { + t.Fatalf("get source watch: %v", err) + } + if got != nil { + t.Fatalf("expected source watch to be dropped after update-time collision, got %+v", got) + } +} + +func TestStore_ResetPRWatch_DropsDiscoveredSourceOnSearchingCollision(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + now := time.Now().UTC() + source := &PRWatch{ + ID: "watch-source", WorkspaceID: "ws-1", SessionID: "session-1", TaskID: "task-1", + RepositoryID: "repo-1", Owner: "owner", Repo: "repo", PRNumber: 7, Branch: "feature/A", + CreatedAt: now, UpdatedAt: now, + } + sibling := &PRWatch{ + ID: "watch-sibling", WorkspaceID: "ws-1", SessionID: "session-2", TaskID: "task-1", + RepositoryID: "repo-1", Owner: "owner", Repo: "repo", PRNumber: 0, Branch: "feature/B", + CreatedAt: now, UpdatedAt: now, + } + for _, watch := range []*PRWatch{source, sibling} { + if err := store.CreatePRWatch(ctx, watch); err != nil { + t.Fatalf("create watch %s: %v", watch.ID, err) + } + } + + if err := store.ResetPRWatch(ctx, source.ID, sibling.Branch); err != nil { + t.Fatalf("ResetPRWatch: %v", err) + } + if got, err := store.GetPRWatch(ctx, source.ID); err != nil { + t.Fatalf("get source watch: %v", err) + } else if got != nil { + t.Fatalf("source watch still exists after reset collision: %+v", got) + } + if got, err := store.GetPRWatch(ctx, sibling.ID); err != nil { + t.Fatalf("get sibling watch: %v", err) + } else if got == nil { + t.Fatal("sibling searching watch was removed; want it to survive") + } +} + // TestStore_ReviewWatch_ListTaskIDsAndReset pins the contract used by the // review watch reset flow: every dedup row's task_id (including empty // reservations) is enumerable, and ResetReviewWatchState wipes those rows diff --git a/apps/backend/internal/health/checks.go b/apps/backend/internal/health/checks.go index c4ef33cfdd..f4e4672dcf 100644 --- a/apps/backend/internal/health/checks.go +++ b/apps/backend/internal/health/checks.go @@ -181,6 +181,61 @@ func rateLimitMessage(status GitHubRateLimitStatus) string { return fmt.Sprintf("PR/issue checks are paused; resets in %s.", wait) } +// WorkflowSyncChecker checks whether any configured workflow-sync target has +// an open auth/config circuit (internal/common/authcircuit) — a credential +// or repository/branch/path that will keep failing until a human fixes it, +// as opposed to a transient blip that resolves on its own. +type WorkflowSyncChecker struct { + provider WorkflowSyncStatusProvider +} + +// NewWorkflowSyncChecker creates a checker for workflow-sync circuit health. +// The provider may be nil if the workflow-sync service was not initialized; +// the checker then produces no issues (workflow sync is optional). +func NewWorkflowSyncChecker(provider WorkflowSyncStatusProvider) *WorkflowSyncChecker { + return &WorkflowSyncChecker{provider: provider} +} + +// Name returns the user-facing label for this check. +func (c *WorkflowSyncChecker) Name() string { return "Workflow sync" } + +// Category returns the issue category this checker emits issues under. +func (c *WorkflowSyncChecker) Category() string { return "workflow_sync" } + +func (c *WorkflowSyncChecker) Check(ctx context.Context) []Issue { + if c.provider == nil { + return nil + } + summary, err := c.provider.WorkflowSyncCircuitSummary(ctx) + if err != nil { + return []Issue{{ + ID: "workflow_sync_status_unavailable", + Category: "workflow_sync", + Title: "Workflow sync status unavailable", + Message: "Kandev could not read workflow sync circuit status.", + Severity: SeverityWarning, + FixURL: "/settings/system/status", + FixLabel: "View status", + }} + } + degraded := summary.OpenAuth + summary.OpenConfig + if degraded == 0 { + return nil + } + return []Issue{{ + ID: "workflow_sync_circuit_open", + Category: "workflow_sync", + Title: "Workflow sync needs attention", + Message: fmt.Sprintf( + "%d of %d workflow sync targets are paused pending a fix (%d auth, %d config).", + degraded, summary.Total, summary.OpenAuth, summary.OpenConfig, + ), + Severity: SeverityWarning, + FixURL: "/settings/system/status", + FixLabel: "View status", + }} +} + // GitExecutableChecker checks whether the host git executable is available. type GitExecutableChecker struct { lookPath func(string) (string, error) diff --git a/apps/backend/internal/health/checks_test.go b/apps/backend/internal/health/checks_test.go index 7fb08c5719..430188fb8d 100644 --- a/apps/backend/internal/health/checks_test.go +++ b/apps/backend/internal/health/checks_test.go @@ -191,6 +191,59 @@ func TestGitHubChecker_StatusFailureDoesNotInspectAmbientAuth(t *testing.T) { } } +// --- WorkflowSyncChecker tests --- + +type mockWorkflowSyncProvider struct { + summary WorkflowSyncCircuitSummary + err error +} + +func (m *mockWorkflowSyncProvider) WorkflowSyncCircuitSummary(context.Context) (WorkflowSyncCircuitSummary, error) { + return m.summary, m.err +} + +func TestWorkflowSyncChecker_NilProviderReportsNothing(t *testing.T) { + checker := NewWorkflowSyncChecker(nil) + issues := checker.Check(context.Background()) + if len(issues) != 0 { + t.Fatalf("expected 0 issues for a nil (optional) provider, got %+v", issues) + } +} + +func TestWorkflowSyncChecker_NoOpenCircuitsReportsNothing(t *testing.T) { + checker := NewWorkflowSyncChecker(&mockWorkflowSyncProvider{ + summary: WorkflowSyncCircuitSummary{Total: 3}, + }) + issues := checker.Check(context.Background()) + if len(issues) != 0 { + t.Fatalf("expected 0 issues when nothing is open, got %+v", issues) + } +} + +func TestWorkflowSyncChecker_OpenAuthOrConfigReportsIssue(t *testing.T) { + checker := NewWorkflowSyncChecker(&mockWorkflowSyncProvider{ + summary: WorkflowSyncCircuitSummary{Total: 3, OpenAuth: 1, OpenConfig: 1, OpenTransient: 1}, + }) + issues := checker.Check(context.Background()) + if len(issues) != 1 || issues[0].ID != "workflow_sync_circuit_open" { + t.Fatalf("issues = %+v", issues) + } + if issues[0].Severity != SeverityWarning { + t.Errorf("severity = %q, want %q", issues[0].Severity, SeverityWarning) + } + if !strings.Contains(issues[0].Message, "2 of 3") { + t.Errorf("message should count only auth+config as degraded, got %q", issues[0].Message) + } +} + +func TestWorkflowSyncChecker_StatusFailureReportsIssue(t *testing.T) { + checker := NewWorkflowSyncChecker(&mockWorkflowSyncProvider{err: errors.New("db unavailable")}) + issues := checker.Check(context.Background()) + if len(issues) != 1 || issues[0].ID != "workflow_sync_status_unavailable" { + t.Fatalf("issues = %+v", issues) + } +} + // --- GitExecutableChecker tests --- func TestGitExecutableChecker_Found(t *testing.T) { diff --git a/apps/backend/internal/health/types.go b/apps/backend/internal/health/types.go index 881843ab39..c3286c48c5 100644 --- a/apps/backend/internal/health/types.go +++ b/apps/backend/internal/health/types.go @@ -48,3 +48,23 @@ type Checker interface { Name() string Category() string } + +// WorkflowSyncStatusProvider abstracts the workflow-sync service status +// check. Implemented by *workflowsync.Service; the health package does not +// import workflowsync directly to avoid a dependency cycle — every checker's +// provider interface stays local to this package (see GitHubStatusProvider). +type WorkflowSyncStatusProvider interface { + WorkflowSyncCircuitSummary(ctx context.Context) (WorkflowSyncCircuitSummary, error) +} + +// WorkflowSyncCircuitSummary aggregates the auth/config circuit-breaker +// state (internal/common/authcircuit) across every configured workflow-sync +// target. It intentionally contains no workspace IDs, repository/project +// identifiers, or error text — only bounded counts, so it is safe to expose +// verbatim in a health response. +type WorkflowSyncCircuitSummary struct { + Total int + OpenAuth int + OpenConfig int + OpenTransient int +} diff --git a/apps/backend/internal/launcher/launcher.go b/apps/backend/internal/launcher/launcher.go index 1df0befa1a..740d71a071 100644 --- a/apps/backend/internal/launcher/launcher.go +++ b/apps/backend/internal/launcher/launcher.go @@ -18,6 +18,9 @@ func Run(args []string, build BuildInfo) int { if len(args) > 0 && args[0] == "service" { return runService(args[1:], build) } + if len(args) > 0 && args[0] == "maintenance" { + return runMaintenance(args[1:], build) + } opts, err := parseArgs(args) if err != nil { fmt.Fprintln(os.Stderr, "[kandev] "+err.Error()) diff --git a/apps/backend/internal/launcher/maintenance_database.go b/apps/backend/internal/launcher/maintenance_database.go new file mode 100644 index 0000000000..922e650006 --- /dev/null +++ b/apps/backend/internal/launcher/maintenance_database.go @@ -0,0 +1,234 @@ +package launcher + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/maintenance" +) + +const maintenanceHelp = `kandev maintenance database — offline SQLite retention and compaction + +Usage: + kandev maintenance database [--execute] [--compact] + [--keep-plan-revisions ] [--candidate-limit ] [--home-dir ] + +Without --execute, this is a read-only dry run: it reports how many +duplicate git-snapshot, obsolete plan-revision, and orphaned message-payload +rows are eligible for removal (and their estimated bytes), and never +acquires a lock, takes a backup, or changes anything. + +--execute requires exclusive access to the database (kandev must not be +running against the same home/database) and always takes and verifies a +fresh backup before deleting anything. Backups are written under +/backups/kandev-pre-maintenance-.db. + +--compact additionally stages a VACUUM INTO compacted copy after retention +deletes commit, verifies it with an integrity check, and atomically +replaces the live database, preserving the pre-compaction file alongside it +as .pre-compact-.bak for manual rollback (stop +kandev if running, then move that file back over the live database path). + +Options: + --execute Perform retention deletes (default: dry run). + --compact Also stage and swap in a compacted copy (requires --execute). + --keep-plan-revisions Additionally protect the n most recent non-HEAD plan revisions per task (default 0). + --candidate-limit Cap candidates reported/deleted per category (default: unlimited). + --home-dir Kandev home directory override (default: $KANDEV_HOME_DIR or ~/.kandev). + --help, -h Print this help. +` + +type maintenanceDatabaseArgs struct { + Execute bool + Compact bool + KeepPlanRevisions int + CandidateLimit int + HomeDir string + ShowHelp bool +} + +func runMaintenance(argv []string, _ BuildInfo) int { + if len(argv) == 0 || argv[0] == flagHelp || argv[0] == "-h" { + fmt.Print(maintenanceHelp) + return 0 + } + if argv[0] != "database" { + fmt.Fprintf(os.Stderr, "[kandev] unknown maintenance target %q (only \"database\" is supported)\n", argv[0]) + return 2 + } + + args, err := parseMaintenanceDatabaseArgs(argv[1:]) + if err != nil { + fmt.Fprintln(os.Stderr, "[kandev] "+err.Error()) + return 2 + } + if args.ShowHelp { + fmt.Print(maintenanceHelp) + return 0 + } + + cfg, err := loadBootstrapConfigWithHome(args.HomeDir) + if err != nil { + fmt.Fprintln(os.Stderr, "[kandev] "+err.Error()) + return 1 + } + applyMaintenanceHomeDirOverride(cfg, args.HomeDir) + + homeDir := resolveHomeDirForConfig(cfg) + databasePath := resolveDatabasePathForConfig(cfg) + opts := maintenance.RunOptions{ + Execute: args.Execute, + Compact: args.Compact, + KeepPlanRevisions: args.KeepPlanRevisions, + CandidateLimit: args.CandidateLimit, + } + + outcome, err := maintenance.Run(context.Background(), homeDir, cfg.Database.Driver, databasePath, opts, nil) + if err != nil { + if outcome.Executed { + printMaintenanceOutcome(outcome) + } + printMaintenanceFailure(err) + return 1 + } + printMaintenanceOutcome(outcome) + return 0 +} + +func applyMaintenanceHomeDirOverride(cfg *config.Config, homeDir string) { + if cfg != nil && strings.TrimSpace(homeDir) != "" { + cfg.HomeDir = homeDir + } +} + +func parseMaintenanceDatabaseArgs(argv []string) (maintenanceDatabaseArgs, error) { + var out maintenanceDatabaseArgs + for i := 0; i < len(argv); i++ { + arg := argv[i] + if arg == flagHelp || arg == "-h" { + out.ShowHelp = true + return out, nil + } + if arg == "--execute" { + out.Execute = true + continue + } + if arg == "--compact" { + out.Compact = true + continue + } + next, err := applyMaintenanceValueFlag(&out, argv, i, arg) + if err != nil { + return out, err + } + i = next + } + return out, nil +} + +// applyMaintenanceValueFlag handles the value-taking flags +// (--keep-plan-revisions, --candidate-limit, --home-dir), returning the +// argv index to resume parsing from. Split out of +// parseMaintenanceDatabaseArgs to keep both functions under the +// cyclomatic-complexity limit. +func applyMaintenanceValueFlag(out *maintenanceDatabaseArgs, argv []string, i int, arg string) (int, error) { + switch { + case arg == "--keep-plan-revisions" || strings.HasPrefix(arg, "--keep-plan-revisions="): + value, next, err := maintenanceFlagIntValue(argv, i, arg, "--keep-plan-revisions") + if err != nil { + return i, err + } + out.KeepPlanRevisions = value + return next, nil + case arg == "--candidate-limit" || strings.HasPrefix(arg, "--candidate-limit="): + value, next, err := maintenanceFlagIntValue(argv, i, arg, "--candidate-limit") + if err != nil { + return i, err + } + out.CandidateLimit = value + return next, nil + case arg == "--home-dir" || strings.HasPrefix(arg, "--home-dir="): + value, next, err := maintenanceFlagStringValue(argv, i, arg, "--home-dir") + if err != nil { + return i, err + } + out.HomeDir = value + return next, nil + default: + return i, ParseError{Message: fmt.Sprintf("unknown maintenance database flag %q", arg)} + } +} + +func maintenanceFlagStringValue(argv []string, i int, arg, flagName string) (string, int, error) { + if value, ok := strings.CutPrefix(arg, flagName+"="); ok { + return value, i, nil + } + if i+1 >= len(argv) { + return "", i, ParseError{Message: flagName + " requires a value"} + } + return argv[i+1], i + 1, nil +} + +func maintenanceFlagIntValue(argv []string, i int, arg, flagName string) (int, int, error) { + raw, next, err := maintenanceFlagStringValue(argv, i, arg, flagName) + if err != nil { + return 0, i, err + } + value, err := strconv.Atoi(raw) + if err != nil || value < 0 { + return 0, i, ParseError{Message: flagName + " requires a non-negative integer"} + } + return value, next, nil +} + +func printMaintenanceFailure(err error) { + switch { + case errors.Is(err, maintenance.ErrUnsupportedDriver): + fmt.Fprintln(os.Stderr, "[kandev] "+err.Error()) + case errors.Is(err, maintenance.ErrOwnershipConflict): + fmt.Fprintln(os.Stderr, "[kandev] "+err.Error()) + case errors.Is(err, maintenance.ErrBackupUnverified): + fmt.Fprintln(os.Stderr, "[kandev] refusing to execute: "+err.Error()) + case errors.Is(err, maintenance.ErrInsufficientDiskSpace): + fmt.Fprintln(os.Stderr, "[kandev] refusing to compact: "+err.Error()) + case errors.Is(err, maintenance.ErrIntegrityCheckFailed): + fmt.Fprintln(os.Stderr, "[kandev] refusing to compact: "+err.Error()) + default: + fmt.Fprintln(os.Stderr, "[kandev] maintenance failed: "+err.Error()) + } +} + +func printMaintenanceOutcome(outcome maintenance.Outcome) { + report := outcome.Report + fmt.Printf("kandev maintenance database report (generated %s)\n", report.GeneratedAt.Format("2006-01-02T15:04:05Z")) + fmt.Printf(" database size: %d bytes\n", report.DatabaseSizeBytes) + fmt.Printf(" duplicate git snapshots: %d rows, ~%d bytes\n", report.DuplicateGitSnapshots.RowCount, report.DuplicateGitSnapshots.EstBytes) + fmt.Printf(" obsolete plan revisions: %d rows, ~%d bytes\n", report.ObsoletePlanRevisions.RowCount, report.ObsoletePlanRevisions.EstBytes) + fmt.Printf(" orphaned message payloads: %d rows, ~%d bytes\n", report.OrphanedMessagePayloads.RowCount, report.OrphanedMessagePayloads.EstBytes) + fmt.Printf(" total candidates: %d rows, ~%d bytes\n", report.TotalCandidateRows(), report.TotalEstBytes()) + + if !outcome.Executed { + fmt.Println("mode: dry run (no lock acquired, nothing changed). Re-run with --execute to apply retention.") + return + } + + fmt.Println("mode: execute") + fmt.Printf(" backup: %s\n", outcome.BackupPath) + fmt.Printf(" deleted git snapshots: %d\n", outcome.Execution.DeletedGitSnapshots) + fmt.Printf(" deleted plan revisions: %d\n", outcome.Execution.DeletedPlanRevisions) + fmt.Printf(" deleted message payloads: %d\n", outcome.Execution.DeletedMessagePayloads) + + if outcome.Compaction == nil { + return + } + fmt.Println("compaction: applied") + fmt.Printf(" size before: %d bytes\n", outcome.Compaction.SizeBeforeBytes) + fmt.Printf(" size after: %d bytes\n", outcome.Compaction.SizeAfterBytes) + fmt.Printf(" rollback artifact: %s\n", outcome.Compaction.RollbackPath) + fmt.Println(" to roll back: stop kandev (if running), then move the rollback artifact back over the live database path.") +} diff --git a/apps/backend/internal/launcher/maintenance_database_test.go b/apps/backend/internal/launcher/maintenance_database_test.go new file mode 100644 index 0000000000..b4602b05b1 --- /dev/null +++ b/apps/backend/internal/launcher/maintenance_database_test.go @@ -0,0 +1,270 @@ +package launcher + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/backendapp/ownershiplock" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/maintenance" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// TestParseMaintenanceDatabaseArgsDefaults confirms no flags parses to the +// dry-run default (Execute=false, Compact=false, zero limits). +func TestParseMaintenanceDatabaseArgsDefaults(t *testing.T) { + args, err := parseMaintenanceDatabaseArgs(nil) + if err != nil { + t.Fatalf("parseMaintenanceDatabaseArgs(nil): %v", err) + } + if args.Execute || args.Compact || args.KeepPlanRevisions != 0 || args.CandidateLimit != 0 || args.HomeDir != "" { + t.Fatalf("args = %+v, want all zero/false", args) + } +} + +// TestParseMaintenanceDatabaseArgsFlags covers every recognized flag in +// both "--flag value" and "--flag=value" spellings. +func TestParseMaintenanceDatabaseArgsFlags(t *testing.T) { + args, err := parseMaintenanceDatabaseArgs([]string{ + "--execute", "--compact", "--keep-plan-revisions", "3", "--candidate-limit=50", "--home-dir=/tmp/kandev-home", + }) + if err != nil { + t.Fatalf("parseMaintenanceDatabaseArgs: %v", err) + } + if !args.Execute || !args.Compact { + t.Fatalf("args = %+v, want Execute=true Compact=true", args) + } + if args.KeepPlanRevisions != 3 { + t.Fatalf("KeepPlanRevisions = %d, want 3", args.KeepPlanRevisions) + } + if args.CandidateLimit != 50 { + t.Fatalf("CandidateLimit = %d, want 50", args.CandidateLimit) + } + if args.HomeDir != "/tmp/kandev-home" { + t.Fatalf("HomeDir = %q, want /tmp/kandev-home", args.HomeDir) + } +} + +// TestParseMaintenanceDatabaseArgsRejectsUnknownFlag confirms an +// unrecognized flag is a usage error (exit status 2 at the runMaintenance +// call site), not silently ignored. +func TestParseMaintenanceDatabaseArgsRejectsUnknownFlag(t *testing.T) { + if _, err := parseMaintenanceDatabaseArgs([]string{"--not-a-real-flag"}); err == nil { + t.Fatal("expected an error for an unrecognized flag") + } +} + +// TestParseMaintenanceDatabaseArgsRejectsNegativeLimit confirms +// --candidate-limit/--keep-plan-revisions reject negative values rather +// than silently passing them through to the domain layer. +func TestParseMaintenanceDatabaseArgsRejectsNegativeLimit(t *testing.T) { + if _, err := parseMaintenanceDatabaseArgs([]string{"--candidate-limit", "-1"}); err == nil { + t.Fatal("expected an error for a negative --candidate-limit") + } +} + +// TestParseMaintenanceDatabaseArgsRequiresValue confirms a flag expecting a +// value at the end of argv is a usage error rather than a panic or silent +// zero value. +func TestParseMaintenanceDatabaseArgsRequiresValue(t *testing.T) { + if _, err := parseMaintenanceDatabaseArgs([]string{"--home-dir"}); err == nil { + t.Fatal("expected an error when --home-dir has no value") + } +} + +// TestRunMaintenanceShowsHelpWithoutArgsOrFlag confirms both bare +// `kandev maintenance` and `kandev maintenance database --help` print help +// and exit 0 without touching any database. +func TestRunMaintenanceShowsHelpWithoutArgsOrFlag(t *testing.T) { + output := captureLauncherStdout(t, func() { + if code := runMaintenance(nil, BuildInfo{}); code != 0 { + t.Fatalf("runMaintenance(nil) code = %d, want 0", code) + } + }) + if !strings.Contains(output, "kandev maintenance database") { + t.Fatalf("help output = %q, want it to mention the command", output) + } + + output = captureLauncherStdout(t, func() { + if code := runMaintenance([]string{"database", "--help"}, BuildInfo{}); code != 0 { + t.Fatalf("runMaintenance(database --help) code = %d, want 0", code) + } + }) + if !strings.Contains(output, "--execute") { + t.Fatalf("help output = %q, want it to document --execute", output) + } +} + +// TestRunMaintenanceRejectsUnknownTarget confirms `kandev maintenance foo` +// is a usage error (exit 2), matching runService's convention for an +// unrecognized subcommand action. +func TestRunMaintenanceRejectsUnknownTarget(t *testing.T) { + if code := runMaintenance([]string{"not-a-target"}, BuildInfo{}); code != 2 { + t.Fatalf("code = %d, want 2", code) + } +} + +// TestRunMaintenanceDryRunEndToEndReportsCandidates exercises the full CLI +// path: config loading, driver/path resolution, and Run's dry-run report, +// against a real SQLite database seeded with one duplicate git snapshot. +func TestRunMaintenanceDryRunEndToEndReportsCandidates(t *testing.T) { + clearLauncherConfigurationEnvironment(t) + dir := t.TempDir() + t.Chdir(dir) + dbPath := filepath.Join(dir, "state", "kandev.db") + writeLauncherConfig(t, filepath.Join(dir, "config.yaml"), "database:\n path: "+dbPath+"\n") + + seedMaintenanceFixtureDB(t, dbPath) + + output := captureLauncherStdout(t, func() { + if code := runMaintenance([]string{"database"}, BuildInfo{}); code != 0 { + t.Fatalf("runMaintenance(database) code = %d, want 0", code) + } + }) + if !strings.Contains(output, "duplicate git snapshots: 1 rows") { + t.Fatalf("output = %q, want it to report 1 duplicate git snapshot candidate", output) + } + if !strings.Contains(output, "mode: dry run") { + t.Fatalf("output = %q, want a dry-run mode line", output) + } +} + +func TestMaintenanceHomeDirOverrideSetsDefaultDatabaseLocation(t *testing.T) { + clearLauncherConfigurationEnvironment(t) + homeDir := t.TempDir() + writeLauncherConfig(t, filepath.Join(homeDir, "config.yaml"), "database:\n driver: sqlite\n") + + cfg, err := loadBootstrapConfigWithHome(homeDir) + if err != nil { + t.Fatalf("loadBootstrapConfigWithHome: %v", err) + } + if cfg.HomeDir != "" { + t.Fatalf("config home directory = %q, want config to rely on the CLI override", cfg.HomeDir) + } + applyMaintenanceHomeDirOverride(cfg, homeDir) + if got := resolveHomeDirForConfig(cfg); got != homeDir { + t.Fatalf("resolved home = %q, want %q", got, homeDir) + } + if got, want := resolveDatabasePathForConfig(cfg), filepath.Join(homeDir, "data", "kandev.db"); got != want { + t.Fatalf("default database path = %q, want %q", got, want) + } +} + +func TestPrintMaintenanceOutcomeReportsExecutedPartialProgress(t *testing.T) { + output := captureLauncherStdout(t, func() { + printMaintenanceOutcome(maintenance.Outcome{ + Executed: true, + BackupPath: "/tmp/backup.db", + Execution: maintenance.ExecutionResult{DeletedGitSnapshots: 2}, + }) + }) + if !strings.Contains(output, "backup: /tmp/backup.db") || !strings.Contains(output, "deleted git snapshots: 2") { + t.Fatalf("partial progress output = %q, want backup and deletion details", output) + } +} + +// TestRunMaintenanceExecuteEndToEndDeletesAndBacksUp exercises --execute +// through the full CLI path against a real SQLite database, confirming the +// command reports deletions and a verified backup path. +func TestRunMaintenanceExecuteEndToEndDeletesAndBacksUp(t *testing.T) { + clearLauncherConfigurationEnvironment(t) + dir := t.TempDir() + t.Chdir(dir) + dbPath := filepath.Join(dir, "state", "kandev.db") + writeLauncherConfig(t, filepath.Join(dir, "config.yaml"), "homeDir: "+dir+"\ndatabase:\n path: "+dbPath+"\n") + + seedMaintenanceFixtureDB(t, dbPath) + + output := captureLauncherStdout(t, func() { + if code := runMaintenance([]string{"database", "--execute"}, BuildInfo{}); code != 0 { + t.Fatalf("runMaintenance(database --execute) code = %d, want 0", code) + } + }) + if !strings.Contains(output, "deleted git snapshots: 1") { + t.Fatalf("output = %q, want 1 deleted git snapshot", output) + } + if !strings.Contains(output, "backup:") { + t.Fatalf("output = %q, want a backup path line", output) + } +} + +// TestRunMaintenanceExecuteRefusesWhileOwnershipLockHeld proves the CLI +// path surfaces ownership-conflict refusals (e.g. kandev is running) as +// exit status 1 with a clear stderr message, without touching the +// database. +func TestRunMaintenanceExecuteRefusesWhileOwnershipLockHeld(t *testing.T) { + clearLauncherConfigurationEnvironment(t) + dir := t.TempDir() + t.Chdir(dir) + dbPath := filepath.Join(dir, "state", "kandev.db") + writeLauncherConfig(t, filepath.Join(dir, "config.yaml"), "homeDir: "+dir+"\ndatabase:\n path: "+dbPath+"\n") + seedMaintenanceFixtureDB(t, dbPath) + + targets, err := ownershiplock.Targets(dir, "sqlite", dbPath) + if err != nil { + t.Fatalf("ownershiplock.Targets: %v", err) + } + owner, err := ownershiplock.Acquire(targets) + if err != nil { + t.Fatalf("ownershiplock.Acquire (simulated live backend): %v", err) + } + defer func() { _ = owner.Close() }() + + if code := runMaintenance([]string{"database", "--execute"}, BuildInfo{}); code != 1 { + t.Fatalf("code = %d, want 1 (ownership conflict)", code) + } +} + +// seedMaintenanceFixtureDB creates a fresh SQLite database at dbPath and +// seeds exactly one duplicate git snapshot candidate (older of two +// identical-content rows for the same session). +func seedMaintenanceFixtureDB(t *testing.T, dbPath string) { + t.Helper() + conn, err := db.OpenSQLite(dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqlxDB := sqlx.NewDb(conn, "sqlite3") + repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + ctx := context.Background() + if err := repo.CreateTask(ctx, &models.Task{ID: "task-cli-fixture", WorkspaceID: "workspace-cli-fixture", Title: "t"}); err != nil { + t.Fatalf("CreateTask: %v", err) + } + if err := repo.CreateTaskEnvironment(ctx, &models.TaskEnvironment{ + ID: "environment-cli-fixture", TaskID: "task-cli-fixture", + ExecutorType: string(models.ExecutorTypeLocal), + Status: models.TaskEnvironmentStatusReady, + }); err != nil { + t.Fatalf("CreateTaskEnvironment: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-cli-fixture", TaskID: "task-cli-fixture", + TaskEnvironmentID: "environment-cli-fixture", + }); err != nil { + t.Fatalf("CreateTaskSession: %v", err) + } + snap := func(id string) *models.GitSnapshot { + return &models.GitSnapshot{ + ID: id, SessionID: "session-cli-fixture", SnapshotType: models.SnapshotTypeStatusUpdate, + Branch: "feature/x", HeadCommit: "head-sha", BaseCommit: "base-sha", + Files: map[string]interface{}{"a.go": map[string]interface{}{"status": "modified"}}, + } + } + if err := repo.CreateGitSnapshot(ctx, snap("snap-cli-1")); err != nil { + t.Fatalf("CreateGitSnapshot(1): %v", err) + } + if err := repo.CreateGitSnapshot(ctx, snap("snap-cli-2")); err != nil { + t.Fatalf("CreateGitSnapshot(2): %v", err) + } + if err := sqlxDB.Close(); err != nil { + t.Fatalf("close fixture db: %v", err) + } +} diff --git a/apps/backend/internal/maintenance/backup.go b/apps/backend/internal/maintenance/backup.go new file mode 100644 index 0000000000..4a22752df4 --- /dev/null +++ b/apps/backend/internal/maintenance/backup.go @@ -0,0 +1,70 @@ +package maintenance + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/persistence" +) + +// defaultKeepMaintenanceBackups bounds how many pre-execution maintenance +// backups accumulate in the backups directory over repeated runs, matching +// this plan's "bound SQLite storage growth" goal rather than adding another +// unbounded-growth source of its own. +const defaultKeepMaintenanceBackups = 5 + +// createVerifiedBackup takes a fresh VACUUM INTO snapshot into +// backupDir/kandev-pre-maintenance-.db, verifies it with +// PRAGMA integrity_check, and prunes old maintenance backups beyond +// defaultKeepMaintenanceBackups. Returns the backup path on success. On any +// failure, no destructive retention/compaction step is permitted to run - +// callers must treat a non-nil error here as an unconditional abort. +func createVerifiedBackup(writer *sqlx.DB, backupDir string) (string, error) { + if err := os.MkdirAll(backupDir, 0o755); err != nil { + return "", fmt.Errorf("%w: create backup dir %s: %v", ErrBackupUnverified, backupDir, err) + } + path, err := uniqueBackupPath(backupDir) + if err != nil { + return "", fmt.Errorf("%w: %v", ErrBackupUnverified, err) + } + + if _, err := persistence.SnapshotSQLite(writer, path); err != nil { + return "", fmt.Errorf("%w: %v", ErrBackupUnverified, err) + } + if err := verifySQLiteIntegrity(path); err != nil { + _ = os.Remove(path) + return "", fmt.Errorf("%w: backup failed integrity check: %v", ErrBackupUnverified, err) + } + + // Best-effort: a prune failure never invalidates the backup just + // verified above, so it is logged by the caller rather than returned. + _ = persistence.PruneBackups(backupDir, defaultKeepMaintenanceBackups) + + return path, nil +} + +// uniqueBackupPath picks a backup file name that does not yet exist. +// VACUUM INTO refuses to write over an existing file, and a second-level +// timestamp alone can collide when two runs happen inside the same wall +// clock second (e.g. immediate back-to-back --execute invocations, or a +// test); this appends nanosecond precision and, defensively, a numeric +// suffix if that still collides. +func uniqueBackupPath(backupDir string) (string, error) { + ts := time.Now().UTC().Format("20060102T150405.000000000Z") + base := fmt.Sprintf("kandev-pre-maintenance-%s", ts) + for attempt := 0; attempt < 100; attempt++ { + name := base + ".db" + if attempt > 0 { + name = fmt.Sprintf("%s-%d.db", base, attempt) + } + path := filepath.Join(backupDir, name) + if _, err := os.Stat(path); os.IsNotExist(err) { + return path, nil + } + } + return "", fmt.Errorf("could not find an available backup filename under %s", backupDir) +} diff --git a/apps/backend/internal/maintenance/compact.go b/apps/backend/internal/maintenance/compact.go new file mode 100644 index 0000000000..c5235746b0 --- /dev/null +++ b/apps/backend/internal/maintenance/compact.go @@ -0,0 +1,197 @@ +package maintenance + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/system/metrics" +) + +// compactionSafetyMarginBytes is added on top of the live database's +// current size when checking free disk space for a staged compaction copy, +// so a near-exact-size compacted output still leaves headroom for the +// filesystem's own bookkeeping. +const compactionSafetyMarginBytes = 10 * 1024 * 1024 // 10 MiB + +// CompactionResult reports the outcome of a successful staged compaction: +// where the pre-compaction database was preserved, and the size delta. +type CompactionResult struct { + // RollbackPath is the pre-compaction database file, renamed aside + // rather than deleted. Restoring it is: stop kandev (if running), + // then move RollbackPath back over the live database path. + RollbackPath string + SizeBeforeBytes int64 + SizeAfterBytes int64 +} + +// diskUsageFunc matches internal/system/metrics.DiskUsage's signature so +// tests can substitute a deterministic stand-in without touching the real +// filesystem's free space. +type diskUsageFunc func(ctx context.Context, path string) (metrics.DiskCapacity, error) + +// snapshotFunc matches internal/persistence.SnapshotSQLite's signature so +// tests can inject a VACUUM INTO failure, or a "succeeded but produced +// corrupt output" scenario, without depending on actually corrupting a +// SQLite file on disk. +type snapshotFunc func(writer *sqlx.DB, path string) (int64, error) + +// compact stages a fresh VACUUM INTO copy of the live database, verifies it +// with PRAGMA integrity_check, and only then atomically replaces the live +// file - retaining the pre-compaction file as a rollback artifact rather +// than deleting it. closeConnections must close every open handle to +// dbPath (writer and reader pools) before compact attempts the swap: open +// handles can silently block or corrupt a rename on some platforms. +// +// On any failure before the swap (insufficient disk space, VACUUM INTO +// failure, integrity-check failure), the live database at dbPath is left +// completely untouched and closeConnections is not called - the caller's +// existing connections remain valid. +func compact(ctx context.Context, writer *sqlx.DB, closeConnections func() error, dbPath string, diskUsage diskUsageFunc, snapshot snapshotFunc) (CompactionResult, error) { + info, err := os.Stat(dbPath) + if err != nil { + return CompactionResult{}, fmt.Errorf("stat live database: %w", err) + } + sizeBefore := info.Size() + + capacity, err := diskUsage(ctx, filepath.Dir(dbPath)) + if err != nil { + return CompactionResult{}, fmt.Errorf("check staging disk space: %w", err) + } + required := uint64(sizeBefore) + compactionSafetyMarginBytes + if capacity.AvailableBytes < required { + return CompactionResult{}, fmt.Errorf("%w: have %d bytes free, need at least %d", + ErrInsufficientDiskSpace, capacity.AvailableBytes, required) + } + + ts := time.Now().UTC().Format("20060102T150405.000000000Z") + stagedPath := dbPath + ".maintenance-staging-" + ts + ".db" + sizeAfter, err := snapshot(writer, stagedPath) + if err != nil { + _ = os.Remove(stagedPath) + return CompactionResult{}, fmt.Errorf("stage compacted copy: %w", err) + } + + if err := verifySQLiteIntegrity(stagedPath); err != nil { + _ = os.Remove(stagedPath) + return CompactionResult{}, err + } + + // Every handle to dbPath must close before the swap: an open connection + // can block (Windows) or silently keep writing to the old inode + // (POSIX, if anything re-opened it) through the rename. + if err := closeConnections(); err != nil { + _ = os.Remove(stagedPath) + return CompactionResult{}, fmt.Errorf("close database connections before compaction swap: %w", err) + } + + rollbackPath := dbPath + ".pre-compact-" + ts + ".bak" + if err := os.Rename(dbPath, rollbackPath); err != nil { + _ = os.Remove(stagedPath) + return CompactionResult{}, fmt.Errorf("move live database aside for rollback: %w", err) + } + // A VACUUM INTO target has no WAL/SHM sidecars of its own, but the live + // database being replaced may - move them alongside the rollback file + // (rather than leave them stranded at the old name) so the rollback + // artifact is self-consistent if ever restored. + movedSidecars, err := moveCompactionSidecars(dbPath, rollbackPath) + if err != nil { + restoreErr := restoreCompactionFiles(rollbackPath, dbPath, movedSidecars) + _ = os.Remove(stagedPath) + return CompactionResult{}, fmt.Errorf("move live database sidecars aside for rollback: %w", errors.Join(err, restoreErr)) + } + + if err := os.Rename(stagedPath, dbPath); err != nil { + restoreErr := restoreCompactionFiles(rollbackPath, dbPath, movedSidecars) + return CompactionResult{}, fmt.Errorf("swap staged database into place: %w", errors.Join(err, restoreErr)) + } + fsyncDir(filepath.Dir(dbPath)) + + return CompactionResult{RollbackPath: rollbackPath, SizeBeforeBytes: sizeBefore, SizeAfterBytes: sizeAfter}, nil +} + +// verifySQLiteIntegrity opens path read-only and runs PRAGMA +// integrity_check, treating anything other than a single "ok" row as a +// failure. Used both for the staged compaction copy and (via +// verifyBackup) the pre-execution backup. +func verifySQLiteIntegrity(path string) error { + conn, err := db.OpenSQLiteReader(path) + if err != nil { + return fmt.Errorf("open %s for integrity check: %w", path, err) + } + defer func() { _ = conn.Close() }() + + rows, err := conn.Query(`PRAGMA integrity_check`) + if err != nil { + return fmt.Errorf("%w: run integrity_check on %s: %v", ErrIntegrityCheckFailed, path, err) + } + defer func() { _ = rows.Close() }() + + var messages []string + for rows.Next() { + var msg string + if err := rows.Scan(&msg); err != nil { + return fmt.Errorf("scan integrity_check row: %w", err) + } + messages = append(messages, msg) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate integrity_check rows: %w", err) + } + if len(messages) == 1 && messages[0] == "ok" { + return nil + } + return fmt.Errorf("%w: %s", ErrIntegrityCheckFailed, strings.Join(messages, "; ")) +} + +// moveCompactionSidecars moves existing WAL/SHM files to the rollback name. +// A missing sidecar is normal; every other inspection or rename error aborts +// the swap so an old sidecar cannot be paired with the compacted database. +func moveCompactionSidecars(dbPath, rollbackPath string) ([]string, error) { + var moved []string + for _, suffix := range []string{"-wal", "-shm"} { + from, to := dbPath+suffix, rollbackPath+suffix + if _, err := os.Stat(from); errors.Is(err, os.ErrNotExist) { + continue + } else if err != nil { + return moved, err + } + if err := os.Rename(from, to); err != nil { + return moved, err + } + moved = append(moved, suffix) + } + return moved, nil +} + +func restoreCompactionFiles(rollbackPath, dbPath string, movedSidecars []string) error { + var restoreErr error + if err := os.Rename(rollbackPath, dbPath); err != nil { + restoreErr = errors.Join(restoreErr, fmt.Errorf("restore database: %w", err)) + } + for _, suffix := range movedSidecars { + if err := os.Rename(rollbackPath+suffix, dbPath+suffix); err != nil { + restoreErr = errors.Join(restoreErr, fmt.Errorf("restore %s sidecar: %w", suffix, err)) + } + } + return restoreErr +} + +// fsyncDir best-effort fsyncs a directory so the preceding renames are +// durable across a crash immediately after compaction. Failures are not +// fatal - the renames themselves already succeeded. +func fsyncDir(dir string) { + f, err := os.Open(dir) + if err != nil { + return + } + defer func() { _ = f.Close() }() + _ = f.Sync() +} diff --git a/apps/backend/internal/maintenance/execute.go b/apps/backend/internal/maintenance/execute.go new file mode 100644 index 0000000000..ab80c917e3 --- /dev/null +++ b/apps/backend/internal/maintenance/execute.go @@ -0,0 +1,106 @@ +package maintenance + +import ( + "context" + "database/sql" + "fmt" + "strings" +) + +// ExecutionResult reports how many rows were actually deleted per +// retention category during a --execute run. Row counts only - never IDs +// or content. +type ExecutionResult struct { + DeletedGitSnapshots int + DeletedPlanRevisions int + DeletedMessagePayloads int +} + +// TotalDeleted sums every category's deleted row count. +func (e ExecutionResult) TotalDeleted() int { + return e.DeletedGitSnapshots + e.DeletedPlanRevisions + e.DeletedMessagePayloads +} + +// executeDeleteChunkSize bounds each IN clause below SQLite's default +// SQLITE_MAX_VARIABLE_NUMBER (999 in most builds), leaving headroom. +const executeDeleteChunkSize = 400 + +// execute deletes exactly the rows identified by set inside a single +// transaction: either every configured retention category is applied, or +// none is (on any error the transaction rolls back and the database is left +// unchanged). Both task_session_git_snapshots and task_plan_revisions have +// no inbound foreign-key references from any other table (confirmed at +// design time), and task_message_payloads is only ever referenced by +// task_session_messages.payload_digest - so no FK cascade side effects are +// possible here beyond what set's callers (Analyze) already excluded. +// +// Deletes are idempotent by construction: applying execute a second time +// with the same (now empty, since the prior run already deleted them) ID +// sets is a correct no-op. +func execute(ctx context.Context, writer *sql.DB, set candidateSet) (ExecutionResult, error) { + tx, err := writer.BeginTx(ctx, nil) + if err != nil { + return ExecutionResult{}, fmt.Errorf("begin retention transaction: %w", err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + + deletedSnapshots, err := deleteByColumn(ctx, tx, "task_session_git_snapshots", "id", set.gitSnapshotIDs) + if err != nil { + return ExecutionResult{}, fmt.Errorf("delete duplicate git snapshots: %w", err) + } + deletedRevisions, err := deleteByColumn(ctx, tx, "task_plan_revisions", "id", set.planRevisionIDs) + if err != nil { + return ExecutionResult{}, fmt.Errorf("delete obsolete plan revisions: %w", err) + } + deletedPayloads, err := deleteByColumn(ctx, tx, "task_message_payloads", "digest", set.payloadDigests) + if err != nil { + return ExecutionResult{}, fmt.Errorf("delete orphaned message payloads: %w", err) + } + + if err := tx.Commit(); err != nil { + return ExecutionResult{}, fmt.Errorf("commit retention transaction: %w", err) + } + committed = true + + return ExecutionResult{ + DeletedGitSnapshots: deletedSnapshots, + DeletedPlanRevisions: deletedRevisions, + DeletedMessagePayloads: deletedPayloads, + }, nil +} + +// deleteByColumn deletes every row in table whose column value is in keys, +// batching the IN clause to stay under SQLite's bound-parameter limit. +// table and column are always compile-time literals supplied by execute +// above, never derived from external input. +func deleteByColumn(ctx context.Context, tx *sql.Tx, table, column string, keys []string) (int, error) { + total := 0 + for start := 0; start < len(keys); start += executeDeleteChunkSize { + end := start + executeDeleteChunkSize + if end > len(keys) { + end = len(keys) + } + batch := keys[start:end] + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",") + query := fmt.Sprintf(`DELETE FROM %s WHERE %s IN (%s)`, table, column, placeholders) //nolint:gosec // table/column are literal call-site constants, not user input + args := make([]interface{}, len(batch)) + for i, key := range batch { + args[i] = key + } + res, err := tx.ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + affected, err := res.RowsAffected() + if err != nil { + return 0, err + } + total += int(affected) + } + return total, nil +} diff --git a/apps/backend/internal/maintenance/fixture_test.go b/apps/backend/internal/maintenance/fixture_test.go new file mode 100644 index 0000000000..abb2eb5b55 --- /dev/null +++ b/apps/backend/internal/maintenance/fixture_test.go @@ -0,0 +1,214 @@ +package maintenance + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// testFixture is a real on-disk SQLite database (maintenance opens +// connections by file path, not by in-memory handle, so tests need a real +// file to point Run/compact/backup at) seeded through the task repository's +// public API wherever one exists, falling back to minimal raw inserts only +// for rows with no exported constructor (turns). +type testFixture struct { + t *testing.T + dbPath string + repo *sqlite.Repository + sqlxDB *sqlx.DB +} + +func newTestFixture(t *testing.T) *testFixture { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "maintenance-test.db") + conn, err := db.OpenSQLite(dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqlxDB := sqlx.NewDb(conn, "sqlite3") + repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + t.Fatalf("new repo: %v", err) + } + f := &testFixture{t: t, dbPath: dbPath, repo: repo, sqlxDB: sqlxDB} + t.Cleanup(func() { _ = sqlxDB.Close() }) + return f +} + +// closeForSwap closes the fixture's own connection so a compaction rename +// (or a second Run invocation opening its own connections) can proceed +// without a stale open handle. Safe to call at most meaningfully once; +// later calls (e.g. via t.Cleanup) are no-ops on an already-closed *sql.DB. +func (f *testFixture) closeForSwap() error { + return f.sqlxDB.Close() +} + +func (f *testFixture) seedTask(taskID string) { + f.t.Helper() + // WorkspaceID must be non-empty: initSchema's ensureDefaultWorkspace + // backfill runs on every repository open (including the second + // connection Run() opens against this same file) and would otherwise + // try to derive it from a (non-existent, in this minimal fixture) + // workflow row, violating tasks.workspace_id's NOT NULL constraint. + if err := f.repo.CreateTask(context.Background(), &models.Task{ID: taskID, WorkspaceID: "workspace-test", Title: taskID}); err != nil { + f.t.Fatalf("CreateTask(%s): %v", taskID, err) + } +} + +func (f *testFixture) seedSession(taskID, sessionID string) { + f.t.Helper() + environmentID := "environment-" + taskID + if err := f.repo.CreateTaskEnvironment(context.Background(), &models.TaskEnvironment{ + ID: environmentID, TaskID: taskID, + ExecutorType: string(models.ExecutorTypeLocal), + Status: models.TaskEnvironmentStatusReady, + }); err != nil { + f.t.Fatalf("CreateTaskEnvironment(%s): %v", environmentID, err) + } + if err := f.repo.CreateTaskSession(context.Background(), &models.TaskSession{ + ID: sessionID, TaskID: taskID, TaskEnvironmentID: environmentID, + }); err != nil { + f.t.Fatalf("CreateTaskSession(%s): %v", sessionID, err) + } +} + +// seedDuplicateGitSnapshots inserts two identical-content snapshots for +// sessionID, which ListDuplicateGitSnapshotCandidates reports the older of +// as a retention candidate (CreateGitSnapshot always records every poll - +// no write-time dedup skip - so this reliably produces one candidate row). +func (f *testFixture) seedDuplicateGitSnapshots(sessionID string, base time.Time) { + f.t.Helper() + ctx := context.Background() + snap := func(id string, createdAt time.Time) *models.GitSnapshot { + return &models.GitSnapshot{ + ID: id, SessionID: sessionID, SnapshotType: models.SnapshotTypeStatusUpdate, + Branch: "feature/x", RemoteBranch: "origin/feature/x", + HeadCommit: "head-sha", BaseCommit: "base-sha", + Files: map[string]interface{}{"a.go": map[string]interface{}{"status": "modified"}}, + CreatedAt: createdAt, + } + } + if err := f.repo.CreateGitSnapshot(ctx, snap(sessionID+"-snap-1", base)); err != nil { + f.t.Fatalf("CreateGitSnapshot(1): %v", err) + } + if err := f.repo.CreateGitSnapshot(ctx, snap(sessionID+"-snap-2", base.Add(time.Minute))); err != nil { + f.t.Fatalf("CreateGitSnapshot(2): %v", err) + } +} + +// seedObsoletePlanRevisions inserts count revisions for taskID; every +// revision except the last (HEAD) is a retention candidate. +func (f *testFixture) seedObsoletePlanRevisions(taskID string, count int, base time.Time) { + f.t.Helper() + ctx := context.Background() + for i := 1; i <= count; i++ { + rev := &models.TaskPlanRevision{ + ID: fmt.Sprintf("%s-rev-%d", taskID, i), TaskID: taskID, + RevisionNumber: i, Title: "v", Content: "revision content", + CreatedAt: base.Add(time.Duration(i) * time.Hour), + } + if err := f.repo.InsertTaskPlanRevision(ctx, rev); err != nil { + f.t.Fatalf("InsertTaskPlanRevision(%d): %v", i, err) + } + } +} + +// seedOrphanedMessagePayload creates one large externalized payload and +// then deletes its only referencing message, leaving the payload row +// orphaned. Returns the digest. +func (f *testFixture) seedOrphanedMessagePayload(taskID, sessionID, turnID string) string { + f.t.Helper() + ctx := context.Background() + f.seedTurn(sessionID, taskID, turnID) + + msg := &models.Message{ + ID: taskID + "-orphan-msg", TaskSessionID: sessionID, AuthorType: models.MessageAuthorAgent, + Type: models.MessageTypeToolCall, TurnID: turnID, + Metadata: shellOutputMetadataFixture(strings.Repeat("o", 5000)), + } + if err := f.repo.CreateMessage(ctx, msg); err != nil { + f.t.Fatalf("CreateMessage: %v", err) + } + if msg.PayloadDigest == "" { + f.t.Fatal("expected message to externalize a payload digest") + } + if _, err := f.repo.DB().ExecContext(ctx, `DELETE FROM task_session_messages WHERE id = ?`, msg.ID); err != nil { + f.t.Fatalf("delete message: %v", err) + } + return msg.PayloadDigest +} + +func (f *testFixture) seedTurn(sessionID, taskID, turnID string) { + f.t.Helper() + now := time.Now().UTC() + if _, err := f.repo.DB().Exec(` + INSERT OR IGNORE INTO task_session_turns + (id, task_session_id, task_id, started_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `, turnID, sessionID, taskID, now, now, now); err != nil { + f.t.Fatalf("seed turn %s: %v", turnID, err) + } +} + +func shellOutputMetadataFixture(stdout string) map[string]interface{} { + return map[string]interface{}{ + "tool_call_id": "call-1", + "normalized": map[string]interface{}{ + "shell_exec": map[string]interface{}{ + "command": "echo", + "output": map[string]interface{}{ + "stdout": stdout, + "stderr": "", + "truncated": false, + }, + }, + }, + } +} + +func (f *testFixture) countRows(query string, args ...interface{}) int { + f.t.Helper() + var n int + if err := f.repo.DB().QueryRow(query, args...).Scan(&n); err != nil { + f.t.Fatalf("count query %q: %v", query, err) + } + return n +} + +// openRepoForVerification opens a fresh connection to an existing database +// file (e.g. one just replaced by a compaction swap) so a test can confirm +// the post-swap file is a fully valid, readable database with the expected +// rows - independent of any connection the code under test held open. +func openRepoForVerification(t *testing.T, dbPath string) *sqlite.Repository { + t.Helper() + conn, err := db.OpenSQLite(dbPath) + if err != nil { + t.Fatalf("open sqlite for verification: %v", err) + } + sqlxDB := sqlx.NewDb(conn, "sqlite3") + t.Cleanup(func() { _ = sqlxDB.Close() }) + repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + t.Fatalf("new repo for verification: %v", err) + } + return repo +} + +func (f *testFixture) fileSize() int64 { + f.t.Helper() + info, err := os.Stat(f.dbPath) + if err != nil { + f.t.Fatalf("stat %s: %v", f.dbPath, err) + } + return info.Size() +} diff --git a/apps/backend/internal/maintenance/maintenance.go b/apps/backend/internal/maintenance/maintenance.go new file mode 100644 index 0000000000..086f62f2ec --- /dev/null +++ b/apps/backend/internal/maintenance/maintenance.go @@ -0,0 +1,55 @@ +// Package maintenance implements the offline `kandev maintenance database` +// command: a CLI-invoked, exclusive-access retention and compaction pass +// over a stopped backend's SQLite database. It is deliberately independent +// of internal/system/database, which is a live, HTTP-triggered facade for +// the running backend's System > Database settings page - the two must +// never be conflated, since this package assumes exclusive access to the +// database file and internal/system/database assumes the opposite. +// +// Safety model (see docs/plans/pr-watch-and-storage-bounds/ +// task-06-database-maintenance-command.md for the full contract): +// - Dry-run (Analyze) is always read-only and is the default mode; it +// never requires exclusive access. +// - Destructive execution (Run with Execute=true) requires exclusive +// access to the database (enforced via the same +// internal/backendapp/ownershiplock mechanism the live backend uses at +// boot, so a running backend and a maintenance run can never overlap) +// and always takes a fresh, verified VACUUM INTO backup before deleting +// anything. +// - Optional staged compaction (Run with Compact=true) stages a second +// VACUUM INTO copy, verifies it with PRAGMA integrity_check, and only +// then atomically replaces the live file, retaining the pre-compaction +// file as a documented rollback artifact. +// - Retention only ever targets the three candidate categories exposed by +// Task 05's read-only candidate queries (duplicate git snapshots, +// obsolete plan revisions, orphaned message payloads); it never touches +// task_session_messages, task_sessions, or any other conversational or +// task-identity row. +package maintenance + +import "errors" + +// ErrUnsupportedDriver is returned when the configured database driver is +// not sqlite. Database maintenance is a SQLite-only capability; a +// PostgreSQL deployment manages retention/compaction through its own +// operator tooling. +var ErrUnsupportedDriver = errors.New("database maintenance requires the sqlite driver") + +// ErrOwnershipConflict is returned when --execute is requested while +// another kandev process (the live backend, or a concurrent maintenance +// run) already owns the target database. +var ErrOwnershipConflict = errors.New("another kandev process owns this database; stop it before running --execute") + +// ErrInsufficientDiskSpace is returned when the staging area does not have +// enough free space to hold a full compacted copy of the database +// alongside the live file. +var ErrInsufficientDiskSpace = errors.New("insufficient free disk space for a staged compaction copy") + +// ErrIntegrityCheckFailed is returned when PRAGMA integrity_check reports a +// problem with a freshly staged (backup or compaction) snapshot. The +// original database is left untouched whenever this is returned. +var ErrIntegrityCheckFailed = errors.New("staged database failed PRAGMA integrity_check") + +// ErrBackupUnverified is returned when the pre-execution backup could not +// be created or verified. Execution never proceeds past this point. +var ErrBackupUnverified = errors.New("could not create or verify a pre-execution backup") diff --git a/apps/backend/internal/maintenance/report.go b/apps/backend/internal/maintenance/report.go new file mode 100644 index 0000000000..69222273a4 --- /dev/null +++ b/apps/backend/internal/maintenance/report.go @@ -0,0 +1,193 @@ +package maintenance + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" + + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// CandidateGroup summarizes one retention category's policy-eligible rows. +// Only counts and byte estimates are carried - never row content, IDs, task +// identifiers, or any other field that could leak conversation or PR data +// into a report that might be logged or printed. +type CandidateGroup struct { + RowCount int + EstBytes int64 +} + +// Report is the dry-run (and pre-execution) summary of retention +// candidates and current storage size. Safe to print or log verbatim. +type Report struct { + GeneratedAt time.Time + DatabaseSizeBytes int64 + DuplicateGitSnapshots CandidateGroup + ObsoletePlanRevisions CandidateGroup + OrphanedMessagePayloads CandidateGroup +} + +// TotalCandidateRows sums the row counts across every retention category. +func (r Report) TotalCandidateRows() int { + return r.DuplicateGitSnapshots.RowCount + r.ObsoletePlanRevisions.RowCount + r.OrphanedMessagePayloads.RowCount +} + +// TotalEstBytes sums the estimated reclaimable bytes across every retention +// category. This is an estimate of row content size, not the eventual disk +// space reclaimed by a VACUUM (which depends on page layout); the report +// clarifies that distinction to callers. +func (r Report) TotalEstBytes() int64 { + return r.DuplicateGitSnapshots.EstBytes + r.ObsoletePlanRevisions.EstBytes + r.OrphanedMessagePayloads.EstBytes +} + +// candidateSet holds the concrete row identifiers behind a Report so a +// later Execute call deletes exactly what Analyze reported, without +// re-querying and risking a report/execution race within the same run. +type candidateSet struct { + gitSnapshotIDs []string + planRevisionIDs []string + payloadDigests []string +} + +// AnalyzeOptions configures how Analyze selects retention candidates. +type AnalyzeOptions struct { + // KeepPlanRevisions additionally protects this many of the most recent + // non-HEAD plan revisions per task, on top of the HEAD/ancestry + // protections ListObsoletePlanRevisionCandidates always applies. Zero + // means "no extra recency window" (only HEAD/ancestry are protected). + KeepPlanRevisions int + // CandidateLimit caps how many rows each category query returns. Zero + // means unlimited. Intended for very large databases where a bounded + // dry-run report is preferable to scanning every candidate row. + CandidateLimit int +} + +// analyze computes the current Report and the concrete candidate ID sets +// behind it, using repo's read-only candidate queries plus a handful of +// direct byte-estimate aggregate queries against the same connection. +// Purely read-only: no row is ever written or deleted. +func analyze(ctx context.Context, repo *sqlite.Repository, opts AnalyzeOptions) (Report, candidateSet, error) { + report := Report{GeneratedAt: time.Now().UTC()} + var set candidateSet + + dupSnapshots, err := repo.ListDuplicateGitSnapshotCandidates(ctx, opts.CandidateLimit) + if err != nil { + return Report{}, candidateSet{}, fmt.Errorf("list duplicate git snapshot candidates: %w", err) + } + for _, c := range dupSnapshots { + set.gitSnapshotIDs = append(set.gitSnapshotIDs, c.ID) + } + gitBytes, err := sumGitSnapshotBytes(ctx, repo.DB(), set.gitSnapshotIDs) + if err != nil { + return Report{}, candidateSet{}, fmt.Errorf("estimate duplicate git snapshot bytes: %w", err) + } + report.DuplicateGitSnapshots = CandidateGroup{RowCount: len(dupSnapshots), EstBytes: gitBytes} + + taskIDs, err := distinctPlanRevisionTaskIDs(ctx, repo.DB()) + if err != nil { + return Report{}, candidateSet{}, fmt.Errorf("list plan revision task ids: %w", err) + } + var planRevisionBytes int64 + remaining := opts.CandidateLimit + for _, taskID := range taskIDs { + perTaskLimit := 0 + if opts.CandidateLimit > 0 { + if remaining <= 0 { + break + } + perTaskLimit = remaining + } + revisions, err := repo.ListObsoletePlanRevisionCandidates(ctx, taskID, opts.KeepPlanRevisions, perTaskLimit) + if err != nil { + return Report{}, candidateSet{}, fmt.Errorf("list obsolete plan revision candidates for task %s: %w", taskID, err) + } + for _, rev := range revisions { + set.planRevisionIDs = append(set.planRevisionIDs, rev.ID) + planRevisionBytes += rev.ContentBytes + } + if opts.CandidateLimit > 0 { + remaining -= len(revisions) + } + } + report.ObsoletePlanRevisions = CandidateGroup{RowCount: len(set.planRevisionIDs), EstBytes: planRevisionBytes} + + orphanedPayloads, err := repo.ListOrphanedMessagePayloadCandidates(ctx, opts.CandidateLimit) + if err != nil { + return Report{}, candidateSet{}, fmt.Errorf("list orphaned message payload candidates: %w", err) + } + var payloadBytes int64 + for _, c := range orphanedPayloads { + set.payloadDigests = append(set.payloadDigests, c.Digest) + payloadBytes += c.CompressedSize + } + report.OrphanedMessagePayloads = CandidateGroup{RowCount: len(orphanedPayloads), EstBytes: payloadBytes} + + return report, set, nil +} + +// distinctPlanRevisionTaskIDs enumerates every task_id present in +// task_plan_revisions, so the maintenance run can call the existing +// per-task ListObsoletePlanRevisionCandidates once per task. Read-only. +func distinctPlanRevisionTaskIDs(ctx context.Context, db *sql.DB) ([]string, error) { + rows, err := db.QueryContext(ctx, `SELECT DISTINCT task_id FROM task_plan_revisions ORDER BY task_id ASC`) + if err != nil { + return nil, fmt.Errorf("query distinct plan revision task ids: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []string + for rows.Next() { + var taskID string + if err := rows.Scan(&taskID); err != nil { + return nil, fmt.Errorf("scan plan revision task id: %w", err) + } + out = append(out, taskID) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate plan revision task ids: %w", err) + } + return out, nil +} + +// sumGitSnapshotBytesChunkSize bounds each IN clause below SQLite's default +// SQLITE_MAX_VARIABLE_NUMBER (999 in most builds), leaving headroom. +const sumGitSnapshotBytesChunkSize = 400 + +// sumGitSnapshotBytes estimates the on-disk content size of the given git +// snapshot rows by summing the length of their text columns. This is an +// estimate (it ignores SQLite's own row/page overhead), not an exact +// reclaimable-byte count; DuplicateGitSnapshotCandidate carries no +// pre-computed size column today; adding one would touch Task 05's already +// tested struct shape, so this query - scoped to exactly the candidate IDs +// Analyze already selected - is a cheaper approach. +func sumGitSnapshotBytes(ctx context.Context, db *sql.DB, ids []string) (int64, error) { + var total int64 + for start := 0; start < len(ids); start += sumGitSnapshotBytesChunkSize { + end := start + sumGitSnapshotBytesChunkSize + if end > len(ids) { + end = len(ids) + } + batch := ids[start:end] + placeholders := strings.TrimSuffix(strings.Repeat("?,", len(batch)), ",") + query := fmt.Sprintf(` + SELECT COALESCE(SUM( + COALESCE(LENGTH(files), 0) + COALESCE(LENGTH(metadata), 0) + + COALESCE(LENGTH(branch), 0) + COALESCE(LENGTH(remote_branch), 0) + + COALESCE(LENGTH(head_commit), 0) + COALESCE(LENGTH(base_commit), 0) + + COALESCE(LENGTH(triggered_by), 0) + 96 + ), 0) + FROM task_session_git_snapshots WHERE id IN (%s)`, placeholders) + args := make([]interface{}, len(batch)) + for i, id := range batch { + args[i] = id + } + var sum int64 + if err := db.QueryRowContext(ctx, query, args...).Scan(&sum); err != nil { + return 0, fmt.Errorf("sum git snapshot bytes: %w", err) + } + total += sum + } + return total, nil +} diff --git a/apps/backend/internal/maintenance/run.go b/apps/backend/internal/maintenance/run.go new file mode 100644 index 0000000000..4e5c24f727 --- /dev/null +++ b/apps/backend/internal/maintenance/run.go @@ -0,0 +1,212 @@ +package maintenance + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/backendapp/ownershiplock" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/persistence" + "github.com/kandev/kandev/internal/system/metrics" + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +// RunOptions configures one `kandev maintenance database` invocation. +type RunOptions struct { + // Execute, when false (the default), performs a read-only Analyze and + // returns without acquiring any lock, taking any backup, or deleting or + // compacting anything. + Execute bool + // Compact additionally stages a VACUUM INTO compaction and atomic + // replacement after retention deletes commit. Ignored when Execute is + // false. + Compact bool + // KeepPlanRevisions is forwarded to AnalyzeOptions.KeepPlanRevisions. + KeepPlanRevisions int + // CandidateLimit is forwarded to AnalyzeOptions.CandidateLimit. + CandidateLimit int +} + +// Outcome is the full result of one Run call, covering both dry-run and +// execute modes; fields not applicable to the mode that ran are left at +// their zero value. +type Outcome struct { + Executed bool + Report Report + BackupPath string + Execution ExecutionResult + Compaction *CompactionResult +} + +// Run performs one `kandev maintenance database` invocation against the +// SQLite database at databasePath. homeDir/databaseDriver/databasePath are +// the same triple internal/backendapp/ownershiplock.Targets uses at backend +// boot, so a live backend and a --execute maintenance run can never +// overlap. log receives only structural events (counts, paths, durations); +// no row content or identifier is ever logged. +func Run(ctx context.Context, homeDir, databaseDriver, databasePath string, opts RunOptions, log *logger.Logger) (Outcome, error) { + if !strings.EqualFold(databaseDriver, "sqlite") { + return Outcome{}, ErrUnsupportedDriver + } + + owner, err := acquireExecuteLock(homeDir, databaseDriver, databasePath, opts.Execute) + if err != nil { + return Outcome{}, err + } + if owner != nil { + defer func() { _ = owner.Close() }() + } + + var repo *sqlite.Repository + var writer *sqlx.DB + var reader *sqlx.DB + var closeConnections func() error + var backupPath string + if opts.Execute { + writer, reader, closeConnections, err = openMaintenanceRawConnections(databasePath) + if err != nil { + return Outcome{}, err + } + backupDir := filepath.Join(filepath.Dir(databasePath), "backups") + backupPath, err = createVerifiedBackup(writer, backupDir) + if err != nil { + _ = closeConnections() + return Outcome{}, err + } + repo, err = openMaintenanceRepository(writer, reader, log) + if err != nil { + _ = closeConnections() + return Outcome{}, err + } + } else { + repo, closeConnections, err = openMaintenanceReadOnlyConnection(databasePath, log) + } + if err != nil { + return Outcome{}, err + } + defer func() { _ = closeConnections() }() + + report, set, err := analyze(ctx, repo, AnalyzeOptions{KeepPlanRevisions: opts.KeepPlanRevisions, CandidateLimit: opts.CandidateLimit}) + if err != nil { + return Outcome{}, fmt.Errorf("analyze retention candidates: %w", err) + } + if info, statErr := os.Stat(databasePath); statErr == nil { + report.DatabaseSizeBytes = info.Size() + } + + if !opts.Execute { + return Outcome{Executed: false, Report: report}, nil + } + + return runExecuteAndCompact(ctx, writer, closeConnections, databasePath, backupPath, report, set, opts.Compact) +} + +// openMaintenanceReadOnlyConnection opens an existing SQLite database in +// read-only mode and deliberately skips repository schema initialization. A +// missing database or schema that predates the retention queries therefore +// fails closed without creating, migrating, or healing anything. +func openMaintenanceReadOnlyConnection(databasePath string, log *logger.Logger) (*sqlite.Repository, func() error, error) { + readerConn, err := db.OpenSQLiteReader(databasePath) + if err != nil { + return nil, nil, fmt.Errorf("open database for reading: %w", err) + } + reader := sqlx.NewDb(readerConn, "sqlite3") + return sqlite.NewReadOnlyWithDB(reader, log), reader.Close, nil +} + +// acquireExecuteLock takes the same exclusive-access ownership lock a live +// backend takes at boot, but only when execute is true: a dry run must +// remain safely runnable concurrently with a live backend. +func acquireExecuteLock(homeDir, databaseDriver, databasePath string, execute bool) (*ownershiplock.Owner, error) { + if !execute { + return nil, nil + } + targets, err := ownershiplock.Targets(homeDir, databaseDriver, databasePath) + if err != nil { + return nil, fmt.Errorf("compute ownership lock targets: %w", err) + } + owner, err := ownershiplock.Acquire(targets) + if err != nil { + var conflict *ownershiplock.ConflictError + if errors.As(err, &conflict) { + return nil, fmt.Errorf("%w: %s", ErrOwnershipConflict, conflict.Error()) + } + return nil, fmt.Errorf("acquire exclusive database access: %w", err) + } + return owner, nil +} + +// openMaintenanceConnections opens the CLI's own raw writer/reader SQLite +// connections (deliberately bypassing persistence.Provide's boot-specific +// version-migration side effects) and wraps them in a task repository. +// The returned close func is idempotent and safe to call from a defer even +// after compact() has already closed the connections itself. +func openMaintenanceRawConnections(databasePath string) (*sqlx.DB, *sqlx.DB, func() error, error) { + writerConn, err := db.OpenSQLite(databasePath) + if err != nil { + return nil, nil, nil, fmt.Errorf("open database for writing: %w", err) + } + readerConn, err := db.OpenSQLiteReader(databasePath) + if err != nil { + _ = writerConn.Close() + return nil, nil, nil, fmt.Errorf("open database for reading: %w", err) + } + writer := sqlx.NewDb(writerConn, "sqlite3") + reader := sqlx.NewDb(readerConn, "sqlite3") + closed := false + closeConnections := func() error { + if closed { + return nil + } + closed = true + writerErr := writer.Close() + readerErr := reader.Close() + if writerErr != nil { + return writerErr + } + return readerErr + } + return writer, reader, closeConnections, nil +} + +func openMaintenanceRepository(writer, reader *sqlx.DB, log *logger.Logger) (*sqlite.Repository, error) { + repo, err := sqlite.NewWithDB(writer, reader, log) + if err != nil { + return nil, fmt.Errorf("initialize task repository: %w", err) + } + return repo, nil +} + +// runExecuteAndCompact performs the --execute path: a fresh verified backup, +// the retention deletes, and (only when compactRequested) the staged +// VACUUM INTO compaction and atomic replacement. A compaction failure is +// reported as a partial-success Outcome (deletes already committed and are +// backed by backupPath) rather than a total abort. +func runExecuteAndCompact(ctx context.Context, writer *sqlx.DB, closeConnections func() error, databasePath, backupPath string, report Report, set candidateSet, compactRequested bool) (Outcome, error) { + execResult, err := execute(ctx, writer.DB, set) + if err != nil { + return Outcome{}, fmt.Errorf("execute retention deletes: %w", err) + } + + outcome := Outcome{Executed: true, Report: report, BackupPath: backupPath, Execution: execResult} + if !compactRequested { + return outcome, nil + } + + compactionResult, err := compact(ctx, writer, closeConnections, databasePath, metrics.DiskUsage, persistence.SnapshotSQLite) + if err != nil { + // Retention deletes already committed and are backed by backupPath; + // only the optional compaction step failed, so the run is reported + // as a partial failure rather than a total abort. + return outcome, fmt.Errorf("compact database: %w", err) + } + outcome.Compaction = &compactionResult + return outcome, nil +} diff --git a/apps/backend/internal/maintenance/run_test.go b/apps/backend/internal/maintenance/run_test.go new file mode 100644 index 0000000000..679de4be82 --- /dev/null +++ b/apps/backend/internal/maintenance/run_test.go @@ -0,0 +1,448 @@ +package maintenance + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/backendapp/ownershiplock" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/system/metrics" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository/sqlite" +) + +func TestRunDryRunMissingDatabaseCreatesNothing(t *testing.T) { + databaseDir := t.TempDir() + databasePath := filepath.Join(databaseDir, "kandev.db") + + _, err := Run(context.Background(), t.TempDir(), "sqlite", databasePath, RunOptions{Execute: false}, nil) + if err == nil { + t.Fatal("Run(dry-run) error = nil, want missing database error") + } + if _, statErr := os.Stat(databasePath); !os.IsNotExist(statErr) { + t.Fatalf("dry-run created missing database file; stat err = %v", statErr) + } +} + +func TestRunDryRunLegacySchemaDoesNotMigrateOrHeal(t *testing.T) { + databasePath := filepath.Join(t.TempDir(), "legacy.db") + conn, err := db.OpenSQLite(databasePath) + if err != nil { + t.Fatalf("open legacy sqlite: %v", err) + } + if _, err := conn.Exec(`CREATE TABLE legacy_marker (id TEXT PRIMARY KEY)`); err != nil { + _ = conn.Close() + t.Fatalf("create legacy schema: %v", err) + } + if err := conn.Close(); err != nil { + t.Fatalf("close legacy sqlite: %v", err) + } + + before, err := os.ReadFile(databasePath) + if err != nil { + t.Fatalf("read legacy database before dry-run: %v", err) + } + _, err = Run(context.Background(), t.TempDir(), "sqlite", databasePath, RunOptions{Execute: false}, nil) + if err == nil { + t.Fatal("Run(dry-run) error = nil, want unsupported legacy schema error") + } + after, err := os.ReadFile(databasePath) + if err != nil { + t.Fatalf("read legacy database after dry-run: %v", err) + } + if !bytes.Equal(after, before) { + t.Fatal("dry-run changed legacy database bytes") + } +} + +// TestRunDryRunReportsCandidatesWithoutMutating is this wave's core dry-run +// contract test: Analyze must report every category's exact candidate +// count/byte estimate while leaving every row, and the database file +// itself, completely unchanged. +func TestRunDryRunReportsCandidatesWithoutMutating(t *testing.T) { + f := newTestFixture(t) + base := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + + f.seedTask("task-dryrun") + f.seedSession("task-dryrun", "session-dryrun") + f.seedDuplicateGitSnapshots("session-dryrun", base) + f.seedObsoletePlanRevisions("task-dryrun", 3, base) + f.seedOrphanedMessagePayload("task-dryrun", "session-dryrun", "turn-dryrun") + + snapshotsBefore := f.countRows(`SELECT COUNT(*) FROM task_session_git_snapshots`) + revisionsBefore := f.countRows(`SELECT COUNT(*) FROM task_plan_revisions`) + payloadsBefore := f.countRows(`SELECT COUNT(*) FROM task_message_payloads`) + sizeBefore := f.fileSize() + + outcome, err := Run(context.Background(), t.TempDir(), "sqlite", f.dbPath, RunOptions{Execute: false}, nil) + if err != nil { + t.Fatalf("Run(dry-run): %v", err) + } + if outcome.Executed { + t.Fatal("Executed = true, want false for a dry-run") + } + if outcome.Report.DuplicateGitSnapshots.RowCount != 1 { + t.Fatalf("DuplicateGitSnapshots.RowCount = %d, want 1", outcome.Report.DuplicateGitSnapshots.RowCount) + } + if outcome.Report.ObsoletePlanRevisions.RowCount != 2 { + t.Fatalf("ObsoletePlanRevisions.RowCount = %d, want 2 (3 revisions, HEAD protected)", outcome.Report.ObsoletePlanRevisions.RowCount) + } + if outcome.Report.OrphanedMessagePayloads.RowCount != 1 { + t.Fatalf("OrphanedMessagePayloads.RowCount = %d, want 1", outcome.Report.OrphanedMessagePayloads.RowCount) + } + if outcome.Report.DatabaseSizeBytes <= 0 { + t.Fatalf("DatabaseSizeBytes = %d, want positive", outcome.Report.DatabaseSizeBytes) + } + + if got := f.countRows(`SELECT COUNT(*) FROM task_session_git_snapshots`); got != snapshotsBefore { + t.Fatalf("git snapshot rows changed: before=%d after=%d", snapshotsBefore, got) + } + if got := f.countRows(`SELECT COUNT(*) FROM task_plan_revisions`); got != revisionsBefore { + t.Fatalf("plan revision rows changed: before=%d after=%d", revisionsBefore, got) + } + if got := f.countRows(`SELECT COUNT(*) FROM task_message_payloads`); got != payloadsBefore { + t.Fatalf("message payload rows changed: before=%d after=%d", payloadsBefore, got) + } + if got := f.fileSize(); got != sizeBefore { + t.Fatalf("database file size changed: before=%d after=%d", sizeBefore, got) + } + + backupDir := filepath.Join(filepath.Dir(f.dbPath), "backups") + if _, err := os.Stat(backupDir); !os.IsNotExist(err) { + t.Fatalf("dry-run must not create a backups dir; stat err = %v", err) + } +} + +// TestRunUnsupportedDriverRefusesImmediately confirms a non-sqlite driver +// (e.g. postgres) is rejected before any connection is opened or lock +// attempted, since database maintenance is a SQLite-only capability. +func TestRunUnsupportedDriverRefusesImmediately(t *testing.T) { + _, err := Run(context.Background(), t.TempDir(), "postgres", "unused", RunOptions{Execute: true}, nil) + if !errors.Is(err, ErrUnsupportedDriver) { + t.Fatalf("err = %v, want ErrUnsupportedDriver", err) + } +} + +// TestRunExecuteRefusesWhenAnotherProcessHoldsOwnership proves --execute +// refuses to run while a live backend (simulated here by acquiring the same +// ownership targets directly) already owns the database, so a maintenance +// run and a running backend can never race on destructive operations. +func TestRunExecuteRefusesWhenAnotherProcessHoldsOwnership(t *testing.T) { + f := newTestFixture(t) + homeDir := t.TempDir() + + targets, err := ownershiplock.Targets(homeDir, "sqlite", f.dbPath) + if err != nil { + t.Fatalf("ownershiplock.Targets: %v", err) + } + owner, err := ownershiplock.Acquire(targets) + if err != nil { + t.Fatalf("ownershiplock.Acquire (simulated live backend): %v", err) + } + defer func() { _ = owner.Close() }() + + _, err = Run(context.Background(), homeDir, "sqlite", f.dbPath, RunOptions{Execute: true}, nil) + if !errors.Is(err, ErrOwnershipConflict) { + t.Fatalf("err = %v, want ErrOwnershipConflict", err) + } + + // The refusal must be a pure no-op: no backup, no deletion attempt. + backupDir := filepath.Join(filepath.Dir(f.dbPath), "backups") + if _, statErr := os.Stat(backupDir); !os.IsNotExist(statErr) { + t.Fatalf("ownership-conflict refusal must not create a backup dir; stat err = %v", statErr) + } +} + +// TestRunExecuteDeletesCandidatesBackupAndIsIdempotent is the end-to-end +// happy path: --execute takes a verified backup, deletes exactly the +// reported candidates, and a second run against the now-converged database +// deletes nothing further (idempotent retention). +func TestRunExecuteDeletesCandidatesBackupAndIsIdempotent(t *testing.T) { + f := newTestFixture(t) + base := time.Date(2026, 3, 2, 0, 0, 0, 0, time.UTC) + + f.seedTask("task-exec") + f.seedSession("task-exec", "session-exec") + f.seedDuplicateGitSnapshots("session-exec", base) + f.seedObsoletePlanRevisions("task-exec", 3, base) + f.seedOrphanedMessagePayload("task-exec", "session-exec", "turn-exec") + + homeDir := t.TempDir() + outcome, err := Run(context.Background(), homeDir, "sqlite", f.dbPath, RunOptions{Execute: true}, nil) + if err != nil { + t.Fatalf("Run(execute): %v", err) + } + if !outcome.Executed { + t.Fatal("Executed = false, want true") + } + if outcome.Execution.DeletedGitSnapshots != 1 { + t.Fatalf("DeletedGitSnapshots = %d, want 1", outcome.Execution.DeletedGitSnapshots) + } + if outcome.Execution.DeletedPlanRevisions != 2 { + t.Fatalf("DeletedPlanRevisions = %d, want 2", outcome.Execution.DeletedPlanRevisions) + } + if outcome.Execution.DeletedMessagePayloads != 1 { + t.Fatalf("DeletedMessagePayloads = %d, want 1", outcome.Execution.DeletedMessagePayloads) + } + if outcome.BackupPath == "" { + t.Fatal("BackupPath is empty, want a verified backup path") + } + if _, err := os.Stat(outcome.BackupPath); err != nil { + t.Fatalf("backup file missing at %s: %v", outcome.BackupPath, err) + } + if err := verifySQLiteIntegrity(outcome.BackupPath); err != nil { + t.Fatalf("backup failed integrity check: %v", err) + } + + if got := f.countRows(`SELECT COUNT(*) FROM task_session_git_snapshots`); got != 1 { + t.Fatalf("git snapshot rows after execute = %d, want 1 (the retained newest row)", got) + } + if got := f.countRows(`SELECT COUNT(*) FROM task_plan_revisions`); got != 1 { + t.Fatalf("plan revision rows after execute = %d, want 1 (HEAD)", got) + } + if got := f.countRows(`SELECT COUNT(*) FROM task_message_payloads`); got != 0 { + t.Fatalf("message payload rows after execute = %d, want 0", got) + } + + // Second execute against the now-converged database must delete + // nothing further - idempotent retention. + second, err := Run(context.Background(), homeDir, "sqlite", f.dbPath, RunOptions{Execute: true}, nil) + if err != nil { + t.Fatalf("Run(execute, second): %v", err) + } + if second.Execution.TotalDeleted() != 0 { + t.Fatalf("second execute deleted %d rows, want 0 (idempotent)", second.Execution.TotalDeleted()) + } +} + +func TestRunExecutePreservesAuthoritativeGitSnapshots(t *testing.T) { + tests := []struct { + name string + snapshotType models.SnapshotType + triggeredBy string + archiveTask bool + }{ + {name: "archive", snapshotType: models.SnapshotTypeArchive, triggeredBy: "task_archived", archiveTask: true}, + {name: "agent completed", snapshotType: models.SnapshotTypeStatusUpdate, triggeredBy: "agent_completed"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := newTestFixture(t) + base := time.Date(2026, 3, 2, 12, 0, 0, 0, time.UTC) + taskID := "task-protected-" + strings.ReplaceAll(tt.name, " ", "-") + sessionID := "session-protected-" + strings.ReplaceAll(tt.name, " ", "-") + f.seedTask(taskID) + f.seedSession(taskID, sessionID) + if tt.archiveTask { + if _, err := f.repo.DB().Exec(`UPDATE tasks SET archived_at = ? WHERE id = ?`, base, taskID); err != nil { + t.Fatalf("archive task: %v", err) + } + } + + seedSnapshot := func(id string, snapshotType models.SnapshotType, triggeredBy string, createdAt time.Time) { + t.Helper() + err := f.repo.CreateGitSnapshot(context.Background(), &models.GitSnapshot{ + ID: id, SessionID: sessionID, SnapshotType: snapshotType, TriggeredBy: triggeredBy, + Branch: "feature/protected", RemoteBranch: "origin/feature/protected", + HeadCommit: "protected-head", BaseCommit: "protected-base", + Files: map[string]interface{}{"protected.go": map[string]interface{}{"status": "modified"}}, + CreatedAt: createdAt, + }) + if err != nil { + t.Fatalf("CreateGitSnapshot(%s): %v", id, err) + } + } + + protectedID := sessionID + "-authoritative" + seedSnapshot(protectedID, tt.snapshotType, tt.triggeredBy, base) + seedSnapshot(sessionID+"-live-old", models.SnapshotTypeStatusUpdate, sqlite.TriggeredByLiveMonitor, base.Add(time.Minute)) + seedSnapshot(sessionID+"-live-new", models.SnapshotTypeStatusUpdate, sqlite.TriggeredByLiveMonitor, base.Add(2*time.Minute)) + + before, err := f.repo.GetLatestGitSnapshot(context.Background(), sessionID) + if err != nil { + t.Fatalf("GetLatestGitSnapshot(before): %v", err) + } + if before.ID != protectedID { + t.Fatalf("authoritative snapshot before maintenance = %s, want %s", before.ID, protectedID) + } + + outcome, err := Run(context.Background(), t.TempDir(), "sqlite", f.dbPath, RunOptions{Execute: true}, nil) + if err != nil { + t.Fatalf("Run(execute): %v", err) + } + if outcome.Execution.DeletedGitSnapshots != 1 { + t.Fatalf("DeletedGitSnapshots = %d, want only the older live duplicate", outcome.Execution.DeletedGitSnapshots) + } + + after, err := f.repo.GetLatestGitSnapshot(context.Background(), sessionID) + if err != nil { + t.Fatalf("GetLatestGitSnapshot(after): %v", err) + } + if after.ID != protectedID { + t.Fatalf("authoritative snapshot after maintenance = %s, want %s", after.ID, protectedID) + } + }) + } +} + +// TestRunExecuteWithCompactReplacesDatabaseAndPreservesRollback proves the +// staged-compaction path: after retention deletes commit, --compact stages +// a VACUUM INTO copy, verifies it, and atomically swaps it into place while +// preserving the pre-compaction file as a named rollback artifact that +// itself passes an integrity check. +func TestRunExecuteWithCompactReplacesDatabaseAndPreservesRollback(t *testing.T) { + f := newTestFixture(t) + base := time.Date(2026, 3, 3, 0, 0, 0, 0, time.UTC) + + f.seedTask("task-compact") + f.seedSession("task-compact", "session-compact") + f.seedDuplicateGitSnapshots("session-compact", base) + + homeDir := t.TempDir() + outcome, err := Run(context.Background(), homeDir, "sqlite", f.dbPath, RunOptions{Execute: true, Compact: true}, nil) + if err != nil { + t.Fatalf("Run(execute+compact): %v", err) + } + if outcome.Compaction == nil { + t.Fatal("Compaction result is nil, want a populated CompactionResult") + } + if outcome.Compaction.RollbackPath == "" { + t.Fatal("RollbackPath is empty") + } + if _, err := os.Stat(outcome.Compaction.RollbackPath); err != nil { + t.Fatalf("rollback artifact missing at %s: %v", outcome.Compaction.RollbackPath, err) + } + if err := verifySQLiteIntegrity(outcome.Compaction.RollbackPath); err != nil { + t.Fatalf("rollback artifact failed integrity check: %v", err) + } + if err := verifySQLiteIntegrity(f.dbPath); err != nil { + t.Fatalf("post-compaction live database failed integrity check: %v", err) + } + + // A fresh connection against the replaced file must still see the + // post-retention row counts (compaction must not lose data). + verifyRepo := openRepoForVerification(t, f.dbPath) + var count int + if err := verifyRepo.DB().QueryRow(`SELECT COUNT(*) FROM task_session_git_snapshots`).Scan(&count); err != nil { + t.Fatalf("count git snapshots after compaction: %v", err) + } + if count != 1 { + t.Fatalf("git snapshot rows after compaction = %d, want 1", count) + } +} + +// TestCompactRefusesWithInsufficientDiskSpace injects a diskUsageFunc +// reporting almost no free space and confirms compact refuses before +// touching the live database file at all. +func TestCompactRefusesWithInsufficientDiskSpace(t *testing.T) { + f := newTestFixture(t) + f.seedTask("task-disk") + sizeBefore := f.fileSize() + + starvedDiskUsage := func(_ context.Context, _ string) (metrics.DiskCapacity, error) { + return metrics.DiskCapacity{TotalBytes: 1024, AvailableBytes: 1, UsedBytes: 1023, UsedPercent: 99.9}, nil + } + _, err := compact(context.Background(), f.sqlxDB, f.closeForSwap, f.dbPath, starvedDiskUsage, fakeSnapshotSuccess) + if !errors.Is(err, ErrInsufficientDiskSpace) { + t.Fatalf("err = %v, want ErrInsufficientDiskSpace", err) + } + if got := f.fileSize(); got != sizeBefore { + t.Fatalf("live database size changed after a disk-space refusal: before=%d after=%d", sizeBefore, got) + } +} + +// TestCompactAbortsWhenSnapshotFails injects a failing snapshot function +// (simulating a VACUUM INTO failure) and confirms the live database is left +// completely untouched and no rollback file is created. +func TestCompactAbortsWhenSnapshotFails(t *testing.T) { + f := newTestFixture(t) + f.seedTask("task-snapfail") + sizeBefore := f.fileSize() + + _, err := compact(context.Background(), f.sqlxDB, f.closeForSwap, f.dbPath, alwaysEnoughDiskUsage, fakeSnapshotFailure) + if err == nil { + t.Fatal("expected an error from a failing snapshot function") + } + if got := f.fileSize(); got != sizeBefore { + t.Fatalf("live database size changed after a snapshot failure: before=%d after=%d", sizeBefore, got) + } + rollbackMatches, globErr := filepath.Glob(f.dbPath + ".pre-compact-*") + if globErr != nil { + t.Fatalf("glob for rollback artifacts: %v", globErr) + } + if len(rollbackMatches) != 0 { + t.Fatalf("no rollback artifact should exist after a snapshot failure, found %v", rollbackMatches) + } +} + +// TestCompactAbortsOnIntegrityCheckFailure injects a snapshot function that +// "succeeds" but writes corrupt (non-SQLite) bytes to the staged path, +// proving PRAGMA integrity_check catches the corruption before the swap and +// the live database is left untouched. +func TestCompactAbortsOnIntegrityCheckFailure(t *testing.T) { + f := newTestFixture(t) + f.seedTask("task-corrupt") + sizeBefore := f.fileSize() + + _, err := compact(context.Background(), f.sqlxDB, f.closeForSwap, f.dbPath, alwaysEnoughDiskUsage, fakeSnapshotCorrupt) + if !errors.Is(err, ErrIntegrityCheckFailed) { + t.Fatalf("err = %v, want ErrIntegrityCheckFailed", err) + } + if got := f.fileSize(); got != sizeBefore { + t.Fatalf("live database size changed after an integrity-check failure: before=%d after=%d", sizeBefore, got) + } +} + +func TestMoveCompactionSidecarsMovesRollbackSetTogether(t *testing.T) { + dir := t.TempDir() + dbPath := filepath.Join(dir, "live.db") + rollbackPath := filepath.Join(dir, "rollback.db") + for _, suffix := range []string{"-wal", "-shm"} { + if err := os.WriteFile(dbPath+suffix, []byte(suffix), 0o600); err != nil { + t.Fatalf("write sidecar %s: %v", suffix, err) + } + } + moved, err := moveCompactionSidecars(dbPath, rollbackPath) + if err != nil { + t.Fatalf("moveCompactionSidecars: %v", err) + } + if got, want := strings.Join(moved, ","), "-wal,-shm"; got != want { + t.Fatalf("moved = %q, want %q", got, want) + } + for _, suffix := range moved { + if _, err := os.Stat(rollbackPath + suffix); err != nil { + t.Fatalf("rollback sidecar %s missing: %v", suffix, err) + } + } +} + +func alwaysEnoughDiskUsage(_ context.Context, _ string) (metrics.DiskCapacity, error) { + return metrics.DiskCapacity{TotalBytes: 1 << 40, AvailableBytes: 1 << 40, UsedBytes: 0, UsedPercent: 0}, nil +} + +func fakeSnapshotSuccess(_ *sqlx.DB, path string) (int64, error) { + if err := os.WriteFile(path, []byte("not a real database"), 0o600); err != nil { + return 0, err + } + return 20, nil +} + +func fakeSnapshotFailure(_ *sqlx.DB, _ string) (int64, error) { + return 0, errors.New("simulated VACUUM INTO failure") +} + +func fakeSnapshotCorrupt(_ *sqlx.DB, path string) (int64, error) { + if err := os.WriteFile(path, []byte("this is not a valid sqlite file"), 0o600); err != nil { + return 0, err + } + return 31, nil +} diff --git a/apps/backend/internal/orchestrator/event_handlers_github.go b/apps/backend/internal/orchestrator/event_handlers_github.go index c3d4466dac..3866f7b195 100644 --- a/apps/backend/internal/orchestrator/event_handlers_github.go +++ b/apps/backend/internal/orchestrator/event_handlers_github.go @@ -1456,15 +1456,114 @@ func (s *Service) ListTasksNeedingPRWatch(ctx context.Context) ([]github.TaskBra return s.buildTaskBranchList(ctx, store) } +// ResolveBranchForRepository returns the current desired branch for a task's +// repository, independent of which session (if any) created the watch. +// Session identity does not participate in canonical PR-watch identity or +// polling (task/repository/branch owns searching watches), so a still- +// searching watch's branch must be re-derived from the task's own +// configuration, not from whichever session happened to create it — a prior +// per-session resolver ignored watch.RepositoryID entirely and always +// resolved against the task's PRIMARY repository, corrupting secondary-repo +// watches on multi-repo tasks. This satisfies the github.TaskBranchProvider +// interface. +func (s *Service) ResolveBranchForRepository(ctx context.Context, taskID, repositoryID string) string { + store, ok := s.repo.(repoStore) + if !ok { + return "" + } + if branch := s.checkoutBranchForRepository(ctx, store, taskID, repositoryID); branch != "" { + return branch + } + // Fallback: any currently-active session's worktree branch for this + // specific repository. Session identity is a best-effort provenance + // candidate only here (checkout_branch is authoritative once set). + return s.activeWorktreeBranchForRepository(ctx, taskID, repositoryID) +} + +// ResolveBranchesForRepository returns every authoritative checkout branch +// configured for a task/repository pair. Multi-branch tasks may legitimately +// have several rows for the same repository, so PR-watch reconciliation must +// preserve each matching watch instead of collapsing the repository to the +// first row returned by ListTaskRepositories. +func (s *Service) ResolveBranchesForRepository(ctx context.Context, taskID, repositoryID string) []string { + store, ok := s.repo.(repoStore) + if !ok { + return nil + } + branches := s.checkoutBranchesForRepository(ctx, store, taskID, repositoryID) + if len(branches) > 0 { + return branches + } + if branch := s.activeWorktreeBranchForRepository(ctx, taskID, repositoryID); branch != "" { + return []string{branch} + } + return nil +} + +// checkoutBranchForRepository returns the authoritative checkout_branch +// configured for a specific task/repository pair, or "" if unset or missing. +func (s *Service) checkoutBranchForRepository(ctx context.Context, store repoStore, taskID, repositoryID string) string { + branches := s.checkoutBranchesForRepository(ctx, store, taskID, repositoryID) + if len(branches) == 0 { + return "" + } + return branches[0] +} + +func (s *Service) checkoutBranchesForRepository(ctx context.Context, store repoStore, taskID, repositoryID string) []string { + taskRepos, err := store.ListTaskRepositories(ctx, taskID) + if err != nil { + return nil + } + seen := make(map[string]bool) + var branches []string + for _, tr := range taskRepos { + if tr.RepositoryID == repositoryID { + branch := strings.TrimSpace(tr.CheckoutBranch) + if branch == "" || seen[branch] { + continue + } + seen[branch] = true + branches = append(branches, branch) + } + } + return branches +} + +// activeWorktreeBranchForRepository returns the worktree branch of any +// currently-active session on the task whose worktree targets the given +// repository. Best-effort fallback only. +func (s *Service) activeWorktreeBranchForRepository(ctx context.Context, taskID, repositoryID string) string { + sessions, err := s.repo.ListActiveTaskSessionsByTaskID(ctx, taskID) + if err != nil { + return "" + } + for _, sess := range sessions { + for _, wt := range sess.Worktrees { + if wt.RepositoryID == repositoryID && strings.TrimSpace(wt.WorktreeBranch) != "" { + return strings.TrimSpace(wt.WorktreeBranch) + } + } + } + return "" +} + // buildTaskBranchList walks sessions × their repositories and emits one -// TaskBranchInfo per (session, repository) that doesn't already have a PR -// watch. Multi-repo: previously dedup was keyed by sessionID, which silently -// dropped non-primary repos as soon as the primary one got a watch. +// TaskBranchInfo per (task, repository, branch) that doesn't already have a +// PR watch. Session identity does not own canonical PR-watch identity, so +// multiple sessions resolving to the same (task, repository, branch) target +// collapse to a single emission here — one lookup per canonical target per +// cycle, regardless of how many sessions are active on it. // -// TaskID is redirected via resolveEffectivePushTaskID before the caller +// TaskID is redirected via resolveEffectivePushTaskID (to the workspace-group +// owner task) before the dedup key is computed and before the caller // (poller.reconcileWatches) creates the watch, same as ensureSessionPRWatch: // this is the third and last producer that would otherwise write a -// member-attributed watch for a shared-worktree subtask. +// member-attributed watch for a shared-worktree subtask. Deduping on the +// pre-redirect sess.TaskID would let a member and its owner both emit a +// TaskBranchInfo for the same underlying repository/branch, so the redirect +// must run first — canonical identity is (effective task, repository, +// branch), never session or pre-redirect task. func (s *Service) buildTaskBranchList(ctx context.Context, store repoStore) ([]github.TaskBranchInfo, error) { sessions, err := store.ListSessionsWithBranches(ctx) if err != nil { @@ -1475,7 +1574,8 @@ func (s *Service) buildTaskBranchList(ctx context.Context, store repoStore) ([]g } // Batch fetch existing watches to avoid N+1 queries. - watchedKeys := s.buildWatchedSessionRepoSet(ctx) + watchedKeys := s.buildWatchedTaskRepoBranchSet(ctx) + emitted := make(map[string]bool) var result []github.TaskBranchInfo seenSessions := make(map[string]struct{}) @@ -1488,10 +1588,12 @@ func (s *Service) buildTaskBranchList(ctx context.Context, store repoStore) ([]g // once from its per-repository targets, as watch refresh does. targets := s.resolveSessionWatchTargets(ctx, sess.TaskID, sess.SessionID, "") for _, t := range targets { - if watchedKeys[watchedSessionRepoKey(sess.SessionID, t.RepositoryID, t.Branch)] { + effectiveTaskID := s.resolveEffectivePushTaskIDForSession(ctx, sess.SessionID, sess.TaskID, t.RepositoryID) + key := watchedTaskRepoBranchKey(effectiveTaskID, t.RepositoryID, t.Branch) + if watchedKeys[key] || emitted[key] { continue } - effectiveTaskID := s.resolveEffectivePushTaskIDForSession(ctx, sess.SessionID, sess.TaskID, t.RepositoryID) + emitted[key] = true result = append(result, github.TaskBranchInfo{ WorkspaceID: sess.WorkspaceID, TaskID: effectiveTaskID, @@ -1506,12 +1608,12 @@ func (s *Service) buildTaskBranchList(ctx context.Context, store repoStore) ([]g return result, nil } -// buildWatchedSessionRepoSet returns the set of (session_id, repository_id, -// branch) triples that already have a PR watch row. Multi-branch tasks -// hold one watch per (session, repo, branch); dedup keyed on -// (session, repo) alone would silently drop secondary branches whenever -// the primary already had a watch. -func (s *Service) buildWatchedSessionRepoSet(ctx context.Context) map[string]bool { +// buildWatchedTaskRepoBranchSet returns the set of (task_id, repository_id, +// branch) triples that already have a PR watch row. Canonical searching +// watches are owned by task/repository/branch, not session_id, so dedup here +// must key on the same triple to avoid re-creating a watch that another +// session already established for the same task/repo/branch. +func (s *Service) buildWatchedTaskRepoBranchSet(ctx context.Context) map[string]bool { watches, err := s.githubService.ListActivePRWatches(ctx) if err != nil { s.logger.Debug("failed to list PR watches for reconciliation", zap.Error(err)) @@ -1519,14 +1621,14 @@ func (s *Service) buildWatchedSessionRepoSet(ctx context.Context) map[string]boo } set := make(map[string]bool, len(watches)) for _, w := range watches { - set[watchedSessionRepoKey(w.SessionID, w.RepositoryID, w.Branch)] = true + set[watchedTaskRepoBranchKey(w.TaskID, w.RepositoryID, w.Branch)] = true } return set } -// watchedSessionRepoKey builds the per-(session, repo, branch) dedup key. -func watchedSessionRepoKey(sessionID, repositoryID, branch string) string { - return sessionID + "|" + repositoryID + "|" + branch +// watchedTaskRepoBranchKey builds the per-(task, repo, branch) dedup key. +func watchedTaskRepoBranchKey(taskID, repositoryID, branch string) string { + return taskID + "|" + repositoryID + "|" + branch } // subscribeGitHubEvents subscribes to GitHub-related events on the event bus. diff --git a/apps/backend/internal/orchestrator/event_handlers_github_test.go b/apps/backend/internal/orchestrator/event_handlers_github_test.go index d460f6ccf9..06c29fb345 100644 --- a/apps/backend/internal/orchestrator/event_handlers_github_test.go +++ b/apps/backend/internal/orchestrator/event_handlers_github_test.go @@ -1320,6 +1320,157 @@ func TestListTasksNeedingPRWatch(t *testing.T) { t.Errorf("secondary entry mis-resolved: %+v", back) } }) + + // Task 02: canonical PR-watch identity is task/repository/branch, not + // session_id. Two active sessions resolving to the SAME (task, + // repository, branch) target must collapse to a single TaskBranchInfo + // emission, so the poller performs one GitHub lookup per canonical + // target per cycle regardless of how many sessions are attached to it. + t.Run("dedupes multiple sessions on the same task/repo/branch", func(t *testing.T) { + testRepo := setupTestRepo(t) + seedSession(t, testRepo, "t1", "s1", "step1") + + // Second session on the SAME task, sharing the task's one + // environment (task_environments.task_id is UNIQUE) and worktree. + session2 := &models.TaskSession{ + ID: "s2", TaskID: "t1", + State: models.TaskSessionStateRunning, StartedAt: now, UpdatedAt: now, + } + if err := testRepo.CreateTaskSession(ctx, session2); err != nil { + t.Fatalf("failed to create session s2: %v", err) + } + + repoObj := &models.Repository{ID: "repo1", WorkspaceID: "ws1", Name: "myrepo", SourceType: "provider", Provider: "github", ProviderOwner: "myorg", ProviderName: "myrepo", CreatedAt: now, UpdatedAt: now} + if err := testRepo.CreateRepository(ctx, repoObj); err != nil { + t.Fatalf("create repo: %v", err) + } + if err := testRepo.CreateTaskRepository(ctx, &models.TaskRepository{ID: "tr-1", TaskID: "t1", RepositoryID: "repo1", Position: 0, CreatedAt: now, UpdatedAt: now}); err != nil { + t.Fatalf("link repo: %v", err) + } + if err := testRepo.CreateTaskEnvironment(ctx, &models.TaskEnvironment{ + ID: "env-t1", TaskID: "t1", ExecutorType: "worktree", WorkspacePath: "/tmp", Status: models.TaskEnvironmentStatusReady, + Repos: []*models.TaskEnvironmentRepo{ + {ID: "wt-1", WorktreeID: "wtree-1", RepositoryID: "repo1", WorktreeBranch: "feature-a", CreatedAt: now}, + }, + }); err != nil { + t.Fatalf("create environment: %v", err) + } + for _, sessionID := range []string{"s1", "s2"} { + session, err := testRepo.GetTaskSession(ctx, sessionID) + if err != nil { + t.Fatalf("load session %s: %v", sessionID, err) + } + session.TaskEnvironmentID = "env-t1" + if err := testRepo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("link session %s to environment: %v", sessionID, err) + } + } + + svc := createTestService(testRepo, newMockStepGetter(), newMockTaskRepo()) + svc.SetGitHubService(&mockGitHubService{prWatch: nil}) + + tasks, err := svc.ListTasksNeedingPRWatch(ctx) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + matches := 0 + for _, ti := range tasks { + if ti.TaskID == "t1" && ti.RepositoryID == "repo1" && ti.Branch == "feature-a" { + matches++ + } + } + if matches != 1 { + t.Errorf("expected exactly 1 entry for the shared (task, repo, branch) target, got %d", matches) + } + }) +} + +// TestResolveBranchForRepository verifies the Task 02 fix: branch resolution +// is scoped to a specific (task, repository) pair, independent of session +// identity. It replaces a resolver that always used the task's PRIMARY +// repository's checkout_branch regardless of which repository the caller +// asked about — a real bug that could corrupt a secondary repo's +// still-searching PR watch with the primary repo's branch name. +func TestResolveBranchForRepository(t *testing.T) { + ctx := context.Background() + now := time.Now().UTC() + + t.Run("uses checkout_branch for the requested repository, not the primary", func(t *testing.T) { + testRepo := setupTestRepo(t) + seedSession(t, testRepo, "t1", "s1", "step1") + + for _, r := range []*models.Repository{ + {ID: "repo-front", WorkspaceID: "ws1", Name: "frontend", SourceType: "provider", Provider: "github", ProviderOwner: "myorg", ProviderName: "frontend", CreatedAt: now, UpdatedAt: now}, + {ID: "repo-back", WorkspaceID: "ws1", Name: "backend", SourceType: "provider", Provider: "github", ProviderOwner: "myorg", ProviderName: "backend", CreatedAt: now, UpdatedAt: now}, + } { + if err := testRepo.CreateRepository(ctx, r); err != nil { + t.Fatalf("create repo %s: %v", r.ID, err) + } + } + links := []*models.TaskRepository{ + {ID: "tr-front", TaskID: "t1", RepositoryID: "repo-front", Position: 0, CheckoutBranch: "main-front", CreatedAt: now, UpdatedAt: now}, + {ID: "tr-back", TaskID: "t1", RepositoryID: "repo-back", Position: 1, CheckoutBranch: "main-back", CreatedAt: now, UpdatedAt: now}, + } + for _, l := range links { + if err := testRepo.CreateTaskRepository(ctx, l); err != nil { + t.Fatalf("link %s: %v", l.ID, err) + } + } + + svc := createTestService(testRepo, newMockStepGetter(), newMockTaskRepo()) + + if got := svc.ResolveBranchForRepository(ctx, "t1", "repo-back"); got != "main-back" { + t.Errorf("expected secondary repo branch %q, got %q", "main-back", got) + } + if got := svc.ResolveBranchForRepository(ctx, "t1", "repo-front"); got != "main-front" { + t.Errorf("expected primary repo branch %q, got %q", "main-front", got) + } + }) + + t.Run("falls back to an active session worktree branch when checkout_branch is unset", func(t *testing.T) { + testRepo := setupTestRepo(t) + seedSession(t, testRepo, "t1", "s1", "step1") + + repoObj := &models.Repository{ID: "repo1", WorkspaceID: "ws1", Name: "myrepo", SourceType: "provider", Provider: "github", ProviderOwner: "myorg", ProviderName: "myrepo", CreatedAt: now, UpdatedAt: now} + if err := testRepo.CreateRepository(ctx, repoObj); err != nil { + t.Fatalf("create repo: %v", err) + } + link := &models.TaskRepository{ID: "tr-1", TaskID: "t1", RepositoryID: "repo1", Position: 0, CreatedAt: now, UpdatedAt: now} + if err := testRepo.CreateTaskRepository(ctx, link); err != nil { + t.Fatalf("link repo: %v", err) + } + if err := testRepo.CreateTaskEnvironment(ctx, &models.TaskEnvironment{ + ID: "env-s1", TaskID: "t1", ExecutorType: "worktree", WorkspacePath: "/tmp", Status: models.TaskEnvironmentStatusReady, + Repos: []*models.TaskEnvironmentRepo{ + {ID: "wt-1", WorktreeID: "wtree-1", RepositoryID: "repo1", WorktreeBranch: "session-branch", CreatedAt: now}, + }, + }); err != nil { + t.Fatalf("create environment: %v", err) + } + session, err := testRepo.GetTaskSession(ctx, "s1") + if err != nil { + t.Fatalf("load session: %v", err) + } + session.TaskEnvironmentID = "env-s1" + if err := testRepo.UpdateTaskSession(ctx, session); err != nil { + t.Fatalf("link session to environment: %v", err) + } + + svc := createTestService(testRepo, newMockStepGetter(), newMockTaskRepo()) + if got := svc.ResolveBranchForRepository(ctx, "t1", "repo1"); got != "session-branch" { + t.Errorf("expected fallback branch %q, got %q", "session-branch", got) + } + }) + + t.Run("returns empty when nothing can be resolved", func(t *testing.T) { + testRepo := setupTestRepo(t) + seedSession(t, testRepo, "t1", "s1", "step1") + svc := createTestService(testRepo, newMockStepGetter(), newMockTaskRepo()) + if got := svc.ResolveBranchForRepository(ctx, "t1", "missing-repo"); got != "" { + t.Errorf("expected empty branch, got %q", got) + } + }) } // TestEnsureSessionPRWatch_MultiRepo guards the same multi-repo gap on the diff --git a/apps/backend/internal/orchestrator/messagequeue/count_pending_test.go b/apps/backend/internal/orchestrator/messagequeue/count_pending_test.go index ebb4ca45e7..985b884c66 100644 --- a/apps/backend/internal/orchestrator/messagequeue/count_pending_test.go +++ b/apps/backend/internal/orchestrator/messagequeue/count_pending_test.go @@ -2,12 +2,64 @@ package messagequeue import ( "context" + "expvar" "reflect" "testing" "github.com/kandev/kandev/internal/common/logger" ) +func TestMessageQueueDepthMetricPublished(t *testing.T) { + if expvar.Get("message_queue_depth") == nil { + t.Fatal("message_queue_depth expvar is not published") + } +} + +func TestMessageQueueDepthGaugeCountsVisibleEntries(t *testing.T) { + repo := NewMemoryRepository() + provider := repo.(queueDepthCounter) + registerQueueDepthProvider(provider) + ctx := context.Background() + for _, content := range []string{"first", "second"} { + message := &QueuedMessage{SessionID: "session-depth", TaskID: "task-depth", Content: content, QueuedBy: QueuedByUser} + if err := repo.Insert(ctx, message, 0); err != nil { + t.Fatalf("Insert(%s): %v", content, err) + } + } + if got := currentMessageQueueDepth(); got != 2 { + t.Fatalf("message queue depth = %v, want 2", got) + } +} + +func TestSQLiteRepositoryCountQueueDepthSkipsReservedRowsWithoutDecodingAttachments(t *testing.T) { + ctx := context.Background() + repo := newTestSQLiteRepo(t).(*sqliteRepository) + message := &QueuedMessage{ + SessionID: "session-depth", + TaskID: "task-depth", + Content: "reserved", + QueuedBy: QueuedByWorkflow, + Metadata: map[string]interface{}{MetadataLifecycleDurable: true}, + } + if err := repo.Insert(ctx, message, 0); err != nil { + t.Fatalf("insert: %v", err) + } + if _, err := repo.ReserveHead(ctx, message.SessionID); err != nil { + t.Fatalf("reserve: %v", err) + } + if _, err := repo.db.ExecContext(ctx, `UPDATE queued_messages SET attachments_json = 'not-json' WHERE id = ?`, message.ID); err != nil { + t.Fatalf("corrupt attachments: %v", err) + } + + depth, err := repo.CountQueueDepth(ctx) + if err != nil { + t.Fatalf("CountQueueDepth: %v", err) + } + if depth != 0 { + t.Fatalf("queue depth = %d, want 0", depth) + } +} + // repositoriesUnderTest returns one SQLite and one memory Repository so the // pending-count contract is proven on both storage backends. func repositoriesUnderTest(t *testing.T) map[string]Repository { diff --git a/apps/backend/internal/orchestrator/messagequeue/metrics_vars.go b/apps/backend/internal/orchestrator/messagequeue/metrics_vars.go new file mode 100644 index 0000000000..0d5ec7636a --- /dev/null +++ b/apps/backend/internal/orchestrator/messagequeue/metrics_vars.go @@ -0,0 +1,43 @@ +package messagequeue + +import ( + "context" + "expvar" + "sync" + "time" +) + +type queueDepthCounter interface { + CountQueueDepth(context.Context) (int, error) +} + +var queueDepthSource struct { + sync.RWMutex + provider queueDepthCounter +} + +func init() { + expvar.Publish("message_queue_depth", expvar.Func(currentMessageQueueDepth)) +} + +func registerQueueDepthProvider(provider queueDepthCounter) { + queueDepthSource.Lock() + queueDepthSource.provider = provider + queueDepthSource.Unlock() +} + +func currentMessageQueueDepth() interface{} { + queueDepthSource.RLock() + provider := queueDepthSource.provider + queueDepthSource.RUnlock() + if provider == nil { + return -1 + } + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + defer cancel() + depth, err := provider.CountQueueDepth(ctx) + if err != nil { + return -1 + } + return depth +} diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_memory.go b/apps/backend/internal/orchestrator/messagequeue/repository_memory.go index 29f2d22a9d..abac190441 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_memory.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_memory.go @@ -252,6 +252,20 @@ func (r *memoryRepository) CountPendingByTaskIDs(_ context.Context, taskIDs []st return counts, nil } +func (r *memoryRepository) CountQueueDepth(_ context.Context) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + count := 0 + for _, entries := range r.entries { + for _, entry := range entries { + if !entry.IsReservedInFlight() { + count++ + } + } + } + return count, nil +} + // Insert appends a new entry at the tail of the session's FIFO queue. func (r *memoryRepository) Insert(_ context.Context, msg *QueuedMessage, maxPerSession int) error { r.mu.Lock() diff --git a/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go b/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go index 3aae5c2ced..0b6adc6331 100644 --- a/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go +++ b/apps/backend/internal/orchestrator/messagequeue/repository_sqlite.go @@ -2474,6 +2474,34 @@ func (r *sqliteRepository) CountBySession(ctx context.Context, sessionID string) return n, err } +func (r *sqliteRepository) CountQueueDepth(ctx context.Context) (int, error) { + rows, err := r.ro.QueryxContext(ctx, `SELECT metadata_json FROM queued_messages`) + if err != nil { + return 0, err + } + defer func() { _ = rows.Close() }() + count := 0 + for rows.Next() { + var metadataJSON string + if err := rows.Scan(&metadataJSON); err != nil { + return 0, err + } + metadata := make(map[string]interface{}) + if metadataJSON != "" && metadataJSON != "{}" { + if err := json.Unmarshal([]byte(metadataJSON), &metadata); err != nil { + return 0, fmt.Errorf("unmarshal queue depth metadata: %w", err) + } + } + if reserved, _ := metadata[MetadataLifecycleReserved].(bool); !reserved { + count++ + } + } + if err := rows.Err(); err != nil { + return 0, err + } + return count, nil +} + // CountPendingByTaskIDs counts pending entries per task, excluding durable // lifecycle rows reserved in flight (filtered in Go via IsReservedInFlight). func (r *sqliteRepository) CountPendingByTaskIDs(ctx context.Context, taskIDs []string) (map[string]int, error) { diff --git a/apps/backend/internal/orchestrator/messagequeue/service.go b/apps/backend/internal/orchestrator/messagequeue/service.go index 06d08c93d3..c03ae80bdb 100644 --- a/apps/backend/internal/orchestrator/messagequeue/service.go +++ b/apps/backend/internal/orchestrator/messagequeue/service.go @@ -138,6 +138,9 @@ func NewService(repo Repository, maxPerSession int, log *logger.Logger) *Service service.SetMaxPerSession(maxPerSession) service.mergeEnabled.Store(true) service.SetAutoMergePolicy(true, 0) + if provider, ok := repo.(queueDepthCounter); ok { + registerQueueDepthProvider(provider) + } return service } diff --git a/apps/backend/internal/persistence/storeconformance/owner_actions.go b/apps/backend/internal/persistence/storeconformance/owner_actions.go index 6b68a2c08d..87678a3ff1 100644 --- a/apps/backend/internal/persistence/storeconformance/owner_actions.go +++ b/apps/backend/internal/persistence/storeconformance/owner_actions.go @@ -16,6 +16,7 @@ import ( "github.com/kandev/kandev/internal/automation" "github.com/kandev/kandev/internal/azuredevops" "github.com/kandev/kandev/internal/canvas" + "github.com/kandev/kandev/internal/common/authcircuit" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/delivery" "github.com/kandev/kandev/internal/github" @@ -2985,7 +2986,7 @@ func workflowSyncAction() apiAction { if err != nil { return nil, err } - if err := store.(*workflowsync.Store).RecordSyncStatus(s.Context, id, enabled, "", nil, "hash", time.Now().UTC()); err != nil { + if err := store.(*workflowsync.Store).RecordSyncStatus(s.Context, id, enabled, "", nil, "hash", time.Now().UTC(), authcircuit.State{}); err != nil { return nil, err } return readConfig(s, store.(*workflowsync.Store), id) diff --git a/apps/backend/internal/system/database/metrics_vars.go b/apps/backend/internal/system/database/metrics_vars.go new file mode 100644 index 0000000000..1b25432ec5 --- /dev/null +++ b/apps/backend/internal/system/database/metrics_vars.go @@ -0,0 +1,21 @@ +package database + +import "expvar" + +var ( + databaseSizeBytes = expvar.NewInt("database_size_bytes") + databaseWALSizeBytes = expvar.NewInt("database_wal_size_bytes") + messageContentBytes = expvar.NewInt("task_message_content_bytes") + messageMetadataBytes = expvar.NewInt("task_message_metadata_bytes") + messagePayloadCompressedBytes = expvar.NewInt("task_message_payload_compressed_bytes") + gitSnapshotBytes = expvar.NewInt("task_git_snapshot_bytes") +) + +func recordStorageMetrics(stats Stats) { + databaseSizeBytes.Set(stats.SizeBytes) + databaseWALSizeBytes.Set(stats.WALSizeBytes) + messageContentBytes.Set(stats.MessageContentBytes) + messageMetadataBytes.Set(stats.MessageMetadataBytes) + messagePayloadCompressedBytes.Set(stats.MessagePayloadBytes) + gitSnapshotBytes.Set(stats.GitSnapshotBytes) +} diff --git a/apps/backend/internal/system/database/stats.go b/apps/backend/internal/system/database/stats.go index dfb7f7de14..379ed73fdc 100644 --- a/apps/backend/internal/system/database/stats.go +++ b/apps/backend/internal/system/database/stats.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "sort" + "strings" "time" "github.com/gin-gonic/gin" @@ -35,13 +36,17 @@ const ( // exists. Serialising a zero time.Time as "0001-01-01T00:00:00Z" would // defeat the frontend's "Never" fallback in database-stats-card.tsx. type Stats struct { - Driver string `json:"driver"` - Path string `json:"path"` - BackupDirectory string `json:"backup_directory"` - SizeBytes int64 `json:"size_bytes"` - WALSizeBytes int64 `json:"wal_size_bytes"` - SchemaVersion string `json:"schema_version"` - LastBackupAt *time.Time `json:"last_backup_at"` + Driver string `json:"driver"` + Path string `json:"path"` + BackupDirectory string `json:"backup_directory"` + SizeBytes int64 `json:"size_bytes"` + WALSizeBytes int64 `json:"wal_size_bytes"` + MessageContentBytes int64 `json:"message_content_bytes"` + MessageMetadataBytes int64 `json:"message_metadata_bytes"` + MessagePayloadBytes int64 `json:"message_payload_bytes"` + GitSnapshotBytes int64 `json:"git_snapshot_bytes"` + SchemaVersion string `json:"schema_version"` + LastBackupAt *time.Time `json:"last_backup_at"` } // ResetDirs lists the on-disk directories factory-reset wipes. The Service @@ -136,6 +141,15 @@ func (s *Service) Stats() (Stats, error) { return Stats{}, err } out.SchemaVersion = version + + storage, err := readLogicalStorageStats(s.pool.Reader()) + if err != nil { + return Stats{}, err + } + out.MessageContentBytes = storage.messageContent + out.MessageMetadataBytes = storage.messageMetadata + out.MessagePayloadBytes = storage.messagePayload + out.GitSnapshotBytes = storage.gitSnapshot } if driver == databaseDriverSQLite { @@ -147,9 +161,53 @@ func (s *Service) Stats() (Stats, error) { out.LastBackupAt = &last } } + recordStorageMetrics(out) return out, nil } +type logicalStorageStats struct { + messageContent int64 + messageMetadata int64 + messagePayload int64 + gitSnapshot int64 +} + +// readLogicalStorageStats uses portable LENGTH aggregates as a deterministic +// fallback when SQLite dbstat or PostgreSQL relation-size extensions are not +// available. A missing table during early boot or a partial test fixture is +// reported as zero for that category; database/WAL sizes remain independent. +func readLogicalStorageStats(db *sqlx.DB) (logicalStorageStats, error) { + var out logicalStorageStats + queries := []struct { + query string + dest *int64 + }{ + {`SELECT COALESCE(SUM(LENGTH(content)), 0) FROM task_session_messages`, &out.messageContent}, + {`SELECT COALESCE(SUM(LENGTH(metadata)), 0) FROM task_session_messages`, &out.messageMetadata}, + {`SELECT COALESCE(SUM(LENGTH(compressed_content)), 0) FROM task_message_payloads`, &out.messagePayload}, + {`SELECT COALESCE(SUM(LENGTH(files) + LENGTH(metadata)), 0) FROM task_session_git_snapshots`, &out.gitSnapshot}, + } + for _, metric := range queries { + if err := db.QueryRow(metric.query).Scan(metric.dest); err != nil { + if isMissingLogicalStorageTableError(err) { + continue + } + return logicalStorageStats{}, fmt.Errorf("read logical storage stats: %w", err) + } + } + return out, nil +} + +func isMissingLogicalStorageTableError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "no such table") || + strings.Contains(msg, "does not exist") || + strings.Contains(msg, "undefined_table") +} + func (s *Service) databaseDriver() string { if s.pool == nil || s.pool.Writer() == nil { return databaseDriverSQLite diff --git a/apps/backend/internal/system/database/stats_test.go b/apps/backend/internal/system/database/stats_test.go index f787bfa97b..4adeb81102 100644 --- a/apps/backend/internal/system/database/stats_test.go +++ b/apps/backend/internal/system/database/stats_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "database/sql/driver" "encoding/json" + "expvar" "fmt" "io" "os" @@ -62,6 +63,11 @@ func (fakePostgresStatsConn) QueryContext( return nil, fmt.Errorf("unexpected args for schema version: %#v", args) } return newFakeRows([]string{"value"}, []driver.Value{"v0.99.0"}), nil + case "SELECT COALESCE(SUM(LENGTH(content)), 0) FROM task_session_messages", + "SELECT COALESCE(SUM(LENGTH(metadata)), 0) FROM task_session_messages", + "SELECT COALESCE(SUM(LENGTH(compressed_content)), 0) FROM task_message_payloads", + "SELECT COALESCE(SUM(LENGTH(files) + LENGTH(metadata)), 0) FROM task_session_git_snapshots": + return newFakeRows([]string{"sum"}, []driver.Value{int64(0)}), nil default: if strings.HasPrefix(normalized, "PRAGMA ") { return nil, fmt.Errorf(`ERROR: syntax error at or near "PRAGMA" (SQLSTATE 42601)`) @@ -268,6 +274,51 @@ func TestStats_ReturnsPathSizeAndSchemaVersion(t *testing.T) { } } +func TestStatsReportsLogicalStorageAndDatabaseGauges(t *testing.T) { + svc, _, _, _ := newTestService(t) + for _, statement := range []string{ + `CREATE TABLE task_session_messages (content TEXT, metadata TEXT)`, + `CREATE TABLE task_message_payloads (compressed_content BLOB)`, + `CREATE TABLE task_session_git_snapshots (files TEXT, metadata TEXT)`, + `INSERT INTO task_session_messages VALUES ('hello', '{"a":1}')`, + `INSERT INTO task_message_payloads VALUES (x'01020304')`, + `INSERT INTO task_session_git_snapshots VALUES ('{"f":1}', '{"m":2}')`, + } { + if _, err := svc.pool.Writer().Exec(statement); err != nil { + t.Fatalf("seed storage metrics with %q: %v", statement, err) + } + } + + stats, err := svc.Stats() + if err != nil { + t.Fatalf("Stats: %v", err) + } + payload := statsPayload(t, stats) + for field, want := range map[string]float64{ + "message_content_bytes": 5, + "message_metadata_bytes": 7, + "message_payload_bytes": 4, + "git_snapshot_bytes": 14, + } { + if got := payload[field]; got != want { + t.Errorf("%s = %v, want %.0f", field, got, want) + } + } + + for _, name := range []string{ + "database_size_bytes", + "database_wal_size_bytes", + "task_message_content_bytes", + "task_message_metadata_bytes", + "task_message_payload_compressed_bytes", + "task_git_snapshot_bytes", + } { + if expvar.Get(name) == nil { + t.Errorf("expvar %q is not published", name) + } + } +} + func TestStats_ReturnsConfiguredSQLiteBackupDirectory(t *testing.T) { svc, _, _, dataDir := newTestService(t) svc.databasePath = filepath.Join(dataDir, "nested", "custom.db") diff --git a/apps/backend/internal/task/handlers/message_handlers.go b/apps/backend/internal/task/handlers/message_handlers.go index e2de2098aa..92a56a75c1 100644 --- a/apps/backend/internal/task/handlers/message_handlers.go +++ b/apps/backend/internal/task/handlers/message_handlers.go @@ -304,8 +304,22 @@ func (h *MessageHandlers) httpGetShellOutput(c *gin.Context) { c.JSON(http.StatusGone, gin.H{"code": "tool_payload_removed", "message_id": message.ID, "removed_at": marker["removed_at"], "summary": message.Content}) return } + if message.PayloadDigest != "" { + // Large output was externalized at write time (see + // externalizeMessagePayload) and message.Metadata currently holds + // only the small has_output/stdout_bytes/... summary + // ProjectMessageMetadata produces - not the full body. This route is + // the explicit, authorized single-message load that resolves it + // back before extraction. + if err := h.service.RehydrateMessagePayload(c.Request.Context(), message); err != nil { + h.logger.Error("failed to rehydrate message payload", + zap.String("message_id", message.ID), zap.Error(err)) + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to load shell output"}) + return + } + } output, ok := models.ExtractShellExecOutput(message.Metadata) - if !ok || message.TaskSessionID != c.Param("id") { + if !ok { c.JSON(http.StatusNotFound, gin.H{"error": "message not found"}) return } @@ -331,7 +345,6 @@ type listMessagesParams struct { sort string authorType string limit int - paginated bool } const ( @@ -386,6 +399,13 @@ func (h *MessageHandlers) parseListMessageParams(c *gin.Context) (listMessagesPa } limit = parsed } + if !limitProvided && before == "" && after == "" && sort == "" { + // No pagination parameters at all: default to a bounded first page + // instead of an unbounded full-session read. See + // service.DefaultMessagesPageSize and wsListMessages's matching + // default for the WebSocket path. + limit = service.DefaultMessagesPageSize + } return listMessagesParams{ before: before, after: after, @@ -393,7 +413,6 @@ func (h *MessageHandlers) parseListMessageParams(c *gin.Context) (listMessagesPa sort: sort, authorType: authorType, limit: limit, - paginated: limitProvided || before != "" || after != "" || aroundProvided || sort != "" || authorTypeProvided, }, true } @@ -402,15 +421,7 @@ func (h *MessageHandlers) fetchMessages( sessionID string, params listMessagesParams, ) (dto.ListMessagesResponse, error) { - if params.paginated { - return h.fetchMessagesPaginated(ctx, sessionID, params) - } - messages, err := h.service.ListMessages(ctx, sessionID) - if err != nil { - return dto.ListMessagesResponse{}, err - } - result := messagesToAPI(messages) - return dto.ListMessagesResponse{Messages: result, Total: len(result)}, nil + return h.fetchMessagesPaginated(ctx, sessionID, params) } // fetchMessagesPaginated loads a filtered or around-window message page. @@ -1915,6 +1926,11 @@ func (h *MessageHandlers) wsListMessages(ctx context.Context, msg *ws.Message) ( if req.Sort != "" && req.Sort != messageSortAsc && req.Sort != messageSortDesc { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "sort must be asc or desc", nil) } + if req.Limit <= 0 && req.Before == "" && req.After == "" && req.Sort == "" { + // No pagination parameters at all: default to a bounded first page, + // matching httpListMessages's parseListMessageParams default. + req.Limit = service.DefaultMessagesPageSize + } messages, hasMore, err := h.service.ListMessagesPaginated(ctx, service.ListMessagesRequest{ TaskSessionID: req.TaskSessionID, diff --git a/apps/backend/internal/task/handlers/message_list_handlers_test.go b/apps/backend/internal/task/handlers/message_list_handlers_test.go index a856ae2f65..da43b10d49 100644 --- a/apps/backend/internal/task/handlers/message_list_handlers_test.go +++ b/apps/backend/internal/task/handlers/message_list_handlers_test.go @@ -105,10 +105,10 @@ func TestHTTPListMessagesRequiresSessionID(t *testing.T) { require.Zero(t, repo.listCalls) } -// TestHTTPListMessagesUsesTheUnpaginatedPathByDefault pins the mode switch: -// with no cursor, sort or limit the handler must take the plain list, whose -// response carries no cursor. -func TestHTTPListMessagesUsesTheUnpaginatedPathByDefault(t *testing.T) { +// TestHTTPListMessagesDefaultsToABoundedFirstPage pins the mode switch: with +// no cursor, sort or limit the handler must still take the bounded paginated +// path (DefaultMessagesPageSize), never the unbounded full-session read. +func TestHTTPListMessagesDefaultsToABoundedFirstPage(t *testing.T) { repo := &messageListRepo{} h := newMessageListHandlers(t, repo) c, rec := messageRequestAs(t, "", "/api/v1/task-sessions/sess-b/messages", "sess-b") @@ -118,11 +118,9 @@ func TestHTTPListMessagesUsesTheUnpaginatedPathByDefault(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) var resp dto.ListMessagesResponse require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - require.Equal(t, 2, resp.Total) - require.Equal(t, 1, repo.listCalls) - require.Empty(t, repo.paginatedOptions, "the paginated query must not run") - require.Empty(t, resp.Cursor) - require.False(t, resp.HasMore) + require.Zero(t, repo.listCalls, "the unbounded query must not run") + require.Len(t, repo.paginatedOptions, 1) + require.Equal(t, service.DefaultMessagesPageSize, repo.paginatedOptions[0].Limit) } // TestHTTPListMessagesSwitchesToPaginationOnAnyCursorParam covers each param @@ -262,7 +260,7 @@ func TestHTTPListMessagesPassesAuthorFilterAndAround(t *testing.T) { } func TestHTTPListMessagesSurfacesRepositoryFailure(t *testing.T) { - repo := &messageListRepo{listErr: errors.New("database unavailable")} + repo := &messageListRepo{paginatedErr: errors.New("database unavailable")} h := newMessageListHandlers(t, repo) c, rec := messageRequestAs(t, "", "/api/v1/task-sessions/sess-b/messages", "sess-b") @@ -367,6 +365,23 @@ func TestWSListMessagesReturnsPageAndCursor(t *testing.T) { require.Equal(t, "asc", repo.paginatedOptions[0].Sort) } +// TestWSListMessagesDefaultsToABoundedFirstPage mirrors +// TestHTTPListMessagesDefaultsToABoundedFirstPage for the WebSocket path: a +// request with no limit/before/after/sort must still bound the query. +func TestWSListMessagesDefaultsToABoundedFirstPage(t *testing.T) { + repo := &messageListRepo{} + h := newMessageListHandlers(t, repo) + + resp, err := h.wsListMessages(context.Background(), + wsWorkflowRequest(t, ws.ActionMessageList, map[string]any{"session_id": "sess-b"})) + + require.NoError(t, err) + var page dto.ListMessagesResponse + wsWorkflowResponse(t, resp, &page) + require.Len(t, repo.paginatedOptions, 1) + require.Equal(t, service.DefaultMessagesPageSize, repo.paginatedOptions[0].Limit) +} + func TestWSListMessagesSurfacesRepositoryFailure(t *testing.T) { repo := &messageListRepo{paginatedErr: errors.New("database unavailable")} h := newMessageListHandlers(t, repo) diff --git a/apps/backend/internal/task/handlers/process_handlers_test.go b/apps/backend/internal/task/handlers/process_handlers_test.go index 5ff24a84b0..40417f1194 100644 --- a/apps/backend/internal/task/handlers/process_handlers_test.go +++ b/apps/backend/internal/task/handlers/process_handlers_test.go @@ -274,6 +274,12 @@ func (m *mockRepository) GetMessage(ctx context.Context, id string) (*models.Mes return nil, nil } +// RehydrateMessagePayload is a no-op for the mock repository: none of its +// callers construct messages with an externalized PayloadDigest. +func (m *mockRepository) RehydrateMessagePayload(ctx context.Context, message *models.Message) error { + return nil +} + // GetMessageWithPromptIndex returns the message for id with its derived prompt index, mirroring the repository contract. func (m *mockRepository) GetMessageWithPromptIndex(ctx context.Context, id string) (*models.Message, error) { return nil, nil diff --git a/apps/backend/internal/task/models/git.go b/apps/backend/internal/task/models/git.go index 988980a8f1..0a6577e7ec 100644 --- a/apps/backend/internal/task/models/git.go +++ b/apps/backend/internal/task/models/git.go @@ -36,6 +36,16 @@ type GitSnapshot struct { TriggeredBy string `json:"triggered_by"` Metadata map[string]interface{} `json:"metadata,omitempty"` CreatedAt time.Time `json:"created_at"` + // ContentDigest is a SHA-256 digest over the fields that define two + // snapshots as equivalent (branch, remote branch, head/base commit, + // ahead/behind counts, and the serialized files diff), independent of + // session/triggered_by/created_at. CreateGitSnapshot uses it to skip + // inserting a duplicate of the immediately preceding snapshot for the + // same session (repeated no-op polling), and it identifies historical + // duplicate rows as retention candidates for the maintenance command. + // Internal repository bookkeeping only - not serialized to API/WS + // clients. + ContentDigest string `json:"-"` } // SessionCommit represents a commit made during a task session diff --git a/apps/backend/internal/task/models/message_shell_output.go b/apps/backend/internal/task/models/message_shell_output.go index e6fd4c7111..b70fe8f59a 100644 --- a/apps/backend/internal/task/models/message_shell_output.go +++ b/apps/backend/internal/task/models/message_shell_output.go @@ -51,6 +51,42 @@ func ExtractShellExecOutput(metadata map[string]any) (ShellExecOutputSnapshot, b return shellOutputFromMetadata(metadata) } +// RehydrateShellOutput restores a full shell command output snapshot into +// metadata previously projected by ProjectMessageMetadata (or the equivalent +// persist-time projection CreateMessage/UpdateMessage apply before storing +// externalized large payloads). Read-time metadata is always the +// map[string]any shape (it comes from json.Unmarshal), unlike +// ProjectMessageMetadata's write path which must also accept the +// *streams.NormalizedPayload shape emitted live by the agent adapter - so +// only that shape needs handling here. Returns metadata unchanged (ok=false) +// if it doesn't have the normalized shell shape ProjectMessageMetadata +// produces, so a caller can distinguish "nothing to rehydrate" from success. +func RehydrateShellOutput(metadata map[string]any, full ShellExecOutputSnapshot) (map[string]any, bool) { + normalized, ok := metadata["normalized"].(map[string]any) + if !ok { + return metadata, false + } + shell, ok := normalized["shell_exec"].(map[string]any) + if !ok { + return metadata, false + } + output := map[string]any{ + "stdout": full.Stdout, + "stderr": full.Stderr, + "truncated": full.Truncated, + } + if full.ExitCode != nil { + output["exit_code"] = *full.ExitCode + } + rehydratedShell := copyMetadata(shell) + rehydratedShell["output"] = output + rehydratedNormalized := copyMetadata(normalized) + rehydratedNormalized["shell_exec"] = rehydratedShell + rehydrated := copyMetadata(metadata) + rehydrated["normalized"] = rehydratedNormalized + return rehydrated, true +} + func shellOutputFromMetadata(metadata map[string]any) (ShellExecOutputSnapshot, bool) { if metadata == nil { return ShellExecOutputSnapshot{}, false diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index e9d0ebea9b..7b52b32b53 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -1741,6 +1741,20 @@ type Message struct { // non-user messages and for legacy 12-column reads, and serialized only // when > 0 (json omitempty). PromptIndex int `json:"prompt_index,omitempty"` + // PayloadDigest is the SHA-256 digest of a large tool payload (currently + // shell command stdout/stderr) that CreateMessage/UpdateMessage + // externalized into task_message_payloads instead of inlining in + // Metadata. Empty when the message's metadata was small enough to stay + // inline. Internal repository/service bookkeeping only - never + // serialized to API/WS clients (json "-"): the client-visible signal is + // already the has_output/stdout_bytes summary ProjectMessageMetadata + // leaves in Metadata, and GetMessage is the only reader that resolves + // this digest, via RehydrateMessagePayload, for explicit authorized + // detail loading (see httpGetShellOutput). + PayloadDigest string `json:"-"` + // PayloadSize is the uncompressed byte size of the externalized payload + // referenced by PayloadDigest. Zero when PayloadDigest is empty. + PayloadSize int64 `json:"-"` } // ToAPI converts internal Message to API type. diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index e02f76d19f..5992bd7c30 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -292,6 +292,12 @@ type WorkflowRepository interface { type MessageRepository interface { CreateMessage(ctx context.Context, message *models.Message) error GetMessage(ctx context.Context, id string) (*models.Message, error) + // RehydrateMessagePayload loads and verifies the externally stored + // payload for a message whose large tool output (e.g. shell command + // stdout/stderr) was moved out of the metadata column at write time, and + // merges the restored content back into message.Metadata. No-op when the + // message has no external payload (message.PayloadDigest == ""). + RehydrateMessagePayload(ctx context.Context, message *models.Message) error // GetLastMessageTimeBySessionIDs returns the newest task_session_messages // updated_at for each requested session, in one chunked query. Sessions // with no messages are absent from the result; callers fall back to the diff --git a/apps/backend/internal/task/repository/sqlite/base.go b/apps/backend/internal/task/repository/sqlite/base.go index ab2e815cbb..a8fe331686 100644 --- a/apps/backend/internal/task/repository/sqlite/base.go +++ b/apps/backend/internal/task/repository/sqlite/base.go @@ -245,6 +245,19 @@ func NewWithDBContext(ctx context.Context, writer, reader *sqlx.DB, log *logger. return newRepositoryContext(ctx, writer, reader, log, false) } +// NewReadOnlyWithDB creates a repository over an existing read-only connection +// without initializing or migrating its schema. Write methods remain guarded by +// the connection's SQLite read-only mode. +func NewReadOnlyWithDB(reader *sqlx.DB, log *logger.Logger) *Repository { + return &Repository{ + db: reader, + ro: reader, + ownsDB: false, + log: log, + migrate: db.NewMigrateLogger(reader, log), + } +} + func newRepository(writer, reader *sqlx.DB, log *logger.Logger, ownsDB bool) (*Repository, error) { return newRepositoryContext(context.Background(), writer, reader, log, ownsDB) } diff --git a/apps/backend/internal/task/repository/sqlite/base_migrations.go b/apps/backend/internal/task/repository/sqlite/base_migrations.go index 9e1f3e065f..a45189fae8 100644 --- a/apps/backend/internal/task/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/task/repository/sqlite/base_migrations.go @@ -17,6 +17,28 @@ import ( "github.com/kandev/kandev/internal/startup" ) +// migrateMessagePayloadStorage adds digest-backed external storage for large +// tool-message payloads (currently shell command stdout/stderr) so +// task_session_messages.metadata stays bounded while preserving lazy, +// integrity-verified detail loading. See externalizeMessagePayload (write +// path) and RehydrateMessagePayload (explicit authorized read path) in +// message_payload.go. task_message_payloads is content-addressed by SHA-256 +// digest, so an identical payload referenced by more than one message is +// stored exactly once. +func (r *Repository) migrateMessagePayloadStorage() { + _ = r.migrate.Apply("task_session_messages.payload_digest", `ALTER TABLE task_session_messages ADD COLUMN payload_digest TEXT NOT NULL DEFAULT ''`) + _ = r.migrate.Apply("task_session_messages.payload_size", `ALTER TABLE task_session_messages ADD COLUMN payload_size INTEGER NOT NULL DEFAULT 0`) + _ = r.migrate.Apply("idx_messages_payload_digest", `CREATE INDEX IF NOT EXISTS idx_messages_payload_digest ON task_session_messages(payload_digest)`) + _ = r.migrate.Apply("task_message_payloads.table", fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS task_message_payloads ( + digest TEXT PRIMARY KEY, + compressed_content %s NOT NULL, + uncompressed_size INTEGER NOT NULL, + compressed_size INTEGER NOT NULL, + created_at TIMESTAMP NOT NULL + )`, dialect.BlobType(r.db.DriverName()))) +} + // migrateExecutorProfiles adds mcp_policy column and drops is_default from executor_profiles. func (r *Repository) migrateExecutorProfiles() error { r.migrate.Apply("executor_profiles.mcp_policy", `ALTER TABLE executor_profiles ADD COLUMN mcp_policy TEXT DEFAULT ''`) @@ -406,6 +428,18 @@ func (r *Repository) runMigrations(ctx context.Context) error { return err } + // Bounded operational payload storage (PR-watch/storage-bounds plan, + // wave 2): digest-backed external storage for large tool-message + // payloads (currently shell command stdout/stderr - see + // externalizeMessagePayload/RehydrateMessagePayload) and a content + // digest on git snapshots so content-equivalent rows become + // identifiable via ListDuplicateGitSnapshotCandidates for a later, + // explicit maintenance pass to prune. + r.migrateMessagePayloadStorage() + if err := r.migrateGitSnapshotContentDigest(); err != nil { + return err + } + // Workflow step display snapshot on plan revisions, same pattern as // author_name: the step a task was on when the revision was written. // Pre-existing revisions get empty strings, matching the fresh-DB default. diff --git a/apps/backend/internal/task/repository/sqlite/base_queries.go b/apps/backend/internal/task/repository/sqlite/base_queries.go index c35b36d326..cb99d6c4de 100644 --- a/apps/backend/internal/task/repository/sqlite/base_queries.go +++ b/apps/backend/internal/task/repository/sqlite/base_queries.go @@ -1,7 +1,14 @@ // Package sqlite provides SQLite-based repository implementations. package sqlite -import "strings" +import ( + "context" + "database/sql" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" +) // sqlLimitClause is the SQL fragment appended to dynamic queries when a row // limit is requested. Shared across plan.go, document.go, and message.go. @@ -37,6 +44,39 @@ func buildInPlaceholders(ids []string) (string, []interface{}) { return placeholders, args } +// listLimitedCandidates runs query against ro (appending sqlLimitClause and +// limit as a trailing bound parameter when limit > 0), scanning each row +// with scanRow. It centralizes the query/scan-loop shape shared by the +// read-only maintenance-candidate listers (duplicate git snapshots, +// orphaned message payloads, ...) so each lister only supplies its SQL and +// per-row scan logic. +func listLimitedCandidates[T any](ctx context.Context, ro *sqlx.DB, query string, limit int, errLabel string, scanRow func(*sql.Rows) (T, error)) ([]T, error) { + args := []interface{}{} + if limit > 0 { + query += sqlLimitClause + args = append(args, limit) + } + + rows, err := ro.QueryContext(ctx, ro.Rebind(query), args...) + if err != nil { + return nil, fmt.Errorf("failed to list %s: %w", errLabel, err) + } + defer func() { _ = rows.Close() }() + + var out []T + for rows.Next() { + item, err := scanRow(rows) + if err != nil { + return nil, fmt.Errorf("failed to scan %s: %w", errLabel, err) + } + out = append(out, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate %s: %w", errLabel, err) + } + return out, nil +} + // chunkIDs splits ids into sub-slices of at most size entries. Useful when // callers want to keep IN-clause queries below sqliteMaxHostParams. An empty // input returns an empty outer slice (NOT a single nil chunk) so callers can diff --git a/apps/backend/internal/task/repository/sqlite/git_snapshot_digest.go b/apps/backend/internal/task/repository/sqlite/git_snapshot_digest.go new file mode 100644 index 0000000000..3534185d13 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/git_snapshot_digest.go @@ -0,0 +1,140 @@ +// Package sqlite provides SQLite-based repository implementations. +package sqlite + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" +) + +// gitSnapshotDigestPayload is the canonical (field-order-stable, since +// encoding/json marshals struct fields in declaration order) shape hashed by +// computeGitSnapshotDigest. It deliberately excludes session_id, +// snapshot_type, triggered_by and created_at: those describe *why* and *for +// whom* a snapshot was captured, not the repository state it describes, so +// two rows with an identical state but different lifecycle context are +// still identified as content-equivalent by this digest. +type gitSnapshotDigestPayload struct { + Branch string `json:"branch"` + RemoteBranch string `json:"remote_branch"` + HeadCommit string `json:"head_commit"` + BaseCommit string `json:"base_commit"` + Ahead int `json:"ahead"` + Behind int `json:"behind"` + Files string `json:"files"` +} + +// computeGitSnapshotDigest returns the SHA-256 hex digest identifying a git +// snapshot's repository-state content, independent of session/lifecycle +// metadata. filesJSON is the already-serialized Files column value (see +// serializeSnapshotJSON) so callers never need to re-marshal it. +func computeGitSnapshotDigest(branch, remoteBranch, headCommit, baseCommit string, ahead, behind int, filesJSON string) string { + // json.Marshal on a fixed struct never errors (no channels/funcs/cycles), + // so the error is deliberately ignored rather than threaded through every + // caller of what is otherwise a pure function. + payload, _ := json.Marshal(gitSnapshotDigestPayload{ + Branch: branch, RemoteBranch: remoteBranch, HeadCommit: headCommit, + BaseCommit: baseCommit, Ahead: ahead, Behind: behind, Files: filesJSON, + }) + sum := sha256.Sum256(payload) + return hex.EncodeToString(sum[:]) +} + +// migrateGitSnapshotContentDigest adds task_session_git_snapshots. +// content_digest and backfills it for every existing row, so historical +// content-duplicate rows become identifiable via +// ListDuplicateGitSnapshotCandidates for a later, explicit maintenance pass +// to prune. +func (r *Repository) migrateGitSnapshotContentDigest() error { + _ = r.migrate.Apply("task_session_git_snapshots.content_digest", `ALTER TABLE task_session_git_snapshots ADD COLUMN content_digest TEXT NOT NULL DEFAULT ''`) + _ = r.migrate.Apply("idx_git_snapshots_session_digest", `CREATE INDEX IF NOT EXISTS idx_git_snapshots_session_digest ON task_session_git_snapshots(session_id, content_digest)`) + return r.backfillGitSnapshotContentDigests() +} + +// backfillGitSnapshotContentDigests computes content_digest for every row +// that doesn't have one yet (fresh installs never do; upgraded installs +// have historical rows to backfill once). SQLite/Postgres have no portable +// SHA-256 SQL function, so this iterates in Go rather than a single UPDATE. +func (r *Repository) backfillGitSnapshotContentDigests() error { + rows, err := r.db.Queryx(` + SELECT id, branch, remote_branch, head_commit, base_commit, ahead, behind, files + FROM task_session_git_snapshots WHERE content_digest = ''`) + if err != nil { + return fmt.Errorf("query git snapshots for content_digest backfill: %w", err) + } + type pending struct { + id string + digest string + } + var updates []pending + for rows.Next() { + var id, branch, remoteBranch, headCommit, baseCommit, filesJSON string + var ahead, behind int + if err := rows.Scan(&id, &branch, &remoteBranch, &headCommit, &baseCommit, &ahead, &behind, &filesJSON); err != nil { + _ = rows.Close() + return fmt.Errorf("scan git snapshot for content_digest backfill: %w", err) + } + updates = append(updates, pending{id: id, digest: computeGitSnapshotDigest(branch, remoteBranch, headCommit, baseCommit, ahead, behind, filesJSON)}) + } + if err := rows.Err(); err != nil { + _ = rows.Close() + return fmt.Errorf("iterate git snapshots for content_digest backfill: %w", err) + } + _ = rows.Close() + + for _, u := range updates { + if _, err := r.db.Exec(r.db.Rebind(` + UPDATE task_session_git_snapshots SET content_digest = ? WHERE id = ? + `), u.digest, u.id); err != nil { + return fmt.Errorf("backfill content_digest for git snapshot %s: %w", u.id, err) + } + } + return nil +} + +// DuplicateGitSnapshotCandidate identifies a non-authoritative git snapshot +// row that shares its (session_id, content_digest) pair with a newer row for +// the same session. Archive and agent-completed rows are never candidates: +// their lifecycle purpose can make them authoritative even when a newer live +// observation has identical Git content. +type DuplicateGitSnapshotCandidate struct { + ID string + SessionID string + ContentDigest string +} + +// ListDuplicateGitSnapshotCandidates returns content-duplicate git snapshot +// rows across all sessions: for every (session_id, content_digest) group +// with more than one row, superseded live/status rows are reported while +// archive and agent-completed rows remain protected. Read-only and +// non-destructive - CreateGitSnapshot always records every poll (per the +// plan's "destructive maintenance is explicit, dry-run-first" constraint), +// so this is the sole place duplicates are identified for explicit +// maintenance to act on. +func (r *Repository) ListDuplicateGitSnapshotCandidates(ctx context.Context, limit int) ([]DuplicateGitSnapshotCandidate, error) { + query := ` + SELECT s.id, s.session_id, s.content_digest + FROM task_session_git_snapshots s + WHERE s.content_digest <> '' + AND s.snapshot_type <> 'archive' + AND s.triggered_by <> 'agent_completed' + AND EXISTS ( + SELECT 1 FROM task_session_git_snapshots newer + WHERE newer.session_id = s.session_id + AND newer.content_digest = s.content_digest + AND ( + newer.created_at > s.created_at + OR (newer.created_at = s.created_at AND newer.id > s.id) + ) + ) + ORDER BY s.session_id, s.created_at ASC` + return listLimitedCandidates(ctx, r.ro, query, limit, "duplicate git snapshot candidates", + func(rows *sql.Rows) (DuplicateGitSnapshotCandidate, error) { + var c DuplicateGitSnapshotCandidate + err := rows.Scan(&c.ID, &c.SessionID, &c.ContentDigest) + return c, err + }) +} diff --git a/apps/backend/internal/task/repository/sqlite/git_snapshot_digest_test.go b/apps/backend/internal/task/repository/sqlite/git_snapshot_digest_test.go new file mode 100644 index 0000000000..ead674e158 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/git_snapshot_digest_test.go @@ -0,0 +1,170 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +// snapshotForDigestTest builds a minimal, otherwise-identical git snapshot +// for a session so tests can vary just the fields that matter to them. +func snapshotForDigestTest(id, sessionID, triggeredBy string, createdAt time.Time) *models.GitSnapshot { + return &models.GitSnapshot{ + ID: id, + SessionID: sessionID, + SnapshotType: models.SnapshotTypeStatusUpdate, + Branch: "feature/digest", + RemoteBranch: "origin/feature/digest", + HeadCommit: "head-sha-digest", + BaseCommit: "base-sha-digest", + Ahead: 1, + Behind: 0, + Files: map[string]interface{}{ + "apps/backend/main.go": map[string]interface{}{"status": "modified"}, + }, + TriggeredBy: triggeredBy, + CreatedAt: createdAt, + } +} + +// TestCreateGitSnapshotSetsContentDigestAndAlwaysRecords proves +// CreateGitSnapshot computes a stable content digest for identical +// repository state and a different one when content changes, while still +// recording every poll (content-based dedup for this wave is read-only via +// ListDuplicateGitSnapshotCandidates, not a write-time skip - see that +// function's docs for why). +func TestCreateGitSnapshotSetsContentDigestAndAlwaysRecords(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedSessionForGit(t, repo, "task-digest-basic", "session-digest-basic") + + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + first := snapshotForDigestTest("snap-digest-1", "session-digest-basic", TriggeredByLiveMonitor, base) + if err := repo.CreateGitSnapshot(ctx, first); err != nil { + t.Fatalf("CreateGitSnapshot(first): %v", err) + } + if first.ContentDigest == "" { + t.Fatal("CreateGitSnapshot did not set ContentDigest") + } + + // Identical repository state, later poll: content_digest matches and the + // row is still recorded (no write-time skip in this wave). + second := snapshotForDigestTest("snap-digest-2", "session-digest-basic", TriggeredByLiveMonitor, base.Add(time.Minute)) + if err := repo.CreateGitSnapshot(ctx, second); err != nil { + t.Fatalf("CreateGitSnapshot(second, identical content): %v", err) + } + if second.ContentDigest != first.ContentDigest { + t.Fatalf("ContentDigest = %q, want %q (identical repository state)", second.ContentDigest, first.ContentDigest) + } + count := countRows(t, repo, `SELECT COUNT(*) FROM task_session_git_snapshots WHERE session_id = ?`, "session-digest-basic") + if count != 2 { + t.Fatalf("row count = %d, want 2 (every poll is still recorded)", count) + } + + // Changed content produces a different digest. + changed := snapshotForDigestTest("snap-digest-3", "session-digest-basic", TriggeredByLiveMonitor, base.Add(2*time.Minute)) + changed.HeadCommit = "head-sha-digest-changed" + if err := repo.CreateGitSnapshot(ctx, changed); err != nil { + t.Fatalf("CreateGitSnapshot(changed): %v", err) + } + if changed.ContentDigest == first.ContentDigest { + t.Fatal("changed snapshot content digest matched the original despite a different head commit") + } +} + +func TestUpsertLatestLiveGitSnapshotSetsContentDigest(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedSessionForGit(t, repo, "task-digest-upsert", "session-digest-upsert") + + snapshot := snapshotForDigestTest("snap-upsert-1", "session-digest-upsert", "", time.Now().UTC()) + if err := repo.UpsertLatestLiveGitSnapshot(ctx, snapshot); err != nil { + t.Fatalf("UpsertLatestLiveGitSnapshot: %v", err) + } + if snapshot.ContentDigest == "" { + t.Fatal("UpsertLatestLiveGitSnapshot did not set ContentDigest") + } + + var stored string + if err := repo.db.QueryRowContext(ctx, repo.db.Rebind( + `SELECT content_digest FROM task_session_git_snapshots WHERE id = ?`), + snapshot.ID).Scan(&stored); err != nil { + t.Fatalf("query stored content_digest: %v", err) + } + if stored != snapshot.ContentDigest { + t.Fatalf("stored content_digest = %q, want %q", stored, snapshot.ContentDigest) + } +} + +// TestListDuplicateGitSnapshotCandidatesKeepsNewestPerGroup exercises the +// read-only retention-candidate selection: every row created via +// CreateGitSnapshot with content-identical repository state is recorded +// (this wave never skips a write), and the candidate query must report all +// but the newest row in each (session, content_digest) group for a later, +// explicit maintenance pass to prune. +func TestListDuplicateGitSnapshotCandidatesKeepsNewestPerGroup(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedSessionForGit(t, repo, "task-digest-candidates", "session-digest-candidates") + + base := time.Date(2026, 1, 3, 0, 0, 0, 0, time.UTC) + ids := []string{"snap-cand-1", "snap-cand-2", "snap-cand-3"} + for i, offset := range []time.Duration{0, time.Minute, 2 * time.Minute} { + snap := snapshotForDigestTest(ids[i], "session-digest-candidates", TriggeredByLiveMonitor, base.Add(offset)) + if err := repo.CreateGitSnapshot(ctx, snap); err != nil { + t.Fatalf("CreateGitSnapshot(%d): %v", i, err) + } + } + + count := countRows(t, repo, `SELECT COUNT(*) FROM task_session_git_snapshots WHERE session_id = ?`, "session-digest-candidates") + if count != 3 { + t.Fatalf("seeded row count = %d, want 3", count) + } + + candidates, err := repo.ListDuplicateGitSnapshotCandidates(ctx, 0) + if err != nil { + t.Fatalf("ListDuplicateGitSnapshotCandidates: %v", err) + } + var seenIDs []string + for _, c := range candidates { + if c.SessionID != "session-digest-candidates" { + continue + } + seenIDs = append(seenIDs, c.ID) + if c.ID == "snap-cand-3" { + t.Fatal("newest duplicate row (snap-cand-3) must not be reported as a candidate") + } + } + if len(seenIDs) != 2 { + t.Fatalf("candidates for this session = %v, want exactly [snap-cand-1 snap-cand-2]", seenIDs) + } +} + +// TestListDuplicateGitSnapshotCandidatesIsNonDestructive confirms the +// selection is read-only: calling it does not remove or alter any snapshot +// row, matching the plan's "non-destructive until maintenance executes" +// constraint. +func TestListDuplicateGitSnapshotCandidatesIsNonDestructive(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedSessionForGit(t, repo, "task-digest-nondestructive", "session-digest-nondestructive") + + base := time.Date(2026, 1, 4, 0, 0, 0, 0, time.UTC) + for i, offset := range []time.Duration{0, time.Minute} { + snap := snapshotForDigestTest([]string{"snap-nd-1", "snap-nd-2"}[i], "session-digest-nondestructive", TriggeredByLiveMonitor, base.Add(offset)) + if err := repo.CreateGitSnapshot(ctx, snap); err != nil { + t.Fatalf("CreateGitSnapshot(%d): %v", i, err) + } + } + + before := countRows(t, repo, `SELECT COUNT(*) FROM task_session_git_snapshots WHERE session_id = ?`, "session-digest-nondestructive") + if _, err := repo.ListDuplicateGitSnapshotCandidates(ctx, 0); err != nil { + t.Fatalf("ListDuplicateGitSnapshotCandidates: %v", err) + } + after := countRows(t, repo, `SELECT COUNT(*) FROM task_session_git_snapshots WHERE session_id = ?`, "session-digest-nondestructive") + if before != after { + t.Fatalf("row count changed from %d to %d after a read-only candidate listing", before, after) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/git_snapshots.go b/apps/backend/internal/task/repository/sqlite/git_snapshots.go index 6ebf45e9c3..8c2710d51b 100644 --- a/apps/backend/internal/task/repository/sqlite/git_snapshots.go +++ b/apps/backend/internal/task/repository/sqlite/git_snapshots.go @@ -171,15 +171,20 @@ func (r *Repository) UpsertLatestLiveGitSnapshot(ctx context.Context, snapshot * if err != nil { return err } + snapshot.ContentDigest = computeGitSnapshotDigest( + snapshot.Branch, snapshot.RemoteBranch, snapshot.HeadCommit, snapshot.BaseCommit, + snapshot.Ahead, snapshot.Behind, filesJSON, + ) if _, err := tx.ExecContext(ctx, r.db.Rebind(` INSERT INTO task_session_git_snapshots ( id, task_environment_id, session_id, snapshot_type, branch, remote_branch, head_commit, base_commit, - ahead, behind, files, triggered_by, metadata, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ahead, behind, files, triggered_by, metadata, created_at, content_digest + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), snapshot.ID, snapshot.TaskEnvironmentID, nullableGitSnapshotSessionID(snapshot.SessionID), string(snapshot.SnapshotType), snapshot.Branch, snapshot.RemoteBranch, snapshot.HeadCommit, snapshot.BaseCommit, snapshot.Ahead, - snapshot.Behind, filesJSON, snapshot.TriggeredBy, metadataJSON, snapshot.CreatedAt); err != nil { + snapshot.Behind, filesJSON, snapshot.TriggeredBy, metadataJSON, snapshot.CreatedAt, + snapshot.ContentDigest); err != nil { return fmt.Errorf("insert live snapshot: %w", err) } @@ -232,7 +237,14 @@ func (r *Repository) DeleteLiveMonitorSnapshotsByTaskEnvironmentID(ctx context.C return err } -// CreateGitSnapshot inserts a new git snapshot into the database. +// CreateGitSnapshot inserts a new git snapshot into the database. It always +// records the row (matching prior behavior, so ordering/history reads stay +// exact): content-based deduplication for this wave is intentionally +// read-only-candidate-based rather than skip-on-write, per the plan's +// "destructive maintenance is explicit, dry-run-first" constraint. The +// computed content_digest lets ListDuplicateGitSnapshotCandidates identify +// content-equivalent rows for a later, explicit maintenance pass (Task 06) +// to prune instead of silently altering what every write persists. func (r *Repository) CreateGitSnapshot(ctx context.Context, snapshot *models.GitSnapshot) error { if snapshot == nil { return fmt.Errorf("snapshot is nil") @@ -251,16 +263,21 @@ func (r *Repository) CreateGitSnapshot(ctx context.Context, snapshot *models.Git if err != nil { return err } + snapshot.ContentDigest = computeGitSnapshotDigest( + snapshot.Branch, snapshot.RemoteBranch, snapshot.HeadCommit, snapshot.BaseCommit, + snapshot.Ahead, snapshot.Behind, filesJSON, + ) if snapshot.TriggeredBy != triggeredByAgentCompleted { _, err = r.db.ExecContext(ctx, r.db.Rebind(` INSERT INTO task_session_git_snapshots ( id, task_environment_id, session_id, snapshot_type, branch, remote_branch, head_commit, base_commit, - ahead, behind, files, triggered_by, metadata, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ahead, behind, files, triggered_by, metadata, created_at, content_digest + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), snapshot.ID, snapshot.TaskEnvironmentID, nullableGitSnapshotSessionID(snapshot.SessionID), string(snapshot.SnapshotType), snapshot.Branch, snapshot.RemoteBranch, snapshot.HeadCommit, snapshot.BaseCommit, snapshot.Ahead, - snapshot.Behind, filesJSON, snapshot.TriggeredBy, metadataJSON, snapshot.CreatedAt) + snapshot.Behind, filesJSON, snapshot.TriggeredBy, metadataJSON, snapshot.CreatedAt, + snapshot.ContentDigest) return err } @@ -300,11 +317,11 @@ func insertGitSnapshot( return exec.ExecContext(ctx, db.Rebind(` INSERT INTO task_session_git_snapshots ( id, task_environment_id, session_id, snapshot_type, branch, remote_branch, head_commit, base_commit, - ahead, behind, files, triggered_by, metadata, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ahead, behind, files, triggered_by, metadata, created_at, content_digest + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `), snapshot.ID, snapshot.TaskEnvironmentID, nullableGitSnapshotSessionID(snapshot.SessionID), string(snapshot.SnapshotType), snapshot.Branch, snapshot.RemoteBranch, snapshot.HeadCommit, snapshot.BaseCommit, snapshot.Ahead, - snapshot.Behind, filesJSON, snapshot.TriggeredBy, metadataJSON, snapshot.CreatedAt) + snapshot.Behind, filesJSON, snapshot.TriggeredBy, metadataJSON, snapshot.CreatedAt, snapshot.ContentDigest) } func (r *Repository) lockGitSnapshotEnvironment(tx *sqlx.Tx, taskEnvironmentID string) error { diff --git a/apps/backend/internal/task/repository/sqlite/message.go b/apps/backend/internal/task/repository/sqlite/message.go index 9313b6864b..9d08aef390 100644 --- a/apps/backend/internal/task/repository/sqlite/message.go +++ b/apps/backend/internal/task/repository/sqlite/message.go @@ -86,13 +86,9 @@ func (r *Repository) CreateMessage(ctx context.Context, message *models.Message) messageType = string(models.MessageTypeMessage) } - metadataJSON := "{}" - if message.Metadata != nil { - metadataBytes, err := json.Marshal(message.Metadata) - if err != nil { - return fmt.Errorf("failed to serialize message metadata: %w", err) - } - metadataJSON = string(metadataBytes) + metadataJSON, err := r.externalizeMessagePayload(ctx, message) + if err != nil { + return err } if message.AuthorType == models.MessageAuthorUser { @@ -163,9 +159,9 @@ func (r *Repository) insertMessageRow( messageType, metadataJSON string, ) error { _, err := execer.ExecContext(ctx, r.db.Rebind(` - INSERT INTO task_session_messages (id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at, prompt_seq) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `), message.ID, message.TaskSessionID, message.TaskID, message.TurnID, message.AuthorType, message.AuthorID, message.Content, requestsInput, messageType, metadataJSON, message.CreatedAt, message.UpdatedAt, message.PromptIndex) + INSERT INTO task_session_messages (id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at, prompt_seq, payload_digest, payload_size) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `), message.ID, message.TaskSessionID, message.TaskID, message.TurnID, message.AuthorType, message.AuthorID, message.Content, requestsInput, messageType, metadataJSON, message.CreatedAt, message.UpdatedAt, message.PromptIndex, message.PayloadDigest, message.PayloadSize) return err } @@ -206,9 +202,9 @@ func (r *Repository) GetMessage(ctx context.Context, id string) (*models.Message var messageType string var metadataJSON string err := r.ro.QueryRowContext(ctx, r.ro.Rebind(` - SELECT id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at + SELECT id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at, payload_digest, payload_size FROM task_session_messages WHERE id = ? - `), id).Scan(&message.ID, &message.TaskSessionID, &message.TaskID, &message.TurnID, &message.AuthorType, &message.AuthorID, &message.Content, &requestsInput, &messageType, &metadataJSON, &message.CreatedAt, &message.UpdatedAt) + `), id).Scan(&message.ID, &message.TaskSessionID, &message.TaskID, &message.TurnID, &message.AuthorType, &message.AuthorID, &message.Content, &requestsInput, &messageType, &metadataJSON, &message.CreatedAt, &message.UpdatedAt, &message.PayloadDigest, &message.PayloadSize) if err != nil { return nil, err } @@ -1004,9 +1000,9 @@ func (r *Repository) CompletePendingToolCallsForTurn(ctx context.Context, turnID // UpdateMessage updates an existing message func (r *Repository) UpdateMessage(ctx context.Context, message *models.Message) error { - metadataJSON, err := json.Marshal(message.Metadata) + metadataJSON, err := r.externalizeMessagePayload(ctx, message) if err != nil { - return fmt.Errorf("failed to marshal metadata: %w", err) + return err } requestsInput := 0 @@ -1016,12 +1012,12 @@ func (r *Repository) UpdateMessage(ctx context.Context, message *models.Message) message.UpdatedAt = time.Now().UTC() if !dialect.IsPostgres(r.db.DriverName()) { - return r.updateMessageWithPayloadGuard(ctx, message, metadataJSON, requestsInput) + return r.updateMessageWithPayloadGuard(ctx, message, []byte(metadataJSON), requestsInput) } result, err := r.db.ExecContext(ctx, r.db.Rebind(` - UPDATE task_session_messages SET content = ?, requests_input = ?, type = ?, metadata = ?, updated_at = ? + UPDATE task_session_messages SET content = ?, requests_input = ?, type = ?, metadata = ?, payload_digest = ?, payload_size = ?, updated_at = ? WHERE id = ? - `), message.Content, requestsInput, string(message.Type), string(metadataJSON), message.UpdatedAt, message.ID) + `), message.Content, requestsInput, string(message.Type), metadataJSON, message.PayloadDigest, message.PayloadSize, message.UpdatedAt, message.ID) if err != nil { return err } diff --git a/apps/backend/internal/task/repository/sqlite/message_history_load_test.go b/apps/backend/internal/task/repository/sqlite/message_history_load_test.go new file mode 100644 index 0000000000..61a6bf6710 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/message_history_load_test.go @@ -0,0 +1,166 @@ +package sqlite + +import ( + "context" + "sort" + "strings" + "testing" + "time" + + "github.com/kandev/kandev/internal/db/dialect" + "github.com/kandev/kandev/internal/task/models" +) + +const ( + loadHistoryMessageCount = 1_000_000 + loadHistoryPayloadCount = 64 + loadHistoryPayloadBytes = 64 * 1024 * 1024 +) + +// TestListMessagesPaginatedBoundsMultiGigabyteEquivalentHistory proves the +// normal history path stays page-bounded with a million-row transcript and +// four GiB of logically externalized tool output. The payload bodies are +// deliberately tiny sentinels: payload integrity/rehydration has dedicated +// tests, while this fixture isolates the list path's cardinality and I/O +// contract without committing or generating a multi-gigabyte test artifact. +func TestListMessagesPaginatedBoundsMultiGigabyteEquivalentHistory(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-load-history" + sessionID = "session-load-history" + turnID = "turn-load-history" + pageSize = 50 + ) + seedForMsgTest(t, repo, taskID, sessionID, turnID) + seedLargeHistoryFixture(t, repo, taskID, sessionID, turnID) + assertIndexedBoundedMessageQuery(t, repo, sessionID, pageSize) + + hydrationsBefore := totalPayloadHydrations(t) + durations := make([]time.Duration, 0, 15) + for i := 0; i < cap(durations); i++ { + started := time.Now() + messages, hasMore, err := repo.ListMessagesPaginated(ctx, sessionID, models.ListMessagesOptions{ + Limit: pageSize, + Sort: "desc", + }) + durations = append(durations, time.Since(started)) + if err != nil { + t.Fatalf("read page %d: %v", i, err) + } + if len(messages) != pageSize || !hasMore { + t.Fatalf("read page %d = (%d rows, hasMore=%v), want (%d, true)", i, len(messages), hasMore, pageSize) + } + assertLightweightMessagePage(t, messages) + } + if delta := totalPayloadHydrations(t) - hydrationsBefore; delta != 0 { + t.Fatalf("payload hydrations during list = %d, want 0", delta) + } + + sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] }) + t.Logf("bounded-history load: rows=%d logical_payload_bytes=%d page=%d samples=%d p50=%s p95=%s max=%s", + loadHistoryMessageCount, + int64(loadHistoryPayloadCount)*loadHistoryPayloadBytes, + pageSize, + len(durations), + durations[len(durations)/2], + durations[(len(durations)*95-1)/100], + durations[len(durations)-1], + ) +} + +func seedLargeHistoryFixture(t *testing.T, repo *Repository, taskID, sessionID, turnID string) { + t.Helper() + tx, err := repo.db.Beginx() + if err != nil { + t.Fatalf("begin large-history seed: %v", err) + } + defer func() { _ = tx.Rollback() }() + + if _, err = tx.Exec(` + WITH RECURSIVE payloads(n) AS ( + SELECT 1 UNION ALL SELECT n + 1 FROM payloads WHERE n < ? + ) + INSERT INTO task_message_payloads + (digest, compressed_content, uncompressed_size, compressed_size, created_at) + SELECT printf('load-payload-%03d', n), x'1f8b08000000000000ff03000000000000000000', ?, 20, CURRENT_TIMESTAMP + FROM payloads + `, loadHistoryPayloadCount, loadHistoryPayloadBytes); err != nil { + t.Fatalf("seed payloads: %v", err) + } + if _, err = tx.Exec(` + WITH RECURSIVE messages(n) AS ( + SELECT 1 UNION ALL SELECT n + 1 FROM messages WHERE n < ? + ) + INSERT INTO task_session_messages + (id, task_session_id, task_id, turn_id, author_type, author_id, content, + requests_input, type, metadata, created_at, updated_at, prompt_seq, + payload_digest, payload_size) + SELECT printf('load-message-%07d', n), ?, ?, ?, 'agent', '', 'ok', + 0, 'tool_call', '{}', + printf('2026-01-%02d %02d:%02d:%02d.000000+00:00', + ((n - 1) / 86400) % 28 + 1, ((n - 1) / 3600) % 24, + ((n - 1) / 60) % 60, (n - 1) % 60), + CURRENT_TIMESTAMP, 0, + CASE WHEN n <= ? THEN printf('load-payload-%03d', n) ELSE '' END, + CASE WHEN n <= ? THEN ? ELSE 0 END + FROM messages + `, loadHistoryMessageCount, sessionID, taskID, turnID, + loadHistoryPayloadCount, loadHistoryPayloadCount, loadHistoryPayloadBytes); err != nil { + t.Fatalf("seed messages: %v", err) + } + if err = tx.Commit(); err != nil { + t.Fatalf("commit large-history seed: %v", err) + } +} + +func assertIndexedBoundedMessageQuery(t *testing.T, repo *Repository, sessionID string, limit int) { + t.Helper() + query, args := buildListMessagesQuery( + repo.ro.DriverName(), sessionID, models.ListMessagesOptions{Sort: "desc"}, nil, "", "DESC", limit, + ) + if strings.Contains(query, "task_message_payloads") || !strings.Contains(query, "LIMIT ?") { + t.Fatalf("history query is not payload-independent and bounded: %s", query) + } + rows, err := repo.ro.Query("EXPLAIN QUERY PLAN "+query, args...) + if err != nil { + t.Fatalf("explain bounded history query: %v", err) + } + defer func() { _ = rows.Close() }() + var details []string + for rows.Next() { + var id, parent, unused int + var detail string + if err := rows.Scan(&id, &parent, &unused, &detail); err != nil { + t.Fatalf("scan query plan: %v", err) + } + details = append(details, detail) + } + joined := strings.Join(details, "\n") + if !strings.Contains(joined, "idx_messages_prompt_order") { + t.Fatalf("bounded history query did not use prompt-order index:\n%s", joined) + } + if !strings.Contains(query, dialect.NormalizedMicrosecond(repo.ro.DriverName(), "created_at")) { + t.Fatal("history query ordering diverged from the indexed normalized timestamp expression") + } +} + +func assertLightweightMessagePage(t *testing.T, messages []*models.Message) { + t.Helper() + returnedBytes := 0 + for _, message := range messages { + returnedBytes += len(message.Content) + if message.PayloadDigest != "" || message.PayloadSize != 0 { + t.Fatalf("list eagerly materialized payload reference: %+v", message) + } + } + if returnedBytes > len(messages)*len("ok") { + t.Fatalf("returned content bytes = %d, want <= %d", returnedBytes, len(messages)*len("ok")) + } +} + +func totalPayloadHydrations(t *testing.T) int64 { + t.Helper() + return metricMapValue(t, messagePayloadHydrationsTotal, "outcome=success") + + metricMapValue(t, messagePayloadHydrationsTotal, "outcome=error") +} diff --git a/apps/backend/internal/task/repository/sqlite/message_payload.go b/apps/backend/internal/task/repository/sqlite/message_payload.go new file mode 100644 index 0000000000..7a2cff571e --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/message_payload.go @@ -0,0 +1,279 @@ +// Package sqlite provides SQLite-based repository implementations. +package sqlite + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +// largeMessagePayloadThresholdBytes is the combined stdout+stderr size above +// which a shell command's output is externalized into task_message_payloads +// instead of staying inline in task_session_messages.metadata. Chosen to keep +// the overwhelming majority of tool calls (short commands, status checks, +// small diffs) fully inline while bounding the handful of large build/test +// logs that otherwise dominate SQLite growth. +const largeMessagePayloadThresholdBytes = 4096 + +// maxMessagePayloadRehydrateBytes caps one explicit shell-output expansion. +// The writer normally receives already-bounded agentctl output, but the read +// path must also defend against corrupt or hostile local database rows. +const maxMessagePayloadRehydrateBytes = 64 * 1024 * 1024 + +// externalizeMessagePayload computes the metadata JSON CreateMessage/ +// UpdateMessage should persist for a message, externalizing a large shell +// command output snapshot into task_message_payloads (content-addressed by +// SHA-256 digest, so byte-identical output across messages naturally +// deduplicates) and replacing it with the same small summary +// ProjectMessageMetadata already computes for client responses. Small +// metadata (the common case) is marshaled unchanged. On success, message's +// PayloadDigest/PayloadSize are set to reflect what was persisted (empty/zero +// when nothing was externalized). +// +// The payload row is written before this function returns so a concurrent or +// later message insert referencing its digest can never observe a dangling +// reference; content-addressing makes writing it more than once (a +// duplicate output from a different message) an idempotent no-op. +func (r *Repository) externalizeMessagePayload(ctx context.Context, message *models.Message) (string, error) { + existingDigest := message.PayloadDigest + existingSize := message.PayloadSize + message.PayloadDigest = "" + message.PayloadSize = 0 + if message.Metadata == nil { + return "{}", nil + } + + output, ok := models.ExtractShellExecOutput(message.Metadata) + if !ok || len(output.Stdout)+len(output.Stderr) <= largeMessagePayloadThresholdBytes { + if existingDigest != "" && isProjectedShellOutputMetadata(message.Metadata) { + message.PayloadDigest = existingDigest + message.PayloadSize = existingSize + } + metadataBytes, err := json.Marshal(message.Metadata) + if err != nil { + return "", fmt.Errorf("failed to serialize message metadata: %w", err) + } + return string(metadataBytes), nil + } + + payloadBytes, err := json.Marshal(output) + if err != nil { + return "", fmt.Errorf("failed to serialize shell output payload: %w", err) + } + digest := sha256Hex(payloadBytes) + if err := r.putMessagePayload(ctx, digest, payloadBytes); err != nil { + return "", fmt.Errorf("failed to store externalized message payload: %w", err) + } + + projected := models.ProjectMessageMetadata(message.Metadata) + metadataBytes, err := json.Marshal(projected) + if err != nil { + return "", fmt.Errorf("failed to serialize projected message metadata: %w", err) + } + message.PayloadDigest = digest + message.PayloadSize = int64(len(payloadBytes)) + return string(metadataBytes), nil +} + +func isProjectedShellOutputMetadata(metadata map[string]interface{}) bool { + normalized, _ := metadata["normalized"].(map[string]interface{}) + shellExec, _ := normalized["shell_exec"].(map[string]interface{}) + output, _ := shellExec["output"].(map[string]interface{}) + if output == nil { + return false + } + if _, hasStdout := output["stdout"]; hasStdout { + return false + } + if _, hasStderr := output["stderr"]; hasStderr { + return false + } + if _, ok := output["has_output"]; ok { + return true + } + if _, ok := output["stdout_bytes"]; ok { + return true + } + if _, ok := output["stderr_bytes"]; ok { + return true + } + return false +} + +// putMessagePayload gzip-compresses payload and upserts it into +// task_message_payloads keyed by its digest. ON CONFLICT DO NOTHING makes a +// repeat write of the same content (a duplicate output from another +// message, or a retry) a no-op rather than an error. +func (r *Repository) putMessagePayload(ctx context.Context, digest string, payload []byte) error { + compressed, err := gzipCompress(payload) + if err != nil { + return err + } + _, err = r.db.ExecContext(ctx, r.db.Rebind(` + INSERT INTO task_message_payloads (digest, compressed_content, uncompressed_size, compressed_size, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (digest) DO NOTHING + `), digest, compressed, len(payload), len(compressed), time.Now().UTC()) + return err +} + +// RehydrateMessagePayload resolves message's externalized payload (if any) +// and merges the restored shell output snapshot back into message.Metadata, +// verifying the stored digest against the decompressed content before +// trusting it. A no-op (nil error) when message.PayloadDigest is empty - +// the common case, since only large shell outputs are ever externalized. +// This is the explicit, authorized single-message detail load: callers must +// already have resolved and authorized message (see httpGetShellOutput, +// which fetches by ID under the requesting session). +func (r *Repository) RehydrateMessagePayload(ctx context.Context, message *models.Message) (err error) { + if message == nil || message.PayloadDigest == "" { + return nil + } + startedAt := time.Now() + defer func() { recordMessagePayloadHydration(time.Since(startedAt), err) }() + var uncompressedSize, compressedSize, actualCompressedSize int64 + err = r.ro.QueryRowContext(ctx, r.ro.Rebind(` + SELECT uncompressed_size, compressed_size, length(compressed_content) FROM task_message_payloads WHERE digest = ? + `), message.PayloadDigest).Scan(&uncompressedSize, &compressedSize, &actualCompressedSize) + if err != nil { + return fmt.Errorf("failed to load message payload %s: %w", message.PayloadDigest, err) + } + if uncompressedSize < 0 { + return fmt.Errorf("message payload %s has invalid rehydrate size: %d", + message.PayloadDigest, uncompressedSize) + } + if uncompressedSize > maxMessagePayloadRehydrateBytes { + return fmt.Errorf("message payload %s exceeds maximum rehydrate size: %d > %d", + message.PayloadDigest, uncompressedSize, maxMessagePayloadRehydrateBytes) + } + if compressedSize < 0 { + return fmt.Errorf("message payload %s has invalid compressed size: %d", + message.PayloadDigest, compressedSize) + } + if compressedSize > maxMessagePayloadRehydrateBytes { + return fmt.Errorf("message payload %s exceeds maximum compressed size: %d > %d", + message.PayloadDigest, compressedSize, maxMessagePayloadRehydrateBytes) + } + if actualCompressedSize != compressedSize { + return fmt.Errorf("message payload %s compressed size mismatch: %d != %d", + message.PayloadDigest, actualCompressedSize, compressedSize) + } + + var compressed []byte + err = r.ro.QueryRowContext(ctx, r.ro.Rebind(` + SELECT compressed_content FROM task_message_payloads WHERE digest = ? + `), message.PayloadDigest).Scan(&compressed) + if err != nil { + return fmt.Errorf("failed to load message payload %s: %w", message.PayloadDigest, err) + } + payloadBytes, err := gzipDecompressExpectedSize(compressed, maxMessagePayloadRehydrateBytes, uncompressedSize) + if err != nil { + return fmt.Errorf("failed to decompress message payload %s: %w", message.PayloadDigest, err) + } + if int64(len(payloadBytes)) != uncompressedSize { + return fmt.Errorf("message payload %s size mismatch: %d != %d", + message.PayloadDigest, len(payloadBytes), uncompressedSize) + } + if actual := sha256Hex(payloadBytes); actual != message.PayloadDigest { + return fmt.Errorf("message payload %s failed integrity check (got %s)", message.PayloadDigest, actual) + } + var output models.ShellExecOutputSnapshot + if err := json.Unmarshal(payloadBytes, &output); err != nil { + return fmt.Errorf("failed to deserialize message payload %s: %w", message.PayloadDigest, err) + } + if rehydrated, ok := models.RehydrateShellOutput(message.Metadata, output); ok { + message.Metadata = rehydrated + } + return nil +} + +// OrphanedMessagePayloadCandidate identifies a task_message_payloads row no +// longer referenced by any message's payload_digest. Content-addressing +// means a row can be shared by many messages; it only becomes eligible once +// every referencing message has been deleted or edited away from it. +type OrphanedMessagePayloadCandidate struct { + Digest string + CompressedSize int64 + UncompressedSize int64 +} + +// ListOrphanedMessagePayloadCandidates returns task_message_payloads rows no +// task_session_messages row currently references. Read-only and +// non-destructive - reported for a later, explicit maintenance command +// (Task 06) to act on, mirroring ListDuplicateGitSnapshotCandidates and +// ListObsoletePlanRevisionCandidates. +func (r *Repository) ListOrphanedMessagePayloadCandidates(ctx context.Context, limit int) ([]OrphanedMessagePayloadCandidate, error) { + query := ` + SELECT p.digest, p.compressed_size, p.uncompressed_size + FROM task_message_payloads p + WHERE NOT EXISTS ( + SELECT 1 FROM task_session_messages m WHERE m.payload_digest = p.digest + ) + ORDER BY p.digest ASC` + return listLimitedCandidates(ctx, r.ro, query, limit, "orphaned message payload candidates", + func(rows *sql.Rows) (OrphanedMessagePayloadCandidate, error) { + var c OrphanedMessagePayloadCandidate + err := rows.Scan(&c.Digest, &c.CompressedSize, &c.UncompressedSize) + return c, err + }) +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func gzipCompress(data []byte) ([]byte, error) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + if _, err := gw.Write(data); err != nil { + _ = gw.Close() + return nil, fmt.Errorf("gzip compress: %w", err) + } + if err := gw.Close(); err != nil { + return nil, fmt.Errorf("gzip compress: close: %w", err) + } + return buf.Bytes(), nil +} + +func gzipDecompress(data []byte, maxBytes int64) ([]byte, error) { + return gzipDecompressExpectedSize(data, maxBytes, -1) +} + +func gzipDecompressExpectedSize(data []byte, maxBytes, expectedSize int64) ([]byte, error) { + if maxBytes < 0 || expectedSize < -1 { + return nil, fmt.Errorf("gzip decompress: invalid size limit") + } + if expectedSize > maxBytes { + return nil, fmt.Errorf("gzip decompress: expected size %d exceeds maximum size %d", expectedSize, maxBytes) + } + gr, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, fmt.Errorf("gzip decompress: %w", err) + } + defer func() { _ = gr.Close() }() + readLimit := maxBytes + 1 + if expectedSize >= 0 && expectedSize < readLimit { + readLimit = expectedSize + 1 + } + out, err := io.ReadAll(io.LimitReader(gr, readLimit)) + if err != nil { + return nil, fmt.Errorf("gzip decompress: read: %w", err) + } + if int64(len(out)) > maxBytes { + return nil, fmt.Errorf("gzip decompress: payload exceeds maximum size %d", maxBytes) + } + if expectedSize >= 0 && int64(len(out)) != expectedSize { + return nil, fmt.Errorf("gzip decompress: payload size mismatch: %d != %d", len(out), expectedSize) + } + return out, nil +} diff --git a/apps/backend/internal/task/repository/sqlite/message_payload_replay.go b/apps/backend/internal/task/repository/sqlite/message_payload_replay.go index 2c8771db15..80e606efe0 100644 --- a/apps/backend/internal/task/repository/sqlite/message_payload_replay.go +++ b/apps/backend/internal/task/repository/sqlite/message_payload_replay.go @@ -47,14 +47,18 @@ func (r *Repository) updateMessageWithPayloadGuardTx(ctx context.Context, tx *sq return fmt.Errorf("message not found: %s", message.ID) } } - query := `SELECT metadata,type,updated_at FROM task_session_messages WHERE id = ?` + query := `SELECT metadata,type,updated_at,payload_digest,payload_size FROM task_session_messages WHERE id = ?` if dialect.IsPostgres(r.db.DriverName()) { query += forUpdateClause } var raw string var storedType models.MessageType var storedUpdatedAt time.Time - if err := tx.QueryRowContext(ctx, tx.Rebind(query), message.ID).Scan(&raw, &storedType, &storedUpdatedAt); err != nil { + var storedPayloadDigest string + var storedPayloadSize int64 + if err := tx.QueryRowContext(ctx, tx.Rebind(query), message.ID).Scan( + &raw, &storedType, &storedUpdatedAt, &storedPayloadDigest, &storedPayloadSize, + ); err != nil { if errors.Is(err, sql.ErrNoRows) && dialect.IsPostgres(r.db.DriverName()) { return fmt.Errorf("message not found: %s", message.ID) } @@ -71,8 +75,10 @@ func (r *Repository) updateMessageWithPayloadGuardTx(ctx context.Context, tx *sq updatedAt := message.UpdatedAt if sanitized != nil { updatedAt = storedUpdatedAt + message.PayloadDigest = storedPayloadDigest + message.PayloadSize = storedPayloadSize } - _, err = tx.ExecContext(ctx, tx.Rebind(`UPDATE task_session_messages SET content = ?, requests_input = ?, type = ?, metadata = ?, updated_at = ? WHERE id = ?`), message.Content, requestsInput, string(nextType), string(metadataJSON), updatedAt, message.ID) + _, err = tx.ExecContext(ctx, tx.Rebind(`UPDATE task_session_messages SET content = ?, requests_input = ?, type = ?, metadata = ?, payload_digest = ?, payload_size = ?, updated_at = ? WHERE id = ?`), message.Content, requestsInput, string(nextType), string(metadataJSON), message.PayloadDigest, message.PayloadSize, updatedAt, message.ID) if err != nil { return err } diff --git a/apps/backend/internal/task/repository/sqlite/message_payload_retention_test.go b/apps/backend/internal/task/repository/sqlite/message_payload_retention_test.go new file mode 100644 index 0000000000..d75ace25e0 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/message_payload_retention_test.go @@ -0,0 +1,95 @@ +package sqlite + +import ( + "context" + "strings" + "testing" +) + +// TestListOrphanedMessagePayloadCandidatesReportsUnreferencedRowsOnly is +// this wave's coverage for the "stale payload" retention category (Task +// 06): a task_message_payloads row still referenced by a message must never +// be reported, even if another byte-identical (deduped) payload row that +// nothing references any longer exists alongside it. +func TestListOrphanedMessagePayloadCandidatesReportsUnreferencedRowsOnly(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-orphan", "sess-payload-orphan", "turn-1") + + referenced := newShellMessage("msg-orphan-referenced", "sess-payload-orphan", strings.Repeat("r", largeMessagePayloadThresholdBytes+1), "") + referenced.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, referenced); err != nil { + t.Fatalf("CreateMessage(referenced): %v", err) + } + + toOrphan := newShellMessage("msg-orphan-deleted", "sess-payload-orphan", strings.Repeat("o", largeMessagePayloadThresholdBytes+1), "") + toOrphan.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, toOrphan); err != nil { + t.Fatalf("CreateMessage(toOrphan): %v", err) + } + orphanDigest := toOrphan.PayloadDigest + if orphanDigest == "" { + t.Fatal("expected toOrphan to externalize a payload digest") + } + + // Delete the only message referencing orphanDigest so its payload row + // becomes unreferenced, without touching the still-referenced payload + // row's message. + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `DELETE FROM task_session_messages WHERE id = ?`), toOrphan.ID); err != nil { + t.Fatalf("delete toOrphan message: %v", err) + } + + candidates, err := repo.ListOrphanedMessagePayloadCandidates(ctx, 0) + if err != nil { + t.Fatalf("ListOrphanedMessagePayloadCandidates: %v", err) + } + if len(candidates) != 1 { + t.Fatalf("candidates = %+v, want exactly 1 orphaned payload", candidates) + } + if candidates[0].Digest != orphanDigest { + t.Fatalf("candidate digest = %q, want %q", candidates[0].Digest, orphanDigest) + } + if candidates[0].CompressedSize <= 0 || candidates[0].UncompressedSize <= 0 { + t.Fatalf("candidate sizes = %+v, want positive compressed/uncompressed sizes", candidates[0]) + } + + // The still-referenced payload's digest must never appear. + for _, c := range candidates { + if c.Digest == referenced.PayloadDigest { + t.Fatalf("still-referenced digest %q reported as orphaned", referenced.PayloadDigest) + } + } +} + +// TestListOrphanedMessagePayloadCandidatesIsNonDestructive confirms the +// selection is read-only: calling it does not remove or alter any payload +// row, matching the plan's "non-destructive until maintenance executes" +// constraint. +func TestListOrphanedMessagePayloadCandidatesIsNonDestructive(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-orphan-nd", "sess-payload-orphan-nd", "turn-1") + + msg := newShellMessage("msg-orphan-nd", "sess-payload-orphan-nd", strings.Repeat("n", largeMessagePayloadThresholdBytes+1), "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `DELETE FROM task_session_messages WHERE id = ?`), msg.ID); err != nil { + t.Fatalf("delete message: %v", err) + } + + before := countRows(t, repo, `SELECT COUNT(*) FROM task_message_payloads`) + if _, err := repo.ListOrphanedMessagePayloadCandidates(ctx, 0); err != nil { + t.Fatalf("ListOrphanedMessagePayloadCandidates: %v", err) + } + after := countRows(t, repo, `SELECT COUNT(*) FROM task_message_payloads`) + if before != after { + t.Fatalf("row count changed from %d to %d after a read-only candidate listing", before, after) + } + if before == 0 { + t.Fatal("expected at least one payload row to exist for this test to be meaningful") + } +} diff --git a/apps/backend/internal/task/repository/sqlite/message_payload_test.go b/apps/backend/internal/task/repository/sqlite/message_payload_test.go new file mode 100644 index 0000000000..7e2380e159 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/message_payload_test.go @@ -0,0 +1,466 @@ +package sqlite + +import ( + "context" + "strings" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// shellOutputMetadata builds the metadata shape ProjectMessageMetadata/ +// ExtractShellExecOutput/RehydrateShellOutput all understand: a +// normalized.shell_exec.output nested map holding stdout/stderr. +func shellOutputMetadata(stdout, stderr string) map[string]interface{} { + return map[string]interface{}{ + "tool_call_id": "call-1", + "normalized": map[string]interface{}{ + "shell_exec": map[string]interface{}{ + "command": "echo", + "output": map[string]interface{}{ + "stdout": stdout, + "stderr": stderr, + "truncated": false, + }, + }, + }, + } +} + +func newShellMessage(id, sessionID string, stdout, stderr string) *models.Message { + return &models.Message{ + ID: id, + TaskSessionID: sessionID, + AuthorType: models.MessageAuthorAgent, + Type: models.MessageTypeToolCall, + Metadata: shellOutputMetadata(stdout, stderr), + } +} + +// TestCreateMessageLeavesSmallMetadataInline confirms the common case (small +// tool output) is untouched: no externalization, no digest, other metadata +// keys preserved verbatim (message_clarification_response.go and friends +// read several of them via direct SQL json_extract). +func TestCreateMessageLeavesSmallMetadataInline(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-small", "sess-payload-small", "turn-1") + + msg := newShellMessage("msg-small", "sess-payload-small", "ok", "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + if msg.PayloadDigest != "" { + t.Fatalf("PayloadDigest = %q, want empty for small output", msg.PayloadDigest) + } + + got, err := repo.GetMessage(ctx, "msg-small") + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + if got.PayloadDigest != "" { + t.Fatalf("persisted PayloadDigest = %q, want empty", got.PayloadDigest) + } + output, ok := models.ExtractShellExecOutput(got.Metadata) + if !ok || output.Stdout != "ok" { + t.Fatalf("ExtractShellExecOutput = (%+v, %v), want stdout=ok", output, ok) + } + if got.Metadata["tool_call_id"] != "call-1" { + t.Fatalf("tool_call_id metadata key was lost: %+v", got.Metadata) + } +} + +// TestCreateMessageExternalizesLargeShellOutputAndRehydrates covers the +// write -> project -> lazy-load -> verify round trip end to end. +func TestCreateMessageExternalizesLargeShellOutputAndRehydrates(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-large", "sess-payload-large", "turn-1") + + largeStdout := strings.Repeat("x", largeMessagePayloadThresholdBytes+1) + msg := newShellMessage("msg-large", "sess-payload-large", largeStdout, "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + if msg.PayloadDigest == "" { + t.Fatal("PayloadDigest was not set for output over the externalization threshold") + } + + // GetMessage (the lazy-detail read path) must return the small projected + // summary, not the full stdout, until explicitly rehydrated. + got, err := repo.GetMessage(ctx, "msg-large") + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + if got.PayloadDigest != msg.PayloadDigest { + t.Fatalf("persisted PayloadDigest = %q, want %q", got.PayloadDigest, msg.PayloadDigest) + } + normalized, _ := got.Metadata["normalized"].(map[string]interface{}) + shellExec, _ := normalized["shell_exec"].(map[string]interface{}) + if shellExec == nil || shellExec["output"] == nil { + t.Fatalf("projected metadata missing shell_exec output summary: %+v", got.Metadata) + } + if summary, ok := shellExec["output"].(map[string]interface{}); ok { + if hasOutput, ok := summary["has_output"].(bool); !ok || !hasOutput { + t.Fatalf("projected summary lost output presence: %+v", summary) + } + if stdoutBytes, ok := summary["stdout_bytes"].(float64); !ok || int(stdoutBytes) != len(largeStdout) { + t.Fatalf("projected summary stdout_bytes = %v, want %d", summary["stdout_bytes"], len(largeStdout)) + } + if _, hasStdout := summary["stdout"]; hasStdout { + t.Fatal("projected summary must not include the raw stdout body") + } + } + + // Explicit authorized rehydration restores the full output. + hydrationsBefore := metricMapValue(t, messagePayloadHydrationsTotal, "outcome=success") + if err := repo.RehydrateMessagePayload(ctx, got); err != nil { + t.Fatalf("RehydrateMessagePayload: %v", err) + } + output, ok := models.ExtractShellExecOutput(got.Metadata) + if !ok || output.Stdout != largeStdout { + t.Fatalf("rehydrated stdout length = %d, want %d (ok=%v)", len(output.Stdout), len(largeStdout), ok) + } + if delta := metricMapValue(t, messagePayloadHydrationsTotal, "outcome=success") - hydrationsBefore; delta != 1 { + t.Fatalf("successful hydration metric delta = %d, want 1", delta) + } +} + +func TestUpdateMessagePreservesExternalizedProjectedPayload(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-update", "sess-payload-update", "turn-1") + + largeStdout := strings.Repeat("u", largeMessagePayloadThresholdBytes+1) + msg := newShellMessage("msg-payload-update", "sess-payload-update", largeStdout, "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + if msg.PayloadDigest == "" { + t.Fatal("PayloadDigest was not set for large output") + } + + projected, err := repo.GetMessage(ctx, msg.ID) + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + projected.Content = "metadata-only update" + projected.Metadata["status"] = "complete" + if err := repo.UpdateMessage(ctx, projected); err != nil { + t.Fatalf("UpdateMessage: %v", err) + } + + got, err := repo.GetMessage(ctx, msg.ID) + if err != nil { + t.Fatalf("GetMessage after update: %v", err) + } + if got.PayloadDigest != msg.PayloadDigest { + t.Fatalf("PayloadDigest = %q, want preserved %q", got.PayloadDigest, msg.PayloadDigest) + } + if err := repo.RehydrateMessagePayload(ctx, got); err != nil { + t.Fatalf("RehydrateMessagePayload: %v", err) + } + output, ok := models.ExtractShellExecOutput(got.Metadata) + if !ok || output.Stdout != largeStdout { + t.Fatalf("rehydrated stdout length = %d, want %d (ok=%v)", len(output.Stdout), len(largeStdout), ok) + } +} + +func TestUpdateMessagePersistsNewExternalizedPayload(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-update-new", "sess-payload-update-new", "turn-1") + + message := newShellMessage("msg-payload-update-new", "sess-payload-update-new", "short", "") + message.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, message); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + + updated, err := repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("GetMessage before update: %v", err) + } + largeStdout := strings.Repeat("n", largeMessagePayloadThresholdBytes+1) + updated.Metadata = shellOutputMetadata(largeStdout, "") + updated.Content = "completed" + if err := repo.UpdateMessage(ctx, updated); err != nil { + t.Fatalf("UpdateMessage: %v", err) + } + + got, err := repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("GetMessage after update: %v", err) + } + if got.PayloadDigest == "" { + t.Fatal("PayloadDigest was not persisted for large output added by an update") + } + if got.PayloadSize <= 0 { + t.Fatalf("PayloadSize = %d, want a positive stored payload size", got.PayloadSize) + } + if err := repo.RehydrateMessagePayload(ctx, got); err != nil { + t.Fatalf("RehydrateMessagePayload: %v", err) + } + output, ok := models.ExtractShellExecOutput(got.Metadata) + if !ok || output.Stdout != largeStdout { + t.Fatalf("rehydrated stdout length = %d, want %d (ok=%v)", len(output.Stdout), len(largeStdout), ok) + } +} + +func TestUpdateMessageReplacesAndClearsExternalizedPayload(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-update-replace", "sess-payload-update-replace", "turn-1") + + firstOutput := strings.Repeat("a", largeMessagePayloadThresholdBytes+1) + message := newShellMessage("msg-payload-update-replace", "sess-payload-update-replace", firstOutput, "") + message.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, message); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + firstDigest := message.PayloadDigest + + updated, err := repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("GetMessage before replacement: %v", err) + } + secondOutput := strings.Repeat("b", largeMessagePayloadThresholdBytes+2) + updated.Metadata = shellOutputMetadata(secondOutput, "") + if err := repo.UpdateMessage(ctx, updated); err != nil { + t.Fatalf("UpdateMessage with replacement: %v", err) + } + + got, err := repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("GetMessage after replacement: %v", err) + } + if got.PayloadDigest == "" || got.PayloadDigest == firstDigest { + t.Fatalf("replacement PayloadDigest = %q, want a new non-empty digest", got.PayloadDigest) + } + if err := repo.RehydrateMessagePayload(ctx, got); err != nil { + t.Fatalf("RehydrateMessagePayload after replacement: %v", err) + } + output, ok := models.ExtractShellExecOutput(got.Metadata) + if !ok || output.Stdout != secondOutput { + t.Fatalf("replacement stdout length = %d, want %d (ok=%v)", len(output.Stdout), len(secondOutput), ok) + } + + updated, err = repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("GetMessage before inline transition: %v", err) + } + updated.Metadata = shellOutputMetadata("small", "") + if err := repo.UpdateMessage(ctx, updated); err != nil { + t.Fatalf("UpdateMessage with inline output: %v", err) + } + + got, err = repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("GetMessage after inline transition: %v", err) + } + if got.PayloadDigest != "" || got.PayloadSize != 0 { + t.Fatalf("inline transition retained payload reference %q/%d", got.PayloadDigest, got.PayloadSize) + } + output, ok = models.ExtractShellExecOutput(got.Metadata) + if !ok || output.Stdout != "small" { + t.Fatalf("inline stdout = %q, want small (ok=%v)", output.Stdout, ok) + } +} + +// TestExternalizeMessagePayloadDedupesIdenticalContentAcrossMessages proves +// the content-addressed store never writes the same payload bytes twice: +// two different messages with byte-identical large output share one +// task_message_payloads row. +func TestExternalizeMessagePayloadDedupesIdenticalContentAcrossMessages(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-dedup", "sess-payload-dedup", "turn-1") + + largeStdout := strings.Repeat("y", largeMessagePayloadThresholdBytes+1) + first := newShellMessage("msg-dedup-1", "sess-payload-dedup", largeStdout, "") + first.TurnID = "turn-1" + second := newShellMessage("msg-dedup-2", "sess-payload-dedup", largeStdout, "") + second.TurnID = "turn-1" + + if err := repo.CreateMessage(ctx, first); err != nil { + t.Fatalf("CreateMessage(first): %v", err) + } + if err := repo.CreateMessage(ctx, second); err != nil { + t.Fatalf("CreateMessage(second): %v", err) + } + + if first.PayloadDigest == "" || first.PayloadDigest != second.PayloadDigest { + t.Fatalf("expected identical payload digests, got %q and %q", first.PayloadDigest, second.PayloadDigest) + } + + count := countRows(t, repo, `SELECT COUNT(*) FROM task_message_payloads WHERE digest = ?`, first.PayloadDigest) + if count != 1 { + t.Fatalf("task_message_payloads rows for shared digest = %d, want 1 (content-addressed dedup)", count) + } +} + +// TestRehydrateMessagePayloadRejectsTamperedContent proves the integrity +// check is enforced: if the stored bytes no longer match the message's +// recorded digest (corruption, or a hostile write to the payload table), the +// rehydration must fail rather than silently return altered content. +func TestRehydrateMessagePayloadRejectsTamperedContent(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-tamper", "sess-payload-tamper", "turn-1") + + largeStdout := strings.Repeat("z", largeMessagePayloadThresholdBytes+1) + msg := newShellMessage("msg-tamper", "sess-payload-tamper", largeStdout, "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + + tampered, err := gzipCompress([]byte(`{"stdout":"not the original content"}`)) + if err != nil { + t.Fatalf("gzipCompress: %v", err) + } + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE task_message_payloads SET compressed_content = ? WHERE digest = ?`), + tampered, msg.PayloadDigest); err != nil { + t.Fatalf("tamper with stored payload: %v", err) + } + + got, err := repo.GetMessage(ctx, "msg-tamper") + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + if err := repo.RehydrateMessagePayload(ctx, got); err == nil { + t.Fatal("RehydrateMessagePayload accepted tampered content; want an integrity-check error") + } +} + +func TestRehydrateMessagePayloadRejectsOversizedStoredPayload(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-oversized", "sess-payload-oversized", "turn-1") + + largeStdout := strings.Repeat("z", largeMessagePayloadThresholdBytes+1) + msg := newShellMessage("msg-oversized", "sess-payload-oversized", largeStdout, "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + + replacementPayload := []byte(`{"stdout":"small but metadata claims it is too large"}`) + oversizedCompressed, err := gzipCompress(replacementPayload) + if err != nil { + t.Fatalf("gzipCompress: %v", err) + } + oversizedDigest := sha256Hex(replacementPayload) + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE task_message_payloads + SET digest = ?, compressed_content = ?, uncompressed_size = ?, compressed_size = ? + WHERE digest = ?`), + oversizedDigest, oversizedCompressed, maxMessagePayloadRehydrateBytes+1, len(oversizedCompressed), msg.PayloadDigest); err != nil { + t.Fatalf("replace stored payload: %v", err) + } + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE task_session_messages SET payload_digest = ?, payload_size = ? WHERE id = ?`), + oversizedDigest, maxMessagePayloadRehydrateBytes+1, msg.ID); err != nil { + t.Fatalf("point message at oversized payload: %v", err) + } + + got, err := repo.GetMessage(ctx, "msg-oversized") + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + if err := repo.RehydrateMessagePayload(ctx, got); err == nil { + t.Fatal("RehydrateMessagePayload accepted an oversized payload; want an error") + } +} + +func TestRehydrateMessagePayloadRejectsMismatchedStoredPayloadSize(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-size-mismatch", "sess-payload-size-mismatch", "turn-1") + + largeStdout := strings.Repeat("s", largeMessagePayloadThresholdBytes+1) + msg := newShellMessage("msg-size-mismatch", "sess-payload-size-mismatch", largeStdout, "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE task_message_payloads SET uncompressed_size = ? WHERE digest = ?`), + len(largeStdout)-1, msg.PayloadDigest); err != nil { + t.Fatalf("corrupt stored payload size: %v", err) + } + + got, err := repo.GetMessage(ctx, "msg-size-mismatch") + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + if err := repo.RehydrateMessagePayload(ctx, got); err == nil { + t.Fatal("RehydrateMessagePayload accepted mismatched payload size metadata") + } +} + +func TestRehydrateMessagePayloadRejectsMismatchedCompressedPayloadSize(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-payload-compressed-size-mismatch", "sess-payload-compressed-size-mismatch", "turn-1") + + largeStdout := strings.Repeat("c", largeMessagePayloadThresholdBytes+1) + msg := newShellMessage("msg-compressed-size-mismatch", "sess-payload-compressed-size-mismatch", largeStdout, "") + msg.TurnID = "turn-1" + if err := repo.CreateMessage(ctx, msg); err != nil { + t.Fatalf("CreateMessage: %v", err) + } + + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind( + `UPDATE task_message_payloads SET compressed_size = ? WHERE digest = ?`), + 1, msg.PayloadDigest); err != nil { + t.Fatalf("corrupt stored compressed payload size: %v", err) + } + + got, err := repo.GetMessage(ctx, "msg-compressed-size-mismatch") + if err != nil { + t.Fatalf("GetMessage: %v", err) + } + if err := repo.RehydrateMessagePayload(ctx, got); err == nil { + t.Fatal("RehydrateMessagePayload accepted mismatched compressed payload size metadata") + } +} + +func TestGzipDecompressRejectsActualOversize(t *testing.T) { + compressed, err := gzipCompress([]byte("123456789")) + if err != nil { + t.Fatalf("gzipCompress: %v", err) + } + if _, err := gzipDecompress(compressed, 8); err == nil { + t.Fatal("gzipDecompress accepted output beyond the configured cap") + } +} + +func TestGzipDecompressRejectsCompressedExpansionBomb(t *testing.T) { + plain := strings.Repeat("b", 1<<20) + compressed, err := gzipCompress([]byte(plain)) + if err != nil { + t.Fatalf("gzipCompress: %v", err) + } + if len(compressed) >= len(plain) { + t.Fatalf("fixture did not compress: compressed=%d plain=%d", len(compressed), len(plain)) + } + if _, err := gzipDecompress(compressed, 1024); err == nil { + t.Fatal("gzipDecompress accepted compressed expansion beyond the configured cap") + } +} + +func TestGzipDecompressRejectsExpectedOversize(t *testing.T) { + compressed, err := gzipCompress([]byte("123456789")) + if err != nil { + t.Fatalf("gzipCompress: %v", err) + } + if _, err := gzipDecompressExpectedSize(compressed, 8, 9); err == nil { + t.Fatal("gzipDecompressExpectedSize accepted an expected size above the configured cap") + } +} diff --git a/apps/backend/internal/task/repository/sqlite/metrics_vars.go b/apps/backend/internal/task/repository/sqlite/metrics_vars.go new file mode 100644 index 0000000000..65b36dc687 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/metrics_vars.go @@ -0,0 +1,29 @@ +package sqlite + +import ( + "expvar" + "strconv" + "time" +) + +var ( + messagePayloadHydrationsTotal = expvar.NewMap("task_message_payload_hydrations_total") + messagePayloadHydrationMS = expvar.NewMap("task_message_payload_hydration_latency_ms") +) + +var hydrationBuckets = [...]int64{10, 50, 250, 1_000} + +func recordMessagePayloadHydration(elapsed time.Duration, err error) { + outcome := "success" + if err != nil { + outcome = "error" + } + messagePayloadHydrationsTotal.Add("outcome="+outcome, 1) + millis := elapsed.Milliseconds() + for _, upperBound := range hydrationBuckets { + if millis <= upperBound { + messagePayloadHydrationMS.Add("outcome="+outcome+";le="+strconv.FormatInt(upperBound, 10), 1) + } + } + messagePayloadHydrationMS.Add("outcome="+outcome+";le=+Inf", 1) +} diff --git a/apps/backend/internal/task/repository/sqlite/metrics_vars_test.go b/apps/backend/internal/task/repository/sqlite/metrics_vars_test.go new file mode 100644 index 0000000000..7dd7484489 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/metrics_vars_test.go @@ -0,0 +1,44 @@ +package sqlite + +import ( + "expvar" + "strconv" + "testing" + "time" +) + +func TestMessagePayloadHydrationMetricsPublished(t *testing.T) { + for _, name := range []string{ + "task_message_payload_hydrations_total", + "task_message_payload_hydration_latency_ms", + } { + if expvar.Get(name) == nil { + t.Errorf("expvar %q is not published", name) + } + } +} + +func metricMapValue(t *testing.T, metric *expvar.Map, key string) int64 { + t.Helper() + value := metric.Get(key) + if value == nil { + return 0 + } + got, err := strconv.ParseInt(value.String(), 10, 64) + if err != nil { + t.Fatalf("metric %q value %q is not an integer: %v", key, value.String(), err) + } + return got +} + +func TestRecordMessagePayloadHydrationUsesBoundedOutcomeAndLatencyLabels(t *testing.T) { + successBefore := metricMapValue(t, messagePayloadHydrationsTotal, "outcome=success") + bucketBefore := metricMapValue(t, messagePayloadHydrationMS, "outcome=success;le=50") + recordMessagePayloadHydration(25*time.Millisecond, nil) + if got := metricMapValue(t, messagePayloadHydrationsTotal, "outcome=success") - successBefore; got != 1 { + t.Fatalf("success hydration delta = %d, want 1", got) + } + if got := metricMapValue(t, messagePayloadHydrationMS, "outcome=success;le=50") - bucketBefore; got != 1 { + t.Fatalf("50ms hydration bucket delta = %d, want 1", got) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/plan.go b/apps/backend/internal/task/repository/sqlite/plan.go index f140609253..8cdd2bd93f 100644 --- a/apps/backend/internal/task/repository/sqlite/plan.go +++ b/apps/backend/internal/task/repository/sqlite/plan.go @@ -237,6 +237,79 @@ func (r *Repository) ListTaskPlanRevisions(ctx context.Context, taskID string, l return out, nil } +// ObsoletePlanRevisionCandidate identifies a plan revision that is +// policy-eligible for future pruning by the maintenance command: never the +// task's current HEAD, never referenced by another revision's revert-of +// ancestry link, and outside the caller-supplied recency window. +type ObsoletePlanRevisionCandidate struct { + ID string + TaskID string + RevisionNumber int + ContentBytes int64 +} + +// ListObsoletePlanRevisionCandidates returns policy-eligible superseded plan +// revisions for a task. It always protects the current HEAD +// (MAX(revision_number)) and any revision that some other revision's +// revert_of_revision_id points to (so revert ancestry - the "restore this +// earlier plan" chain - is never broken). When keepLastN > 0, it further +// protects that many of the most recent non-HEAD revisions from being +// reported, regardless of ancestry. This is a read-only, non-destructive +// selection: it reports candidates for a later maintenance command to act +// on, and never deletes or modifies anything itself. +func (r *Repository) ListObsoletePlanRevisionCandidates(ctx context.Context, taskID string, keepLastN int, limit int) ([]ObsoletePlanRevisionCandidate, error) { + query := ` + SELECT rev.id, rev.task_id, rev.revision_number, LENGTH(rev.content) + FROM task_plan_revisions rev + WHERE rev.task_id = ? + AND rev.revision_number < ( + SELECT MAX(revision_number) FROM task_plan_revisions WHERE task_id = ? + ) + AND NOT EXISTS ( + SELECT 1 FROM task_plan_revisions child + WHERE child.task_id = rev.task_id AND child.revert_of_revision_id = rev.id + )` + args := []interface{}{taskID, taskID} + if keepLastN > 0 { + query += ` + AND rev.id NOT IN ( + SELECT recent.id + FROM task_plan_revisions recent + WHERE recent.task_id = ? + AND recent.revision_number < ( + SELECT MAX(revision_number) FROM task_plan_revisions WHERE task_id = ? + ) + ORDER BY recent.revision_number DESC + ` + sqlLimitClause + ` + )` + args = append(args, taskID, taskID, keepLastN) + } + query += ` ORDER BY rev.revision_number ASC` + if limit > 0 { + query += sqlLimitClause + args = append(args, limit) + } + + rows, err := r.ro.QueryContext(ctx, r.ro.Rebind(query), args...) + if err != nil { + return nil, fmt.Errorf("failed to list obsolete plan revision candidates: %w", err) + } + defer func() { _ = rows.Close() }() + + var out []ObsoletePlanRevisionCandidate + for rows.Next() { + var c ObsoletePlanRevisionCandidate + if err := rows.Scan(&c.ID, &c.TaskID, &c.RevisionNumber, &c.ContentBytes); err != nil { + return nil, fmt.Errorf("failed to scan obsolete plan revision candidate: %w", err) + } + out = append(out, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate obsolete plan revision candidates: %w", err) + } + return out, nil +} + // ListTaskPlanRevisionMetadata returns bounded newest-first revision metadata // without loading revision content. beforeRevisionNumber is an exclusive, // positive cursor; a non-positive value starts at the newest revision. diff --git a/apps/backend/internal/task/repository/sqlite/plan_retention_test.go b/apps/backend/internal/task/repository/sqlite/plan_retention_test.go new file mode 100644 index 0000000000..18feae71eb --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/plan_retention_test.go @@ -0,0 +1,176 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestListObsoletePlanRevisionCandidatesProtectsHeadAndAncestry is this +// wave's coverage for the plan-revision half of "non-destructive retention +// candidate selection": HEAD and any revision referenced by another +// revision's revert-of ancestry link must never be reported, no matter how +// old they are. +func TestListObsoletePlanRevisionCandidatesProtectsHeadAndAncestry(t *testing.T) { + repo := newRepoForEntityTests(t) + ctx := context.Background() + seedTaskForDocs(t, repo, "task-planrev-retention") + + base := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC) + revertOf1 := "planrev-ret-1" + revisions := []*models.TaskPlanRevision{ + {ID: "planrev-ret-1", TaskID: "task-planrev-retention", RevisionNumber: 1, Title: "v1", Content: "one", CreatedAt: base}, + {ID: "planrev-ret-2", TaskID: "task-planrev-retention", RevisionNumber: 2, Title: "v2", Content: "two", CreatedAt: base.Add(time.Hour)}, + // Revision 3 reverts back to revision 1's content, so revision 1 must + // stay protected by ancestry despite not being HEAD. + {ID: "planrev-ret-3", TaskID: "task-planrev-retention", RevisionNumber: 3, Title: "v1-restored", Content: "one", + RevertOfRevisionID: &revertOf1, CreatedAt: base.Add(2 * time.Hour)}, + {ID: "planrev-ret-4", TaskID: "task-planrev-retention", RevisionNumber: 4, Title: "v4 (HEAD)", Content: "four", CreatedAt: base.Add(3 * time.Hour)}, + } + for _, rev := range revisions { + if err := repo.InsertTaskPlanRevision(ctx, rev); err != nil { + t.Fatalf("InsertTaskPlanRevision(%s): %v", rev.ID, err) + } + } + + candidates, err := repo.ListObsoletePlanRevisionCandidates(ctx, "task-planrev-retention", 0, 0) + if err != nil { + t.Fatalf("ListObsoletePlanRevisionCandidates: %v", err) + } + // Revision 1 is protected because revision 3 reverts to it (ancestry); + // revision 4 is protected because it's HEAD. Revision 3 itself is + // neither HEAD nor referenced by any other revision, so it is reported + // too - the ancestry protection is expected to preserve the *target* of + // a revert, not the revert action itself, since the target already + // carries the content anyone would want to restore. + want := map[string]bool{"planrev-ret-2": true, "planrev-ret-3": true} + if len(candidates) != len(want) { + t.Fatalf("candidates = %+v, want exactly %v", candidates, want) + } + for _, c := range candidates { + if !want[c.ID] { + t.Fatalf("unexpected candidate %q; HEAD (planrev-ret-4) and ancestry target (planrev-ret-1) must be protected", c.ID) + } + if c.ID == "planrev-ret-2" && c.ContentBytes != int64(len("two")) { + t.Fatalf("ContentBytes = %d, want %d", c.ContentBytes, len("two")) + } + } +} + +// TestListObsoletePlanRevisionCandidatesRespectsRecencyWindow confirms +// keepLastN additionally protects the most recent non-HEAD revisions from +// being reported, on top of the HEAD/ancestry protections. +func TestListObsoletePlanRevisionCandidatesRespectsRecencyWindow(t *testing.T) { + repo := newRepoForEntityTests(t) + ctx := context.Background() + seedTaskForDocs(t, repo, "task-planrev-window") + + base := time.Date(2026, 2, 2, 0, 0, 0, 0, time.UTC) + for i := 1; i <= 5; i++ { + rev := &models.TaskPlanRevision{ + ID: "planrev-win-" + string(rune('0'+i)), TaskID: "task-planrev-window", + RevisionNumber: i, Title: "v", Content: "c", + CreatedAt: base.Add(time.Duration(i) * time.Hour), + } + if err := repo.InsertTaskPlanRevision(ctx, rev); err != nil { + t.Fatalf("InsertTaskPlanRevision(%d): %v", i, err) + } + } + + // HEAD is revision 5. keepLastN=2 additionally protects revisions 4 + // and 3, leaving revisions 1-2 as candidates. + candidates, err := repo.ListObsoletePlanRevisionCandidates(ctx, "task-planrev-window", 2, 0) + if err != nil { + t.Fatalf("ListObsoletePlanRevisionCandidates: %v", err) + } + if len(candidates) != 2 { + t.Fatalf("candidates = %d, want 2 (revisions 1-2)", len(candidates)) + } + for _, c := range candidates { + if c.RevisionNumber > 2 { + t.Fatalf("candidate revision_number = %d, want <= 2 (recency window must protect revisions 3 and 4)", c.RevisionNumber) + } + } +} + +func TestListObsoletePlanRevisionCandidatesProtectsRecentRowsWithGaps(t *testing.T) { + repo := newRepoForEntityTests(t) + ctx := context.Background() + seedTaskForDocs(t, repo, "task-planrev-gap-window") + + base := time.Date(2026, 2, 3, 0, 0, 0, 0, time.UTC) + for _, number := range []int{1, 2, 10, 11} { + rev := &models.TaskPlanRevision{ + ID: "planrev-gap-" + string(rune('a'+number)), + TaskID: "task-planrev-gap-window", + RevisionNumber: number, + Title: "v", + Content: "c", + CreatedAt: base.Add(time.Duration(number) * time.Hour), + } + if err := repo.InsertTaskPlanRevision(ctx, rev); err != nil { + t.Fatalf("InsertTaskPlanRevision(%d): %v", number, err) + } + } + + candidates, err := repo.ListObsoletePlanRevisionCandidates(ctx, "task-planrev-gap-window", 2, 0) + if err != nil { + t.Fatalf("ListObsoletePlanRevisionCandidates: %v", err) + } + if len(candidates) != 1 || candidates[0].RevisionNumber != 1 { + t.Fatalf("candidates = %+v, want only revision 1; revisions 10 and 2 are the two newest non-HEAD rows", candidates) + } +} + +func TestListObsoletePlanRevisionCandidatesAppliesLimitInQuery(t *testing.T) { + repo := newRepoForEntityTests(t) + ctx := context.Background() + seedTaskForDocs(t, repo, "task-planrev-limit") + + for i := 1; i <= 5; i++ { + if err := repo.InsertTaskPlanRevision(ctx, &models.TaskPlanRevision{ + ID: "planrev-limit-" + string(rune('0'+i)), TaskID: "task-planrev-limit", + RevisionNumber: i, Title: "v", Content: "c", + }); err != nil { + t.Fatalf("InsertTaskPlanRevision(%d): %v", i, err) + } + } + + candidates, err := repo.ListObsoletePlanRevisionCandidates(ctx, "task-planrev-limit", 0, 2) + if err != nil { + t.Fatalf("ListObsoletePlanRevisionCandidates: %v", err) + } + if len(candidates) != 2 { + t.Fatalf("candidates = %d, want SQL-limited 2", len(candidates)) + } +} + +// TestListObsoletePlanRevisionCandidatesIsNonDestructive confirms the +// selection is read-only: calling it does not remove or alter any revision +// row, matching the plan's "non-destructive until maintenance executes" +// constraint. +func TestListObsoletePlanRevisionCandidatesIsNonDestructive(t *testing.T) { + repo := newRepoForEntityTests(t) + ctx := context.Background() + seedTaskForDocs(t, repo, "task-planrev-nondestructive") + + for i := 1; i <= 2; i++ { + if err := repo.InsertTaskPlanRevision(ctx, &models.TaskPlanRevision{ + ID: "planrev-nd-" + string(rune('0'+i)), TaskID: "task-planrev-nondestructive", + RevisionNumber: i, Title: "v", Content: "c", + }); err != nil { + t.Fatalf("InsertTaskPlanRevision(%d): %v", i, err) + } + } + + before := countRows(t, repo, `SELECT COUNT(*) FROM task_plan_revisions WHERE task_id = ?`, "task-planrev-nondestructive") + if _, err := repo.ListObsoletePlanRevisionCandidates(ctx, "task-planrev-nondestructive", 0, 0); err != nil { + t.Fatalf("ListObsoletePlanRevisionCandidates: %v", err) + } + after := countRows(t, repo, `SELECT COUNT(*) FROM task_plan_revisions WHERE task_id = ?`, "task-planrev-nondestructive") + if before != after { + t.Fatalf("row count changed from %d to %d after a read-only candidate listing", before, after) + } +} diff --git a/apps/backend/internal/task/service/constants.go b/apps/backend/internal/task/service/constants.go index b8c224c2fa..59b9f6f4c2 100644 --- a/apps/backend/internal/task/service/constants.go +++ b/apps/backend/internal/task/service/constants.go @@ -1,6 +1,11 @@ package service const ( - DefaultMessagesPageSize = 10 + // DefaultMessagesPageSize is the bounded first-page size returned when a + // history request supplies no pagination parameters at all (see + // httpListMessages/wsListMessages), matching the PR-watch/storage-bounds + // plan's requirement that normal history hydration never eagerly + // materializes a whole session's messages by default. + DefaultMessagesPageSize = 50 MaxMessagesPageSize = 100 ) diff --git a/apps/backend/internal/task/service/service_messages.go b/apps/backend/internal/task/service/service_messages.go index dbceae0ab2..3b0b67850e 100644 --- a/apps/backend/internal/task/service/service_messages.go +++ b/apps/backend/internal/task/service/service_messages.go @@ -817,6 +817,16 @@ func (s *Service) GetMessage(ctx context.Context, id string) (*models.Message, e return message, nil } +// RehydrateMessagePayload resolves an externalized large tool-output +// payload (see PayloadDigest) back into message.Metadata for the explicit, +// single-message lazy-detail routes (e.g. httpGetShellOutput). Callers must +// have already authorized access to message (typically via GetMessage), so +// this performs no additional authorization itself. A no-op when the +// message has no external payload. +func (s *Service) RehydrateMessagePayload(ctx context.Context, message *models.Message) error { + return s.messages.RehydrateMessagePayload(ctx, message) +} + // GetMessageWithPromptIndex retrieves a message by ID with its computed // prompt ordinal, scoped like GetMessage. Used by the idempotent WS // replay/response path so a retried prompt answers with its stable index. diff --git a/apps/backend/internal/task/statussummary/metrics_vars.go b/apps/backend/internal/task/statussummary/metrics_vars.go new file mode 100644 index 0000000000..cfbb800485 --- /dev/null +++ b/apps/backend/internal/task/statussummary/metrics_vars.go @@ -0,0 +1,15 @@ +package statussummary + +import "expvar" + +// Aggregate counters are intentionally unlabeled: a task or workspace label +// would create unbounded cardinality and expose user-controlled identifiers. +var ( + casRetriesTotal = expvar.NewInt("task_status_summary_cas_retries_total") + casExhaustionsTotal = expvar.NewInt("task_status_summary_cas_exhaustions_total") + eventHandlerFailuresTotal = expvar.NewInt("task_status_summary_event_handler_failures_total") +) + +func recordCASRetry() { casRetriesTotal.Add(1) } +func recordCASExhaustion() { casExhaustionsTotal.Add(1) } +func recordHandlerFailure() { eventHandlerFailuresTotal.Add(1) } diff --git a/apps/backend/internal/task/statussummary/metrics_vars_test.go b/apps/backend/internal/task/statussummary/metrics_vars_test.go new file mode 100644 index 0000000000..e53f42dc62 --- /dev/null +++ b/apps/backend/internal/task/statussummary/metrics_vars_test.go @@ -0,0 +1,34 @@ +package statussummary + +import ( + "context" + "expvar" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" +) + +func TestStatusSummaryOperationalMetricsPublished(t *testing.T) { + for _, name := range []string{ + "task_status_summary_cas_retries_total", + "task_status_summary_cas_exhaustions_total", + "task_status_summary_event_handler_failures_total", + } { + if expvar.Get(name) == nil { + t.Errorf("expvar %q is not published", name) + } + } +} + +func TestSubscribedEventHandlerRecordsFailures(t *testing.T) { + _, _, eventBus, _, _ := newProjectorTest(t) + before := eventHandlerFailuresTotal.Value() + event := bus.NewEvent(events.TaskCreated, "test", make(chan int)) + if err := eventBus.Publish(context.Background(), events.TaskCreated, event); err != nil { + t.Fatalf("Publish: %v", err) + } + if delta := eventHandlerFailuresTotal.Value() - before; delta != 1 { + t.Fatalf("subscribed handler failure metric delta = %d, want 1", delta) + } +} diff --git a/apps/backend/internal/task/statussummary/projector.go b/apps/backend/internal/task/statussummary/projector.go index 8b2e94b8ca..8b7a603743 100644 --- a/apps/backend/internal/task/statussummary/projector.go +++ b/apps/backend/internal/task/statussummary/projector.go @@ -3,6 +3,7 @@ package statussummary import ( "context" "fmt" + "math/rand" "sync" "time" @@ -108,6 +109,14 @@ type ProjectorConfig struct { CountQueuedPrompts func(context.Context, string) (int, error) Logger *logger.Logger Now func() time.Time + // RetryBackoff paces genuine compare-and-set retries so two writers that + // collide do not immediately collide again. It receives the zero-based + // retry attempt (0 for the wait before the second try) and must return + // once the caller should retry, or ctx.Err() if ctx is done first. Nil + // selects the built-in bounded exponential backoff with jitter; tests may + // inject a fast/no-op implementation to keep unit tests instantaneous + // while still exercising the retry path. + RetryBackoff func(ctx context.Context, attempt int) error } // Projector converts authoritative, bounded occurrences into one complete @@ -128,6 +137,7 @@ type Projector struct { countQueuedPrompts func(context.Context, string) (int, error) logger *logger.Logger now func() time.Time + retryBackoff func(ctx context.Context, attempt int) error mu sync.Mutex state map[string]*projectionState @@ -210,6 +220,10 @@ func NewProjector(cfg ProjectorConfig) *Projector { if now == nil { now = func() time.Time { return time.Now().UTC() } } + retryBackoff := cfg.RetryBackoff + if retryBackoff == nil { + retryBackoff = defaultCASRetryBackoff + } return &Projector{ store: cfg.Store, eventBus: cfg.EventBus, @@ -224,11 +238,65 @@ func NewProjector(cfg ProjectorConfig) *Projector { countQueuedPrompts: cfg.CountQueuedPrompts, logger: log.WithFields(zap.String("component", "task-status-summary-projector")), now: now, + retryBackoff: retryBackoff, state: make(map[string]*projectionState), taskLocks: make(map[string]*taskProjectionLock), } } +// casRetryBaseDelay and casRetryMaxDelay bound the in-process wait between a +// rejected compare-and-set and the next retry. These are intentionally small +// (single-digit milliseconds): the goal is only to avoid two writers +// retrying in lockstep, not to throttle event handling. A per-task mutex +// already serializes this projector's own writers, so this backoff exists for +// the cross-writer case (a concurrent boot reconciliation or HTTP rebuild +// racing the live projector on the same row). +const ( + casRetryBaseDelay = 4 * time.Millisecond + casRetryMaxDelay = 64 * time.Millisecond +) + +// defaultCASRetryBackoff waits a bounded exponential delay with up to +25% +// jitter before a genuine compare-and-set retry. It never increases the +// caller's attempt bound on its own; it only paces the attempts that already +// exist so a competing writer has a chance to finish first. +func defaultCASRetryBackoff(ctx context.Context, attempt int) error { + delay := casRetryBaseDelay << attempt + if delay <= 0 || delay > casRetryMaxDelay { + delay = casRetryMaxDelay + } + delay += casRetryJitter(delay) + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// casRetryJitter returns a random offset in [0, base/4] so concurrent +// retriers spread out instead of colliding again on the same tick. +func casRetryJitter(base time.Duration) time.Duration { + quarter := int64(base / 4) + if quarter <= 0 { + return 0 + } + return time.Duration(rand.Int63n(quarter + 1)) //nolint:gosec +} + +// waitForCASRetry backs off before a genuine compare-and-set retry. Callers +// pass the zero-based index of the retry about to be attempted (0 for the +// wait before the second overall attempt). +func (p *Projector) waitForCASRetry(ctx context.Context, attempt int) error { + backoff := p.retryBackoff + if backoff == nil { + backoff = defaultCASRetryBackoff + } + return backoff(ctx, attempt) +} + // Start subscribes only to source occurrences that can affect a summary. // In particular, it does not subscribe to agent.stream, shell, process, // model, MCP, or per-message streaming subjects. @@ -259,7 +327,7 @@ func (p *Projector) Start(ctx context.Context) error { events.MessageQueueStatusChanged, } for _, pattern := range patterns { - sub, err := p.eventBus.Subscribe(pattern, p.handleEvent) + sub, err := p.eventBus.Subscribe(pattern, p.HandleEvent) if err != nil { p.Close() return fmt.Errorf("subscribe task status summary source %q: %w", pattern, err) @@ -291,7 +359,11 @@ func (p *Projector) Close() { // HandleEvent is exported for focused tests and for callers that already // multiplex event-bus subscriptions. Start normally installs it directly. func (p *Projector) HandleEvent(ctx context.Context, event *bus.Event) error { - return p.handleEvent(ctx, event) + err := p.handleEvent(ctx, event) + if err != nil { + recordHandlerFailure() + } + return err } func (p *Projector) handleEvent(ctx context.Context, event *bus.Event) error { @@ -435,7 +507,14 @@ func (p *Projector) persistPendingRefreshLocked( if eventType != "" { p.applySourceEventLocked(state, eventType, eventData) } + if attempt < maxPendingPersistAttempts-1 { + recordCASRetry() + if err := p.waitForCASRetry(ctx, attempt); err != nil { + return fmt.Errorf("wait before CAS retry refreshing pending task status %q: %w", taskID, err) + } + } } + recordCASExhaustion() p.logger.Warn("exhausted CAS retries refreshing pending task status", zap.String("task_id", taskID), zap.Int("attempts", maxPendingPersistAttempts)) @@ -487,7 +566,14 @@ func (p *Projector) applyQueueStatusEvent( return err } sourceChanged = true + if attempt < maxQueueCountPersistAttempts-1 { + recordCASRetry() + if err := p.waitForCASRetry(ctx, attempt); err != nil { + return fmt.Errorf("wait before CAS retry updating queued prompt count %q: %w", taskID, err) + } + } } + recordCASExhaustion() // The count self-corrects on the next queue event or list load, but a // sustained contention run is worth surfacing so a repeated rejector is not // silently starved. diff --git a/apps/backend/internal/task/statussummary/projector_contention_test.go b/apps/backend/internal/task/statussummary/projector_contention_test.go new file mode 100644 index 0000000000..810591c8bc --- /dev/null +++ b/apps/backend/internal/task/statussummary/projector_contention_test.go @@ -0,0 +1,249 @@ +package statussummary + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" +) + +// alternatingRejectProjectorStore models a second, unsynchronized writer (a +// boot reconciliation pass or an HTTP rebuild request) racing the live +// projector on the same row. It rejects every odd-numbered attempt across the +// whole store by bumping the stored revision out from under the caller, and +// accepts every even-numbered attempt. Because the projector's per-task lock +// makes every attempt within one HandleEvent call (including its own +// rebase-and-retry sequence) contiguous in this global order, this +// deterministically forces at least one genuine CAS rejection under +// concurrency without ever requiring more than one retry per event - keeping +// the test itself non-flaky while still exercising real contention. +type alternatingRejectProjectorStore struct { + *projectorTestStore + counter int64 +} + +func (s *alternatingRejectProjectorStore) CompareAndUpdateTaskStatusSummary( + ctx context.Context, + stored *StoredTaskStatusSummary, +) (bool, error) { + n := atomic.AddInt64(&s.counter, 1) + if n%2 == 1 { + s.mu.Lock() + bumped := StoredTaskStatusSummary{TaskID: stored.TaskID, WorkspaceID: stored.WorkspaceID} + if previous := s.rows[stored.TaskID]; previous != nil { + bumped = *previous + } + bumped.Summary.Revision++ + s.rows[stored.TaskID] = &bumped + s.mu.Unlock() + return false, nil + } + return s.projectorTestStore.CompareAndUpdateTaskStatusSummary(ctx, stored) +} + +// TestProjectorConcurrentSessionEventsConvergeWithoutExhaustedCAS drives many +// goroutines applying distinct session observations for one task while a +// simulated external writer (boot reconciliation/HTTP rebuild analog) races +// the live projector on the same row. It asserts zero exhausted-CAS handler +// errors, that the jittered backoff hook actually fires under contention, and +// that the final summary reflects every concurrently applied session - +// proving a genuine CAS loss reloads/rebases authoritative state instead of +// silently dropping concurrent work. +func TestProjectorConcurrentSessionEventsConvergeWithoutExhaustedCAS(t *testing.T) { + const taskID = "task-concurrent-cas" + base := newProjectorTestStore() + base.rows[taskID] = &StoredTaskStatusSummary{ + TaskID: taskID, + WorkspaceID: "workspace-1", + Summary: TaskStatusSummary{Revision: 1}, + } + store := &alternatingRejectProjectorStore{projectorTestStore: base} + + var registryMu sync.Mutex + registry := make(map[string]RebuildSession) + registerSession := func(id string) { + registryMu.Lock() + registry[id] = RebuildSession{ID: id, State: sessionStateRunning, ActiveSubagentCount: 1} + registryMu.Unlock() + } + loadSessions := func(context.Context, string) (SessionObservationSnapshot, error) { + registryMu.Lock() + defer registryMu.Unlock() + sessions := make([]RebuildSession, 0, len(registry)) + for _, session := range registry { + sessions = append(sessions, session) + } + return SessionObservationSnapshot{Sessions: sessions, ActivityObserved: true}, nil + } + + var backoffCalls int64 + projector := NewProjector(ProjectorConfig{ + Store: store, + LoadSessionObservations: loadSessions, + RetryBackoff: func(context.Context, int) error { + atomic.AddInt64(&backoffCalls, 1) + return nil + }, + }) + + const sessionCount = 24 + var wg sync.WaitGroup + errs := make(chan error, sessionCount) + for i := 0; i < sessionCount; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + sessionID := fmt.Sprintf("session-%02d", i) + registerSession(sessionID) + err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{ + "task_id": taskID, + "workspace_id": "workspace-1", + "session_id": sessionID, + "new_state": sessionStateRunning, + "foreground_activity": activityGenerating, + "active_subagent_count": 1, + })) + errs <- err + }(i) + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent HandleEvent returned error (want zero exhausted-CAS handler errors): %v", err) + } + } + + if atomic.LoadInt64(&backoffCalls) == 0 { + t.Fatal("expected the CAS-retry backoff hook to fire at least once under contention") + } + + got := base.summary(taskID) + if got == nil { + t.Fatal("expected a persisted summary after concurrent contention") + } + if got.ActiveSubagentCount != sessionCount { + t.Fatalf("ActiveSubagentCount = %d, want %d (every concurrently applied session preserved)", + got.ActiveSubagentCount, sessionCount) + } + if got.ForegroundActivity != activityGenerating { + t.Fatalf("ForegroundActivity = %q, want %q", got.ForegroundActivity, activityGenerating) + } +} + +// TestProjectorCoalescesConcurrentEquivalentPendingRefreshes drives many +// goroutines that all apply the exact same session observation for one task +// concurrently. Because the projector serializes per-task application and +// treats a semantically-equal derived summary as an already-applied no-op, +// only the first genuinely-changing call should persist or publish; every +// other concurrent duplicate must coalesce into that same outcome instead of +// producing its own store write or event. +func TestProjectorCoalescesConcurrentEquivalentPendingRefreshes(t *testing.T) { + const taskID = "task-coalesce" + store := newProjectorTestStore() + eventBus := bus.NewMemoryEventBus(logger.Default()) + defer eventBus.Close() + var updates atomic.Int64 + if _, err := eventBus.Subscribe(events.TaskStatusSummaryUpdated, func(context.Context, *bus.Event) error { + updates.Add(1) + return nil + }); err != nil { + t.Fatal(err) + } + + projector := NewProjector(ProjectorConfig{ + Store: store, + EventBus: eventBus, + }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if err := projector.Start(ctx); err != nil { + t.Fatal(err) + } + defer projector.Close() + + const callers = 20 + var wg sync.WaitGroup + errs := make(chan error, callers) + for i := 0; i < callers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := projector.HandleEvent(context.Background(), bus.NewEvent(events.TaskSessionStateChanged, "test", map[string]interface{}{ + "task_id": taskID, + "workspace_id": "workspace-1", + "session_id": "session-shared", + "new_state": sessionStateRunning, + "is_primary": true, + "foreground_activity": activityGenerating, + "active_subagent_count": 3, + })) + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent equivalent HandleEvent returned error: %v", err) + } + } + + // Publish is synchronous in the in-memory bus, so by the time every + // goroutine's HandleEvent call has returned, every publish it triggered + // has already been delivered. + if got := updates.Load(); got != 1 { + t.Fatalf("SummaryUpdated publish count = %d, want exactly 1 (coalesced)", got) + } + got := store.summary(taskID) + if got == nil || got.ActiveSubagentCount != 3 || got.ForegroundActivity != activityGenerating { + t.Fatalf("summary = %+v, want single coalesced application", got) + } + if got.Revision != 1 { + t.Fatalf("revision = %d, want 1 (only the first change persisted)", got.Revision) + } +} + +func TestDefaultCASRetryBackoffRespectsContextCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + start := time.Now() + err := defaultCASRetryBackoff(ctx, 5) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if elapsed := time.Since(start); elapsed > 50*time.Millisecond { + t.Fatalf("cancelled backoff took %s, want near-instant", elapsed) + } +} + +func TestCASRetryJitterStaysWithinBounds(t *testing.T) { + base := 40 * time.Millisecond + for i := 0; i < 200; i++ { + jitter := casRetryJitter(base) + if jitter < 0 || jitter > base/4 { + t.Fatalf("jitter = %s, want within [0, %s]", jitter, base/4) + } + } +} + +func TestDefaultCASRetryBackoffDelayGrowsAndCaps(t *testing.T) { + for attempt := 0; attempt < 6; attempt++ { + start := time.Now() + if err := defaultCASRetryBackoff(context.Background(), attempt); err != nil { + t.Fatalf("attempt %d: unexpected error: %v", attempt, err) + } + elapsed := time.Since(start) + maxAllowed := casRetryMaxDelay + casRetryMaxDelay/4 + 30*time.Millisecond // scheduling slack + if elapsed > maxAllowed { + t.Fatalf("attempt %d backoff took %s, want <= %s", attempt, elapsed, maxAllowed) + } + } +} diff --git a/apps/backend/internal/task/statussummary/projector_load_test.go b/apps/backend/internal/task/statussummary/projector_load_test.go new file mode 100644 index 0000000000..56aabf9365 --- /dev/null +++ b/apps/backend/internal/task/statussummary/projector_load_test.go @@ -0,0 +1,186 @@ +package statussummary + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" +) + +// perTaskAlternatingRejectStore models the same "second, unsynchronized +// writer" contention as alternatingRejectProjectorStore (a boot +// reconciliation pass or an HTTP rebuild request racing the live projector), +// but keeps a per-task counter instead of one global counter. The projector's +// per-task lock only guarantees that one task's own CAS attempts stay +// contiguous in a shared *global* ordering when no other task's calls can +// interleave between them; with many tasks contending concurrently on one +// global counter, an unrelated task's accepted call can land between two of +// this task's attempts and break the "reject-then-accept" guarantee within +// maxPendingPersistAttempts. Keying the alternation per task preserves that +// guarantee at scale while still deterministically forcing a genuine CAS +// rejection (and therefore a real retry/rebase) for every task, every cycle. +type perTaskAlternatingRejectStore struct { + *projectorTestStore + counterMu sync.Mutex + counters map[string]int64 +} + +func (s *perTaskAlternatingRejectStore) CompareAndUpdateTaskStatusSummary( + ctx context.Context, + stored *StoredTaskStatusSummary, +) (bool, error) { + s.counterMu.Lock() + if s.counters == nil { + s.counters = make(map[string]int64) + } + s.counters[stored.TaskID]++ + n := s.counters[stored.TaskID] + s.counterMu.Unlock() + if n%2 == 1 { + s.mu.Lock() + bumped := StoredTaskStatusSummary{TaskID: stored.TaskID, WorkspaceID: stored.WorkspaceID} + if previous := s.rows[stored.TaskID]; previous != nil { + bumped = *previous + } + bumped.Summary.Revision++ + s.rows[stored.TaskID] = &bumped + s.mu.Unlock() + return false, nil + } + return s.projectorTestStore.CompareAndUpdateTaskStatusSummary(ctx, stored) +} + +// TestProjectorSustainedLoadAcrossManyTasksConvergesWithoutExhaustedCAS is the +// Task 07 statussummary half of the deterministic load-validation fixture +// (plan.md Task 07, "Document operations and validate sustained load"). It +// scales TestProjectorConcurrentSessionEventsConvergeWithoutExhaustedCAS's +// single-task contention harness up to the same 25-canonical-identity, ten +// simulated-cycle shape as the github package's +// TestLoadValidation_TenSimulatedMinutes_NoWatchCreationLoopBoundedLookups, +// standing in for the many simultaneous task/coordinator boards a real +// deployment runs concurrently. +// +// Each of ten simulated poll cycles fires one concurrent burst of session +// observations across 25 tasks (4 sessions per task per cycle = 100 +// concurrent HandleEvent calls per cycle, 1000 total) while the shared +// alternatingRejectProjectorStore keeps forcing every other +// CompareAndUpdateTaskStatusSummary call to lose its CAS race - modeling a +// second, unsynchronized writer (a boot reconciliation pass or an HTTP +// rebuild request) racing the live projector throughout the run, not just in +// a single burst. It asserts zero exhausted-CAS handler errors across the +// entire sustained run and that every task's final summary reflects its last +// cycle's session count, proving sustained contention never drops or +// misapplies concurrent work. +func TestProjectorSustainedLoadAcrossManyTasksConvergesWithoutExhaustedCAS(t *testing.T) { + const ( + taskCount = 25 + simulatedCycles = 10 + sessionsPerCycle = 4 + workspaceID = "workspace-load" + activeSubagentCnt = 1 + ) + + base := newProjectorTestStore() + taskIDs := make([]string, taskCount) + for i := 0; i < taskCount; i++ { + taskID := fmt.Sprintf("task-load-%02d", i) + taskIDs[i] = taskID + base.rows[taskID] = &StoredTaskStatusSummary{ + TaskID: taskID, + WorkspaceID: workspaceID, + Summary: TaskStatusSummary{Revision: 1}, + } + } + store := &perTaskAlternatingRejectStore{projectorTestStore: base} + + var registryMu sync.Mutex + registry := make(map[string]map[string]RebuildSession) // taskID -> sessionID -> session + registerSession := func(taskID, sessionID string) { + registryMu.Lock() + if registry[taskID] == nil { + registry[taskID] = make(map[string]RebuildSession) + } + registry[taskID][sessionID] = RebuildSession{ID: sessionID, State: sessionStateRunning, ActiveSubagentCount: activeSubagentCnt} + registryMu.Unlock() + } + loadSessions := func(_ context.Context, taskID string) (SessionObservationSnapshot, error) { + registryMu.Lock() + defer registryMu.Unlock() + sessions := make([]RebuildSession, 0, len(registry[taskID])) + for _, session := range registry[taskID] { + sessions = append(sessions, session) + } + return SessionObservationSnapshot{Sessions: sessions, ActivityObserved: true}, nil + } + + var backoffCalls int64 + projector := NewProjector(ProjectorConfig{ + Store: store, + LoadSessionObservations: loadSessions, + RetryBackoff: func(context.Context, int) error { + atomic.AddInt64(&backoffCalls, 1) + return nil + }, + }) + + for cycle := 0; cycle < simulatedCycles; cycle++ { + var wg sync.WaitGroup + errs := make(chan error, taskCount*sessionsPerCycle) + for _, taskID := range taskIDs { + for s := 0; s < sessionsPerCycle; s++ { + wg.Add(1) + go func(taskID string, cycle, s int) { + defer wg.Done() + sessionID := fmt.Sprintf("session-%d-%02d", cycle, s) + registerSession(taskID, sessionID) + err := projector.HandleEvent(context.Background(), bus.NewEvent( + events.TaskSessionStateChanged, "test", map[string]interface{}{ + "task_id": taskID, + "workspace_id": workspaceID, + "session_id": sessionID, + "new_state": sessionStateRunning, + "foreground_activity": activityGenerating, + "active_subagent_count": activeSubagentCnt, + })) + errs <- err + }(taskID, cycle, s) + } + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("cycle %d: concurrent HandleEvent returned error (want zero exhausted-CAS handler errors across the sustained run): %v", + cycle, err) + } + } + } + + if atomic.LoadInt64(&backoffCalls) == 0 { + t.Fatal("expected the CAS-retry backoff hook to fire at least once across the sustained run") + } + + // Every task must reflect the cumulative session count from every cycle: + // registerSession keys by (cycle, s), so each task accumulates + // simulatedCycles*sessionsPerCycle distinct running sessions overall. + wantActive := simulatedCycles * sessionsPerCycle + for _, taskID := range taskIDs { + got := base.summary(taskID) + if got == nil { + t.Fatalf("task %s: expected a persisted summary after sustained contention", taskID) + } + if got.ActiveSubagentCount != wantActive { + t.Fatalf("task %s: ActiveSubagentCount = %d, want %d (every concurrent session across all cycles preserved)", + taskID, got.ActiveSubagentCount, wantActive) + } + if got.ForegroundActivity != activityGenerating { + t.Fatalf("task %s: ForegroundActivity = %q, want %q", taskID, got.ForegroundActivity, activityGenerating) + } + } + t.Logf("sustained load: %d tasks x %d cycles x %d sessions/cycle = %d concurrent HandleEvent calls, %d CAS backoff retries", + taskCount, simulatedCycles, sessionsPerCycle, taskCount*simulatedCycles*sessionsPerCycle, atomic.LoadInt64(&backoffCalls)) +} diff --git a/apps/backend/internal/task/statussummary/projector_pending_authority_test.go b/apps/backend/internal/task/statussummary/projector_pending_authority_test.go index f84abbf249..f8e4d80624 100644 --- a/apps/backend/internal/task/statussummary/projector_pending_authority_test.go +++ b/apps/backend/internal/task/statussummary/projector_pending_authority_test.go @@ -220,6 +220,9 @@ func TestProjectorPendingLoaderFailureRetainsStoredState(t *testing.T) { } func TestProjectorPendingRefreshSurfacesExhaustedCASRetries(t *testing.T) { + retriesBefore := casRetriesTotal.Value() + exhaustionsBefore := casExhaustionsTotal.Value() + failuresBefore := eventHandlerFailuresTotal.Value() base := newProjectorTestStore() base.rows["task-cas-exhausted"] = &StoredTaskStatusSummary{ TaskID: "task-cas-exhausted", @@ -252,9 +255,19 @@ func TestProjectorPendingRefreshSurfacesExhaustedCASRetries(t *testing.T) { if got := base.summary("task-cas-exhausted"); got == nil || got.PendingAction != "" { t.Fatalf("stored summary after exhausted CAS retries = %+v, want unchanged", got) } + if got := casRetriesTotal.Value() - retriesBefore; got != maxPendingPersistAttempts-1 { + t.Fatalf("CAS retry metric delta = %d, want %d", got, maxPendingPersistAttempts-1) + } + if got := casExhaustionsTotal.Value() - exhaustionsBefore; got != 1 { + t.Fatalf("CAS exhaustion metric delta = %d, want 1", got) + } + if got := eventHandlerFailuresTotal.Value() - failuresBefore; got != 1 { + t.Fatalf("event-handler failure metric delta = %d, want 1", got) + } } func TestProjectorPendingRefreshRetriesAfterCASRejection(t *testing.T) { + retriesBefore := casRetriesTotal.Value() base := newProjectorTestStore() storedAt := time.Date(2026, 8, 14, 12, 0, 0, 0, time.UTC) base.rows["task-cas"] = &StoredTaskStatusSummary{ @@ -365,6 +378,9 @@ func TestProjectorPendingRefreshRetriesAfterCASRejection(t *testing.T) { if prLoaderCalls != 2 { t.Fatalf("PR loader calls = %d, want reload after rejection", prLoaderCalls) } + if got := casRetriesTotal.Value() - retriesBefore; got != 1 { + t.Fatalf("CAS retry metric delta = %d, want 1", got) + } err = projector.HandleEvent(context.Background(), bus.NewEvent(events.GitEvent, "test", map[string]interface{}{ "task_id": "task-cas", diff --git a/apps/backend/internal/workflowsync/circuit_test.go b/apps/backend/internal/workflowsync/circuit_test.go new file mode 100644 index 0000000000..37bedd99c6 --- /dev/null +++ b/apps/backend/internal/workflowsync/circuit_test.go @@ -0,0 +1,191 @@ +package workflowsync + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/common/authcircuit" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/github" +) + +// fakeFingerprintGitHubClients wraps fakeGitHubClients with a +// CredentialFingerprintProvider implementation, so tests can drive the +// "credential rotated" reset path without a real github.Service. +type fakeFingerprintGitHubClients struct { + fakeGitHubClients + fingerprint string + err error +} + +func (f fakeFingerprintGitHubClients) WorkspaceConnectionFingerprint( + _ context.Context, _ string, +) (string, error) { + return f.fingerprint, f.err +} + +var _ CredentialFingerprintProvider = fakeFingerprintGitHubClients{} + +func setupCircuitTestService(t *testing.T, clients GitHubClientProvider) (*Service, *fakeApplier) { + t.Helper() + store := setupTestStore(t) + applier := &fakeApplier{} + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "console"}) + require.NoError(t, err) + return NewService(store, clients, nil, applier, log), applier +} + +// TestSyncDueConfigs_OpenCircuitSkipsWithoutCallingProvider is the core +// AC-9 behavior: once a config's circuit is open, SyncDueConfigs must not +// call the provider at all — not even to fail again — until the backoff +// window elapses or the credential/config changes. +func TestSyncDueConfigs_OpenCircuitSkipsWithoutCallingProvider(t *testing.T) { + svc, applier := setupCircuitTestService(t, fakeGitHubClients{client: seededMockClient()}) + configureWorkspace(t, svc, "ws-1") + + future := time.Now().UTC().Add(time.Hour) + require.NoError(t, svc.store.RecordSyncStatus( + context.Background(), "ws-1", false, "boom", nil, "", time.Now().UTC(), + authcircuit.State{FailureClass: authcircuit.FailureClassAuth, ConsecutiveFailures: 3, NextRetryAt: &future}, + )) + + svc.SyncDueConfigs(context.Background()) + assert.Empty(t, applier.calls, "an open circuit must skip the sync attempt entirely") +} + +// TestSyncDueConfigs_RepeatedFailuresOpenCircuit confirms a config that +// keeps failing (here: a 404 "directory not found" from the mock client, +// classified as config) eventually stops retrying every tick, rather than +// costing one GitHub call per tick forever. +func TestSyncDueConfigs_RepeatedFailuresOpenCircuit(t *testing.T) { + svc, applier := setupCircuitTestService(t, fakeGitHubClients{client: github.NewMockClient()}) // nothing seeded → 404 every time + configureWorkspace(t, svc, "ws-1") + + // First attempt: due (never synced), fails, opens the circuit. + svc.SyncDueConfigs(context.Background()) + cfg, err := svc.GetConfigForWorkspace(context.Background(), "ws-1") + require.NoError(t, err) + assert.Equal(t, authcircuit.FailureClassConfig, cfg.FailureClass) + require.NotNil(t, cfg.NextRetryAt) + assert.True(t, cfg.NextRetryAt.After(time.Now().UTC()), "backoff should schedule a future retry") + + // A config-class failure uses the long "permanent" backoff schedule + // (minutes, not the config's own 5-minute poll interval alone) — so a + // tick that would otherwise be "due" per the plain interval must still + // be skipped while the circuit is open. + svc.SyncDueConfigs(context.Background()) + assert.Empty(t, applier.calls, "no successful apply ever happened, but no second GitHub call either") +} + +// TestSyncDueConfigs_CredentialFingerprintChangeResetsCircuitAndForcesSync +// confirms Task 04's headline behavior: rotating/reconnecting a credential +// (a fingerprint change) resumes syncing on the very next tick rather than +// waiting out the remaining permanent backoff. +func TestSyncDueConfigs_CredentialFingerprintChangeResetsCircuitAndForcesSync(t *testing.T) { + clients := fakeFingerprintGitHubClients{ + fakeGitHubClients: fakeGitHubClients{client: seededMockClient()}, + fingerprint: "active:1", + } + svc, applier := setupCircuitTestService(t, clients) + configureWorkspace(t, svc, "ws-1") + + future := time.Now().UTC().Add(time.Hour) + require.NoError(t, svc.store.RecordCircuitState( + context.Background(), "ws-1", + authcircuit.State{ + FailureClass: authcircuit.FailureClassAuth, ConsecutiveFailures: 3, NextRetryAt: &future, + Fingerprint: "active:1", + }, + )) + + // Same fingerprint: circuit stays open, still no sync. + svc.SyncDueConfigs(context.Background()) + assert.Empty(t, applier.calls, "an unchanged fingerprint must not reset an open circuit") + + // Credential rotates (generation bumps). + clients.fingerprint = "active:2" + svc.githubClients = clients + + svc.SyncDueConfigs(context.Background()) + assert.Len(t, applier.calls, 1, "a changed fingerprint must reset the circuit and sync immediately") + + cfg, err := svc.GetConfigForWorkspace(context.Background(), "ws-1") + require.NoError(t, err) + assert.Empty(t, cfg.FailureClass, "a successful sync after reset clears the failure class") +} + +// TestSyncDueConfigs_EmptyFingerprintNeverResets confirms a provider that +// cannot currently determine the fingerprint (transient lookup error, or a +// workspace with no connection at all) never accidentally clears an open +// circuit — see authcircuit.State.ResetIfFingerprintChanged's contract. +func TestSyncDueConfigs_EmptyFingerprintNeverResets(t *testing.T) { + clients := fakeFingerprintGitHubClients{ + fakeGitHubClients: fakeGitHubClients{client: seededMockClient()}, + fingerprint: "", + } + svc, applier := setupCircuitTestService(t, clients) + configureWorkspace(t, svc, "ws-1") + + future := time.Now().UTC().Add(time.Hour) + require.NoError(t, svc.store.RecordSyncStatus( + context.Background(), "ws-1", false, "boom", nil, "", time.Now().UTC(), + authcircuit.State{FailureClass: authcircuit.FailureClassAuth, ConsecutiveFailures: 3, NextRetryAt: &future}, + )) + + svc.SyncDueConfigs(context.Background()) + assert.Empty(t, applier.calls, "an empty fingerprint must never reset an open circuit") +} + +func TestSyncDueConfigs_CredentialChangeRespectsPollingDisabled(t *testing.T) { + clients := fakeFingerprintGitHubClients{ + fakeGitHubClients: fakeGitHubClients{client: seededMockClient()}, + fingerprint: "active:1", + } + svc, applier := setupCircuitTestService(t, clients) + disabled := false + _, err := svc.SetConfigForWorkspace(context.Background(), "ws-1", &SetConfigRequest{ + RepoOwner: "acme", + RepoName: "flows", + PollEnabled: &disabled, + }) + require.NoError(t, err) + + // The first tick records the fingerprint baseline, but polling is disabled. + svc.SyncDueConfigs(context.Background()) + assert.Empty(t, applier.calls) + + // A rotated credential must not turn a manual-only config into an automatic + // sync. The explicit SyncWorkspace path remains available to the user. + clients.fingerprint = "active:2" + svc.githubClients = clients + svc.SyncDueConfigs(context.Background()) + assert.Empty(t, applier.calls, "credential changes must not bypass PollEnabled") +} + +// TestWorkflowSyncCircuitSummary_AggregatesByClass confirms the health +// checker's data source counts only open circuits, grouped by class, with +// no per-workspace identifiers leaking into the aggregate. +func TestWorkflowSyncCircuitSummary_AggregatesByClass(t *testing.T) { + svc, _ := setupCircuitTestService(t, fakeGitHubClients{client: seededMockClient()}) + configureWorkspace(t, svc, "ws-auth") + configureWorkspace(t, svc, "ws-config") + configureWorkspace(t, svc, "ws-ok") + + future := time.Now().UTC().Add(time.Hour) + ctx := context.Background() + require.NoError(t, svc.store.RecordSyncStatus(ctx, "ws-auth", false, "boom", nil, "", time.Now().UTC(), + authcircuit.State{FailureClass: authcircuit.FailureClassAuth, ConsecutiveFailures: 1, NextRetryAt: &future})) + require.NoError(t, svc.store.RecordSyncStatus(ctx, "ws-config", false, "boom", nil, "", time.Now().UTC(), + authcircuit.State{FailureClass: authcircuit.FailureClassConfig, ConsecutiveFailures: 1, NextRetryAt: &future})) + + summary, err := svc.WorkflowSyncCircuitSummary(ctx) + require.NoError(t, err) + assert.Equal(t, 3, summary.Total) + assert.Equal(t, 1, summary.OpenAuth) + assert.Equal(t, 1, summary.OpenConfig) + assert.Equal(t, 0, summary.OpenTransient) +} diff --git a/apps/backend/internal/workflowsync/failure.go b/apps/backend/internal/workflowsync/failure.go new file mode 100644 index 0000000000..97114369ad --- /dev/null +++ b/apps/backend/internal/workflowsync/failure.go @@ -0,0 +1,69 @@ +package workflowsync + +import ( + "errors" + "net/http" + "strings" + + "github.com/kandev/kandev/internal/common/authcircuit" + "github.com/kandev/kandev/internal/github" + "github.com/kandev/kandev/internal/gitlab" +) + +// classifySyncErr maps a sync failure to an authcircuit.FailureClass so the +// caller can decide whether to back off on the short transient schedule or +// the long permanent one that only a credential/config change shortens. +// +// GitHub and GitLab clients report auth/permission failures as typed API +// errors carrying the HTTP status; this package has no other way to learn +// "the token is bad" versus "GitHub had a bad minute", so those status codes +// are the classification boundary. Client-provider-not-configured errors +// (returned by listGitHubEntries/listGitLabEntries when the workspace has no +// connection at all) are a config failure: retrying on any schedule cannot +// fix "nobody has connected this integration yet". +func classifySyncErr(err error) authcircuit.FailureClass { + if err == nil { + return authcircuit.FailureClassNone + } + if errors.Is(err, errGitHubClientNotConfigured) || errors.Is(err, errGitLabClientNotConfigured) { + return authcircuit.FailureClassConfig + } + + var ghErr *github.GitHubAPIError + if errors.As(err, &ghErr) { + if ghErr.StatusCode == http.StatusForbidden && githubRateLimitBody(ghErr.Body) { + return authcircuit.FailureClassTransient + } + return classifyStatusCode(ghErr.StatusCode) + } + var glErr *gitlab.APIError + if errors.As(err, &glErr) { + return classifyStatusCode(glErr.StatusCode) + } + if errors.Is(err, github.ErrGitHubConnectionInvalid) || errors.Is(err, github.ErrGitHubNotConfigured) { + return authcircuit.FailureClassAuth + } + if errors.Is(err, gitlab.ErrInvalidToken) { + return authcircuit.FailureClassAuth + } + return authcircuit.FailureClassTransient +} + +func githubRateLimitBody(body string) bool { + body = strings.ToLower(body) + return strings.Contains(body, "rate limit") || strings.Contains(body, "abuse detection") +} + +func classifyStatusCode(status int) authcircuit.FailureClass { + switch status { + case http.StatusUnauthorized, http.StatusForbidden: + return authcircuit.FailureClassAuth + case http.StatusNotFound, http.StatusUnprocessableEntity, http.StatusBadRequest: + // The configured repo/project/branch/path does not exist or is not + // reachable with the current (valid) credential — a configuration + // problem, not a credential one. + return authcircuit.FailureClassConfig + default: + return authcircuit.FailureClassTransient + } +} diff --git a/apps/backend/internal/workflowsync/failure_test.go b/apps/backend/internal/workflowsync/failure_test.go new file mode 100644 index 0000000000..8ceab49c40 --- /dev/null +++ b/apps/backend/internal/workflowsync/failure_test.go @@ -0,0 +1,95 @@ +package workflowsync + +import ( + "errors" + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/kandev/kandev/internal/common/authcircuit" + "github.com/kandev/kandev/internal/github" + "github.com/kandev/kandev/internal/gitlab" +) + +func TestClassifySyncErr_NilIsNone(t *testing.T) { + assert.Equal(t, authcircuit.FailureClassNone, classifySyncErr(nil)) +} + +func TestClassifySyncErr_NotConfiguredIsConfig(t *testing.T) { + assert.Equal(t, authcircuit.FailureClassConfig, classifySyncErr(errGitHubClientNotConfigured)) + assert.Equal(t, authcircuit.FailureClassConfig, classifySyncErr(errGitLabClientNotConfigured)) + // Wrapped sentinels must still classify correctly — fetchFiles/list + // helpers wrap these with additional context. + assert.Equal(t, authcircuit.FailureClassConfig, + classifySyncErr(fmt.Errorf("listing failed: %w", errGitHubClientNotConfigured))) +} + +func TestClassifySyncErr_GitHubAPIErrorByStatus(t *testing.T) { + tests := []struct { + name string + status int + want authcircuit.FailureClass + }{ + {"unauthorized", http.StatusUnauthorized, authcircuit.FailureClassAuth}, + {"forbidden", http.StatusForbidden, authcircuit.FailureClassAuth}, + {"not_found", http.StatusNotFound, authcircuit.FailureClassConfig}, + {"unprocessable", http.StatusUnprocessableEntity, authcircuit.FailureClassConfig}, + {"bad_request", http.StatusBadRequest, authcircuit.FailureClassConfig}, + {"server_error", http.StatusInternalServerError, authcircuit.FailureClassTransient}, + {"rate_limited", http.StatusTooManyRequests, authcircuit.FailureClassTransient}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := &github.GitHubAPIError{StatusCode: tt.status, Endpoint: "/repos/x/y", Body: "irrelevant"} + assert.Equal(t, tt.want, classifySyncErr(err)) + // Wrapped through a %w chain (the real call sites always wrap). + wrapped := fmt.Errorf("failed to list x/y: %w", err) + assert.Equal(t, tt.want, classifySyncErr(wrapped)) + }) + } +} + +func TestClassifySyncErr_GitHubRateLimit403IsTransient(t *testing.T) { + err := &github.GitHubAPIError{StatusCode: http.StatusForbidden, Endpoint: "/repos/x/y", Body: "API rate limit exceeded for installation"} + assert.Equal(t, authcircuit.FailureClassTransient, classifySyncErr(err)) +} + +func TestClassifySyncErr_GitLabAPIErrorByStatus(t *testing.T) { + tests := []struct { + name string + status int + want authcircuit.FailureClass + }{ + {"unauthorized", http.StatusUnauthorized, authcircuit.FailureClassAuth}, + {"forbidden", http.StatusForbidden, authcircuit.FailureClassAuth}, + {"not_found", http.StatusNotFound, authcircuit.FailureClassConfig}, + {"server_error", http.StatusBadGateway, authcircuit.FailureClassTransient}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := &gitlab.APIError{StatusCode: tt.status, Endpoint: "/projects/x"} + assert.Equal(t, tt.want, classifySyncErr(err)) + }) + } +} + +func TestClassifySyncErr_SentinelFallbacksAreAuth(t *testing.T) { + assert.Equal(t, authcircuit.FailureClassAuth, classifySyncErr(github.ErrGitHubConnectionInvalid)) + assert.Equal(t, authcircuit.FailureClassAuth, classifySyncErr(github.ErrGitHubNotConfigured)) + assert.Equal(t, authcircuit.FailureClassAuth, classifySyncErr(gitlab.ErrInvalidToken)) +} + +func TestClassifySyncErr_UnrecognizedErrorIsTransient(t *testing.T) { + assert.Equal(t, authcircuit.FailureClassTransient, classifySyncErr(errors.New("connection reset by peer"))) +} + +// TestClassifySyncErr_TypedErrorTakesPrecedenceOverSentinel confirms the +// documented precedence: a typed API error's status code wins even when the +// underlying transport also happens to satisfy one of the coarser sentinel +// checks, because the status code is the more specific signal. +func TestClassifySyncErr_TypedErrorTakesPrecedenceOverSentinel(t *testing.T) { + err := &github.GitHubAPIError{StatusCode: http.StatusNotFound, Endpoint: "/repos/x/y"} + assert.Equal(t, authcircuit.FailureClassConfig, classifySyncErr(err)) +} diff --git a/apps/backend/internal/workflowsync/metrics_vars.go b/apps/backend/internal/workflowsync/metrics_vars.go new file mode 100644 index 0000000000..54d6c89d1f --- /dev/null +++ b/apps/backend/internal/workflowsync/metrics_vars.go @@ -0,0 +1,51 @@ +package workflowsync + +import ( + "expvar" + "strings" + + "github.com/kandev/kandev/internal/common/authcircuit" +) + +// expvar maps published at package init, exposed via stdlib's /debug/vars +// handler in dev mode. Process-local and dev-mode-visible only; labels are +// bounded to provider name ("github"/"gitlab") and failure class +// ("transient"/"auth"/"config"), never workspace IDs, repository/project +// identifiers, branches, or error text. Mirrors the label idiom in +// internal/github/metrics_vars.go. +var ( + syncFailuresTotal = expvar.NewMap("workflowsync_failures_total") + circuitSkipsTotal = expvar.NewMap("workflowsync_circuit_skips_total") + circuitResetTotal = expvar.NewMap("workflowsync_circuit_resets_total") +) + +// metricLabel builds a "k1=v1;k2=v2;..." label string for an expvar map +// key, matching the idiom in internal/github/metrics_vars.go. +func metricLabel(pairs ...string) string { + if len(pairs)%2 != 0 { + return "" + } + parts := make([]string, 0, len(pairs)/2) + for i := 0; i < len(pairs); i += 2 { + parts = append(parts, pairs[i]+"="+pairs[i+1]) + } + return strings.Join(parts, ";") +} + +// incSyncFailure records a classified sync failure, labeled only by +// provider and failure class. +func incSyncFailure(provider string, class authcircuit.FailureClass) { + syncFailuresTotal.Add(metricLabel("provider", provider, "class", string(class)), 1) +} + +// incCircuitSkip records that a due sync was skipped because its circuit +// was still open, labeled only by provider. +func incCircuitSkip(provider string) { + circuitSkipsTotal.Add(metricLabel("provider", provider), 1) +} + +// incCircuitReset records a circuit reset, labeled by provider and the +// trigger ("credential" or "config"). +func incCircuitReset(provider, trigger string) { + circuitResetTotal.Add(metricLabel("provider", provider, "trigger", trigger), 1) +} diff --git a/apps/backend/internal/workflowsync/models.go b/apps/backend/internal/workflowsync/models.go index 7f1023ffc2..99d71fefd1 100644 --- a/apps/backend/internal/workflowsync/models.go +++ b/apps/backend/internal/workflowsync/models.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/kandev/kandev/internal/common/authcircuit" "github.com/kandev/kandev/internal/common/securityutil" ) @@ -60,6 +61,56 @@ type Config struct { LastHash string `json:"-"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` + + // FailureClass, ConsecutiveFailures, and NextRetryAt persist the + // generation-aware circuit-breaker state for this config (see + // internal/common/authcircuit). Exposed read-only to operators so a + // permanently-failing config is visibly distinguishable from one that is + // simply between polls. + FailureClass authcircuit.FailureClass `json:"failure_class,omitempty"` + ConsecutiveFailures int `json:"consecutive_failures,omitempty"` + NextRetryAt *time.Time `json:"next_retry_at,omitempty"` + // ConfigFingerprint and CredentialFingerprint are internal-only circuit + // identity keys (never secrets — see authcircuit.State.Fingerprint) and + // are not surfaced over the API. + ConfigFingerprint string `json:"-"` + CredentialFingerprint string `json:"-"` +} + +// circuitState extracts the persisted authcircuit.State embedded in this +// config's flat columns. +func (c *Config) circuitState() authcircuit.State { + return authcircuit.State{ + FailureClass: c.FailureClass, + ConsecutiveFailures: c.ConsecutiveFailures, + NextRetryAt: c.NextRetryAt, + Fingerprint: c.CredentialFingerprint, + } +} + +// applyCircuitState writes an updated authcircuit.State back onto the +// config's flat columns (the in-memory mirror of what Store persists). +func (c *Config) applyCircuitState(state authcircuit.State) { + c.FailureClass = state.FailureClass + c.ConsecutiveFailures = state.ConsecutiveFailures + c.NextRetryAt = state.NextRetryAt + c.CredentialFingerprint = state.Fingerprint +} + +// circuitOpen reports whether this config's circuit is currently open (skip +// syncing) at the given time. +func (c *Config) circuitOpen(now time.Time) bool { + return c.circuitState().Open(now) +} + +// fingerprint is a stable, non-secret digest of the fields that define +// "what this config points at". Changing any of these fields (a deliberate +// SetConfig call) always resets the circuit — see Store.UpsertConfigForWorkspace. +func (r *SetConfigRequest) fingerprint() string { + return strings.Join([]string{ + r.Provider, r.RepoOwner, r.RepoName, r.ProjectPath, r.Branch, r.Path, + fmt.Sprintf("%d", r.IntervalSeconds), + }, "\x1f") } // SetConfigRequest is the payload for creating or updating a workspace's diff --git a/apps/backend/internal/workflowsync/service.go b/apps/backend/internal/workflowsync/service.go index 30bb496fa3..64ffc0b321 100644 --- a/apps/backend/internal/workflowsync/service.go +++ b/apps/backend/internal/workflowsync/service.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "sort" "strings" @@ -14,6 +15,7 @@ import ( "go.uber.org/zap" "gopkg.in/yaml.v3" + "github.com/kandev/kandev/internal/common/authcircuit" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/github" "github.com/kandev/kandev/internal/gitlab" @@ -58,6 +60,32 @@ var ( _ GitLabClientProvider = (*gitlab.Service)(nil) ) +// CredentialFingerprintProvider optionally exposes a non-secret fingerprint +// that changes whenever a workspace's integration credential is replaced, +// rotated, reconnected, or revoked. When the configured GitHubClientProvider +// or GitLabClientProvider also implements this, SyncDueConfigs resets an +// open circuit as soon as the fingerprint changes instead of waiting out the +// backoff schedule. Implementing it is optional: *github.Service does; a +// provider that does not (GitLab today has no credential-generation concept) +// simply keeps the "resume promptly on credential change" fast path +// unavailable — its circuit still resets on any explicit SetConfig call and +// self-probes at the capped backoff interval regardless. +type CredentialFingerprintProvider interface { + WorkspaceConnectionFingerprint(ctx context.Context, workspaceID string) (string, error) +} + +var _ CredentialFingerprintProvider = (*github.Service)(nil) + +// errGitHubClientNotConfigured and errGitLabClientNotConfigured are the +// sentinels wrapped by listGitHubEntries/listGitLabEntries when the +// workspace has no usable client for the configured provider. Kept distinct +// from other fetch failures so classifySyncErr can recognize "not +// configured" as a config-class failure independent of error string text. +var ( + errGitHubClientNotConfigured = errors.New("GitHub is not authenticated; configure a GitHub token to sync workflows") + errGitLabClientNotConfigured = errors.New("GitLab is not authenticated; configure a GitLab connection to sync workflows") +) + // dirEntry is a provider-neutral directory listing entry. It exists only to // share the file-selection and content-fetch loop in fetchFiles between the // two providers' native listing shapes. @@ -206,6 +234,10 @@ func (s *Service) SyncWorkspace(ctx context.Context, workspaceID string) (*SyncR lock := s.workspaceLock(workspaceID) lock.Lock() defer lock.Unlock() + return s.syncWorkspaceLocked(ctx, workspaceID) +} + +func (s *Service) syncWorkspaceLocked(ctx context.Context, workspaceID string) (*SyncResult, error) { cfg, err := s.store.GetConfigForWorkspace(ctx, workspaceID) if err != nil { return nil, err @@ -216,18 +248,22 @@ func (s *Service) SyncWorkspace(ctx context.Context, workspaceID string) (*SyncR files, err := s.fetchFiles(ctx, cfg) if err != nil { - s.recordFailure(ctx, workspaceID, err) + s.recordFailure(ctx, workspaceID, cfg, err) return nil, err } parsed, warnings := parseFiles(files) applied, err := s.applier.ApplySyncedWorkflows(ctx, workspaceID, parsed) if err != nil { - s.recordFailure(ctx, workspaceID, err) + s.recordFailure(ctx, workspaceID, cfg, err) return nil, err } warnings = append(warnings, applied.Warnings...) - if err := s.store.RecordSyncStatus(ctx, workspaceID, true, "", warnings, contentHash(files), time.Now().UTC()); err != nil { + successState := cfg.circuitState() + successState.RecordSuccess() + if err := s.store.RecordSyncStatus( + ctx, workspaceID, true, "", warnings, contentHash(files), time.Now().UTC(), successState, + ); err != nil { return nil, err } return &SyncResult{ @@ -239,9 +275,15 @@ func (s *Service) SyncWorkspace(ctx context.Context, workspaceID string) (*SyncR }, nil } -func (s *Service) recordFailure(ctx context.Context, workspaceID string, syncErr error) { +func (s *Service) recordFailure(ctx context.Context, workspaceID string, cfg *Config, syncErr error) { + class := classifySyncErr(syncErr) + state := cfg.circuitState() + state.RecordFailure(time.Now().UTC(), class, nil) + incSyncFailure(cfg.Provider, class) // Clear the hash so the next successful fetch re-applies from scratch. - if err := s.store.RecordSyncStatus(ctx, workspaceID, false, syncErr.Error(), nil, "", time.Now().UTC()); err != nil { + if err := s.store.RecordSyncStatus( + ctx, workspaceID, false, syncErr.Error(), nil, "", time.Now().UTC(), state, + ); err != nil { s.logger.Warn("failed to record sync failure", zap.String("workspace_id", workspaceID), zap.Error(err)) } @@ -286,7 +328,7 @@ func (s *Service) listProviderEntries(ctx context.Context, cfg *Config) ([]dirEn func (s *Service) listGitHubEntries(ctx context.Context, cfg *Config) ([]dirEntry, fileGetter, error) { if s.githubClients == nil { - return nil, nil, fmt.Errorf("GitHub is not authenticated; configure a GitHub token to sync workflows") + return nil, nil, errGitHubClientNotConfigured } raw, err := s.githubClients.ListRepoDirectoryForWorkspace( ctx, cfg.WorkspaceID, cfg.RepoOwner, cfg.RepoName, cfg.Path, cfg.Branch, @@ -306,7 +348,7 @@ func (s *Service) listGitHubEntries(ctx context.Context, cfg *Config) ([]dirEntr func (s *Service) listGitLabEntries(ctx context.Context, cfg *Config) ([]dirEntry, fileGetter, error) { if s.gitlabClients == nil { - return nil, nil, fmt.Errorf("GitLab is not authenticated; configure a GitLab connection to sync workflows") + return nil, nil, errGitLabClientNotConfigured } raw, err := s.gitlabClients.ListRepoTreeForWorkspace(ctx, cfg.WorkspaceID, cfg.ProjectPath, cfg.Path, cfg.Branch) if err != nil { @@ -371,6 +413,14 @@ func parseExport(path string, data []byte) (*workflowmodels.WorkflowExport, erro // SyncDueConfigs runs a periodic sync for every workspace whose interval has // elapsed. Failures are recorded on the config row and logged, never fatal. +// Before deciding whether to sync, each config's credential fingerprint is +// re-checked: a changed fingerprint (reconnect/rotate/revoke) resets an open +// circuit and forces an immediate attempt, so a fixed credential resumes +// syncing on this tick rather than waiting out the remaining backoff. A +// config whose circuit is still open after that check is skipped without +// ever calling the provider — this is the behavior AC-9 (auth backoff) +// requires: a broken credential/config does not keep costing GitHub/GitLab +// requests every tick. func (s *Service) SyncDueConfigs(ctx context.Context) { configs, err := s.store.ListConfigs(ctx) if err != nil { @@ -382,16 +432,120 @@ func (s *Service) SyncDueConfigs(ctx context.Context) { if ctx.Err() != nil { return } - if !isSyncDue(cfg, now) { + lock := s.workspaceLock(cfg.WorkspaceID) + lock.Lock() + forceSync := s.refreshCredentialFingerprint(ctx, cfg, now) + if cfg.circuitOpen(now) { + incCircuitSkip(cfg.Provider) + lock.Unlock() continue } - if _, err := s.SyncWorkspace(ctx, cfg.WorkspaceID); err != nil { + if !cfg.PollEnabled { + lock.Unlock() + continue + } + if !forceSync && !isSyncDue(cfg, now) { + lock.Unlock() + continue + } + if _, err := s.syncWorkspaceLocked(ctx, cfg.WorkspaceID); err != nil { s.logger.Warn("periodic workflow sync failed", zap.String("workspace_id", cfg.WorkspaceID), zap.Error(err)) } + lock.Unlock() } } +// refreshCredentialFingerprint re-derives the workspace's current credential +// fingerprint (when the configured provider exposes one) and resets cfg's +// circuit if it changed, persisting the reset immediately so it survives +// even if this tick does not go on to attempt a sync. Returns true when a +// reset happened, telling the caller to sync now regardless of the normal +// interval. +func (s *Service) refreshCredentialFingerprint(ctx context.Context, cfg *Config, now time.Time) bool { + provider := s.credentialFingerprintProvider(cfg.Provider) + if provider == nil { + return false + } + fingerprint, err := provider.WorkspaceConnectionFingerprint(ctx, cfg.WorkspaceID) + if err != nil || fingerprint == "" { + return false + } + state := cfg.circuitState() + previousFingerprint := state.Fingerprint + if !state.ResetIfFingerprintChanged(fingerprint) { + cfg.applyCircuitState(state) // still record the first-observed fingerprint + if previousFingerprint == "" && state.Fingerprint == fingerprint { + if err := s.store.RecordCircuitState(ctx, cfg.WorkspaceID, state); err != nil { + s.logger.Warn("failed to persist credential fingerprint", + zap.String("workspace_id", cfg.WorkspaceID), zap.Error(err)) + } + } + return false + } + cfg.applyCircuitState(state) + incCircuitReset(cfg.Provider, "credential") + if err := s.store.RecordCircuitState(ctx, cfg.WorkspaceID, state); err != nil { + s.logger.Warn("failed to persist credential-triggered circuit reset", + zap.String("workspace_id", cfg.WorkspaceID), zap.Error(err)) + } + return true +} + +func (s *Service) credentialFingerprintProvider(provider string) CredentialFingerprintProvider { + if provider == ProviderGitLab { + if p, ok := s.gitlabClients.(CredentialFingerprintProvider); ok { + return p + } + return nil + } + if p, ok := s.githubClients.(CredentialFingerprintProvider); ok { + return p + } + return nil +} + +// WorkflowSyncCircuitSummary aggregates the current circuit-breaker state +// across every configured workflow-sync target, for the health checker +// (internal/health.WorkflowSyncStatusProvider). It returns only bounded +// counts by failure class — never workspace IDs, repository/project +// identifiers, or error text. +func (s *Service) WorkflowSyncCircuitSummary(ctx context.Context) (workflowSyncCircuitSummary, error) { + configs, err := s.store.ListConfigs(ctx) + if err != nil { + return workflowSyncCircuitSummary{}, err + } + now := time.Now().UTC() + summary := workflowSyncCircuitSummary{Total: len(configs)} + for _, cfg := range configs { + if !cfg.circuitOpen(now) { + continue + } + switch cfg.FailureClass { + case authcircuit.FailureClassAuth: + summary.OpenAuth++ + case authcircuit.FailureClassConfig: + summary.OpenConfig++ + default: + summary.OpenTransient++ + } + } + return summary, nil +} + +// workflowSyncCircuitSummary mirrors health.WorkflowSyncCircuitSummary +// field-for-field. It is defined locally (rather than importing the health +// package here) to avoid making workflowsync depend on health; the health +// package's checker consumes this through structural typing via its own +// WorkflowSyncStatusProvider interface, matching the rest of this file's +// provider-interface pattern. +type workflowSyncCircuitSummary struct { + Total int + OpenAuth int + OpenConfig int + OpenTransient int +} + func isSyncDue(cfg *Config, now time.Time) bool { if !cfg.PollEnabled { return false diff --git a/apps/backend/internal/workflowsync/store.go b/apps/backend/internal/workflowsync/store.go index ab94e486e4..b21a805505 100644 --- a/apps/backend/internal/workflowsync/store.go +++ b/apps/backend/internal/workflowsync/store.go @@ -10,6 +10,7 @@ import ( "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/common/authcircuit" "github.com/kandev/kandev/internal/db" "github.com/kandev/kandev/internal/db/dialect" ) @@ -57,7 +58,31 @@ func (s *Store) initSchema() error { if err := s.addPollEnabledColumn(); err != nil { return err } - return s.addProviderColumns() + if err := s.addProviderColumns(); err != nil { + return err + } + return s.addCircuitColumns() +} + +// addCircuitColumns brings databases created before the auth/config +// circuit-breaker (Task 04) up to the current schema. Idempotent and +// race-safe, matching addPollEnabledColumn/addProviderColumns. A pre-existing +// row's implicit state is "circuit closed, no known fingerprint" — exactly +// the zero values these defaults produce, so no explicit backfill is needed. +func (s *Store) addCircuitColumns() error { + statements := []string{ + `ALTER TABLE workflow_sync_configs ADD COLUMN failure_class TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE workflow_sync_configs ADD COLUMN consecutive_failures INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE workflow_sync_configs ADD COLUMN next_retry_at DATETIME`, + `ALTER TABLE workflow_sync_configs ADD COLUMN config_fingerprint TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE workflow_sync_configs ADD COLUMN credential_fingerprint TEXT NOT NULL DEFAULT ''`, + } + for _, stmt := range statements { + if _, err := s.db.Exec(schemaSQLForDriver(stmt, s.db.DriverName())); err != nil && !db.IsDuplicateColumnError(err) { + return err + } + } + return nil } // addProviderColumns brings databases created before GitLab sync up to the @@ -89,7 +114,8 @@ func (s *Store) addPollEnabledColumn() error { const configSelectColumns = `workspace_id, provider, repo_owner, repo_name, project_path, branch, path, interval_seconds, poll_enabled, last_synced_at, last_ok, last_error, last_warnings, last_hash, - created_at, updated_at` + created_at, updated_at, failure_class, consecutive_failures, next_retry_at, + config_fingerprint, credential_fingerprint` type configScanner interface { Scan(dest ...interface{}) error @@ -97,9 +123,9 @@ type configScanner interface { func scanConfig(row configScanner) (*Config, error) { cfg := &Config{} - var lastOk, pollEnabled int - var lastSyncedAt sql.NullTime - var warningsJSON string + var lastOk, pollEnabled, consecutiveFailures int + var lastSyncedAt, nextRetryAt sql.NullTime + var warningsJSON, failureClass string if err := row.Scan( &cfg.WorkspaceID, &cfg.Provider, @@ -117,11 +143,22 @@ func scanConfig(row configScanner) (*Config, error) { &cfg.LastHash, &cfg.CreatedAt, &cfg.UpdatedAt, + &failureClass, + &consecutiveFailures, + &nextRetryAt, + &cfg.ConfigFingerprint, + &cfg.CredentialFingerprint, ); err != nil { return nil, err } cfg.LastOk = lastOk != 0 cfg.PollEnabled = pollEnabled != 0 + cfg.FailureClass = authcircuit.FailureClass(failureClass) + cfg.ConsecutiveFailures = consecutiveFailures + if nextRetryAt.Valid { + t := nextRetryAt.Time + cfg.NextRetryAt = &t + } // A row written before the provider column existed carries the implicit // GitHub meaning. The migration default covers the normal path; this // guards any row that still reads back empty. @@ -175,15 +212,21 @@ func (s *Store) ListConfigs(ctx context.Context) ([]*Config, error) { } // UpsertConfigForWorkspace creates or replaces a workspace's config. The sync -// status columns are reset so the next sync re-fetches and re-applies. +// status and circuit-breaker columns are reset so the next sync re-fetches, +// re-applies, and gets an immediate attempt regardless of any previously +// open circuit — an explicit SetConfig call is exactly the kind of change +// Author Decision #5 says must reset backoff. credential_fingerprint is +// intentionally left untouched: it tracks the workspace's connection +// identity, not this config, and is refreshed independently by the poller. func (s *Store) UpsertConfigForWorkspace(ctx context.Context, workspaceID string, req *SetConfigRequest) (*Config, error) { now := time.Now().UTC() _, err := s.db.ExecContext(ctx, s.db.Rebind(` INSERT INTO workflow_sync_configs ( workspace_id, provider, repo_owner, repo_name, project_path, branch, path, interval_seconds, poll_enabled, - last_synced_at, last_ok, last_error, last_warnings, last_hash, created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, 0, '', '[]', '', ?, ?) + last_synced_at, last_ok, last_error, last_warnings, last_hash, created_at, updated_at, + failure_class, consecutive_failures, next_retry_at, config_fingerprint + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, 0, '', '[]', '', ?, ?, '', 0, NULL, ?) ON CONFLICT(workspace_id) DO UPDATE SET provider = excluded.provider, repo_owner = excluded.repo_owner, @@ -198,17 +241,27 @@ func (s *Store) UpsertConfigForWorkspace(ctx context.Context, workspaceID string last_error = '', last_warnings = '[]', last_hash = '', - updated_at = excluded.updated_at + updated_at = excluded.updated_at, + failure_class = '', + consecutive_failures = 0, + next_retry_at = NULL, + config_fingerprint = excluded.config_fingerprint `), workspaceID, req.Provider, req.RepoOwner, req.RepoName, req.ProjectPath, req.Branch, req.Path, - req.IntervalSeconds, boolToInt(req.PollEnabled != nil && *req.PollEnabled), now, now) + req.IntervalSeconds, boolToInt(req.PollEnabled != nil && *req.PollEnabled), now, now, req.fingerprint()) if err != nil { return nil, err } return s.GetConfigForWorkspace(ctx, workspaceID) } -// RecordSyncStatus persists the outcome of a sync attempt. -func (s *Store) RecordSyncStatus(ctx context.Context, workspaceID string, ok bool, errMsg string, warnings []string, hash string, at time.Time) error { +// RecordSyncStatus persists the outcome of a sync attempt, along with the +// resulting circuit-breaker state (failure class, consecutive-failure +// count, and next-retry time), so a restart resumes backoff instead of +// hammering a known-broken credential/config from a clean slate. +func (s *Store) RecordSyncStatus( + ctx context.Context, workspaceID string, ok bool, errMsg string, warnings []string, + hash string, at time.Time, circuit authcircuit.State, +) error { warningsJSON, err := json.Marshal(warnings) if err != nil { warningsJSON = []byte("[]") @@ -219,9 +272,25 @@ func (s *Store) RecordSyncStatus(ctx context.Context, workspaceID string, ok boo } _, err = s.db.ExecContext(ctx, s.db.Rebind(` UPDATE workflow_sync_configs - SET last_synced_at = ?, last_ok = ?, last_error = ?, last_warnings = ?, last_hash = ?, updated_at = ? + SET last_synced_at = ?, last_ok = ?, last_error = ?, last_warnings = ?, last_hash = ?, updated_at = ?, + failure_class = ?, consecutive_failures = ?, next_retry_at = ? WHERE workspace_id = ? - `), at, okInt, errMsg, string(warningsJSON), hash, at, workspaceID) + `), at, okInt, errMsg, string(warningsJSON), hash, at, + string(circuit.FailureClass), circuit.ConsecutiveFailures, circuit.NextRetryAt, workspaceID) + return err +} + +// RecordCircuitState persists only the circuit-breaker fields, used when a +// credential-fingerprint change resets an open circuit outside of a full +// sync attempt (SyncDueConfigs may reset-and-skip in the same tick when the +// reset alone does not yet prove the credential works). +func (s *Store) RecordCircuitState(ctx context.Context, workspaceID string, circuit authcircuit.State) error { + _, err := s.db.ExecContext(ctx, s.db.Rebind(` + UPDATE workflow_sync_configs + SET failure_class = ?, consecutive_failures = ?, next_retry_at = ?, credential_fingerprint = ?, updated_at = ? + WHERE workspace_id = ? + `), string(circuit.FailureClass), circuit.ConsecutiveFailures, circuit.NextRetryAt, circuit.Fingerprint, + time.Now().UTC(), workspaceID) return err } diff --git a/apps/backend/internal/workflowsync/store_postgres_test.go b/apps/backend/internal/workflowsync/store_postgres_test.go index 73e50c1157..d0b99e5c81 100644 --- a/apps/backend/internal/workflowsync/store_postgres_test.go +++ b/apps/backend/internal/workflowsync/store_postgres_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/kandev/kandev/internal/common/authcircuit" "github.com/kandev/kandev/internal/testutil" ) @@ -38,7 +39,7 @@ func TestPostgresStoreSchemaReplay(t *testing.T) { t.Fatalf("upsert workflow sync config: %v", err) } updatedAt := time.Now().UTC().Truncate(time.Microsecond) - if err := replayed.RecordSyncStatus(ctx, "ws-1", true, "", []string{"warning"}, "hash-1", updatedAt); err != nil { + if err := replayed.RecordSyncStatus(ctx, "ws-1", true, "", []string{"warning"}, "hash-1", updatedAt, authcircuit.State{}); err != nil { t.Fatalf("record workflow sync status: %v", err) } config, err := replayed.GetConfigForWorkspace(ctx, "ws-1") diff --git a/apps/backend/internal/workflowsync/store_test.go b/apps/backend/internal/workflowsync/store_test.go index b6eda9c47a..e368cba9da 100644 --- a/apps/backend/internal/workflowsync/store_test.go +++ b/apps/backend/internal/workflowsync/store_test.go @@ -10,6 +10,8 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/common/authcircuit" ) func setupTestStore(t *testing.T) *Store { @@ -61,7 +63,11 @@ func TestStore_UpsertResetsSyncStatus(t *testing.T) { ctx := context.Background() _, err := store.UpsertConfigForWorkspace(ctx, "ws-1", testRequest()) require.NoError(t, err) - require.NoError(t, store.RecordSyncStatus(ctx, "ws-1", true, "", []string{"w1"}, "hash-1", time.Now().UTC())) + failAt := time.Now().UTC().Truncate(time.Second) + require.NoError(t, store.RecordSyncStatus( + ctx, "ws-1", false, "boom", nil, "", failAt, + authcircuit.State{FailureClass: authcircuit.FailureClassConfig, ConsecutiveFailures: 3, NextRetryAt: &failAt}, + )) req := testRequest() req.RepoName = "other" @@ -71,6 +77,10 @@ func TestStore_UpsertResetsSyncStatus(t *testing.T) { assert.Nil(t, cfg.LastSyncedAt, "changing the config resets sync status") assert.Empty(t, cfg.LastHash) assert.Empty(t, cfg.LastWarnings) + assert.Empty(t, cfg.FailureClass, "an explicit config change always resets an open circuit") + assert.Zero(t, cfg.ConsecutiveFailures) + assert.Nil(t, cfg.NextRetryAt) + assert.Equal(t, req.fingerprint(), cfg.ConfigFingerprint) } func TestStore_RecordSyncStatusRoundtrip(t *testing.T) { @@ -81,7 +91,10 @@ func TestStore_RecordSyncStatusRoundtrip(t *testing.T) { at := time.Now().UTC().Truncate(time.Second) warnings := []string{"workflow \"X\" not updated", "flows/broken.yml: bad yaml"} - require.NoError(t, store.RecordSyncStatus(ctx, "ws-1", false, "boom", warnings, "hash-2", at)) + require.NoError(t, store.RecordSyncStatus( + ctx, "ws-1", false, "boom", warnings, "hash-2", at, + authcircuit.State{FailureClass: authcircuit.FailureClassTransient, ConsecutiveFailures: 1, NextRetryAt: &at}, + )) cfg, err := store.GetConfigForWorkspace(ctx, "ws-1") require.NoError(t, err) @@ -90,6 +103,87 @@ func TestStore_RecordSyncStatusRoundtrip(t *testing.T) { assert.Equal(t, "boom", cfg.LastError) assert.Equal(t, warnings, cfg.LastWarnings) assert.Equal(t, "hash-2", cfg.LastHash) + assert.Equal(t, authcircuit.FailureClassTransient, cfg.FailureClass) + assert.Equal(t, 1, cfg.ConsecutiveFailures) + require.NotNil(t, cfg.NextRetryAt) + assert.True(t, at.Equal(*cfg.NextRetryAt)) +} + +// TestStore_RecordSyncStatus_SuccessClearsCircuit confirms a subsequent +// successful sync clears a previously-recorded circuit-open state, matching +// authcircuit.State.RecordSuccess's contract. +func TestStore_RecordSyncStatus_SuccessClearsCircuit(t *testing.T) { + store := setupTestStore(t) + ctx := context.Background() + _, err := store.UpsertConfigForWorkspace(ctx, "ws-1", testRequest()) + require.NoError(t, err) + + failAt := time.Now().UTC().Truncate(time.Second) + require.NoError(t, store.RecordSyncStatus( + ctx, "ws-1", false, "boom", nil, "", failAt, + authcircuit.State{FailureClass: authcircuit.FailureClassAuth, ConsecutiveFailures: 2, NextRetryAt: &failAt}, + )) + + cfg, err := store.GetConfigForWorkspace(ctx, "ws-1") + require.NoError(t, err) + require.Equal(t, authcircuit.FailureClassAuth, cfg.FailureClass) + + okAt := failAt.Add(time.Minute) + require.NoError(t, store.RecordSyncStatus(ctx, "ws-1", true, "", nil, "hash-ok", okAt, authcircuit.State{})) + + cfg, err = store.GetConfigForWorkspace(ctx, "ws-1") + require.NoError(t, err) + assert.Empty(t, cfg.FailureClass) + assert.Zero(t, cfg.ConsecutiveFailures) + assert.Nil(t, cfg.NextRetryAt) +} + +// TestStore_RecordCircuitState confirms the standalone circuit-only writer +// used by the credential-fingerprint reset path persists exactly the given +// state, including the fingerprint (which RecordSyncStatus never touches). +func TestStore_RecordCircuitState(t *testing.T) { + store := setupTestStore(t) + ctx := context.Background() + _, err := store.UpsertConfigForWorkspace(ctx, "ws-1", testRequest()) + require.NoError(t, err) + + require.NoError(t, store.RecordCircuitState(ctx, "ws-1", authcircuit.State{ + FailureClass: authcircuit.FailureClassNone, + ConsecutiveFailures: 0, + Fingerprint: "active:3", + })) + + cfg, err := store.GetConfigForWorkspace(ctx, "ws-1") + require.NoError(t, err) + assert.Empty(t, cfg.FailureClass) + assert.Equal(t, "active:3", cfg.CredentialFingerprint) +} + +// TestStore_addCircuitColumns_Idempotent confirms re-running the migration +// (as happens on every backend boot) is a no-op, matching +// addPollEnabledColumn/addProviderColumns. +func TestStore_addCircuitColumns_Idempotent(t *testing.T) { + store := setupTestStore(t) + require.NoError(t, store.addCircuitColumns()) + require.NoError(t, store.addCircuitColumns()) +} + +func TestStore_addCircuitColumns_UsesPostgresTimestampType(t *testing.T) { + rawDB, err := sql.Open("sqlite3", ":memory:") + require.NoError(t, err) + rawDB.SetMaxOpenConns(1) + db := sqlx.NewDb(rawDB, "pgx") + t.Cleanup(func() { _ = db.Close() }) + _, err = db.Exec(`CREATE TABLE workflow_sync_configs (workspace_id TEXT PRIMARY KEY)`) + require.NoError(t, err) + + store := &Store{db: db, ro: db} + require.NoError(t, store.addCircuitColumns()) + + var columnType string + err = db.Get(&columnType, `SELECT type FROM pragma_table_info('workflow_sync_configs') WHERE name = 'next_retry_at'`) + require.NoError(t, err) + assert.Equal(t, "TIMESTAMPTZ", columnType) } func TestStore_ListConfigs(t *testing.T) { diff --git a/apps/web/e2e/helpers/api-client.ts b/apps/web/e2e/helpers/api-client.ts index 4b522bb95f..eac7e9be81 100644 --- a/apps/web/e2e/helpers/api-client.ts +++ b/apps/web/e2e/helpers/api-client.ts @@ -2510,7 +2510,31 @@ export class ApiClient { metadata?: Record; }>; }> { - return this.request("GET", `/api/v1/task-sessions/${sessionId}/messages`); + // The production endpoint intentionally caps explicit pages at 100. E2E + // callers use this helper for authoritative fixture inspection, so follow + // the cursor explicitly instead of relying on the bounded default page. + const messages: Array<{ + id: string; + content: string; + author_type: string; + type?: string; + raw_content?: string; + metadata?: Record; + }> = []; + let after = ""; + for (;;) { + const query = new URLSearchParams({ limit: "100", sort: "asc" }); + if (after) query.set("after", after); + const page = await this.request<{ + messages: typeof messages; + has_more?: boolean; + cursor?: string; + }>("GET", `/api/v1/task-sessions/${sessionId}/messages?${query.toString()}`); + messages.push(...page.messages); + if (!page.has_more || !page.cursor || page.cursor === after) break; + after = page.cursor; + } + return { messages }; } async listSessionTurns(sessionId: string): Promise<{ diff --git a/apps/web/e2e/tests/pr/pr-detection.spec.ts b/apps/web/e2e/tests/pr/pr-detection.spec.ts index 3cc9ce9dee..22c3d84625 100644 --- a/apps/web/e2e/tests/pr/pr-detection.spec.ts +++ b/apps/web/e2e/tests/pr/pr-detection.spec.ts @@ -309,8 +309,8 @@ test.describe("PR auto-detection", () => { // --- Verify PR topbar button updates to merged state --- // The backend poller (every 30s) syncs the merged state and broadcasts - // via WS. The PRTopbarButton re-renders with the purple merged icon. - await expect(session.prTopbarButton().locator(".text-purple-500").first()).toBeVisible({ + // via WS. Assert the semantic state rather than an icon styling class. + await expect(session.prTopbarButton()).toHaveAttribute("data-pr-state", "merged", { timeout: 90_000, }); }); diff --git a/apps/web/hooks/domains/session/use-shell-command-output.test.ts b/apps/web/hooks/domains/session/use-shell-command-output.test.ts index b40a375776..b090b47dc7 100644 --- a/apps/web/hooks/domains/session/use-shell-command-output.test.ts +++ b/apps/web/hooks/domains/session/use-shell-command-output.test.ts @@ -11,6 +11,8 @@ vi.mock("@/lib/api/domains/session-api", () => ({ })); const fetchOutputMock = vi.mocked(fetchShellCommandOutput); +let messageSequence = 0; +let messageId = ""; function snapshot( status: string, @@ -18,7 +20,7 @@ function snapshot( output: ShellCommandOutputSnapshot["output"] = {}, ): ShellCommandOutputSnapshot { return { - message_id: "message-1", + message_id: messageId, status, updated_at: "2026-07-16T12:00:00Z", output: { ...output, ...(stdout ? { stdout } : {}) }, @@ -45,6 +47,8 @@ async function flushPromises() { beforeEach(() => { vi.useFakeTimers(); fetchOutputMock.mockReset(); + messageSequence += 1; + messageId = `message-${messageSequence}`; }); afterEach(() => { @@ -58,7 +62,7 @@ describe("useShellCommandOutput fetching", () => { ({ isOpen }) => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen, messageStatus: "complete", }), @@ -85,7 +89,7 @@ describe("useShellCommandOutput fetching", () => { const { result } = renderHook(() => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen: true, messageStatus: "running", }), @@ -112,7 +116,7 @@ describe("useShellCommandOutput fetching", () => { const { result } = renderHook(() => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen: true, messageStatus: "pending", }), @@ -132,7 +136,7 @@ describe("useShellCommandOutput fetching", () => { const { result } = renderHook(() => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen: true, messageStatus: "running", }), @@ -156,6 +160,110 @@ describe("useShellCommandOutput fetching", () => { }); }); +describe("useShellCommandOutput shared cache", () => { + it("deduplicates a concurrent request across hook instances", async () => { + const pending = deferred(); + fetchOutputMock.mockReturnValue(pending.promise); + + const { result } = renderHook(() => { + const first = useShellCommandOutput({ + sessionId: "session-1", + messageId, + isOpen: true, + messageStatus: "complete", + }); + const second = useShellCommandOutput({ + sessionId: "session-1", + messageId, + isOpen: true, + messageStatus: "complete", + }); + return { first, second }; + }); + + expect(fetchOutputMock).toHaveBeenCalledTimes(1); + pending.resolve(snapshot("complete", "shared", { exit_code: 0 })); + await flushPromises(); + expect(result.current.first.snapshot?.output.stdout).toBe("shared"); + expect(result.current.second.snapshot?.output.stdout).toBe("shared"); + }); + + it("reuses a terminal payload after the renderer is closed and reopened", async () => { + fetchOutputMock.mockResolvedValue(snapshot("complete", "cached", { exit_code: 0 })); + const first = renderHook(() => + useShellCommandOutput({ + sessionId: "session-1", + messageId, + isOpen: true, + messageStatus: "complete", + }), + ); + await flushPromises(); + expect(first.result.current.snapshot?.output.stdout).toBe("cached"); + first.unmount(); + + const reopened = renderHook(() => + useShellCommandOutput({ + sessionId: "session-1", + messageId, + isOpen: true, + messageStatus: "complete", + }), + ); + await flushPromises(); + + expect(reopened.result.current.snapshot?.output.stdout).toBe("cached"); + expect(fetchOutputMock).toHaveBeenCalledTimes(1); + }); + + it("replaces a shared running request once when separate renderers become terminal", async () => { + const finalOutput = "final shared output"; + const running = deferred(); + const terminal = deferred(); + const signals: AbortSignal[] = []; + fetchOutputMock.mockImplementation((_sessionId, _messageId, options) => { + const signal = options?.init?.signal; + if (signal) signals.push(signal); + return signals.length === 1 ? running.promise : terminal.promise; + }); + + const renderOutput = () => + renderHook( + ({ messageStatus }) => + useShellCommandOutput({ + sessionId: "session-1", + messageId, + isOpen: true, + messageStatus, + }), + { initialProps: { messageStatus: "running" } }, + ); + const first = renderOutput(); + const second = renderOutput(); + expect(fetchOutputMock).toHaveBeenCalledTimes(1); + + first.rerender({ messageStatus: "complete" }); + second.rerender({ messageStatus: "complete" }); + + expect(signals[0]?.aborted).toBe(true); + expect(fetchOutputMock).toHaveBeenCalledTimes(2); + terminal.resolve(snapshot("complete", finalOutput, { exit_code: 0 })); + await flushPromises(); + expect(first.result.current.snapshot?.output.stdout).toBe(finalOutput); + expect(second.result.current.snapshot?.output.stdout).toBe(finalOutput); + + running.resolve(snapshot("running", "stale shared output")); + await flushPromises(); + first.unmount(); + second.unmount(); + + const reopened = renderOutput(); + await flushPromises(); + expect(reopened.result.current.snapshot?.output.stdout).toBe(finalOutput); + expect(fetchOutputMock).toHaveBeenCalledTimes(2); + }); +}); + describe("useShellCommandOutput cleanup", () => { it("aborts and ignores stale output after collapse", async () => { const pending = deferred(); @@ -168,7 +276,7 @@ describe("useShellCommandOutput cleanup", () => { ({ isOpen }) => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen, messageStatus: "running", }), @@ -196,7 +304,7 @@ describe("useShellCommandOutput cleanup", () => { ({ messageStatus }) => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen: true, messageStatus, }), @@ -227,7 +335,7 @@ describe("useShellCommandOutput terminal lifecycle", () => { const unmounted = renderHook(() => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen: true, messageStatus: "running", }), @@ -247,7 +355,7 @@ describe("useShellCommandOutput terminal lifecycle", () => { ({ messageStatus }) => useShellCommandOutput({ sessionId: "session-1", - messageId: "message-1", + messageId, isOpen: true, messageStatus, }), @@ -271,7 +379,7 @@ it("drops cached output and stops retrying when the server reports removal", asy .mockRejectedValue(new ApiError("gone", 410, { code: "tool_payload_removed" })); const { result } = renderHook(() => useShellCommandOutput({ - sessionId: "session-1", + sessionId: "removed-session-1", messageId: "message-1", isOpen: true, messageStatus: "running", diff --git a/apps/web/hooks/domains/session/use-shell-command-output.ts b/apps/web/hooks/domains/session/use-shell-command-output.ts index 0d8e3ca3c1..3e2becdcfd 100644 --- a/apps/web/hooks/domains/session/use-shell-command-output.ts +++ b/apps/web/hooks/domains/session/use-shell-command-output.ts @@ -8,6 +8,7 @@ import { isTerminalToolCallStatus } from "@/lib/utils/tool-call-status"; const POLL_INTERVAL_MS = 1_000; const MAX_RETRY_INTERVAL_MS = 5_000; +const MAX_CACHED_OUTPUTS = 200; export type UseShellCommandOutputOptions = { sessionId: string; @@ -25,10 +26,25 @@ export type UseShellCommandOutputResult = { type PollOperation = { generation: number; - controller: AbortController | null; + outputKey: string; + consumer: symbol | null; timer: ReturnType | null; }; +type SharedRequest = { + controller: AbortController; + consumers: Set; + terminalProjection: boolean; + promise: Promise; +}; + +type SharedOutput = { + snapshot: ShellCommandOutputSnapshot | null; + request: SharedRequest | null; +}; + +const outputCache = new Map(); + function retryDelay(failureCount: number) { return Math.min(POLL_INTERVAL_MS * 2 ** Math.max(0, failureCount - 1), MAX_RETRY_INTERVAL_MS); } @@ -37,6 +53,82 @@ function asError(error: unknown) { return error instanceof Error ? error : new Error("Command output unavailable"); } +function touchOutput(outputKey: string, output: SharedOutput) { + outputCache.delete(outputKey); + outputCache.set(outputKey, output); +} + +function pruneOutputCache() { + if (outputCache.size <= MAX_CACHED_OUTPUTS) return; + for (const [outputKey, output] of outputCache) { + if (output.request) continue; + outputCache.delete(outputKey); + if (outputCache.size <= MAX_CACHED_OUTPUTS) return; + } +} + +function readCachedOutput(outputKey: string) { + const output = outputCache.get(outputKey); + if (output) touchOutput(outputKey, output); + return output?.snapshot ?? null; +} + +function requestSharedOutput( + outputKey: string, + sessionId: string, + messageId: string, + consumer: symbol, + terminalProjection: boolean, +) { + const output = outputCache.get(outputKey) ?? { snapshot: null, request: null }; + touchOutput(outputKey, output); + if (output.snapshot && isTerminalToolCallStatus(output.snapshot.status)) { + return Promise.resolve(output.snapshot); + } + if (output.request) { + if (!terminalProjection || output.request.terminalProjection) { + output.request.consumers.add(consumer); + return output.request.promise; + } + output.request.controller.abort(); + output.request = null; + } + + const controller = new AbortController(); + const request: SharedRequest = { + controller, + consumers: new Set([consumer]), + terminalProjection, + promise: fetchShellCommandOutput(sessionId, messageId, { + init: { signal: controller.signal }, + }) + .then((snapshot) => { + if (output.request === request) { + output.snapshot = snapshot; + touchOutput(outputKey, output); + } + return snapshot; + }) + .finally(() => { + if (output.request === request) output.request = null; + pruneOutputCache(); + }), + }; + output.request = request; + pruneOutputCache(); + return request.promise; +} + +function releaseSharedOutput(outputKey: string, consumer: symbol) { + const output = outputCache.get(outputKey); + const request = output?.request; + if (!output || !request) return; + request.consumers.delete(consumer); + if (request.consumers.size > 0) return; + output.request = null; + request.controller.abort(); +} + export function useShellCommandOutput({ sessionId, messageId, @@ -50,7 +142,12 @@ export function useShellCommandOutput({ const snapshotRef = useRef(null); const outputKeyRef = useRef(""); const messageStatusRef = useRef(messageStatus); - const operationRef = useRef({ generation: 0, controller: null, timer: null }); + const operationRef = useRef({ + generation: 0, + outputKey: "", + consumer: null, + timer: null, + }); messageStatusRef.current = messageStatus; const stop = useCallback(() => { @@ -58,8 +155,11 @@ export function useShellCommandOutput({ operation.generation += 1; if (operation.timer) clearTimeout(operation.timer); operation.timer = null; - operation.controller?.abort(); - operation.controller = null; + if (operation.outputKey && operation.consumer) { + releaseSharedOutput(operation.outputKey, operation.consumer); + } + operation.outputKey = ""; + operation.consumer = null; }, []); useEffect(() => { @@ -72,25 +172,34 @@ export function useShellCommandOutput({ const outputKey = `${sessionId}:${messageId}`; if (outputKeyRef.current !== outputKey) { outputKeyRef.current = outputKey; - snapshotRef.current = null; - setSnapshot(null); + snapshotRef.current = readCachedOutput(outputKey); + setSnapshot(snapshotRef.current); setError(null); } const operation = operationRef.current; const generation = operation.generation; + const consumer = Symbol(outputKey); + operation.outputKey = outputKey; + operation.consumer = consumer; let failureCount = 0; + if (snapshotRef.current && isTerminalToolCallStatus(snapshotRef.current.status)) { + setIsLoading(false); + return stop; + } + const requestSnapshot = async () => { - const controller = new AbortController(); - operation.controller = controller; if (!snapshotRef.current) setIsLoading(true); try { - const nextSnapshot = await fetchShellCommandOutput(sessionId, messageId, { - init: { signal: controller.signal }, - }); - if (operation.generation !== generation || controller.signal.aborted) return; - operation.controller = null; + const nextSnapshot = await requestSharedOutput( + outputKey, + sessionId, + messageId, + consumer, + isTerminalToolCallStatus(messageStatusRef.current), + ); + if (operation.generation !== generation) return; failureCount = 0; snapshotRef.current = nextSnapshot; setSnapshot(nextSnapshot); @@ -103,8 +212,7 @@ export function useShellCommandOutput({ operation.timer = setTimeout(requestSnapshot, POLL_INTERVAL_MS); } } catch (requestError) { - if (operation.generation !== generation || controller.signal.aborted) return; - operation.controller = null; + if (operation.generation !== generation) return; if (isToolPayloadRemovedError(requestError)) { snapshotRef.current = null; setSnapshot(null); diff --git a/docs/decisions/2026-08-31-task-owned-pr-watch-identity.md b/docs/decisions/2026-08-31-task-owned-pr-watch-identity.md new file mode 100644 index 0000000000..2510300d9f --- /dev/null +++ b/docs/decisions/2026-08-31-task-owned-pr-watch-identity.md @@ -0,0 +1,32 @@ +# ADR-2026-08-31-task-owned-pr-watch-identity: Keep PR watches task owned + +**Status:** accepted +**Date:** 2026-08-31 +**Area:** backend, GitHub, workflow + +## Context + +Agent sessions are attempts within a task, not independent ownership domains for a pull request. A session-scoped PR-watch key lets resumed or concurrent sessions create duplicate watches for the same task branch. Each duplicate polls GitHub and emits equivalent events, which creates unnecessary task-status projection contention. + +Review work can outlive the session that initially discovered the pull request, so deleting watches solely because their provenance session has completed would stop required monitoring. + +## Decision + +`github_pr_watches` is task-owned. A searching watch is canonical by `(task_id, repository_id, branch)` and a discovered watch is canonical by `(task_id, repository_id, pr_number)`. `session_id` remains optional provenance only and must not participate in canonical identity, polling fan-out, or task-status publication. + +The active-watch query admits watches for existing, non-archived tasks, including Review tasks whose originating session is complete, and excludes watches without a valid task/repository relationship. Reconciliation resolves one desired branch per task/repository and performs an atomic transition only when it differs from the canonical searching watch. Polling and publication coalesce state by task/repository/PR and observable PR state. + +Upgrade cleanup is a one-time, idempotent, transactional migration. It requires a database snapshot before destructive work, retains a deterministic canonical row, prefers discovered rows, preserves the newest status and feedback watermarks, and removes only duplicate or orphaned watch rows. + +## Consequences + +- Multiple historical or concurrent sessions no longer multiply GitHub requests, feedback events, or status-summary work. +- A watch is retained based on task lifecycle instead of session completion, preserving Review monitoring. +- Session-level callers must be migrated to task/repository identity and cannot rely on a separate watch per session. +- Legacy duplicate cleanup has a documented backup and rollback requirement. + +## Alternatives Considered + +- **Keep session-scoped watches and deduplicate only at polling time.** Rejected because duplicate persistent rows still create reconciliation writes and every future consumer must reproduce deduplication. +- **Delete every completed-session watch.** Rejected because a Review task can legitimately require monitoring after its original session finishes. +- **Use only `(task_id, repository_id)` as the key.** Rejected because multi-branch tasks require independent searching watches until a PR is discovered. diff --git a/docs/plans/pr-watch-and-storage-bounds/plan.md b/docs/plans/pr-watch-and-storage-bounds/plan.md new file mode 100644 index 0000000000..4571f5556b --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/plan.md @@ -0,0 +1,125 @@ +--- +created: 2026-08-31 +updated: 2026-09-14 +status: done +requirements: + - REQ-PLATFORM-PR-WATCH-IDENTITY-001 + - REQ-PLATFORM-PR-WATCH-IDENTITY-002 + - REQ-PLATFORM-TASK-SUMMARY-CONTENTION-001 + - REQ-PLATFORM-BOUNDED-SESSION-HISTORY-001 + - REQ-PLATFORM-BOUNDED-SESSION-HISTORY-002 + - REQ-PLATFORM-DATABASE-MAINTENANCE-001 + - REQ-PLATFORM-PROVIDER-BACKOFF-001 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +legacy_specs: + - ../../specs/platform/pr-watch-and-storage-bounds.md +--- + +# Implementation Plan: Canonical PR Monitoring and Bounded Task Storage + +## Overview + +Replace session-scoped PR-watch ownership with task/repository canonical identities, migrate legacy data safely, and prevent unchanged reconciliation and PR status from producing writes or event fan-out. Then make summary projection resilient to real cross-writer contention and make GitHub authentication failures visibly back off. + +Bound operational storage without deleting normal conversation history: separate heavy tool payloads from normal hydration, deduplicate snapshots, add explicit retention maintenance with verified backups and safe SQLite compaction, and expose the relevant health signals. This follows [ADR-2026-08-31-task-owned-pr-watch-identity](../../decisions/2026-08-31-task-owned-pr-watch-identity.md) and the existing [ADR 0045](../../decisions/0045-install-wide-storage-maintenance.md). + +## Confirmed root cause + +`github_pr_watches` is currently unique by `(session_id, repository_id, branch)`, and `Service.EnsurePRWatch` looks up the same session-level identity. `Poller.reconcileWatches` iterates session-level branch candidates while `refreshStaleBranches` resolves branch by task/session. A resumed or parallel session can therefore create its own row and drive duplicate polling even though `ListActivePRWatches` selects all non-archived task watches. + +The task-status projector has an in-process per-task lock and semantic no-op check, but its fixed three-attempt CAS loop returns a handler error when another writer repeatedly wins. Duplicate PR events make that pressure much more likely. Message pagination exists in the repository/service, but legacy hydration paths still use full message reads and load inline metadata. Git snapshots and plan revisions have no equivalent payload-dedup/retention maintenance policy. + +## Backend + +### Canonical PR-watch schema and migration + +Update `apps/backend/internal/github/store.go` and its migration helpers to replace the session key with canonical searching/discovered identities. Preserve `session_id` as provenance, validate task/repository ownership in active queries, and add transactional migration that snapshots first, resolves duplicates deterministically, preserves freshest status/timestamps, removes orphans, logs safe counts, and is idempotent. + +Use the persistence snapshot implementation in `apps/backend/internal/persistence/snapshot.go`; do not introduce another backup mechanism. Reconcile existing multi-repository backfills and task cleanup behavior with the new lifetime contract. + +### Reconciliation, polling, and events + +Change `TaskBranchProvider` and `Poller` so branch resolution has one desired result per task/repository. Replace session-level ensure/reset operations with atomic task-level transitions. Batch/check only canonical watches and gate `GitHubPRFeedback` plus `GitHubTaskPRUpdated` publication on a durable, semantic state fingerprint that includes task, repository, PR, head SHA, and exposed status fields. + +Classify GitHub auth/configuration failures separately from transient provider/rate-limit failures. Persist/report degraded integration state and use generation-aware exponential backoff/circuit reopening when credentials/configuration changes. + +### Task status and observability + +Extend `internal/task/statussummary` with per-task coalescing/single-flight pending refreshes and a contention policy: reload/rebase authoritative state, jittered bounded backoff for a real CAS race, and semantic success for equivalent pending work. Keep the existing revision contract in `docs/specs/platform/bounded-task-status-delivery.md`. + +Add metrics/structured health through the existing health/debug metric wiring for canonical/searching/duplicate/orphan watch counts, per-canonical PR requests, CAS retries/exhaustion, event-handler failures, storage sizes, SQLite/WAL size, history hydration latency, queue depth, and active runtime count. Metric labels must not include credentials, branches with sensitive content, or unbounded IDs. + +### Bounded storage and history hydration + +Introduce a repository-owned heavy tool-payload representation, digesting and compressing/externalizing payloads above a configured threshold. Return a lightweight metadata projection by default and add explicit, scoped detail retrieval. Update all normal session hydration callers to cursor pagination; preserve stable ordering and authorization. + +Deduplicate equivalent snapshot writes using a stable content fingerprint, and add retention selection for redundant snapshots, superseded tool payloads, and obsolete plan revisions. Retention is opt-in through maintenance and never targets ordinary human/agent messages. + +### Operator maintenance and documentation + +Add the database-maintenance command to the native Go launcher/backend command surface, reusing `internal/system/storage` ownership and `persistence.SnapshotSQLite`. It must plan/dry-run estimates by table, verify or create a backup, perform deletions transactionally, compact into a staged file, atomically replace only after validation, and print exact rollback instructions. + +Document the operator workflow under `docs/public/**`, including backup of `kandev.db` and `master.key`, dry-run review, execute/compact, rollback, and post-release verification. Update relevant public configuration/operations references and backend guidance if command or metrics conventions change. + +## Frontend + +No new product interaction is required for the first iteration. If existing System health/status surfaces expose the new degradation and database-maintenance telemetry, add only localized status presentation following the current System-page patterns; otherwise the command and structured health contract are the supported operator interface. + +## Tests + +- **Canonical identity/migration:** `apps/backend/internal/github/store_*_test.go` and `service_pr_watch*_test.go` use duplicated legacy records, multiple sessions, branch changes, multi-repository tasks, and completed provenance sessions to prove uniqueness, idempotency, and Review monitoring. +- **Polling/events/auth:** `apps/backend/internal/github/poller_test.go`, batched-watch tests, and workflow-sync/GitHub health tests count GitHub calls/events and prove unchanged cycles are write-free, duplicate PR state emits once, and invalid credentials back off until generation changes. +- **Status projector:** `apps/backend/internal/task/statussummary/projector*_test.go` drives concurrent equal/different PR events and competing CAS writers; it asserts no exhausted retry handler error, monotonic revision, and correct final authoritative summary. +- **Storage/history:** task SQLite repository/service/controller tests build a large fixture, assert cursor-bounded lightweight history reads and explicit payload retrieval, digest deduplication, retention selection, and preservation of ordinary messages. +- **Maintenance:** command/system/persistence tests assert dry run makes no write, backup precondition fails closed, per-table estimates are stable, transaction rollback preserves data, staged compaction validates and atomically swaps, and the documented backup restores the fixture. +- **Load validation:** a deterministic multi-coordinator/task integration fixture runs ten simulated minutes and asserts no watch creation loop, no recurring projector failure, bounded lookup count, and stable handler latency. + +## E2E Tests + +No browser E2E is required unless task 06 adds a System-page health UI. In that case, add a desktop and mobile Playwright scenario that verifies the localized degraded GitHub state and links to the non-destructive maintenance instructions; run `mobile-parity` before implementing that UI task. + +## Verification Results + +All seven tasks (01-07) are implemented, tested, documented, and committed. +See each task-0N-*.md file's "Results" section for detailed +implementation/test/verification evidence. Summary: canonical PR-watch +identity and migration (01), idempotent polling/events (02), contention-safe +projection (03), GitHub auth backoff/observability (04), bounded history +storage (05), offline database maintenance command (06), and operator +upgrade documentation plus deterministic sustained-load validation (07). + +## Implementation Waves And Parallel Candidates + +Wave 1 (sequential, shared schema and state contracts): + +- [x] [task-01-canonical-watch-migration](task-01-canonical-watch-migration.md) +- [x] [task-02-idempotent-polling-events](task-02-idempotent-polling-events.md) +- [x] [task-03-contention-safe-projection](task-03-contention-safe-projection.md) + +Wave 2 (sequential, depends on stable event/storage contracts): + +- [x] [task-04-github-auth-backoff-observability](task-04-github-auth-backoff-observability.md) +- [x] [task-05-bounded-history-storage](task-05-bounded-history-storage.md) + +Wave 3 (sequential, destructive operator surface depends on retention implementation): + +- [x] [task-06-database-maintenance-command](task-06-database-maintenance-command.md) +- [x] [task-07-operator-docs-load-validation](task-07-operator-docs-load-validation.md) + +No task is parallel-safe: the watch schema, projector/event semantics, storage representation, and maintenance command all share persistence and operation contracts. + +## Risks + +- Changing unique constraints on SQLite requires a table rebuild and careful preservation of all watch columns; migration must not assume completed sessions are disposable. +- A canonical discovered-watch uniqueness constraint needs an explicit searching-row representation so multiple branches remain possible before discovery. +- Heavy tool payload externalization must preserve authorization, integrity verification, backup behavior, and existing transcript/export consumers. +- Atomic compaction needs sufficient free storage for a staged copy and must coordinate database pools/WAL; failures must leave the live database usable. +- Exact retention defaults are intentionally operator-configured and disabled for destructive execution until an explicit maintenance run. The command must print its effective policies before it acts. + +## Out of scope + +- Automatic deletion of normal conversation messages. +- Hardware resizing or a container restart as a remediation. +- Disabling GitHub PR monitoring. +- A new third-party object-store integration for payloads. diff --git a/docs/plans/pr-watch-and-storage-bounds/task-01-canonical-watch-migration.md b/docs/plans/pr-watch-and-storage-bounds/task-01-canonical-watch-migration.md new file mode 100644 index 0000000000..f78d21a016 --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-01-canonical-watch-migration.md @@ -0,0 +1,100 @@ +--- +id: "01-canonical-watch-migration" +title: "Migrate PR watches to task ownership" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLATFORM-PR-WATCH-IDENTITY-001 +acceptance_criteria: + - AC-PLATFORM-PR-WATCH-IDENTITY-001.1 + - AC-PLATFORM-PR-WATCH-IDENTITY-001.2 + - AC-PLATFORM-PR-WATCH-IDENTITY-001.3 + - AC-PLATFORM-PR-WATCH-IDENTITY-001.4 + - AC-PLATFORM-PR-WATCH-IDENTITY-001.5 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 01: Migrate PR watches to task ownership + +## Intent + +Make `github_pr_watches` canonical at task/repository scope and safely collapse legacy session duplicates without breaking Review monitoring. + +## Acceptance + +- Searching and discovered watch identities enforce the spec keys; provenance `session_id` cannot create a second canonical watch. +- Upgrade migration snapshots before destructive work, is transactional/idempotent, removes only duplicates/orphans, prefers discovered rows, and preserves newest status/check/review/comment fields. +- Active listing excludes invalid/orphaned watches but retains a non-archived Review task whose provenance session is completed. + +## Files likely touched + +- `apps/backend/internal/github/store.go` +- `apps/backend/internal/github/store_multi_repo_test.go` +- `apps/backend/internal/github/store_watch_reset_test.go` +- `apps/backend/internal/github/store_task_cleanup_qa_test.go` +- `apps/backend/internal/github/service_task_events.go` +- `apps/backend/internal/persistence/snapshot.go` +- New focused GitHub migration tests beside `store.go` + +## Dependencies + +None. + +## Parallelism + +Sequential. This task owns the shared watch schema and migration contract. + +## Verification + +Write the duplicated-legacy and completed-session Review regressions first. Run targeted tests before and after implementation: + +```bash +cd apps/backend && go test ./internal/github -run 'Test.*(PRWatch|WatchMigration|BackfillPRWatches|Review).*' -count=1 -v +cd apps/backend && go test ./internal/persistence ./internal/github -count=1 +git diff --check +``` + +## Output contract + +Report migration before/after fixture counts, the expected red failure, backup boundary, exact files, and test results. Update task and plan status in the same conversation. + +## Results + +**Status: done.** + +`github_pr_watches` now uses task-owned canonical identities. Searching rows +are unique by `(task_id, repository_id, branch)` and discovered rows by +`(task_id, repository_id, pr_number)` through partial unique indexes; +`session_id` remains optional provenance and cannot create another watch. +Service lookups, create/ensure behavior, and collision-safe PR-number updates +all use those task-owned keys, so concurrent and resumed sessions converge on +one row. + +The startup migration runs transactionally after the existing SQLite snapshot +boundary. It removes watches for missing tasks or detached repositories, +clears missing-session provenance, prefers a discovered row while merging +newest status/check/review/comment watermarks, resolves cross-branch PR-number +collisions, and is idempotent on a second boot. A completed provenance session +does not make a non-archived Review task ineligible for monitoring. + +Focused migration and service regressions cover 50 historical sessions +collapsing to one watch, discovered-row preference, PR-number collisions, +orphan cleanup, provenance clearing, database-enforced uniqueness, +create/ensure deduplication, and completed-session Review monitoring. The +later sustained-load fixture additionally proves those 50 historical sessions +still produce one canonical polling target per cycle. + +Verification: + +~~~text +go test ./internal/github -run 'Test.*(PRWatch|WatchMigration|BackfillPRWatches|Review).*' -count=1 -v +go test ./internal/persistence ./internal/github -count=1 +golangci-lint run ./internal/github/... +~~~ + +All commands passed when this wave was delivered; Task 07's integrated load +validation and the final PR remediation rerun the affected canonical-watch +coverage against the current branch head. diff --git a/docs/plans/pr-watch-and-storage-bounds/task-02-idempotent-polling-events.md b/docs/plans/pr-watch-and-storage-bounds/task-02-idempotent-polling-events.md new file mode 100644 index 0000000000..934bebcda2 --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-02-idempotent-polling-events.md @@ -0,0 +1,146 @@ +--- +id: "02-idempotent-polling-events" +title: "Reconcile and publish canonical PR state" +status: done +wave: 1 +depends_on: ["01-canonical-watch-migration"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-PR-WATCH-IDENTITY-001 + - REQ-PLATFORM-PR-WATCH-IDENTITY-002 +acceptance_criteria: + - AC-PLATFORM-PR-WATCH-IDENTITY-001.3 + - AC-PLATFORM-PR-WATCH-IDENTITY-001.6 + - AC-PLATFORM-PR-WATCH-IDENTITY-002.1 + - AC-PLATFORM-PR-WATCH-IDENTITY-002.2 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 02: Reconcile and publish canonical PR state + +## Intent + +Stop session-driven branch oscillation and duplicate GitHub/event work after canonical identity is available. + +## Acceptance + +- One branch result per task/repository drives one atomic searching-watch transition; unchanged reconciliation has no inserts or updates. +- Multiple sessions on one task/branch result in one lookup per cycle; multi-repository and multi-branch behavior remains correct. +- `GitHubPRFeedback` and `GitHubTaskPRUpdated` publish only for durable relevant state changes, coalesced by task/repository/PR/head SHA/status. + +## Files likely touched + +- `apps/backend/internal/github/poller.go` +- `apps/backend/internal/github/poller_test.go` +- `apps/backend/internal/github/service_pr_watch.go` +- `apps/backend/internal/github/service_pr_watch_batched.go` +- `apps/backend/internal/github/service_pr_watch_test.go` +- `apps/backend/internal/github/service_pr_watch_multi_branch_test.go` +- `apps/backend/internal/github/service_pr_sync_test.go` + +## Dependencies + +Task 01. + +## Parallelism + +Sequential. It consumes the task-owned store contract and event semantics used by projection. + +## Verification + +```bash +cd apps/backend && go test ./internal/github -run 'Test.*(Poller|Reconcile|Branch|PRWatch|Sync|Feedback|MultiRepo).*' -count=1 -v +cd apps/backend && go test ./internal/github -count=1 +git diff --check +``` + +## Output contract + +Record red/green call and event counts for repeated resume, branch switch, unchanged cycles, and multi-repository cases. Update task and plan status. + +## Results + +Commit: `e7cf5e8184c1df4e03efe56298afa2bf176ba269` (branch +`feature/fix-pr-watch-amplifi-u6z`, pushed to origin). + +**Fixed (code changed):** + +- `TaskBranchProvider.ResolveBranchForSession(taskID, sessionID)` replaced + with `ResolveBranchForRepository(taskID, repositoryID)`. The old resolver + ignored `watch.RepositoryID` and always resolved the task's PRIMARY + repository's `checkout_branch`; on multi-repo tasks this could silently + overwrite a still-searching secondary-repo watch's branch with the primary + repo's branch. New behavior resolves the specific repository's + `checkout_branch` first, falling back to any active session's worktree + branch for that repository only as a last resort. Locked in by + `TestRefreshStaleBranches_ResolvesByRepositoryNotSession` (two watches, same + session, different repos, provider returns different per-repo branches — + both update to their own branch) and `TestResolveBranchForRepository` + (checkout_branch precedence, per-repo scoping, worktree fallback, "" when + unresolvable). +- `buildTaskBranchList`/watch dedup rekeyed from + `(session_id, repository_id, branch)` to `(task_id, repository_id, branch)` + (`watchedTaskRepoBranchKey`, `buildWatchedTaskRepoBranchSet`), plus an + in-pass `emitted` set inside `buildTaskBranchList`. N active sessions + resolving to the same (task, repository, branch) target now collapse to + one `TaskBranchInfo` emission and therefore one + `EnsurePRWatchForWorkspace` call per cycle. Locked in by + `TestListTasksNeedingPRWatch/dedupes_multiple_sessions_on_the_same_task/repo/branch` + (two sessions, one shared task environment/worktree, one repo/branch -> + exactly 1 emitted entry). + +**Verified already-satisfied (no code change needed):** + +- "One branch result per task/repository drives one atomic searching-watch + transition; unchanged reconciliation has no inserts or updates" — already + covered by Task 01's `UpdatePRWatchBranchIfSearching`/`EnsurePRWatchForWorkspace` + idempotency (existing `TestUpdatePRWatchBranchIfSearching_*` tests pass + unchanged). +- "`GitHubPRFeedback` and `GitHubTaskPRUpdated` publish only for durable + relevant state changes, coalesced by task/repository/PR/head SHA/status" — + `checkSinglePRWatch`/`checkPRWatchWithClient` already gate `GitHubPRFeedback` + on `hasNew` (checks_state/review_state diff or PR `updated_at` advancing + past the watch's watermark); the batched path + (`tryBatchedPRWatchCheck`) already gates on `PRWatchSyncResult.Changed` or a + terminal merged/closed transition. `GitHubTaskPRUpdated` is already gated by + `taskPRChangedFields` (Task 01). Existing tests + (`TestCheckSinglePRWatch_OpenPR_NoChange_NoSync`, + `TestSyncTaskPR_NoEventWhenUnchanged`, + `TestSyncTaskPR_SecondIdenticalSyncNoEvent`, + `TestSyncTaskPR_EquivalentRESTAndGraphQLStatusNoSecondEvent`) already cover + this and pass unmodified. No literal `head_sha` column exists on `PRWatch`; + head-SHA-level changes are already captured transitively through + `checks_state` (a new commit triggers a fresh CI run) and PR `updated_at`. + +**Branch-switch collision coverage.** Live branch changes coalesce into the +existing task-owned destination row. `TestUpdatePRWatchBranchIfSearching_CollidesWithSibling_DropsSource` +and `TestUpdatePRWatchBranchIfSearching_CollidesWithSiblingHasPR_DropsSource` +cover searching and discovered destinations; `TestResetPRWatch_ConcurrentSessionsCoalesceDestination` +proves concurrent transitions leave one canonical destination. The stale-source +regression confirms a late source observation cannot recreate or switch that +destination back. + +**Test evidence:** + +``` +cd apps/backend && go test ./internal/github -run 'Test.*(Poller|Reconcile|Branch|PRWatch|Sync|Feedback|MultiRepo).*' -count=1 -v +... all PASS (no failures) + +cd apps/backend && go test ./internal/github -count=1 +ok github.com/kandev/kandev/internal/github 11.277s + +cd apps/backend && go test ./internal/orchestrator/... -count=1 +ok github.com/kandev/kandev/internal/orchestrator 56.896s +ok github.com/kandev/kandev/internal/orchestrator/executor 1.163s +ok github.com/kandev/kandev/internal/orchestrator/handlers 0.075s +ok github.com/kandev/kandev/internal/orchestrator/messagequeue 0.337s +ok github.com/kandev/kandev/internal/orchestrator/queue 0.003s +ok github.com/kandev/kandev/internal/orchestrator/scheduler 0.469s +ok github.com/kandev/kandev/internal/orchestrator/watcher 0.017s + +git diff --check # exit 0, no whitespace issues +go build ./... # exit 0 +gofmt -l # no output +golangci-lint run ./internal/github/... ./internal/orchestrator/... # 0 issues +``` diff --git a/docs/plans/pr-watch-and-storage-bounds/task-03-contention-safe-projection.md b/docs/plans/pr-watch-and-storage-bounds/task-03-contention-safe-projection.md new file mode 100644 index 0000000000..6639e2bd48 --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-03-contention-safe-projection.md @@ -0,0 +1,133 @@ +--- +id: "03-contention-safe-projection" +title: "Coalesce task-status projection contention" +status: done +wave: 1 +depends_on: ["02-idempotent-polling-events"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-TASK-SUMMARY-CONTENTION-001 +acceptance_criteria: + - AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.1 + - AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.2 + - AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.3 + - AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.4 + - AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.5 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 03: Coalesce task-status projection contention + +## Intent + +Ensure duplicate/equivalent PR signals and genuine CAS races converge without handler errors or lost final summaries. + +## Acceptance + +- Equivalent pending refreshes for one task are single-flight/coalesced and result in at most one effective summary persistence/publication. +- Genuine CAS loss reloads/rebases authoritative state and uses bounded exponential backoff with jitter; it does not merely raise the retry count. +- Concurrent PR-event tests preserve the correct final summary and report zero exhausted-CAS handler errors. + +## Files likely touched + +- `apps/backend/internal/task/statussummary/projector.go` +- `apps/backend/internal/task/statussummary/projector_events.go` +- `apps/backend/internal/task/statussummary/projector_helpers.go` +- `apps/backend/internal/task/statussummary/projector_test.go` +- `apps/backend/internal/task/statussummary/projector_pending_authority_test.go` +- New concurrent projector regression test + +## Dependencies + +Task 02. + +## Parallelism + +Sequential. It depends on the canonical PR-event semantics from task 02. + +## Verification + +```bash +cd apps/backend && go test -race ./internal/task/statussummary -run 'Test.*(Projector|CAS|Concurrent|PullRequest).*' -count=1 -v +cd apps/backend && go test ./internal/task/statussummary -count=1 +git diff --check +``` + +## Output contract + +Report expected red failure, contention test worker/count setup, final revision/summary assertions, and no-exhaustion result. Update task and plan status. + +## Results + +Implemented and verified. + +**What was already satisfied (no code change needed):** +- Per-task in-process serialization already existed (`Projector.lockTask`), so + concurrent events for one task are already sequenced, never interleaved. +- `persistAndPublishLocked` already short-circuits a semantically-equal derived + summary (`TaskStatusSummary.SemanticEqual`) before touching the store or + publishing, so N concurrent equivalent refreshes for one task already + converge to at most one real persist/publish (`TestProjectorCoalescesConcurrentEquivalentPendingRefreshes` + proves this holds under real goroutine concurrency with `-race`). +- Reload/rebase-on-CAS-rejection (`rebaseProjectionStateFromCurrent`) already + existed and is exercised by the pre-existing `projector_pending_authority_test.go` + suite. + +**Gap fixed:** `persistPendingRefreshLocked`'s and `applyQueueStatusEvent`'s +fixed-attempt CAS retry loops previously retried immediately with zero delay, +so a repeatedly-losing writer (e.g. a live projector racing an unsynchronized +boot reconciliation or HTTP rebuild pass, which write to the same +`CompareAndUpdateTaskStatusSummary` row without going through the projector's +lock) had no way to let the other writer settle before colliding again - +"exhausted CAS retries" only raised the attempt count, never changed the +collision dynamics. Added a small (4ms-64ms, capped) bounded exponential +backoff with up to +25% jitter (`defaultCASRetryBackoff`/`casRetryJitter`), +injected between each rejected attempt and the next retry in both loops. The +attempt bound (3) is unchanged; only the pacing between attempts changed. +`ProjectorConfig.RetryBackoff` allows tests to inject a fast/no-op backoff so +unit tests stay instantaneous while still exercising the real retry path. + +**New tests** (`projector_contention_test.go`): +- `TestProjectorConcurrentSessionEventsConvergeWithoutExhaustedCAS`: 24 + goroutines apply distinct session observations for one task concurrently + while a deterministic "alternating reject" store simulates an external + writer (boot reconciliation/HTTP rebuild analog) racing the live projector + on the same row. Asserts zero handler errors, that the backoff hook fires + under contention, and that the final summary reflects every concurrently + applied session (`ActiveSubagentCount == 24`, correct `ForegroundActivity`). +- `TestProjectorCoalescesConcurrentEquivalentPendingRefreshes`: 20 goroutines + fire the exact same session-state event concurrently; asserts exactly one + `SummaryUpdated` publish and one persisted revision. +- `TestDefaultCASRetryBackoffRespectsContextCancellation`, + `TestCASRetryJitterStaysWithinBounds`, `TestDefaultCASRetryBackoffDelayGrowsAndCaps`: + unit coverage of the backoff/jitter helper itself (bounds, cancellation, + cap). + +**Verification:** +``` +go build ./... -> ok +go vet ./internal/task/... -> ok +go test -race ./internal/task/statussummary -count=1 -v -> ok, all PASS (incl. 5x repeat of new + concurrent/backoff tests under -race) +go test ./internal/orchestrator/... -count=1 -> ok +go test ./internal/backendapp -run "StatusSummary|PRWatch|Projector" -count=1 -> ok +gofmt -l internal/task/statussummary/projector.go internal/task/statussummary/projector_contention_test.go -> no output +golangci-lint run ./internal/task/statussummary/... -> 0 issues +git diff --check -> exit 0 +``` +Pre-existing, unrelated failures (`TestServiceInitializeLocalRepository*`, +`TestCreateDirectoryRejectsInvalidOrExistingChild`, +`TestHTTPInitializeLocalRepositoryCreatesRepository`, +`TestHTTPInitializeLocalRepositoryMapsClientErrors`, +`TestHTTPCreateDirectoryCreatesFolder`, +`TestRepositoryMutationsRejectedInImproveKandevWorkspace`) were confirmed via +`git stash` to fail identically without this change - a sandbox filesystem +permission limitation ("parent directory cannot be accessed"), not a +regression from this task. + +**Deferred/out of scope:** the boot-reconciliation/HTTP-rebuild retry loop in +`internal/task/service/service_status_summary_rebuild.go` already has bounded +exponential backoff (`waitForSummaryReconcileRetry`) but no jitter; this task's +file list scoped the fix to `internal/task/statussummary`, so that loop's +jitter was left untouched to avoid broadening scope beyond this wave. diff --git a/docs/plans/pr-watch-and-storage-bounds/task-04-github-auth-backoff-observability.md b/docs/plans/pr-watch-and-storage-bounds/task-04-github-auth-backoff-observability.md new file mode 100644 index 0000000000..9ed2fcc452 --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-04-github-auth-backoff-observability.md @@ -0,0 +1,192 @@ +--- +id: "04-github-auth-backoff-observability" +title: "Back off failed GitHub integrations and expose health" +status: done +wave: 2 +depends_on: ["02-idempotent-polling-events", "03-contention-safe-projection"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-PROVIDER-BACKOFF-001 +acceptance_criteria: + - AC-PLATFORM-PROVIDER-BACKOFF-001.1 + - AC-PLATFORM-PROVIDER-BACKOFF-001.2 + - AC-PLATFORM-PROVIDER-BACKOFF-001.3 + - AC-PLATFORM-PROVIDER-BACKOFF-001.4 + - AC-PLATFORM-PROVIDER-BACKOFF-001.5 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 04: Back off failed GitHub integrations and expose health + +## Intent + +Treat permanent GitHub credential/configuration failures as degraded state, not continuous work, and expose operator-visible bounded metrics. + +## Acceptance + +- Authentication/configuration errors enter generation-aware exponential backoff/circuit breaking and resume after credentials/configuration changes. +- Health/status distinguishes healthy, degraded, and disabled GitHub integration without exposing secrets. +- Required canonical-watch, poll-request, CAS/event failure, queue/runtime, storage, database/WAL, and hydration-latency measures are emitted with bounded safe labels. + +## Files likely touched + +- `apps/backend/internal/github/errors.go` +- `apps/backend/internal/github/poller.go` +- `apps/backend/internal/github/service*.go` +- `apps/backend/internal/workflowsync/*.go` +- `apps/backend/internal/health/*.go` +- `apps/backend/internal/backendapp/*.go` +- Relevant `*_test.go` files in those packages + +## Dependencies + +Tasks 02 and 03. + +## Parallelism + +Sequential. Health measures consume the finalized event/projector semantics. + +## Verification + +```bash +cd apps/backend && go test ./internal/github ./internal/workflowsync ./internal/health ./internal/backendapp -run 'Test.*(Auth|Credential|Backoff|Health|Metric|Poller).*' -count=1 -v +cd apps/backend && go test ./internal/github ./internal/workflowsync ./internal/health -count=1 +git diff --check +``` + +## Output contract + +Report invalid-credential call counts before/within/after backoff, generation-change recovery, metric names/labels, and secret-redaction evidence. Update task and plan status. + +## Results + +**Implementation.** Both authenticated-integration surfaces named in scope now classify +failures and circuit-break on them, sharing one value type +(`internal/common/authcircuit`: `FailureClass`, `Backoff`, `State`) that already existed +from a prior wave: + +- **workflowsync** (`internal/workflowsync/failure.go`, `service.go`, `store.go`): + `classifySyncErr` maps not-configured sentinels and typed GitHub/GitLab API errors + (`classifyStatusCode`, 401/403 → auth, 404/422 → config, else → transient) to a + `FailureClass`. `SyncWorkspace`'s failure path (`recordFailure`) and success path both + persist circuit state (`failure_class`, `consecutive_failures`, `next_retry_at`, + `config_fingerprint`, `credential_fingerprint` — 5 new columns via + `addCircuitColumns`). `SyncDueConfigs` refreshes each config's credential fingerprint + first (`refreshCredentialFingerprint`/`CredentialFingerprintProvider`, backed by the + new `github.Service.WorkspaceConnectionFingerprint`, an opaque + `status:credential_generation` string with no secret material) and forces an + immediate sync if it changed; otherwise it skips due-but-circuit-open configs before + falling through to the existing interval gate. `Service.WorkflowSyncCircuitSummary` + aggregates open circuits by class for health reporting. +- **GitHub PR-watch poller** (`internal/github/errors.go`, `poller_circuit.go`, + `poller.go`): `classifyPollErr` classifies connection-invalid sentinels as auth, + `ErrRepoNotResolvable` as config, typed `*GitHubAPIError` by status code, else + transient. A new in-memory, Poller-scoped `pollerCircuits` (map keyed by workspace + ID, mutex-guarded) is consulted by `filterOpenCircuitWatches` before every + `checkPRWatches` cycle — after first refreshing each active workspace's fingerprint + once per cycle and resetting any circuit whose fingerprint changed. Outcomes are + recorded at all three GitHub-call sites in the poll path (batched sync, single-watch + check, branch-based PR detection). This circuit is deliberately narrower than + workflowsync's: it protects only the background poll loop, not on-demand/HTTP- + triggered syncs, matching the existing `Service.rateTracker` vs. Poller-only-loop + precedent already in the codebase. + +**Health.** `internal/health` gained `WorkflowSyncStatusProvider` / +`WorkflowSyncCircuitSummary` / `WorkflowSyncChecker`, mirroring the existing +`GitHubChecker` pattern exactly (nil-provider-safe, aggregate-only, no workspace IDs +in messages). `internal/backendapp/helpers.go` wires it into `registerHealthRoutes` +via a `workflowSyncHealthAdapter`, the same adapter-per-package-boundary idiom already +used for `githubWorkspaceHealthAdapter`/`githubRateLimitAdapter` (avoids an import +cycle since `github`/`workflowsync` don't import `health`). + +**Metrics** (all expvar counters, bounded label sets — provider/class/trigger only, +never secrets/tokens/branches/task/workspace IDs): +- `workflowsync_failures_total{provider,class}` +- `workflowsync_circuit_skips_total{provider}` +- `workflowsync_circuit_resets_total{provider}` +- `github_pr_watch_auth_circuit_skips_total` +- `github_pr_watch_auth_circuit_resets_total` + +**Invalid-credential call-count evidence** (from +`internal/workflowsync/circuit_test.go` and `internal/github/poller_circuit_test.go`): +- Before backoff: a config that has never synced calls the provider once and fails + (`TestSyncDueConfigs_RepeatedFailuresOpenCircuit`, first `SyncDueConfigs` call). +- Within backoff: with the circuit open, a second `SyncDueConfigs` call over the same + due config makes **zero** additional provider calls + (`applier.calls` stays empty) — same for workflowsync's + `TestSyncDueConfigs_OpenCircuitSkipsWithoutCallingProvider` and the poller's + `TestCheckPRWatches_OpenCircuitSkipsSearchEntirely` (`FindPRByBranchCallCount() == 0` + vs. 1 in the closed-circuit control `TestCheckPRWatches_ClosedCircuitStillSearches`). +- Generation-change recovery: `TestSyncDueConfigs_CredentialFingerprintChangeResetsCircuitAndForcesSync` + shows an unchanged fingerprint (`"active:1"` → `"active:1"`) leaves the circuit open + (0 calls), while a changed fingerprint (`"active:1"` → `"active:2"`) resets the + circuit and forces exactly 1 immediate sync call on the very next tick, after which + the config's `FailureClass` clears on success. +- `TestSyncDueConfigs_EmptyFingerprintNeverResets` confirms a provider that cannot + currently determine the fingerprint (empty string) never accidentally resets an + open circuit. +- `TestWorkflowSyncCircuitSummary_AggregatesByClass` confirms the health aggregate + counts by class only (1 auth + 1 config + 1 ok → `Total=3, OpenAuth=1, OpenConfig=1, + OpenTransient=0`), with no per-workspace data in the DTO. + +**Secret-redaction confirmation.** Reviewed every new metric label and log/health +message: all use provider name, failure class (`auth`/`config`/`transient`), or a +plain integer count only. `WorkspaceConnectionFingerprint`'s opaque string +(`status:credential_generation`) is stored as a DB column and used only for equality +comparison inside the circuit-reset logic — it is never emitted as a metric label, +log field, or included in any health-check message. + +**Deliberate deferrals** (out of this wave's scope, consistent with the task-04 doc's +own file list and verification command targeting only `github`/`workflowsync`/ +`health`/`backendapp`): +- Wiring the GitHub poller's `pollerCircuits.summary()` into the `/api/v1/system/health` + HTTP endpoint. `ghPoller` is constructed as a function-local variable inside + `main.go`'s `startGitHubPoller` closure with no field on `services.GitHub` or + elsewhere to expose it through; doing so would require a structural change beyond + this narrow wave. workflowsync's circuit IS wired into health because + `services.WorkflowSync` is already a stable, accessible field. +- Queue/runtime, storage, database/WAL, and hydration-latency measures named in this + task's Acceptance bullet — these belong to the plan's later storage/hydration waves + (Tasks 05-07) per the plan's AC14. Final QA remediation completed those + aggregate metrics through the existing debug/status surfaces without + adding identifiers or user-controlled values as metric labels. +- CAS retries/exhaustions metrics — already covered by Task 03's contention-safe + projection work; not duplicated here. +- Review/issue watch loops (if any exist outside the PR-watch poller) are not + circuit-protected by this wave; only `checkPRWatches`'s three GitHub-call sites are. + +**Tests added.** +- `internal/workflowsync/failure_test.go` — `classifySyncErr`/`classifyStatusCode` + unit coverage (nil, not-configured sentinels incl. wrapped, typed API errors by + status code, sentinel fallbacks, unrecognized-as-transient, typed-precedence). +- `internal/workflowsync/circuit_test.go` — service-level circuit behavior (5 tests, + listed above). +- `internal/workflowsync/store_test.go` — 3 new tests + (`TestStore_RecordSyncStatus_SuccessClearsCircuit`, `TestStore_RecordCircuitState`, + `TestStore_addCircuitColumns_Idempotent`) plus 2 extended existing tests. +- `internal/github/poller_circuit_test.go` — `pollerCircuits` unit tests (record/open/ + reset/summary), `classifyPollErr` unit tests (5 cases), and 2 end-to-end poller tests + (`TestCheckPRWatches_OpenCircuitSkipsSearchEntirely`, + `TestCheckPRWatches_ClosedCircuitStillSearches`). +- `internal/health/checks_test.go` — 4 new `WorkflowSyncChecker` tests (nil provider, + no-open-circuits, open-auth-or-config, status-failure). + +**Verification evidence** (apps/backend, from this session): +``` +go build ./... # pass +go test ./internal/github ./internal/workflowsync ./internal/health \ + ./internal/backendapp \ + -run 'Test.*(Auth|Credential|Backoff|Health|Metric|Poller|WorkflowSync|Circuit).*' \ + -count=1 -v # all pass +go test ./internal/github ./internal/workflowsync ./internal/health -count=1 # ok (14.5s/0.05s/0.01s) +go test ./internal/github/... ./internal/workflowsync/... ./internal/health/... \ + ./internal/backendapp/... ./internal/common/authcircuit/... -race -count=1 # all pass (github 29.7s, backendapp 22.9s) +golangci-lint run ./internal/github/... ./internal/workflowsync/... \ + ./internal/health/... ./internal/backendapp/... ./internal/common/authcircuit/... # 0 issues +gofmt -l # clean (after formatting 3 files) +git diff --check # clean, exit 0 +``` + +Preserves Tasks 01-03 unchanged (verified `git log` head still shows `0f80b3069` as +parent, no history rewritten). Does not start Tasks 05-07. diff --git a/docs/plans/pr-watch-and-storage-bounds/task-05-bounded-history-storage.md b/docs/plans/pr-watch-and-storage-bounds/task-05-bounded-history-storage.md new file mode 100644 index 0000000000..2f8a7ba622 --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-05-bounded-history-storage.md @@ -0,0 +1,248 @@ +--- +id: "05-bounded-history-storage" +title: "Bound history hydration and operational payload storage" +status: done +wave: 2 +depends_on: ["01-canonical-watch-migration"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-BOUNDED-SESSION-HISTORY-001 + - REQ-PLATFORM-BOUNDED-SESSION-HISTORY-002 +acceptance_criteria: + - AC-PLATFORM-BOUNDED-SESSION-HISTORY-001.1 + - AC-PLATFORM-BOUNDED-SESSION-HISTORY-001.2 + - AC-PLATFORM-BOUNDED-SESSION-HISTORY-001.3 + - AC-PLATFORM-BOUNDED-SESSION-HISTORY-002.1 + - AC-PLATFORM-BOUNDED-SESSION-HISTORY-002.2 + - AC-PLATFORM-BOUNDED-SESSION-HISTORY-002.3 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 05: Bound history hydration and operational payload storage + +## Intent + +Keep normal history responsive while retaining conversations and making large operational data deduplicable and retention-eligible. + +## Acceptance + +- Every normal session-history hydration path uses cursor-bounded reads and does not eagerly materialize large tool metadata. +- Large tool payloads use digest-backed compressed/external storage; explicit authorized detail loading verifies integrity and ordinary user/agent messages remain retained. +- Equivalent Git snapshots deduplicate, while superseded payload/snapshot/plan-revision selection respects configurable retention and is non-destructive until maintenance executes. + +## Files likely touched + +- apps/backend/internal/task/models/message.go +- apps/backend/internal/task/repository/sqlite/message.go +- apps/backend/internal/task/repository/sqlite/git_snapshots.go +- apps/backend/internal/task/repository/sqlite/plan.go +- apps/backend/internal/task/repository/sqlite/base_schema.go +- apps/backend/internal/task/service/service_messages.go +- Relevant task controller/DTO hydration callers and focused repository/service tests + +## Dependencies + +Task 01 for shared migration/snapshot conventions. + +## Parallelism + +Sequential. It establishes the retention data contract consumed by the maintenance command. + +## Verification + +Build a multi-gigabyte-equivalent fixture through bounded blobs rather than committing a giant test database: + +~~~bash +cd apps/backend && go test ./internal/task/repository/sqlite ./internal/task/service -run 'Test.*(Message|History|Payload|Snapshot|Retention|PlanRevision).*' -count=1 -v +cd apps/backend && go test ./internal/task/repository/sqlite ./internal/task/service -count=1 +git diff --check +~~~ + +## Output contract + +Report page limits, query/payload assertions, digest collision/dedup behavior, retention candidates, message-preservation evidence, and test outcomes. Update task and plan status. + +## Results + +**Bounded default page (cursor reads).** `service.DefaultMessagesPageSize` moved +10 → 50 (matching AC10's "bounded first page of 50"). `parseListMessageParams` +(HTTP) and `wsListMessages` (WS) now default to that page size whenever a +caller supplies *zero* pagination params (no `limit`/`before`/`after`/`sort`); +any explicit param (including bare `sort`) still runs the existing paginated +path unchanged. The shared service-layer `ListMessagesPaginated` default-limit +guard was deliberately left untouched — `apps/web/hooks/use-summarize-session.ts` +relies on `sort=asc` with no limit to fetch a session's *entire* history for +summarization, so bounding at the service layer would have silently broken +that caller for sessions over 50 messages. `fetchMessages` no longer has an +unbounded branch at all. Normal list reads (`ListMessages`, +`ListMessagesPaginated`, etc.) were not changed to add payload columns, so +paging never selects/decompresses large blobs. + +**Digest-backed externalized payloads.** `task_session_messages` gained +`payload_digest`/`payload_size` (migration `migrateMessagePayloadStorage`, +`base_migrations.go`), and a new content-addressed `task_message_payloads` +table (`digest TEXT PRIMARY KEY, compressed_content , +uncompressed_size, compressed_size, created_at`; `dialect.BlobType` keeps the +column type Postgres-portable per the `dynamic_installation_keys` precedent). +`externalizeMessagePayload` (new file, `message_payload.go`) is the sole write +path for `metadata`: only a shell command's combined stdout+stderr over 4096 +bytes (`largeMessagePayloadThresholdBytes`) is gzip-compressed, SHA-256 +digested, and upserted (`ON CONFLICT (digest) DO NOTHING`, so byte-identical +output from different messages or retries is naturally deduplicated and never +errors) before the persisted `metadata` is replaced with the same small +`has_output`/`stdout_bytes`/`stderr_bytes`/`truncated`/`exit_code` summary +`ProjectMessageMetadata` already computes for client responses. Every other +metadata key (`tool_call_id`, `pending_id`, `status`, `agent_disconnected`, +...), read directly via SQL `json_extract` elsewhere, is left untouched — the +externalization is scoped to exactly the +`metadata.normalized.shell_exec.output` nested field. `CreateMessage`, +`insertMessageRow` (shared by the user-message boundary path), and +`UpdateMessage` all route through it; `GetMessage` — the only read path used +by the lazy shell-output detail route — now also selects +`payload_digest`/`payload_size`. `RehydrateMessagePayload` is the explicit, +authorized detail-load inverse: it loads the compressed row, decompresses, +**recomputes and verifies the SHA-256 digest before trusting the content**, +then merges the restored stdout/stderr back into `message.Metadata` via the +new `models.RehydrateShellOutput`. `Service.RehydrateMessagePayload` is a thin +passthrough (no extra authorization — callers already went through +`GetMessage`'s session-scoped check). `httpGetShellOutput` now rehydrates +whenever `message.PayloadDigest != ""` before extracting shell output, so a +large output that was externalized at write time is no longer invisible to +that endpoint. Ordinary user/agent message content (never routed through +`externalizeMessagePayload`'s shell-output branch) and small tool metadata +(the common case) are stored inline exactly as before — nothing is dropped or +truncated for either. + +The web client's `use-shell-command-output.ts` uses one module-shared, +200-entry bounded payload cache keyed by session/message identity. Concurrent +renderers join the same in-flight request, an underlying request is aborted +only after its last consumer closes, and a terminal payload survives +unmount/reopen without another download. Running snapshots continue polling +and terminal transitions still replace an obsolete in-flight read with one +final fetch. Hook regressions cover concurrent instances and reopen reuse in +addition to the existing polling, backoff, abort, and terminal-transition +cases. + +**Git snapshot content dedup (read-only retention candidates).** +`task_session_git_snapshots` gained `content_digest` (migration +`migrateGitSnapshotContentDigest` + a Go-side backfill for historical rows, +since neither SQLite nor Postgres has a portable SHA-256 SQL function). +`computeGitSnapshotDigest` hashes only the repository-state fields (branch, +remote_branch, head_commit, base_commit, ahead, behind, files) — deliberately +excluding `session_id`/`snapshot_type`/`triggered_by`/`created_at`, which +describe *why* and *for whom* a snapshot was captured, not what it contains. +`CreateGitSnapshot` and `UpsertLatestLiveGitSnapshot` both compute and persist +it on every write. Per the plan's "destructive maintenance is explicit, +dry-run-first" constraint, this wave does **not** skip a write when the +computed digest matches the immediately preceding row (an initial write-time +skip design was tried and reverted after it silently broke two pre-existing +ordering tests — `TestGetLatestGitSnapshotFallsBackToNewestWithoutAgentCompleted` +and `TestGetGitSnapshotsBySessionOrdersDescendingAndHonoursLimit` — that +depend on every historical poll being recorded). Instead, +`ListDuplicateGitSnapshotCandidates` (new, read-only) reports every row in a +per-session `(session_id, content_digest)` group except the newest (matching +`snapshotRankExpr`'s tie-break), for a later, explicit maintenance pass +(Task 06) to prune. Equal snapshots from different sessions remain distinct +history and are not candidates. A dangling-session-reference "orphan" query was +considered and dropped: `task_session_git_snapshots.session_id` has +`FOREIGN KEY ... REFERENCES task_sessions(id) ON DELETE CASCADE`, so that +condition is schema-guaranteed impossible to reach (confirmed by a test that +tried to insert one and hit the FK constraint). + +**Plan-revision retention candidates (read-only).** +`ListObsoletePlanRevisionCandidates(ctx, taskID, keepLastN)` (new, in +`plan.go`) reports superseded revisions: it always excludes the task's +current HEAD (`MAX(revision_number)`) and any revision some other revision's +`revert_of_revision_id` points to (so the "restore an earlier plan" ancestry +chain is never broken), and additionally excludes the most recent `keepLastN` +non-HEAD revisions when `keepLastN > 0`. The revision *doing* the revert is +not itself protected by ancestry (only the revision it points *to* is) — its +content already duplicates the protected target, so losing it doesn't lose +information. Purely a `SELECT`; a dedicated non-destructiveness test asserts +row count is unchanged after calling it. + +**Message/conversation preservation.** No conversational message row, and no +message's human/agent-authored `content`, was touched by any change in this +wave — only the `metadata` JSON of shell-tool-call messages over the +externalization threshold is ever rewritten (to the summary shape), and the +original bytes remain fully recoverable via `RehydrateMessagePayload`'s +verified round trip. + +**Tests** (`internal/task/repository/sqlite`, `internal/task/service`, +`internal/task/handlers`; all new tests use `newRepoForSessionTests`/ +`newRepoForEntityTests` against the real migrated SQLite schema, not mocks): +- `message_payload_test.go`: small metadata stays inline + (`TestCreateMessageLeavesSmallMetadataInline`); large output is + externalized then correctly rehydrated end-to-end + (`TestCreateMessageExternalizesLargeShellOutputAndRehydrates`); identical + large payloads across two messages dedupe to one + `task_message_payloads` row + (`TestExternalizeMessagePayloadDedupesIdenticalContentAcrossMessages`); + tampering with stored bytes is rejected by the integrity check + (`TestRehydrateMessagePayloadRejectsTamperedContent`). +- `git_snapshot_digest_test.go`: digest computed and always recorded per + poll (`TestCreateGitSnapshotSetsContentDigestAndAlwaysRecords`), upsert + path sets the digest too (`TestUpsertLatestLiveGitSnapshotSetsContentDigest`), + duplicate-candidate selection keeps only the newest of a group + (`TestListDuplicateGitSnapshotCandidatesKeepsNewestPerGroup`) and is + read-only (`TestListDuplicateGitSnapshotCandidatesIsNonDestructive`). +- `plan_retention_test.go`: HEAD + revert-ancestry protection + (`TestListObsoletePlanRevisionCandidatesProtectsHeadAndAncestry`), + `keepLastN` recency window + (`TestListObsoletePlanRevisionCandidatesRespectsRecencyWindow`), and + non-destructiveness + (`TestListObsoletePlanRevisionCandidatesIsNonDestructive`). +- `message_list_handlers_test.go`: `TestHTTPListMessagesDefaultsToABoundedFirstPage`, + `TestWSListMessagesDefaultsToABoundedFirstPage` (both assert the paginated + path runs with `limit == service.DefaultMessagesPageSize` when no params + are given); pre-existing `TestHTTPShellOutputSnapshot` / + `TestHTTPGetShellOutputDeniesForeignMessageWith404` continue to pass + unmodified, confirming no regression to the lazy shell-output route. + +**Verification run:** +~~~text +$ go test ./internal/task/repository/sqlite ./internal/task/service \ + -run 'Test.*(Message|History|Payload|Snapshot|Retention|PlanRevision).*' -count=1 -v +ok github.com/kandev/kandev/internal/task/repository/sqlite +ok github.com/kandev/kandev/internal/task/service + +$ go test ./internal/task/repository/sqlite ./internal/task/service -count=1 +ok github.com/kandev/kandev/internal/task/repository/sqlite +ok github.com/kandev/kandev/internal/task/service + +$ go test ./internal/task/repository/sqlite ./internal/task/service \ + -run 'Test.*(Message|History|Payload|Snapshot|Retention|PlanRevision).*' -count=1 -race +ok github.com/kandev/kandev/internal/task/repository/sqlite +ok github.com/kandev/kandev/internal/task/service + +$ go build ./... # apps/backend, clean +$ go vet ./... # apps/backend, clean +$ golangci-lint run ./internal/task/repository/sqlite/... ./internal/task/service/... \ + ./internal/task/handlers/... ./internal/task/models/... ./internal/task/repository/... +0 issues. +$ gofmt -l # empty +$ git diff --cached --check # empty +~~~ + +Two pre-existing failures observed in `internal/task/service`/ +`internal/task/handlers` full-package runs +(`TestServiceInitializeLocalRepositoryCreatesMainRepository` and siblings, +"parent directory cannot be accessed") are environmental — reproduced +identically with this wave's changes fully stashed, so unrelated to this +task. + +**Completed by Tasks 06-07 and final QA remediation.** Task 06 acts on +`ListDuplicateGitSnapshotCandidates` / `ListObsoletePlanRevisionCandidates` / +stale `task_message_payloads` rows through the dry-run-first, backup-gated, +offline command. The final retention regression additionally proves an older +authoritative archive or `agent_completed` snapshot remains selected and +preserved when a newer live-monitor row has the same content digest. + +The Task 07 large-history fixture now creates one million lightweight message +rows and 4 GiB of logically externalized payload history in a temporary +SQLite database. Fifteen page reads return exactly 50 lightweight messages, +use the prompt-order index, never reference the payload table, and increment +no payload-hydration metric. It records p50/p95/max latency as diagnostic +evidence while keeping pass/fail assertions on deterministic query, row, byte, +and hydration bounds rather than a host-dependent absolute latency threshold. diff --git a/docs/plans/pr-watch-and-storage-bounds/task-06-database-maintenance-command.md b/docs/plans/pr-watch-and-storage-bounds/task-06-database-maintenance-command.md new file mode 100644 index 0000000000..e8c075af95 --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-06-database-maintenance-command.md @@ -0,0 +1,153 @@ +--- +id: "06-database-maintenance-command" +title: "Provide safe database retention maintenance" +status: done +wave: 3 +depends_on: ["04-github-auth-backoff-observability", "05-bounded-history-storage"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-DATABASE-MAINTENANCE-001 +acceptance_criteria: + - AC-PLATFORM-DATABASE-MAINTENANCE-001.1 + - AC-PLATFORM-DATABASE-MAINTENANCE-001.2 + - AC-PLATFORM-DATABASE-MAINTENANCE-001.3 + - AC-PLATFORM-DATABASE-MAINTENANCE-001.4 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 06: Provide safe database retention maintenance + +## Intent + +Give operators a backup-gated command to report, execute, compact, and roll back bounded-storage cleanup. + +## Acceptance + +- The native maintenance database command supports dry run, explicit retention settings, per-table candidate/reclaim estimates, and fails closed when a verified backup is absent. +- Execution removes only selected redundant payloads/snapshots/revisions in transactions, stages/validates compaction with VACUUM INTO or equivalent, atomically replaces only after success, and prints rollback steps. +- The command reports SQLite/WAL and table-storage measurements without contents, credentials, or destructive default behavior. + +## Files likely touched + +- apps/backend/internal/maintenance/*.go (new package: analyze/execute/backup/compact/run) +- apps/backend/internal/launcher/maintenance_database.go (new CLI subcommand) +- apps/backend/internal/launcher/launcher.go (dispatch wiring) +- apps/backend/internal/task/repository/sqlite/message_payload.go (`ListOrphanedMessagePayloadCandidates`) +- apps/backend/internal/task/repository/sqlite/base_queries.go (shared candidate-query helper) +- apps/backend/internal/task/repository/sqlite/git_snapshot_digest.go (reuses shared helper) +- apps/backend/internal/persistence/snapshot.go (reused, not modified) +- docs/public/cli.md +- New `*_test.go` files in the above packages + +## Dependencies + +Tasks 04 and 05. + +## Parallelism + +Sequential. It consumes finalized retention data and health/configuration contracts. + +## Verification + +~~~bash +cd apps/backend && go test ./internal/maintenance/... ./internal/launcher/... ./internal/task/repository/sqlite/... -run 'Test.*(Maintenance|Retention|DryRun|Backup|Vacuum|Compact|Rollback|Orphaned|Duplicate).*' -count=1 -v +cd apps/backend && go test ./internal/maintenance/... ./internal/launcher/... ./internal/task/repository/sqlite/... -count=1 +cd apps/backend && go test ./internal/maintenance/... ./internal/launcher/... ./internal/task/repository/sqlite/... -race -count=1 +cd apps/backend && go build ./... && go vet ./... +cd apps/backend && gofmt -l internal/maintenance internal/launcher internal/task/repository/sqlite +cd apps/backend && golangci-lint run ./internal/maintenance/... ./internal/launcher/... ./internal/task/repository/sqlite/... +git diff --check +~~~ + +## Output contract + +Report dry-run versus execution row/byte counts, backup verification, staged-compaction failure preservation, successful restore/rollback, command output redaction, and test results. Update task and plan status. + +## Results + +**Status: done.** + +Implemented as a new offline `internal/maintenance` package (`maintenance.go`, +`report.go`, `execute.go`, `backup.go`, `compact.go`, `run.go`) plus the +`kandev maintenance database` CLI subcommand +(`internal/launcher/maintenance_database.go`, dispatched from +`internal/launcher/launcher.go`). Added the missing Task-05-adjacent +`ListOrphanedMessagePayloadCandidates` query +(`internal/task/repository/sqlite/message_payload.go`) so the "stale +payload" retention category has real data to report/delete, and extracted a +shared `listLimitedCandidates` generic helper +(`internal/task/repository/sqlite/base_queries.go`) reused by both the +orphaned-payload and duplicate-git-snapshot listers to keep them +lint-clean under `dupl`. + +**Acceptance:** +- Dry run (no `--execute`) never acquires a lock, takes a backup, or + mutates anything; it reports row/byte estimates per category + (duplicate git snapshots, obsolete plan revisions, orphaned message + payloads) plus current database size. Verified by + `TestRunOptionsDryRun...`-style tests in `internal/maintenance` and the + CLI-level `TestRunMaintenanceDryRunEndToEndReportsCandidates`. +- `--execute` takes the same `ownershiplock` exclusive-access lock a live + backend takes at boot (refuses to run concurrently with `kandev`), + always takes and verifies a fresh `VACUUM INTO` backup before deleting + anything, and performs all retention deletes inside one transaction. + Verified by ownership-conflict-refusal, snapshot-failure-abort, and + execute+idempotency tests in both `internal/maintenance` and the + CLI-level `TestRunMaintenanceExecuteRefusesWhileOwnershipLockHeld` / + `TestRunMaintenanceExecuteEndToEndDeletesAndBacksUp`. +- `--compact` stages a `VACUUM INTO` compacted copy after retention + deletes commit, verifies it with an integrity check, requires + sufficient free disk, and atomically replaces the live database, + preserving a `.pre-compact-.bak` rollback artifact. A + compaction failure after committed deletes returns a partial-success + `Outcome` (populated backup path/delete counts) rather than a total + abort. Verified by insufficient-disk-refusal, integrity-check-failure, + and execute+compact+rollback tests. +- Reconciliation/candidate selection is read-only and non-destructive + until `--execute`; normal conversation rows, current plan HEAD, revert + ancestry, and protected snapshots are never candidates (candidate + queries only ever select superseded/orphaned rows). +- No secrets, tokens, branches, task IDs, or payload contents appear in + CLI output or logs; the CLI prints only structural counts, byte + estimates, and file paths. + +**Tests:** 8 focused tests in `internal/maintenance/run_test.go` (dry-run +non-mutation, unsupported-driver refusal, ownership-conflict refusal, +execute+backup+idempotency, execute+compact+rollback, +insufficient-disk-space refusal, snapshot-failure abort, +integrity-check-failure abort) plus 2 tests in +`internal/task/repository/sqlite/message_payload_retention_test.go` and 9 +new tests in `internal/launcher/maintenance_database_test.go` (arg-parsing +unit tests + help/unknown-target + 3 end-to-end CLI-path tests covering +dry run, execute+backup, and ownership-lock refusal). All pass, including +under `-race`. + +**Verification commands run** (from `apps/backend`): +- `go build ./...` — clean. +- `go vet ./...` — clean. +- `go test ./internal/maintenance/... ./internal/task/repository/sqlite/... -race -count=1` — `ok` both packages. +- `go test ./internal/launcher/... -race -skip 'TestRunDevAbortsOnBackupFailureBeforeLaunch|TestRunDevLaunchesBackendThenWebAndOpensBrowser|TestRunDevUsesConfiguredHealthTimeoutForBackendAndWeb' -count=1` — `ok` both `internal/launcher` and `internal/launcher/cli`. +- `gofmt -l` on all new/changed files — no output (clean). +- `golangci-lint run ./internal/maintenance/... ./internal/launcher/... ./internal/task/repository/sqlite/...` — `0 issues`. +- `git diff --check` — clean. + +**Deliberate deferral / pre-existing issue (not part of this task's +scope):** `TestRunDevAbortsOnBackupFailureBeforeLaunch`, +`TestRunDevLaunchesBackendThenWebAndOpensBrowser`, and +`TestRunDevUsesConfiguredHealthTimeoutForBackendAndWeb` in +`internal/launcher/dev_test.go` fail on this Go toolchain +(`go1.26.0`) with a nil-pointer panic in +`restartableBackend.Exited`/`waitForHealth`, reproduced identically +against the pre-Task-06 commit +(`e0768ba67423c5563f5c27d514076a4f053e80e1`) with this task's new files +moved aside — confirmed pre-existing and unrelated to this change, not +introduced or fixed by Task 06. + +`internal/system/storage` (the live, HTTP-triggered System→Database +maintenance UI) and `internal/persistence` (boot-time version-change +snapshot) were deliberately left unmodified: Task 06 is a separate, +offline/exclusive CLI path that reuses `persistence.SnapshotSQLite` and +the same `ownershiplock` targets a live backend uses, but does not extend +either existing package, per the plan's separation of the "live" and +"offline" maintenance concepts. diff --git a/docs/plans/pr-watch-and-storage-bounds/task-07-operator-docs-load-validation.md b/docs/plans/pr-watch-and-storage-bounds/task-07-operator-docs-load-validation.md new file mode 100644 index 0000000000..cf519bc64a --- /dev/null +++ b/docs/plans/pr-watch-and-storage-bounds/task-07-operator-docs-load-validation.md @@ -0,0 +1,176 @@ +--- +id: "07-operator-docs-load-validation" +title: "Document operations and validate sustained load" +status: done +wave: 3 +depends_on: ["06-database-maintenance-command"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-PR-WATCH-IDENTITY-002 + - REQ-PLATFORM-TASK-SUMMARY-CONTENTION-001 + - REQ-PLATFORM-DATABASE-MAINTENANCE-001 +acceptance_criteria: + - AC-PLATFORM-PR-WATCH-IDENTITY-002.3 + - AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.4 + - AC-PLATFORM-DATABASE-MAINTENANCE-001.5 +system_design: + - ../../specs/platform/system-design/pr-watch-and-bounded-storage.md +--- + +# Task 07: Document operations and validate sustained load + +## Intent + +Publish the safe upgrade/maintenance procedure and prove the combined change is stable under the production-like duplicate-watch load. + +## Acceptance + +- Public operations documentation describes backup of kandev.db and master.key, image upgrade, migration/maintenance dry run, execute/compact, rollback, and post-release health checks. +- A deterministic ten-minute multi-task/coordinator load test proves no watch-creation loop, one lookup per canonical identity, no recurring exhausted CAS errors, and stable API/history latency. +- Documentation identifies the maintenance command’s required backup and operator-selected retention, without claiming automatic deletion of conversations. + +## Files likely touched + +- docs/public/operations.md (new "Upgrading past the canonical PR-watch migration" subsection) +- apps/backend/internal/github/load_validation_test.go (new load-validation fixture) +- apps/backend/internal/task/statussummary/projector_load_test.go (new sustained-contention fixture) + +## Dependencies + +Task 06. + +## Parallelism + +Sequential. Documentation must describe the shipped command and load validation exercises the integrated behavior. + +## Verification + +~~~bash +cd apps/backend && go test ./internal/github ./internal/task/statussummary -run 'Test.*(Canonical|Load|TenMinute|Concurrent|CAS).*' -count=1 -v +cd apps/backend && go test ./internal/github ./internal/task/statussummary -count=1 +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +git diff --check +~~~ + +## Output contract + +Report the exact load fixture scale, lookup/watch/CAS/error/latency measurements, public-doc pages and Diátaxis type, validation outcomes, and upgrade/rollback instructions verified. Update task and plan status. + +## Results + +**Status: done.** + +**Operator documentation** (`docs/public/operations.md`, new "Upgrading past +the canonical PR-watch migration" subsection inside the existing "Database +operation details" `
` block, alongside the existing PostgreSQL +schema-cutover upgrade pattern): describes the one-time, idempotent, +transactional `github_pr_watches` migration +(ADR-2026-08-31-task-owned-pr-watch-identity), that it runs after the +existing version-change SQLite snapshot and aborts startup before any +destructive change on a failed snapshot or migration, and that it only ever +removes duplicate/orphaned watch rows while preserving Review-task +monitoring and never touching conversation messages/snapshots/plans. Gives +the four-step safe-upgrade procedure (verify a backup, stop all writers +before starting the new binary, start exactly one instance and spot-check +health plus an open-PR task and a completed-session Review task, optionally +run `kandev maintenance database` afterward) and cross-links to the +existing [Database maintenance](../../public/cli.md#database-maintenance) +section (Task 06) for the maintenance command's own dry-run/execute/compact/ +rollback semantics, plus a rollback note (restore the pre-migration snapshot +only if an older binary is needed again; the migration is otherwise safe to +retry because it is idempotent). This satisfies the acceptance criterion's +required backup/upgrade/migration-dry-run/execute-compact/rollback/health-check +coverage and explicitly states the maintenance command's own retention is +operator-selected and backup-gated, not automatic conversation deletion. + +**Load validation** (new deterministic fixtures, no shared test-scaffold +package needed — both reused existing package test helpers): + +- `apps/backend/internal/github/load_validation_test.go` — + `TestLoadValidation_TenSimulatedMinutes_NoWatchCreationLoopBoundedLookups`. + Seeds one task resumed across 50 historical sessions (mirroring the plan's + AC1 fixture) plus 24 other concurrently-active tasks (25 canonical + task/repository/branch identities total, standing in for many + simultaneous coordinator-driven boards), then drives ten simulated + `checkPRWatches` poll cycles (the poller's real `defaultPRPollInterval` + cadence, without sleeping wall-clock time between cycles). Asserts: (1) + the exact watch-ID set is unchanged from cycle 0 through cycle 9 and the + canonical watch count is always 25, never the 74 total historical-session + entries the provider reports (no watch-creation loop); (2) each cycle + performs exactly one batched GraphQL branch-search call for all 25 + still-searching watches, so ten cycles produce exactly 10 branch calls and + 0 numbered-PR calls (bounded lookups, independent of session-history + size); (3) no single cycle's wall-clock duration exceeds a 2-second + absolute bound (stable latency, ruling out a duplicate-row leak's + quadratic-cost shape). Reuses the package's existing + `setupBatchedPollerTest`/`mockTaskBranchProvider`/`graphQLMockClient`/ + `seedTask` test helpers. +- `apps/backend/internal/task/statussummary/projector_load_test.go` — + `TestProjectorSustainedLoadAcrossManyTasksConvergesWithoutExhaustedCAS`. + Scales the existing single-task + `TestProjectorConcurrentSessionEventsConvergeWithoutExhaustedCAS` harness + up to the same 25-task, ten-simulated-cycle shape as the github-package + fixture: each cycle fires one concurrent burst of 4 sessions x 25 tasks + (100 concurrent `HandleEvent` calls/cycle, 1000 total across the run) + while a new per-task `perTaskAlternatingRejectStore` continuously forces + every other `CompareAndUpdateTaskStatusSummary` call to lose its CAS race + for every task throughout the entire sustained run (not just a single + burst) — modeling a boot-reconciliation pass or HTTP rebuild racing the + live projector continuously. A new per-task store was required because + the existing single-task `alternatingRejectProjectorStore`'s one *global* + counter only guarantees "reject-then-accept" ordering when no other + task's calls can interleave between a given task's own attempts; with 25 + tasks contending on one global counter, that guarantee breaks and + produces spurious CAS-exhaustion errors unrelated to the code under test + (confirmed by first reproducing the failure, then fixing it with the + per-task counter). Asserts zero exhausted-CAS handler errors across all + 1000 calls and that every task's final summary reflects the cumulative + 40 sessions applied to it over the ten cycles, proving sustained, + many-task contention never drops or misapplies concurrent work. + +**Verification commands run** (from `apps/backend`): +- `go test ./internal/github ./internal/task/statussummary -run 'Test.*(Canonical|Load|TenMinute|Concurrent|CAS).*' -count=1 -v` — all pass (includes both new load fixtures and the existing canonical-migration/CAS/contention suites). +- `go test ./internal/github ./internal/task/statussummary -count=1` — `ok` both packages. +- `go test ./internal/github/... -run 'TestLoadValidation' -race -count=10` and `go test ./internal/task/statussummary/... -run 'TestProjectorSustainedLoad' -race -count=10` — both stable across 10 repeated runs under `-race`. +- `go build ./...` and `go vet ./...` (full repo) — clean. +- `gofmt -l` on both new test files — no output (clean). +- `golangci-lint run ./internal/github/... ./internal/task/statussummary/...` — `0 issues` (one `staticcheck` QF1008 finding on the first draft of the per-task store was fixed by renaming its own mutex field to avoid embedded-field ambiguity, then re-verified). +- `node scripts/validate-public-docs.mjs` — validated 41 published docs pages. +- `node --test scripts/validate-public-docs.test.mjs` — 61 tests, all pass. +- `git diff --check` — clean. + +**Deliberate deferrals:** none new. The three pre-existing, unrelated +`internal/launcher/dev_test.go` toolchain failures documented in Task 06's +Results remain untouched and out of scope for this wave; this task's +verification commands do not target `internal/launcher`. + +**Final QA amendments.** The initial load suite covered PR polling and status +projection but did not exercise the large-history half of the accepted +storage contract. `message_history_load_test.go` now creates 1,000,000 +lightweight message rows and 64 external payload records representing 4 GiB +of logical tool-output history in a temporary SQLite database. The payload +bodies are bounded sentinels because integrity and real gzip round trips are +already covered separately; this fixture isolates normal list amplification +without allocating a multi-gigabyte artifact. It asserts the SQL has a +`LIMIT`, does not reference the payload table, uses +`idx_messages_prompt_order`, returns exactly 50 rows plus the `hasMore` +sentinel, returns only bounded lightweight content, and records zero explicit +payload hydrations over 15 repeated reads. A focused run on 2026-09-05 +measured p50 `262.638µs`, p95 `355.927µs`, and max `355.927µs` after fixture +creation. These latency values are diagnostic rather than a brittle absolute +pass threshold: the approved risk policy treats structural query/row/payload +bounds as authoritative across heterogeneous CI hosts. + +The final observability pass publishes aggregate counters/gauges for active, +searching, duplicate, and orphan PR watches; canonical poll targets; status +projection CAS retries/exhaustions/handler failures; logical message, +metadata, compressed payload, and Git snapshot bytes; database/WAL bytes; +bounded payload-hydration latency buckets; current unreserved queue depth; and +active runtimes. Tests pin every public metric name and update path. Labels +are absent or limited to bounded success/error and latency-bucket values. +The operator contract and typed database-stat fields are documented in +[PR-watch and bounded-storage diagnostics](../../public/cli.md#pr-watch-and-bounded-storage-diagnostics). + +This is the final task in the plan; all seven waves (Tasks 01-07) are now +implemented, tested, documented, and committed. diff --git a/docs/public/cli.md b/docs/public/cli.md index a91a669565..cba9545e7e 100644 --- a/docs/public/cli.md +++ b/docs/public/cli.md @@ -246,6 +246,94 @@ Supported actions are `install`, `uninstall`, `start`, `stop`, `restart`, `statu
+## Database maintenance + +`kandev maintenance database` is an offline SQLite retention and compaction +command for the same file the backend uses (`--home-dir`, `database.path`, +`KANDEV_HOME_DIR`, and `KANDEV_DATABASE_PATH` resolve identically to the +normal launcher): + +```text +kandev maintenance database [--execute] [--compact] + [--keep-plan-revisions ] [--candidate-limit ] [--home-dir ] +``` + +| Option | Meaning | +|---|---| +| `--execute` | Perform retention deletes. Without it, the command is a read-only dry run (the default). | +| `--compact` | After retention deletes commit, additionally stage a compacted copy and atomically replace the live database. Only takes effect together with `--execute`. | +| `--keep-plan-revisions ` | Additionally protect the `n` most recent non-HEAD plan revisions per task (default `0`). | +| `--candidate-limit ` | Cap how many rows are reported/deleted per retention category (default: unlimited). | +| `--home-dir ` | Kandev home directory override (default: `$KANDEV_HOME_DIR` or `~/.kandev`). | +| `--help`, `-h` | Print command help. | + +**Dry run (default).** Without `--execute`, the command never acquires a +lock, takes a backup, or changes anything; it is always safe to run +alongside a live `kandev` process. It reports a row count and estimated +byte size for each retention category: duplicate git snapshots (content- +identical repository-state polls for the same session), obsolete plan +revisions (superseded, non-HEAD plan history beyond `--keep-plan-revisions`), +and orphaned message payloads (externalized tool output no message still +references). Normal conversation rows, the current plan HEAD, revert +ancestry, and any protected snapshot are never candidates. + +**Execute.** `--execute` requires exclusive access to the database: it takes +the same ownership lock a live backend takes at boot, so it refuses to run +while `kandev` is already running against the same home/database. It always +takes and verifies a fresh backup before deleting anything, written to +`/backups/kandev-pre-maintenance-.db`; a failed or +unverified backup aborts before any delete. Retention deletes run inside a +single transaction, so partial deletion across categories cannot happen. + +**Compaction and rollback.** `--compact` (with `--execute`) stages a +`VACUUM INTO` compacted copy after retention deletes commit, verifies it +with an integrity check, requires enough free disk for the staged copy, and +atomically replaces the live database file. The pre-compaction file is kept +alongside the live database as +`.pre-compact-.bak` for manual rollback: stop +`kandev` if it is running, then move that file back over the live database +path. If compaction fails after retention deletes already committed, the +command still reports the completed backup path and deleted-row counts; only +the optional compaction step is aborted. + +### PR-watch and bounded-storage diagnostics + +Kandev publishes aggregate operational counters through Go's `/debug/vars` +endpoint when diagnostics are enabled with `--debug`, +`KANDEV_DEBUG_DEV_MODE=true`, or `KANDEV_DEBUG_PPROF_ENABLED=true`. Bind the +backend to loopback while diagnostics are enabled and treat the complete +response as sensitive process data. The counters below never use task, +workspace, repository, branch, session, or execution identifiers as labels. + +| Metric | Meaning | +|---|---| +| `github_pr_watch_active` | Canonical watches eligible for polling in the latest poll census. | +| `github_pr_watch_searching` | Eligible watches still searching by branch. | +| `github_pr_watch_duplicates` | Rows that violate the canonical searching or discovered identity in the latest census. Expected value: `0`. | +| `github_pr_watch_orphans` | Watches whose task or task/repository relationship is missing. Expected value: `0`. | +| `github_pr_watch_canonical_poll_requests_total` | Canonical watch targets submitted to GitHub polling, including targets combined into one batch request. | +| `task_status_summary_cas_retries_total` | Projection compare-and-swap losses retried after reloading authoritative state. | +| `task_status_summary_cas_exhaustions_total` | Projection updates that exhausted the bounded retry policy. Expected steady-state value: no increase. | +| `task_status_summary_event_handler_failures_total` | Status-summary events that returned a handler error. | +| `task_message_payload_hydrations_total` | Explicit payload hydration outcomes, labelled only by `outcome=success` or `outcome=error`. | +| `task_message_payload_hydration_latency_ms` | Cumulative hydration counts in the bounded `10`, `50`, `250`, `1000`, and `+Inf` millisecond buckets, labelled by outcome. | +| `task_message_content_bytes` | Logical bytes in message content from the latest database-stat read. | +| `task_message_metadata_bytes` | Logical bytes in inline message metadata from the latest database-stat read. | +| `task_message_payload_compressed_bytes` | Compressed external payload bytes from the latest database-stat read. | +| `task_git_snapshot_bytes` | Logical Git snapshot file and metadata bytes from the latest database-stat read. | +| `database_size_bytes` / `database_wal_size_bytes` | Database and SQLite WAL bytes from the latest database-stat read. PostgreSQL reports `0` for the SQLite-only WAL gauge. | +| `message_queue_depth` | Current unreserved prompt queue depth. `-1` means the queue is unavailable or the bounded read failed. | +| `agent_active_runtimes` | Agent executions currently owned by the lifecycle runtime. | + +`GET /api/v1/system/database` refreshes the database and storage gauges and +returns the same values as typed fields: `size_bytes`, `wal_size_bytes`, +`message_content_bytes`, `message_metadata_bytes`, `message_payload_bytes`, +and `git_snapshot_bytes`. Storage categories use portable logical byte +aggregates, so they remain available when SQLite `dbstat` or PostgreSQL +relation-size facilities are unavailable. Use them for trends and retention +decisions; use filesystem or database-provider metrics for physical quota and +capacity alerts. + ## Ports and network exposure | Process | Preferred port | Automatic behavior | diff --git a/docs/public/operations.md b/docs/public/operations.md index febd2fbbdf..0d4650d16c 100644 --- a/docs/public/operations.md +++ b/docs/public/operations.md @@ -528,6 +528,48 @@ The normalized schema is intentionally incompatible with older binaries. To down Switching `database.driver` does not migrate data. PostgreSQL and shared NATS remove two single-process data constraints, but they do not make Kandev horizontally scalable: WebSocket subscriptions, execution lifecycle/control state, and task workspaces remain process- or filesystem-local. The current product and supplied deployment validate one backend replica only; do not add replicas based on the database and event bus alone. +### Upgrading past the canonical PR-watch migration + +A release that makes GitHub PR watches task-owned instead of session-owned (see +[ADR-2026-08-31-task-owned-pr-watch-identity](../decisions/2026-08-31-task-owned-pr-watch-identity.md)) +runs a one-time, idempotent, transactional cleanup of `github_pr_watches` +during SQLite startup, after the existing version-change snapshot described +above. A failed snapshot or a failed migration aborts startup before any +destructive change; a database that has already been migrated, or a fresh +install, skips it entirely. The migration only ever removes duplicate rows +for the same task/repository/branch (keeping the newest check/comment/review +status and any discovered PR number) and rows orphaned by a deleted task or a +detached repository; it never removes normal conversation messages, snapshots, +or plans, and Review-task monitoring survives the originating session's +completion. + +To upgrade safely: + +1. Take and verify a SQLite backup as described above (manual snapshot from + **Settings > System > Backups**, or `sqlite3` `VACUUM INTO`); the automatic + pre-migration snapshot is taken during the new binary's startup, so it + cannot be verified beforehand. +2. Stop all backend writers before starting the new binary; do not run a + mixed-version fleet across the upgrade. +3. Start exactly one instance of the new binary and let it reach a healthy + state. Check **System > Status** or `/health`, then spot-check a task that + has an open pull request and a Review-stage task whose originating session + already completed: both should still show a monitored PR. +4. Optionally, after confirming health, run `kandev maintenance database` + (dry run first) to review and, if desired, reclaim storage from the + superseded rows the migration and prior duplicate polling left behind; see + [Database maintenance](cli.md#database-maintenance) for its full dry-run, + execute, compact, and rollback behavior. The maintenance command never + deletes ordinary human/agent conversation messages, and its own retention + deletes still require an explicit `--execute` plus a verified backup + independent of the migration's own snapshot. + +If startup aborts during the migration, the pre-migration snapshot in +`backups/` is untouched and the live database was never partially modified: +restore from that snapshot only if you need to run an older binary again, +otherwise fix the reported cause and restart the same new binary; the +migration is safe to retry because it is idempotent. + ## SQLite backups diff --git a/docs/specs/platform/README.md b/docs/specs/platform/README.md index 4482f2e9cb..7150478aaa 100644 --- a/docs/specs/platform/README.md +++ b/docs/specs/platform/README.md @@ -41,6 +41,7 @@ retains ownership of its values, validation, authority, and persistence. ### Requirements +- [Task Status Summary Projection Contention](requirements/task-summary-contention.md) - [CI performance](requirements/ci-performance.md) - [Startup lifecycle](requirements/startup-lifecycle.md) @@ -92,6 +93,7 @@ retains ownership of its values, validation, authority, and persistence. ### System design +- [Task Status Summary Projection Contention](system-design/task-summary-contention.md) - [CI performance](system-design/ci-performance.md) - [Startup lifecycle](system-design/startup-lifecycle.md) diff --git a/docs/specs/platform/pr-watch-and-storage-bounds.md b/docs/specs/platform/pr-watch-and-storage-bounds.md new file mode 100644 index 0000000000..d0c8ea83fb --- /dev/null +++ b/docs/specs/platform/pr-watch-and-storage-bounds.md @@ -0,0 +1,76 @@ +--- +status: draft +created: 2026-08-31 +owner: kandev +--- + +# Canonical PR Monitoring and Bounded Task Storage + +## Why + +Long-lived Kandev installations must remain responsive as tasks are resumed and agents generate rich tool and Git history. Duplicate PR monitoring and unbounded large operational payloads create database growth and background work unrelated to available host CPU, memory, or I/O capacity. + +## What + +- Kandev monitors each searching pull request once per task, repository, and branch, and each discovered pull request once per task, repository, and PR number. +- A Review task continues to receive PR monitoring after its originating session completes. Orphaned, archived-task, and invalid repository watches are not polled. +- Unchanged branch state causes no watch creation or watch update. A branch change transitions one searching watch atomically and never oscillates between equivalent session observations. +- PR feedback and task-PR events are emitted only for a changed task/repository/PR/head-SHA/status state. Equivalent events cause no task-summary revision or handler error. +- Task-status projection coalesces equivalent pending work per task, serializes projection per task, rebases from authoritative state after genuine contention, and uses bounded jittered backoff for a cross-writer CAS race. +- Session history APIs are cursor-paginated and normal history hydration excludes large tool-call metadata. Detailed tool payloads are retrieved only when explicitly requested. +- Large tool execution payloads are compressed or externalized with an integrity digest/reference. Human and agent conversational messages remain preserved by default. +- Every Git snapshot write is retained. Explicit maintenance may remove redundant live/status snapshots only within the same session and content-digest group; snapshots from different sessions remain distinct history. Configurable retention also selects superseded tool payloads and obsolete plan revisions without silently deleting them. +- An operator-run database maintenance command reports a dry-run reclaim estimate by table, requires a current database backup before destructive cleanup, supports configured retention, safely compacts SQLite, and documents rollback. Upgrade does not silently delete task history. +- GitHub authentication and configuration failures enter a visible degraded/backoff state and resume probing after credentials or configuration change. +- Health/metrics expose watch cardinality and duplication, per-canonical-PR polling, projection contention and failures, storage and SQLite/WAL size, hydration latency, queue depth, and active runtime count. + +## Data model + +`github_pr_watches` is task-owned as defined by [ADR-2026-08-31-task-owned-pr-watch-identity](../../decisions/2026-08-31-task-owned-pr-watch-identity.md). Searching identity is `(task_id, repository_id, branch)`; discovered identity is `(task_id, repository_id, pr_number)`. `session_id` is nullable provenance and never a unique identity component. + +Tool payload storage records digest, encoding, byte size, and an internal reference when a payload exceeds the configured inline threshold. `task_session_messages` retains a lightweight metadata projection sufficient for transcript rendering. Retention records report, but do not implicitly delete, superseded payloads, duplicate snapshots, and plan revisions older than their configured retention policies. + +## API and operations surface + +- Existing session-message list responses accept and return stable cursor pagination. Normal list/detail hydration returns lightweight tool metadata; a scoped explicit payload request returns the stored detail when it is retained. +- `kandev maintenance database` supports `--dry-run`, explicit retention options, `--backup ` or a verified freshly-created snapshot, `--compact`, and prints per-table candidate/reclaim estimates and rollback instructions. Destructive execution fails closed without a verified backup. +- Health/status output reports integration state (`healthy`, `degraded`, `disabled`) and the backoff reason without credentials. Debug metrics and structured logs expose the required watch, projection, storage, latency, queue, and runtime measures. + +## Failure modes + +- A watch migration or maintenance deletion is not started without a successful database snapshot. A failed transaction leaves all affected rows unchanged; a failed compaction leaves the original database in place and reports the staged artifact for removal or inspection. +- A missing task/repository relationship makes a watch ineligible for polling and eligible for the safe migration report; it never triggers a GitHub request. +- Invalid/unchanged GitHub credentials do not cause continuous polling. The integration reports degradation and only retries after its backoff elapses or the credential/configuration generation changes. +- A status-summary CAS loss reloads and rebases authoritative state. Equivalent queued refreshes succeed as no-ops and do not surface an event-handler error. + +## Persistence guarantees + +- Canonical watches, PR status watermarks, integration health/backoff state, and retention configuration survive restart. +- Conversational messages are not removed by upgrade or default retention. A retained compressed/external tool payload remains integrity-verifiable and is either retrievable or explicitly reported as expired. +- Database maintenance preserves a pre-operation backup and supplies a deterministic rollback procedure. SQLite compaction uses an atomically staged replacement or leaves the original file untouched. + +## Scenarios + +- **GIVEN** fifty historical sessions for one task, repository, and branch, **WHEN** reconciliation and polling run, **THEN** exactly one canonical searching or discovered watch exists and one GitHub lookup is issued for that identity. +- **GIVEN** a Review task whose provenance session is complete, **WHEN** its discovered PR changes, **THEN** Kandev continues monitoring and publishes one changed feedback/update event. +- **GIVEN** unchanged task/repository Git state, **WHEN** reconciliation runs repeatedly, **THEN** it creates no rows and updates no branch. +- **GIVEN** a canonical searching watch and a changed canonical branch, **WHEN** reconciliation runs, **THEN** one atomic transition occurs and later identical cycles make no write. +- **GIVEN** repeated equal PR events, **WHEN** the task-status projector receives them concurrently, **THEN** it emits at most one effective summary update, returns no exhausted-CAS handler error, and retains the correct final summary. +- **GIVEN** duplicated and orphaned legacy watch rows, **WHEN** upgrade migration runs after a snapshot, **THEN** it keeps one deterministic canonical row with the newest observed state, retains Review monitoring, removes only redundant/orphan rows transactionally, and a second run makes no change. +- **GIVEN** invalid GitHub credentials, **WHEN** workflow synchronization polls repeatedly, **THEN** it enters visible backoff and does not retry unchanged credentials continuously; a credential generation change resumes probing. +- **GIVEN** a multi-gigabyte fixture database, **WHEN** a client loads one history page, **THEN** the response is page-bounded, avoids eager large tool metadata, and records hydration latency. +- **GIVEN** a verified backup and a maintenance dry run, **WHEN** an operator requests retention cleanup, **THEN** Kandev reports candidates and estimated reclaim by table without deleting data; a subsequent explicit compact run can be rolled back from that backup. + +## Success criteria + +- The ten-minute multi-coordinator load test has no PR-watch creation loop, no recurring exhausted-CAS handler errors, and stable history/API latency. +- A canonical PR identity produces no more than one GitHub lookup per poll cycle. +- Maintenance output identifies reclaimable bytes for tool metadata/payloads, snapshots, and plan revisions before any destructive operation. + +## Out of scope + +- Increasing host hardware as a workaround. +- Disabling PR monitoring. +- Deleting human or agent conversation messages by default. +- Silent upgrade deletion beyond the documented canonical-watch migration. +- Retrofitting arbitrary external storage providers for tool payloads in this iteration. diff --git a/docs/specs/platform/requirements/pr-watch-and-bounded-storage.md b/docs/specs/platform/requirements/pr-watch-and-bounded-storage.md new file mode 100644 index 0000000000..7b77eaf46e --- /dev/null +++ b/docs/specs/platform/requirements/pr-watch-and-bounded-storage.md @@ -0,0 +1,214 @@ +--- +status: active +system: platform +created: 2026-09-14 +owners: + - kandev +--- + +# Canonical PR watch and bounded storage Requirements + +## Overview + +A Kandev task can be resumed across many sessions and worked by parallel +sessions. Legacy `github_pr_watches` identity was keyed by session, so each +resumed or parallel session could create its own watch row for the same task, +repository, and branch. Duplicate rows multiplied GitHub polling, event +publication, and status-summary work, and completing a session could silently +stop monitoring a Review task. + +Long-lived installations also accumulate large tool payloads, Git snapshots, +and plan revisions, and background integration loops retried broken +credentials every cycle. This change makes watch identity task-owned, +reconciliation idempotent, projection contention-safe, operational storage +bounded and retention-eligible, maintenance backup-gated, and provider +failures visibly backed off. + +Platform owns this cross-cutting contract because it covers the shared +task-status summary projector, the install-wide storage and maintenance +guarantees, and the background integration loops' health. The integration +system's GitHub provider contracts and the task system's conversation history +consume and feed these behaviors. + +## Requirements + +### REQ-PLATFORM-PR-WATCH-IDENTITY-001: Task-owned pull request watch identity + +**Intent:** One task, repository, and branch (or pull request) shall have at +most one canonical watch row regardless of how many sessions observed it, +and monitoring shall survive completion of the session that created it. + +#### Acceptance criteria + +- **AC-PLATFORM-PR-WATCH-IDENTITY-001.1:** A still-searching watch (no pull + request number yet) shall be unique per task, repository, and branch; a + discovered watch shall be unique per task, repository, and pull request + number; `session_id` shall remain optional provenance that cannot create a + second canonical watch. +- **AC-PLATFORM-PR-WATCH-IDENTITY-001.2:** The database shall enforce both + identities with partial unique indexes, and watch creation and ensure paths + shall deduplicate by task-owned keys so resumed or concurrent sessions reuse + the one canonical watch. +- **AC-PLATFORM-PR-WATCH-IDENTITY-001.3:** Moving a searching watch to a + branch or pull-request number that already has a canonical watch shall merge + into that sibling instead of violating the unique constraint, and concurrent + transitions of equivalent sessions shall coalesce to one destination. +- **AC-PLATFORM-PR-WATCH-IDENTITY-001.4:** The upgrade migration shall run + transactionally after the existing database snapshot boundary, collapse + legacy session duplicates preferring discovered rows while preserving the + newest status, check, review, and comment watermarks, remove watches for + missing tasks or detached repositories, clear provenance for missing + sessions, and be idempotent on a second boot. +- **AC-PLATFORM-PR-WATCH-IDENTITY-001.5:** Active watch listing shall exclude + invalid or orphaned watches but retain a non-archived Review task whose + provenance session has completed. +- **AC-PLATFORM-PR-WATCH-IDENTITY-001.6:** Branch resolution shall be keyed by + task and repository, so a branch observation for one repository cannot + overwrite a still-searching watch of another repository of the same task. + +### REQ-PLATFORM-PR-WATCH-IDENTITY-002: Unchanged watch state causes no writes or duplicate lookups + +**Intent:** Reconciliation that observes no change shall not write rows or +publish events, and repeated poll cycles shall not amplify lookups with +session history. + +#### Acceptance criteria + +- **AC-PLATFORM-PR-WATCH-IDENTITY-002.1:** A branch observation equivalent to + a watch's current state shall perform no insert and no watch update. +- **AC-PLATFORM-PR-WATCH-IDENTITY-002.2:** Pull request feedback and task + pull request events shall publish only for durable relevant state changes; + an unchanged sync shall publish nothing on both single-watch REST and + batched GraphQL paths. +- **AC-PLATFORM-PR-WATCH-IDENTITY-002.3:** A task resumed across many + sessions shall drive one branch or pull request lookup per canonical target + per poll cycle, and a simulated sustained-load run shall keep the canonical + watch set and per-cycle lookup counts constant across cycles. + +### REQ-PLATFORM-TASK-SUMMARY-CONTENTION-001: Coalesce and rebase task summary projection under contention + +**Intent:** Equivalent projection work for one task shall converge to one +effective persistence and publication, and a writer that genuinely loses a +compare-and-set race shall rebase and retry with bounded pacing instead of +failing the handler. + +#### Acceptance criteria + +- **AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.1:** Concurrent equivalent pending + refreshes for one task shall be single-flight coalesced and result in at + most one effective summary persistence and publication. +- **AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.2:** A writer whose + compare-and-set attempt is rejected shall reload the authoritative summary + state and rebase its derivation on it before retrying. +- **AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.3:** Retry pacing between rejected + attempts shall use a bounded exponential backoff with jitter and respect + context cancellation; the attempt bound shall remain unchanged and pacing + shall be injectable so tests can run instantaneously. +- **AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.4:** Under sustained per-task + contention where an external writer forces repeated compare-and-set losses, + concurrent events for many tasks shall preserve each task's correct final + summary and report zero exhausted-retry handler errors. +- **AC-PLATFORM-TASK-SUMMARY-CONTENTION-001.5:** Per-task in-process + serialization and the semantic no-op short-circuit for equivalent derived + summaries shall keep a semantically-equal refresh from touching the store or + publishing a new revision. + +### REQ-PLATFORM-BOUNDED-SESSION-HISTORY-001: Cursor-bounded history hydration + +**Intent:** Normal session-history reads shall stay responsive by fetching +only the requested window and never eagerly materializing large tool +metadata. + +#### Acceptance criteria + +- **AC-PLATFORM-BOUNDED-SESSION-HISTORY-001.1:** Session history list responses + shall accept and return a stable cursor so each hydration reads only the + requested window. +- **AC-PLATFORM-BOUNDED-SESSION-HISTORY-001.2:** Normal history hydration shall + return lightweight tool metadata without the full payload; the stored detail + of a retained tool payload shall be returned only by a scoped explicit + request. +- **AC-PLATFORM-BOUNDED-SESSION-HISTORY-001.3:** A reader on a separate pool + shall observe committed writer updates, so paginated history stays fresh + across readers and writers. + +### REQ-PLATFORM-BOUNDED-SESSION-HISTORY-002: Digest-backed operational payload storage + +**Intent:** Large operational payloads shall be stored once, verifiably, and +retainably without enlarging every transcript row. + +#### Acceptance criteria + +- **AC-PLATFORM-BOUNDED-SESSION-HISTORY-002.1:** A tool payload exceeding the + configured inline threshold shall be compressed or externalized with a + recorded digest, encoding, byte size, and internal reference; payload + metadata keyed by payload identity shall be deduplicated between sessions. +- **AC-PLATFORM-BOUNDED-SESSION-HISTORY-002.2:** Equivalent Git snapshots + within the same session and content-digest group shall deduplicate; + snapshots from different sessions remain distinct history. +- **AC-PLATFORM-BOUNDED-SESSION-HISTORY-002.3:** Retention selection shall + identify superseded tool payloads, redundant live and status Git snapshots, + and obsolete plan revisions only as reportable candidates; selection respects + configured retention and is non-destructive until maintenance executes, and + human and agent conversational messages shall remain preserved by default. + +### REQ-PLATFORM-DATABASE-MAINTENANCE-001: Backup-gated database maintenance command + +**Intent:** The native maintenance database command shall report, execute, +and roll back bounded-storage cleanup only behind a verified backup, never +silently deleting conversation history. + +#### Acceptance criteria + +- **AC-PLATFORM-DATABASE-MAINTENANCE-001.1:** The command shall support a dry + run and explicit retention settings, and report per-table candidate and + reclaim estimates without contents or credentials. +- **AC-PLATFORM-DATABASE-MAINTENANCE-001.2:** Destructive execution shall fail + closed when a verified backup is absent; the verified backup may be an + operator-supplied path or a freshly-created snapshot. +- **AC-PLATFORM-DATABASE-MAINTENANCE-001.3:** Execution shall remove only + selected redundant payloads, snapshots, and plan revisions inside + transactions; compaction shall stage and validate its result (`VACUUM INTO` + or equivalent) and atomically replace only after success. +- **AC-PLATFORM-DATABASE-MAINTENANCE-001.4:** The command shall report + SQLite/WAL and table-storage measurements and rollback steps without + database contents, credentials, or destructive default behavior. +- **AC-PLATFORM-DATABASE-MAINTENANCE-001.5:** Upgrade shall not silently delete + task history; public operations documentation shall describe backup of + `kandev.db` and `master.key`, image upgrade, migration and maintenance dry + run, execute and compact, rollback, and post-release health checks. + +### REQ-PLATFORM-PROVIDER-BACKOFF-001: Generation-aware provider failure backoff + +**Intent:** Authentication and configuration failures on background +integration loops shall enter exponential backoff with fingerprint-aware +reset, so a credential or configuration change resumes probing on the very +next cycle. + +#### Acceptance criteria + +- **AC-PLATFORM-PROVIDER-BACKOFF-001.1:** A background loop observing an + authentication or configuration failure shall open a per-target circuit + keyed by the owning workspace or configuration and skip further provider + calls for that target until its backoff expires. +- **AC-PLATFORM-PROVIDER-BACKOFF-001.2:** A non-secret connection fingerprint + (status and credential generation) shall reset an open circuit when it + changes, forcing an immediate probe after a credential rotate, reconnect, or + re-auth; an empty or unknown fingerprint shall never reset an open circuit. +- **AC-PLATFORM-PROVIDER-BACKOFF-001.3:** While a circuit is open, the loop + shall make zero additional provider calls for that target; after a reset or + expiry, one probe resumes evaluation. +- **AC-PLATFORM-PROVIDER-BACKOFF-001.4:** Failures shall be classified as auth, + config, or transient; a client capability fallback (for example, no GraphQL + support) is deliberately not a failure and shall not open any circuit. +- **AC-PLATFORM-PROVIDER-BACKOFF-001.5:** Health and status output shall + distinguish healthy, degraded, and disabled integration states, aggregate + open circuits by failure class without per-workspace identifiers or secret + material, and expose circuit skip, reset, and failure counters as + bounded-label metrics (provider and class only). + +## Out of scope + +- Agent identity and runtime profiles, owned by the agent system. +- Executor runtime environments, owned by the executor system. +- Frontend settings presentation, owned by the UI system. diff --git a/docs/specs/platform/system-design/pr-watch-and-bounded-storage.md b/docs/specs/platform/system-design/pr-watch-and-bounded-storage.md new file mode 100644 index 0000000000..053f3f4192 --- /dev/null +++ b/docs/specs/platform/system-design/pr-watch-and-bounded-storage.md @@ -0,0 +1,134 @@ +--- +status: current +system: platform +requirements: + - REQ-PLATFORM-PR-WATCH-IDENTITY-001 + - REQ-PLATFORM-PR-WATCH-IDENTITY-002 + - REQ-PLATFORM-TASK-SUMMARY-CONTENTION-001 + - REQ-PLATFORM-BOUNDED-SESSION-HISTORY-001 + - REQ-PLATFORM-BOUNDED-SESSION-HISTORY-002 + - REQ-PLATFORM-DATABASE-MAINTENANCE-001 + - REQ-PLATFORM-PROVIDER-BACKOFF-001 +--- + +# Canonical PR watch and bounded storage System Design + +## Purpose and boundaries + +Platform owns the cross-cutting contracts of this change: the task-status +summary projector, install-wide storage and maintenance guarantees, and +background integration loop health. The GitHub provider implementation +(`apps/backend/internal/github`) and the workflow-sync loop +(`apps/backend/internal/workflowsync`) realize the watch and backoff +contracts against their providers; conversation persistence +(`apps/backend/internal/task`) realizes the bounded-history contracts. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| `REQ-PLATFORM-PR-WATCH-IDENTITY-001` | Watch data model and migration; Reconciliation | +| `REQ-PLATFORM-PR-WATCH-IDENTITY-002` | Event publication; Sustained load | +| `REQ-PLATFORM-TASK-SUMMARY-CONTENTION-001` | Projection coalescing and backoff | +| `REQ-PLATFORM-BOUNDED-SESSION-HISTORY-001` | History hydration | +| `REQ-PLATFORM-BOUNDED-SESSION-HISTORY-002` | Payload storage and retention candidates | +| `REQ-PLATFORM-DATABASE-MAINTENANCE-001` | Maintenance command | +| `REQ-PLATFORM-PROVIDER-BACKOFF-001` | Provider failure backoff and health | + +## Watch data model and migration + +Searching rows are unique on `(task_id, repository_id, branch)` where +`pr_number = 0`; discovered rows are unique on `(task_id, repository_id, +pr_number)` where `pr_number != 0`. `session_id` is optional provenance +stored as `TEXT NOT NULL DEFAULT ''` (empty string when unknown), never a +unique-key component. The +transactional upgrade migration runs after the existing version-change +snapshot boundary, prefers discovered rows while merging the newest status, +check, review, and comment watermarks, removes orphaned rows, and is +idempotent. + +## Reconciliation + +Per cycle, resolve each task's per-repository branch (checkout branch first, +worktree fallback within the same repository), collapse equivalent sessions +to one target, and transition searching watches atomically. A transition that +collides with an existing canonical sibling merges into the sibling instead of +violating the unique constraint, so concurrent transitions of equivalent +sessions coalesce to one destination. + +## Event publication + +`GitHubPRFeedback` and `GitHubTaskPRUpdated` publish only when the observed +state differs from the watch watermarks (checks or review state change, or PR +update time advancing). A sync of unchanged state performs no watch writes and +publishes nothing on both single-watch REST and batched GraphQL paths. + +## Sustained load + +`apps/backend/internal/github/load_validation_test.go` seeds a task resumed +across fifty historical sessions plus concurrent tasks and drives simulated +poll cycles, asserting a constant canonical watch set, one batched branch +lookup per cycle, and bounded cycle latency. + +## Projection coalescing and backoff + +`apps/backend/internal/task/statussummary/projector.go` keeps per-task +in-process serialization, single-flight pending-refresh coalescing, and the +semantic-equal short-circuit. Rejected compare-and-set attempts are paced by a +small capped exponential backoff with up to +25% jitter +(`defaultCASRetryBackoff`, `casRetryJitter`, injectable through +`ProjectorConfig.RetryBackoff`) and rebase from authoritative state +(`rebaseProjectionStateFromCurrent`) before retrying. The sustained-contention +fixture applies a per-task alternating-reject store across many tasks and +asserts zero exhausted-retry handler errors. + +## History hydration + +`task_session_messages` keeps a lightweight metadata projection per message. +List endpoints read by cursor (keyset pagination), and tool payload detail is +fetched only through the explicit payload endpoint. No legacy full-row +hydration path remains. Separate reader and writer pools observe committed +writes. + +## Payload storage and retention candidates + +A payload above the inline threshold is stored in +`task_message_payloads` keyed by digest with compression encoding, byte +size, and an internal reference; message rows retain the lightweight +projection. Explicit detail loading verifies the digest before returning +content. Equivalent Git snapshots within one session and digest group share +one stored snapshot. Candidate queries for orphaned payloads, redundant +snapshots, and obsolete plan revisions expose counts and references without +deleting; retention windows protect HEAD revisions, revert ancestry, recency +defaults, and query limits. + +## Maintenance command + +`kandev maintenance database` (dispatched by +`apps/backend/internal/launcher/maintenance_database.go`) supports dry run, +explicit retention options, verified backup, and compaction. +`apps/backend/internal/maintenance` verifies the backup before destructive +work (failing closed otherwise), removes only retention-eligible candidates +in transactions, stages compaction with `VACUUM INTO`, validates, and +atomically replaces only after success. Rollback steps are printed; storage +gauges report database, WAL, and per-table logical sizes without contents or +credentials. + +## Provider failure backoff and health + +`apps/backend/internal/common/authcircuit` provides the shared `FailureClass` +(auth, config, transient), `Backoff`, and `State`. The GitHub PR-watch poller +(`poller_circuit.go`) holds a poller-scoped, mutex-guarded per-workspace +circuit consulted by `filterOpenCircuitWatches` before each cycle, refreshing +each present workspace's connection fingerprint once per cycle +(`Service.WorkspaceConnectionFingerprint`) and recording outcomes at all +three poll-path call sites through `classifyPollErr`. The workflow-sync loop +persists circuit state and fingerprint per config and skips due-but-open +configs. `apps/backend/internal/health` exposes aggregate health (counts by +class, no workspace identifiers or secrets), and bounded-label expvar counters +report skips, resets, and failures. + +## Related decisions + +- [Keep PR watches task owned](../../../decisions/2026-08-31-task-owned-pr-watch-identity.md) +- [Install-wide storage maintenance uses typed ownership providers and quarantine](../../../decisions/0045-install-wide-storage-maintenance.md) diff --git a/docs/specs/tasks/README.md b/docs/specs/tasks/README.md index 96460010a1..9294c98f41 100644 --- a/docs/specs/tasks/README.md +++ b/docs/specs/tasks/README.md @@ -38,6 +38,7 @@ signals, and task-scoped scheduling contracts. - [Initial task brief](requirements/initial-task-brief.md) +- [Bounded Session History Storage](requirements/bounded-session-history.md) - [Task completion, conversation follow-ups, and workspace access](requirements/task-completion.md) - [Agent-Generated Task Titles](requirements/agent-generated-titles.md) @@ -149,6 +150,7 @@ signals, and task-scoped scheduling contracts. ### System design +- [Bounded Session History Storage](system-design/bounded-session-history.md) - [Initial task brief](system-design/initial-task-brief.md) - [Task completion, conversation follow-ups, and workspace access](system-design/task-completion.md)