diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 2a93f72d32..e01b401f7b 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -132,6 +132,7 @@ import ( // Database "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/delivery" "github.com/kandev/kandev/internal/common/ports" ) @@ -481,9 +482,10 @@ func startAgentInfrastructure( cancelContext context.CancelFunc, ) bool { restoreCleanups := make([]func() error, 0) - addRuntimeCleanup := func(fn func() error) { + var databaseQuiesce func() error + addRuntimeCleanup := func(fn func() error) func() error { if fn == nil { - return + return nil } var stopOnce sync.Once var stopErr error @@ -493,6 +495,7 @@ func startAgentInfrastructure( } addCleanup(stop) restoreCleanups = append(restoreCleanups, stop) + return stop } userSecretStore := secrets.NewUserVisibleStore(repos.Secrets) mcpScopeResolver := mcpscope.NewResolver( @@ -752,11 +755,27 @@ func startAgentInfrastructure( // Start the plugin system's event delivery and health monitor // background loops. if services.Plugins != nil { - startPluginsSubsystems(ctx, services.Plugins, lifecycleMgr, eventBus, log, addRuntimeCleanup) + startPluginsSubsystems(ctx, services.Plugins, lifecycleMgr, eventBus, log, + func(fn func() error) { addRuntimeCleanup(fn) }) + } + + // Start the task delivery ledger sweep. Must run after task/repository + // tables exist (already true here, provided in provideRepositories) — + // the ledger's foreign keys require them present at CREATE TABLE time + // on PostgreSQL. services.Task satisfies delivery.CheckoutResolver. + if _, deliveryCleanup, err := delivery.Provide(dbPool.Writer(), dbPool.Reader(), services.Task, log); err != nil { + log.Warn("delivery ledger sweep unavailable", zap.Error(err)) + } else { + // Must be addRuntimeCleanup, not addCleanup: RestoreQuiesce only + // stops workers registered here, and a restore checkpoints, closes, + // and replaces the shared database pool. A five-minute sweep pass + // overlapping that would race the pool swap. + databaseQuiesce = addRuntimeCleanup(deliveryCleanup) } return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services, - agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath, addRuntimeCleanup, runCleanups, cancelContext, restoreCleanups) + agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath, + func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelContext, restoreCleanups, databaseQuiesce) } // startOrchestratorAndAutomationConsumers establishes the startup chain in @@ -823,6 +842,7 @@ func startGatewayAndServe( runCleanups func(), cancelContext context.CancelFunc, restoreCleanups []func() error, + databaseQuiesce func() error, ) bool { // ============================================ // WEBSOCKET GATEWAY @@ -1032,6 +1052,7 @@ func startGatewayAndServe( BuildTime: BuildTime, }, systemsvc.Wiring{ OrchestratorShutdown: func() { _ = orchestratorSvc.Stop() }, + DatabaseQuiesce: databaseQuiesce, RestoreQuiesce: restoreQuiesce, MessageQueue: orchestratorSvc.GetMessageQueue(), MessageQueueConfig: queueConfiguration(cfg), diff --git a/apps/backend/internal/backendapp/orchestrator.go b/apps/backend/internal/backendapp/orchestrator.go index 8433029f2b..9f5a6ccdee 100644 --- a/apps/backend/internal/backendapp/orchestrator.go +++ b/apps/backend/internal/backendapp/orchestrator.go @@ -20,6 +20,7 @@ import ( "github.com/kandev/kandev/internal/common/gitref" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/delivery" "github.com/kandev/kandev/internal/events/bus" "github.com/kandev/kandev/internal/gitcredentials" githubpkg "github.com/kandev/kandev/internal/github" @@ -1084,10 +1085,26 @@ func (a *repositoryResolverAdapter) persistDetectedDefaultBranch( if strings.TrimSpace(repo.DefaultBranch) == detected { return detected } + // detected is still the correct base branch for the review in progress + // even when the write below fails, so it is always returned. But a + // failure here is not a transient blip to shrug off: it leaves + // repositories.default_branch empty, which (per spec "Degraded + // evaluation") permanently degrades every future delivery-ledger + // evaluation of this repository to default_branch_unknown until some + // later write succeeds — and this call site retries with the same + // detected value on every future invocation, so a rejection driven by + // validation (as opposed to a transient DB error) will repeat forever. + // Review round 3, finding #4: this used to log at Warn and nothing + // else, making that permanent degradation invisible. Error level plus + // the dedicated counter make it observable the same way ancestry/write + // failures already are in internal/delivery/metrics.go. if _, err := a.taskSvc.UpdateRepository(ctx, repo.ID, &taskservice.UpdateRepositoryRequest{ DefaultBranch: &detected, }); err != nil { - a.logger.Warn("failed to persist detected default branch", + delivery.RecordDefaultBranchPersistError() + a.logger.Error("failed to persist detected default branch: repository row keeps its prior "+ + "default_branch and will read as default_branch_unknown in the delivery ledger until a "+ + "future write succeeds", zap.String("repository_id", repo.ID), zap.String(branchFieldKey, detected), zap.Error(err)) diff --git a/apps/backend/internal/backendapp/orchestrator_default_branch_persist_test.go b/apps/backend/internal/backendapp/orchestrator_default_branch_persist_test.go new file mode 100644 index 0000000000..f8cc1493f6 --- /dev/null +++ b/apps/backend/internal/backendapp/orchestrator_default_branch_persist_test.go @@ -0,0 +1,116 @@ +package backendapp + +import ( + "context" + "expvar" + "os" + "path/filepath" + "testing" + + taskservice "github.com/kandev/kandev/internal/task/service" +) + +// readDeliveryExpvarInt reads a package-level expvar.Int published by +// internal/delivery/metrics.go, mirroring internal/delivery's own +// readExpvarInt test helper (sweep_test.go). The value is process-global +// and never resets between tests, so callers must compare a before/after +// delta rather than an absolute value. +func readDeliveryExpvarInt(t *testing.T, name string) int64 { + t.Helper() + v := expvar.Get(name) + if v == nil { + t.Fatalf("expvar %q not published", name) + } + iv, ok := v.(*expvar.Int) + if !ok { + t.Fatalf("expvar %q is not an *expvar.Int", name) + } + return iv.Value() +} + +// TestResolveReviewBaseBranchRecordsPersistFailureForDetectedBranch covers +// Review round 3, finding #4: persistDetectedDefaultBranch used to log a +// bare Warn and otherwise swallow a failure to persist a newly detected +// default branch, leaving repositories.default_branch permanently empty +// (and the repository's delivery-ledger classification permanently +// default_branch_unknown, per spec "Degraded evaluation") with no way for +// an operator to notice. "_weird" is a branch name real git accepts (it +// only forbids a leading "-", not "_") but IsValidBranchName's allowlist +// regex rejects (it requires a leading alnum), so UpdateRepository rejects +// the persist attempt with ErrInvalidRepositorySettings — exactly the "git +// allows it, our validator doesn't" gap the finding describes, as opposed +// to a transient DB error. +func TestResolveReviewBaseBranchRecordsPersistFailureForDetectedBranch(t *testing.T) { + harness := newBootStateTestHarness(t) + ctx := context.Background() + workspace, err := harness.taskSvc.CreateWorkspace(ctx, &taskservice.CreateWorkspaceRequest{ + Name: "Workspace", + }) + if err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + repoPath := t.TempDir() + gitDir := filepath.Join(repoPath, ".git") + if err := os.MkdirAll(filepath.Join(gitDir, "refs", "remotes", "origin"), 0o755); err != nil { + t.Fatalf("mkdir origin refs: %v", err) + } + if err := os.WriteFile(filepath.Join(gitDir, "HEAD"), []byte("ref: refs/heads/feature/x\n"), 0o644); err != nil { + t.Fatalf("write HEAD: %v", err) + } + if err := os.WriteFile( + filepath.Join(gitDir, "refs", "remotes", "origin", "HEAD"), + []byte("ref: refs/remotes/origin/_weird\n"), + 0o644, + ); err != nil { + t.Fatalf("write origin HEAD: %v", err) + } + if err := os.WriteFile( + filepath.Join(gitDir, "refs", "remotes", "origin", "_weird"), + []byte("0000000\n"), + 0o644, + ); err != nil { + t.Fatalf("write _weird ref: %v", err) + } + repo, err := harness.taskSvc.CreateRepository(ctx, &taskservice.CreateRepositoryRequest{ + WorkspaceID: workspace.ID, + Name: "owner/repo", + SourceType: "provider", + LocalPath: repoPath, + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + }) + if err != nil { + t.Fatalf("CreateRepository: %v", err) + } + if repo.DefaultBranch != "" { + t.Fatalf("precondition failed: DefaultBranch = %q, want empty", repo.DefaultBranch) + } + + before := readDeliveryExpvarInt(t, "delivery_ledger_default_branch_persist_errors_total") + + adapter := &repositoryResolverAdapter{ + taskSvc: harness.taskSvc, + logger: newTestLogger(), + } + got := adapter.resolveReviewBaseBranch(ctx, repo, repoPath, "") + if got != "_weird" { + t.Fatalf("resolveReviewBaseBranch = %q, want %q (the detected value must still be used for this operation "+ + "even though persistence is rejected)", got, "_weird") + } + + after := readDeliveryExpvarInt(t, "delivery_ledger_default_branch_persist_errors_total") + if after != before+1 { + t.Fatalf("delivery_ledger_default_branch_persist_errors_total = %d, want %d "+ + "(a rejected persist must be recorded, not silently swallowed)", after, before+1) + } + + stored, err := harness.taskSvc.GetRepository(ctx, repo.ID) + if err != nil { + t.Fatalf("GetRepository: %v", err) + } + if stored.DefaultBranch != "" { + t.Fatalf("stored default_branch = %q, want empty (persistence must actually have been rejected)", + stored.DefaultBranch) + } +} diff --git a/apps/backend/internal/common/securityutil/git.go b/apps/backend/internal/common/securityutil/git.go index fa44a0d55c..ea930b50e4 100644 --- a/apps/backend/internal/common/securityutil/git.go +++ b/apps/backend/internal/common/securityutil/git.go @@ -51,6 +51,41 @@ func IsValidBaseBranchRef(ref string) bool { return IsValidBranchName(rest) } +// gitSymbolicRefs are git's well-known symbolic refs: pure alnum/underscore +// with no leading dash and no "~"/"^", so validBranchNameRegex (and +// therefore IsValidBaseBranchRef) admits them even though none is ever a +// real branch name. A caller that persists one of these as a repository's +// default branch and later feeds it to a local rev-parse fallback (worktree +// FallbackBaseBranch, or internal/delivery's ancestry check) gets whatever +// commit that pseudo-ref currently resolves to in that checkout — silently +// the wrong commit, not a resolution failure. +var gitSymbolicRefs = map[string]struct{}{ + "HEAD": {}, + "ORIG_HEAD": {}, + "FETCH_HEAD": {}, + "MERGE_HEAD": {}, +} + +// IsGitSymbolicRef reports whether ref is one of git's reserved pseudo-refs +// above. +func IsGitSymbolicRef(ref string) bool { + _, ok := gitSymbolicRefs[ref] + return ok +} + +// IsValidDefaultBranchName validates a value bound for +// repositories.default_branch: the IsValidBaseBranchRef allowlist (branch +// syntax, no trailing/double slash) plus a reject on git's symbolic +// pseudo-refs, which that allowlist alone admits. Every ingestion site for +// this field (repository create, update, and the FindOrCreateRepository +// backfill) must use this rather than IsValidBranchName or +// IsValidBaseBranchRef directly, since a default branch — unlike an +// ordinary base-branch override — is read back through a local-fallback +// rev-parse with no upstream ref to disambiguate it from a pseudo-ref. +func IsValidDefaultBranchName(branch string) bool { + return IsValidBaseBranchRef(branch) && !IsGitSymbolicRef(branch) +} + // IsKnownSafeGitFlag returns true if the argument is a known safe git flag used by this codebase. // This prevents argument injection where user input could introduce malicious flags. // Only flags actually used by the Kandev codebase are whitelisted. diff --git a/apps/backend/internal/common/securityutil/git_test.go b/apps/backend/internal/common/securityutil/git_test.go index 284300ef7c..177cc8d35d 100644 --- a/apps/backend/internal/common/securityutil/git_test.go +++ b/apps/backend/internal/common/securityutil/git_test.go @@ -39,6 +39,40 @@ func TestIsValidBaseBranchRef(t *testing.T) { } } +func TestIsValidDefaultBranchName(t *testing.T) { + cases := map[string]bool{ + "main": true, + "release/v2": true, + "feature/foo-bar": true, + "": false, + "main/": false, // trailing slash, via IsValidBaseBranchRef + "a//b": false, // consecutive slashes, via IsValidBaseBranchRef + "HEAD": false, // git symbolic ref + "ORIG_HEAD": false, // git symbolic ref + "FETCH_HEAD": false, // git symbolic ref + "MERGE_HEAD": false, // git symbolic ref + "head": true, // lowercase is a real, distinct ref name + } + for branch, want := range cases { + if got := IsValidDefaultBranchName(branch); got != want { + t.Errorf("IsValidDefaultBranchName(%q) = %v, want %v", branch, got, want) + } + } +} + +func TestIsGitSymbolicRef(t *testing.T) { + for _, ref := range []string{"HEAD", "ORIG_HEAD", "FETCH_HEAD", "MERGE_HEAD"} { + if !IsGitSymbolicRef(ref) { + t.Errorf("IsGitSymbolicRef(%q) = false, want true", ref) + } + } + for _, ref := range []string{"main", "head", "MERGE_HEAD_2"} { + if IsGitSymbolicRef(ref) { + t.Errorf("IsGitSymbolicRef(%q) = true, want false", ref) + } + } +} + func TestIsKnownSafeGitFlagAllowsRequiredGitOperationFlags(t *testing.T) { for _, flag := range []string{"--dry-run", "--first-parent", "--is-ancestor"} { if !IsKnownSafeGitFlag(flag) { diff --git a/apps/backend/internal/delivery/ancestry.go b/apps/backend/internal/delivery/ancestry.go new file mode 100644 index 0000000000..12d2680458 --- /dev/null +++ b/apps/backend/internal/delivery/ancestry.go @@ -0,0 +1,196 @@ +package delivery + +import ( + "context" + "errors" + "os/exec" + "regexp" + "time" + + "github.com/kandev/kandev/internal/common/securityutil" + "github.com/kandev/kandev/internal/common/subproc" +) + +// commitSHAPattern matches a plausible git object name: hex digits only, +// long enough to rule out a bare option-like fragment and short enough to +// cover both SHA-1 (40 hex chars) and a future SHA-256 repository (64). +// commit comes from task_session_git_snapshots.head_commit, which is not +// format-validated anywhere upstream of this seam. +var commitSHAPattern = regexp.MustCompile(`^[0-9a-fA-F]{4,64}$`) + +// looksLikeCommitSHA rejects a commit value before it ever reaches a git +// subprocess as raw argv. apps/backend/internal/common/subproc/git_command.go +// documents that callers are responsible for validating user-controlled +// refs before they reach that seam — a value beginning with "-" could +// otherwise be parsed as a git option rather than a revision (Review +// round 1, finding #7). +func looksLikeCommitSHA(commit string) bool { + return commitSHAPattern.MatchString(commit) +} + +// CheckoutResolver is the narrow, read-only, single-method port the +// ancestry check depends on for the repository's local checkout path +// (spec "Which checkout"). It is satisfied by +// task/service.Service.ResolveRepositoryLocalPath, wired in at +// construction so this package never depends on the concrete task +// service type or duplicates its canonicalization/containment checks. +type CheckoutResolver interface { + ResolveRepositoryLocalPath(ctx context.Context, repositoryID string) (string, error) +} + +// AncestryCallTimeout is the fixed per-call timeout (spec "Boundary +// values"). +const AncestryCallTimeout = 10 * time.Second + +// AncestryChecker runs the ancestry check through the class-aware git +// subprocess admission pool at subproc.GitBackground — the sweep is +// background work and must not contend with user-facing git (spec +// "Concurrency", "Ancestry admission class"). +type AncestryChecker struct { + Checkout CheckoutResolver +} + +// Check resolves the repository's checkout, resolves the default branch +// ref (remote-tracking first, falling back to the local branch), and runs +// `git merge-base --is-ancestor commit ref`. It never returns a negative +// result as an error under Errored; Errored is set only when the check +// could not be performed at all — no readable local checkout, neither +// ref resolves, or the git call itself fails or times out (spec "Failure +// modes"). +func (a *AncestryChecker) Check(ctx context.Context, repositoryID, defaultBranch, commit string) AncestryOutcome { + out := AncestryOutcome{Attempted: true, Commit: commit} + if !looksLikeCommitSHA(commit) { + out.Errored = true + return out + } + // repositories.default_branch is attacker-reachable through ordinary + // repository create/update requests and is not format-validated + // upstream of this seam either (Review round 2, finding #1). A + // flag-like value risks option-injection against refExists's rev-parse + // call, which cannot take a "--" separator (it would turn the argument + // into a pathspec). More dangerously, a value like "main~1" is not + // flag-like at all but is valid git revision syntax: appended to a ref + // that does exist (e.g. "refs/remotes/origin/main~1"), it resolves + // successfully to the wrong commit with no error, rather than failing + // as "does not exist" — defense-in-depth alongside the ingestion-time + // validation in task/service.applyRepositoryUpdates / + // CreateRepository, mirroring looksLikeCommitSHA's treatment of + // commit above. IsValidDefaultBranchName's allowlist admits bare commit + // SHAs (both pure alnum/underscore, no leading dash, no "~"/"^"), which + // is not a real branch name and resolves via refExists's local-fallback + // rev-parse to a commit that is not what the caller's default_branch + // value names — the same silent wrong-commit failure mode this guard + // exists to close (Review round 3, finding #2). Git's well-known + // symbolic refs (HEAD and friends) are rejected by + // IsValidDefaultBranchName itself. + if !securityutil.IsValidDefaultBranchName(defaultBranch) || looksLikeCommitSHA(defaultBranch) { + out.Errored = true + return out + } + if a.Checkout == nil { + out.Errored = true + return out + } + path, err := a.Checkout.ResolveRepositoryLocalPath(ctx, repositoryID) + if err != nil { + out.Errored = true + return out + } + ref, err := resolveDefaultBranchRef(ctx, path, defaultBranch) + if err != nil { + out.Errored = true + return out + } + positive, err := isAncestor(ctx, path, commit, ref) + if err != nil { + out.Errored = true + return out + } + out.Positive = positive + return out +} + +// resolveDefaultBranchRef implements spec "Which default branch": +// refs/remotes/origin/, falling back to the local branch +// when the remote-tracking ref genuinely does not exist. +// A failure to CHECK the remote ref (as opposed to a confirmed absence) +// must never fall through to the local branch — spec "Failure modes" +// requires an inability to check to surface as an error, and silently +// preferring a possibly-stale local branch over an unchecked remote- +// tracking ref could produce a wrong-direction answer on the write-once +// ancestry observation (Review round 1, finding #4). +func resolveDefaultBranchRef(ctx context.Context, checkoutPath, defaultBranch string) (string, error) { + remote := "refs/remotes/origin/" + defaultBranch + remoteExists, err := refExists(ctx, checkoutPath, remote) + if err != nil { + return "", err + } + if remoteExists { + return remote, nil + } + localExists, err := refExists(ctx, checkoutPath, defaultBranch) + if err != nil { + return "", err + } + if localExists { + return defaultBranch, nil + } + return "", errors.New("neither the remote-tracking ref nor the local default branch resolves") +} + +// refExists runs `git rev-parse --verify --quiet ^{commit}` and +// interprets its exit code the same way isAncestor does: 0 = exists, +// 1 = genuinely absent (a routine negative, not an error), anything else +// (acquisition failure, timeout, or another git error) is a failure to +// check and is returned as an error rather than folded into `false`. +func refExists(ctx context.Context, checkoutPath, ref string) (bool, error) { + runErr, acquireErr := subproc.RunGitAfterAcquire(ctx, subproc.GitBackground, AncestryCallTimeout, + func(execCtx context.Context) *exec.Cmd { + cmd := subproc.NewGitCommand(execCtx, "rev-parse", "--verify", "--quiet", ref+"^{commit}") + cmd.Dir = checkoutPath + return cmd + }) + if acquireErr != nil { + return false, acquireErr + } + if runErr == nil { + return true, nil + } + var exitErr *exec.ExitError + if errors.As(runErr, &exitErr) && exitErr.ExitCode() == 1 { + return false, nil + } + return false, runErr +} + +// isAncestor runs `git merge-base --is-ancestor -- commit ref` and +// interprets its exit code: 0 = true, 1 = false (a routine negative +// result, not an error — see spec "Evidence", squash-merge finding 2), +// anything else is an ancestry error. The `--` separator (unlike +// refExists's rev-parse call, which must NOT take one: it changes +// rev-parse's ^{commit} argument from a revision expression to a +// pathspec and breaks resolution entirely) forces both positional +// arguments to be parsed as revisions, never as options, regardless of +// content — commit is already checked against commitSHAPattern by the +// time it reaches here, but this is the same caller-validation contract +// apps/backend/internal/common/subproc/git_command.go documents, applied +// at the seam that actually shells out (Review round 1, finding #7). +func isAncestor(ctx context.Context, checkoutPath, commit, ref string) (bool, error) { + runErr, acquireErr := subproc.RunGitAfterAcquire(ctx, subproc.GitBackground, AncestryCallTimeout, + func(execCtx context.Context) *exec.Cmd { + cmd := subproc.NewGitCommand(execCtx, "merge-base", "--is-ancestor", "--", commit, ref) + cmd.Dir = checkoutPath + return cmd + }) + if acquireErr != nil { + return false, acquireErr + } + if runErr == nil { + return true, nil + } + var exitErr *exec.ExitError + if errors.As(runErr, &exitErr) && exitErr.ExitCode() == 1 { + return false, nil + } + return false, runErr +} diff --git a/apps/backend/internal/delivery/ancestry_internal_test.go b/apps/backend/internal/delivery/ancestry_internal_test.go new file mode 100644 index 0000000000..ab80fce892 --- /dev/null +++ b/apps/backend/internal/delivery/ancestry_internal_test.go @@ -0,0 +1,123 @@ +package delivery + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// runInternalGit is a package-internal twin of ancestry_test.go's runGit +// (package delivery_test), needed here because this file exercises +// refExists directly, which is unexported. +func runInternalGit(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } +} + +func newInternalAncestryRepo(t *testing.T) string { + t.Helper() + work := t.TempDir() + runInternalGit(t, work, "init", "-b", "main") + runInternalGit(t, work, "config", "user.email", "test@example.com") + runInternalGit(t, work, "config", "user.name", "test") + if err := os.WriteFile(filepath.Join(work, "README.md"), []byte("hello"), 0o644); err != nil { + t.Fatalf("write README: %v", err) + } + runInternalGit(t, work, "add", "README.md") + runInternalGit(t, work, "commit", "-m", "initial") + return work +} + +// TestRefExists_GenuineAbsenceIsNotAnError covers the negative case +// refExists must still handle exactly as before the fix: a ref that +// simply does not resolve (git rev-parse --verify --quiet exits 1) is a +// routine "does not exist" result, not a failure to check. +func TestRefExists_GenuineAbsenceIsNotAnError(t *testing.T) { + work := newInternalAncestryRepo(t) + + exists, err := refExists(context.Background(), work, "does-not-exist") + if err != nil { + t.Fatalf("err = %v, want nil (a genuinely absent ref is not an error)", err) + } + if exists { + t.Fatal("exists = true, want false") + } +} + +// TestRefExists_GenuinePresenceIsNotAnError is the positive-case +// companion, confirming the fix does not regress the ordinary success +// path. +func TestRefExists_GenuinePresenceIsNotAnError(t *testing.T) { + work := newInternalAncestryRepo(t) + + exists, err := refExists(context.Background(), work, "main") + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if !exists { + t.Fatal("exists = false, want true (main was just committed to)") + } +} + +// TestRefExists_FailureToCheckIsAnErrorNotAbsence is Review round 1, +// finding #4: refExists used to collapse "the ref genuinely does not +// exist" (git rev-parse --verify --quiet exits 1, a routine negative) and +// "the check itself could not be performed" (any other failure — here, a +// checkout path that is not a git repository at all, which exits 128) +// into the same bare `false`. resolveDefaultBranchRef then falls back +// from the remote-tracking ref to the local branch on either condition, +// so a transient failure checking the remote ref could make the check run +// against a stale local branch instead of erroring, per spec "Failure +// modes": "the git call fails or times out" must be an error, never a +// negative result. refExists must now distinguish the two: a failure to +// check is returned as a non-nil error, not folded into `false`. +func TestRefExists_FailureToCheckIsAnErrorNotAbsence(t *testing.T) { + notARepo := t.TempDir() // deliberately never `git init`-ed + + exists, err := refExists(context.Background(), notARepo, "main") + if err == nil { + t.Fatal("err = nil, want a non-nil error (the checkout path is not a git repository at all)") + } + if exists { + t.Fatal("exists = true, want false alongside the error") + } +} + +// TestLooksLikeCommitSHA covers Review round 1, finding #7: the commit +// value submitted to the ancestry check comes from +// task_session_git_snapshots.head_commit, which is not format-validated +// anywhere upstream, and apps/backend/internal/common/subproc/git_command.go +// documents that callers must validate a value before it reaches a git +// subprocess as raw argv — a value beginning with "-" could otherwise be +// parsed as a git option rather than a revision. This is the guard that +// makes Check reject such a value before it ever reaches git. +func TestLooksLikeCommitSHA(t *testing.T) { + cases := []struct { + name string + in string + want bool + }{ + {"full sha-1", "a1b2c3d4e5f60718293a4b5c6d7e8f9012345678", true}, + {"short sha", "a1b2c3d", true}, + {"uppercase hex", "A1B2C3D4", true}, + {"leading dash", "-1", false}, + {"long option", "--all", false}, + {"double dash alone", "--", false}, + {"empty", "", false}, + {"non-hex letters", "not-a-sha", false}, + {"too short", "a1b", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := looksLikeCommitSHA(tc.in); got != tc.want { + t.Fatalf("looksLikeCommitSHA(%q) = %v, want %v", tc.in, got, tc.want) + } + }) + } +} diff --git a/apps/backend/internal/delivery/ancestry_test.go b/apps/backend/internal/delivery/ancestry_test.go new file mode 100644 index 0000000000..ad4e0d9e4c --- /dev/null +++ b/apps/backend/internal/delivery/ancestry_test.go @@ -0,0 +1,313 @@ +package delivery_test + +import ( + "bytes" + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/kandev/kandev/internal/delivery" +) + +// fakeCheckoutResolver satisfies delivery.CheckoutResolver for tests +// without pulling in task/service. calls, when non-nil, records how many +// times ResolveRepositoryLocalPath was invoked, so a test can assert a +// rejection happened before the checkout was even resolved rather than +// merely that Check ended up Errored (which a later, unrelated failure could +// also produce). +type fakeCheckoutResolver struct { + path string + err error + calls *int +} + +func (f fakeCheckoutResolver) ResolveRepositoryLocalPath(_ context.Context, _ string) (string, error) { + if f.calls != nil { + *f.calls++ + } + return f.path, f.err +} + +// runGit runs a git command in dir for test repo setup, failing the test +// on error. This is test fixture setup, not the production seam under +// test (which goes through delivery.AncestryChecker). +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + var out bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out.String()) + } + return strings.TrimSpace(out.String()) +} + +// newAncestryTestRepo creates a bare "origin" plus a working checkout with +// a remote-tracking branch, so refs/remotes/origin/ +// resolves the way a real clone would. +func newAncestryTestRepo(t *testing.T) (checkoutPath, defaultBranch string) { + t.Helper() + origin := t.TempDir() + runGit(t, origin, "init", "--bare", "-b", "main") + + work := t.TempDir() + runGit(t, work, "init", "-b", "main") + runGit(t, work, "config", "user.email", "test@example.com") + runGit(t, work, "config", "user.name", "test") + runGit(t, work, "remote", "add", "origin", origin) + writeFile(t, work, "README.md", "hello") + runGit(t, work, "add", "README.md") + runGit(t, work, "commit", "-m", "initial") + runGit(t, work, "push", "origin", "main") + runGit(t, work, "fetch", "origin") + + return work, "main" +} + +func writeFile(t *testing.T, dir, name, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } +} + +func TestAncestryChecker_PositiveResult(t *testing.T) { + work, defaultBranch := newAncestryTestRepo(t) + head := runGit(t, work, "rev-parse", "HEAD") + + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + out := checker.Check(context.Background(), "repo-1", defaultBranch, head) + + if !out.Attempted || out.Errored { + t.Fatalf("out = %+v, want Attempted, not Errored", out) + } + if !out.Positive { + t.Fatal("HEAD of the default branch must be an ancestor of itself") + } +} + +func TestAncestryChecker_NegativeResultIsNotAnError(t *testing.T) { + work, defaultBranch := newAncestryTestRepo(t) + runGit(t, work, "checkout", "-b", "feature") + writeFile(t, work, "feature.txt", "wip") + runGit(t, work, "add", "feature.txt") + runGit(t, work, "commit", "-m", "feature work") + featureHead := runGit(t, work, "rev-parse", "HEAD") + + // Advance main independently so the feature commit is not an ancestor. + runGit(t, work, "checkout", defaultBranch) + writeFile(t, work, "main.txt", "unrelated") + runGit(t, work, "add", "main.txt") + runGit(t, work, "commit", "-m", "unrelated main commit") + runGit(t, work, "push", "origin", defaultBranch) + runGit(t, work, "fetch", "origin") + + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + out := checker.Check(context.Background(), "repo-1", defaultBranch, featureHead) + + if out.Errored { + t.Fatalf("a negative ancestry result must not be an error: %+v", out) + } + if out.Positive { + t.Fatal("an unmerged feature commit must not be an ancestor of main") + } +} + +func TestAncestryChecker_LocalBranchFallbackWhenRemoteTrackingRefMissing(t *testing.T) { + work, defaultBranch := newAncestryTestRepo(t) + // Simulate a repository with no remote-tracking ref at all (never + // fetched, or a purely local checkout). + runGit(t, work, "update-ref", "-d", "refs/remotes/origin/"+defaultBranch) + head := runGit(t, work, "rev-parse", "HEAD") + + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + out := checker.Check(context.Background(), "repo-1", defaultBranch, head) + + if out.Errored { + t.Fatalf("expected the local branch fallback to succeed, got errored: %+v", out) + } + if !out.Positive { + t.Fatal("HEAD must be an ancestor of its own local default branch") + } +} + +func TestAncestryChecker_NeitherRefResolvesIsAnError(t *testing.T) { + work, _ := newAncestryTestRepo(t) + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + out := checker.Check(context.Background(), "repo-1", "does-not-exist", "HEAD") + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true", out) + } +} + +// TestAncestryChecker_FlagLikeCommitIsRejectedBeforeReachingGit covers +// Review round 1, finding #7: task_session_git_snapshots.head_commit is +// not format-validated anywhere upstream of this seam, and +// apps/backend/internal/common/subproc/git_command.go documents that +// callers are responsible for validating a value before it reaches a git +// subprocess as raw argv. A commit value beginning with "-" previously +// went straight to `git merge-base --is-ancestor ` with no +// `--` separator, so it could be parsed as a git option rather than a +// revision. Check must now reject a non-SHA-shaped commit before it ever +// reaches a git subprocess, rather than relying on git's own error +// handling for the malformed value. +func TestAncestryChecker_FlagLikeCommitIsRejectedBeforeReachingGit(t *testing.T) { + work, defaultBranch := newAncestryTestRepo(t) + + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + out := checker.Check(context.Background(), "repo-1", defaultBranch, "--all") + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true (a flag-like commit must never reach the git subprocess)", out) + } +} + +// TestAncestryChecker_FlagLikeDefaultBranchIsRejectedBeforeReachingGit +// covers Review round 2, finding #1 (SEC-005): repositories.default_branch +// is attacker-reachable through ordinary repository create/update requests +// and was never validated before reaching refExists as raw argv. Round 1's +// finding #7 guarded the commit argument with looksLikeCommitSHA but left +// defaultBranch unguarded — a flag-like value here risks option-injection +// against `git rev-parse --verify --quiet ^{commit}`, which cannot +// take a `--` separator (it would turn the argument into a pathspec), so +// the fix must be validation before the call, mirroring looksLikeCommitSHA's +// treatment of commit. +// +// Review round 3, finding #3: this test originally only asserted +// Errored=true, which is phantom-green — "-x" already produces Errored=true +// via the pre-existing "neither ref resolves" path (refExists's rev-parse +// calls both exit non-zero for an unrecognized flag), identically on +// pre-fix code, so the assertion could not tell "rejected before reaching +// git" apart from "reached git and git errored anyway". Asserting the +// checkout was never even resolved makes the test actually discriminate: +// pre-fix, Check calls a.Checkout.ResolveRepositoryLocalPath unconditionally +// before ever touching defaultBranch. +func TestAncestryChecker_FlagLikeDefaultBranchIsRejectedBeforeReachingGit(t *testing.T) { + work, _ := newAncestryTestRepo(t) + head := runGit(t, work, "rev-parse", "HEAD") + + calls := 0 + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work, calls: &calls}} + out := checker.Check(context.Background(), "repo-1", "-x", head) + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true (a flag-like default branch must never reach the git subprocess)", out) + } + if calls != 0 { + t.Fatalf("ResolveRepositoryLocalPath was called %d time(s), want 0 "+ + "(the guard must reject defaultBranch before resolving the checkout at all)", calls) + } +} + +// TestAncestryChecker_CommitSHALikeDefaultBranchIsRejectedBeforeReachingGit +// covers Review round 3, finding #2: IsValidBranchName's allowlist regex +// admits a bare commit SHA (pure hex, no leading dash, no "~"/"^"), which is +// not a real branch name but resolves via refExists's local-fallback +// rev-parse to whatever commit that SHA names — the silent wrong-commit +// failure mode the guard exists to close, not merely a rejection based on +// "no such ref". +func TestAncestryChecker_CommitSHALikeDefaultBranchIsRejectedBeforeReachingGit(t *testing.T) { + work, _ := newAncestryTestRepo(t) + head := runGit(t, work, "rev-parse", "HEAD") + + calls := 0 + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work, calls: &calls}} + // head is a real, resolvable commit SHA in this checkout — if it were + // treated as a branch name instead of rejected, resolveDefaultBranchRef + // would resolve it successfully rather than failing "does not exist". + out := checker.Check(context.Background(), "repo-1", head, head) + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true (a commit-SHA-shaped default branch must never reach the git subprocess)", out) + } + if calls != 0 { + t.Fatalf("ResolveRepositoryLocalPath was called %d time(s), want 0", calls) + } +} + +// TestAncestryChecker_SymbolicRefDefaultBranchIsRejectedBeforeReachingGit +// covers the second half of Review round 3, finding #2: git's well-known +// symbolic refs (HEAD, ORIG_HEAD, FETCH_HEAD, MERGE_HEAD) are also pure +// alnum/underscore and pass IsValidBranchName, but none is a real branch +// name — HEAD in particular always resolves in a non-empty checkout, to +// whatever commit is currently checked out, independent of the caller's +// intended default branch. +func TestAncestryChecker_SymbolicRefDefaultBranchIsRejectedBeforeReachingGit(t *testing.T) { + work, _ := newAncestryTestRepo(t) + head := runGit(t, work, "rev-parse", "HEAD") + + for _, ref := range []string{"HEAD", "ORIG_HEAD", "FETCH_HEAD", "MERGE_HEAD"} { + t.Run(ref, func(t *testing.T) { + calls := 0 + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work, calls: &calls}} + out := checker.Check(context.Background(), "repo-1", ref, head) + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true (%q must never reach the git subprocess as a default branch)", out, ref) + } + if calls != 0 { + t.Fatalf("ResolveRepositoryLocalPath was called %d time(s), want 0", calls) + } + }) + } +} + +// TestAncestryChecker_RevisionSyntaxDefaultBranchIsRejectedBeforeReachingGit +// covers the second, more dangerous consequence named in finding #1: a +// value like "main~1" is not flag-like, but git parses it as revision syntax +// (a parent-commit modifier) rather than a literal ref name. This is not a +// theoretical concern — with a real second commit on the default branch, +// "refs/remotes/origin/main~1^{commit}" resolves successfully (exit 0) to +// the PARENT commit, silently, with no error at all: the exact +// "nonexistent ref resolves to the wrong commit entirely" failure mode the +// finding describes. A repo with only one commit cannot reproduce this +// (main~1 has no parent to resolve to, so it fails safe by accident), so +// this test seeds a second commit specifically to exercise the real defect. +func TestAncestryChecker_RevisionSyntaxDefaultBranchIsRejectedBeforeReachingGit(t *testing.T) { + work, defaultBranch := newAncestryTestRepo(t) + writeFile(t, work, "second.txt", "second commit") + runGit(t, work, "add", "second.txt") + runGit(t, work, "commit", "-m", "second commit") + runGit(t, work, "push", "origin", defaultBranch) + runGit(t, work, "fetch", "origin") + head := runGit(t, work, "rev-parse", "HEAD") + + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + out := checker.Check(context.Background(), "repo-1", defaultBranch+"~1", head) + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true (revision syntax must never reach the git subprocess as a literal ref, "+ + "even when it happens to resolve to a real, but wrong, commit)", out) + } +} + +// TestAncestryChecker_CheckoutResolverErrorIsAnAncestryError is Review +// round 2, finding #5: the original version of this test passed commit +// "HEAD", which looksLikeCommitSHA rejects before Check ever reaches +// a.Checkout.ResolveRepositoryLocalPath (ancestry.go's very first guard) — +// so it was phantom-green, passing for the commit-format-validation +// reason covered elsewhere, never exercising the checkout-resolver error +// path its name claims. This version uses a real-looking 40-hex commit +// SHA and a valid branch name so Check's earlier guards let the call +// through, and asserts via calls that the resolver was actually invoked. +func TestAncestryChecker_CheckoutResolverErrorIsAnAncestryError(t *testing.T) { + calls := 0 + checker := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{err: errTestCheckout, calls: &calls}} + out := checker.Check(context.Background(), "repo-1", "main", strings.Repeat("a", 40)) + + if !out.Errored || out.Positive { + t.Fatalf("out = %+v, want Errored=true (no readable local checkout)", out) + } + if calls != 1 { + t.Fatalf("ResolveRepositoryLocalPath calls = %d, want 1 (Check must actually reach the checkout resolver)", calls) + } +} + +var errTestCheckout = errors.New("repository local path is empty") diff --git a/apps/backend/internal/delivery/evaluator.go b/apps/backend/internal/delivery/evaluator.go new file mode 100644 index 0000000000..c3e44734e1 --- /dev/null +++ b/apps/backend/internal/delivery/evaluator.go @@ -0,0 +1,369 @@ +package delivery + +import ( + "strings" + "time" +) + +// Snapshot is one task_session_git_snapshots row read for a candidate +// pair. HeadCommit and Ahead are the raw column values; Classify performs +// the normalization spec "Classification" requires (a NULL/empty/ +// whitespace-only head is not a distinct value; a NULL/negative ahead +// reads as 0) so those rules are exercised as pure Go logic rather than +// trusted to whichever caller built the slice. +type Snapshot struct { + SessionID string + Branch string + HeadCommit string + Ahead *int + CreatedAt time.Time +} + +// ProviderRequest is one provider pull/merge request row read for a +// candidate pair, already normalized to the "Provider predicates" table +// (Merged/Detached are the per-provider SQL predicates evaluated at the +// read layer, since the three provider tables share no schema). +type ProviderRequest struct { + Provider string // ProviderAzureDevOps | ProviderGitHub | ProviderGitLab + Merged bool + Detached bool + MergeInstant time.Time + RequestNumber int + URL string + BaseBranch string + ScopeValue string // github: owner+"/"+repo; gitlab: project_path; azure: azure_repository_id +} + +// AncestryOutcome is the result of an ancestry check attempted this +// evaluation, or the zero value when no check was attempted (precondition +// not met, or the caller decided not to run one). +type AncestryOutcome struct { + Attempted bool + Errored bool + Positive bool + Commit string // the head commit submitted to the check +} + +// PairInput is every input Classify reads for one (task, repository) +// pair, gathered by the caller before calling Classify. DefaultBranch ==" +// (already normalized from NULL) triggers Degraded evaluation. +type PairInput struct { + DefaultBranch string + Snapshots []Snapshot + Providers []ProviderRequest + Ancestry AncestryOutcome +} + +// Classification is the result of evaluating one pair once. It is a pure +// function of PairInput — re-running Classify on unchanged inputs +// produces a byte-identical result (spec "Idempotency"). +type Classification struct { + Outcome Outcome + Basis Basis + Ref *string + Rank int + + // ReachedDefault reports whether THIS evaluation produced a + // default-branch observation (by any basis), independent of whether a + // write-once column already holds an earlier one. Rule 3 reads this + // value, never the stored reached_default_at (spec "Classification", + // "What a predicate may read"). + ReachedDefault bool + ReachedBasis ReachedBasis + ReachedRef string + + // ObservedBranchCommits is nil when the pair has no snapshots at all + // (spec "Ordering": the incoming value is NULL, not 0, for an empty + // snapshot set). + ObservedBranchCommits *int +} + +// AncestryPrecondition reports whether an ancestry check should be +// attempted for this pair, per spec "Default-branch observation § +// Ancestry precondition": at least one session shows two or more distinct +// non-empty head_commit values among its own snapshots, evaluated across +// every branch (not restricted to the default branch). +func AncestryPrecondition(snapshots []Snapshot) bool { + return len(movingSessions(snapshots, nil)) > 0 +} + +// SelectAncestryHead picks the commit to submit to the ancestry check. +// +// Deviation from the literal spec text, recorded as a documented Build +// decision (R5-F1 in the plan): the spec's "Which commit" paragraph +// selects pair-wide, across every session's snapshots, not restricted to +// the session(s) that satisfied the precondition. That is unsafe on a +// multi-session pair — an idle session sitting on the default branch can +// supply a newer, unrelated snapshot whose inherited head trivially +// satisfies --is-ancestor, permanently misattributing a write-once +// column. This implementation restricts selection to the precondition- +// satisfying session(s), which is the fix the spec's own precondition +// paragraph describes wanting. No existing scenario in the spec +// contradicts this (both ancestry head-selection scenarios there are +// single-session). +func SelectAncestryHead(snapshots []Snapshot) string { + moving := movingSessions(snapshots, nil) + if len(moving) == 0 { + return "" + } + return selectHeadCommit(snapshots, func(s Snapshot) bool { + return moving[s.SessionID] + }) +} + +// Classify evaluates one (task, repository) pair once, applying the +// six-rule precedence, degraded evaluation, and the default-branch +// observation basis precedence, all as pure computation over the already- +// gathered inputs (spec "Classification", "Default-branch observation"). +func Classify(in PairInput) Classification { + degraded := in.DefaultBranch == "" + + reached, reachedBasis, reachedRef := computeDefaultBranchObservation(in, degraded) + outcome, basis, ref := matchRules(in, degraded, reached) + + if degraded { + basis = BasisDefaultBranchUnknown + // delivery_ref is NULL for every degraded row, not just rules 3-6 + // (spec "Classification": degraded is its own category regardless of + // which rule matched) — rule 1 still runs unconditionally above and + // would otherwise leak a real PR URL into a degraded row. + ref = "" + } + + return Classification{ + Outcome: outcome, + Basis: basis, + Ref: refPtr(ref), + Rank: Rank(outcome, basis), + ReachedDefault: reached, + ReachedBasis: reachedBasis, + ReachedRef: reachedRef, + ObservedBranchCommits: observedBranchCommits(in.Snapshots), + } +} + +// computeDefaultBranchObservation applies the basis precedence table in +// spec "Default-branch observation": provider_pr_merged (1, strongest), +// default_branch_commit (2), ancestor_of_default (3). All three steps are +// suspended when degraded (spec "Degraded evaluation"). +func computeDefaultBranchObservation(in PairInput, degraded bool) (bool, ReachedBasis, string) { + if degraded { + return false, "", "" + } + if ref, ok := selectProviderRef(in.Providers, func(p ProviderRequest) bool { + return p.BaseBranch == in.DefaultBranch + }); ok { + return true, ReachedBasisProviderPRMerged, ref + } + if ref := selectDefaultBranchCommitRef(in.Snapshots, in.DefaultBranch); ref != "" { + return true, ReachedBasisDefaultBranchCommit, ref + } + if in.Ancestry.Attempted && !in.Ancestry.Errored && in.Ancestry.Positive { + return true, ReachedBasisAncestorOfDefault, in.Ancestry.Commit + } + return false, "", "" +} + +// matchRules applies the six classification rules in order; the first +// match wins. Rule 1 runs even when degraded (whether a pull request +// merged does not depend on knowing the default branch); rules 2 and the +// reached-based half of rule 3 naturally fall through when degraded +// because branchMatches never matches an empty DefaultBranch and reached +// is forced false by computeDefaultBranchObservation. +func matchRules(in PairInput, degraded bool, reached bool) (Outcome, Basis, string) { + if ref, ok := selectProviderRef(in.Providers, nil); ok { + return OutcomePRMerge, BasisProviderPRMerged, ref + } + if !degraded { + if ref := selectDefaultBranchCommitRef(in.Snapshots, in.DefaultBranch); ref != "" { + return OutcomeDirectCommit, BasisDefaultBranchCommit, ref + } + } + if len(in.Snapshots) > 0 { + if maxAhead(in.Snapshots) > 0 || anySessionHasMovingHead(in.Snapshots, nil) { + if reached { + return OutcomeUnknown, BasisReachedDefaultUnattributed, "" + } + return OutcomeUnknown, BasisBranchCommitsUnmerged, "" + } + return OutcomeNoDeliveryObserved, BasisNoCommitsObserved, "" + } + if len(in.Providers) > 0 { + return OutcomeUnknown, BasisProviderPRUnmerged, "" + } + return OutcomeUnknown, BasisNoObservations, "" +} + +// selectDefaultBranchCommitRef implements rule 2's predicate and the +// Ordering "direct_commit ref" rule together: at least one session with +// >=2 distinct non-empty heads on the default branch, restricted (R5-F5, +// same reasoning as SelectAncestryHead) to the session(s) that earned it, +// then greatest created_at, tie broken by lexicographically greatest +// head_commit. +func selectDefaultBranchCommitRef(snapshots []Snapshot, defaultBranch string) string { + moving := movingSessions(snapshots, &defaultBranch) + if len(moving) == 0 { + return "" + } + return selectHeadCommit(snapshots, func(s Snapshot) bool { + return branchMatches(s.Branch, defaultBranch) && moving[s.SessionID] + }) +} + +// branchMatches is the exact, case-sensitive branch comparison spec +// "Classification" requires: an empty defaultBranch never matches any +// branch, even a snapshot whose own branch column is also empty. +func branchMatches(branch, defaultBranch string) bool { + return defaultBranch != "" && branch == defaultBranch +} + +// normalizeHead applies the document-wide head_commit normalization: NULL +// (never reaches Go), empty, or whitespace-only is not a distinct value. +func normalizeHead(raw string) string { + return strings.TrimSpace(raw) +} + +// normalizeAhead floors a NULL or negative ahead value at 0. +func normalizeAhead(raw *int) int { + if raw == nil || *raw < 0 { + return 0 + } + return *raw +} + +// distinctNonEmptyHeadsPerSession groups snapshots by SessionID and, +// when branchFilter is non-nil, restricts to snapshots whose Branch +// equals *branchFilter (via branchMatches, so an empty branchFilter +// matches nothing). Returns the set of distinct non-empty HeadCommit +// values per session. +func distinctNonEmptyHeadsPerSession(snapshots []Snapshot, branchFilter *string) map[string]map[string]bool { + out := map[string]map[string]bool{} + for _, s := range snapshots { + if branchFilter != nil && !branchMatches(s.Branch, *branchFilter) { + continue + } + head := normalizeHead(s.HeadCommit) + if head == "" { + continue + } + if out[s.SessionID] == nil { + out[s.SessionID] = map[string]bool{} + } + out[s.SessionID][head] = true + } + return out +} + +// movingSessions returns the set of session IDs with two or more distinct +// non-empty heads (optionally restricted to branchFilter) — the +// precondition-satisfying sessions spec "Head movement is a per-session +// fact" describes. Distinct heads are never pooled across sessions. +func movingSessions(snapshots []Snapshot, branchFilter *string) map[string]bool { + out := map[string]bool{} + for sid, heads := range distinctNonEmptyHeadsPerSession(snapshots, branchFilter) { + if len(heads) >= 2 { + out[sid] = true + } + } + return out +} + +// anySessionHasMovingHead is movingSessions's boolean form, used by rule +// 3's predicate. +func anySessionHasMovingHead(snapshots []Snapshot, branchFilter *string) bool { + return len(movingSessions(snapshots, branchFilter)) > 0 +} + +// selectHeadCommit picks the ref commit per Ordering: among snapshots +// satisfying keep, those with non-empty HeadCommit (filtered before the +// created_at comparison, so a newer empty-headed snapshot can never +// displace a real commit), greatest CreatedAt, ties broken by +// lexicographically greatest HeadCommit. Returns "" if no snapshot +// qualifies. +func selectHeadCommit(snapshots []Snapshot, keep func(Snapshot) bool) string { + var bestHead string + var bestAt time.Time + found := false + for _, s := range snapshots { + if !keep(s) { + continue + } + head := normalizeHead(s.HeadCommit) + if head == "" { + continue + } + if !found || s.CreatedAt.After(bestAt) || (s.CreatedAt.Equal(bestAt) && head > bestHead) { + bestHead, bestAt, found = head, s.CreatedAt, true + } + } + return bestHead +} + +// selectProviderRef implements the shared provider tiebreak (spec +// "Ordering", pr_merge ref rule / default-branch observation ref rule) +// over the subset of merged, non-detached rows satisfying filter (nil = +// no filter, i.e. rule 1's ref selection). Earliest merge instant, then +// request number ascending, then provider name ascending +// (azure_devops, github, gitlab), then that provider's scope column +// ascending. +func selectProviderRef(prs []ProviderRequest, filter func(ProviderRequest) bool) (string, bool) { + var best ProviderRequest + found := false + for _, p := range prs { + if !p.Merged || p.Detached { + continue + } + if filter != nil && !filter(p) { + continue + } + if !found || providerLess(p, best) { + best, found = p, true + } + } + return best.URL, found +} + +func providerLess(a, b ProviderRequest) bool { + if !a.MergeInstant.Equal(b.MergeInstant) { + return a.MergeInstant.Before(b.MergeInstant) + } + if a.RequestNumber != b.RequestNumber { + return a.RequestNumber < b.RequestNumber + } + if providerOrder[a.Provider] != providerOrder[b.Provider] { + return providerOrder[a.Provider] < providerOrder[b.Provider] + } + return a.ScopeValue < b.ScopeValue +} + +// maxAhead returns the greatest normalized ahead across snapshots, or 0 +// for an empty slice (callers must not treat that 0 as "no snapshots" — +// they check len(in.Snapshots) separately, matching observedBranchCommits +// which returns nil, not 0, for an empty set). +func maxAhead(snapshots []Snapshot) int { + max := 0 + for _, s := range snapshots { + if a := normalizeAhead(s.Ahead); a > max { + max = a + } + } + return max +} + +// observedBranchCommits is the incoming value for the high-water column: +// the greatest normalized ahead, or nil when the pair has no snapshots at +// all (spec "Ordering": the greatest of an empty set is not zero). +func observedBranchCommits(snapshots []Snapshot) *int { + if len(snapshots) == 0 { + return nil + } + v := maxAhead(snapshots) + return &v +} + +func refPtr(s string) *string { + if s == "" { + return nil + } + return &s +} diff --git a/apps/backend/internal/delivery/evaluator_test.go b/apps/backend/internal/delivery/evaluator_test.go new file mode 100644 index 0000000000..65608c1419 --- /dev/null +++ b/apps/backend/internal/delivery/evaluator_test.go @@ -0,0 +1,521 @@ +package delivery_test + +import ( + "testing" + "time" + + "github.com/kandev/kandev/internal/delivery" +) + +func at(minutesFromEpoch int) time.Time { + return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(minutesFromEpoch) * time.Minute) +} + +func intp(v int) *int { return &v } + +func TestClassify_Rule1PRMerge(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://pr/1", BaseBranch: "main", MergeInstant: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomePRMerge || got.Basis != delivery.BasisProviderPRMerged { + t.Fatalf("got %+v", got) + } + if got.Rank != 8 { + t.Fatalf("rank = %d, want 8", got.Rank) + } + if got.Ref == nil || *got.Ref != "https://pr/1" { + t.Fatalf("ref = %v, want pr url", got.Ref) + } +} + +func TestClassify_Rule1GitLabMergedAtNonNull(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitLab, Merged: true, URL: "https://gitlab/mr/1", BaseBranch: "main", MergeInstant: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomePRMerge || got.Basis != delivery.BasisProviderPRMerged { + t.Fatalf("got %+v", got) + } +} + +func TestClassify_AzureUnrecognisedStatusFallsThroughWithoutError(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderAzureDevOps, Merged: false, URL: "https://azure/pr/1", BaseBranch: "main"}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisProviderPRUnmerged { + t.Fatalf("got %+v, want unknown/provider_pr_unmerged", got) + } + if got.Rank != 3 { + t.Fatalf("rank = %d, want 3", got.Rank) + } +} + +func TestClassify_Rule2DirectCommit(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "main", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "main", HeadCommit: "bbb", CreatedAt: at(1)}, + {SessionID: "s1", Branch: "main", HeadCommit: "ccc", CreatedAt: at(2)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeDirectCommit || got.Basis != delivery.BasisDefaultBranchCommit { + t.Fatalf("got %+v", got) + } + if got.Ref == nil || *got.Ref != "ccc" { + t.Fatalf("ref = %v, want ccc (newest)", got.Ref) + } + if !got.ReachedDefault || got.ReachedBasis != delivery.ReachedBasisDefaultBranchCommit || got.ReachedRef != "ccc" { + t.Fatalf("reached observation = %+v", got) + } +} + +// TestClassify_Rule2DirectCommitRestrictedToPreconditionSatisfyingSession +// covers R5-F5 (spec revision 5, Ordering § direct_commit ref): the +// selectDefaultBranchCommitRef restriction to the session(s) satisfying +// rule 2's own moving-head-on-default-branch predicate had no test — +// its ancestry twin (SelectAncestryHead) was covered but this one was +// not. s1 authored two distinct heads on the default branch (satisfies +// the predicate); s2 sits idle on the default branch and contributes one +// newer, never-moved inherited snapshot. Restricted selection must stay +// within s1 and pick its newest head, not s2's newer inherited tip. This +// already passes against the shipped implementation — recorded as a +// regression guard for a real gap in test coverage, not a fix. +func TestClassify_Rule2DirectCommitRestrictedToPreconditionSatisfyingSession(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "main", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "main", HeadCommit: "bbb", CreatedAt: at(1)}, + {SessionID: "s2", Branch: "main", HeadCommit: "inherited-tip", CreatedAt: at(2)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeDirectCommit || got.Basis != delivery.BasisDefaultBranchCommit { + t.Fatalf("got %+v", got) + } + if got.Ref == nil || *got.Ref != "bbb" { + t.Fatalf("ref = %v, want bbb (s1's newest head, not s2's newer inherited tip)", got.Ref) + } + if !got.ReachedDefault || got.ReachedRef != "bbb" { + t.Fatalf("reached observation = %+v, want ReachedRef bbb", got) + } +} + +func TestClassify_Rule3UnknownBranchCommitsUnmerged(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(12), CreatedAt: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisBranchCommitsUnmerged { + t.Fatalf("got %+v", got) + } + if got.Rank != 5 { + t.Fatalf("rank = %d, want 5", got.Rank) + } + if got.ObservedBranchCommits == nil || *got.ObservedBranchCommits != 12 { + t.Fatalf("observed_branch_commits = %v, want 12", got.ObservedBranchCommits) + } + if got.Ref != nil { + t.Fatalf("ref = %v, want nil", got.Ref) + } +} + +func TestClassify_Rule3PromotesToReachedDefaultUnattributedWhenObservedThisEvaluation(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(12), CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "bbb", Ahead: intp(12), CreatedAt: at(1)}, + }, + Ancestry: delivery.AncestryOutcome{Attempted: true, Positive: true, Commit: "bbb"}, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisReachedDefaultUnattributed { + t.Fatalf("got %+v", got) + } + if got.Rank != 6 { + t.Fatalf("rank = %d, want 6", got.Rank) + } + if !got.ReachedDefault || got.ReachedBasis != delivery.ReachedBasisAncestorOfDefault || got.ReachedRef != "bbb" { + t.Fatalf("reached observation = %+v", got) + } +} + +func TestClassify_Rule4NoDeliveryObserved(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(0), CreatedAt: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeNoDeliveryObserved || got.Basis != delivery.BasisNoCommitsObserved { + t.Fatalf("got %+v", got) + } + if got.Rank != 4 { + t.Fatalf("rank = %d, want 4", got.Rank) + } +} + +func TestClassify_Rule4EmptyHeadCountsAsNoDistinctHead(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "", Ahead: intp(0), CreatedAt: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeNoDeliveryObserved { + t.Fatalf("got %+v, want no_delivery_observed", got) + } +} + +func TestClassify_AheadNullAndNegativeNormalizeToZero(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: nil, CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(-1), CreatedAt: at(1)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeNoDeliveryObserved { + t.Fatalf("got %+v, want no_delivery_observed (NULL/negative ahead read as 0)", got) + } +} + +func TestClassify_Rule5ProviderPRUnmergedNoSnapshot(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: false, URL: "https://pr/2", BaseBranch: "main"}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisProviderPRUnmerged { + t.Fatalf("got %+v", got) + } + if got.Ref != nil { + t.Fatalf("ref = %v, want nil (unmerged PR URL is not a delivery ref)", got.Ref) + } +} + +func TestClassify_Rule6NoObservations(t *testing.T) { + got := delivery.Classify(delivery.PairInput{DefaultBranch: "main"}) + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisNoObservations { + t.Fatalf("got %+v", got) + } + if got.Rank != 2 { + t.Fatalf("rank = %d, want 2", got.Rank) + } +} + +func TestClassify_DegradedEvaluationRank1SupersedesMatchedBasis(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(12), CreatedAt: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeUnknown { + t.Fatalf("outcome = %v, want unknown (rule 3 still matched)", got.Outcome) + } + if got.Basis != delivery.BasisDefaultBranchUnknown { + t.Fatalf("basis = %v, want default_branch_unknown", got.Basis) + } + if got.Rank != 1 { + t.Fatalf("rank = %d, want 1", got.Rank) + } + if got.ReachedDefault { + t.Fatal("reached default observation must be suspended while degraded") + } +} + +func TestClassify_DegradedProviderPRWithEmptyBaseBranchNeverReachesDefault(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://pr/3", BaseBranch: "", MergeInstant: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomePRMerge { + t.Fatalf("outcome = %v, want pr_merge (rule 1 still runs while degraded)", got.Outcome) + } + if got.Basis != delivery.BasisDefaultBranchUnknown || got.Rank != 1 { + t.Fatalf("got %+v, want degraded rank 1", got) + } + if got.ReachedDefault { + t.Fatal("empty default_branch must never match an equally-empty base_branch") + } + if got.Ref != nil { + t.Fatalf("ref = %v, want nil: every degraded row must have a NULL delivery_ref, not just rules 3-6", *got.Ref) + } +} + +func TestClassify_TwoInheritedBranchesNeverPooledAcrossSessions(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature-a", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s2", Branch: "feature-b", HeadCommit: "bbb", CreatedAt: at(1)}, + }, + } + got := delivery.Classify(in) + if got.Outcome == delivery.OutcomeDirectCommit { + t.Fatal("distinct heads from different sessions must not be pooled into direct_commit") + } + if got.ReachedDefault { + t.Fatal("no ancestry check should have been attempted, so nothing should be reached") + } + if !delivery.AncestryPrecondition(in.Snapshots) { + return // fine either way for this assertion; the real check is below + } + t.Fatal("neither session individually has >=2 distinct heads, so the precondition must not be met") +} + +func TestClassify_RulesAreExhaustive(t *testing.T) { + // A pair with at least one snapshot and no providers must always match + // rule 2, 3, or 4 — never fall through to 5/6, and never leave + // delivery_outcome unset. + cases := []delivery.PairInput{ + {DefaultBranch: "main"}, + {DefaultBranch: "main", Snapshots: []delivery.Snapshot{{SessionID: "s1", Branch: "f", HeadCommit: "a", Ahead: intp(0), CreatedAt: at(0)}}}, + {DefaultBranch: "main", Snapshots: []delivery.Snapshot{{SessionID: "s1", Branch: "f", HeadCommit: "a", Ahead: intp(1), CreatedAt: at(0)}}}, + {DefaultBranch: "main", Providers: []delivery.ProviderRequest{{Provider: delivery.ProviderGitHub, Merged: false, URL: "u"}}}, + } + for i, in := range cases { + got := delivery.Classify(in) + if got.Outcome == "" || got.Basis == "" { + t.Errorf("case %d: outcome/basis unset: %+v", i, got) + } + } +} + +// --- Squash-merge, ancestry preconditions and negative evidence --- + +func TestAncestryPrecondition_SameSingleHeadNeverMet(t *testing.T) { + snaps := []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(4), CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(4), CreatedAt: at(1)}, + } + if delivery.AncestryPrecondition(snaps) { + t.Fatal("a single repeated head (ahead>0, inherited branch) must not satisfy the precondition") + } +} + +func TestAncestryPrecondition_MetWhenSessionHeadMoves(t *testing.T) { + snaps := []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "bbb", CreatedAt: at(1)}, + } + if !delivery.AncestryPrecondition(snaps) { + t.Fatal("two distinct heads within one session must satisfy the precondition") + } +} + +func TestSelectAncestryHead_TieBrokenByLexicographicallyGreatestHead(t *testing.T) { + snaps := []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "zzz", CreatedAt: at(5)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "yyy", CreatedAt: at(5)}, + } + if got := delivery.SelectAncestryHead(snaps); got != "zzz" { + t.Fatalf("got %q, want zzz", got) + } +} + +func TestSelectAncestryHead_SkipsWhitespaceOnlyNewestSnapshot(t *testing.T) { + snaps := []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "bbb", CreatedAt: at(1)}, + {SessionID: "s1", Branch: "feature", HeadCommit: " ", CreatedAt: at(2)}, + } + if got := delivery.SelectAncestryHead(snaps); got != "bbb" { + t.Fatalf("got %q, want bbb (newest non-empty)", got) + } +} + +func TestSelectAncestryHead_RestrictedToPreconditionSatisfyingSession(t *testing.T) { + // s1 authored two distinct heads on an unmerged feature branch + // (satisfies the precondition). s2 sits idle on the default branch and + // writes a newer snapshot whose head_commit is its inherited tip. Per + // the R5-F1 Build decision, selection must stay within s1 and must not + // pick s2's newer, unrelated head. + snaps := []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "bbb", CreatedAt: at(1)}, + {SessionID: "s2", Branch: "main", HeadCommit: "inherited-tip", CreatedAt: at(2)}, + } + if got := delivery.SelectAncestryHead(snaps); got != "bbb" { + t.Fatalf("got %q, want bbb (s1's newest head, not s2's newer inherited tip)", got) + } +} + +func TestClassify_NegativeAncestryResultDiscarded(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "bbb", CreatedAt: at(1)}, + }, + Ancestry: delivery.AncestryOutcome{Attempted: true, Positive: false, Commit: "bbb"}, + } + got := delivery.Classify(in) + if got.ReachedDefault { + t.Fatal("a negative ancestry result must not produce a default-branch observation") + } + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisBranchCommitsUnmerged { + t.Fatalf("got %+v, want unknown/branch_commits_unmerged", got) + } +} + +func TestClassify_AncestryErrorDoesNotProduceObservation(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "feature", HeadCommit: "bbb", CreatedAt: at(1)}, + }, + Ancestry: delivery.AncestryOutcome{Attempted: true, Errored: true}, + } + got := delivery.Classify(in) + if got.ReachedDefault { + t.Fatal("an errored ancestry check must not produce an observation") + } + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisBranchCommitsUnmerged { + t.Fatalf("got %+v, want unknown/branch_commits_unmerged (rest of classification still persisted)", got) + } +} + +func TestClassify_DifferentBaseBranchSplitsRefFromReachedRef(t *testing.T) { + // A pair with a merged PR into a release branch (earlier merge instant, + // wins delivery_ref because rule 1 has no base-branch filter) and a + // later merged PR into main (wins reached_default_ref because the + // observation ref rule filters on base branch). + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://pr/A", BaseBranch: "release/1.2", MergeInstant: at(0), RequestNumber: 1}, + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://pr/B", BaseBranch: "main", MergeInstant: at(10), RequestNumber: 2}, + }, + } + got := delivery.Classify(in) + if got.Ref == nil || *got.Ref != "https://pr/A" { + t.Fatalf("delivery_ref = %v, want pr A (earliest merge, no base-branch filter)", got.Ref) + } + if got.ReachedRef != "https://pr/B" || got.ReachedBasis != delivery.ReachedBasisProviderPRMerged { + t.Fatalf("reached ref/basis = %q/%v, want pr B / provider_pr_merged", got.ReachedRef, got.ReachedBasis) + } +} + +func TestClassify_DetachedMergedPRExcludedFromRule1AndObservation(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "feature", HeadCommit: "aaa", Ahead: intp(3), CreatedAt: at(0)}, + }, + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: true, Detached: true, URL: "https://pr/4", BaseBranch: "main", MergeInstant: at(0)}, + }, + } + got := delivery.Classify(in) + if got.Outcome != delivery.OutcomeUnknown || got.Basis != delivery.BasisBranchCommitsUnmerged { + t.Fatalf("got %+v, want unknown/branch_commits_unmerged", got) + } + if got.Rank != 5 { + t.Fatalf("rank = %d, want 5", got.Rank) + } + if got.ReachedDefault { + t.Fatal("a detached row must be excluded from the observation exactly as from rule 1") + } +} + +func TestClassify_TwoMergedPRsTieBrokenByLowerRequestNumber(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://pr/hi", BaseBranch: "main", MergeInstant: at(0), RequestNumber: 9}, + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://pr/lo", BaseBranch: "main", MergeInstant: at(0), RequestNumber: 2}, + }, + } + got := delivery.Classify(in) + if got.Ref == nil || *got.Ref != "https://pr/lo" { + t.Fatalf("ref = %v, want pr/lo (lower request number on exact tie)", got.Ref) + } +} + +func TestClassify_GitHubAndAzureTieBrokenByProviderOrder(t *testing.T) { + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitHub, Merged: true, URL: "https://gh", BaseBranch: "main", MergeInstant: at(0), RequestNumber: 1}, + {Provider: delivery.ProviderAzureDevOps, Merged: true, URL: "https://az", BaseBranch: "main", MergeInstant: at(0), RequestNumber: 1}, + }, + } + got := delivery.Classify(in) + if got.Ref == nil || *got.Ref != "https://az" { + t.Fatalf("ref = %v, want azure (azure_devops sorts before github on exact tie)", got.Ref) + } +} + +func TestClassify_GitLabSameMRIIDTiebreakByProjectPathOnEqualMergeInstant(t *testing.T) { + // Per R5-F7 (Build decision), the literal spec GIVEN omits "sharing the + // same merge instant" — the Ordering rule selects by earliest merge + // instant first, so this test seeds an exact tie to actually exercise + // the project_path tiebreak, rather than translating the scenario + // literally (which could fail a correct implementation). + in := delivery.PairInput{ + DefaultBranch: "main", + Providers: []delivery.ProviderRequest{ + {Provider: delivery.ProviderGitLab, Merged: true, URL: "https://gl/z-path", BaseBranch: "main", MergeInstant: at(0), RequestNumber: 5, ScopeValue: "z-path"}, + {Provider: delivery.ProviderGitLab, Merged: true, URL: "https://gl/a-path", BaseBranch: "main", MergeInstant: at(0), RequestNumber: 5, ScopeValue: "a-path"}, + }, + } + got := delivery.Classify(in) + if got.Ref == nil || *got.Ref != "https://gl/a-path" { + t.Fatalf("ref = %v, want the lower project_path", got.Ref) + } +} + +func TestClassify_ObservedBranchCommitsNilWithNoSnapshots(t *testing.T) { + got := delivery.Classify(delivery.PairInput{DefaultBranch: "main"}) + if got.ObservedBranchCommits != nil { + t.Fatalf("observed_branch_commits = %v, want nil", got.ObservedBranchCommits) + } +} + +func TestClassify_RemoteBranchFilterOnDefaultBranchDoesNotMatchEmptyBranchColumn(t *testing.T) { + // Guards against the "empty default_branch matches an equally empty + // branch column" trap called out in spec "Degraded evaluation": here + // DefaultBranch is set but a snapshot's own Branch happens to be empty + // — that must never satisfy branchMatches either. + in := delivery.PairInput{ + DefaultBranch: "main", + Snapshots: []delivery.Snapshot{ + {SessionID: "s1", Branch: "", HeadCommit: "aaa", CreatedAt: at(0)}, + {SessionID: "s1", Branch: "", HeadCommit: "bbb", CreatedAt: at(1)}, + }, + } + got := delivery.Classify(in) + if got.Outcome == delivery.OutcomeDirectCommit { + t.Fatal("an empty branch column must never match a non-empty default_branch") + } +} diff --git a/apps/backend/internal/delivery/metrics.go b/apps/backend/internal/delivery/metrics.go new file mode 100644 index 0000000000..d19a4d9770 --- /dev/null +++ b/apps/backend/internal/delivery/metrics.go @@ -0,0 +1,197 @@ +package delivery + +import ( + "context" + "expvar" + "fmt" + "time" +) + +// expvar maps/ints published at package init, exposed via the stdlib +// /debug/vars handler, following the subproc_* / routing_* precedent +// (apps/backend/internal/common/subproc/metrics.go, +// internal/office/scheduler/metrics_vars.go). See spec "Writer health". +var ( + evaluationsTotal = expvar.NewMap("delivery_ledger_evaluations_total") // keyed by delivery_outcome + rowsWrittenTotal = expvar.NewInt("delivery_ledger_rows_written_total") + demotionsSuppressedTotal = expvar.NewInt("delivery_ledger_demotions_suppressed_total") + evaluationErrorsTotal = expvar.NewInt("delivery_ledger_evaluation_errors_total") + ancestryErrorsTotal = expvar.NewInt("delivery_ledger_ancestry_errors_total") + ancestrySkippedTotal = expvar.NewInt("delivery_ledger_ancestry_skipped_total") + writeErrorsTotal = expvar.NewInt("delivery_ledger_write_errors_total") + degradedOutcomeChangedTotal = expvar.NewInt("delivery_ledger_degraded_outcome_changed_total") + pairsMissingRepositoryGauge = expvar.NewInt("delivery_ledger_pairs_missing_repository_total") + pairsMissingTaskGauge = expvar.NewInt("delivery_ledger_pairs_missing_task_total") + sessionsUnattributedGauge = expvar.NewInt("delivery_ledger_sessions_unattributed_total") + lastEvaluatedUnixVar = expvar.NewInt("delivery_ledger_last_evaluated_unix") + stallSecondsVar = expvar.NewInt("delivery_ledger_stall_seconds") + + defaultBranchPersistErrorsTotal = expvar.NewInt("delivery_ledger_default_branch_persist_errors_total") +) + +func recordEvaluation(outcome Outcome) { evaluationsTotal.Add(string(outcome), 1) } +func recordRowWritten() { rowsWrittenTotal.Add(1) } +func recordDemotionSuppressed() { demotionsSuppressedTotal.Add(1) } +func recordEvaluationError() { evaluationErrorsTotal.Add(1) } +func recordAncestryError() { ancestryErrorsTotal.Add(1) } +func recordAncestrySkipped() { ancestrySkippedTotal.Add(1) } +func recordWriteError() { writeErrorsTotal.Add(1) } +func recordDegradedOutcomeChanged() { degradedOutcomeChangedTotal.Add(1) } + +// RecordDefaultBranchPersistError increments +// delivery_ledger_default_branch_persist_errors_total. Exported for call +// sites outside this package (backendapp's review base-branch resolution) +// that fail to write a detected repositories.default_branch value: per spec +// "Degraded evaluation", that repository's pairs keep reading as +// default_branch_unknown until some future write succeeds, and Review round +// 3 finding #4 was exactly that this could happen silently, with no signal +// anywhere that it was happening. +func RecordDefaultBranchPersistError() { defaultBranchPersistErrorsTotal.Add(1) } + +func setPairsMissingRepositoryGauge(n int) { pairsMissingRepositoryGauge.Set(int64(n)) } +func setPairsMissingTaskGauge(n int) { pairsMissingTaskGauge.Set(int64(n)) } +func setSessionsUnattributedGauge(n int) { sessionsUnattributedGauge.Set(int64(n)) } + +// StallSignal is what the sweep publishes at the end of every pass (spec +// "Writer health § The data-side stall signal"). LedgerErr / ComparandErr +// are non-nil exactly when their respective query failed; the sentinel +// values (-1) are what LastEvaluatedUnix / StallSeconds hold in that case, +// but the caller logs the distinct WARN messages, not this type. +type StallSignal struct { + LastEvaluatedUnix int64 + StallSeconds int64 + LedgerErr error + ComparandErr error +} + +// ComputeStallSignal implements the five-row state table in spec "Writer +// health": the ledger query and the comparand query are independent and +// either can fail or return empty on its own, so the sentinels +// distinguish "unavailable" from "healthy and empty" — publishing 0/0 for +// a missing table would be indistinguishable from a fresh, healthy +// database. +func (r *Repository) ComputeStallSignal(ctx context.Context) StallSignal { + maxEvaluated, hasRows, err := r.maxLastEvaluatedAt(ctx) + if err != nil { + return StallSignal{LastEvaluatedUnix: -1, StallSeconds: -1, LedgerErr: err} + } + if !hasRows { + return StallSignal{LastEvaluatedUnix: 0, StallSeconds: 0} + } + + sig := StallSignal{LastEvaluatedUnix: maxEvaluated.Unix()} + maxSession, hasSessions, err := r.stallComparand(ctx) + if err != nil { + sig.StallSeconds = -1 + sig.ComparandErr = err + return sig + } + if !hasSessions { + sig.StallSeconds = 0 + return sig + } + diff := maxSession.Sub(maxEvaluated) + if diff < 0 { + diff = 0 + } + sig.StallSeconds = int64(diff.Seconds()) + return sig +} + +func publishStallSignal(sig StallSignal) { + lastEvaluatedUnixVar.Set(sig.LastEvaluatedUnix) + stallSecondsVar.Set(sig.StallSeconds) +} + +func (r *Repository) maxLastEvaluatedAt(ctx context.Context) (time.Time, bool, error) { + var raw interface{} + err := r.ro.QueryRowxContext(ctx, `SELECT MAX(last_evaluated_at) FROM task_delivery_ledger`).Scan(&raw) + if err != nil { + return time.Time{}, false, err + } + return scanAggregateTime(raw) +} + +// stallComparand computes MAX(task_sessions.updated_at) restricted to +// sessions that could have produced work for the writer (spec "Writer +// health"). The INNER JOIN against repositories — rather than a LEFT +// JOIN — is the R5-F2-adjacent Build decision for R5-F4: a session whose +// repository_id is non-empty but matches no repositories row must drop +// out of the comparand exactly like the two named filters, or a dangling +// repository_id would publish a permanent false stall on a healthy +// writer. deleted_at IS NULL then excludes soft-deleted repositories, +// whose pairs are frozen by design. +func (r *Repository) stallComparand(ctx context.Context) (time.Time, bool, error) { + var raw interface{} + err := r.ro.QueryRowxContext(ctx, ` + SELECT MAX(s.updated_at) + FROM task_sessions s + JOIN repositories rep ON rep.id = s.repository_id + WHERE s.repository_id != '' AND rep.deleted_at IS NULL + `).Scan(&raw) + if err != nil { + return time.Time{}, false, err + } + return scanAggregateTime(raw) +} + +// sqliteAggregateTimeLayouts are the text formats an aggregate (MAX/MIN) +// over a TIMESTAMP column can come back as on SQLite: aggregate functions +// erase the column's declared type affinity that go-sqlite3 normally uses +// to auto-parse a scanned value into time.Time, so the driver hands back +// the raw stored text instead. This is the exact set of layouts +// mattn/go-sqlite3 itself writes when binding a time.Time parameter +// (SQLiteTimestampFormats), tried in the same order it does. +var sqliteAggregateTimeLayouts = []string{ + "2006-01-02 15:04:05.999999999-07:00", + "2006-01-02T15:04:05.999999999-07:00", + "2006-01-02 15:04:05.999999999", + "2006-01-02T15:04:05.999999999", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05", + "2006-01-02 15:04", + "2006-01-02T15:04", + "2006-01-02", +} + +// scanAggregateTime interprets the driver value of a MAX/MIN aggregate +// over a TIMESTAMP column, which on Postgres is already a time.Time +// (aggregates do not erase Postgres's wire-protocol type info) but on +// SQLite may be a plain string or []byte. A NULL aggregate (empty input +// set) reports ok=false, never a zero time mistaken for a real one. +func scanAggregateTime(raw interface{}) (time.Time, bool, error) { + switch v := raw.(type) { + case nil: + return time.Time{}, false, nil + case time.Time: + return v, true, nil + case string: + t, err := parseAggregateTimeString(v) + return t, err == nil, err + case []byte: + t, err := parseAggregateTimeString(string(v)) + return t, err == nil, err + default: + return time.Time{}, false, fmt.Errorf("delivery: unsupported aggregate time value type %T", raw) + } +} + +func parseAggregateTimeString(s string) (time.Time, error) { + for _, layout := range sqliteAggregateTimeLayouts { + if t, err := time.ParseInLocation(layout, s, time.UTC); err == nil { + return t.UTC(), nil + } + } + return time.Time{}, fmt.Errorf("delivery: cannot parse aggregate time %q", s) +} + +// CountUnattributedSessions counts distinct sessions with an empty +// repository_id, for the sessions_unattributed_total gauge (set, not +// incremented, once per sweep pass). +func (r *Repository) CountUnattributedSessions(ctx context.Context) (int, error) { + var n int + err := r.ro.QueryRowxContext(ctx, ` + SELECT COUNT(*) FROM task_sessions WHERE repository_id = '' OR repository_id IS NULL + `).Scan(&n) + return n, err +} diff --git a/apps/backend/internal/delivery/metrics_test.go b/apps/backend/internal/delivery/metrics_test.go new file mode 100644 index 0000000000..7228f5bcb7 --- /dev/null +++ b/apps/backend/internal/delivery/metrics_test.go @@ -0,0 +1,276 @@ +package delivery_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/delivery" +) + +func TestComputeStallSignal_EmptyLedgerIsNotAnInfiniteStall(t *testing.T) { + repo, _ := newTestRepo(t) + sig := repo.ComputeStallSignal(context.Background()) + if sig.LastEvaluatedUnix != 0 || sig.StallSeconds != 0 { + t.Fatalf("sig = %+v, want 0/0 for an empty table", sig) + } + if sig.LedgerErr != nil || sig.ComparandErr != nil { + t.Fatalf("sig = %+v, want no errors", sig) + } +} + +func TestComputeStallSignal_LedgerRowsWithEmptyComparandIsNotAStall(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + // No session at all: the comparand set is empty. + + sig := repo.ComputeStallSignal(context.Background()) + if sig.LastEvaluatedUnix == 0 { + t.Fatal("expected the true MAX(last_evaluated_at), not 0") + } + if sig.StallSeconds != 0 { + t.Fatalf("stall_seconds = %d, want 0 (no work to be behind on)", sig.StallSeconds) + } +} + +// TestComputeStallSignal_DanglingRepositoryIDExcludedFromComparand is the +// R5-F4 regression test: a session whose repository_id is non-empty but +// matches no repositories row must not drive the comparand, or a broken +// reference publishes a permanent false stall on a healthy writer. +func TestComputeStallSignal_DanglingRepositoryIDExcludedFromComparand(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + // A session whose repository_id points at nothing. + seedTaskSession(t, db, "sess-dangling", "task-1", "repo-does-not-exist") + + sig := repo.ComputeStallSignal(context.Background()) + if sig.StallSeconds != 0 { + t.Fatalf("stall_seconds = %d, want 0 (dangling repository_id must not drive the comparand)", sig.StallSeconds) + } +} + +// TestComputeStallSignal_ExceedsThresholdWithValidComparand covers the +// firing state Review round 1 finding #4 flagged as untested: every +// existing TestComputeStallSignal_* case keeps StallSeconds at 0 (an +// empty/dangling/deleted/unattributed comparand), so nothing before this +// proved the >900s branch (spec.md:2179-2183) actually computes a +// positive value once a valid, live session drives the comparand ahead +// of the ledger's last_evaluated_at. +func TestComputeStallSignal_ExceedsThresholdWithValidComparand(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + // A live session on a real, non-deleted repository: it drives the + // comparand to ~now, an hour ahead of the ledger's last_evaluated_at. + seedTaskSession(t, db, "sess-live", "task-1", "repo-1") + + sig := repo.ComputeStallSignal(context.Background()) + if sig.LedgerErr != nil || sig.ComparandErr != nil { + t.Fatalf("sig = %+v, want no errors", sig) + } + const stallThresholdSeconds = 900 + if sig.StallSeconds <= stallThresholdSeconds { + t.Fatalf("stall_seconds = %d, want > %d (session is ~1h ahead of last_evaluated_at)", sig.StallSeconds, stallThresholdSeconds) + } +} + +// TestComputeStallSignal_ArchivedTaskSessionStillDrivesComparand is +// Review round 2, finding #9 (spec.md:2213-2217): archived tasks are still +// evaluated, so a stalled writer must still surface as stalled when the +// only live session belongs to an archived task. Unlike the soft-deleted +// repository and unattributed-session cases above, task archival must NOT +// be a comparand exclusion — stallComparand has no join against tasks at +// all, and this test guards against ever adding one. +func TestComputeStallSignal_ArchivedTaskSessionStillDrivesComparand(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-archived", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE tasks SET archived_at = ? WHERE id = ?`), time.Now().UTC(), "task-archived"); err != nil { + t.Fatalf("archive task: %v", err) + } + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-archived", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + // The writer has stopped: this session updated only 20 minutes ago, + // well inside the writer's own hour-old last_evaluated_at, but it is + // the only thing driving the comparand and must not be excluded for + // belonging to an archived task. + seedTaskSession(t, db, "sess-archived-task", "task-archived", "repo-1") + if _, err := db.Exec(db.Rebind(`UPDATE task_sessions SET updated_at = ? WHERE id = ?`), time.Now().UTC().Add(-20*time.Minute), "sess-archived-task"); err != nil { + t.Fatalf("backdate session: %v", err) + } + + sig := repo.ComputeStallSignal(context.Background()) + if sig.LedgerErr != nil || sig.ComparandErr != nil { + t.Fatalf("sig = %+v, want no errors", sig) + } + const stallThresholdSeconds = 900 + if sig.StallSeconds <= stallThresholdSeconds { + t.Fatalf("stall_seconds = %d, want > %d (archived task's session must still drive the comparand)", sig.StallSeconds, stallThresholdSeconds) + } +} + +func TestComputeStallSignal_SoftDeletedRepositorySessionExcludedFromComparand(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + seedRepository(t, db, "repo-deleted", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET deleted_at = ? WHERE id = ?`), time.Now().UTC(), "repo-deleted"); err != nil { + t.Fatalf("soft delete: %v", err) + } + seedTaskSession(t, db, "sess-deleted-repo", "task-1", "repo-deleted") + + sig := repo.ComputeStallSignal(context.Background()) + if sig.StallSeconds != 0 { + t.Fatalf("stall_seconds = %d, want 0 (a soft-deleted repository's sessions are frozen by design)", sig.StallSeconds) + } +} + +func TestComputeStallSignal_UnattributedSessionExcludedFromComparand(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + seedTaskSession(t, db, "sess-unattributed", "task-1", "") + + sig := repo.ComputeStallSignal(context.Background()) + if sig.StallSeconds != 0 { + t.Fatalf("stall_seconds = %d, want 0 (an unattributed session contributes to no pair)", sig.StallSeconds) + } +} + +// TestComputeStallSignal_MissingLedgerTableReportsUnavailable covers spec +// "Writer health": a genuinely broken ledger (its migration swallowed at +// WARN) must report -1/-1, never 0/0 — the healthy-and-empty state — or a +// broken writer would be indistinguishable from a fresh, working one. +func TestComputeStallSignal_MissingLedgerTableReportsUnavailable(t *testing.T) { + repo, db := newTestRepo(t) + if _, err := db.Exec(`DROP TABLE task_delivery_ledger`); err != nil { + t.Fatalf("drop ledger table: %v", err) + } + + sig := repo.ComputeStallSignal(context.Background()) + if sig.LastEvaluatedUnix != -1 || sig.StallSeconds != -1 { + t.Fatalf("sig = %+v, want -1/-1 for a missing ledger table", sig) + } + if sig.LedgerErr == nil { + t.Fatal("expected LedgerErr, got nil") + } + if sig.ComparandErr != nil { + t.Fatalf("ComparandErr = %v, want nil (never reached)", sig.ComparandErr) + } +} + +// TestComputeStallSignal_ComparandQueryErrorReportsUnknownStall covers spec +// "Writer health": when the ledger query succeeds but the comparand query +// errors, the true last-evaluated instant is still published — it did not +// fail — but the stall figure is unknown (-1) rather than 0, so a +// repeatedly failing comparand read can never report a healthy writer. +func TestComputeStallSignal_ComparandQueryErrorReportsUnknownStall(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + if _, err := db.Exec(`DROP TABLE task_sessions`); err != nil { + t.Fatalf("drop task_sessions: %v", err) + } + + sig := repo.ComputeStallSignal(context.Background()) + if sig.LastEvaluatedUnix == 0 || sig.LastEvaluatedUnix == -1 { + t.Fatalf("last_evaluated_unix = %d, want the true MAX from the still-healthy ledger query", sig.LastEvaluatedUnix) + } + if sig.StallSeconds != -1 { + t.Fatalf("stall_seconds = %d, want -1 (unknown, not 0)", sig.StallSeconds) + } + if sig.LedgerErr != nil { + t.Fatalf("LedgerErr = %v, want nil", sig.LedgerErr) + } + if sig.ComparandErr == nil { + t.Fatal("expected ComparandErr, got nil") + } +} + +// TestSessionsUnattributedGauge_PublishesLatestNotAccumulated replaces a +// prior tautological version of this test (Review round 1 recommended +// item) that only asserted CountUnattributedSessions returns the same +// number when called twice in a row — true of any pure query and never +// touching setSessionsUnattributedGauge (metrics.go:53) at all. This +// drives the gauge through two real Sweep.RunPass calls with a shrinking +// unattributed-session count and asserts the published expvar reflects +// only the latest pass (a Set, not an Add). +func TestSessionsUnattributedGauge_PublishesLatestNotAccumulated(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTaskSession(t, db, "s1", "task-1", "") + seedTaskSession(t, db, "s2", "task-1", "") + seedTaskSession(t, db, "s3", "task-1", "") + + sweep := delivery.NewSweep(repo, nil, nil) + sweep.RunPass(context.Background()) + if got := readExpvarInt(t, "delivery_ledger_sessions_unattributed_total"); got != 3 { + t.Fatalf("gauge after first pass = %d, want 3", got) + } + + if _, err := db.Exec(db.Rebind(`DELETE FROM task_sessions WHERE id IN (?, ?)`), "s2", "s3"); err != nil { + t.Fatalf("delete sessions: %v", err) + } + + sweep.RunPass(context.Background()) + if got := readExpvarInt(t, "delivery_ledger_sessions_unattributed_total"); got != 1 { + t.Fatalf("gauge after second pass = %d, want 1 (a gauge reflects the latest count, not 3+1 accumulated)", got) + } +} diff --git a/apps/backend/internal/delivery/models.go b/apps/backend/internal/delivery/models.go new file mode 100644 index 0000000000..aa591a1a30 --- /dev/null +++ b/apps/backend/internal/delivery/models.go @@ -0,0 +1,95 @@ +// Package delivery implements the task delivery ledger: classifying how, +// and whether, each (task, repository) pair delivered +// (docs/specs/task-delivery-ledger/spec.md). +package delivery + +// Outcome is task_delivery_ledger.delivery_outcome. The zero value is used +// only as a "no outcome computed" sentinel in Go; the stored column is +// NULL for a never-evaluated row, never the empty string. +type Outcome string + +const ( + OutcomeUnknown Outcome = "unknown" + OutcomeNoDeliveryObserved Outcome = "no_delivery_observed" + OutcomeDirectCommit Outcome = "direct_commit" + OutcomePRMerge Outcome = "pr_merge" +) + +// Basis is task_delivery_ledger.delivery_basis. See spec "Basis +// vocabulary". +type Basis string + +const ( + BasisProviderPRMerged Basis = "provider_pr_merged" + BasisDefaultBranchCommit Basis = "default_branch_commit" + BasisReachedDefaultUnattributed Basis = "reached_default_unattributed" + BasisBranchCommitsUnmerged Basis = "branch_commits_unmerged" + BasisNoCommitsObserved Basis = "no_commits_observed" + BasisProviderPRUnmerged Basis = "provider_pr_unmerged" + BasisNoObservations Basis = "no_observations" + BasisDefaultBranchUnknown Basis = "default_branch_unknown" +) + +// ReachedBasis is task_delivery_ledger.reached_default_basis. See spec +// "Basis vocabulary". ReachedBasisPushWebhookDefault is defined for a +// later card; nothing in this package writes it. +type ReachedBasis string + +const ( + ReachedBasisProviderPRMerged ReachedBasis = "provider_pr_merged" + ReachedBasisDefaultBranchCommit ReachedBasis = "default_branch_commit" + ReachedBasisAncestorOfDefault ReachedBasis = "ancestor_of_default" + ReachedBasisPushWebhookDefault ReachedBasis = "push_webhook_default" +) + +// Provider names, used for the ordering tiebreak (spec "Ordering", +// provider name ascending in the order azure_devops, github, gitlab). +const ( + ProviderAzureDevOps = "azure_devops" + ProviderGitHub = "github" + ProviderGitLab = "gitlab" +) + +var providerOrder = map[string]int{ + ProviderAzureDevOps: 0, + ProviderGitHub: 1, + ProviderGitLab: 2, +} + +// evidenceRankTable keys a (Outcome, Basis) pair to its rank (2-8). Rank 0 +// (never evaluated) is a DB default, never computed here. Rank 1 +// (degraded) applies to every (outcome, BasisDefaultBranchUnknown) pair +// and is handled by Rank directly rather than through this table, because +// it is deliberately non-injective (spec "Evidence rank"). +var evidenceRankTable = map[Outcome]map[Basis]int{ + OutcomeUnknown: { + BasisNoObservations: 2, + BasisProviderPRUnmerged: 3, + BasisBranchCommitsUnmerged: 5, + BasisReachedDefaultUnattributed: 6, + }, + OutcomeNoDeliveryObserved: { + BasisNoCommitsObserved: 4, + }, + OutcomeDirectCommit: { + BasisDefaultBranchCommit: 7, + }, + OutcomePRMerge: { + BasisProviderPRMerged: 8, + }, +} + +// Rank returns the evidence_rank for (outcome, basis), per spec "Evidence +// rank". Degraded evaluations (basis == BasisDefaultBranchUnknown) always +// rank 1, regardless of which rule matched. +func Rank(outcome Outcome, basis Basis) int { + if basis == BasisDefaultBranchUnknown { + return 1 + } + if m, ok := evidenceRankTable[outcome]; ok { + if r, ok := m[basis]; ok { + return r + } + } + return 0 +} diff --git a/apps/backend/internal/delivery/observe.go b/apps/backend/internal/delivery/observe.go new file mode 100644 index 0000000000..540b77c9c1 --- /dev/null +++ b/apps/backend/internal/delivery/observe.go @@ -0,0 +1,337 @@ +package delivery + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" + + dbutil "github.com/kandev/kandev/internal/db" +) + +// CandidatePair is a (task, repository) pair eligible for evaluation. +type CandidatePair struct { + TaskID string + RepositoryID string +} + +// CandidateResult is the outcome of one candidacy pass: the resolvable +// pairs plus the two exclusion gauges spec "Candidate pairs" requires — +// each pair is excluded and counted, never silently dropped. +type CandidateResult struct { + Pairs []CandidatePair + MissingRepository int + MissingTask int +} + +type pairKey struct{ taskID, repositoryID string } + +// candidateSourceQueries enumerates every table condition 1 of "Candidate +// pairs" names as a candidacy source. Each is queried independently and +// tolerantly of a missing table (dbutil.IsMissingTableError) so an +// integration whose store never initialized cannot break candidacy for +// every other source. +var candidateSourceQueries = []string{ + `SELECT task_id, repository_id FROM task_repositories WHERE repository_id != ''`, + `SELECT task_id, repository_id FROM task_sessions WHERE repository_id != ''`, + `SELECT task_id, repository_id FROM github_task_prs WHERE repository_id != ''`, + `SELECT task_id, repository_id FROM gitlab_task_mrs WHERE repository_id != ''`, + `SELECT task_id, repository_id FROM azure_devops_task_prs WHERE repository_id != ''`, +} + +// Candidates computes candidacy per spec "Candidate pairs": a pair is a +// candidate only when its repository_id resolves to a repositories row +// AND its task_id resolves to a tasks row. This stage never reads +// task_delivery_ledger, so a missing ledger table cannot suppress it (see +// "Sweep selection predicate § When the ledger itself cannot be read"). +func (r *Repository) Candidates(ctx context.Context) (CandidateResult, error) { + raw, err := r.discoverCandidatePairs(ctx) + if err != nil { + return CandidateResult{}, err + } + repoIDs, err := r.existingIDs(ctx, "repositories") + if err != nil { + return CandidateResult{}, err + } + taskIDs, err := r.existingIDs(ctx, "tasks") + if err != nil { + return CandidateResult{}, err + } + + var result CandidateResult + for k := range raw { + if _, ok := repoIDs[k.repositoryID]; !ok { + result.MissingRepository++ + continue + } + if _, ok := taskIDs[k.taskID]; !ok { + result.MissingTask++ + continue + } + result.Pairs = append(result.Pairs, CandidatePair{TaskID: k.taskID, RepositoryID: k.repositoryID}) + } + return result, nil +} + +func (r *Repository) discoverCandidatePairs(ctx context.Context) (map[pairKey]struct{}, error) { + out := map[pairKey]struct{}{} + for _, q := range candidateSourceQueries { + if err := r.collectPairs(ctx, q, out); err != nil { + if dbutil.IsMissingTableError(err) { + continue + } + return nil, err + } + } + return out, nil +} + +func (r *Repository) collectPairs(ctx context.Context, query string, out map[pairKey]struct{}) error { + rows, err := r.ro.QueryContext(ctx, query) + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var k pairKey + if err := rows.Scan(&k.taskID, &k.repositoryID); err != nil { + return err + } + out[k] = struct{}{} + } + return rows.Err() +} + +// existingIDs is a caller-controlled table-name lookup (never +// user-supplied), used to resolve candidacy conditions 2 and 3. +func (r *Repository) existingIDs(ctx context.Context, table string) (map[string]struct{}, error) { + rows, err := r.ro.QueryContext(ctx, fmt.Sprintf("SELECT id FROM %s", table)) //nolint:gosec // table is a package-internal constant, never user input + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + out := map[string]struct{}{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out[id] = struct{}{} + } + return out, rows.Err() +} + +// RepositoryInfo is the subset of a repositories row the evaluator and +// sweep need. +type RepositoryInfo struct { + DefaultBranch string + LocalPath string + DeletedAt *time.Time + UpdatedAt time.Time +} + +// RepositoryInfo reads the repository row's evaluation-relevant columns. +// Returns sql.ErrNoRows if the id does not resolve (should not happen for +// a pair Candidates() already returned, since that stage just verified +// it, but a repository can be hard-deleted between candidacy and +// evaluation in a maximally adversarial timing). +func (r *Repository) RepositoryInfo(ctx context.Context, id string) (RepositoryInfo, error) { + var row struct { + DefaultBranch sql.NullString `db:"default_branch"` + LocalPath sql.NullString `db:"local_path"` + DeletedAt sql.NullTime `db:"deleted_at"` + UpdatedAt time.Time `db:"updated_at"` + } + err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(` + SELECT default_branch, local_path, deleted_at, updated_at FROM repositories WHERE id = ? + `), id).StructScan(&row) + if err != nil { + return RepositoryInfo{}, err + } + info := RepositoryInfo{ + DefaultBranch: strings.TrimSpace(row.DefaultBranch.String), + LocalPath: row.LocalPath.String, + UpdatedAt: row.UpdatedAt, + } + if row.DeletedAt.Valid { + t := row.DeletedAt.Time + info.DeletedAt = &t + } + return info, nil +} + +// TaskInfo is the subset of a tasks row the evaluator and sweep ordering +// need. +type TaskInfo struct { + WorkspaceID string + CreatedAt time.Time + UpdatedAt time.Time +} + +func (r *Repository) TaskInfo(ctx context.Context, id string) (TaskInfo, error) { + var row struct { + WorkspaceID string `db:"workspace_id"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` + } + err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(` + SELECT workspace_id, created_at, updated_at FROM tasks WHERE id = ? + `), id).StructScan(&row) + if err != nil { + return TaskInfo{}, err + } + return TaskInfo{WorkspaceID: row.WorkspaceID, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil +} + +// SnapshotsForPair reads every task_session_git_snapshots row for the +// pair's sessions, joined through task_sessions. +func (r *Repository) SnapshotsForPair(ctx context.Context, taskID, repositoryID string) ([]Snapshot, error) { + var rows []struct { + SessionID string `db:"session_id"` + Branch string `db:"branch"` + // head_commit is TEXT DEFAULT '', not NOT NULL + // (base_schema.go), and spec "Classification" normalization + // explicitly anticipates a real NULL there — scan into + // sql.NullString rather than a bare string, or a literal NULL + // row fails the whole query (Review round 1, finding #6). + HeadCommit sql.NullString `db:"head_commit"` + Ahead *int `db:"ahead"` + CreatedAt time.Time `db:"created_at"` + } + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT g.session_id, g.branch, g.head_commit, g.ahead, g.created_at + FROM task_session_git_snapshots g + JOIN task_sessions s ON s.id = g.session_id + WHERE s.task_id = ? AND s.repository_id = ? + `), taskID, repositoryID) + if err != nil { + return nil, err + } + out := make([]Snapshot, len(rows)) + for i, row := range rows { + out[i] = Snapshot{ + SessionID: row.SessionID, Branch: row.Branch, HeadCommit: row.HeadCommit.String, + Ahead: row.Ahead, CreatedAt: row.CreatedAt, + } + } + return out, nil +} + +// ProvidersForPair reads every provider pull/merge request row for the +// pair across all three provider tables, normalized to the "Provider +// predicates" table. Tolerant of a missing provider table. +func (r *Repository) ProvidersForPair(ctx context.Context, taskID, repositoryID string) ([]ProviderRequest, error) { + var all []ProviderRequest + for _, fetch := range []func(context.Context, string, string) ([]ProviderRequest, error){ + r.githubPRs, r.gitlabMRs, r.azureDevOpsPRs, + } { + rows, err := fetch(ctx, taskID, repositoryID) + if err != nil { + if dbutil.IsMissingTableError(err) { + continue + } + return nil, err + } + all = append(all, rows...) + } + return all, nil +} + +func (r *Repository) githubPRs(ctx context.Context, taskID, repositoryID string) ([]ProviderRequest, error) { + var rows []struct { + MergedAt sql.NullTime `db:"merged_at"` + DetachedAt sql.NullTime `db:"detached_at"` + PRNumber int `db:"pr_number"` + PRURL string `db:"pr_url"` + BaseBranch string `db:"base_branch"` + Owner string `db:"owner"` + Repo string `db:"repo"` + } + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT merged_at, detached_at, pr_number, pr_url, base_branch, owner, repo + FROM github_task_prs WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID) + if err != nil { + return nil, err + } + out := make([]ProviderRequest, len(rows)) + for i, row := range rows { + out[i] = ProviderRequest{ + Provider: ProviderGitHub, Merged: row.MergedAt.Valid, Detached: row.DetachedAt.Valid, + MergeInstant: row.MergedAt.Time, RequestNumber: row.PRNumber, URL: row.PRURL, + BaseBranch: row.BaseBranch, ScopeValue: row.Owner + "/" + row.Repo, + } + } + return out, nil +} + +func (r *Repository) gitlabMRs(ctx context.Context, taskID, repositoryID string) ([]ProviderRequest, error) { + var rows []struct { + MergedAt sql.NullTime `db:"merged_at"` + MRIID int `db:"mr_iid"` + MRURL string `db:"mr_url"` + BaseBranch string `db:"base_branch"` + ProjectPath string `db:"project_path"` + } + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT merged_at, mr_iid, mr_url, base_branch, project_path + FROM gitlab_task_mrs WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID) + if err != nil { + return nil, err + } + out := make([]ProviderRequest, len(rows)) + for i, row := range rows { + out[i] = ProviderRequest{ + // gitlab_task_mrs has no detached_at column: a GitLab detach + // deletes the row, so Detached is always false here. + Provider: ProviderGitLab, Merged: row.MergedAt.Valid, Detached: false, + MergeInstant: row.MergedAt.Time, RequestNumber: row.MRIID, URL: row.MRURL, + BaseBranch: row.BaseBranch, ScopeValue: row.ProjectPath, + } + } + return out, nil +} + +func (r *Repository) azureDevOpsPRs(ctx context.Context, taskID, repositoryID string) ([]ProviderRequest, error) { + var rows []struct { + Status string `db:"status"` + PullRequestID int `db:"pull_request_id"` + PullRequestURL string `db:"pull_request_url"` + TargetBranch string `db:"target_branch"` + AzureRepositoryID string `db:"azure_repository_id"` + UpdatedAt time.Time `db:"updated_at"` + } + err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` + SELECT status, pull_request_id, pull_request_url, target_branch, azure_repository_id, updated_at + FROM azure_devops_task_prs WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID) + if err != nil { + return nil, err + } + out := make([]ProviderRequest, len(rows)) + for i, row := range rows { + out[i] = ProviderRequest{ + // azure_devops_task_prs has no merged_at: updated_at is the + // merge instant used for ordering only, per spec "Provider + // predicates" — it is never written to reached_default_at. + Provider: ProviderAzureDevOps, Merged: isAzureMergedStatus(row.Status), Detached: false, + MergeInstant: row.UpdatedAt, RequestNumber: row.PullRequestID, URL: row.PullRequestURL, + BaseBranch: row.TargetBranch, ScopeValue: row.AzureRepositoryID, + } + } + return out, nil +} + +// isAzureMergedStatus implements spec "Provider predicates": "completed" +// and "merged" are the merged states; every other value — including an +// unrecognised one — is not-merged, and is never guessed at. +func isAzureMergedStatus(status string) bool { + switch strings.ToLower(strings.TrimSpace(status)) { + case "completed", "merged": + return true + default: + return false + } +} diff --git a/apps/backend/internal/delivery/observe_test.go b/apps/backend/internal/delivery/observe_test.go new file mode 100644 index 0000000000..3fdb9e6b46 --- /dev/null +++ b/apps/backend/internal/delivery/observe_test.go @@ -0,0 +1,295 @@ +package delivery_test + +import ( + "context" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/delivery" +) + +func seedTaskSession(t *testing.T, db *sqlx.DB, id, taskID, repositoryID string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_sessions (id, task_id, repository_id, started_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), id, taskID, repositoryID, now, now); err != nil { + t.Fatalf("seed task_session %s: %v", id, err) + } +} + +func seedTaskRepository(t *testing.T, db *sqlx.DB, id, taskID, repositoryID string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_repositories (id, task_id, repository_id, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), id, taskID, repositoryID, now, now); err != nil { + t.Fatalf("seed task_repository %s: %v", id, err) + } +} + +func seedGitSnapshot(t *testing.T, db *sqlx.DB, id, sessionID, branch, headCommit string, ahead int, createdAt time.Time) { + t.Helper() + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_git_snapshots (id, session_id, snapshot_type, branch, head_commit, ahead, created_at) + VALUES (?, ?, 'test', ?, ?, ?, ?) + `), id, sessionID, branch, headCommit, ahead, createdAt); err != nil { + t.Fatalf("seed snapshot %s: %v", id, err) + } +} + +func TestCandidates_ExcludesMissingRepositoryAndTask(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-real", "ws-1") + seedTask(t, db, "task-real", "ws-1") + + // A valid pair. + seedTaskRepository(t, db, "tr-1", "task-real", "repo-real") + // A pair whose repository row does not exist (dangling repository_id). + seedTaskSession(t, db, "sess-missing-repo", "task-real", "repo-ghost") + // A pair whose task row does not exist (orphaned provider row via a + // session with no matching task — approximated here via task_sessions + // since seeding an orphaned github_task_prs row needs the github + // schema; task_sessions has no FK to tasks either). + + ctx := context.Background() + result, err := repo.Candidates(ctx) + if err != nil { + t.Fatalf("Candidates: %v", err) + } + if len(result.Pairs) != 1 || result.Pairs[0].TaskID != "task-real" || result.Pairs[0].RepositoryID != "repo-real" { + t.Fatalf("pairs = %+v, want exactly the valid pair", result.Pairs) + } + if result.MissingRepository != 1 { + t.Fatalf("MissingRepository = %d, want 1", result.MissingRepository) + } +} + +// TestCandidates_ExcludesMissingTask covers spec "Candidate pairs", "Why +// condition 3 exists": a github_task_prs row whose task_id matches no tasks +// row — a hard-deleted task the GitHub task-deleted subscriber did not +// prune the provider row for — is excluded from candidacy and counted via +// MissingTask, never attempted, exactly parallel to the MissingRepository +// case above but reachable only through condition 3 (task_sessions has no +// FK to tasks either, but the spec's own example is a provider row). +func TestCandidates_ExcludesMissingTask(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-real", "ws-1") + seedGitHubStore(t, db) + now := time.Now().UTC() + seedGitHubPR(t, db, "pr-orphan", "task-ghost", "repo-real", "acme", "widgets", + 1, "https://gh/1", "main", &now, nil) + + ctx := context.Background() + result, err := repo.Candidates(ctx) + if err != nil { + t.Fatalf("Candidates: %v", err) + } + for _, p := range result.Pairs { + if p.TaskID == "task-ghost" { + t.Fatalf("orphaned-task pair must not be a candidate, got %+v", p) + } + } + if result.MissingTask != 1 { + t.Fatalf("MissingTask = %d, want 1", result.MissingTask) + } +} + +func TestCandidates_TaskWithNoRepositoryProducesNoRow(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedTask(t, db, "task-bare", "ws-1") + // No task_repositories, task_sessions, or provider row references + // task-bare at all. + + ctx := context.Background() + result, err := repo.Candidates(ctx) + if err != nil { + t.Fatalf("Candidates: %v", err) + } + for _, p := range result.Pairs { + if p.TaskID == "task-bare" { + t.Fatalf("task with no repository must produce no candidate pair, got %+v", p) + } + } +} + +func TestSnapshotsForPair_JoinsThroughSessions(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTaskSession(t, db, "sess-1", "task-1", "repo-1") + seedTaskSession(t, db, "sess-other-repo", "task-1", "repo-other") + + at0 := time.Now().UTC().Add(-time.Hour) + seedGitSnapshot(t, db, "snap-1", "sess-1", "feature", "aaa", 3, at0) + seedGitSnapshot(t, db, "snap-2", "sess-other-repo", "feature", "zzz", 9, at0) + + ctx := context.Background() + snaps, err := repo.SnapshotsForPair(ctx, "task-1", "repo-1") + if err != nil { + t.Fatalf("SnapshotsForPair: %v", err) + } + if len(snaps) != 1 || snaps[0].HeadCommit != "aaa" { + t.Fatalf("snaps = %+v, want exactly snap-1", snaps) + } +} + +// TestSnapshotsForPair_NullHeadCommitNormalizesToEmpty covers Review round +// 1, finding #6: task_session_git_snapshots.head_commit defaults to an +// empty string but is not NOT NULL (base_schema.go), and spec +// "Classification"'s normalization rule explicitly anticipates a real SQL +// NULL there ("A head_commit that is NULL, empty, or whitespace-only is +// not a distinct head value"). Scanning the column into a bare Go string +// instead of sql.NullString used to fail the whole query on a literal +// NULL row, aborting the pair's evaluation rather than normalizing to +// empty. seedGitSnapshot cannot express a literal NULL (a Go empty string +// binds as a SQL empty string, not NULL), so this test inserts one +// directly. +func TestSnapshotsForPair_NullHeadCommitNormalizesToEmpty(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTaskSession(t, db, "sess-1", "task-1", "repo-1") + + at0 := time.Now().UTC().Add(-time.Hour) + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_session_git_snapshots (id, session_id, snapshot_type, branch, head_commit, ahead, created_at) + VALUES (?, ?, 'test', ?, NULL, ?, ?) + `), "snap-null", "sess-1", "feature", 2, at0); err != nil { + t.Fatalf("seed null-head snapshot: %v", err) + } + + ctx := context.Background() + snaps, err := repo.SnapshotsForPair(ctx, "task-1", "repo-1") + if err != nil { + t.Fatalf("SnapshotsForPair must tolerate a literal NULL head_commit, got: %v", err) + } + if len(snaps) != 1 || snaps[0].HeadCommit != "" { + t.Fatalf("snaps = %+v, want exactly one snapshot with HeadCommit normalized to empty", snaps) + } +} + +func TestProvidersForPair_MissingTablesTolerated(t *testing.T) { + // A database with only the task schema (no github/gitlab/azuredevops + // store initialized) must not error; it simply contributes no rows. + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + _ = db + + ctx := context.Background() + prs, err := repo.ProvidersForPair(ctx, "task-1", "repo-1") + if err != nil { + t.Fatalf("ProvidersForPair must tolerate missing provider tables, got: %v", err) + } + if len(prs) != 0 { + t.Fatalf("prs = %+v, want none", prs) + } +} + +func TestGitHubPRs_MergedAndDetachedFromColumns(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedGitHubStore(t, db) + + now := time.Now().UTC() + seedGitHubPR(t, db, "pr-1", "task-1", "repo-1", "acme", "widgets", 42, "https://gh/1", "main", &now, nil) + seedGitHubPR(t, db, "pr-2", "task-1", "repo-1", "acme", "widgets", 43, "https://gh/2", "main", &now, &now) + + ctx := context.Background() + prs, err := repo.ProvidersForPair(ctx, "task-1", "repo-1") + if err != nil { + t.Fatalf("ProvidersForPair: %v", err) + } + if len(prs) != 2 { + t.Fatalf("prs = %+v, want 2", prs) + } + byURL := map[string]delivery.ProviderRequest{} + for _, p := range prs { + byURL[p.URL] = p + } + if !byURL["https://gh/1"].Merged || byURL["https://gh/1"].Detached { + t.Fatalf("pr-1 = %+v, want merged and not detached", byURL["https://gh/1"]) + } + if !byURL["https://gh/2"].Merged || !byURL["https://gh/2"].Detached { + t.Fatalf("pr-2 = %+v, want merged and detached", byURL["https://gh/2"]) + } + if byURL["https://gh/1"].ScopeValue != "acme/widgets" { + t.Fatalf("scope value = %q, want acme/widgets", byURL["https://gh/1"].ScopeValue) + } +} + +func TestGitLabMRs_MergedFromColumnAndScopeIsProjectPath(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedGitLabStore(t, db) + + now := time.Now().UTC() + seedGitLabMR(t, db, "mr-1", "task-1", "repo-1", "group/project", 7, "https://gitlab/mr/7", "main", &now) + seedGitLabMR(t, db, "mr-2", "task-1", "repo-1", "group/project", 8, "https://gitlab/mr/8", "main", nil) + + ctx := context.Background() + prs, err := repo.ProvidersForPair(ctx, "task-1", "repo-1") + if err != nil { + t.Fatalf("ProvidersForPair: %v", err) + } + byURL := map[string]delivery.ProviderRequest{} + for _, p := range prs { + byURL[p.URL] = p + } + merged, ok := byURL["https://gitlab/mr/7"] + if !ok || !merged.Merged || merged.Detached { + t.Fatalf("mr-1 = %+v, want merged and never detached (GitLab has no detached_at column)", merged) + } + if merged.ScopeValue != "group/project" { + t.Fatalf("scope value = %q, want group/project", merged.ScopeValue) + } + if unmerged := byURL["https://gitlab/mr/8"]; unmerged.Merged { + t.Fatalf("mr-2 = %+v, want not merged (NULL merged_at)", unmerged) + } +} + +func TestAzureDevOpsPRs_StatusCaseInsensitiveAndUnrecognisedIsNotMerged(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedAzureDevOpsStore(t, db) + + seedAzurePR(t, db, "az-1", "task-1", "repo-1", "azrepo-1", 1, "https://az/1", "main", "Completed") + seedAzurePR(t, db, "az-2", "task-1", "repo-1", "azrepo-1", 2, "https://az/2", "main", "abandoned") + seedAzurePR(t, db, "az-3", "task-1", "repo-1", "azrepo-1", 3, "https://az/3", "main", "some-future-status") + + ctx := context.Background() + prs, err := repo.ProvidersForPair(ctx, "task-1", "repo-1") + if err != nil { + t.Fatalf("ProvidersForPair: %v", err) + } + byURL := map[string]delivery.ProviderRequest{} + for _, p := range prs { + byURL[p.URL] = p + } + if !byURL["https://az/1"].Merged { + t.Fatal("Completed (case-insensitive) must be merged") + } + if byURL["https://az/2"].Merged { + t.Fatal("abandoned must not be merged") + } + if byURL["https://az/3"].Merged { + t.Fatal("an unrecognised status must not be merged and must not error") + } +} diff --git a/apps/backend/internal/delivery/postgres_schema_test.go b/apps/backend/internal/delivery/postgres_schema_test.go new file mode 100644 index 0000000000..3a57f10650 --- /dev/null +++ b/apps/backend/internal/delivery/postgres_schema_test.go @@ -0,0 +1,189 @@ +package delivery_test + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/delivery" + "github.com/kandev/kandev/internal/persistence" + taskrepo "github.com/kandev/kandev/internal/task/repository" + "github.com/kandev/kandev/internal/testutil" +) + +// TestPostgresDeliveryLedgerMigration_FreshAndReplay is the PostgreSQL +// twin ADR 0027 requires for every schema-touching change: fresh init and +// replay both succeed, the table and its foreign keys land as declared, +// and the activation key is written. The task schema (tasks, +// repositories — the ledger's FK targets) must exist before +// delivery.NewWithDB runs: unlike SQLite, PostgreSQL requires FK target +// tables to exist at CREATE TABLE time. Skips unless +// KANDEV_TEST_POSTGRES_DSN is set. +func TestPostgresDeliveryLedgerMigration_FreshAndReplay(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + if _, _, err := taskrepo.Provide(db, db, nil); err != nil { + t.Fatalf("init task schema: %v", err) + } + + if _, err := delivery.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init delivery schema: %v", err) + } + // Replay against the same database. + if _, err := delivery.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay delivery schema: %v", err) + } + + var exists bool + err := db.QueryRow(` + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = 'task_delivery_ledger' + ) + `).Scan(&exists) + if err != nil { + t.Fatalf("inspect task_delivery_ledger: %v", err) + } + if !exists { + t.Fatal("task_delivery_ledger table missing after init") + } + + var fkCount int + err = db.QueryRow(` + SELECT COUNT(*) FROM information_schema.table_constraints + WHERE table_schema = current_schema() + AND table_name = 'task_delivery_ledger' + AND constraint_type = 'FOREIGN KEY' + `).Scan(&fkCount) + if err != nil { + t.Fatalf("inspect foreign keys: %v", err) + } + if fkCount != 2 { + t.Fatalf("foreign key count = %d, want 2 (task_id, repository_id)", fkCount) + } + + val, err := persistence.ReadMetaKey(db, "telemetry.delivery_ledger.activated_at") + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if val == "" { + t.Fatal("expected activation key to be written on postgres") + } +} + +// TestPostgresUpsert_RankGuardedBehavior is Review round 2, finding #6: +// Repository.Upsert's dialect-sensitive rank-guard/high-water/write-once +// SQL (upsert.go's CASE expressions, chosen specifically because SQLite's +// two-arg max() and Postgres's GREATEST handle NULL differently — see +// highWaterObservedExpr's comment) had only a schema-existence Postgres +// test, never a behavioral one, despite ADR 0027 requiring a dialect-gated +// behavior test for every dialect-sensitive method. This drives the same +// scenarios as the SQLite-only tests in upsert_test.go — fresh insert, +// demotion suppression, equal-rank ref reselection, and the high-water +// NULL floor in both directions — against a real Postgres instance so a +// SQLite-only regression (e.g. IS DISTINCT FROM or the CASE expressions +// behaving differently under Postgres's stricter typing) cannot hide +// behind SQLite-only coverage. Skips unless KANDEV_TEST_POSTGRES_DSN is +// set. +func TestPostgresUpsert_RankGuardedBehavior(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + if _, _, err := taskrepo.Provide(db, db, nil); err != nil { + t.Fatalf("init task schema: %v", err) + } + repo, err := delivery.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init delivery schema: %v", err) + } + + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-a", "ws-1") + seedTask(t, db, "task-b", "ws-1") + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + // Fresh insert. + prA := "https://pr/A" + result, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: "task-a", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{ + Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Ref: &prA, Rank: 8, + ObservedBranchCommits: intp(12), + }, + EvaluatedAt: t0, + }) + if err != nil { + t.Fatalf("fresh insert: %v", err) + } + if !result.RowChanged { + t.Fatal("fresh insert must report RowChanged") + } + row := readLedgerRow(t, db, "task-a", "repo-1") + if row.Outcome.String != string(delivery.OutcomePRMerge) || row.Rank != 8 || row.Ref.String != prA { + t.Fatalf("row after fresh insert = %+v", row) + } + + // Demotion suppressed: a lower-rank re-evaluation, and its NULL + // observed_branch_commits, must not move the stored higher-rank row — + // both directions of the rank guard and the high-water floor at once. + result, err = repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: "task-a", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, + ObservedBranchCommits: nil, + }, + EvaluatedAt: t0.Add(time.Minute), + }) + if err != nil { + t.Fatalf("demotion upsert: %v", err) + } + if !result.Demoted || result.RowChanged { + t.Fatalf("result = %+v, want Demoted=true RowChanged=false", result) + } + row = readLedgerRow(t, db, "task-a", "repo-1") + if row.Outcome.String != string(delivery.OutcomePRMerge) || row.Rank != 8 || row.Ref.String != prA { + t.Fatalf("row after suppressed demotion = %+v, want unchanged pr_merge at rank 8", row) + } + if !row.ObservedBranchCommits.Valid || row.ObservedBranchCommits.Int64 != 12 { + t.Fatalf("observed_branch_commits = %v, want unchanged 12 (NULL incoming must not lower a stored value)", row.ObservedBranchCommits) + } + + // Equal-rank ref reselection: another rank-8 evaluation with a + // different ref must overwrite, never guaranteed-unchanged. + prB := "https://pr/B" + result, err = repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: "task-a", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Ref: &prB, Rank: 8}, + EvaluatedAt: t0.Add(2 * time.Minute), + }) + if err != nil { + t.Fatalf("equal-rank upsert: %v", err) + } + if !result.RowChanged { + t.Fatal("a ref change at equal rank must report RowChanged") + } + row = readLedgerRow(t, db, "task-a", "repo-1") + if row.Ref.String != prB { + t.Fatalf("ref = %q, want prB after equal-rank reselection", row.Ref.String) + } + + // High-water floor, other direction: a NULL stored observed_branch_commits + // must accept a later real value rather than swallowing it. + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: "task-b", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2, ObservedBranchCommits: nil}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("task-b fresh insert: %v", err) + } + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: "task-b", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, ObservedBranchCommits: intp(7)}, + EvaluatedAt: t0.Add(time.Minute), + }); err != nil { + t.Fatalf("task-b second upsert: %v", err) + } + row = readLedgerRow(t, db, "task-b", "repo-1") + if !row.ObservedBranchCommits.Valid || row.ObservedBranchCommits.Int64 != 7 { + t.Fatalf("observed_branch_commits = %v, want 7 (a NULL stored value must not swallow an incoming real value)", row.ObservedBranchCommits) + } +} diff --git a/apps/backend/internal/delivery/provider.go b/apps/backend/internal/delivery/provider.go new file mode 100644 index 0000000000..87d3b2460a --- /dev/null +++ b/apps/backend/internal/delivery/provider.go @@ -0,0 +1,41 @@ +package delivery + +import ( + "context" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/common/logger" +) + +// Provide constructs the delivery ledger repository, wires the ancestry +// checker to the supplied checkout resolver, and starts the sweep +// immediately (spec "Boundary values": first sweep runs at boot, no +// initial delay). writer and reader must already have the tasks and +// repositories tables present — Provide must therefore be called after +// task/repository.Provide in the boot sequence, since the ledger's +// foreign keys require those tables to exist at CREATE TABLE time on +// PostgreSQL. checkout may be nil (e.g. in a deployment with no task +// service wired), in which case the ancestry check is never attempted +// and every pair's default-branch observation is limited to the +// provider/direct-commit bases. +// +// Returns the Sweep (exposed for tests/diagnostics) and a cleanup +// function that stops it. +func Provide(writer, reader *sqlx.DB, checkout CheckoutResolver, log *logger.Logger) (*Sweep, func() error, error) { + repo, err := NewWithDB(writer, reader, log) + if err != nil { + return nil, nil, err + } + var ancestry *AncestryChecker + if checkout != nil { + ancestry = &AncestryChecker{Checkout: checkout} + } + sweep := NewSweep(repo, ancestry, log) + sweep.Start(context.Background()) + cleanup := func() error { + sweep.Stop() + return nil + } + return sweep, cleanup, nil +} diff --git a/apps/backend/internal/delivery/provider_seed_test.go b/apps/backend/internal/delivery/provider_seed_test.go new file mode 100644 index 0000000000..fd9b5e4c80 --- /dev/null +++ b/apps/backend/internal/delivery/provider_seed_test.go @@ -0,0 +1,82 @@ +package delivery_test + +import ( + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/azuredevops" + "github.com/kandev/kandev/internal/github" + "github.com/kandev/kandev/internal/gitlab" +) + +func seedGitHubStore(t *testing.T, db *sqlx.DB) { + t.Helper() + if _, err := github.NewStore(db, db); err != nil { + t.Fatalf("init github store: %v", err) + } +} + +func seedGitHubPR( + t *testing.T, db *sqlx.DB, id, taskID, repositoryID, owner, repo string, + prNumber int, prURL, baseBranch string, mergedAt, detachedAt *time.Time, +) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO github_task_prs ( + id, task_id, repository_id, owner, repo, pr_number, pr_url, pr_title, + head_branch, base_branch, author_login, created_at, merged_at, detached_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, '', '', ?, '', ?, ?, ?, ?) + `), id, taskID, repositoryID, owner, repo, prNumber, prURL, baseBranch, now, mergedAt, detachedAt, now); err != nil { + t.Fatalf("seed github_task_prs %s: %v", id, err) + } +} + +func seedGitLabStore(t *testing.T, db *sqlx.DB) { + t.Helper() + if _, err := gitlab.NewStore(db, db); err != nil { + t.Fatalf("init gitlab store: %v", err) + } +} + +func seedGitLabMR( + t *testing.T, db *sqlx.DB, id, taskID, repositoryID, projectPath string, + mrIID int, mrURL, baseBranch string, mergedAt *time.Time, +) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO gitlab_task_mrs ( + id, task_id, repository_id, project_path, mr_iid, mr_url, mr_title, + head_branch, base_branch, created_at, merged_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, '', '', ?, ?, ?, ?) + `), id, taskID, repositoryID, projectPath, mrIID, mrURL, baseBranch, now, mergedAt, now); err != nil { + t.Fatalf("seed gitlab_task_mrs %s: %v", id, err) + } +} + +func seedAzureDevOpsStore(t *testing.T, db *sqlx.DB) { + t.Helper() + if _, err := azuredevops.NewStore(db, db); err != nil { + t.Fatalf("init azuredevops store: %v", err) + } +} + +func seedAzurePR( + t *testing.T, db *sqlx.DB, id, taskID, repositoryID, azureRepositoryID string, + pullRequestID int, pullRequestURL, targetBranch, status string, +) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO azure_devops_task_prs ( + id, task_id, repository_id, organization_url, project_id, azure_repository_id, + pull_request_id, pull_request_url, title, source_branch, target_branch, + author_id, author_name, status, created_at, updated_at + ) VALUES (?, ?, ?, '', '', ?, ?, ?, '', '', ?, '', '', ?, ?, ?) + `), id, taskID, repositoryID, azureRepositoryID, pullRequestID, pullRequestURL, targetBranch, status, now, now); err != nil { + t.Fatalf("seed azure_devops_task_prs %s: %v", id, err) + } +} diff --git a/apps/backend/internal/delivery/store.go b/apps/backend/internal/delivery/store.go new file mode 100644 index 0000000000..3fcc4e3478 --- /dev/null +++ b/apps/backend/internal/delivery/store.go @@ -0,0 +1,156 @@ +package delivery + +import ( + "database/sql" + "errors" + "fmt" + "time" + + "github.com/jmoiron/sqlx" + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/db/dialect" + "github.com/kandev/kandev/internal/persistence" +) + +// Repository owns the task_delivery_ledger table: its schema, its +// migration, its activation instant, and the single upsert statement that +// is the only writer. See docs/specs/task-delivery-ledger/spec.md, "Data +// model". +type Repository struct { + db *sqlx.DB // writer + ro *sqlx.DB // reader + log *logger.Logger + migrate *db.MigrateLogger +} + +// activationKey is the kandev_meta key published once a boot has verified +// task_delivery_ledger is present and queryable. See spec "Activation +// points". +const activationKey = "telemetry.delivery_ledger.activated_at" + +// createTableSQL declares the FK on repository_id ON DELETE CASCADE +// (Build decision R5-F2, the highest-risk of the spec's open gaps): the +// spec's repository_id row states no ON DELETE clause, but repositories +// itself cascades from workspaces (FOREIGN KEY (workspace_id) REFERENCES +// workspaces(id) ON DELETE CASCADE), and DELETE FROM workspaces is a live +// production path (internal/task/repository/sqlite/workspace.go). A bare +// FK (dialect default NO ACTION) would make workspace deletion start +// failing the moment any ledger row exists, invisibly, because the +// migration runner swallows errors. CASCADE matches task_id's own +// documented clause and the spec's persistence guarantee that deleting a +// task deletes its ledger rows. +// +// Both this table and its FK targets (tasks, repositories) must already +// exist at CREATE TABLE time on PostgreSQL (unlike SQLite, which allows +// forward-declared FK targets), so Provide must run after +// task/repository.Provide in the boot sequence. +const createTableSQL = ` +CREATE TABLE IF NOT EXISTS task_delivery_ledger ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + repository_id TEXT NOT NULL, + workspace_id TEXT NOT NULL, + delivery_outcome TEXT, + delivery_basis TEXT, + delivery_ref TEXT, + evidence_rank INTEGER NOT NULL DEFAULT 0, + reached_default_at TIMESTAMP, + reached_default_basis TEXT, + reached_default_ref TEXT, + observed_branch_commits INTEGER, + first_classified_at TIMESTAMP, + last_evaluated_at TIMESTAMP NOT NULL, + evaluation_seq INTEGER NOT NULL, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + UNIQUE(task_id, repository_id), + FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE, + FOREIGN KEY (repository_id) REFERENCES repositories(id) ON DELETE CASCADE +)` + +const createIndexSQL = ` +CREATE INDEX IF NOT EXISTS idx_task_delivery_ledger_workspace_evaluated + ON task_delivery_ledger(workspace_id, last_evaluated_at)` + +// NewWithDB creates the delivery ledger repository and initializes its +// schema. writer and reader must already have the tasks and repositories +// tables present (see createTableSQL). +func NewWithDB(writer, reader *sqlx.DB, log *logger.Logger) (*Repository, error) { + r := &Repository{ + db: writer, + ro: reader, + log: log, + migrate: db.NewMigrateLogger(writer, log), + } + if err := r.initSchema(); err != nil { + return nil, fmt.Errorf("delivery ledger schema init: %w", err) + } + return r, nil +} + +func (r *Repository) initSchema() error { + r.migrate.Apply("task_delivery_ledger.table", createTableSQL) + r.migrate.Apply("task_delivery_ledger.idx_workspace_evaluated", createIndexSQL) + r.activate() + return nil +} + +// activate writes activationKey only after a positive probe confirms +// task_delivery_ledger exists. See spec "Activation points": the +// migration runner swallows failures at WARN, so this probe is the only +// thing standing between a failed migration and a consumer wrongly +// believing the mechanism is live. WriteMetaKeyIfAbsent makes the write +// replay-safe. Any failure here is logged and swallowed — activation is a +// published fact for consumers, never a boot-blocking requirement, and it +// does not gate the sweep. +func (r *Repository) activate() { + if err := persistence.EnsureMetaTable(r.db); err != nil { + r.logWarn("ensure kandev_meta table failed", err) + return + } + exists, err := tableExists(r.db, "task_delivery_ledger") + if err != nil { + r.logWarn("task_delivery_ledger schema probe failed", err) + return + } + if !exists { + return + } + if _, err := persistence.WriteMetaKeyIfAbsent( + r.db, activationKey, time.Now().UTC().Format(time.RFC3339), + ); err != nil { + r.logWarn("write delivery ledger activation key failed", err) + } +} + +func (r *Repository) logWarn(msg string, err error) { + if r.log != nil { + r.log.Warn(msg, zap.Error(err)) + } +} + +// tableExists reports whether table exists, on either dialect. +func tableExists(conn *sqlx.DB, table string) (bool, error) { + if dialect.IsPostgres(conn.DriverName()) { + var exists bool + err := conn.QueryRow(conn.Rebind(` + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = current_schema() AND table_name = ? + ) + `), table).Scan(&exists) + return exists, err + } + var name string + err := conn.QueryRow(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`, table).Scan(&name) + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} diff --git a/apps/backend/internal/delivery/store_test.go b/apps/backend/internal/delivery/store_test.go new file mode 100644 index 0000000000..e5789ab7c5 --- /dev/null +++ b/apps/backend/internal/delivery/store_test.go @@ -0,0 +1,161 @@ +package delivery_test + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + dbutil "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/delivery" + "github.com/kandev/kandev/internal/persistence" + taskrepo "github.com/kandev/kandev/internal/task/repository" +) + +// newTestDB opens a file-backed SQLite database with foreign key +// enforcement on (required for the R5-F2 cascade behaviour under test), +// initializes the task schema (workspaces/repositories/tasks, the FK +// targets), and returns the raw handle. +func newTestDB(t *testing.T) *sqlx.DB { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "delivery-ledger.db") + rawConn, err := dbutil.OpenSQLite(dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db := sqlx.NewDb(rawConn, "sqlite3") + t.Cleanup(func() { _ = db.Close() }) + + if _, _, err := taskrepo.Provide(db, db, nil); err != nil { + t.Fatalf("init task schema: %v", err) + } + return db +} + +// newTestRepo returns a delivery repository and the underlying db handle. +func newTestRepo(t *testing.T) (*delivery.Repository, *sqlx.DB) { + t.Helper() + db := newTestDB(t) + repo, err := delivery.NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init delivery schema: %v", err) + } + return repo, db +} + +func seedWorkspace(t *testing.T, db *sqlx.DB, id string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO workspaces (id, name, created_at, updated_at) VALUES (?, ?, ?, ?) + `), id, id, now, now); err != nil { + t.Fatalf("seed workspace %s: %v", id, err) + } +} + +func seedRepository(t *testing.T, db *sqlx.DB, id, workspaceID string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO repositories (id, workspace_id, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), id, workspaceID, id, now, now); err != nil { + t.Fatalf("seed repository %s: %v", id, err) + } +} + +func seedTask(t *testing.T, db *sqlx.DB, id, workspaceID string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), id, workspaceID, id, now, now); err != nil { + t.Fatalf("seed task %s: %v", id, err) + } +} + +func TestNewWithDB_CreatesTableAndActivates(t *testing.T) { + _, db := newTestRepo(t) + + var name string + if err := db.Get(&name, `SELECT name FROM sqlite_master WHERE type='table' AND name='task_delivery_ledger'`); err != nil { + t.Fatalf("table missing: %v", err) + } + + val, err := persistence.ReadMetaKey(db, "telemetry.delivery_ledger.activated_at") + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if val == "" { + t.Fatal("expected telemetry.delivery_ledger.activated_at to be written after first boot") + } + if _, err := time.Parse(time.RFC3339, val); err != nil { + t.Fatalf("activation value %q not RFC3339: %v", val, err) + } +} + +func TestNewWithDB_ReplaySafeAndActivationNeverOverwritten(t *testing.T) { + db := newTestDB(t) + if _, err := delivery.NewWithDB(db, db, nil); err != nil { + t.Fatalf("first init: %v", err) + } + if _, err := db.Exec(db.Rebind(`UPDATE kandev_meta SET value = ? WHERE key = ?`), + "sentinel-value", "telemetry.delivery_ledger.activated_at"); err != nil { + t.Fatalf("seed sentinel: %v", err) + } + + if _, err := delivery.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay 1: %v", err) + } + if _, err := delivery.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay 2: %v", err) + } + + got, err := persistence.ReadMetaKey(db, "telemetry.delivery_ledger.activated_at") + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if got != "sentinel-value" { + t.Fatalf("activation key = %q after replay, want unchanged sentinel-value", got) + } +} + +// TestWorkspaceDeletion_CascadesLedgerRows is the R5-F2 regression test: +// deleting a workspace must not start failing once a ledger row exists, +// because repositories cascade from workspaces and the ledger's +// repository_id FK must cascade transitively. +func TestWorkspaceDeletion_CascadesLedgerRows(t *testing.T) { + repo, db := newTestRepo(t) + taskRepo, _, err := taskrepo.Provide(db, db, nil) + if err != nil { + t.Fatalf("task repo: %v", err) + } + + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + + ctx := context.Background() + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + if err := taskRepo.DeleteWorkspace(ctx, "ws-1"); err != nil { + t.Fatalf("delete workspace: %v (workspace deletion must not fail once a ledger row exists)", err) + } + + var count int + if err := db.Get(&count, `SELECT COUNT(*) FROM task_delivery_ledger`); err != nil { + t.Fatalf("count ledger rows: %v", err) + } + if count != 0 { + t.Fatalf("ledger rows after workspace delete = %d, want 0 (cascade)", count) + } +} diff --git a/apps/backend/internal/delivery/sweep.go b/apps/backend/internal/delivery/sweep.go new file mode 100644 index 0000000000..31ebbf9280 --- /dev/null +++ b/apps/backend/internal/delivery/sweep.go @@ -0,0 +1,447 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "sort" + "sync" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" + dbutil "github.com/kandev/kandev/internal/db" +) + +// Boundary values fixed by spec "Evaluation triggers and cadence § +// Boundary values" so they are not invented downstream. +const ( + // SweepInterval is completion-relative: the next pass starts this long + // after the previous pass finished, never fixed-rate. + SweepInterval = 5 * time.Minute + // StaleRefreshInterval is the "Stale refresh" due condition's window. + StaleRefreshInterval = 24 * time.Hour + // StallThreshold is the writer-health stall threshold. + StallThreshold = 15 * time.Minute +) + +// Sweep owns the periodic evaluation loop: Start/Stop lifecycle, +// completion-relative cadence, and no overlapping passes — a single +// goroutine running one pass at a time (spec "Boundary values", +// "Concurrent sweep passes: never"). Mirrors the healthpoll.Poller +// Start/Stop shape. +type Sweep struct { + repo *Repository + ancestry *AncestryChecker + log *logger.Logger + + mu sync.Mutex + cancel context.CancelFunc + wg sync.WaitGroup + started bool +} + +// NewSweep constructs a Sweep. ancestry may be nil, in which case the +// ancestry check is never attempted and every pair's default-branch +// observation is limited to the provider/direct-commit bases. +func NewSweep(repo *Repository, ancestry *AncestryChecker, log *logger.Logger) *Sweep { + return &Sweep{repo: repo, ancestry: ancestry, log: log} +} + +// Start launches the background loop, running the first pass immediately +// (spec: "First sweep: runs at boot ... with no initial delay"). Calling +// Start more than once without Stop is a no-op. +func (s *Sweep) Start(ctx context.Context) { + s.mu.Lock() + defer s.mu.Unlock() + if s.started { + return + } + s.started = true + ctx, s.cancel = context.WithCancel(ctx) + s.wg.Add(1) + go s.loop(ctx) +} + +// Stop cancels the loop and waits for the in-flight pass, if any, to +// finish. +func (s *Sweep) Stop() { + s.mu.Lock() + if !s.started { + s.mu.Unlock() + return + } + cancel := s.cancel + s.mu.Unlock() + if cancel != nil { + cancel() + } + s.wg.Wait() + s.mu.Lock() + s.started = false + s.mu.Unlock() +} + +func (s *Sweep) loop(ctx context.Context) { + defer s.wg.Done() + s.RunPass(ctx) + for { + timer := time.NewTimer(SweepInterval) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + s.RunPass(ctx) + } + } +} + +// RunPass executes one complete sweep pass: candidacy, dueness (with the +// missing-ledger fallback), ordering, per-pair evaluation, and the +// writer-health publication at the end. Exported so tests and a manual +// trigger can drive a single deterministic pass. +func (s *Sweep) RunPass(ctx context.Context) { + passStart := time.Now().UTC() + + candidates, err := s.repo.Candidates(ctx) + if err != nil { + s.warn("delivery_ledger.candidates_unavailable", zap.Error(err)) + return + } + setPairsMissingRepositoryGauge(candidates.MissingRepository) + setPairsMissingTaskGauge(candidates.MissingTask) + + due, fallback, ledgerErr := s.repo.SelectDuePairs(ctx, candidates.Pairs, passStart) + if fallback { + s.warn("delivery_ledger.dueness_unavailable", zap.Error(ledgerErr), zap.Int("candidates", len(candidates.Pairs))) + } + + ordered := s.repo.OrderPairs(ctx, due) + + writeErrors := 0 + var lastWriteErr error + for _, pair := range ordered { + if err := s.evaluatePair(ctx, pair); err != nil { + writeErrors++ + lastWriteErr = err + } + } + if writeErrors > 0 { + s.warn("delivery_ledger.write_failed", zap.Error(lastWriteErr), zap.Int("count", writeErrors)) + } + + if n, err := s.repo.CountUnattributedSessions(ctx); err == nil { + setSessionsUnattributedGauge(n) + } + + s.publishWriterHealth(ctx) +} + +func (s *Sweep) publishWriterHealth(ctx context.Context) { + sig := s.repo.ComputeStallSignal(ctx) + publishStallSignal(sig) + switch { + case sig.LedgerErr != nil: + s.warn("delivery_ledger.unavailable", zap.Error(sig.LedgerErr)) + case sig.ComparandErr != nil: + s.warn("delivery_ledger.comparand_unavailable", zap.Error(sig.ComparandErr)) + case sig.StallSeconds > int64(StallThreshold.Seconds()): + s.warn("delivery_ledger.stalled", + zap.Int64("last_evaluated_unix", sig.LastEvaluatedUnix), + zap.Int64("stall_seconds", sig.StallSeconds)) + } +} + +func (s *Sweep) warn(msg string, fields ...zap.Field) { + if s.log != nil { + s.log.Warn(msg, fields...) + } +} + +// evaluatePair gathers every input for one pair, computes the ancestry +// precondition/check, classifies, and upserts. Returns an error only for +// the upsert-failure mode (spec "Failure modes"); an input-query failure +// abandons the evaluation (no column written, pair stays due) and is +// counted by delivery_ledger_evaluation_errors_total, not returned as a +// pass-level write error. +func (s *Sweep) evaluatePair(ctx context.Context, pair CandidatePair) error { + evaluatedAt := time.Now().UTC() // T0: the instant this evaluation began reading its inputs + + snapshots, err := s.repo.SnapshotsForPair(ctx, pair.TaskID, pair.RepositoryID) + if err != nil { + recordEvaluationError() + return nil + } + providers, err := s.repo.ProvidersForPair(ctx, pair.TaskID, pair.RepositoryID) + if err != nil { + recordEvaluationError() + return nil + } + repoInfo, err := s.repo.RepositoryInfo(ctx, pair.RepositoryID) + if err != nil { + recordEvaluationError() + return nil + } + // Re-check the freeze: SelectDuePairs enforced it at due-selection + // time, but the two reads are not in the same transaction, so a + // repository soft-deleted in the window between due-selection and + // this evaluation must still be honored here (Review round 1, + // finding #5) — otherwise the freeze promised under "Persistence + // guarantees" can be silently voided by an ordinary race. + if repoInfo.DeletedAt != nil { + return nil + } + taskInfo, err := s.repo.TaskInfo(ctx, pair.TaskID) + if err != nil { + recordEvaluationError() + return nil + } + + ancestry := s.runAncestryIfDue(ctx, pair.RepositoryID, repoInfo.DefaultBranch, snapshots) + + classification := Classify(PairInput{ + DefaultBranch: repoInfo.DefaultBranch, + Snapshots: snapshots, + Providers: providers, + Ancestry: ancestry, + }) + + result, err := s.repo.Upsert(ctx, UpsertInput{ + TaskID: pair.TaskID, RepositoryID: pair.RepositoryID, WorkspaceID: taskInfo.WorkspaceID, + Classification: classification, EvaluatedAt: evaluatedAt, + }) + if err != nil { + if errors.Is(err, errRepositoryNotLive) { + return nil + } + recordWriteError() + return err + } + // recordEvaluation counts persisted evaluations only, per spec's + // "once per persisted evaluation" definition — it must run after a + // successful Upsert, not before (Review round 1, finding #2). + recordEvaluation(classification.Outcome) + if result.RowChanged { + recordRowWritten() + } + if result.Demoted { + recordDemotionSuppressed() + } + if result.DegradedOutcomeChanged { + recordDegradedOutcomeChanged() + } + return nil +} + +// runAncestryIfDue applies the ancestry precondition (spec "Default- +// branch observation § Ancestry precondition") and, when met, runs the +// check. Suspended entirely while degraded (empty default branch), which +// is neither a skip nor an error and is not counted by either counter. +func (s *Sweep) runAncestryIfDue(ctx context.Context, repositoryID, defaultBranch string, snapshots []Snapshot) AncestryOutcome { + if defaultBranch == "" || s.ancestry == nil { + return AncestryOutcome{} + } + if !AncestryPrecondition(snapshots) { + recordAncestrySkipped() + return AncestryOutcome{} + } + commit := SelectAncestryHead(snapshots) + if commit == "" { + return AncestryOutcome{} + } + out := s.ancestry.Check(ctx, repositoryID, defaultBranch, commit) + if out.Errored { + recordAncestryError() + } + return out +} + +// SelectDuePairs applies the freeze and the three due conditions (spec +// "Sweep selection predicate"), including the two-stage dueness fallback +// under "When the ledger itself cannot be read": if the bulk ledger read +// fails, every non-frozen candidate is treated as due. The returned error +// is the underlying ledger-read failure that triggered the fallback (nil +// when fallback is false), so a caller logging the fallback can carry the +// actual error rather than just the fact that one occurred (spec "Sweep +// selection predicate": the dueness_unavailable warning "carries the error +// and the candidate count"). +func (r *Repository) SelectDuePairs(ctx context.Context, candidates []CandidatePair, passStart time.Time) ([]CandidatePair, bool, error) { + ledgerRows, ledgerErr := r.allLedgerRows(ctx) + fallback := ledgerErr != nil + staleRefreshBoundary := passStart.Add(-StaleRefreshInterval) + + var due []CandidatePair + for _, pair := range candidates { + repoInfo, err := r.RepositoryInfo(ctx, pair.RepositoryID) + if err != nil { + // Not a spec-defined evaluation_errors_total case: this pair + // never reached evaluatePair, so it is not an "abandoned + // evaluation" under that counter's definition. Log only — + // self-healing, since Candidates()/SelectDuePairs reruns every + // pass and will pick the pair back up. + r.logWarn("delivery_ledger.due_selection_repository_info_failed", err) + continue + } + if repoInfo.DeletedAt != nil { + continue // freeze evaluated first, overrides every due condition + } + if fallback { + due = append(due, pair) + continue + } + if r.isDue(ctx, pair, ledgerRows, repoInfo, staleRefreshBoundary) { + due = append(due, pair) + } + } + return due, fallback, ledgerErr +} + +func (r *Repository) isDue( + ctx context.Context, pair CandidatePair, ledgerRows map[pairKey]dueLedgerInfo, + repoInfo RepositoryInfo, staleRefreshBoundary time.Time, +) bool { + existing, hasRow := ledgerRows[pairKey{pair.TaskID, pair.RepositoryID}] + if !hasRow { + return true + } + // Fail OPEN on a per-pair query error here, consistent with + // SelectDuePairs's own bulk-ledger-read fallback one level up + // (Review round 1, finding #3): these queries read the same input + // tables spec "Failure modes" names under "an input query fails", + // so a transient failure here must not silently exclude a pair with + // zero observability. Selecting it as due lets evaluatePair's own + // read of the same tables discover and count the failure via + // delivery_ledger_evaluation_errors_total. + taskInfo, err := r.TaskInfo(ctx, pair.TaskID) + if err != nil { + return true + } + mostRecent, err := r.mostRecentInputObservation(ctx, pair.TaskID, pair.RepositoryID, repoInfo.UpdatedAt, taskInfo.UpdatedAt) + if err != nil { + return true + } + if existing.LastEvaluatedAt.Before(mostRecent) { + return true + } + if existing.ReachedDefaultAt == nil && existing.LastEvaluatedAt.Before(staleRefreshBoundary) { + return true + } + return false +} + +type dueLedgerInfo struct { + EvidenceRank int + LastEvaluatedAt time.Time + ReachedDefaultAt *time.Time +} + +// allLedgerRows bulk-reads the ledger in one query, so a missing table +// fails once for the whole pass rather than per-candidate (spec "When the +// ledger itself cannot be read"). +func (r *Repository) allLedgerRows(ctx context.Context) (map[pairKey]dueLedgerInfo, error) { + var rows []struct { + TaskID string `db:"task_id"` + RepositoryID string `db:"repository_id"` + EvidenceRank int `db:"evidence_rank"` + LastEvaluatedAt time.Time `db:"last_evaluated_at"` + ReachedDefaultAt sql.NullTime `db:"reached_default_at"` + } + err := r.ro.SelectContext(ctx, &rows, ` + SELECT task_id, repository_id, evidence_rank, last_evaluated_at, reached_default_at + FROM task_delivery_ledger + `) + if err != nil { + return nil, err + } + out := make(map[pairKey]dueLedgerInfo, len(rows)) + for _, row := range rows { + info := dueLedgerInfo{EvidenceRank: row.EvidenceRank, LastEvaluatedAt: row.LastEvaluatedAt} + if row.ReachedDefaultAt.Valid { + t := row.ReachedDefaultAt.Time + info.ReachedDefaultAt = &t + } + out[pairKey{row.TaskID, row.RepositoryID}] = info + } + return out, nil +} + +// mostRecentInputObservationQueries enumerates every per-pair source in +// spec "Sweep selection predicate"'s due-source table other than the task +// and repository rows themselves (passed in directly, since the caller +// already has them). Tolerant of a missing provider table. +var mostRecentInputObservationQueries = []string{ + `SELECT MAX(g.created_at) FROM task_session_git_snapshots g ` + + `JOIN task_sessions s ON s.id = g.session_id WHERE s.task_id = ? AND s.repository_id = ?`, + `SELECT MAX(updated_at) FROM task_sessions WHERE task_id = ? AND repository_id = ?`, + `SELECT MAX(updated_at) FROM task_repositories WHERE task_id = ? AND repository_id = ?`, + `SELECT MAX(updated_at) FROM github_task_prs WHERE task_id = ? AND repository_id = ?`, + `SELECT MAX(updated_at) FROM gitlab_task_mrs WHERE task_id = ? AND repository_id = ?`, + `SELECT MAX(updated_at) FROM azure_devops_task_prs WHERE task_id = ? AND repository_id = ?`, +} + +func (r *Repository) mostRecentInputObservation( + ctx context.Context, taskID, repositoryID string, repoUpdatedAt, taskUpdatedAt time.Time, +) (time.Time, error) { + max := repoUpdatedAt + if taskUpdatedAt.After(max) { + max = taskUpdatedAt + } + for _, q := range mostRecentInputObservationQueries { + var raw interface{} + err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(q), taskID, repositoryID).Scan(&raw) + if err != nil { + if dbutil.IsMissingTableError(err) { + continue + } + return time.Time{}, err + } + t, ok, err := scanAggregateTime(raw) + if err != nil { + return time.Time{}, err + } + if ok && t.After(max) { + max = t + } + } + return max, nil +} + +// OrderPairs sorts due pairs by tasks.created_at ascending, then +// tasks.id ascending, then repositories.id ascending (spec "Ordering"). A +// pair whose task has since vanished is dropped rather than sorted on a +// missing created_at. +func (r *Repository) OrderPairs(ctx context.Context, pairs []CandidatePair) []CandidatePair { + type item struct { + pair CandidatePair + createdAt time.Time + } + items := make([]item, 0, len(pairs)) + for _, p := range pairs { + info, err := r.TaskInfo(ctx, p.TaskID) + if err != nil { + // Same rationale as SelectDuePairs' RepositoryInfo read above: + // not an "abandoned evaluation" (the pair never reached + // evaluatePair), so log rather than count. Self-healing next + // pass. + r.logWarn("delivery_ledger.ordering_task_info_failed", err) + continue + } + items = append(items, item{pair: p, createdAt: info.CreatedAt}) + } + sort.Slice(items, func(i, j int) bool { + if !items[i].createdAt.Equal(items[j].createdAt) { + return items[i].createdAt.Before(items[j].createdAt) + } + if items[i].pair.TaskID != items[j].pair.TaskID { + return items[i].pair.TaskID < items[j].pair.TaskID + } + return items[i].pair.RepositoryID < items[j].pair.RepositoryID + }) + out := make([]CandidatePair, len(items)) + for i, it := range items { + out[i] = it.pair + } + return out +} diff --git a/apps/backend/internal/delivery/sweep_internal_test.go b/apps/backend/internal/delivery/sweep_internal_test.go new file mode 100644 index 0000000000..9d32024bf7 --- /dev/null +++ b/apps/backend/internal/delivery/sweep_internal_test.go @@ -0,0 +1,240 @@ +package delivery + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/kandev/kandev/internal/common/logger" + dbutil "github.com/kandev/kandev/internal/db" + taskrepo "github.com/kandev/kandev/internal/task/repository" +) + +// newInternalTestRepo is a package-internal twin of the external test +// suite's newTestRepo (store_test.go, package delivery_test), needed here +// solely because evaluatePair is unexported and the soft-delete race this +// file covers must land its DB write between the due-selection read and +// evaluatePair's own RepositoryInfo re-read — a sequencing no black-box +// (RunPass-only) test can express. +func newInternalTestRepo(t *testing.T) (*Repository, *sqlx.DB) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "delivery-ledger.db") + rawConn, err := dbutil.OpenSQLite(dbPath) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db := sqlx.NewDb(rawConn, "sqlite3") + t.Cleanup(func() { _ = db.Close() }) + + if _, _, err := taskrepo.Provide(db, db, nil); err != nil { + t.Fatalf("init task schema: %v", err) + } + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init delivery schema: %v", err) + } + return repo, db +} + +func seedInternalWorkspace(t *testing.T, db *sqlx.DB, id string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO workspaces (id, name, created_at, updated_at) VALUES (?, ?, ?, ?) + `), id, id, now, now); err != nil { + t.Fatalf("seed workspace %s: %v", id, err) + } +} + +func seedInternalRepository(t *testing.T, db *sqlx.DB, id, workspaceID string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO repositories (id, workspace_id, name, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), id, workspaceID, id, now, now); err != nil { + t.Fatalf("seed repository %s: %v", id, err) + } +} + +func seedInternalTask(t *testing.T, db *sqlx.DB, id, workspaceID string) { + t.Helper() + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), id, workspaceID, id, now, now); err != nil { + t.Fatalf("seed task %s: %v", id, err) + } +} + +func countInternalLedgerRows(t *testing.T, db *sqlx.DB, taskID, repositoryID string) int { + t.Helper() + var n int + if err := db.QueryRowxContext(context.Background(), db.Rebind(` + SELECT COUNT(*) FROM task_delivery_ledger WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID).Scan(&n); err != nil { + t.Fatalf("count ledger rows: %v", err) + } + return n +} + +// TestEvaluatePair_RepositorySoftDeletedAfterDueSelectionIsNotWritten covers +// the freeze guarantee under spec "Sweep selection predicate": "A pair +// whose repository has a non-NULL repositories.deleted_at is never due, +// whichever condition below would otherwise select it." SelectDuePairs +// enforces that at due-selection time, but evaluatePair re-reads +// RepositoryInfo independently and, before this fix, never rechecked +// DeletedAt — so a repository soft-deleted in the window between +// due-selection and evaluation (a real race within a single sweep pass, +// since the two reads are not in the same transaction) would still be +// classified and written, silently voiding the freeze (Review round 1, +// finding #5). This test simulates that exact window: the pair is +// selected as due first, the repository is soft-deleted second, and only +// then is the pair evaluated. +func TestEvaluatePair_RepositorySoftDeletedAfterDueSelectionIsNotWritten(t *testing.T) { + repo, db := newInternalTestRepo(t) + seedInternalWorkspace(t, db, "ws-1") + seedInternalRepository(t, db, "repo-1", "ws-1") + seedInternalTask(t, db, "task-1", "ws-1") + + pair := CandidatePair{TaskID: "task-1", RepositoryID: "repo-1"} + + // Due-selection happens while the repository is still live. + due, _, _ := repo.SelectDuePairs(context.Background(), []CandidatePair{pair}, time.Now().UTC()) + if len(due) != 1 { + t.Fatalf("due = %+v, want the one live candidate", due) + } + + // The repository is soft-deleted in the window between due-selection + // and evaluation. + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET deleted_at = ? WHERE id = ?`), + time.Now().UTC(), "repo-1"); err != nil { + t.Fatalf("soft delete: %v", err) + } + + sweep := NewSweep(repo, nil, nil) + if err := sweep.evaluatePair(context.Background(), due[0]); err != nil { + t.Fatalf("evaluatePair: %v", err) + } + + if n := countInternalLedgerRows(t, db, "task-1", "repo-1"); n != 0 { + t.Fatalf("ledger rows = %d, want 0 (a repository soft-deleted after due-selection must never be written)", n) + } +} + +// TestIsDue_PerPairTaskInfoQueryErrorFailsOpen covers the inconsistency +// flagged in Review round 1, finding #3: SelectDuePairs's own bulk-ledger- +// read failure fails OPEN (every non-frozen candidate is treated as due, +// "When the ledger itself cannot be read"), but isDue's per-pair TaskInfo / +// mostRecentInputObservation queries — reading exactly the same input +// tables (task_session_git_snapshots, task_sessions, task_repositories, +// the provider tables) that spec "Failure modes" names as "an input query +// fails" — used to fail CLOSED (silently not due, no counter, no log) on a +// query error, silently skipping a pair already carrying a ledger row. +// isDue now fails open on those errors too, so the pair is selected as due +// and the same underlying failure is discovered and counted by +// evaluatePair's own input-query handling (delivery_ledger_evaluation_errors_total), +// rather than being silently absorbed one layer up with no observability at +// all. +func TestIsDue_PerPairTaskInfoQueryErrorFailsOpen(t *testing.T) { + repo, db := newInternalTestRepo(t) + seedInternalWorkspace(t, db, "ws-1") + seedInternalRepository(t, db, "repo-1", "ws-1") + seedInternalTask(t, db, "task-1", "ws-1") + + // Seed an existing ledger row so isDue reaches its TaskInfo call + // (the "no ledger row" branch returns true before ever touching + // TaskInfo, and would not exercise this fix). + if _, err := repo.Upsert(context.Background(), UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: Classification{Outcome: OutcomeUnknown, Basis: BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("seed ledger row: %v", err) + } + + // Force a genuine query error in isDue's TaskInfo read without + // dropping the tasks table outright: task_delivery_ledger.task_id is + // ON DELETE CASCADE to tasks(id), and SQLite's DROP TABLE cascades + // that FK, which would delete the ledger row just seeded above and + // make the pair due via the "no ledger row" branch instead of + // exercising this fix. Renaming a column away breaks the query + // (TaskInfo selects workspace_id by name) while leaving the table and + // row, and the ledger row, intact. + if _, err := db.Exec(`ALTER TABLE tasks RENAME COLUMN workspace_id TO workspace_id_renamed`); err != nil { + t.Fatalf("rename tasks.workspace_id: %v", err) + } + + due, fallback, _ := repo.SelectDuePairs(context.Background(), + []CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, time.Now().UTC()) + if fallback { + t.Fatal("fallback must be false: the bulk ledger read itself succeeded") + } + if len(due) != 1 { + t.Fatalf("due = %+v, want the pair selected as due (fail open on a per-pair query error)", due) + } +} + +// TestPublishWriterHealth_LogsStalledWarnWhenThresholdExceeded is the +// firing-state half of Review round 1, finding #4: every +// TestComputeStallSignal_* case in the external suite keeps StallSeconds +// at 0, and none of them drives Sweep.publishWriterHealth's own threshold +// comparison (sweep.go), which is where the "delivery_ledger.stalled" warn +// actually lives (ComputeStallSignal only computes the number). Calling +// publishWriterHealth directly — rather than through RunPass — sidesteps +// an unrelated interaction: any session on a resolvable (task, repository) +// pair becomes a Candidate, and a pair with no ledger row is +// unconditionally due, so RunPass would immediately re-evaluate and +// overwrite the very last_evaluated_at gap this test needs to keep +// intact. Testing this unexported method from the same package is exactly +// what evaluatePair's own soft-delete race test above does for the same +// reason. +func TestPublishWriterHealth_LogsStalledWarnWhenThresholdExceeded(t *testing.T) { + repo, db := newInternalTestRepo(t) + seedInternalWorkspace(t, db, "ws-1") + seedInternalRepository(t, db, "repo-1", "ws-1") + seedInternalTask(t, db, "task-1", "ws-1") + if _, err := repo.Upsert(context.Background(), UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: Classification{Outcome: OutcomeUnknown, Basis: BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC().Add(-time.Hour), + }); err != nil { + t.Fatalf("seed ledger row: %v", err) + } + now := time.Now().UTC() + if _, err := db.Exec(db.Rebind(` + INSERT INTO task_sessions (id, task_id, repository_id, started_at, updated_at) + VALUES (?, ?, ?, ?, ?) + `), "sess-live", "task-1", "repo-1", now, now); err != nil { + t.Fatalf("seed task_session: %v", err) + } + + core, observed := observer.New(zapcore.WarnLevel) + testLogger, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("build logger: %v", err) + } + + sweep := NewSweep(repo, nil, testLogger) + sweep.publishWriterHealth(context.Background()) + + entries := observed.FilterMessage("delivery_ledger.stalled").All() + if len(entries) != 1 { + t.Fatalf("got %d %q warnings, want 1 (all warnings: %+v)", len(entries), "delivery_ledger.stalled", observed.All()) + } + fields := entries[0].ContextMap() + stallSeconds, ok := fields["stall_seconds"].(int64) + if !ok || stallSeconds <= int64(StallThreshold.Seconds()) { + t.Fatalf("stall_seconds field = %v, want an int64 > %d", fields["stall_seconds"], int64(StallThreshold.Seconds())) + } + if _, ok := fields["last_evaluated_unix"].(int64); !ok { + t.Fatalf("last_evaluated_unix field = %v, want an int64", fields["last_evaluated_unix"]) + } +} diff --git a/apps/backend/internal/delivery/sweep_test.go b/apps/backend/internal/delivery/sweep_test.go new file mode 100644 index 0000000000..56f3812b8f --- /dev/null +++ b/apps/backend/internal/delivery/sweep_test.go @@ -0,0 +1,712 @@ +package delivery_test + +import ( + "context" + "database/sql" + "expvar" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/delivery" +) + +// readExpvarInt reads a package-level expvar.Int published by +// internal/delivery/metrics.go, following the same convention as +// internal/office/scheduler/metrics_vars_test.go. These are process-global +// and never reset between tests, so callers must compare a before/after +// delta rather than an absolute value. +func readExpvarInt(t *testing.T, name string) int64 { + t.Helper() + v := expvar.Get(name) + if v == nil { + t.Fatalf("expvar %q not published", name) + } + iv, ok := v.(*expvar.Int) + if !ok { + t.Fatalf("expvar %q is not an *expvar.Int", name) + } + return iv.Value() +} + +// readExpvarMapTotal sums every key of a package-level expvar.Map (the +// evaluations-by-outcome map published by metrics.go), following the same +// before/after-delta convention as readExpvarInt since the map is also +// process-global and never reset between tests. +func readExpvarMapTotal(t *testing.T, name string) int64 { + t.Helper() + v := expvar.Get(name) + if v == nil { + t.Fatalf("expvar %q not published", name) + } + mv, ok := v.(*expvar.Map) + if !ok { + t.Fatalf("expvar %q is not an *expvar.Map", name) + } + var total int64 + mv.Do(func(kv expvar.KeyValue) { + iv, ok := kv.Value.(*expvar.Int) + if !ok { + t.Fatalf("expvar map %q key %q value not an *expvar.Int", name, kv.Key) + } + total += iv.Value() + }) + return total +} + +func TestSelectDuePairs_NoLedgerRowIsUnconditionallyDue(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + + due, fallback, ledgerErr := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, time.Now().UTC()) + if fallback { + t.Fatal("a healthy, present ledger table must not report fallback") + } + if ledgerErr != nil { + t.Fatalf("ledgerErr = %v, want nil (a healthy, present ledger table must not report an error)", ledgerErr) + } + if len(due) != 1 { + t.Fatalf("due = %+v, want the pair with no ledger row", due) + } +} + +func TestSelectDuePairs_AlreadyFreshIsNotSelectedAgain(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + now := time.Now().UTC() + + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2, + ReachedDefault: true, ReachedBasis: delivery.ReachedBasisAncestorOfDefault, ReachedRef: "aaa", + }, + EvaluatedAt: now, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + due, _, _ := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, now.Add(time.Second)) + if len(due) != 0 { + t.Fatalf("due = %+v, want none (nothing moved, reached_default_at already set)", due) + } +} + +func TestSelectDuePairs_InputMovementReselects(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + past := time.Now().UTC().Add(-time.Hour) + + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2, + ReachedDefault: true, ReachedBasis: delivery.ReachedBasisAncestorOfDefault, ReachedRef: "aaa", + }, + EvaluatedAt: past, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + seedTaskSession(t, db, "sess-1", "task-1", "repo-1") + seedGitSnapshot(t, db, "snap-1", "sess-1", "feature", "bbb", 3, time.Now().UTC()) + + due, _, _ := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, time.Now().UTC()) + if len(due) != 1 { + t.Fatalf("due = %+v, want the pair selected because a snapshot moved", due) + } +} + +func TestSelectDuePairs_StaleRefreshWhileReachedDefaultIsNull(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + old := time.Now().UTC().Add(-25 * time.Hour) + + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5}, + EvaluatedAt: old, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + due, _, _ := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, time.Now().UTC()) + if len(due) != 1 { + t.Fatalf("due = %+v, want selected by stale refresh (>24h, reached_default_at NULL)", due) + } +} + +func TestSelectDuePairs_ReachedDefaultSetNeverStaleRefreshed(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + old := time.Now().UTC().Add(-25 * time.Hour) + // Backdate the repository/task rows themselves so condition 2 (input + // movement) cannot spuriously fire from their own seed-time + // updated_at — this test isolates condition 3 (stale refresh) alone. + backdateRepoAndTask(t, db, "repo-1", "task-1", old.Add(-time.Hour)) + + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-1", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{ + Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Rank: 8, + ReachedDefault: true, ReachedBasis: delivery.ReachedBasisProviderPRMerged, ReachedRef: "https://pr/1", + }, + EvaluatedAt: old, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + due, _, _ := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, time.Now().UTC()) + if len(due) != 0 { + t.Fatalf("due = %+v, want none (stale refresh applies only while reached_default_at is NULL)", due) + } +} + +func TestSelectDuePairs_FreezeOverridesEveryCondition(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET deleted_at = ? WHERE id = ?`), time.Now().UTC(), "repo-1"); err != nil { + t.Fatalf("soft delete: %v", err) + } + + // No ledger row: would otherwise be unconditionally due. + due, _, _ := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-1", RepositoryID: "repo-1"}}, time.Now().UTC()) + if len(due) != 0 { + t.Fatalf("due = %+v, want none (freeze overrides the unconditionally-due rule)", due) + } +} + +func TestSelectDuePairs_MissingLedgerTableFallsBackToEveryNonFrozenCandidate(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedRepository(t, db, "repo-frozen", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTask(t, db, "task-2", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET deleted_at = ? WHERE id = ?`), time.Now().UTC(), "repo-frozen"); err != nil { + t.Fatalf("soft delete: %v", err) + } + + // Simulate a swallowed migration: the table existed (newTestRepo + // created it) and is now gone. + if _, err := db.Exec(`DROP TABLE task_delivery_ledger`); err != nil { + t.Fatalf("drop ledger table: %v", err) + } + + due, fallback, ledgerErr := repo.SelectDuePairs(context.Background(), []delivery.CandidatePair{ + {TaskID: "task-1", RepositoryID: "repo-1"}, + {TaskID: "task-2", RepositoryID: "repo-frozen"}, + }, time.Now().UTC()) + if !fallback { + t.Fatal("expected fallback=true when the ledger table is missing") + } + if ledgerErr == nil { + t.Fatal("expected a non-nil ledgerErr describing why the fallback fired") + } + if len(due) != 1 || due[0].RepositoryID != "repo-1" { + t.Fatalf("due = %+v, want only the non-frozen candidate (freeze still applies under the fallback)", due) + } +} + +func TestOrderPairs_ByTaskCreatedAtThenTaskIDThenRepositoryID(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-a", "ws-1") + seedRepository(t, db, "repo-b", "ws-1") + + now := time.Now().UTC() + seedTaskAt(t, db, "task-later", "ws-1", now.Add(time.Hour)) + seedTaskAt(t, db, "task-earlier", "ws-1", now) + + pairs := []delivery.CandidatePair{ + {TaskID: "task-later", RepositoryID: "repo-a"}, + {TaskID: "task-earlier", RepositoryID: "repo-b"}, + {TaskID: "task-earlier", RepositoryID: "repo-a"}, + } + ordered := repo.OrderPairs(context.Background(), pairs) + if len(ordered) != 3 { + t.Fatalf("ordered = %+v, want 3", ordered) + } + if ordered[0].TaskID != "task-earlier" || ordered[0].RepositoryID != "repo-a" { + t.Fatalf("ordered[0] = %+v, want task-earlier/repo-a (repository_id ascending within the same task)", ordered[0]) + } + if ordered[1].TaskID != "task-earlier" || ordered[1].RepositoryID != "repo-b" { + t.Fatalf("ordered[1] = %+v, want task-earlier/repo-b", ordered[1]) + } + if ordered[2].TaskID != "task-later" { + t.Fatalf("ordered[2] = %+v, want task-later last (created later)", ordered[2]) + } +} + +func backdateRepoAndTask(t *testing.T, db *sqlx.DB, repositoryID, taskID string, at time.Time) { + t.Helper() + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET updated_at = ? WHERE id = ?`), at, repositoryID); err != nil { + t.Fatalf("backdate repository: %v", err) + } + if _, err := db.Exec(db.Rebind(`UPDATE tasks SET updated_at = ? WHERE id = ?`), at, taskID); err != nil { + t.Fatalf("backdate task: %v", err) + } +} + +func seedTaskAt(t *testing.T, db interface { + Rebind(string) string + Exec(string, ...interface{}) (sql.Result, error) +}, id, workspaceID string, createdAt time.Time) { + t.Helper() + if _, err := db.Exec(db.Rebind(` + INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES (?, ?, ?, ?, ?) + `), id, workspaceID, id, createdAt, createdAt); err != nil { + t.Fatalf("seed task %s: %v", id, err) + } +} + +// TestRunPass_EndToEnd exercises the full pipeline once: candidacy, +// dueness, ordering, evaluation, upsert, and writer-health publication, +// against a pair with a real merged pull request. +func TestRunPass_EndToEnd(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET default_branch = ? WHERE id = ?`), "main", "repo-1"); err != nil { + t.Fatalf("set default_branch: %v", err) + } + seedGitHubStore(t, db) + now := time.Now().UTC() + seedGitHubPR(t, db, "pr-1", "task-1", "repo-1", "acme", "widgets", 1, "https://gh/1", "main", &now, nil) + + sweep := delivery.NewSweep(repo, nil, nil) + sweep.RunPass(context.Background()) + + row := readLedgerRow(t, db, "task-1", "repo-1") + if row.Outcome.String != string(delivery.OutcomePRMerge) || row.Rank != 8 { + t.Fatalf("row = %+v, want pr_merge at rank 8", row) + } + if row.EvaluationSeq != 1 { + t.Fatalf("evaluation_seq = %d, want 1", row.EvaluationSeq) + } + + // A second pass with nothing changed must not re-select the pair. + sweep.RunPass(context.Background()) + after := readLedgerRow(t, db, "task-1", "repo-1") + if after.EvaluationSeq != 1 { + t.Fatalf("evaluation_seq after second pass = %d, want still 1 (not re-selected, nothing moved and reached_default_at unset only matters after 24h)", after.EvaluationSeq) + } +} + +// TestRunPass_AncestrySkippedWhenNoSessionHasMovingHead covers spec +// "Default-branch observation § Ancestry precondition" and +// "delivery_ledger_ancestry_skipped_total" (spec.md:1797-1801), composed +// through the real Sweep with a real, non-nil AncestryChecker rather than +// the nil used by every other RunPass test in this file — every prior +// test passed nil, which left runAncestryIfDue and both ancestry counters +// effectively unexercised. Here the pair's one session shows a single +// snapshot (no distinct non-empty heads of its own), so the precondition +// is not met: the skipped counter must increment and the checkout must +// never even be resolved, proven via the calls counter rather than merely +// asserting Errored=false (which a different bug could also produce). +func TestRunPass_AncestrySkippedWhenNoSessionHasMovingHead(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET default_branch = ? WHERE id = ?`), "main", "repo-1"); err != nil { + t.Fatalf("set default_branch: %v", err) + } + seedTaskSession(t, db, "sess-1", "task-1", "repo-1") + seedGitSnapshot(t, db, "snap-1", "sess-1", "feature", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 3, time.Now().UTC()) + + checkoutCalls := 0 + ancestry := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: t.TempDir(), calls: &checkoutCalls}} + sweep := delivery.NewSweep(repo, ancestry, nil) + + skippedBefore := readExpvarInt(t, "delivery_ledger_ancestry_skipped_total") + errorsBefore := readExpvarInt(t, "delivery_ledger_ancestry_errors_total") + + sweep.RunPass(context.Background()) + + if delta := readExpvarInt(t, "delivery_ledger_ancestry_skipped_total") - skippedBefore; delta != 1 { + t.Fatalf("ancestry_skipped delta = %d, want 1", delta) + } + if delta := readExpvarInt(t, "delivery_ledger_ancestry_errors_total") - errorsBefore; delta != 0 { + t.Fatalf("ancestry_errors delta = %d, want 0 (skipped, not attempted)", delta) + } + if checkoutCalls != 0 { + t.Fatalf("checkout resolved %d times, want 0: a skipped precondition must never reach the checker", checkoutCalls) + } + row := readLedgerRow(t, db, "task-1", "repo-1") + if row.ReachedAt.Valid { + t.Fatalf("reached_default_at = %v, want NULL: no ancestry attempt means no ancestor_of_default observation", row.ReachedAt) + } +} + +// TestRunPass_GenuineAncestorReachesDefaultViaSweep covers the same spec +// sections as the skipped test above from the opposite side: a real +// ancestor relation, composed end-to-end through Sweep.RunPass, a real +// git repository (the same fixture ancestry_test.go's unit tests use), +// and a real AncestryChecker — not merely AncestryChecker.Check in +// isolation. The pair's one session shows two distinct non-empty heads +// (satisfying the moving-head precondition); the more recent of the two +// is the git repo's actual HEAD, which is trivially an ancestor of its +// own default branch. This is the missing link test-supervisor flagged: +// spec.md:2041-2050 warns that "a test that omits the relation is +// asserting nothing", and no prior test joined AncestryChecker to the +// evaluator through the sweep. +func TestRunPass_GenuineAncestorReachesDefaultViaSweep(t *testing.T) { + work, defaultBranch := newAncestryTestRepo(t) + head := runGit(t, work, "rev-parse", "HEAD") + + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET default_branch = ? WHERE id = ?`), defaultBranch, "repo-1"); err != nil { + t.Fatalf("set default_branch: %v", err) + } + seedTaskSession(t, db, "sess-1", "task-1", "repo-1") + older := time.Now().UTC().Add(-time.Hour) + newer := time.Now().UTC() + // Two distinct non-empty heads for the same session satisfies the + // moving-head precondition; the older, unrelated value never reaches + // git (only the most recent snapshot's head is selected), so it does + // not need to resolve to a real commit. + seedGitSnapshot(t, db, "snap-old", "sess-1", "feature", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", 1, older) + seedGitSnapshot(t, db, "snap-new", "sess-1", "feature", head, 2, newer) + + ancestry := &delivery.AncestryChecker{Checkout: fakeCheckoutResolver{path: work}} + sweep := delivery.NewSweep(repo, ancestry, nil) + + sweep.RunPass(context.Background()) + + row := readLedgerRow(t, db, "task-1", "repo-1") + if !row.ReachedAt.Valid { + t.Fatal("reached_default_at = NULL, want set: HEAD is trivially an ancestor of its own default branch") + } + if row.ReachedBasis.String != string(delivery.ReachedBasisAncestorOfDefault) { + t.Fatalf("reached_default_basis = %q, want %q", row.ReachedBasis.String, delivery.ReachedBasisAncestorOfDefault) + } + if row.ReachedRef.String != head { + t.Fatalf("reached_default_ref = %q, want %q (the selected session head)", row.ReachedRef.String, head) + } +} + +// TestRunPass_OrphanedProviderRowMissingTaskExcludedFromUpsert covers spec +// "Candidate pairs" / "Scenarios": a github_task_prs row whose task_id +// matches no tasks row is excluded from candidacy before ever reaching the +// upsert — no ledger row is written for it, on this pass or a second one +// with nothing changed, which is what keeps +// delivery_ledger_write_errors_total at its documented healthy resting +// value of 0 rather than climbing forever on a foreign-key failure. +func TestRunPass_OrphanedProviderRowMissingTaskExcludedFromUpsert(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedGitHubStore(t, db) + now := time.Now().UTC() + seedGitHubPR(t, db, "pr-orphan", "task-ghost", "repo-1", "acme", "widgets", + 1, "https://gh/1", "main", &now, nil) + + sweep := delivery.NewSweep(repo, nil, nil) + sweep.RunPass(context.Background()) + + if countLedgerRows(t, db, "task-ghost", "repo-1") != 0 { + t.Fatal("no ledger row must be written for a pair excluded by a missing task row") + } + + // A second pass with nothing changed must not attempt the upsert either. + sweep.RunPass(context.Background()) + if countLedgerRows(t, db, "task-ghost", "repo-1") != 0 { + t.Fatal("second pass must not write a ledger row for the still-orphaned pair") + } +} + +// TestRunPass_RankOneDegradedOutcomeChangeRecordsCounterAndAdvancesUpdatedAt +// is Review round 2, finding #7: upsert_test.go's rank-1 degraded write +// test only ever calls repo.Upsert directly, which never touches +// delivery_ledger_degraded_outcome_changed_total or +// delivery_ledger_demotions_suppressed_total — those counters are +// incremented one layer up, in RunPass's per-pair loop, off the +// UpsertResult it gets back. This test drives the exact same scenario +// (a degraded pair's outcome changing with its rank pinned at 1) through +// two real sweep passes so the counters actually move, and checks +// updated_at advances on the row that resulted. +func TestRunPass_RankOneDegradedOutcomeChangeRecordsCounterAndAdvancesUpdatedAt(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") // default_branch defaults to '': every evaluation here is degraded + seedTask(t, db, "task-1", "ws-1") + seedGitHubStore(t, db) + + mergedAt := time.Now().UTC() + seedGitHubPR(t, db, "pr-1", "task-1", "repo-1", "acme", "widgets", 1, "https://gh/1", "main", &mergedAt, nil) + + sweep := delivery.NewSweep(repo, nil, nil) + sweep.RunPass(context.Background()) + + before := readLedgerRow(t, db, "task-1", "repo-1") + if before.Outcome.String != string(delivery.OutcomePRMerge) || before.Rank != 1 { + t.Fatalf("row after first pass = %+v, want degraded pr_merge at rank 1", before) + } + + demotionsBefore := readExpvarInt(t, "delivery_ledger_demotions_suppressed_total") + degradedBefore := readExpvarInt(t, "delivery_ledger_degraded_outcome_changed_total") + + // The PR detaches and a snapshot lands at ahead=0 with no moving head: + // still degraded (no default branch), so the pair re-evaluates from + // pr_merge to no_delivery_observed at the SAME rank 1 — a + // DegradedOutcomeChanged, never a Demoted (1 is not less than 1). + detachedAt := mergedAt.Add(time.Minute) + if _, err := db.Exec(db.Rebind(`UPDATE github_task_prs SET detached_at = ?, updated_at = ? WHERE id = ?`), + detachedAt, detachedAt, "pr-1"); err != nil { + t.Fatalf("detach pr: %v", err) + } + seedTaskSession(t, db, "sess-1", "task-1", "repo-1") + seedGitSnapshot(t, db, "snap-1", "sess-1", "feature", "aaa", 0, detachedAt) + + sweep.RunPass(context.Background()) + + if delta := readExpvarInt(t, "delivery_ledger_demotions_suppressed_total") - demotionsBefore; delta != 0 { + t.Fatalf("demotions_suppressed_total delta = %d, want 0 (rank did not fall)", delta) + } + if delta := readExpvarInt(t, "delivery_ledger_degraded_outcome_changed_total") - degradedBefore; delta != 1 { + t.Fatalf("degraded_outcome_changed_total delta = %d, want 1", delta) + } + + after := readLedgerRow(t, db, "task-1", "repo-1") + if after.Outcome.String != string(delivery.OutcomeNoDeliveryObserved) || after.Rank != 1 { + t.Fatalf("row after second pass = %+v, want no_delivery_observed at rank 1", after) + } + if !after.UpdatedAt.After(before.UpdatedAt) { + t.Fatalf("updated_at = %v, want after %v (an outcome change at unchanged rank must still advance it)", after.UpdatedAt, before.UpdatedAt) + } +} + +// countLedgerRows returns the number of task_delivery_ledger rows for a +// pair — used where the pair is expected to have none, since readLedgerRow +// fails the test on no rows. +func countLedgerRows(t *testing.T, db *sqlx.DB, taskID, repositoryID string) int { + t.Helper() + var n int + if err := db.QueryRowxContext(context.Background(), db.Rebind(` + SELECT COUNT(*) FROM task_delivery_ledger WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID).Scan(&n); err != nil { + t.Fatalf("count ledger rows: %v", err) + } + return n +} + +// TestRunPass_MissingLedgerTableCountsWriteErrorsAndReachesEveryCandidate +// covers spec "Scenarios § Idempotency, ordering and concurrency": with the +// ledger table itself missing, every candidate is treated as due (the +// SelectDuePairs fallback, already covered by +// TestSelectDuePairs_MissingLedgerTableFallsBackToEveryNonFrozenCandidate), +// and this test covers what happens next — each due pair's Upsert then +// fails against the missing table, write_errors_total increments once per +// pair (never evaluation_errors_total, since every input read before the +// upsert still succeeds), evaluations_total does NOT increment for a pair +// whose upsert failed (it counts persisted evaluations, not computed +// ones — Review round 1, finding #2), and the pass reaches its LAST +// candidate rather than aborting after the first failure. +func TestRunPass_MissingLedgerTableCountsWriteErrorsAndReachesEveryCandidate(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTask(t, db, "task-2", "ws-1") + seedTaskRepository(t, db, "tr-1", "task-1", "repo-1") + seedTaskRepository(t, db, "tr-2", "task-2", "repo-1") + + if _, err := db.Exec(`DROP TABLE task_delivery_ledger`); err != nil { + t.Fatalf("drop ledger table: %v", err) + } + + writeErrorsBefore := readExpvarInt(t, "delivery_ledger_write_errors_total") + evalErrorsBefore := readExpvarInt(t, "delivery_ledger_evaluation_errors_total") + evaluationsBefore := readExpvarMapTotal(t, "delivery_ledger_evaluations_total") + + sweep := delivery.NewSweep(repo, nil, nil) + sweep.RunPass(context.Background()) + + writeErrorsDelta := readExpvarInt(t, "delivery_ledger_write_errors_total") - writeErrorsBefore + evalErrorsDelta := readExpvarInt(t, "delivery_ledger_evaluation_errors_total") - evalErrorsBefore + evaluationsDelta := readExpvarMapTotal(t, "delivery_ledger_evaluations_total") - evaluationsBefore + + if writeErrorsDelta != 2 { + t.Fatalf("write_errors_total delta = %d, want 2 (both candidates reached and failed their upsert)", writeErrorsDelta) + } + if evalErrorsDelta != 0 { + t.Fatalf("evaluation_errors_total delta = %d, want 0 (input reads succeed; only the upsert fails)", evalErrorsDelta) + } + if evaluationsDelta != 0 { + t.Fatalf("evaluations_total delta = %d, want 0 (a failed upsert must not count as a persisted evaluation)", evaluationsDelta) + } +} + +// TestRunPass_MissingLedgerTableLogsBothWarningsOnceEachCarryingTheError is +// Review round 2, finding #3 (an OBVIOUS production defect: both +// spec-mandated WARN logs discarded their underlying error, contradicting +// spec:962 and spec:1303-1304's explicit "carrying the error" requirement) +// together with finding #8 (nothing verified either log fires exactly once +// per pass rather than once per candidate). A dropped ledger table is a +// single failure that naturally triggers both warnings in the same real +// pass: the bulk dueness read fails first (dueness_unavailable), and then +// every due candidate's Upsert fails against the same missing table +// (write_failed) — so this one scenario proves both logs fire exactly +// once each, and both carry a non-nil, non-empty underlying error rather +// than just the fact that something failed. +func TestRunPass_MissingLedgerTableLogsBothWarningsOnceEachCarryingTheError(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTask(t, db, "task-2", "ws-1") + seedTaskRepository(t, db, "tr-1", "task-1", "repo-1") + seedTaskRepository(t, db, "tr-2", "task-2", "repo-1") + + if _, err := db.Exec(`DROP TABLE task_delivery_ledger`); err != nil { + t.Fatalf("drop ledger table: %v", err) + } + + core, observed := observer.New(zapcore.WarnLevel) + testLogger, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatalf("build logger: %v", err) + } + + sweep := delivery.NewSweep(repo, nil, testLogger) + sweep.RunPass(context.Background()) + + duenessEntries := observed.FilterMessage("delivery_ledger.dueness_unavailable").All() + if len(duenessEntries) != 1 { + t.Fatalf("got %d %q warnings, want exactly 1 (all warnings: %+v)", + len(duenessEntries), "delivery_ledger.dueness_unavailable", observed.All()) + } + duenessErr, ok := duenessEntries[0].ContextMap()["error"].(string) + if !ok || duenessErr == "" { + t.Fatalf("dueness_unavailable error field = %v, want a non-empty underlying error", duenessEntries[0].ContextMap()["error"]) + } + + writeFailedEntries := observed.FilterMessage("delivery_ledger.write_failed").All() + if len(writeFailedEntries) != 1 { + t.Fatalf("got %d %q warnings, want exactly 1 (all warnings: %+v)", + len(writeFailedEntries), "delivery_ledger.write_failed", observed.All()) + } + writeErr, ok := writeFailedEntries[0].ContextMap()["error"].(string) + if !ok || writeErr == "" { + t.Fatalf("write_failed error field = %v, want a non-empty underlying error", writeFailedEntries[0].ContextMap()["error"]) + } + if count, ok := writeFailedEntries[0].ContextMap()["count"].(int64); !ok || count != 2 { + t.Fatalf("write_failed count field = %v, want int64 2 (both candidates reached and failed their upsert)", writeFailedEntries[0].ContextMap()["count"]) + } +} + +// TestRunPass_SnapshotQueryErrorAbandonsEvaluationNoColumnWritten covers +// spec "Scenarios § Idempotency, ordering and concurrency": an input-query +// failure abandons the evaluation entirely — no column is written, +// including last_evaluated_at — evaluation_errors_total increments, +// evaluations_total does not, and the pair is still due on the next pass. +// SnapshotsForPair is the representative case: unlike ProvidersForPair, it +// applies no missing-table tolerance, so dropping its table is a clean, +// deterministic way to force the real (non-tolerated) query error +// evaluatePair's four early-return branches exist to handle. +func TestRunPass_SnapshotQueryErrorAbandonsEvaluationNoColumnWritten(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + seedTaskRepository(t, db, "tr-1", "task-1", "repo-1") + + if _, err := db.Exec(`DROP TABLE task_session_git_snapshots`); err != nil { + t.Fatalf("drop task_session_git_snapshots: %v", err) + } + + evalErrorsBefore := readExpvarInt(t, "delivery_ledger_evaluation_errors_total") + + sweep := delivery.NewSweep(repo, nil, nil) + sweep.RunPass(context.Background()) + + if delta := readExpvarInt(t, "delivery_ledger_evaluation_errors_total") - evalErrorsBefore; delta != 1 { + t.Fatalf("evaluation_errors_total delta = %d, want 1", delta) + } + if countLedgerRows(t, db, "task-1", "repo-1") != 0 { + t.Fatal("no ledger row must be written when the snapshot query fails") + } +} + +// TestSelectDuePairs_ArchivedTaskRemainsEligible covers spec "Persistence +// guarantees" / "Scenarios": archiving a task neither excludes it from +// candidacy nor freezes it — no predicate anywhere reads +// tasks.archived_at, so an archived task's pair is due under exactly the +// same three conditions as any other. +func TestSelectDuePairs_ArchivedTaskRemainsEligible(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-archived", "ws-1") + if _, err := db.Exec(db.Rebind(`UPDATE tasks SET archived_at = ? WHERE id = ?`), + time.Now().UTC(), "task-archived"); err != nil { + t.Fatalf("archive task: %v", err) + } + + // No ledger row yet: unconditionally due, same as an unarchived pair. + due, _, _ := repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-archived", RepositoryID: "repo-1"}}, time.Now().UTC()) + if len(due) != 1 { + t.Fatalf("due = %+v, want the archived task's pair selected", due) + } + + if _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: "task-archived", RepositoryID: "repo-1", WorkspaceID: "ws-1", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: time.Now().UTC(), + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + // A new session on the archived task's pair moves an input: still due, + // exactly like an unarchived pair. + seedTaskSession(t, db, "sess-archived", "task-archived", "repo-1") + seedGitSnapshot(t, db, "snap-archived", "sess-archived", "feature", "aaa", 3, time.Now().UTC()) + due, _, _ = repo.SelectDuePairs(context.Background(), + []delivery.CandidatePair{{TaskID: "task-archived", RepositoryID: "repo-1"}}, time.Now().UTC()) + if len(due) != 1 { + t.Fatalf("due = %+v, want the archived task's pair re-selected on input movement", due) + } +} + +func TestSweep_StartStopLifecycle(t *testing.T) { + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + + sweep := delivery.NewSweep(repo, nil, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sweep.Start(ctx) + sweep.Start(ctx) // second Start must be a no-op, not a second goroutine + sweep.Stop() + sweep.Stop() // idempotent +} diff --git a/apps/backend/internal/delivery/testmain_test.go b/apps/backend/internal/delivery/testmain_test.go new file mode 100644 index 0000000000..69d2b7c8b4 --- /dev/null +++ b/apps/backend/internal/delivery/testmain_test.go @@ -0,0 +1,17 @@ +package delivery + +import ( + "testing" + + "go.uber.org/goleak" +) + +// TestMain wraps internal/delivery's test suite in goleak so Sweep.Start's +// background loop goroutine is verified to drain on Stop, following the +// same convention as internal/gateway/websocket, internal/agentctl/server/ +// process, and the runtime lifecycle package (Review round 2, finding #4; +// docs/plans/task-delivery-ledger/task-06-sweep-and-health.md commits to +// this coverage). +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/apps/backend/internal/delivery/upsert.go b/apps/backend/internal/delivery/upsert.go new file mode 100644 index 0000000000..99f1986065 --- /dev/null +++ b/apps/backend/internal/delivery/upsert.go @@ -0,0 +1,244 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/kandev/kandev/internal/db/dialect" +) + +// The rank-guarded classification columns. Repeated verbatim in both the +// SET clause (the actual write) and the updated_at "did anything change" +// predicate, because SQL's UPSERT gives no other way to compare the +// would-be new value against the pre-update stored value from inside the +// same atomic statement (spec "Concurrency": expressed inside the +// statement's SET clause, never a Go read-then-write). +const ( + rankGuardOutcome = `CASE WHEN excluded.evidence_rank >= task_delivery_ledger.evidence_rank ` + + `THEN excluded.delivery_outcome ELSE task_delivery_ledger.delivery_outcome END` + rankGuardBasis = `CASE WHEN excluded.evidence_rank >= task_delivery_ledger.evidence_rank ` + + `THEN excluded.delivery_basis ELSE task_delivery_ledger.delivery_basis END` + rankGuardRef = `CASE WHEN excluded.evidence_rank >= task_delivery_ledger.evidence_rank ` + + `THEN excluded.delivery_ref ELSE task_delivery_ledger.delivery_ref END` + rankGuardRank = `CASE WHEN excluded.evidence_rank >= task_delivery_ledger.evidence_rank ` + + `THEN excluded.evidence_rank ELSE task_delivery_ledger.evidence_rank END` + + writeOnceAt = `CASE WHEN task_delivery_ledger.reached_default_at IS NULL ` + + `THEN excluded.reached_default_at ELSE task_delivery_ledger.reached_default_at END` + writeOnceBasis = `CASE WHEN task_delivery_ledger.reached_default_at IS NULL ` + + `THEN excluded.reached_default_basis ELSE task_delivery_ledger.reached_default_basis END` + writeOnceRef = `CASE WHEN task_delivery_ledger.reached_default_at IS NULL ` + + `THEN excluded.reached_default_ref ELSE task_delivery_ledger.reached_default_ref END` + + firstClassifiedAtExpr = `CASE WHEN task_delivery_ledger.first_classified_at IS NULL ` + + `THEN excluded.first_classified_at ELSE task_delivery_ledger.first_classified_at END` + + // highWaterObservedExpr avoids SQLite's two-arg max() (NULL-poisons on + // either side being NULL) and Postgres's differently-behaved GREATEST, + // per spec "High-water columns": NULL handling is made explicit with a + // portable CASE rather than either dialect's native function. + highWaterObservedExpr = `CASE + WHEN excluded.observed_branch_commits IS NULL THEN task_delivery_ledger.observed_branch_commits + WHEN task_delivery_ledger.observed_branch_commits IS NULL THEN excluded.observed_branch_commits + WHEN task_delivery_ledger.observed_branch_commits >= excluded.observed_branch_commits + THEN task_delivery_ledger.observed_branch_commits + ELSE excluded.observed_branch_commits + END` + + highWaterEvaluatedAtExpr = `CASE + WHEN task_delivery_ledger.last_evaluated_at >= excluded.last_evaluated_at + THEN task_delivery_ledger.last_evaluated_at + ELSE excluded.last_evaluated_at + END` + + // updatedAtExpr advances updated_at only when a classification or + // observation column actually changes value (spec "Idempotency": a + // re-evaluation with unchanged inputs must not advance updated_at). + // IS DISTINCT FROM is NULL-safe on both dialects (SQLite since 3.39, + // bundled by the go-sqlite3 driver version this module pins). + updatedAtExpr = `CASE WHEN + ` + rankGuardOutcome + ` IS DISTINCT FROM task_delivery_ledger.delivery_outcome + OR ` + rankGuardBasis + ` IS DISTINCT FROM task_delivery_ledger.delivery_basis + OR ` + rankGuardRef + ` IS DISTINCT FROM task_delivery_ledger.delivery_ref + OR ` + highWaterObservedExpr + ` IS DISTINCT FROM task_delivery_ledger.observed_branch_commits + OR ` + writeOnceAt + ` IS DISTINCT FROM task_delivery_ledger.reached_default_at + THEN excluded.updated_at ELSE task_delivery_ledger.updated_at END` +) + +const upsertSQL = ` +INSERT INTO task_delivery_ledger ( + id, task_id, repository_id, workspace_id, + delivery_outcome, delivery_basis, delivery_ref, evidence_rank, + reached_default_at, reached_default_basis, reached_default_ref, + observed_branch_commits, first_classified_at, + last_evaluated_at, evaluation_seq, created_at, updated_at +) +SELECT + ?, ?, repositories.id, ?, + ?, ?, ?, ?, + ?, ?, ?, + ?, ?, + ?, ?, ?, ? +FROM repositories +WHERE repositories.id = ? AND repositories.deleted_at IS NULL +ON CONFLICT (task_id, repository_id) DO UPDATE SET + workspace_id = excluded.workspace_id, + delivery_outcome = ` + rankGuardOutcome + `, + delivery_basis = ` + rankGuardBasis + `, + delivery_ref = ` + rankGuardRef + `, + evidence_rank = ` + rankGuardRank + `, + reached_default_at = ` + writeOnceAt + `, + reached_default_basis = ` + writeOnceBasis + `, + reached_default_ref = ` + writeOnceRef + `, + observed_branch_commits = ` + highWaterObservedExpr + `, + first_classified_at = ` + firstClassifiedAtExpr + `, + last_evaluated_at = ` + highWaterEvaluatedAtExpr + `, + evaluation_seq = task_delivery_ledger.evaluation_seq + 1, + updated_at = ` + updatedAtExpr + ` +RETURNING evidence_rank, delivery_outcome, delivery_basis, delivery_ref, + observed_branch_commits, reached_default_at, updated_at, evaluation_seq +` + +// errRepositoryNotLive means the repository disappeared or became soft +// deleted before the ledger write. It is a no-op for a sweep evaluation, not +// a database failure, so the caller must leave the pair due without advancing +// its ledger row. +var errRepositoryNotLive = errors.New("delivery ledger repository is not live") + +// UpsertInput is everything one persisted evaluation writes for a pair. +// EvaluatedAt is T0, the instant this evaluation began reading its +// inputs — never the commit instant (spec "Sweep selection predicate", +// "Which instant last_evaluated_at records"). It is also used as the +// evaluation's own clock reading for reached_default_at and +// first_classified_at when this evaluation is the one that sets them +// (Build decision R5-F6: same instant source as last_evaluated_at, for +// the same comparability reason the spec argues for that column). +type UpsertInput struct { + TaskID string + RepositoryID string + WorkspaceID string + Classification + EvaluatedAt time.Time +} + +// UpsertResult reports what one Upsert call actually did, for the +// writer-health counters. These three signals are derived from a pre- +// upsert read compared against the post-upsert RETURNING row rather than +// computed atomically inside the statement, unlike every persisted +// column above. That is a deliberate, narrower exception than the +// NULL-handling one the spec calls out for high-water columns: it can +// only ever be inexact under a genuine concurrent write to the very same +// pair, which is unreachable in this deployment because the sweep never +// overlaps itself and no event trigger is wired (Build decision R5-F3). +// The counters are documented as a convenience for a live /debug/vars +// reader, not the ground truth — last_evaluated_at in the database is. +type UpsertResult struct { + RowChanged bool + Demoted bool + DegradedOutcomeChanged bool +} + +type existingRow struct { + EvidenceRank int + Outcome sql.NullString +} + +// Upsert persists one evaluation via the single INSERT ... ON CONFLICT DO +// UPDATE statement described above. See package doc and spec "Ordering, +// idempotency, concurrency". +func (r *Repository) Upsert(ctx context.Context, in UpsertInput) (UpsertResult, error) { + before, beforeErr := r.readExisting(ctx, in.TaskID, in.RepositoryID) + if beforeErr != nil && beforeErr != sql.ErrNoRows { + return UpsertResult{}, beforeErr + } + + id := uuid.New().String() + now := time.Now().UTC() + + var outcome sql.NullString + if in.Outcome != "" { + outcome = sql.NullString{String: string(in.Outcome), Valid: true} + } + var basis sql.NullString + if in.Basis != "" { + basis = sql.NullString{String: string(in.Basis), Valid: true} + } + + var reachedAt *time.Time + var reachedBasis, reachedRef *string + if in.ReachedDefault { + t := in.EvaluatedAt + reachedAt = &t + rb := string(in.ReachedBasis) + reachedBasis = &rb + // An empty ReachedRef (a merged, non-detached provider row with an + // empty pr_url/mr_url/pull_request_url — all TEXT NOT NULL with no + // DEFAULT) must store NULL, not '', in the write-once + // reached_default_ref column (R6-F3). + reachedRef = refPtr(in.ReachedRef) + } + + queryText := upsertSQL + if dialect.IsPostgres(r.db.DriverName()) { + // PostgreSQL must lock the source row while the INSERT ... SELECT is + // evaluated. This serializes the final liveness check with a concurrent + // soft delete; SQLite serializes both writers when the INSERT starts. + queryText = strings.Replace(queryText, "\nON CONFLICT", "\nFOR UPDATE\nON CONFLICT", 1) + } + query := r.db.Rebind(queryText) + var after existingRow + var afterBasis, afterRef sql.NullString + var afterObserved sql.NullInt64 + var afterReachedAt sql.NullTime + var afterUpdatedAt time.Time + var afterSeq int + err := r.db.QueryRowxContext(ctx, query, + id, in.TaskID, in.WorkspaceID, + outcome, basis, in.Ref, in.Rank, + reachedAt, reachedBasis, reachedRef, + in.ObservedBranchCommits, in.EvaluatedAt, + in.EvaluatedAt, 1, now, now, + in.RepositoryID, + ).Scan( + &after.EvidenceRank, &after.Outcome, &afterBasis, &afterRef, + &afterObserved, &afterReachedAt, &afterUpdatedAt, &afterSeq, + ) + if errors.Is(err, sql.ErrNoRows) { + return UpsertResult{}, errRepositoryNotLive + } + if err != nil { + return UpsertResult{}, err + } + + result := UpsertResult{} + if beforeErr == sql.ErrNoRows { + result.RowChanged = true + } else { + result.RowChanged = !afterUpdatedAt.Equal(before.updatedAt) + result.Demoted = in.Rank < before.EvidenceRank + result.DegradedOutcomeChanged = in.Rank == 1 && + before.EvidenceRank == 1 && + before.Outcome.String != after.Outcome.String + } + return result, nil +} + +type existingRowFull struct { + existingRow + updatedAt time.Time +} + +func (r *Repository) readExisting(ctx context.Context, taskID, repositoryID string) (existingRowFull, error) { + var row existingRowFull + err := r.db.QueryRowxContext(ctx, r.db.Rebind(` + SELECT evidence_rank, delivery_outcome, updated_at + FROM task_delivery_ledger + WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID).Scan(&row.EvidenceRank, &row.Outcome, &row.updatedAt) + return row, err +} diff --git a/apps/backend/internal/delivery/upsert_test.go b/apps/backend/internal/delivery/upsert_test.go new file mode 100644 index 0000000000..e62f3f4034 --- /dev/null +++ b/apps/backend/internal/delivery/upsert_test.go @@ -0,0 +1,515 @@ +package delivery_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/delivery" +) + +// ledgerRow is a raw read of every column, used to assert exact stored +// state without going through any repository accessor. +type ledgerRow struct { + Outcome sql.NullString + Basis sql.NullString + Ref sql.NullString + Rank int + ReachedAt sql.NullTime + ReachedBasis sql.NullString + ReachedRef sql.NullString + ObservedBranchCommits sql.NullInt64 + FirstClassifiedAt sql.NullTime + LastEvaluatedAt time.Time + EvaluationSeq int + WorkspaceID string + UpdatedAt time.Time +} + +func readLedgerRow(t *testing.T, db *sqlx.DB, taskID, repositoryID string) ledgerRow { + t.Helper() + var row ledgerRow + err := db.QueryRowxContext(context.Background(), db.Rebind(` + SELECT delivery_outcome, delivery_basis, delivery_ref, evidence_rank, + reached_default_at, reached_default_basis, reached_default_ref, + observed_branch_commits, first_classified_at, + last_evaluated_at, evaluation_seq, workspace_id, updated_at + FROM task_delivery_ledger WHERE task_id = ? AND repository_id = ? + `), taskID, repositoryID).Scan( + &row.Outcome, &row.Basis, &row.Ref, &row.Rank, + &row.ReachedAt, &row.ReachedBasis, &row.ReachedRef, + &row.ObservedBranchCommits, &row.FirstClassifiedAt, + &row.LastEvaluatedAt, &row.EvaluationSeq, &row.WorkspaceID, &row.UpdatedAt, + ) + if err != nil { + t.Fatalf("read ledger row: %v", err) + } + return row +} + +func setupPair(t *testing.T) (*delivery.Repository, *sqlx.DB, string, string, string) { + t.Helper() + repo, db := newTestRepo(t) + seedWorkspace(t, db, "ws-1") + seedRepository(t, db, "repo-1", "ws-1") + seedTask(t, db, "task-1", "ws-1") + return repo, db, "task-1", "repo-1", "ws-1" +} + +func TestUpsert_FreshInsert(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + result, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2, + }, + EvaluatedAt: t0, + }) + if err != nil { + t.Fatalf("upsert: %v", err) + } + if !result.RowChanged { + t.Fatal("first insert must report RowChanged") + } + + row := readLedgerRow(t, db, taskID, repoID) + if row.Outcome.String != string(delivery.OutcomeUnknown) || row.Rank != 2 { + t.Fatalf("row = %+v", row) + } + if row.EvaluationSeq != 1 { + t.Fatalf("evaluation_seq = %d, want 1 on insert", row.EvaluationSeq) + } + if !row.FirstClassifiedAt.Valid || !row.FirstClassifiedAt.Time.Equal(t0) { + t.Fatalf("first_classified_at = %v, want %v", row.FirstClassifiedAt, t0) + } + if !row.LastEvaluatedAt.Equal(t0) { + t.Fatalf("last_evaluated_at = %v, want %v", row.LastEvaluatedAt, t0) + } +} + +func TestUpsert_SoftDeletedRepositoryDoesNotWrite(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + if _, err := db.Exec(db.Rebind(`UPDATE repositories SET deleted_at = ? WHERE id = ?`), + time.Now().UTC(), repoID); err != nil { + t.Fatalf("soft delete repository: %v", err) + } + + _, err := repo.Upsert(context.Background(), delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2, + }, + EvaluatedAt: time.Now().UTC(), + }) + if err == nil { + t.Fatal("upsert should report that a soft-deleted repository was not written") + } + + var count int + if err := db.QueryRow(db.Rebind(` + SELECT COUNT(*) FROM task_delivery_ledger WHERE task_id = ? AND repository_id = ? + `), taskID, repoID).Scan(&count); err != nil { + t.Fatalf("count ledger rows: %v", err) + } + if count != 0 { + t.Fatalf("ledger rows = %d, want 0 for a soft-deleted repository", count) + } +} + +func TestUpsert_IdempotentReevaluationLeavesColumnsUnchanged(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC().Truncate(time.Second) + + in := delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, + ObservedBranchCommits: intp(12), + }, + EvaluatedAt: t0, + } + if _, err := repo.Upsert(ctx, in); err != nil { + t.Fatalf("first upsert: %v", err) + } + before := readLedgerRow(t, db, taskID, repoID) + + t1 := t0.Add(time.Minute) + in.EvaluatedAt = t1 + result, err := repo.Upsert(ctx, in) + if err != nil { + t.Fatalf("second upsert: %v", err) + } + if result.RowChanged { + t.Fatal("re-evaluation with unchanged inputs must not report RowChanged") + } + + after := readLedgerRow(t, db, taskID, repoID) + if after.Outcome != before.Outcome || after.Basis != before.Basis || after.Ref != before.Ref || + after.Rank != before.Rank || after.ObservedBranchCommits != before.ObservedBranchCommits { + t.Fatalf("classification columns changed: before=%+v after=%+v", before, after) + } + if !after.UpdatedAt.Equal(before.UpdatedAt) { + t.Fatalf("updated_at advanced on a no-op evaluation: before=%v after=%v", before.UpdatedAt, after.UpdatedAt) + } + if after.EvaluationSeq != before.EvaluationSeq+1 { + t.Fatalf("evaluation_seq = %d, want %d (advances every persisted evaluation)", after.EvaluationSeq, before.EvaluationSeq+1) + } + if !after.LastEvaluatedAt.Equal(t1) { + t.Fatalf("last_evaluated_at = %v, want %v", after.LastEvaluatedAt, t1) + } +} + +func TestUpsert_PromotionAdvancesRankAndDoesNotSuppressDemotion(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeNoDeliveryObserved, Basis: delivery.BasisNoCommitsObserved, Rank: 4}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + + result, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, ObservedBranchCommits: intp(12), + }, + EvaluatedAt: t0.Add(time.Minute), + }) + if err != nil { + t.Fatalf("promotion upsert: %v", err) + } + if !result.RowChanged || result.Demoted { + t.Fatalf("result = %+v, want RowChanged=true Demoted=false", result) + } + + row := readLedgerRow(t, db, taskID, repoID) + if row.Outcome.String != string(delivery.OutcomeUnknown) || row.Rank != 5 { + t.Fatalf("row = %+v, want promoted to rank 5", row) + } +} + +func TestUpsert_DemotionSuppressed(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + prURL := "https://pr/1" + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Ref: &prURL, Rank: 8, + ObservedBranchCommits: intp(12), + }, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + + // PR later detached: re-evaluates to a lower rank. + result, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5}, + EvaluatedAt: t0.Add(time.Minute), + }) + if err != nil { + t.Fatalf("demotion upsert: %v", err) + } + if !result.Demoted { + t.Fatal("expected Demoted=true") + } + if result.RowChanged { + t.Fatal("a suppressed demotion must not report RowChanged (classification columns are unchanged)") + } + + row := readLedgerRow(t, db, taskID, repoID) + if row.Outcome.String != string(delivery.OutcomePRMerge) || row.Rank != 8 || row.Ref.String != prURL { + t.Fatalf("row = %+v, want unchanged pr_merge at rank 8", row) + } + if !row.ObservedBranchCommits.Valid || row.ObservedBranchCommits.Int64 != 12 { + t.Fatalf("observed_branch_commits = %v, want unchanged 12", row.ObservedBranchCommits) + } +} + +func TestUpsert_EqualRankReselectsRef(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + prA := "https://pr/A" + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Ref: &prA, Rank: 8}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + + prB := "https://pr/B" + result, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Ref: &prB, Rank: 8}, + EvaluatedAt: t0.Add(time.Minute), + }) + if err != nil { + t.Fatalf("second upsert: %v", err) + } + if !result.RowChanged { + t.Fatal("a ref change at equal rank must report RowChanged") + } + + row := readLedgerRow(t, db, taskID, repoID) + if row.Ref.String != prB { + t.Fatalf("ref = %q, want prB (equal rank re-selects, never guaranteed unchanged)", row.Ref.String) + } +} + +func TestUpsert_EqualRankOneOutcomeChangeTracksInputsNotSilent(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisDefaultBranchUnknown, Rank: 1}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert (degraded pr_merge): %v", err) + } + before := readLedgerRow(t, db, taskID, repoID) + + // Detached, ahead=0 everywhere, still degraded: re-evaluates to + // no_delivery_observed at the same rank 1. + result, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeNoDeliveryObserved, Basis: delivery.BasisDefaultBranchUnknown, Rank: 1}, + EvaluatedAt: t0.Add(time.Minute), + }) + if err != nil { + t.Fatalf("second upsert: %v", err) + } + if result.Demoted { + t.Fatal("rank did not fall (1 -> 1); this must not count as a demotion") + } + if !result.DegradedOutcomeChanged { + t.Fatal("expected DegradedOutcomeChanged=true") + } + if !result.RowChanged { + t.Fatal("the write happens: outcome changed, so RowChanged must be true") + } + + row := readLedgerRow(t, db, taskID, repoID) + if row.Outcome.String != string(delivery.OutcomeNoDeliveryObserved) || row.Rank != 1 { + t.Fatalf("row = %+v, want no_delivery_observed at rank 1", row) + } + // Review round 2, finding #7: the original version of this test never + // checked updated_at, so a regression that left it frozen (the same + // bug updatedAtExpr's IS DISTINCT FROM comparison exists to prevent) + // would have gone unnoticed for exactly the scenario its own comment + // calls out — an outcome change with no rank change. + if !row.UpdatedAt.After(before.UpdatedAt) { + t.Fatalf("updated_at = %v, want after %v (an outcome change at unchanged rank must still advance it)", row.UpdatedAt, before.UpdatedAt) + } +} + +func TestUpsert_WriteOnceTripleNeverOverwritten(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, + ReachedDefault: true, ReachedBasis: delivery.ReachedBasisAncestorOfDefault, ReachedRef: "commit-a", + }, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + first := readLedgerRow(t, db, taskID, repoID) + if !first.ReachedAt.Valid || first.ReachedRef.String != "commit-a" { + t.Fatalf("first observation not stored: %+v", first) + } + + // A later evaluation observes the default branch again through a + // different basis: must not move the write-once triple. + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisReachedDefaultUnattributed, Rank: 6, + ReachedDefault: true, ReachedBasis: delivery.ReachedBasisProviderPRMerged, ReachedRef: "https://pr/2", + }, + EvaluatedAt: t0.Add(time.Minute), + }); err != nil { + t.Fatalf("second upsert: %v", err) + } + + second := readLedgerRow(t, db, taskID, repoID) + if !second.ReachedAt.Time.Equal(first.ReachedAt.Time) { + t.Fatalf("reached_default_at changed: %v -> %v", first.ReachedAt.Time, second.ReachedAt.Time) + } + if second.ReachedBasis.String != string(delivery.ReachedBasisAncestorOfDefault) || second.ReachedRef.String != "commit-a" { + t.Fatalf("write-once triple overwritten: %+v", second) + } +} + +func TestUpsert_ObservedBranchCommitsHighWaterNeverDecreases(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Rank: 8, ObservedBranchCommits: intp(12)}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + + // Snapshots deleted with their session: incoming value is nil. + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, ObservedBranchCommits: nil}, + EvaluatedAt: t0.Add(time.Minute), + }); err != nil { + t.Fatalf("second upsert: %v", err) + } + + row := readLedgerRow(t, db, taskID, repoID) + if !row.ObservedBranchCommits.Valid || row.ObservedBranchCommits.Int64 != 12 { + t.Fatalf("observed_branch_commits = %v, want still 12 (a NULL stored value is lower than any incoming value, and this direction must not decrease it)", row.ObservedBranchCommits) + } +} + +func TestUpsert_ObservedBranchCommitsNullStoredAcceptsIncoming(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2, ObservedBranchCommits: nil}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5, ObservedBranchCommits: intp(7)}, + EvaluatedAt: t0.Add(time.Minute), + }); err != nil { + t.Fatalf("second upsert: %v", err) + } + + row := readLedgerRow(t, db, taskID, repoID) + if !row.ObservedBranchCommits.Valid || row.ObservedBranchCommits.Int64 != 7 { + t.Fatalf("observed_branch_commits = %v, want 7 (NULL stored must not swallow an incoming value)", row.ObservedBranchCommits) + } +} + +func TestUpsert_LastEvaluatedAtIsHighWaterNeverRewound(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t3 := time.Now().UTC() + t0 := t3.Add(-time.Minute) + + // The faster evaluation (began at T3) commits first. + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: t3, + }); err != nil { + t.Fatalf("fast upsert: %v", err) + } + + // The slower evaluation, which began earlier at T0, commits second. + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisNoObservations, Rank: 2}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("slow upsert: %v", err) + } + + row := readLedgerRow(t, db, taskID, repoID) + if !row.LastEvaluatedAt.Equal(t3) { + t.Fatalf("last_evaluated_at = %v, want %v (a slow evaluation must not rewind the column)", row.LastEvaluatedAt, t3) + } +} + +// TestUpsert_ReachedDefaultWithEmptyRefStoresNull covers R6-F3: the +// provider tables' URL columns (pr_url / mr_url / pull_request_url) are +// TEXT NOT NULL with no DEFAULT, so a merged, non-detached row can carry +// an empty string. computeDefaultBranchObservation propagates that +// empty string as ReachedRef, and reached_default_ref is a write-once +// column — persisting ” there instead of NULL would violate the +// document-wide "NULL, not empty string" rule in spec "What" and "Data +// model" the moment it is ever written, since the column can never be +// overwritten afterwards. +func TestUpsert_ReachedDefaultWithEmptyRefStoresNull(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{ + Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisReachedDefaultUnattributed, Rank: 6, + ReachedDefault: true, ReachedBasis: delivery.ReachedBasisProviderPRMerged, ReachedRef: "", + }, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("upsert: %v", err) + } + + row := readLedgerRow(t, db, taskID, repoID) + if !row.ReachedAt.Valid { + t.Fatalf("reached_default_at not stored: %+v", row) + } + if row.ReachedRef.Valid { + t.Fatalf("reached_default_ref = %q, want NULL not empty string", row.ReachedRef.String) + } +} + +func TestUpsert_WorkspaceIDRefreshedUnconditionallyEvenOnDemotion(t *testing.T) { + repo, db, taskID, repoID, wsID := setupPair(t) + ctx := context.Background() + t0 := time.Now().UTC() + + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: wsID, + Classification: delivery.Classification{Outcome: delivery.OutcomePRMerge, Basis: delivery.BasisProviderPRMerged, Rank: 8}, + EvaluatedAt: t0, + }); err != nil { + t.Fatalf("first upsert: %v", err) + } + + seedWorkspace(t, db, "ws-2") + if _, err := repo.Upsert(ctx, delivery.UpsertInput{ + TaskID: taskID, RepositoryID: repoID, WorkspaceID: "ws-2", + Classification: delivery.Classification{Outcome: delivery.OutcomeUnknown, Basis: delivery.BasisBranchCommitsUnmerged, Rank: 5}, + EvaluatedAt: t0.Add(time.Minute), + }); err != nil { + t.Fatalf("demoted upsert with new workspace: %v", err) + } + + row := readLedgerRow(t, db, taskID, repoID) + if row.WorkspaceID != "ws-2" { + t.Fatalf("workspace_id = %q, want ws-2 (refreshed unconditionally, even on a suppressed demotion)", row.WorkspaceID) + } + if row.Outcome.String != string(delivery.OutcomePRMerge) || row.Rank != 8 { + t.Fatalf("classification must remain the higher-rank pr_merge: %+v", row) + } +} diff --git a/apps/backend/internal/office/dashboard/agent_summary.go b/apps/backend/internal/office/dashboard/agent_summary.go index 3d8a7394bd..3d572a86ae 100644 --- a/apps/backend/internal/office/dashboard/agent_summary.go +++ b/apps/backend/internal/office/dashboard/agent_summary.go @@ -40,13 +40,18 @@ type AgentLatestRunDTO struct { } // AgentRunActivityDay is one row of the Run Activity chart. -// `total = succeeded + failed + other`. +// `total = succeeded + skipped + unclassified + failed + other`; total is +// unchanged in value from before Skipped/Unclassified existed (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome § Response shapes") — +// only the partition of what used to be counted as succeeded changed. type AgentRunActivityDay struct { - Date string `json:"date"` - Succeeded int `json:"succeeded"` - Failed int `json:"failed"` - Other int `json:"other"` - Total int `json:"total"` + Date string `json:"date"` + Succeeded int `json:"succeeded"` + Skipped int `json:"skipped"` + Unclassified int `json:"unclassified"` + Failed int `json:"failed"` + Other int `json:"other"` + Total int `json:"total"` } // AgentTaskPriorityDay is one row of the Tasks by Priority chart. @@ -70,11 +75,18 @@ type AgentTaskStatusDay struct { Backlog int `json:"backlog"` } -// AgentSuccessRateDay is one row of the Success Rate chart. +// AgentSuccessRateDay is one row of the Success Rate chart. Succeeded now +// counts processed runs only; Unclassified is the marker that lets a +// reader tell a pre-activation day (unclassified accounts for the whole +// non-failed, non-other remainder) from a fully-classified one +// (unclassified == 0) without consulting kandev_meta (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome § Response shapes"). +// Total remains the same full denominator as before this field existed. type AgentSuccessRateDay struct { - Date string `json:"date"` - Succeeded int `json:"succeeded"` - Total int `json:"total"` + Date string `json:"date"` + Succeeded int `json:"succeeded"` + Unclassified int `json:"unclassified"` + Total int `json:"total"` } // AgentRecentTaskDTO is one row of the Recent Tasks list on the agent @@ -315,11 +327,13 @@ func padAgentRunActivity(dates []string, rows []sqlite.AgentRunDayRow) []AgentRu for i, date := range dates { row := byDate[date] out[i] = AgentRunActivityDay{ - Date: date, - Succeeded: row.Succeeded, - Failed: row.Failed, - Other: row.Other, - Total: row.Succeeded + row.Failed + row.Other, + Date: date, + Succeeded: row.Succeeded, + Skipped: row.Skipped, + Unclassified: row.Unclassified, + Failed: row.Failed, + Other: row.Other, + Total: row.Succeeded + row.Skipped + row.Unclassified + row.Failed + row.Other, } } return out @@ -375,9 +389,10 @@ func buildSuccessRate(dates []string, rows []sqlite.AgentRunDayRow) []AgentSucce for i, date := range dates { row := byDate[date] out[i] = AgentSuccessRateDay{ - Date: date, - Succeeded: row.Succeeded, - Total: row.Succeeded + row.Failed + row.Other, + Date: date, + Succeeded: row.Succeeded, + Unclassified: row.Unclassified, + Total: row.Succeeded + row.Skipped + row.Unclassified + row.Failed + row.Other, } } return out diff --git a/apps/backend/internal/office/dashboard/agent_summary_test.go b/apps/backend/internal/office/dashboard/agent_summary_test.go index 5e83e8b07a..f77130500c 100644 --- a/apps/backend/internal/office/dashboard/agent_summary_test.go +++ b/apps/backend/internal/office/dashboard/agent_summary_test.go @@ -24,7 +24,26 @@ const summaryFixtureAgent = "agent-A" // sane values so callers only specify the fields under test. Distinct // from run_detail_test.go's seedRun helper, which writes via the repo // (this one writes raw SQL so we control claimed_at + finished_at). +// +// A status='finished' row is seeded with outcome='processed', so +// existing callers that only cared about status keep reading as +// succeeded under the outcome-aware bucketing (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome"). Tests that need +// to exercise skipped/unclassified use seedSummaryRunWithOutcome. func seedSummaryRun(t *testing.T, db *sqlx.DB, runID, agentID, status, taskID string, requestedAt time.Time) { + t.Helper() + outcome := "" + if status == "finished" { + outcome = "processed" + } + seedSummaryRunWithOutcome(t, db, runID, agentID, status, outcome, taskID, requestedAt) +} + +// seedSummaryRunWithOutcome is seedSummaryRun plus an explicit outcome +// value. outcome == "" writes SQL NULL. +func seedSummaryRunWithOutcome( + t *testing.T, db *sqlx.DB, runID, agentID, status, outcome, taskID string, requestedAt time.Time, +) { t.Helper() payload := `{"task_id":"` + taskID + `"}` // Give the run a synthetic 1h duration: claimed_at = requested_at, @@ -33,12 +52,16 @@ func seedSummaryRun(t *testing.T, db *sqlx.DB, runID, agentID, status, taskID st // seeded a few minutes after the run's start need that window to // be open enough to include them. finishedAt := requestedAt.Add(1 * time.Hour) + var outcomeArg interface{} + if outcome != "" { + outcomeArg = outcome + } _, err := db.Exec(` INSERT INTO runs - (id, agent_profile_id, reason, payload, status, coalesced_count, + (id, agent_profile_id, reason, payload, status, outcome, coalesced_count, retry_count, requested_at, claimed_at, finished_at) - VALUES (?, ?, 'task_assigned', ?, ?, 1, 0, ?, ?, ?) - `, runID, agentID, payload, status, requestedAt.UTC().Format(time.RFC3339), + VALUES (?, ?, 'task_assigned', ?, ?, ?, 1, 0, ?, ?, ?) + `, runID, agentID, payload, status, outcomeArg, requestedAt.UTC().Format(time.RFC3339), requestedAt.UTC().Format(time.RFC3339), finishedAt.UTC().Format(time.RFC3339)) if err != nil { t.Fatalf("seed run %s: %v", runID, err) @@ -223,6 +246,56 @@ func TestGetAgentSummary_RunActivityBuckets(t *testing.T) { } } +// TestGetAgentSummary_RunActivityBuckets_SkippedAndUnclassified covers +// docs/specs/task-delivery-ledger/spec.md, "Scenarios § Office run +// outcome": a day with one processed, one budget_blocked (skipped) and +// one pre-activation NULL-outcome (unclassified) finished run reports all +// three fields and total stays the sum of all five buckets; a day made +// entirely of pre-activation runs reports succeeded=0, unclassified=N and +// total=N, so a reader can tell the day predates the mechanism without +// consulting kandev_meta. +func TestGetAgentSummary_RunActivityBuckets_SkippedAndUnclassified(t *testing.T) { + deps := newTestDeps(t) + now := time.Now().UTC() + d0 := now + d1 := now.AddDate(0, 0, -1) + + seedSummaryRunWithOutcome(t, deps.db, "r-proc", summaryFixtureAgent, "finished", "processed", "t1", d0) + seedSummaryRunWithOutcome(t, deps.db, "r-skip", summaryFixtureAgent, "finished", "budget_blocked", "t1", d0) + seedSummaryRunWithOutcome(t, deps.db, "r-null", summaryFixtureAgent, "finished", "", "t1", d0) + + seedSummaryRunWithOutcome(t, deps.db, "r-y1", summaryFixtureAgent, "finished", "", "t2", d1) + seedSummaryRunWithOutcome(t, deps.db, "r-y2", summaryFixtureAgent, "finished", "", "t2", d1) + seedSummaryRunWithOutcome(t, deps.db, "r-y3", summaryFixtureAgent, "finished", "", "t2", d1) + + resp := fetchAgentSummary(t, deps, summaryFixtureAgent, 14) + + activityByDate := map[string]dashboard.AgentRunActivityDay{} + for _, b := range resp.RunActivity { + activityByDate[b.Date] = b + } + rateByDate := map[string]dashboard.AgentSuccessRateDay{} + for _, sr := range resp.SuccessRate { + rateByDate[sr.Date] = sr + } + d0Key, d1Key := d0.Format("2006-01-02"), d1.Format("2006-01-02") + + if got := activityByDate[d0Key]; got.Succeeded != 1 || got.Skipped != 1 || got.Unclassified != 1 || got.Total != 3 { + t.Errorf("today activity = %#v, want succeeded=1 skipped=1 unclassified=1 total=3", got) + } + if got := rateByDate[d0Key]; got.Succeeded != 1 || got.Unclassified != 1 || got.Total != 3 { + t.Errorf("today success rate = %#v, want succeeded=1 unclassified=1 total=3", got) + } + + if got := activityByDate[d1Key]; got.Succeeded != 0 || got.Skipped != 0 || got.Unclassified != 3 || got.Failed != 0 || got.Other != 0 || got.Total != 3 { + t.Errorf("all-pre-activation day activity = %#v, want succeeded=0 unclassified=3 total=3", got) + } + if got := rateByDate[d1Key]; got.Succeeded != 0 || got.Unclassified != 3 || got.Total != 3 { + t.Errorf("all-pre-activation day success rate = %#v, want succeeded=0 unclassified=3 total=3 "+ + "(reader can tell this day predates the mechanism from the response alone)", got) + } +} + // TestGetAgentSummary_TaskBuckets pins the priority + status charts. func TestGetAgentSummary_TaskBuckets(t *testing.T) { deps := newTestDeps(t) diff --git a/apps/backend/internal/office/models/models.go b/apps/backend/internal/office/models/models.go index af6b4e4a60..3761150459 100644 --- a/apps/backend/internal/office/models/models.go +++ b/apps/backend/internal/office/models/models.go @@ -425,6 +425,14 @@ type Run struct { ClaimedAt *time.Time `json:"claimed_at" db:"claimed_at"` FinishedAt *time.Time `json:"finished_at" db:"finished_at"` + // Outcome records why a finished run ended (docs/specs/ + // task-delivery-ledger/spec.md, "Office run outcome"): one of eight + // values on the finished path, NULL on failed and on every + // pre-activation row. Pointer-typed so StructScan reads the NULL + // every pre-activation row carries, same idiom as the provider- + // routing columns below. + Outcome *string `json:"outcome,omitempty" db:"outcome"` + // Provider-routing columns (office-provider-routing spec). All // optional and ignored when workspace routing is disabled. The TEXT // columns are pointer-typed so SELECT * StructScan handles the NULL diff --git a/apps/backend/internal/office/repository/sqlite/agent_summary.go b/apps/backend/internal/office/repository/sqlite/agent_summary.go index 49a2be72b8..31b8a5ca0a 100644 --- a/apps/backend/internal/office/repository/sqlite/agent_summary.go +++ b/apps/backend/internal/office/repository/sqlite/agent_summary.go @@ -10,11 +10,19 @@ import ( // AgentRunDayRow holds per-day run outcome counts for one agent. // Status values mirror the runs.status column: // queued | claimed | finished | failed | cancelled. +// +// The five buckets are total over every (status, outcome) combination +// (docs/specs/task-delivery-ledger/spec.md, "Office run outcome § +// Repository query"): succeeded and skipped and unclassified partition +// status='finished' by outcome; failed and other are unchanged by that +// card. type AgentRunDayRow struct { - Date string `db:"date"` - Succeeded int `db:"succeeded"` - Failed int `db:"failed"` - Other int `db:"other"` + Date string `db:"date"` + Succeeded int `db:"succeeded"` + Skipped int `db:"skipped"` + Unclassified int `db:"unclassified"` + Failed int `db:"failed"` + Other int `db:"other"` } // RunCountsByDayForAgent returns per-day run outcome counts for an agent @@ -22,9 +30,11 @@ type AgentRunDayRow struct { // the start of (today - days + 1) so the result range is inclusive of // today and `days` days back. // -// finished → succeeded; failed (and timed_out, treated as failed -// upstream) → failed; everything else (cancelled, queued, claimed) → -// other. +// finished + outcome='processed' → succeeded; finished + any other +// non-NULL outcome → skipped; finished + outcome IS NULL → unclassified +// (pre-activation rows, and any finished path with no established +// semantic label); failed (and timed_out, treated as failed upstream) → +// failed; everything else (cancelled, queued, claimed) → other. func (r *Repository) RunCountsByDayForAgent( ctx context.Context, agentID string, days int, ) ([]AgentRunDayRow, error) { @@ -35,7 +45,9 @@ func (r *Repository) RunCountsByDayForAgent( var rows []AgentRunDayRow err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(` SELECT strftime('%Y-%m-%d', requested_at) AS date, - SUM(CASE WHEN status = 'finished' THEN 1 ELSE 0 END) AS succeeded, + SUM(CASE WHEN status = 'finished' AND outcome = 'processed' THEN 1 ELSE 0 END) AS succeeded, + SUM(CASE WHEN status = 'finished' AND outcome IS NOT NULL AND outcome != 'processed' THEN 1 ELSE 0 END) AS skipped, + SUM(CASE WHEN status = 'finished' AND outcome IS NULL THEN 1 ELSE 0 END) AS unclassified, SUM(CASE WHEN status IN ('failed','timed_out') THEN 1 ELSE 0 END) AS failed, SUM(CASE WHEN status NOT IN ('finished','failed','timed_out') THEN 1 ELSE 0 END) AS other FROM runs diff --git a/apps/backend/internal/office/repository/sqlite/agent_summary_test.go b/apps/backend/internal/office/repository/sqlite/agent_summary_test.go index 318f68cecc..d3baa8ab82 100644 --- a/apps/backend/internal/office/repository/sqlite/agent_summary_test.go +++ b/apps/backend/internal/office/repository/sqlite/agent_summary_test.go @@ -36,6 +36,27 @@ func seedSummaryRun( } } +// seedSummaryRunWithOutcome is seedSummaryRun plus an explicit outcome +// value, for the task-delivery-ledger Office run outcome bucketing tests. +// outcome == "" writes SQL NULL (pre-activation / failed-path shape). +func seedSummaryRunWithOutcome( + t *testing.T, repo *sqlite.Repository, + id, agentID, status, requestedAt, outcome string, +) { + t.Helper() + var outcomeArg interface{} + if outcome != "" { + outcomeArg = outcome + } + if _, err := repo.ExecRaw(context.Background(), ` + INSERT INTO runs + (id, agent_profile_id, reason, payload, status, error_message, outcome, requested_at) + VALUES (?, ?, 'test', '{}', ?, '', ?, ?) + `, id, agentID, status, outcomeArg, requestedAt); err != nil { + t.Fatalf("seed run %s: %v", id, err) + } +} + // seedCostEvent inserts an office_cost_events row with every summed column set. func seedCostEvent( t *testing.T, repo *sqlite.Repository, @@ -64,20 +85,29 @@ func seedActivity(t *testing.T, repo *sqlite.Repository, id, actorID, targetType } } +// TestRunCountsByDayForAgent_BucketsStatuses covers the five-bucket +// contract in docs/specs/task-delivery-ledger/spec.md, "Office run +// outcome § Repository query": succeeded requires status='finished' AND +// outcome='processed'; any other non-NULL outcome on a finished run is +// skipped; a finished run with outcome IS NULL (pre-activation, or a +// terminal path with no established semantic label) is unclassified; +// failed/other are unchanged by this card. func TestRunCountsByDayForAgent_BucketsStatuses(t *testing.T) { repo := newTestRepo(t) ctx := context.Background() base := time.Now().UTC() today, yday, ancient := dayStamp(base, 0, 10), dayStamp(base, 1, 10), dayStamp(base, 60, 10) - seedSummaryRun(t, repo, "r-fin", "agent-a", "finished", today, "", "", "") + seedSummaryRunWithOutcome(t, repo, "r-fin", "agent-a", "finished", today, "processed") + seedSummaryRunWithOutcome(t, repo, "r-skip", "agent-a", "finished", today, "budget_blocked") + seedSummaryRunWithOutcome(t, repo, "r-unclassified", "agent-a", "finished", today, "") seedSummaryRun(t, repo, "r-fail", "agent-a", "failed", today, "", "", "") seedSummaryRun(t, repo, "r-timeout", "agent-a", "timed_out", today, "", "", "") seedSummaryRun(t, repo, "r-queued", "agent-a", "queued", today, "", "", "") seedSummaryRun(t, repo, "r-cancel", "agent-a", "cancelled", today, "", "", "") - seedSummaryRun(t, repo, "r-yday", "agent-a", "finished", yday, "", "", "") - seedSummaryRun(t, repo, "r-old", "agent-a", "finished", ancient, "", "", "") - seedSummaryRun(t, repo, "r-other", "agent-b", "finished", today, "", "", "") + seedSummaryRunWithOutcome(t, repo, "r-yday", "agent-a", "finished", yday, "") + seedSummaryRunWithOutcome(t, repo, "r-old", "agent-a", "finished", ancient, "processed") + seedSummaryRunWithOutcome(t, repo, "r-other", "agent-b", "finished", today, "processed") rows, err := repo.RunCountsByDayForAgent(ctx, "agent-a", 14) if err != nil { @@ -90,15 +120,42 @@ func TestRunCountsByDayForAgent_BucketsStatuses(t *testing.T) { if rows[0].Date != dayDate(base, 1) { t.Errorf("rows[0].Date = %q, want %q (ORDER BY date ascending)", rows[0].Date, dayDate(base, 1)) } - if want := (sqlite.AgentRunDayRow{Date: dayDate(base, 1), Succeeded: 1}); rows[0] != want { - t.Errorf("yesterday = %+v, want %+v", rows[0], want) + if want := (sqlite.AgentRunDayRow{Date: dayDate(base, 1), Unclassified: 1}); rows[0] != want { + t.Errorf("yesterday = %+v, want %+v (NULL outcome is unclassified, not succeeded)", rows[0], want) + } + want := sqlite.AgentRunDayRow{ + Date: dayDate(base, 0), Succeeded: 1, Skipped: 1, Unclassified: 1, Failed: 2, Other: 2, } - want := sqlite.AgentRunDayRow{Date: dayDate(base, 0), Succeeded: 1, Failed: 2, Other: 2} if rows[1] != want { t.Errorf("today = %+v, want %+v (timed_out counts as failed; queued+cancelled as other)", rows[1], want) } } +// TestRunCountsByDayForAgent_UnrecognisedOutcomeCountsSkipped covers the +// bucketing-is-total scenario: a finished run whose outcome holds a value +// outside the eight named ones still counts in skipped, and the query +// never errors. +func TestRunCountsByDayForAgent_UnrecognisedOutcomeCountsSkipped(t *testing.T) { + repo := newTestRepo(t) + ctx := context.Background() + base := time.Now().UTC() + today := dayStamp(base, 0, 10) + + seedSummaryRunWithOutcome(t, repo, "r-weird", "agent-a", "finished", today, "some_future_value") + + rows, err := repo.RunCountsByDayForAgent(ctx, "agent-a", 14) + if err != nil { + t.Fatalf("RunCountsByDayForAgent: %v", err) + } + if len(rows) != 1 { + t.Fatalf("rows = %d, want 1: %+v", len(rows), rows) + } + want := sqlite.AgentRunDayRow{Date: dayDate(base, 0), Skipped: 1} + if rows[0] != want { + t.Errorf("today = %+v, want %+v", rows[0], want) + } +} + func TestRunCountsByDayForAgent_NonPositiveDaysDefaultsTo14(t *testing.T) { repo := newTestRepo(t) ctx := context.Background() diff --git a/apps/backend/internal/office/repository/sqlite/base.go b/apps/backend/internal/office/repository/sqlite/base.go index 9b13a514ce..126fb9d9c3 100644 --- a/apps/backend/internal/office/repository/sqlite/base.go +++ b/apps/backend/internal/office/repository/sqlite/base.go @@ -118,6 +118,7 @@ func (r *Repository) initSchema() error { return err } r.runMigrations() + r.activateRunOutcome() return nil } @@ -345,6 +346,11 @@ func (r *Repository) createRunTables() error { scheduled_retry_at TIMESTAMP, error_message TEXT NOT NULL DEFAULT '', cancel_reason TEXT, + -- outcome (task-delivery-ledger): nullable, one of the eight + -- Office run outcome values on the finished path, NULL on failed + -- and on every pre-activation row. See migrateRunOutcome for the + -- ADD COLUMN that converges existing databases. + outcome TEXT, -- Provider routing (office-provider-routing). logical_provider_order TEXT, requested_tier TEXT, diff --git a/apps/backend/internal/office/repository/sqlite/base_migrations.go b/apps/backend/internal/office/repository/sqlite/base_migrations.go index a8cc0bb899..0c1b1e6141 100644 --- a/apps/backend/internal/office/repository/sqlite/base_migrations.go +++ b/apps/backend/internal/office/repository/sqlite/base_migrations.go @@ -39,6 +39,7 @@ func (r *Repository) runMigrations() { // schemas include the columns inline; ALTERs converge existing databases. r.migrateProviderRouting() r.migrateContinuationScope() + r.migrateRunOutcome() r.migrateParentWakeIndexes() r.migrateParentWakeReceiptColumns() } diff --git a/apps/backend/internal/office/repository/sqlite/run_outcome_activation.go b/apps/backend/internal/office/repository/sqlite/run_outcome_activation.go new file mode 100644 index 0000000000..200c6622f1 --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/run_outcome_activation.go @@ -0,0 +1,99 @@ +package sqlite + +import ( + "database/sql" + "fmt" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/db/dialect" + "github.com/kandev/kandev/internal/persistence" +) + +// runOutcomeActivationKey is the kandev_meta key published once a boot has +// verified runs.outcome is present. See docs/specs/task-delivery-ledger/ +// spec.md, "Activation points". +const runOutcomeActivationKey = "telemetry.run_outcome.activated_at" + +// migrateRunOutcome adds the nullable runs.outcome column. Legacy rows and +// every row written before activation carry NULL, never a guessed value. +func (r *Repository) migrateRunOutcome() { + r.migrate.Apply("runs.outcome", "ALTER TABLE runs ADD COLUMN outcome TEXT") +} + +// activateRunOutcome writes telemetry.run_outcome.activated_at only after a +// positive probe confirms runs.outcome exists. The migration runner +// (MigrateLogger.Apply) swallows failures at WARN, so this probe is the +// only thing standing between a failed migration and a consumer wrongly +// believing the mechanism is live. WriteMetaKeyIfAbsent makes the write +// replay-safe: the instant is never overwritten once set. Any failure here +// is logged and swallowed — activation is a best-effort published fact, +// never a boot-blocking requirement, and the sweep-equivalent writer path +// (Office's own FinishRun calls) does not gate on this key. +func (r *Repository) activateRunOutcome() { + if err := persistence.EnsureMetaTable(r.db); err != nil { + r.logActivationWarn("ensure kandev_meta table failed", err) + return + } + exists, err := columnExists(r.db, "runs", "outcome") + if err != nil { + r.logActivationWarn("runs.outcome schema probe failed", err) + return + } + if !exists { + return + } + if _, err := persistence.WriteMetaKeyIfAbsent( + r.db, runOutcomeActivationKey, time.Now().UTC().Format(time.RFC3339), + ); err != nil { + r.logActivationWarn("write run outcome activation key failed", err) + } +} + +func (r *Repository) logActivationWarn(msg string, err error) { + if r.log != nil { + r.log.Warn(msg, zap.Error(err)) + } +} + +// columnExists reports whether table declares column, on either dialect. +// SQLite has no information_schema, so it falls back to PRAGMA table_info; +// Postgres is queried through the standard catalog view scoped to the +// current schema so callers never see another schema's same-named table. +func columnExists(db interface { + DriverName() string + Query(string, ...interface{}) (*sql.Rows, error) + QueryRow(string, ...interface{}) *sql.Row + Rebind(string) string +}, table, column string) (bool, error) { + if dialect.IsPostgres(db.DriverName()) { + var exists bool + err := db.QueryRow(db.Rebind(` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = ? + AND column_name = ? + ) + `), table, column).Scan(&exists) + return exists, err + } + rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) + if err != nil { + return false, err + } + defer func() { _ = rows.Close() }() + for rows.Next() { + var cid, notnull, pk int + var name, ctype string + var dflt sql.NullString + if err := rows.Scan(&cid, &name, &ctype, ¬null, &dflt, &pk); err != nil { + return false, err + } + if name == column { + return true, nil + } + } + return false, rows.Err() +} diff --git a/apps/backend/internal/office/repository/sqlite/run_outcome_migration_test.go b/apps/backend/internal/office/repository/sqlite/run_outcome_migration_test.go new file mode 100644 index 0000000000..04e8da246f --- /dev/null +++ b/apps/backend/internal/office/repository/sqlite/run_outcome_migration_test.go @@ -0,0 +1,128 @@ +package sqlite_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/repository/sqlite" + "github.com/kandev/kandev/internal/persistence" + taskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + "github.com/kandev/kandev/internal/testutil" +) + +const runOutcomeActivationKey = "telemetry.run_outcome.activated_at" + +// TestRunOutcomeMigration_AddsNullableColumn covers the Migration and +// activation scenario: a database created before this feature gets +// runs.outcome, and it is NULL on every pre-existing row (docs/specs/ +// task-delivery-ledger/spec.md, "Scenarios § Migration and activation"). +func TestRunOutcomeMigration_AddsNullableColumn(t *testing.T) { + repo, db := newTestRepoWithDB(t) + seedAgentProfile(t, db, "agent-outcome-migration", "ws-outcome-migration") + ctx := context.Background() + + run := &models.Run{ + AgentProfileID: "agent-outcome-migration", + Reason: "task_assigned", + Payload: `{}`, + Status: "queued", + CoalescedCount: 1, + } + if err := repo.CreateRun(ctx, run); err != nil { + t.Fatalf("create run: %v", err) + } + + var outcome sql.NullString + if err := db.Get(&outcome, db.Rebind(`SELECT outcome FROM runs WHERE id = ?`), run.ID); err != nil { + t.Fatalf("select outcome: %v", err) + } + if outcome.Valid { + t.Fatalf("outcome on freshly-queued run = %q, want NULL", outcome.String) + } +} + +// TestRunOutcomeActivation_WrittenOnceAfterSchemaProbe covers "Activation +// points": the key is written only after a positive probe confirms +// runs.outcome exists, and it is never overwritten on replay. +func TestRunOutcomeActivation_WrittenOnceAfterSchemaProbe(t *testing.T) { + _, db := newTestRepoWithDB(t) + + val, err := persistence.ReadMetaKey(db, runOutcomeActivationKey) + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if val == "" { + t.Fatal("expected telemetry.run_outcome.activated_at to be written after first boot") + } + if _, err := time.Parse(time.RFC3339, val); err != nil { + t.Fatalf("activation value %q is not RFC3339: %v", val, err) + } + + // Overwrite with a sentinel to prove a replayed boot never re-writes + // (spec: "never overwritten on replay"). + if _, err := db.Exec(db.Rebind( + `UPDATE kandev_meta SET value = ? WHERE key = ?`, + ), "sentinel-value", runOutcomeActivationKey); err != nil { + t.Fatalf("seed sentinel: %v", err) + } + + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay init: %v", err) + } + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay init twice: %v", err) + } + + got, err := persistence.ReadMetaKey(db, runOutcomeActivationKey) + if err != nil { + t.Fatalf("read activation key after replay: %v", err) + } + if got != "sentinel-value" { + t.Fatalf("activation key = %q after replay, want unchanged sentinel-value", got) + } +} + +// TestPostgresRunOutcomeMigration_AddsColumnAndActivates is the PostgreSQL +// twin required by ADR 0027: fresh init and replay both succeed and the +// runs.outcome column and activation key exist. Skips unless +// KANDEV_TEST_POSTGRES_DSN is set. +// +// The office schema's runs table has a foreign key onto tasks, so the tasks +// table must exist first — initialize via taskrepo, mirroring production +// boot order (see internal/office/repository/sqlite/workflow_test.go). +func TestPostgresRunOutcomeMigration_AddsColumnAndActivates(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + if _, err := taskrepo.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init task repo: %v", err) + } + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("init postgres schema: %v", err) + } + // Replay against the same database. + if _, err := sqlite.NewWithDB(db, db, nil); err != nil { + t.Fatalf("replay postgres schema: %v", err) + } + + var dataType string + err := db.QueryRow(` + SELECT data_type FROM information_schema.columns + WHERE table_schema = current_schema() AND table_name = 'runs' AND column_name = 'outcome' + `).Scan(&dataType) + if err != nil { + t.Fatalf("inspect runs.outcome: %v", err) + } + if dataType != "text" { + t.Fatalf("runs.outcome data type = %q, want text", dataType) + } + + val, err := persistence.ReadMetaKey(db, runOutcomeActivationKey) + if err != nil { + t.Fatalf("read activation key: %v", err) + } + if val == "" { + t.Fatal("expected activation key to be written on postgres") + } +} diff --git a/apps/backend/internal/office/repository/sqlite/runs_test.go b/apps/backend/internal/office/repository/sqlite/runs_test.go index d7af7cae60..256d2e45bc 100644 --- a/apps/backend/internal/office/repository/sqlite/runs_test.go +++ b/apps/backend/internal/office/repository/sqlite/runs_test.go @@ -242,7 +242,7 @@ func TestCleanExpired(t *testing.T) { t.Fatalf("create: %v", err) } // Finish it. - if err := repo.FinishRun(ctx, req.ID, "finished"); err != nil { + if err := repo.FinishRun(ctx, req.ID, "finished", strPtr("processed")); err != nil { t.Fatalf("finish: %v", err) } diff --git a/apps/backend/internal/office/scheduler/run_processing.go b/apps/backend/internal/office/scheduler/run_processing.go index 5573954992..755d84fa59 100644 --- a/apps/backend/internal/office/scheduler/run_processing.go +++ b/apps/backend/internal/office/scheduler/run_processing.go @@ -27,14 +27,19 @@ func (ss *SchedulerService) ClaimNextRun(ctx context.Context) (*models.Run, erro return req, nil } -// FinishRun marks a claimed run as finished. +// FinishRun marks a claimed run as finished. outcome is written as NULL: +// this dormant parallel copy of the service-layer FinishRun has no +// production caller and therefore no established semantic label for what +// happened, so it is not bucketed as processed (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome"). func (ss *SchedulerService) FinishRun(ctx context.Context, id string) error { - return ss.repo.FinishRun(ctx, id, RunStatusFinished) + return ss.repo.FinishRun(ctx, id, RunStatusFinished, nil) } -// FailRun marks a claimed run as failed. +// FailRun marks a claimed run as failed. outcome is written as NULL, same +// as the service-layer FailRun: a failed run is bucketed on status alone. func (ss *SchedulerService) FailRun(ctx context.Context, id string) error { - return ss.repo.FinishRun(ctx, id, RunStatusFailed) + return ss.repo.FinishRun(ctx, id, RunStatusFailed, nil) } // ProcessRunGuard checks if the agent is still eligible to be woken. diff --git a/apps/backend/internal/office/scheduler/run_processing_test.go b/apps/backend/internal/office/scheduler/run_processing_test.go new file mode 100644 index 0000000000..6b91ab4674 --- /dev/null +++ b/apps/backend/internal/office/scheduler/run_processing_test.go @@ -0,0 +1,149 @@ +package scheduler_test + +// Covers docs/specs/task-delivery-ledger/spec.md, "Office run outcome": +// SchedulerService.FinishRun and .FailRun (run_processing.go) are a dormant +// parallel copy of the service-layer FinishRun/FailRun with no production +// caller today, but the spec names them explicitly (scenarios at lines +// 2077-2085) because they reach the same runs table through a different +// path than transitionRunTerminal: outcome must be written NULL on both, +// never one of the eight-value vocabulary invented for a path with no +// established semantic label. + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/scheduler" +) + +func seedRunForProcessing(t *testing.T, repo interface { + CreateRun(ctx context.Context, req *models.Run) error +}, agentID, reason string) *models.Run { + t.Helper() + run := &models.Run{ + ID: uuid.New().String(), + AgentProfileID: agentID, + Reason: reason, + Payload: `{}`, + Status: scheduler.RunStatusClaimed, + CoalescedCount: 1, + RequestedAt: time.Now().UTC(), + } + if err := repo.CreateRun(context.Background(), run); err != nil { + t.Fatalf("seed run: %v", err) + } + return run +} + +// TestSchedulerService_FinishRun_WritesFinishedWithNilOutcome covers +// run_processing.go:35-37. This path has no established semantic label for +// what happened, so it must never be bucketed as "processed" or any other +// value from the eight-value vocabulary. +func TestSchedulerService_FinishRun_WritesFinishedWithNilOutcome(t *testing.T) { + repo := newTestRepoSched(t) + ss := buildScheduler(t, repo, newFakeTaskStarter()) + ctx := context.Background() + + run := seedRunForProcessing(t, repo, testAgentID, scheduler.RunReasonTaskAssigned) + + if err := ss.FinishRun(ctx, run.ID); err != nil { + t.Fatalf("finish run: %v", err) + } + + got, err := repo.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if got.Status != scheduler.RunStatusFinished { + t.Fatalf("status = %q, want %q", got.Status, scheduler.RunStatusFinished) + } + if got.Outcome != nil { + t.Fatalf("outcome = %q, want nil", *got.Outcome) + } +} + +// TestSchedulerService_FailRun_WritesFailedWithNilOutcome covers +// run_processing.go:39-43 — bucketed on status alone, same as the +// service-layer FailRun. +func TestSchedulerService_FailRun_WritesFailedWithNilOutcome(t *testing.T) { + repo := newTestRepoSched(t) + ss := buildScheduler(t, repo, newFakeTaskStarter()) + ctx := context.Background() + + run := seedRunForProcessing(t, repo, testAgentID, scheduler.RunReasonTaskAssigned) + + if err := ss.FailRun(ctx, run.ID); err != nil { + t.Fatalf("fail run: %v", err) + } + + got, err := repo.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if got.Status != scheduler.RunStatusFailed { + t.Fatalf("status = %q, want %q", got.Status, scheduler.RunStatusFailed) + } + if got.Outcome != nil { + t.Fatalf("outcome = %q, want nil", *got.Outcome) + } +} + +// TestSchedulerService_ClaimNextRun_ReturnsQueuedRun exercises ClaimNextRun +// (run_processing.go:15-28), the third method in this file, so the whole +// dormant-copy surface has direct coverage rather than only the two +// outcome-bearing terminal methods. +func TestSchedulerService_ClaimNextRun_ReturnsQueuedRun(t *testing.T) { + repo := newTestRepoSched(t) + ss := buildScheduler(t, repo, newFakeTaskStarter()) + ctx := context.Background() + + settingsAgent := &models.AgentInstance{ + ID: testAgentID, + WorkspaceID: testWorkspaceID, + Name: "claim-next-agent", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + } + if err := repo.CreateAgentInstance(ctx, settingsAgent); err != nil { + t.Fatalf("create agent: %v", err) + } + + queued := &models.Run{ + ID: uuid.New().String(), + AgentProfileID: testAgentID, + Reason: scheduler.RunReasonTaskAssigned, + Payload: `{}`, + Status: scheduler.RunStatusQueued, + CoalescedCount: 1, + RequestedAt: time.Now().UTC(), + } + if err := repo.CreateRun(ctx, queued); err != nil { + t.Fatalf("seed queued run: %v", err) + } + + claimed, err := ss.ClaimNextRun(ctx) + if err != nil { + t.Fatalf("claim: %v", err) + } + if claimed == nil { + t.Fatal("expected a claimed run") + } + if claimed.ID != queued.ID { + t.Fatalf("claimed id = %q, want %q", claimed.ID, queued.ID) + } + if claimed.Status != scheduler.RunStatusClaimed { + t.Fatalf("status = %q, want %q", claimed.Status, scheduler.RunStatusClaimed) + } + + next, err := ss.ClaimNextRun(ctx) + if err != nil { + t.Fatalf("claim again: %v", err) + } + if next != nil { + t.Fatalf("expected nil on empty queue, got %+v", next) + } +} diff --git a/apps/backend/internal/office/service/event_subscribers.go b/apps/backend/internal/office/service/event_subscribers.go index 2cddebfc08..5b001bdc36 100644 --- a/apps/backend/internal/office/service/event_subscribers.go +++ b/apps/backend/internal/office/service/event_subscribers.go @@ -405,7 +405,7 @@ func (s *Service) handleAgentCompleted(ctx context.Context, event *bus.Event) er // must stay held so ReapStaleCheckouts (not a same-agent race) is what // eventually reclaims it, instead of releasing a lock for a run that // never actually reached a terminal state. - if err := s.FinishRun(ctx, run.ID); err != nil { + if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil { return err } s.releaseTaskCheckoutForRun(ctx, run) @@ -509,7 +509,7 @@ func (s *Service) handleTasklessAgentCompleted( // // Finish before releasing, same as handleAgentCompleted: a failed // FinishRun must not still give up the checkout. - if err := s.FinishRun(ctx, run.ID); err != nil { + if err := s.FinishRun(ctx, run.ID, RunOutcomeProcessed); err != nil { return err } s.releaseTaskCheckoutForRun(ctx, run) diff --git a/apps/backend/internal/office/service/run.go b/apps/backend/internal/office/service/run.go index f0a272f7ef..b21dfbe159 100644 --- a/apps/backend/internal/office/service/run.go +++ b/apps/backend/internal/office/service/run.go @@ -53,6 +53,20 @@ const ( RunStatusCancelled = "cancelled" ) +// Run outcome constants (docs/specs/task-delivery-ledger/spec.md, "Office run +// outcome"). Written into runs.outcome alongside status='finished' at each of +// the six terminal call sites; NULL on the failed path and on every +// pre-activation row. RunOutcomeProcessed is the only value +// RunCountsByDayForAgent counts as succeeded. Every other value buckets into +// skipped. +const ( + RunOutcomeProcessed = "processed" + RunOutcomeBudgetBlocked = "budget_blocked" + RunOutcomeIdleSkipped = "idle_skipped" + RunOutcomeAgentInactive = "agent_inactive" + RunOutcomeTaskTreeHeld = "task_tree_held" +) + // CoalesceWindowSeconds is the default coalescing window. const CoalesceWindowSeconds = 5 diff --git a/apps/backend/internal/office/service/run_lifecycle_events_test.go b/apps/backend/internal/office/service/run_lifecycle_events_test.go index 67a414fa64..dd818d50d9 100644 --- a/apps/backend/internal/office/service/run_lifecycle_events_test.go +++ b/apps/backend/internal/office/service/run_lifecycle_events_test.go @@ -314,7 +314,7 @@ func TestFinishRun_PublishesOfficeRunProcessed(t *testing.T) { } defer func() { _ = sub.Unsubscribe() }() - if err := svc.FinishRun(ctx, run.ID); err != nil { + if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -375,7 +375,7 @@ func TestFinishRun_PublishesOfficeRunProcessedForSourceCommentTask(t *testing.T) } defer func() { _ = sub.Unsubscribe() }() - if err := svc.FinishRun(ctx, run.ID); err != nil { + if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } diff --git a/apps/backend/internal/office/service/scheduler_integration.go b/apps/backend/internal/office/service/scheduler_integration.go index 94319d3e09..e825d96a88 100644 --- a/apps/backend/internal/office/service/scheduler_integration.go +++ b/apps/backend/internal/office/service/scheduler_integration.go @@ -215,7 +215,7 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run) si.logger.Info("run skipped (agent not active)", zap.String("run_id", runID), zap.String("agent_status", string(agent.Status))) - _ = si.svc.FinishRun(ctx, runID) + _ = si.svc.FinishRun(ctx, runID, RunOutcomeAgentInactive) return } @@ -244,7 +244,7 @@ func (si *SchedulerIntegration) processRun(ctx context.Context, run *models.Run) "agent": agent.Name, "agent_id": agent.ID, }), runID, "") - _ = si.svc.FinishRun(ctx, runID) + _ = si.svc.FinishRun(ctx, runID, RunOutcomeIdleSkipped) return } @@ -514,7 +514,7 @@ func (si *SchedulerIntegration) isTaskTreeGated(ctx context.Context, runID, task zap.String("task_id", taskID), zap.String("hold_id", hold.ID), zap.String("mode", hold.Mode)) - _ = si.svc.FinishRun(ctx, runID) + _ = si.svc.FinishRun(ctx, runID, RunOutcomeTaskTreeHeld) return true } @@ -765,7 +765,8 @@ func (si *SchedulerIntegration) tryCheckout( // successful completion: route it through the same // retry-then-escalate path as every other run failure in this // file, rather than FinishRun, which would silently mark a run - // that never executed as finished. + // that never executed as finished. Checkout errors therefore remain + // on the retry/failure path. _ = si.svc.HandleRunFailure(ctx, run, err) return false } @@ -773,6 +774,8 @@ func (si *SchedulerIntegration) tryCheckout( si.logger.Info("run skipped (task checked out by another agent)", zap.String("run_id", run.ID), zap.String("task_id", taskID)) + // Requeue rather than FinishRun: a run that did not acquire the + // checkout must not be reported as completed work. si.requeueContendedCheckout(ctx, run, taskID) return false } @@ -823,7 +826,7 @@ func (si *SchedulerIntegration) checkBudget( si.logger.Info("run skipped (budget exceeded)", zap.String("run_id", run.ID), zap.String("reason", reason)) si.releaseCheckoutIfNeeded(ctx, run) - _ = si.svc.FinishRun(ctx, run.ID) + _ = si.svc.FinishRun(ctx, run.ID, RunOutcomeBudgetBlocked) si.svc.LogActivityWithRun(ctx, agent.WorkspaceID, "scheduler", "office-scheduler", "run_budget_blocked", "run", run.ID, diff --git a/apps/backend/internal/office/service/scheduler_integration_test.go b/apps/backend/internal/office/service/scheduler_integration_test.go index 4b14fadcb9..4c51f73619 100644 --- a/apps/backend/internal/office/service/scheduler_integration_test.go +++ b/apps/backend/internal/office/service/scheduler_integration_test.go @@ -44,7 +44,7 @@ func TestSchedulerIntegration_TickProcessesRun(t *testing.T) { } // Finish the run. - if err := svc.FinishRun(ctx, run.ID); err != nil { + if err := svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -236,7 +236,7 @@ func TestSchedulerIntegration_AtCapacityStaysQueued(t *testing.T) { } // Finish the first run. - if err := svc.FinishRun(ctx, first.ID); err != nil { + if err := svc.FinishRun(ctx, first.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -314,7 +314,7 @@ func TestSchedulerIntegration_PromptBuiltCorrectly(t *testing.T) { t.Errorf("prompt should contain %q, got: %s", tt.contains, prompt) } - _ = svc.FinishRun(ctx, run.ID) + _ = svc.FinishRun(ctx, run.ID, service.RunOutcomeProcessed) }) } } diff --git a/apps/backend/internal/office/service/scheduler_run_outcome_test.go b/apps/backend/internal/office/service/scheduler_run_outcome_test.go new file mode 100644 index 0000000000..21d4a43059 --- /dev/null +++ b/apps/backend/internal/office/service/scheduler_run_outcome_test.go @@ -0,0 +1,381 @@ +package service_test + +// Covers docs/specs/task-delivery-ledger/spec.md, "Office run outcome": +// drives each of the four non-"processed" FinishRun call sites in +// scheduler_integration.go through its real triggering condition (not a +// hardcoded FinishRun call) and asserts the resulting runs.outcome value. +// Existing tests in scheduler_features_test.go already exercise these +// trigger conditions (budget, checkout, idle-skip) for their other side +// effects (activity log entries, StartTask call counts) but never assert +// the outcome column itself, which is the gap this file closes. + +import ( + "context" + "testing" + + "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/events/bus" + "github.com/kandev/kandev/internal/office/models" + "github.com/kandev/kandev/internal/office/service" +) + +// findRunForAgent locates the run queued for agentID with the given reason +// after a scheduler tick has processed it. Mirrors the lookup pattern in +// TestSchedulerIntegration_ResolvesExecutorFromTaskProject. +func findRunForAgent(t *testing.T, svc *service.Service, ctx context.Context, wsID, agentID, reason string) *models.Run { + t.Helper() + runs, err := svc.ListRuns(ctx, wsID) + if err != nil { + t.Fatalf("list runs: %v", err) + } + for _, run := range runs { + if run.AgentProfileID == agentID && run.Reason == reason { + return run + } + } + t.Fatalf("no run found for agent %s reason %s: %#v", agentID, reason, runs) + return nil +} + +func assertOutcome(t *testing.T, run *models.Run, want string) { + t.Helper() + if run.Status != "finished" { + t.Fatalf("status = %q, want finished", run.Status) + } + if run.Outcome == nil { + t.Fatalf("outcome = nil, want %q", want) + } + if *run.Outcome != want { + t.Fatalf("outcome = %q, want %q", *run.Outcome, want) + } +} + +// TestSchedulerOutcome_AgentInactive_WritesOutcome covers scheduler_integration.go:218. +// isAgentActive's guard exists for the race window between claim and +// processing, so the agent is paused after the run is claimed rather than +// before — ClaimNextRun's own query already excludes non-idle/working agents, +// which is exactly why this scenario needs ProcessRunForTest instead of +// RunSchedulerTick. +func TestSchedulerOutcome_AgentInactive_WritesOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("worker-inactive-outcome", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, `{"task_id":"t1"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + run, err := svc.ClaimNextRun(ctx) + if err != nil { + t.Fatalf("claim: %v", err) + } + if run == nil { + t.Fatal("expected a run") + } + + if _, err := svc.UpdateAgentStatus(ctx, agent.ID, models.AgentStatusPaused, "test"); err != nil { + t.Fatalf("pause agent: %v", err) + } + + service.ProcessRunForTest(svc, ctx, run) + + got, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + assertOutcome(t, got, service.RunOutcomeAgentInactive) +} + +// TestSchedulerOutcome_IdleSkipped_WritesOutcome covers scheduler_integration.go:247. +func TestSchedulerOutcome_IdleSkipped_WritesOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := &models.AgentInstance{ + WorkspaceID: "ws-1", + Name: "idle-outcome-worker", + Role: models.AgentRoleWorker, + Status: models.AgentStatusIdle, + ExecutorPreference: `{"type":"worktree"}`, + } + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + // Worker defaults to skip_idle_runs=true, no tasks assigned. + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonHeartbeat, `{}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + service.RunSchedulerTick(svc, ctx) + + run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonHeartbeat) + assertOutcome(t, run, service.RunOutcomeIdleSkipped) +} + +// TestSchedulerOutcome_TaskTreeHeld_WritesOutcome covers scheduler_integration.go:517. +func TestSchedulerOutcome_TaskTreeHeld_WritesOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES ('task-tree-outcome', 'ws-1', 'Tree Held Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + svc.ExecSQL(t, `INSERT INTO office_task_tree_holds + (id, workspace_id, root_task_id, mode, created_at) + VALUES ('hold-outcome-1', 'ws-1', 'task-tree-outcome', ?, CURRENT_TIMESTAMP)`, + models.TreeHoldModePause) + svc.ExecSQL(t, `INSERT INTO office_task_tree_hold_members + (hold_id, task_id, depth) VALUES ('hold-outcome-1', 'task-tree-outcome', 0)`) + + agent := makeAgent("worker-tree-held-outcome", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-tree-outcome"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + service.RunSchedulerTick(svc, ctx) + + run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonTaskAssigned) + assertOutcome(t, run, service.RunOutcomeTaskTreeHeld) +} + +// assertRetriedNotFinished asserts a run was requeued for retry rather than +// finished: still non-terminal, retry_count bumped, a scheduled_retry_at set, +// and no outcome written (outcome is only ever set on the finished path). +func assertRetriedNotFinished(t *testing.T, run *models.Run, wantRetryCount int) { + t.Helper() + if run.Status == "finished" || run.Status == "failed" { + t.Fatalf("status = %q, want a non-terminal (retried) status", run.Status) + } + if run.RetryCount != wantRetryCount { + t.Fatalf("retry_count = %d, want %d", run.RetryCount, wantRetryCount) + } + if run.ScheduledRetryAt == nil { + t.Fatalf("scheduled_retry_at = nil, want a scheduled retry time") + } + if run.Outcome != nil { + t.Fatalf("outcome = %q, want nil (never written on the retry path)", *run.Outcome) + } +} + +// TestSchedulerOutcome_CheckoutError_RetriesInsteadOfFinishing covers +// scheduler_integration.go's tryCheckout error branch. Forces a real DB +// error from CheckoutTask's UPDATE by dropping the tasks table after the +// run is claimed (so ClaimNextRun itself is unaffected) and driving the +// claimed run through processRun directly. +// +// A transient checkout error is retried like any other run failure via +// HandleRunFailure, not finished. This preserves the retry-then-escalate +// behavior already present on the base branch. +func TestSchedulerOutcome_CheckoutError_RetriesInsteadOfFinishing(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES ('task-checkout-error', 'ws-1', 'Checkout Error Task', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + + agent := makeAgent("worker-checkout-error", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-checkout-error"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + run, err := svc.ClaimNextRun(ctx) + if err != nil { + t.Fatalf("claim: %v", err) + } + if run == nil { + t.Fatal("expected a run") + } + + svc.ExecSQL(t, `DROP TABLE tasks`) + + service.ProcessRunForTest(svc, ctx, run) + + got, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + assertRetriedNotFinished(t, got, 1) + + // Past MaxRetryCount, the same error escalates to a permanent failure + // instead of retrying again — FailRun deliberately writes a NULL + // outcome (outcome is not part of the failed-path vocabulary). + svc.ExecSQL(t, `UPDATE runs SET retry_count = ?, status = 'claimed' WHERE id = ?`, + service.MaxRetryCount, got.ID) + got.RetryCount = service.MaxRetryCount + got.Status = "claimed" + service.ProcessRunForTest(svc, ctx, got) + + final, err := svc.GetRun(ctx, got.ID) + if err != nil { + t.Fatalf("get run after escalation: %v", err) + } + if final.Status != "failed" { + t.Fatalf("status = %q, want failed after exhausting retries", final.Status) + } + if final.Outcome != nil { + t.Fatalf("outcome = %q, want nil on the failed path", *final.Outcome) + } +} + +// TestSchedulerOutcome_CheckoutUnavailable_RetriesInsteadOfFinishing covers +// scheduler_integration.go's requeueContendedCheckout path. +// +// A lost checkout race is requeued via requeueContendedCheckout instead of +// being finished, because the run did not acquire the task checkout. +func TestSchedulerOutcome_CheckoutUnavailable_RetriesInsteadOfFinishing(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + svc.ExecSQL(t, `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) + VALUES ('task-checkout-unavail', 'ws-1', 'Checkout Unavailable', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`) + + // Another agent already holds the checkout. + ok, err := svc.CheckoutTask(ctx, "task-checkout-unavail", "someone-else-agent") + if err != nil || !ok { + t.Fatalf("seed checkout: ok=%v err=%v", ok, err) + } + + agent := makeAgent("worker-checkout-unavail", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-checkout-unavail"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + service.RunSchedulerTick(svc, ctx) + + run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonTaskAssigned) + assertRetriedNotFinished(t, run, 1) +} + +// TestSchedulerOutcome_BudgetBlocked_WritesOutcome covers scheduler_integration.go:829. +func TestSchedulerOutcome_BudgetBlocked_WritesOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("worker-budget-outcome", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + + policy := &models.BudgetPolicy{ + WorkspaceID: "ws-1", + ScopeType: "agent", + ScopeID: agent.ID, + LimitSubcents: 100, + Period: "monthly", + AlertThresholdPct: 80, + ActionOnExceed: "pause_agent", + } + if err := svc.CreateBudgetPolicy(ctx, policy); err != nil { + t.Fatalf("create policy: %v", err) + } + insertTestTask(t, svc, "task-budget-outcome", "ws-1") + insertTestCostEvent(t, svc, agent.ID, "task-budget-outcome", int64(600)) + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-budget-outcome"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + service.RunSchedulerTick(svc, ctx) + + run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonTaskAssigned) + assertOutcome(t, run, service.RunOutcomeBudgetBlocked) +} + +// TestSchedulerOutcome_NoTaskStarter_FailsRun covers the fail-fast wiring +// guard in scheduler_integration.go. A missing TaskStarter is a permanent +// process capability gap, so the run must be visible as failed instead of +// being reported as completed work. +func TestSchedulerOutcome_NoTaskStarter_FailsRun(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + insertTestTask(t, svc, "task-no-launch", "ws-1") + + agent := makeAgent("worker-no-launch", models.AgentRoleWorker) + agent.ExecutorPreference = `{"type":"local_pc"}` + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, + `{"task_id":"task-no-launch"}`, ""); err != nil { + t.Fatalf("queue: %v", err) + } + + service.RunSchedulerTick(svc, ctx) + + run := findRunForAgent(t, svc, ctx, "ws-1", agent.ID, service.RunReasonTaskAssigned) + if run.Status != service.RunStatusFailed { + t.Fatalf("status = %q, want %q", run.Status, service.RunStatusFailed) + } + if run.Outcome != nil { + t.Fatalf("outcome = %v, want NULL on failed run", *run.Outcome) + } + if run.ErrorMessage == "" { + t.Fatal("expected failed run to include a diagnostic error message") + } +} + +// TestSchedulerOutcome_Processed_WritesOutcome covers +// event_subscribers.go:408 (handleAgentCompleted), the only path that +// yields "processed" — every other FinishRun call site in this file is +// driven through its own real trigger, but until now nothing drove the +// agent-completed subscriber itself and checked runs.outcome: existing +// coverage (TestRunLifecycle_StepCompleteEventsEmitted) publishes the +// same AgentCompleted event but only asserts the run's Events log, never +// the outcome column (spec.md:2118-2120). +// +// The synthetic event carries agent_profile_id, matching every real +// AgentCompleted publish site (lifecycle/events.go sets it from +// execution.officeProfileID()): resolveLifecycleRun's task+agent fallback +// (used here since the event carries no run_id) scopes its lookup by +// agent_profile_id, so an event missing it would never match the claimed +// run and would leave it stuck at "claimed" instead of "finished". +func TestSchedulerOutcome_Processed_WritesOutcome(t *testing.T) { + svc, eb := newTestServiceWithBus(t) + ctx := context.Background() + + createTestAgent(t, svc, "ws-1", "worker-processed") + taskID := createOfficeTask(t, svc, "ws-1", "worker-processed") + + if err := svc.QueueRun( + ctx, "worker-processed", service.RunReasonTaskAssigned, + `{"task_id":"`+taskID+`"}`, "processed-outcome", + ); err != nil { + t.Fatalf("queue: %v", err) + } + run, err := svc.ClaimNextRun(ctx) + if err != nil || run == nil { + t.Fatalf("claim: %v (run=%v)", err, run) + } + + completed := bus.NewEvent(events.AgentCompleted, "test", map[string]string{ + "task_id": taskID, + "session_id": "sess-processed", + "agent_profile_id": "worker-processed", + }) + if err := eb.Publish(ctx, events.AgentCompleted, completed); err != nil { + t.Fatalf("publish completed: %v", err) + } + + got, err := svc.GetRun(ctx, run.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + assertOutcome(t, got, service.RunOutcomeProcessed) +} diff --git a/apps/backend/internal/office/service/scheduler_runs.go b/apps/backend/internal/office/service/scheduler_runs.go index 69bf85c2bc..cd6ce17c2c 100644 --- a/apps/backend/internal/office/service/scheduler_runs.go +++ b/apps/backend/internal/office/service/scheduler_runs.go @@ -31,25 +31,31 @@ func (s *Service) ClaimNextRun(ctx context.Context) (*models.Run, error) { return req, nil } -// FinishRun marks a claimed run as finished and publishes an -// OfficeRunProcessed bus event. The run row is fetched first so the -// published payload carries enough context (agent, task, comment, -// reason) for downstream WS consumers to scope updates. -func (s *Service) FinishRun(ctx context.Context, id string) error { - return s.transitionRunTerminal(ctx, id, RunStatusFinished) +// FinishRun marks a claimed run as finished, records outcome (one of the +// eight docs/specs/task-delivery-ledger/spec.md "Office run outcome" +// values — the call site's semantic label, never invented), and publishes +// an OfficeRunProcessed bus event. The run row is fetched first so the +// published payload carries enough context (agent, task, comment, reason) +// for downstream WS consumers to scope updates. +func (s *Service) FinishRun(ctx context.Context, id, outcome string) error { + return s.transitionRunTerminal(ctx, id, RunStatusFinished, &outcome) } // FailRun marks a claimed run as failed and publishes an -// OfficeRunProcessed event. See FinishRun for the lifecycle contract. +// OfficeRunProcessed event. outcome is written as NULL: a failed run is +// bucketed by RunCountsByDayForAgent on status alone and never reaches the +// outcome-derived buckets, so no value from the five-value vocabulary is +// invented for it. See FinishRun for the lifecycle contract. func (s *Service) FailRun(ctx context.Context, id string) error { - return s.transitionRunTerminal(ctx, id, RunStatusFailed) + return s.transitionRunTerminal(ctx, id, RunStatusFailed, nil) } -// transitionRunTerminal updates the run row to the given terminal -// status and emits OfficeRunProcessed. Pre-fetching the run keeps the -// published payload self-contained even when the caller doesn't hold a -// reference to the model. Publish errors are logged at debug and -// swallowed; persistence errors are returned to the caller. +// transitionRunTerminal updates the run row to the given terminal status +// and outcome (nil writes SQL NULL) and emits OfficeRunProcessed. +// Pre-fetching the run keeps the published payload self-contained even +// when the caller doesn't hold a reference to the model. Publish errors +// are logged at debug and swallowed; persistence errors are returned to +// the caller. // // Deliberately does NOT release the task checkout: transitionRunTerminal // is reached by every terminal run, including ones that never held the @@ -66,7 +72,7 @@ func (s *Service) FailRun(ctx context.Context, id string) error { // explicitly instead: handleAgentCompleted / handleTasklessAgentCompleted // (event_subscribers.go) for the launched-run completion path, and // HandleAgentFailure (failure.go) for the launched-run failure path. -func (s *Service) transitionRunTerminal(ctx context.Context, id, status string) error { +func (s *Service) transitionRunTerminal(ctx context.Context, id, status string, outcome *string) error { run, getErr := s.repo.GetRunByID(ctx, id) if getErr != nil && !errors.Is(getErr, sql.ErrNoRows) { s.logger.Debug("get run for terminal transition failed", @@ -74,7 +80,7 @@ func (s *Service) transitionRunTerminal(ctx context.Context, id, status string) zap.String("status", status), zap.Error(getErr)) } - if err := s.repo.FinishRun(ctx, id, status); err != nil { + if err := s.repo.FinishRun(ctx, id, status, outcome); err != nil { return err } s.publishRunProcessed(ctx, id, status, run) @@ -84,10 +90,9 @@ func (s *Service) transitionRunTerminal(ctx context.Context, id, status string) // releaseTaskCheckoutForRun releases the run's task checkout. Call this // only when the caller knows run genuinely held the checkout (a launched // run that reached checkoutTask) — see transitionRunTerminal's doc for why -// it is not called unconditionally on every terminal transition. Safe to -// call redundantly alongside SchedulerIntegration's own synchronous -// finishRun/checkBudget call sites (releaseCheckoutIfNeeded): idempotent, -// owner-scoped SET NULL. +// it is not called unconditionally on every terminal transition. Safe to call +// redundantly alongside the scheduler's checkout and terminal paths: the +// update is idempotent and owner-scoped. // // Owner-scoped by run ID and run.AgentProfileID: a run that never // held the checkout (the loser of a checkout-contention race escalating via @@ -112,12 +117,10 @@ func (s *Service) releaseTaskCheckoutForRun(ctx context.Context, run *models.Run } // stampRunFinished records the cooldown timestamp for the agent that ran -// this run. Shared by the synchronous SchedulerIntegration.finishRun path -// and the async AgentCompleted/AgentStopped event subscribers so the two -// completion paths cannot drift apart again — see releaseTaskCheckoutForRun -// for the same split. run.AgentProfileID is the launching agent's own -// identity (processRun resolves the agent from the run), so no extra -// agent load is needed here. +// this run. It is shared by the AgentCompleted/AgentStopped event subscribers +// so their completion paths cannot drift apart. run.AgentProfileID is the +// launching agent's own identity (processRun resolves the agent from the run), +// so no extra agent load is needed here. func (s *Service) stampRunFinished(ctx context.Context, run *models.Run) { if run == nil || run.AgentProfileID == "" { return diff --git a/apps/backend/internal/office/service/scheduler_runs_test.go b/apps/backend/internal/office/service/scheduler_runs_test.go index 1155c94377..56218742e2 100644 --- a/apps/backend/internal/office/service/scheduler_runs_test.go +++ b/apps/backend/internal/office/service/scheduler_runs_test.go @@ -105,7 +105,7 @@ func TestFinishAndFailRun(t *testing.T) { t.Fatal("expected claimed run") } - if err := svc.FinishRun(ctx, req.ID); err != nil { + if err := svc.FinishRun(ctx, req.ID, service.RunOutcomeProcessed); err != nil { t.Fatalf("finish: %v", err) } @@ -115,3 +115,107 @@ func TestFinishAndFailRun(t *testing.T) { t.Error("expected no more runs") } } + +// TestFailRun_WritesFailedStatusAndNullOutcome is Review round 2, finding +// #1: TestFinishAndFailRun never actually calls FailRun, so nothing +// asserted that a failed run's outcome column is written NULL rather than +// one of the eight finished-path outcome values. FailRun's own doc comment +// says outcome must be NULL because a failed run "never reaches the +// outcome-derived buckets" — this pins that contract against a regression +// that starts writing a real outcome string for a failed run. +func TestFailRun_WritesFailedStatusAndNullOutcome(t *testing.T) { + svc := newTestService(t) + ctx := context.Background() + + agent := makeAgent("worker-1", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + t.Fatalf("queue: %v", err) + } + + req, _ := svc.ClaimNextRun(ctx) + if req == nil { + t.Fatal("expected claimed run") + } + + if err := svc.FailRun(ctx, req.ID); err != nil { + t.Fatalf("fail: %v", err) + } + + got, err := svc.GetRun(ctx, req.ID) + if err != nil { + t.Fatalf("get run: %v", err) + } + if got.Status != models.RunStatusFailed { + t.Errorf("status = %q, want %q", got.Status, models.RunStatusFailed) + } + if got.Outcome != nil { + t.Errorf("outcome = %q, want nil (a failed run must never carry a finished-path outcome value)", *got.Outcome) + } +} + +// TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome is Review +// round 2's lower-priority TEST-002 (spec:2130-2132): two terminal callers +// reaching the same run row must never leave status and outcome +// disagreeing, because transitionRunTerminal's underlying FinishRun writes +// both columns in a single UPDATE statement. This drives both orderings — +// FinishRun-then-FailRun and FailRun-then-FinishRun — and asserts the row +// always ends up wholly consistent with whichever call went last. +func TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome(t *testing.T) { + ctx := context.Background() + + queueAndClaim := func(t *testing.T, svc *service.Service) string { + t.Helper() + agent := makeAgent("worker-1", models.AgentRoleWorker) + if err := svc.CreateAgentInstance(ctx, agent); err != nil { + t.Fatalf("create agent: %v", err) + } + if err := svc.QueueRun(ctx, agent.ID, service.RunReasonTaskAssigned, "{}", "k1"); err != nil { + t.Fatalf("queue: %v", err) + } + req, _ := svc.ClaimNextRun(ctx) + if req == nil { + t.Fatal("expected claimed run") + } + return req.ID + } + + t.Run("FailRun after FinishRun", func(t *testing.T) { + svc := newTestService(t) + id := queueAndClaim(t, svc) + if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish: %v", err) + } + if err := svc.FailRun(ctx, id); err != nil { + t.Fatalf("fail: %v", err) + } + got, err := svc.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run: %v", err) + } + if got.Status != models.RunStatusFailed || got.Outcome != nil { + t.Fatalf("row = {status: %q, outcome: %v}, want the later FailRun write: {failed, nil}", got.Status, got.Outcome) + } + }) + + t.Run("FinishRun after FailRun", func(t *testing.T) { + svc := newTestService(t) + id := queueAndClaim(t, svc) + if err := svc.FailRun(ctx, id); err != nil { + t.Fatalf("fail: %v", err) + } + if err := svc.FinishRun(ctx, id, service.RunOutcomeProcessed); err != nil { + t.Fatalf("finish: %v", err) + } + got, err := svc.GetRun(ctx, id) + if err != nil { + t.Fatalf("get run: %v", err) + } + if got.Status != models.RunStatusFinished || got.Outcome == nil || *got.Outcome != service.RunOutcomeProcessed { + t.Fatalf("row = {status: %q, outcome: %v}, want the later FinishRun write: {finished, %q}", got.Status, got.Outcome, service.RunOutcomeProcessed) + } + }) +} diff --git a/apps/backend/internal/office/service/test_helpers.go b/apps/backend/internal/office/service/test_helpers.go index 79735fa819..c3ee6c0b86 100644 --- a/apps/backend/internal/office/service/test_helpers.go +++ b/apps/backend/internal/office/service/test_helpers.go @@ -150,6 +150,17 @@ func RunSchedulerTick(svc *Service, ctx context.Context) { si.tick(ctx) } +// ProcessRunForTest exposes processRun directly for external test packages +// that need to drive a specific run through the pipeline after mutating +// state RunSchedulerTick's atomic claim+process cannot isolate — e.g. the +// agent-inactive race window, where ClaimNextRun's own query already +// excludes non-idle/working agents so the run must be claimed first and the +// agent paused afterward, then handed to processRun directly. +func ProcessRunForTest(svc *Service, ctx context.Context, run *models.Run) { + si := &SchedulerIntegration{svc: svc, logger: svc.logger} + si.processRun(ctx, run) +} + // BuildEnvVarsForTest exposes buildEnvVars for external test packages. func BuildEnvVarsForTest( si *SchedulerIntegration, diff --git a/apps/backend/internal/persistence/meta.go b/apps/backend/internal/persistence/meta.go index c4fff848d2..627cb06a09 100644 --- a/apps/backend/internal/persistence/meta.go +++ b/apps/backend/internal/persistence/meta.go @@ -113,6 +113,23 @@ func WriteMetaKeyIfAbsent(db *sqlx.DB, key, value string) (bool, error) { return affected > 0, nil } +// DeleteKeys removes the given keys from kandev_meta, if present. Used by +// the factory reset path to clear activation instants before dropping user +// tables (docs/specs/task-delivery-ledger/spec.md, "Reset parity"). +func DeleteKeys(db *sqlx.DB, keys ...string) error { + if len(keys) == 0 { + return nil + } + query, args, err := sqlx.In(`DELETE FROM kandev_meta WHERE key IN (?)`, keys) + if err != nil { + return fmt.Errorf("build delete meta keys query: %w", err) + } + if _, err := db.Exec(db.Rebind(query), args...); err != nil { + return fmt.Errorf("delete meta keys: %w", err) + } + return nil +} + // WriteVersion records currentVersion as the binary version that last // successfully completed boot against this DB. Call this only after every // repository initSchema has succeeded. diff --git a/apps/backend/internal/runs/repository/sqlite/base_test.go b/apps/backend/internal/runs/repository/sqlite/base_test.go index 59db04d901..e7dcf9648c 100644 --- a/apps/backend/internal/runs/repository/sqlite/base_test.go +++ b/apps/backend/internal/runs/repository/sqlite/base_test.go @@ -225,7 +225,7 @@ func TestReadPathsUseTheReadOnlyHandle(t *testing.T) { } // Writes must not fall back to the reader. - if err := repo.FinishRun(ctx, run.ID, "finished"); err == nil { + if err := repo.FinishRun(ctx, run.ID, "finished", nil); err == nil { t.Errorf("FinishRun succeeded with a closed writer; the write path is not using r.db") } } diff --git a/apps/backend/internal/runs/repository/sqlite/errors_test.go b/apps/backend/internal/runs/repository/sqlite/errors_test.go index c93811d908..35ea307d50 100644 --- a/apps/backend/internal/runs/repository/sqlite/errors_test.go +++ b/apps/backend/internal/runs/repository/sqlite/errors_test.go @@ -116,7 +116,7 @@ func TestWriteMethodsPropagateDatabaseErrors(t *testing.T) { return repo.UpdateRunOutputSummary(ctx, run.ID, "o", "f") }, "FinishRun": func() error { - return repo.FinishRun(ctx, run.ID, "finished") + return repo.FinishRun(ctx, run.ID, "finished", nil) }, "ClaimRun": func() error { _, err := repo.ClaimRun(ctx, "a1") diff --git a/apps/backend/internal/runs/repository/sqlite/runs.go b/apps/backend/internal/runs/repository/sqlite/runs.go index f45c792cb0..eb2d530de4 100644 --- a/apps/backend/internal/runs/repository/sqlite/runs.go +++ b/apps/backend/internal/runs/repository/sqlite/runs.go @@ -184,12 +184,16 @@ func (r *Repository) ClaimRun(ctx context.Context, agentInstanceID string) (*mod return &req, nil } -// FinishRun marks a run as finished. -func (r *Repository) FinishRun(ctx context.Context, id, status string) error { +// FinishRun marks a run as terminal (status) and records its outcome in the +// same statement, so the row can never hold a terminal status with a stale +// outcome from a different transition. outcome is nil for the failed path +// and for callers with no established semantic label (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome"). +func (r *Repository) FinishRun(ctx context.Context, id, status string, outcome *string) error { now := time.Now().UTC() _, err := r.db.ExecContext(ctx, r.db.Rebind(` - UPDATE runs SET status = ?, finished_at = ? WHERE id = ? - `), status, now, id) + UPDATE runs SET status = ?, outcome = ?, finished_at = ? WHERE id = ? + `), status, outcome, now, id) return err } @@ -706,16 +710,29 @@ func (r *Repository) SetRunRequestedAtForTest( // SetRunStatusForTest forces the status + timing fields for a seeded // run. Test-only: lets the E2E harness land non-queued rows // (claimed/finished/failed/cancelled) without going through the -// production state machine. +// production state machine. A "finished" status defaults outcome to +// "processed", mirroring the ordinary production call site +// (event_subscribers.go's agent-completed path) that "finished" almost +// always represents — the same default already applied by +// office/dashboard's seedSummaryRun test helper (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome"). Every other +// status writes a NULL outcome, matching FailRun and the dormant +// SchedulerService paths. Existing callers that only cared about +// status keep reading as succeeded under the outcome-aware bucketing +// with no call-site changes. func (r *Repository) SetRunStatusForTest( ctx context.Context, runID, status string, claimedAt, finishedAt *time.Time, ) error { + var outcome interface{} + if status == "finished" { + outcome = "processed" + } _, err := r.db.ExecContext(ctx, r.db.Rebind(` UPDATE runs - SET status = ?, claimed_at = ?, finished_at = ? + SET status = ?, outcome = ?, claimed_at = ?, finished_at = ? WHERE id = ? - `), status, claimedAt, finishedAt, runID) + `), status, outcome, claimedAt, finishedAt, runID) return err } diff --git a/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go b/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go index 566e8f734c..ba7036b547 100644 --- a/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go +++ b/apps/backend/internal/runs/repository/sqlite/runs_crud_test.go @@ -275,7 +275,7 @@ func TestFinishRun_SetsStatusAndFinishedAt(t *testing.T) { before := time.Now().UTC().Add(-time.Second) run := mustCreateRun(t, repo, fullRun()) - if err := repo.FinishRun(ctx, run.ID, "finished"); err != nil { + if err := repo.FinishRun(ctx, run.ID, "finished", nil); err != nil { t.Fatalf("finish run: %v", err) } got := mustGetRun(t, repo, run.ID) @@ -305,6 +305,7 @@ func TestSetRunStatusForTest_WritesStatusAndBothTimestamps(t *testing.T) { if got.FinishedAt == nil || !got.FinishedAt.Equal(finished) { t.Errorf("finished_at = %v, want %s", got.FinishedAt, finished) } + checkStringPtr(t, "outcome", got.Outcome, nil) setStatus(t, repo, run.ID, "queued", nil, nil) got = mustGetRun(t, repo, run.ID) @@ -312,6 +313,25 @@ func TestSetRunStatusForTest_WritesStatusAndBothTimestamps(t *testing.T) { if got.ClaimedAt != nil || got.FinishedAt != nil { t.Errorf("timestamps = (%v, %v), want both nil", got.ClaimedAt, got.FinishedAt) } + checkStringPtr(t, "outcome", got.Outcome, nil) +} + +// TestSetRunStatusForTest_FinishedDefaultsOutcomeToProcessed covers the +// one branch the E2E harness actually depends on: a bare +// status="finished" seed (apiClient.seedRun with no outcome override) +// must classify as "succeeded" under RunCountsByDayForAgent's +// outcome-aware bucketing, or every E2E spec asserting a succeeded +// segment count regresses the moment this diff ships (docs/specs/ +// task-delivery-ledger/spec.md, "Office run outcome"). +func TestSetRunStatusForTest_FinishedDefaultsOutcomeToProcessed(t *testing.T) { + repo := newTestRepo(t) + run := mustCreateRun(t, repo, fullRun()) + finished := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + + setStatus(t, repo, run.ID, "finished", nil, timePtr(finished)) + got := mustGetRun(t, repo, run.ID) + checkString(t, "status", string(got.Status), "finished") + checkStringPtr(t, "outcome", got.Outcome, strPtr("processed")) } // TestSetRunRequestedAtAndErrorMessageForTest covers the two remaining diff --git a/apps/backend/internal/system/database/reset.go b/apps/backend/internal/system/database/reset.go index 3c0dba5c54..f02e39bcd3 100644 --- a/apps/backend/internal/system/database/reset.go +++ b/apps/backend/internal/system/database/reset.go @@ -11,6 +11,8 @@ import ( "github.com/jmoiron/sqlx" "go.uber.org/zap" + + "github.com/kandev/kandev/internal/persistence" ) // resetConfirmToken is the literal value the client must POST as @@ -30,9 +32,10 @@ var ErrResetNotConfirmed = errors.New("factory reset requires confirm=\"RESET\"" // // The job: // 1. Calls s.OrchestratorShutdown (if set) to stop running executions. -// 2. Snapshots the live DB to the sibling backups directory. -// 3. Drops every user table from the SQLite schema (kandev_meta is kept). -// 4. os.RemoveAll on worktrees/repos/sessions/tasks/quick-chat subdirs. +// 2. Calls s.DatabaseQuiesce (if set) to stop database-backed workers. +// 3. Snapshots the live DB to the sibling backups directory. +// 4. Drops every user table from the SQLite schema (kandev_meta is kept). +// 5. os.RemoveAll on worktrees/repos/sessions/tasks/quick-chat subdirs. // // On success the result map exposes {"snapshot_path", "tables_dropped", // "restart_required": true}. The frontend dialog uses restart_required to @@ -54,12 +57,34 @@ func (s *Service) runFactoryReset(_ context.Context) (map[string]interface{}, er if s.OrchestratorShutdown != nil { s.OrchestratorShutdown() } + if s.DatabaseQuiesce != nil { + if err := s.DatabaseQuiesce(); err != nil { + return nil, fmt.Errorf("quiesce database workers: %w", err) + } + } snapshotPath, err := s.createPreResetSnapshot() if err != nil { return nil, fmt.Errorf("pre-reset snapshot: %w", err) } + // Delete the delivery-ledger and run-outcome activation keys BEFORE + // dropping any user table (docs/specs/task-delivery-ledger/spec.md, + // "Reset parity"). kandev_meta itself survives the drop loop below, so + // a stale activation key left behind would point at a boot instant + // before data that no longer exists — a post-reset task with no + // ledger row would then read as "observed as nothing" rather than + // "not observed". Ordering matters because the drop loop is not + // transactional: if key deletion fails, the reset fails here and no + // table is dropped, which is the fail-safe order — the alternative + // (drop first, delete keys second) can leave a present key pointing + // at data a later step just removed. + if err := persistence.DeleteKeys(s.pool.Writer(), + "telemetry.delivery_ledger.activated_at", "telemetry.run_outcome.activated_at", + ); err != nil { + return nil, fmt.Errorf("delete activation keys: %w", err) + } + dropped, err := dropUserTables(s.pool.Writer()) if err != nil { return nil, fmt.Errorf("drop tables: %w", err) diff --git a/apps/backend/internal/system/database/reset_test.go b/apps/backend/internal/system/database/reset_test.go index eb1edf7e3f..1ffc9ef765 100644 --- a/apps/backend/internal/system/database/reset_test.go +++ b/apps/backend/internal/system/database/reset_test.go @@ -9,6 +9,7 @@ import ( "strings" "sync/atomic" "testing" + "time" "github.com/gin-gonic/gin" @@ -115,6 +116,147 @@ func TestFactoryReset_Confirmed_RunsFullSequence(t *testing.T) { } } +// TestFactoryReset_DeletesDeliveryLedgerActivationKeys covers +// docs/specs/task-delivery-ledger/spec.md, "Reset parity": an activated +// database resets with both telemetry.*.activated_at keys absent from +// kandev_meta afterward, while the table itself (and unrelated keys) are +// kept. +func TestFactoryReset_DeletesDeliveryLedgerActivationKeys(t *testing.T) { + svc, tracker, _, _ := newTestService(t) + + if _, err := svc.pool.Writer().Exec(` + INSERT OR REPLACE INTO kandev_meta (key, value) VALUES + ('telemetry.delivery_ledger.activated_at', '2026-08-01T00:00:00Z'), + ('telemetry.run_outcome.activated_at', '2026-08-01T00:00:00Z') + `); err != nil { + t.Fatalf("seed activation keys: %v", err) + } + + id, err := svc.FactoryReset(context.Background(), "RESET") + if err != nil { + t.Fatalf("FactoryReset: %v", err) + } + job := waitForState(t, tracker, id, jobs.StateSucceeded) + if job.State != jobs.StateSucceeded { + t.Fatalf("state = %s, want succeeded; message=%s", job.State, job.Message) + } + + var count int + if err := svc.pool.Reader().QueryRow(` + SELECT COUNT(*) FROM kandev_meta + WHERE key IN ('telemetry.delivery_ledger.activated_at', 'telemetry.run_outcome.activated_at') + `).Scan(&count); err != nil { + t.Fatalf("query kandev_meta: %v", err) + } + if count != 0 { + t.Errorf("activation keys remaining = %d, want 0", count) + } + + // kandev_meta itself, and its unrelated keys, must survive the reset. + var version string + if err := svc.pool.Reader().QueryRow( + `SELECT value FROM kandev_meta WHERE key = 'kandev_version'`, + ).Scan(&version); err != nil { + t.Fatalf("kandev_version missing after reset: %v", err) + } + if version != "v0.99.0" { + t.Errorf("kandev_version = %q, want unchanged v0.99.0", version) + } +} + +func TestFactoryReset_WaitsForDatabaseWorkersBeforeMaintenance(t *testing.T) { + svc, _, _, _ := newTestService(t) + + quiesceStarted := make(chan struct{}) + releaseQuiesce := make(chan struct{}) + svc.DatabaseQuiesce = func() error { + close(quiesceStarted) + <-releaseQuiesce + return nil + } + + resultCh := make(chan error, 1) + go func() { + _, err := svc.runFactoryReset(context.Background()) + resultCh <- err + }() + + select { + case <-quiesceStarted: + case <-time.After(time.Second): + t.Fatal("factory reset did not quiesce database workers") + } + + var usersTable int + if err := svc.pool.Reader().QueryRow(` + SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'users' + `).Scan(&usersTable); err != nil { + t.Fatalf("check users table while quiescing: %v", err) + } + if usersTable != 1 { + t.Fatalf("users table count while quiescing = %d, want 1", usersTable) + } + + close(releaseQuiesce) + select { + case err := <-resultCh: + if err != nil { + t.Fatalf("factory reset: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("factory reset did not finish after database workers stopped") + } + +} + +// TestFactoryReset_ActivationKeyDeleteFailureLeavesUserTablesIntact is +// Review round 2, finding #2: runFactoryReset's own comment documents a +// deliberate fail-safe ordering — delete the delivery-ledger/run-outcome +// activation keys BEFORE dropping any user table, specifically so that if +// key deletion fails, the reset aborts before dropUserTables ever runs. +// Nothing exercised that path: this test forces persistence.DeleteKeys to +// fail (by dropping kandev_meta, the table it deletes rows from, before +// the reset runs) and asserts the job surfaces as failed while the seeded +// user tables (users, sessions_t) are still present. +func TestFactoryReset_ActivationKeyDeleteFailureLeavesUserTablesIntact(t *testing.T) { + svc, tracker, _, _ := newTestService(t) + + if _, err := svc.pool.Writer().Exec(`DROP TABLE kandev_meta`); err != nil { + t.Fatalf("drop kandev_meta: %v", err) + } + + id, err := svc.FactoryReset(context.Background(), "RESET") + if err != nil { + t.Fatalf("FactoryReset: %v", err) + } + job := waitForState(t, tracker, id, jobs.StateFailed) + if job.State != jobs.StateFailed { + t.Fatalf("state = %s, want failed; message=%s", job.State, job.Message) + } + + rows, err := svc.pool.Reader().Query(` + SELECT name FROM sqlite_master + WHERE type='table' AND name NOT LIKE 'sqlite_%' + `) + if err != nil { + t.Fatalf("query sqlite_master: %v", err) + } + defer func() { _ = rows.Close() }() + remaining := map[string]bool{} + for rows.Next() { + var n string + if scanErr := rows.Scan(&n); scanErr != nil { + t.Fatalf("scan: %v", scanErr) + } + remaining[n] = true + } + for _, want := range []string{"users", "sessions_t"} { + if !remaining[want] { + t.Errorf("table %q missing after a failed activation-key delete; want it kept (drop must not have run)", want) + } + } +} + func TestHandleReset_WrongConfirm_Returns400(t *testing.T) { svc, _, _, _ := newTestService(t) gin.SetMode(gin.TestMode) diff --git a/apps/backend/internal/system/database/stats.go b/apps/backend/internal/system/database/stats.go index a4a93e37a7..79b510cbb0 100644 --- a/apps/backend/internal/system/database/stats.go +++ b/apps/backend/internal/system/database/stats.go @@ -71,6 +71,9 @@ type Service struct { // OrchestratorShutdown stops the orchestrator and active executions before // the factory-reset job runs. Wired by cmd/kandev. Tests pass a no-op. OrchestratorShutdown func() + // DatabaseQuiesce stops database-backed workers before the factory-reset + // job snapshots or changes the shared schema. Wired by cmd/kandev. + DatabaseQuiesce func() error } // NewService constructs a Service for the configured SQLite database path. diff --git a/apps/backend/internal/system/system.go b/apps/backend/internal/system/system.go index ae3b26cd33..2c261ffd86 100644 --- a/apps/backend/internal/system/system.go +++ b/apps/backend/internal/system/system.go @@ -51,12 +51,14 @@ type BuildInfo struct { // Wiring supplies the runtime hooks and repositories owned by the wider // application. OrchestratorShutdown stops in-flight agent executions before -// destructive resets. RestoreQuiesce stops the complete database-backed -// runtime before a SQLite restore closes the shared pool. TaskSessions is the -// authoritative session reader used by the install-wide sleep-inhibition -// service. +// destructive resets. DatabaseQuiesce stops database-backed workers before a +// factory reset mutates the shared schema. RestoreQuiesce stops the complete +// database-backed runtime before a SQLite restore closes the shared pool. +// TaskSessions is the authoritative session reader used by the install-wide +// sleep-inhibition service. type Wiring struct { OrchestratorShutdown func() + DatabaseQuiesce func() error RestoreQuiesce func() error MessageQueue queuesettings.Target MessageQueueConfig queuesettings.Configuration @@ -107,6 +109,7 @@ func Provide(cfg *config.Config, log *logger.Logger, pool *db.Pool, eventBus bus } dbSvc := database.NewService(pool, databasePath, resetDirs, tracker, log) dbSvc.OrchestratorShutdown = wiring.OrchestratorShutdown + dbSvc.DatabaseQuiesce = wiring.DatabaseQuiesce backupsSvc := backups.NewService(databasePath, pool, tracker, log) backupsSvc.OrchestratorShutdown = wiring.OrchestratorShutdown diff --git a/apps/backend/internal/task/service/repository_discovery.go b/apps/backend/internal/task/service/repository_discovery.go index 6f8c80035e..bcbb9c67bb 100644 --- a/apps/backend/internal/task/service/repository_discovery.go +++ b/apps/backend/internal/task/service/repository_discovery.go @@ -399,6 +399,19 @@ func (s *Service) resolveRepositoryLocalPath(ctx context.Context, repoID string) return resolved, nil } +// ResolveRepositoryLocalPath is the exported form of +// resolveRepositoryLocalPath. It is the narrow, read-only, +// single-method checkout-resolution port docs/specs/task-delivery-ledger/ +// spec.md, "Which checkout" describes: repository ID in, a validated +// absolute checkout path or error out, with no sentinel empty path on +// rejection. internal/delivery's ancestry check depends on this method +// through a local interface rather than importing this package, so it +// inherits the existing canonicalization and containment checks instead +// of duplicating them. +func (s *Service) ResolveRepositoryLocalPath(ctx context.Context, repoID string) (string, error) { + return s.resolveRepositoryLocalPath(ctx, repoID) +} + // RepositoryCurrentBranch returns the current branch for a saved repository, // resolving its exact path grant by repository ID. func (s *Service) RepositoryCurrentBranch(ctx context.Context, repoID string) (string, error) { diff --git a/apps/backend/internal/task/service/service_resources.go b/apps/backend/internal/task/service/service_resources.go index fca02fbf62..a237a03112 100644 --- a/apps/backend/internal/task/service/service_resources.go +++ b/apps/backend/internal/task/service/service_resources.go @@ -15,6 +15,7 @@ import ( "github.com/jmoiron/sqlx" "go.uber.org/zap" + "github.com/kandev/kandev/internal/common/securityutil" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/secrets" "github.com/kandev/kandev/internal/task/models" @@ -760,6 +761,9 @@ func (s *Service) createRepository( if sourceType == "" { sourceType = sourceTypeLocal } + if req.DefaultBranch != "" && !securityutil.IsValidDefaultBranchName(req.DefaultBranch) { + return nil, fmt.Errorf("%w: invalid default branch", ErrInvalidRepositorySettings) + } prefix := strings.TrimSpace(req.WorktreeBranchPrefix) if err := worktree.ValidateBranchPrefix(prefix); err != nil { return nil, fmt.Errorf("%w: %s", ErrInvalidRepositorySettings, err) @@ -966,10 +970,20 @@ func (s *Service) FindOrCreateRepository(ctx context.Context, req *FindOrCreateR // Backfill default_branch when the caller carries one and the existing // row is still empty. Lets the synchronous add_branch probe persist its // answer onto a previously-empty Repository row (e.g. one created by - // an earlier create_task that left default_branch unset). + // an earlier create_task that left default_branch unset). Validated + // here directly: this value reaches git argv (worktree fetch/pull via + // FallbackBaseBranch) on paths that never go through ancestry.Check, + // so that defense-in-depth guard does not cover it. Invalid input + // skips only this field's backfill rather than failing the whole + // find-or-create call, since this is a best-effort backfill. if existing.DefaultBranch == "" && req.DefaultBranch != "" { - existing.DefaultBranch = req.DefaultBranch - dirty = true + if securityutil.IsValidDefaultBranchName(req.DefaultBranch) { + existing.DefaultBranch = req.DefaultBranch + dirty = true + } else { + s.logger.Warn("skipping default_branch backfill: invalid branch name", + zap.String("repository_id", existing.ID)) + } } if existing.RemoteURL == "" && req.RemoteURL != "" { existing.RemoteURL = req.RemoteURL @@ -1181,6 +1195,9 @@ func applyRepositoryUpdates(repository *models.Repository, req *UpdateRepository repository.ProviderName = *req.ProviderName } if req.DefaultBranch != nil { + if *req.DefaultBranch != "" && !securityutil.IsValidDefaultBranchName(*req.DefaultBranch) { + return fmt.Errorf("%w: invalid default branch", ErrInvalidRepositorySettings) + } repository.DefaultBranch = *req.DefaultBranch } if req.WorktreeBranchPrefix != nil { diff --git a/apps/backend/internal/task/service/service_resources_default_branch_test.go b/apps/backend/internal/task/service/service_resources_default_branch_test.go new file mode 100644 index 0000000000..bed32a3092 --- /dev/null +++ b/apps/backend/internal/task/service/service_resources_default_branch_test.go @@ -0,0 +1,366 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +// TestService_CreateRepositoryRejectsInvalidDefaultBranchWithoutPersistence +// covers Review round 2, finding #1 (SEC-005): default_branch reaches the +// git subprocess in internal/delivery/ancestry.go as raw, unvalidated argv +// (both as an option-injection surface and as git revision syntax), so it +// must be validated at ingestion the same way WorktreeBranchPrefix already +// is in this function. +func TestService_CreateRepositoryRejectsInvalidDefaultBranchWithoutPersistence(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + + invalidBranches := []string{"-x", "--all", "main~1", "main^{commit}", "main..other"} + for _, branch := range invalidBranches { + t.Run(branch, func(t *testing.T) { + _, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "Invalid Repo", + SourceType: sourceTypeProvider, + Provider: "github", + DefaultBranch: branch, + }) + if !errors.Is(err, ErrInvalidRepositorySettings) { + t.Fatalf("CreateRepository error = %v, want ErrInvalidRepositorySettings", err) + } + }) + } + repositories, err := repo.ListRepositories(ctx, "ws-1") + if err != nil { + t.Fatalf("ListRepositories: %v", err) + } + if len(repositories) != 0 { + t.Fatalf("invalid repository was persisted: %+v", repositories) + } +} + +// TestService_CreateRepositoryAllowsEmptyDefaultBranch confirms the +// validation added for Review round 2 finding #1 does not reject the +// legitimate "not yet known" state: repository_discovery.go's makeRepo +// leaves DefaultBranch empty when local git probing fails, and +// FindOrCreateRepository's create path already routes through here. +func TestService_CreateRepositoryAllowsEmptyDefaultBranch(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + created, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "Repo", + SourceType: sourceTypeProvider, + Provider: "github", + }) + if err != nil { + t.Fatalf("CreateRepository: %v", err) + } + if created.DefaultBranch != "" { + t.Fatalf("DefaultBranch = %q, want empty", created.DefaultBranch) + } +} + +// TestService_UpdateRepositoryRejectsInvalidDefaultBranchWithoutPersistence +// is UpdateRepository's twin of the CreateRepository test above. +func TestService_UpdateRepositoryRejectsInvalidDefaultBranchWithoutPersistence(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + created, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "Provider Repo", + SourceType: sourceTypeProvider, + Provider: "github", + }) + if err != nil { + t.Fatalf("CreateRepository: %v", err) + } + + invalidBranch := "-x" + _, err = svc.UpdateRepository(ctx, created.ID, &UpdateRepositoryRequest{DefaultBranch: &invalidBranch}) + if !errors.Is(err, ErrInvalidRepositorySettings) { + t.Fatalf("UpdateRepository error = %v, want ErrInvalidRepositorySettings", err) + } + stored, err := repo.GetRepository(ctx, created.ID) + if err != nil { + t.Fatalf("GetRepository: %v", err) + } + if stored.DefaultBranch != "" { + t.Fatalf("invalid DefaultBranch persisted as %q", stored.DefaultBranch) + } +} + +// TestApplyRepositoryUpdates_DefaultBranchRejectsFlagLikeValue covers Review +// round 2, finding #1 (SEC-005): an unvalidated default_branch reaches +// internal/delivery/ancestry.go as raw git argv. A flag-like value must be +// rejected here, mirroring worktree.ValidateBranchPrefix's existing gate in +// this same function, and the repository struct must be left unmodified. +func TestApplyRepositoryUpdates_DefaultBranchRejectsFlagLikeValue(t *testing.T) { + repo := &models.Repository{DefaultBranch: "main"} + invalid := "-x" + err := applyRepositoryUpdates(repo, &UpdateRepositoryRequest{DefaultBranch: &invalid}) + if !errors.Is(err, ErrInvalidRepositorySettings) { + t.Fatalf("applyRepositoryUpdates error = %v, want ErrInvalidRepositorySettings", err) + } + if repo.DefaultBranch != "main" { + t.Errorf("DefaultBranch = %q, want unmodified %q", repo.DefaultBranch, "main") + } +} + +// TestApplyRepositoryUpdates_DefaultBranchRejectsRevisionSyntax covers the +// second consequence named in finding #1: a value like "main~1" is not +// flag-like, but is parsed by git as revision syntax (a parent-commit +// modifier) rather than a literal ref name, which can resolve to the wrong +// commit entirely when it reaches resolveDefaultBranchRef. +func TestApplyRepositoryUpdates_DefaultBranchRejectsRevisionSyntax(t *testing.T) { + repo := &models.Repository{DefaultBranch: "main"} + invalid := "main~1" + err := applyRepositoryUpdates(repo, &UpdateRepositoryRequest{DefaultBranch: &invalid}) + if !errors.Is(err, ErrInvalidRepositorySettings) { + t.Fatalf("applyRepositoryUpdates error = %v, want ErrInvalidRepositorySettings", err) + } + if repo.DefaultBranch != "main" { + t.Errorf("DefaultBranch = %q, want unmodified %q", repo.DefaultBranch, "main") + } +} + +// TestApplyRepositoryUpdates_DefaultBranchEmptyStringClears mirrors the +// CopyFiles empty-string-clears convention: an explicit empty pointer is a +// valid "clear the default branch" update, not a validation failure. +func TestApplyRepositoryUpdates_DefaultBranchEmptyStringClears(t *testing.T) { + repo := &models.Repository{DefaultBranch: "main"} + empty := "" + if err := applyRepositoryUpdates(repo, &UpdateRepositoryRequest{DefaultBranch: &empty}); err != nil { + t.Fatalf("applyRepositoryUpdates: %v", err) + } + if repo.DefaultBranch != "" { + t.Errorf("DefaultBranch = %q, want empty string", repo.DefaultBranch) + } +} + +// TestApplyRepositoryUpdates_DefaultBranchAcceptsValidValue is the positive +// control confirming the new validation does not regress an ordinary +// update. +func TestApplyRepositoryUpdates_DefaultBranchAcceptsValidValue(t *testing.T) { + repo := &models.Repository{DefaultBranch: "main"} + valid := "develop" + if err := applyRepositoryUpdates(repo, &UpdateRepositoryRequest{DefaultBranch: &valid}); err != nil { + t.Fatalf("applyRepositoryUpdates: %v", err) + } + if repo.DefaultBranch != "develop" { + t.Errorf("DefaultBranch = %q, want %q", repo.DefaultBranch, "develop") + } +} + +// TestService_FindOrCreateRepositoryRejectsInvalidDefaultBranchBackfill +// covers Review round 3, finding #1: createRepository and +// applyRepositoryUpdates validate default_branch, but FindOrCreateRepository's +// own backfill block (the third write site) did not, leaving a path to the +// same git argv (internal/delivery ancestry check, and independently the +// worktree fetch/pull FallbackBaseBranch path, which ancestry.Check does not +// cover at all) unvalidated. Unlike the LocalPath backfill above, an invalid +// default_branch must not fail the whole find-or-create call — it is a +// best-effort backfill — so this asserts the call still succeeds, the field +// is simply left unset, and nothing invalid reaches the repository row. +func TestService_FindOrCreateRepositoryRejectsInvalidDefaultBranchBackfill(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + created, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "owner/repo", + SourceType: sourceTypeProvider, + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + }) + if err != nil { + t.Fatalf("CreateRepository: %v", err) + } + if created.DefaultBranch != "" { + t.Fatalf("precondition failed: DefaultBranch = %q, want empty", created.DefaultBranch) + } + + resolved, wasCreated, err := svc.FindOrCreateRepository(ctx, &FindOrCreateRepositoryRequest{ + WorkspaceID: "ws-1", + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + DefaultBranch: "-x", + }) + if err != nil { + t.Fatalf("FindOrCreateRepository: %v", err) + } + if wasCreated || resolved.ID != created.ID { + t.Fatalf("resolved repository = %q (created=%t), want existing %q", resolved.ID, wasCreated, created.ID) + } + if resolved.DefaultBranch != "" { + t.Fatalf("resolved.DefaultBranch = %q, want empty (backfill skipped)", resolved.DefaultBranch) + } + stored, err := repo.GetRepository(ctx, created.ID) + if err != nil { + t.Fatalf("GetRepository: %v", err) + } + if stored.DefaultBranch != "" { + t.Fatalf("invalid DefaultBranch backfill persisted as %q", stored.DefaultBranch) + } +} + +// TestService_CreateRepositoryRejectsGitSymbolicRefDefaultBranch covers a +// PR-review finding on this same field: "HEAD", "main/", and "a//b" all +// pass the plain securityutil.IsValidBranchName allowlist this function +// used before, but "HEAD" in particular is a git symbolic pseudo-ref, not a +// real branch name. Persisting it as default_branch reaches the worktree +// FallbackBaseBranch path (internal/worktree), which resolves it via a +// local rev-parse fallback to whatever commit the checkout's HEAD currently +// points at — silently the wrong revision, not a resolution failure. This +// asserts the ingestion-time reject, mirroring the existing flag-like/ +// revision-syntax coverage above. +func TestService_CreateRepositoryRejectsGitSymbolicRefDefaultBranch(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + + invalidBranches := []string{"HEAD", "ORIG_HEAD", "FETCH_HEAD", "MERGE_HEAD", "main/", "a//b"} + for _, branch := range invalidBranches { + t.Run(branch, func(t *testing.T) { + _, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "Invalid Repo " + branch, + SourceType: sourceTypeProvider, + Provider: "github", + DefaultBranch: branch, + }) + if !errors.Is(err, ErrInvalidRepositorySettings) { + t.Fatalf("CreateRepository error = %v, want ErrInvalidRepositorySettings", err) + } + }) + } + repositories, err := repo.ListRepositories(ctx, "ws-1") + if err != nil { + t.Fatalf("ListRepositories: %v", err) + } + if len(repositories) != 0 { + t.Fatalf("invalid repository was persisted: %+v", repositories) + } +} + +// TestApplyRepositoryUpdates_DefaultBranchRejectsGitSymbolicRef is +// applyRepositoryUpdates' twin of the CreateRepository test above. +func TestApplyRepositoryUpdates_DefaultBranchRejectsGitSymbolicRef(t *testing.T) { + repo := &models.Repository{DefaultBranch: "main"} + invalid := "HEAD" + err := applyRepositoryUpdates(repo, &UpdateRepositoryRequest{DefaultBranch: &invalid}) + if !errors.Is(err, ErrInvalidRepositorySettings) { + t.Fatalf("applyRepositoryUpdates error = %v, want ErrInvalidRepositorySettings", err) + } + if repo.DefaultBranch != "main" { + t.Errorf("DefaultBranch = %q, want unmodified %q", repo.DefaultBranch, "main") + } +} + +// TestService_FindOrCreateRepositoryBackfillRejectsGitSymbolicRef is the +// FindOrCreateRepository backfill's twin of the same coverage, mirroring +// TestService_FindOrCreateRepositoryRejectsInvalidDefaultBranchBackfill's +// "best-effort backfill, call still succeeds" shape. +func TestService_FindOrCreateRepositoryBackfillRejectsGitSymbolicRef(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + created, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "owner/repo", + SourceType: sourceTypeProvider, + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + }) + if err != nil { + t.Fatalf("CreateRepository: %v", err) + } + + resolved, wasCreated, err := svc.FindOrCreateRepository(ctx, &FindOrCreateRepositoryRequest{ + WorkspaceID: "ws-1", + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + DefaultBranch: "HEAD", + }) + if err != nil { + t.Fatalf("FindOrCreateRepository: %v", err) + } + if wasCreated || resolved.ID != created.ID { + t.Fatalf("resolved repository = %q (created=%t), want existing %q", resolved.ID, wasCreated, created.ID) + } + if resolved.DefaultBranch != "" { + t.Fatalf("resolved.DefaultBranch = %q, want empty (backfill skipped)", resolved.DefaultBranch) + } + stored, err := repo.GetRepository(ctx, created.ID) + if err != nil { + t.Fatalf("GetRepository: %v", err) + } + if stored.DefaultBranch != "" { + t.Fatalf("invalid DefaultBranch backfill persisted as %q", stored.DefaultBranch) + } +} + +// TestService_FindOrCreateRepositoryBackfillsValidDefaultBranch is the +// positive control for the test above: a legitimate default_branch value +// still backfills onto a previously-empty row. +func TestService_FindOrCreateRepositoryBackfillsValidDefaultBranch(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-1", Name: "Workspace"}); err != nil { + t.Fatalf("CreateWorkspace: %v", err) + } + created, err := svc.CreateRepository(ctx, &CreateRepositoryRequest{ + WorkspaceID: "ws-1", + Name: "owner/repo", + SourceType: sourceTypeProvider, + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + }) + if err != nil { + t.Fatalf("CreateRepository: %v", err) + } + + resolved, _, err := svc.FindOrCreateRepository(ctx, &FindOrCreateRepositoryRequest{ + WorkspaceID: "ws-1", + Provider: "github", + ProviderOwner: "owner", + ProviderName: "repo", + DefaultBranch: "main", + }) + if err != nil { + t.Fatalf("FindOrCreateRepository: %v", err) + } + if resolved.DefaultBranch != "main" { + t.Fatalf("resolved.DefaultBranch = %q, want %q", resolved.DefaultBranch, "main") + } + stored, err := repo.GetRepository(ctx, created.ID) + if err != nil { + t.Fatalf("GetRepository: %v", err) + } + if stored.DefaultBranch != "main" { + t.Fatalf("stored.DefaultBranch = %q, want %q", stored.DefaultBranch, "main") + } +} diff --git a/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.test.tsx b/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.test.tsx new file mode 100644 index 0000000000..53834f0eec --- /dev/null +++ b/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.test.tsx @@ -0,0 +1,31 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { RunActivityChart } from "./run-activity-chart"; + +afterEach(() => cleanup()); + +describe("RunActivityChart", () => { + it("renders every backend outcome bucket in each bar and the legend", () => { + render( + , + ); + + const bar = screen.getByTestId("stacked-bar"); + for (const key of ["succeeded", "skipped", "unclassified", "failed", "other"]) { + expect(bar.querySelector(`[data-segment-key="${key}"]`)).not.toBeNull(); + } + expect(screen.getByTestId("run-activity-legend").children).toHaveLength(5); + }); +}); diff --git a/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx b/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx index 1a4ffc1357..736b8f06b5 100644 --- a/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx +++ b/apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx @@ -1,6 +1,6 @@ /** * Run Activity stacked-bars chart. One bar per day in the window; - * each bar stacks succeeded (green) / failed (red) / other (muted). + * each bar shows every backend outcome bucket. */ import { Card, CardContent, CardHeader, CardTitle } from "@kandev/ui/card"; @@ -22,6 +22,8 @@ function rowsFromDays(days: AgentRunActivityDay[]): StackedBarRow[] { label: formatBarLabel(d.date), segments: [ { key: "succeeded", value: d.succeeded, className: "bg-emerald-500" }, + { key: "skipped", value: d.skipped, className: "bg-amber-500" }, + { key: "unclassified", value: d.unclassified, className: "bg-slate-400" }, { key: "failed", value: d.failed, className: "bg-red-500" }, { key: "other", value: d.other, className: "bg-muted-foreground/40" }, ], @@ -48,8 +50,11 @@ export function RunActivityChart({ days }: Props) { ariaLabel={t("office:agentRunActivity")} /> }) { +export function ChartLegend({ + items, + "data-testid": dataTestId, +}: { + items: Array<{ label: string; className: string }>; + "data-testid"?: string; +}) { return ( -
+
{items.map((item) => ( diff --git a/apps/web/e2e/tests/office/mobile-agent-dashboard.spec.ts b/apps/web/e2e/tests/office/mobile-agent-dashboard.spec.ts new file mode 100644 index 0000000000..9a9d2505c9 --- /dev/null +++ b/apps/web/e2e/tests/office/mobile-agent-dashboard.spec.ts @@ -0,0 +1,21 @@ +import { expect, test } from "../../fixtures/office-fixture"; + +test.describe("Agent dashboard on mobile", () => { + test("keeps all run outcome buckets reachable on a phone", async ({ testPage, officeSeed }) => { + await testPage.goto(`/office/agents/${officeSeed.agentId}/dashboard`); + + const card = testPage.getByTestId("run-activity-card"); + await expect(card).toBeVisible({ timeout: 15_000 }); + await expect(card.getByTestId("run-activity-legend").locator(":scope > span")).toHaveCount(5); + await expect(card.locator('[data-segment-key="skipped"]').first()).toBeAttached(); + await expect(card.locator('[data-segment-key="unclassified"]').first()).toBeAttached(); + + const viewportWidth = testPage.viewportSize()?.width ?? 0; + const cardBox = await card.boundingBox(); + expect(cardBox).not.toBeNull(); + expect(cardBox!.x + cardBox!.width).toBeLessThanOrEqual(viewportWidth); + expect( + await testPage.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth), + ).toBe(true); + }); +}); diff --git a/apps/web/lib/api/domains/office-runs-api.ts b/apps/web/lib/api/domains/office-runs-api.ts index d2a43620c2..a288120503 100644 --- a/apps/web/lib/api/domains/office-runs-api.ts +++ b/apps/web/lib/api/domains/office-runs-api.ts @@ -157,6 +157,8 @@ export type AgentLatestRun = { export type AgentRunActivityDay = { date: string; succeeded: number; + skipped: number; + unclassified: number; failed: number; other: number; total: number; @@ -184,6 +186,7 @@ export type AgentTaskStatusDay = { export type AgentSuccessRateDay = { date: string; succeeded: number; + unclassified: number; total: number; }; diff --git a/apps/web/src/locales/en/office.json b/apps/web/src/locales/en/office.json index ea02965023..54fd2cf857 100644 --- a/apps/web/src/locales/en/office.json +++ b/apps/web/src/locales/en/office.json @@ -683,6 +683,8 @@ "runStatusFailed": "failed", "runStatusReceived": "received", "runStatusSkipped": "skipped", + "runSkipped": "Skipped", + "runUnclassified": "Unclassified", "runStatusTaskCreated": "task created", "runtime": "Runtime", "runtimeCapabilities": "Runtime capabilities", diff --git a/apps/web/src/locales/pseudo/automations.json b/apps/web/src/locales/pseudo/automations.json index ab9956c868..b2234877a0 100644 --- a/apps/web/src/locales/pseudo/automations.json +++ b/apps/web/src/locales/pseudo/automations.json @@ -32,13 +32,13 @@ "comingSoonSuffix": " (Ćōḿĩńĝ śōōń)", "completeRequiredFields": "Ćōḿƥĺēţē ţĥē ŕēqũĩŕēď àũţōḿàţĩōń ƒĩēĺďś ƀēƒōŕē śàvĩńĝ.", "completed": "Ćōḿƥĺēţēď", - "contextBetweenRunsConcurrencyLock": "Ţĥĩś ōƥţĩōń śũƥƥōŕţś ōńē àćţĩṽē ŕũń àţ à ţĩḿē.", - "contextBetweenRunsDescription": "Ćĥōōśē ŵĥēţĥēŕ ēàćĥ ŕũń śţàŕţś ƒŕēśĥ ōŕ ćōńţĩńũēś ţĥē śàḿē ćōńṽēŕśàţĩōń àńď ƒĩĺēś.", - "contextBetweenRunsNewTask": "Śţàŕţ à ńēŵ ţàśķ ƒōŕ ēṽēŕŷ ŕũń", + "contextBetweenRunsConcurrencyLock": "Ţĥĩś ōƥţĩōń śũƥƥōŕţś ōńē àćţĩvē ŕũń àţ à ţĩḿē.", + "contextBetweenRunsDescription": "Ćĥōōśē ŵĥēţĥēŕ ēàćĥ ŕũń śţàŕţś ƒŕēśĥ ōŕ ćōńţĩńũēś ţĥē śàḿē ćōńvēŕśàţĩōń àńď ƒĩĺēś.", + "contextBetweenRunsNewTask": "Śţàŕţ à ńēŵ ţàśķ ƒōŕ ēvēŕŷ ŕũń", "contextBetweenRunsNewTaskDescription": "Ēàćĥ ŕũń śţàŕţś ŵĩţĥ à śēƥàŕàţē ćōńvēŕśàţĩōń àńď ƒĩĺēś. Ţĥēśē ţàśķś ďō ńōţ àƥƥēàŕ ĩń Ķàńƀàń ōŕ ţĥē śĩďēƀàŕ. Ũśē ţĥĩś ōƥţĩōń ƒōŕ ĩńďēƥēńďēńţ ĵōƀś àńď ćōńćũŕŕēńţ ŕũńś.", "contextBetweenRunsNewTaskNormalDescription": "Ēàćĥ ŕũń śţàŕţś ŵĩţĥ à śēƥàŕàţē ćōńvēŕśàţĩōń àńď ƒĩĺēś. Ţĥēśē ţàśķś àƥƥēàŕ ĩń Ķàńƀàń àńď ţĥē śĩďēƀàŕ. Ũśē ţĥĩś ōƥţĩōń ƒōŕ ĩńďēƥēńďēńţ ĵōƀś àńď ćōńćũŕŕēńţ ŕũńś.", - "contextBetweenRunsReuseThread": "Ćōńţĩńũē ţĥē ƥŕēṽĩōũś śēśśĩōń", - "contextBetweenRunsReuseThreadDescription": "Ŕũńś ćōńţĩńũē ţĥē śàḿē ćōńṽēŕśàţĩōń àńď ƒĩĺēś, śō ţĥē àĝēńţ ķēēƥś ƥŕĩōŕ ćōńţēxţ àńď ćĥàńĝēś. Ŕũńś ēxēćũţē ōńē àţ à ţĩḿē.", + "contextBetweenRunsReuseThread": "Ćōńţĩńũē ţĥē ƥŕēvĩōũś śēśśĩōń", + "contextBetweenRunsReuseThreadDescription": "Ŕũńś ćōńţĩńũē ţĥē śàḿē ćōńvēŕśàţĩōń àńď ƒĩĺēś, śō ţĥē àĝēńţ ķēēƥś ƥŕĩōŕ ćōńţēxţ àńď ćĥàńĝēś. Ŕũńś ēxēćũţē ōńē àţ à ţĩḿē.", "contextBetweenRunsTitle": "Ćōńţēxţ ƀēţŵēēń ŕũńś", "configurationLabel": "Ćōńƒĩĝũŕàţĩōń", "couldNotLoadWorkflows": "Ćōũĺďń'ţ ĺōàď ŵōŕķƒĺōŵś", @@ -122,7 +122,7 @@ "repositoryLabel": "Ŕēƥōśĩţōŕŷ", "repositoryNone": "Ńōńē; ńō ŕēƥōśĩţōŕŷ", "repositoryModeTitle": "Ŕēƥōśĩţōŕŷ àććēśś", - "repositoryModeDescription": "Àďď ōńē ōŕ mōŕē ŕēƥōśĩţōŕŷ àńď ƀàśē ƀŕàńćĥ ƥàĩŕś, ōŕ ŕũń ŵĩţĥ ńō ŕēƥōśĩţōŕŷ.", + "repositoryModeDescription": "Àďď ōńē ōŕ ḿōŕē ŕēƥōśĩţōŕŷ àńď ƀàśē ƀŕàńćĥ ƥàĩŕś, ōŕ ŕũń ŵĩţĥ ńō ŕēƥōśĩţōŕŷ.", "repositoryModeNoneDescription": "Ŕũń ŵĩţĥōũţ ŕēƥōśĩţōŕŷ ƒĩĺēś ĩń à ţàśķ-ōŵńēď śćŕàţćĥ ŵōŕķśƥàćē.", "repositoryPlaceholderAuto": "Àũţō", "required": "ŕēqũĩŕēď", diff --git a/apps/web/src/locales/pseudo/office.json b/apps/web/src/locales/pseudo/office.json index 5d1e853151..36292db52e 100644 --- a/apps/web/src/locales/pseudo/office.json +++ b/apps/web/src/locales/pseudo/office.json @@ -34,7 +34,7 @@ "agentAutoPaused": "Àĝēńţ àũţō-ƥàũśēď", "agentAwaitingApproval": "Àĝēńţ àŵàĩţĩńĝ àƥƥŕōvàĺ", "agentConfigurationUpdated": "Àĝēńţ ćōńƒĩĝũŕàţĩōń ũƥďàţēď", - "agentCeoReportsToError": "À ĆĒŐ àĝēńţ mũśţ ńōţ ħàvē à màńàĝēŕ.", + "agentCeoReportsToError": "À ĆĒŌ àĝēńţ ḿũśţ ńōţ ĥàvē à ḿàńàĝēŕ.", "agentCount_one": "{{count}} àĝēńţ", "agentCount_other": "{{count}} àĝēńţś", "agentCreated": "Àĝēńţ ćŕēàţēď", @@ -48,8 +48,8 @@ "agentRecentTasks": "Ŕēćēńţ ţàśķś", "agentRunActivity": "Ŕũń àćţĩvĩţŷ", "agentRunFailed": "Àĝēńţ ŕũń ƒàĩĺēď", - "agentReportsToCycleError": "Ŧħē ŕēƥōŕţĩńĝ śţŕũćţũŕē ćàńńōţ ćōńţàĩń à ćŷćĺē.", - "agentReportsToInvalidError": "Ŝēĺēćţ àń àĝēńţ ƒŕōm ţħĩś ŵōŕķśƥàćē.", + "agentReportsToCycleError": "Ţĥē ŕēƥōŕţĩńĝ śţŕũćţũŕē ćàńńōţ ćōńţàĩń à ćŷćĺē.", + "agentReportsToInvalidError": "Śēĺēćţ àń àĝēńţ ƒŕōḿ ţĥĩś ŵōŕķśƥàćē.", "agentReportsToSelfError": "Àń àĝēńţ ćàńńōţ ŕēƥōŕţ ţō ĩţśēĺƒ.", "agents": "Àĝēńţś", "agentSections": "Àĝēńţ śēćţĩōńś", @@ -683,6 +683,8 @@ "runStatusFailed": "ƒàĩĺēď", "runStatusReceived": "ŕēćēĩvēď", "runStatusSkipped": "śķĩƥƥēď", + "runSkipped": "Śķĩƥƥēď", + "runUnclassified": "Ũńćĺàśśĩƒĩēď", "runStatusTaskCreated": "ţàśķ ćŕēàţēď", "runtime": "Ŕũńţĩḿē", "runtimeCapabilities": "Ŕũńţĩḿē ćàƥàƀĩĺĩţĩēś", diff --git a/apps/web/src/locales/pseudo/plugins.json b/apps/web/src/locales/pseudo/plugins.json index c0cf294e2a..769fe32554 100644 --- a/apps/web/src/locales/pseudo/plugins.json +++ b/apps/web/src/locales/pseudo/plugins.json @@ -92,7 +92,7 @@ "sortName": "Ńàḿē", "sortRecentlyUpdated": "Ŕēćēńţĺŷ ũƥďàţēď", "sourceNamePlaceholder": "Śōũŕćē ńàḿē (ē.ĝ. Àćḿē Ĩńţēŕńàĺ)", - "sourceUrlPlaceholder": "https://.../index.json", + "sourceUrlPlaceholder": "ĥţţƥś://.../ĩńďēx.ĵśōń", "sourceOfficial": "Ōƒƒĩćĩàĺ", "sourceUnreachable": "Ũńŕēàćĥàƀĺē", "sourceUnreachableDetail": "<0>{{name}} ĩś ũńŕēàćĥàƀĺē{{detail}}; ĩţś ƥĺũĝĩńś àŕē ĥĩďďēń.", diff --git a/apps/web/src/locales/pseudo/settings.json b/apps/web/src/locales/pseudo/settings.json index 41836e0d85..e32b154254 100644 --- a/apps/web/src/locales/pseudo/settings.json +++ b/apps/web/src/locales/pseudo/settings.json @@ -749,6 +749,6 @@ "keyValueAddItem": "Àďď Ĩţēḿ", "keyValueKeyPlaceholder": "Ķēŷ", "keyValueValuePlaceholder": "Vàĺũē", - "externalMcpToolListPendingAgentPermissions": "Ĺĩśţ ĺĩvē ćōḿḿàńď àńď ţōōĺ ƥēŕḿĩśśĩōń ŕēǫũēśţś ƒōŕ à ţàśķ, ĩńćĺũďĩńĝ śàƒē àćţĩōń ďēţàĩĺś àńď ţĥē àĝēńţ'ś ēxàćţ ćĥōĩćēś.", - "externalMcpToolResolveAgentPermission": "Ŕēśōĺvē ōńē ĺĩvē ƥēŕḿĩśśĩōń ŕēǫũēśţ ƀŷ śēĺēćţĩńĝ àń ēxàćţ ōƥţĩōń ŕēţũŕńēď ƀŷ ţĥē ĺĩśţ ţōōĺ." + "externalMcpToolListPendingAgentPermissions": "Ĺĩśţ ĺĩvē ćōḿḿàńď àńď ţōōĺ ƥēŕḿĩśśĩōń ŕēqũēśţś ƒōŕ à ţàśķ, ĩńćĺũďĩńĝ śàƒē àćţĩōń ďēţàĩĺś àńď ţĥē àĝēńţ'ś ēxàćţ ćĥōĩćēś.", + "externalMcpToolResolveAgentPermission": "Ŕēśōĺvē ōńē ĺĩvē ƥēŕḿĩśśĩōń ŕēqũēśţ ƀŷ śēĺēćţĩńĝ àń ēxàćţ ōƥţĩōń ŕēţũŕńēď ƀŷ ţĥē ĺĩśţ ţōōĺ." } diff --git a/apps/web/src/locales/pseudo/task.json b/apps/web/src/locales/pseudo/task.json index 819ba94b20..7ed8358030 100644 --- a/apps/web/src/locales/pseudo/task.json +++ b/apps/web/src/locales/pseudo/task.json @@ -1396,14 +1396,14 @@ "sortCreated": "Ćŕēàţēď", "sortCreatedDescription": "Ŵĥēń ţĥē ţàśķ ŵàś ćŕēàţēď.", "sortCustom": "Ćũśţōḿ", - "sortCustomDescription": "Ŧĥē ḿàńũàĺ ōŕďēŕ ẏōũ śēţ ƒōŕ ţàśķś.", + "sortCustomDescription": "Ţĥē ḿàńũàĺ ōŕďēŕ ŷōũ śēţ ƒōŕ ţàśķś.", "sortDirection": "Śōŕţ ďĩŕēćţĩōń {{direction}}", "sortLastActivity": "Ĺàśţ àćţĩvĩţŷ", "sortLastActivityDescription": "Ĺàśţ ũśēŕ ōŕ àĝēńţ àćţĩōń. Vĩēŵĩńĝ à ţàśķ ďōēś ńōţ ćĥàńĝē ĩţ.", "sortStatus": "Śţàţũś", - "sortStatusDescription": "Ŧàśķ śţàţē, ƒŕōḿ ŕēvĩēŵ ţō ƀàćķĺōĝ.", + "sortStatusDescription": "Ţàśķ śţàţē, ƒŕōḿ ŕēvĩēŵ ţō ƀàćķĺōĝ.", "sortTitle": "Ţĩţĺē", - "sortTitleDescription": "Ŧàśķ ţĩţĺē ĩń àĺƥĥàƀēţĩćàĺ ōŕďēŕ.", + "sortTitleDescription": "Ţàśķ ţĩţĺē ĩń àĺƥĥàƀēţĩćàĺ ōŕďēŕ.", "sortUpdated": "Ũƥďàţēď", "sortUpdatedDescription": "Ĺàśţ ţàśķ śũḿḿàŕŷ ŕēƒŕēśĥ. Ɓàćķĝŕōũńď ēvēńţś ćàń ćĥàńĝē ĩţ.", "source": "Śōũŕćē", diff --git a/apps/web/src/locales/pseudo/tasks.json b/apps/web/src/locales/pseudo/tasks.json index 2e0fc6fe77..f06052da2b 100644 --- a/apps/web/src/locales/pseudo/tasks.json +++ b/apps/web/src/locales/pseudo/tasks.json @@ -23,7 +23,7 @@ "groupByRepository": "Ŕēƥōśĩţōŕŷ", "groupByState": "Śţàţē", "groupByWorkflow": "Ŵōŕķƒĺōŵ", - "ungrouped": "Ŭńĝŕōũƥēđ", + "ungrouped": "Ũńĝŕōũƥēď", "loading": "Ĺōàďĩńĝ...", "loadingTasks": "Ĺōàďĩńĝ ţàśķś...", "noRepository": "Ńō ŕēƥōśĩţōŕŷ", diff --git a/apps/web/src/locales/pt-pt/office.json b/apps/web/src/locales/pt-pt/office.json index f454f36b43..fe30fbe8cb 100644 --- a/apps/web/src/locales/pt-pt/office.json +++ b/apps/web/src/locales/pt-pt/office.json @@ -683,6 +683,8 @@ "runStatusFailed": "falhada", "runStatusReceived": "recebida", "runStatusSkipped": "ignorada", + "runSkipped": "Ignoradas", + "runUnclassified": "Não classificadas", "runStatusTaskCreated": "tarefa criada", "runtime": "Runtime", "runtimeCapabilities": "Capacidades do runtime", diff --git a/apps/web/src/locales/zh-cn/office.json b/apps/web/src/locales/zh-cn/office.json index 4f429a24b0..b7ad6b1ec1 100644 --- a/apps/web/src/locales/zh-cn/office.json +++ b/apps/web/src/locales/zh-cn/office.json @@ -683,6 +683,8 @@ "runStatusFailed": "失败", "runStatusReceived": "已接收", "runStatusSkipped": "已跳过", + "runSkipped": "已跳过", + "runUnclassified": "未分类", "runStatusTaskCreated": "任务已创建", "runtime": "运行时", "runtimeCapabilities": "运行时能力", diff --git a/apps/web/src/locales/zh-hk/office.json b/apps/web/src/locales/zh-hk/office.json index e6275b1d09..1920549adf 100644 --- a/apps/web/src/locales/zh-hk/office.json +++ b/apps/web/src/locales/zh-hk/office.json @@ -683,6 +683,8 @@ "runStatusFailed": "失敗", "runStatusReceived": "已接收", "runStatusSkipped": "已跳過", + "runSkipped": "已跳過", + "runUnclassified": "未分類", "runStatusTaskCreated": "任務已建立", "runtime": "執行時", "runtimeCapabilities": "執行時能力", diff --git a/apps/web/src/locales/zh-tw/office.json b/apps/web/src/locales/zh-tw/office.json index 39b6519054..e4c4d4b980 100644 --- a/apps/web/src/locales/zh-tw/office.json +++ b/apps/web/src/locales/zh-tw/office.json @@ -683,6 +683,8 @@ "runStatusFailed": "失敗", "runStatusReceived": "已接收", "runStatusSkipped": "已跳過", + "runSkipped": "已跳過", + "runUnclassified": "未分類", "runStatusTaskCreated": "任務已建立", "runtime": "執行時", "runtimeCapabilities": "執行時能力", diff --git a/docs/plans/task-delivery-ledger/plan.md b/docs/plans/task-delivery-ledger/plan.md new file mode 100644 index 0000000000..2012a84e70 --- /dev/null +++ b/docs/plans/task-delivery-ledger/plan.md @@ -0,0 +1,647 @@ +--- +spec: docs/specs/task-delivery-ledger/spec.md +created: 2026-08-13 +status: in-progress +--- + +# Implementation Plan: Task delivery ledger + +**Implementation status:** tasks 01-07 (the ledger backend and the Office run +outcome backend) are done and have been through multiple Build/Testing/Review +rounds on `feature/tel-distinguish-dire-34b` — see the `Review round N` / +`R5-Fx` / `R6-Fx` comments across `apps/backend/internal/delivery/` and +`apps/backend/internal/office/service/` for the authoritative, in-code record +of what each round found and fixed. Tasks 08-09 (frontend + E2E) are still +pending, as their own task files state. + +## Overview + +Two delimited changes ship together. The **delivery ledger** is a new +`task_delivery_ledger` table plus a new `internal/delivery` package that owns its +schema, its evaluator, and a 5-minute sweep; it classifies each +`(task, repository)` pair as `pr_merge | direct_commit | no_delivery_observed | +unknown` and separately records whether the pair's work was observed reaching the +repository's default branch. The **Office run outcome** is a nullable +`runs.outcome` column written at the six active `FinishRun` call sites, so a +budget-blocked or idle-skipped run stops being counted as a success. + +Order is dependency-driven: shared migration/meta primitives first, then the +ledger's persistence and its pure evaluator, then the evidence queries and the +git ancestry probe, then the sweep that drives them and the writer-health +signals. The Office half is independent of the ledger and runs in parallel from +wave 1, with its frontend and E2E following. + +--- + +## Design decisions taken here + +The spec fixes the contract; these are the implementation choices it left open. +Each is a decision, not a restatement. + +**D1 — The ledger gets its own package, `internal/delivery/`.** It cannot live in +`internal/task/repository/sqlite` because the evaluator must read +`github_task_prs`, `gitlab_task_mrs` and `azure_devops_task_prs`, which that +package does not own; it cannot live in `internal/analytics` because that package +is a read-side stats + HTTP handler surface with no writer and no background +loop, and the ledger is a writer with a lifecycle. `internal/github/store.go` is +the ownership pattern being followed: a package that owns its own table DDL, +runs its own idempotent migration at boot, and exposes `Provide(...)`. + +**D2 — `runs.outcome` is added by the office repository's own migration.** It is a +column on an existing table owned by `internal/office/repository/sqlite/base.go`, +so per `apps/backend/CLAUDE.md` it is added by an idempotent `ADD COLUMN` in +`base_migrations.go` and *also* listed inline in the `CREATE TABLE` so fresh +databases get it. The migration is the source of truth for evolution. + +**D3 — Detachment is a GitHub-only predicate.** `detached_at` exists on +`github_task_prs` (`internal/github/store.go:137`) and on neither +`gitlab_task_mrs` nor `azure_devops_task_prs`. The spec's rule-1 "is not +detached" is therefore evaluated as `detached_at IS NULL` for GitHub and is +vacuously true for the other two providers. Do not add a `detached_at` column to +the GitLab or Azure tables: that is a provider-table change and this card does +not own it. + +**D4 — Azure DevOps merge is read from `status`.** `azure_devops_task_prs` has no +`merged_at` (`internal/azuredevops/store.go:37-58`). `status = 'completed'` is +the merged signal; `abandoned` and `active` are not. Any other status value +leaves the pair unclassified by rule 1 and falls through to rules 3-5, per the +spec's failure mode. Because Azure rows carry no `merged_at`, they sort **last** +in the `delivery_ref` tiebreak (`merged_at` NULLs last), then by +`pull_request_id` ascending. + +**D5 — Missing provider tables are tolerated, classified through `internal/db`.** +A database where a provider store never initialized has no such table. ADR 0027 +forbids local `strings.Contains` migration classifiers, so this adds +`db.IsMissingTableError` alongside the existing `IsDuplicateColumnError` / +`IsAlreadyExistsError` rather than string-matching in `internal/delivery`. + +**D6 — The snapshot-write trigger is served by the sweep, not a new event.** The +spec lists four evaluation triggers. Three have existing event types +(`task_session.state_changed`, `github.task_pr.updated`, +`gitlab.task_mr.updated`). There is **no** event published when a git snapshot is +written (`internal/task/repository/sqlite/git_snapshots.go:101` writes the row +and publishes nothing), and Azure has no PR-updated event. Rather than add a new +event type on the snapshot hot path, the sweep's freshness predicate covers both: +it evaluates every candidate pair whose `last_evaluated_at` is older than its +most recent input observation, and snapshot `created_at` is one of those +observations. Worst-case latency is one sweep interval, which the spec already +fixes at 5 minutes. See **Open Questions**. + +--- + +## Backend + +### Shared primitives (`internal/db`, `internal/persistence`) + +**`apps/backend/internal/db/errors.go`** — add: + +```go +// IsMissingTableError reports whether err means the referenced table or +// relation does not exist. Classifies the SQLite "no such table" string and +// Postgres SQLSTATE 42P01 (undefined_table). +func IsMissingTableError(err error) bool +``` + +Same shape and test placement as the existing `IsDuplicateColumnError` / +`IsAlreadyExistsError`. Required by D5 so `internal/delivery` never string-matches. + +**`apps/backend/internal/persistence/meta.go`** — add: + +```go +// WriteKeyIfAbsent writes value at key only if key is absent, and reports +// whether this call performed the write. Replay-safe: a second call against a +// key that already exists is a no-op and returns false. +func WriteKeyIfAbsent(db *sqlx.DB, key, value string) (bool, error) +``` + +Implemented as `INSERT INTO kandev_meta (key, value) VALUES (?, ?) ON CONFLICT +(key) DO NOTHING` through `db.Rebind`, reading `RowsAffected`. The existing +unexported `writeKey` (`meta.go:75`) is an upsert and would overwrite an +activation point on every boot, which the spec forbids. `kandev_meta` survives +`internal/system/database/reset.go` (`reset.go:111` keeps it), so activation +points survive a database reset — which is correct: the extract needs them. + +### `internal/delivery` (new package) + +``` +apps/backend/internal/delivery/ +├── models.go # Outcome, Basis, LedgerRow, Observation types + rank() +├── store.go # DDL, migration, activation point, lattice upsert, reads +├── observe.go # candidate pairs + per-pair evidence queries +├── evaluator.go # pure Classify(Observation) -> Classification +├── ancestry.go # git merge-base --is-ancestor probe +├── sweep.go # 5-minute loop + event subscribers, Start/Stop +├── metrics.go # expvar maps +└── provider.go # Provide(writer, reader, deps, log) (*Service, func() error, error) +``` + +**`models.go`** — the closed vocabularies as typed string constants: + +```go +type Outcome string +const ( + OutcomeUnknown Outcome = "unknown" + OutcomeNoDeliveryObserved Outcome = "no_delivery_observed" + OutcomeDirectCommit Outcome = "direct_commit" + OutcomePRMerge Outcome = "pr_merge" +) + +// rank orders the promotion lattice. A higher rank never demotes to a lower. +func rank(o Outcome) int // "" -> 0, unknown -> 1, no_delivery_observed -> 2, + // direct_commit -> 3, pr_merge -> 4 +``` + +`delivery_basis` and `reached_default_basis` constants exactly as the spec's +**Basis vocabulary** tables. `Outcome` and both basis types get a `Valid()` +method, and a table-driven test pins that the constant sets match the spec so a +typo cannot silently widen the enum. + +**`store.go`** — schema, applied through `db.MigrateLogger` at `Provide`: + +```sql +CREATE TABLE IF NOT EXISTS task_delivery_ledger ( + id TEXT PRIMARY KEY, + task_id TEXT NOT NULL, + repository_id TEXT NOT NULL, + workspace_id TEXT NOT NULL, + delivery_outcome TEXT, + delivery_basis TEXT, + delivery_ref TEXT, + reached_default_at TIMESTAMP, + reached_default_basis TEXT, + reached_default_ref TEXT, + observed_branch_commits INTEGER, + first_classified_at TIMESTAMP, + last_evaluated_at TIMESTAMP NOT NULL, + evaluation_seq INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMP NOT NULL, + updated_at TIMESTAMP NOT NULL, + UNIQUE(task_id, repository_id), + FOREIGN KEY (task_id) REFERENCES tasks(id) ON DELETE CASCADE +); +CREATE INDEX IF NOT EXISTS idx_task_delivery_ledger_ws_evaluated + ON task_delivery_ledger(workspace_id, last_evaluated_at); +``` + +Every column the feature adds is nullable except identity, sequence and +timestamps, exactly as the spec's **Data model** table. No `DEFAULT 0` and no +`DEFAULT ''` on any classification or observation column: a legacy or +never-evaluated row must read `NULL`, and a default would make "not observed" +indistinguishable from "observed as nothing". + +Immediately after the DDL, write the activation point exactly once: + +```go +persistence.WriteKeyIfAbsent(writer, "telemetry.delivery_ledger.activated_at", + time.Now().UTC().Format(time.RFC3339)) +``` + +**The lattice lives in SQL, not in Go.** One statement, so two racing evaluators +converge regardless of arrival order and neither can lose the other's write: + +```sql +INSERT INTO task_delivery_ledger ( + id, task_id, repository_id, workspace_id, + delivery_outcome, delivery_basis, delivery_ref, + reached_default_at, reached_default_basis, reached_default_ref, + observed_branch_commits, first_classified_at, + last_evaluated_at, evaluation_seq, created_at, updated_at +) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?) +ON CONFLICT (task_id, repository_id) DO UPDATE SET + -- Promotion only. :rank is the incoming outcome's lattice rank, computed + -- in Go and bound as a parameter; the stored rank is recomputed in SQL so + -- the comparison never depends on a prior read. + delivery_outcome = CASE WHEN :rank > + THEN :outcome ELSE task_delivery_ledger.delivery_outcome END, + delivery_basis = CASE WHEN :rank >= + THEN :basis ELSE task_delivery_ledger.delivery_basis END, + delivery_ref = CASE WHEN :rank > + THEN :ref ELSE task_delivery_ledger.delivery_ref END, + -- Write-once and monotonic. + reached_default_at = COALESCE(task_delivery_ledger.reached_default_at, :reached_at), + reached_default_basis = CASE WHEN task_delivery_ledger.reached_default_at IS NULL + THEN :reached_basis ELSE task_delivery_ledger.reached_default_basis END, + reached_default_ref = CASE WHEN task_delivery_ledger.reached_default_at IS NULL + THEN :reached_ref ELSE task_delivery_ledger.reached_default_ref END, + observed_branch_commits = MAX(COALESCE(task_delivery_ledger.observed_branch_commits, 0), :ahead), + first_classified_at = COALESCE(task_delivery_ledger.first_classified_at, :first_classified), + -- Always advance. + last_evaluated_at = :now, + evaluation_seq = task_delivery_ledger.evaluation_seq + 1, + -- Advance only on a real change. + updated_at = CASE WHEN + THEN :now ELSE task_delivery_ledger.updated_at END +``` + +`` is a `CASE` over `task_delivery_ledger.delivery_outcome` +mapping the four values to 1-4 and `NULL` to 0. `delivery_basis` uses `>=` rather +than `>` so a re-evaluation at the same outcome can refine the basis +(`branch_commits_unmerged` becoming `reached_default_unattributed` once ancestry +lands), which the spec's fifth Classification scenario requires. When the stored +rank exceeds the incoming rank, the caller increments +`delivery_ledger_demotions_suppressed_total`. + +`MAX(a, b)` is SQLite's scalar two-argument `MAX`; on Postgres it is `GREATEST`. +Select the dialect through the existing `internal/db` dialect seam used elsewhere +in the repository rather than branching on a driver-name string in +`internal/delivery`. + +Also on `store.go`: + +- `MaxLastEvaluatedAt(ctx) (time.Time, error)` — the data-side stall signal. +- `ListStalePairs(ctx)` — the sweep's work list; see **`observe.go`**. + +**`observe.go`** — candidate pairs and per-pair evidence, all against the single +shared database (every table below lives in the same SQLite file / Postgres +schema). + +Candidate pairs are the union of `task_repositories`, non-empty +`task_sessions.repository_id`, and the `task_id`/`repository_id` of all three +provider tables, restricted to `repository_id <> ''` and to repositories with +`deleted_at IS NULL` (a soft-deleted repository freezes evaluation per the spec). +A task with no repository yields no row. Ordering is `tasks.created_at ASC, +tasks.id ASC, repositories.id ASC` — deterministic, so an interrupted sweep +resumes on the same order. + +Per pair, gather one `Observation`: + +| Field | Source | +|---|---| +| `DefaultBranch` | `repositories.default_branch` (`internal/task/repository/sqlite/base_schema.go:325`) | +| `MaxAhead` | `MAX(ahead)` over the pair's `task_session_git_snapshots` | +| `DistinctHeads` | `COUNT(DISTINCT head_commit)` where `head_commit <> ''` | +| `DefaultBranchHeads` | distinct non-empty `head_commit` where `branch = default_branch` | +| `SnapshotCount` | row count; zero distinguishes rule 4 from rule 5 | +| `LastHeadCommit` | greatest-`created_at` snapshot's `head_commit`, for ancestry | +| `MergedPRs` | union across the three provider tables (below) | + +Snapshots reach the pair through `task_session_git_snapshots.session_id -> +task_sessions.id`, matching on `task_sessions.task_id` **and** +`task_sessions.repository_id`. A session with an empty `repository_id` +contributes to no pair and increments +`delivery_ledger_sessions_unattributed_total`. + +Merged provider rows, each query wrapped so `db.IsMissingTableError` yields zero +rows rather than failing the evaluation (D5): + +```sql +-- GitHub: detachment applies here and only here (D3). +SELECT pr_url, merged_at, pr_number, base_branch FROM github_task_prs + WHERE task_id = ? AND repository_id = ? + AND merged_at IS NOT NULL AND detached_at IS NULL +-- GitLab: no detachment concept. +SELECT mr_url, merged_at, mr_iid, base_branch FROM gitlab_task_mrs + WHERE task_id = ? AND repository_id = ? AND merged_at IS NOT NULL +-- Azure: no merged_at; status carries it (D4). +SELECT pull_request_url, NULL, pull_request_id, target_branch + FROM azure_devops_task_prs + WHERE task_id = ? AND repository_id = ? AND status = 'completed' +``` + +`delivery_ref` selection follows the spec exactly: earliest `merged_at` (NULLs +last, D4), then number ascending, then provider name ascending in the order +`azure_devops`, `github`, `gitlab`. `direct_commit` ref: among default-branch +snapshots sharing the greatest `created_at`, the lexicographically greatest +`head_commit`. Both orderings are total, so repeated evaluation selects the same +row — which is what makes the idempotency scenario testable. + +**`evaluator.go`** — one pure function, no database and no git: + +```go +func Classify(obs Observation) Classification +``` + +The spec's five rules in their exact order, first match wins. Pure means the +whole **Classification** and **Squash-merge** scenario set is a table-driven unit +test with no fixtures. `reached_default_at` is computed here too, from the +`provider_pr_merged` (merged PR whose base branch is the default branch), +`default_branch_commit` and `ancestor_of_default` bases; `push_webhook_default` +is defined in the vocabulary but never produced (spec, Out of scope). + +When `DefaultBranch == ""`, rules 2 and the ancestry observation are skipped and +the basis records `default_branch_unknown`. + +**`ancestry.go`** — at most one git call per pair per evaluation, 10-second +timeout: + +```go +subproc.NewGitCommand(ctx, "-C", repo.LocalPath, + "merge-base", "--is-ancestor", headCommit, defaultRef) +// run via subproc.RunGitClass(ctx, subproc.GitWorkClassBackground, cmd) +``` + +Production git must go through `subproc` with a work class +(`apps/backend/CLAUDE.md`); `background` is the correct class for a telemetry +sweep. **Only exit code 0 is evidence.** A non-zero exit, a missing checkout, an +absent object, or a timeout produces *nothing*: `reached_default_at` stays +`NULL`, the outcome is untouched, and `delivery_ledger_ancestry_errors_total` +increments on the error cases. This is not defensive coding — the spec's +finding 2 shows a negative ancestry result is a routine false negative under +squash-merge, receipted by PR #2514. + +Skipped entirely when `repositories.local_path` is empty or +`repositories.default_branch` is empty. + +**`sweep.go`** — the goroutine, following the repository's ownership rules +(single owner, `Start(ctx)` / `Stop()`, `sync.WaitGroup`, `time.NewTicker` +inside a `select` that also watches shutdown, idempotent on both ends, +restartable). 5-minute interval, no result cap. Plus bus subscribers for +`events.TaskSessionStateChanged`, `events.GitHubTaskPRUpdated` and +`events.GitLabTaskMRUpdated`, each resolving the affected pair and evaluating it +directly. `internal/delivery` gets `goleak.VerifyTestMain(m)`. + +**`metrics.go`** — `expvar` maps published at package init, following +`internal/common/subproc/metrics.go`: `delivery_ledger_evaluations_total` (keyed +by outcome), `delivery_ledger_rows_written_total`, +`delivery_ledger_demotions_suppressed_total`, +`delivery_ledger_ancestry_errors_total`, +`delivery_ledger_sessions_unattributed_total`. Exposed through the existing +dev-mode `/debug/vars` handler; no new route. + +Counters are necessary but not sufficient — they reset to zero on restart, so a +writer that died before the last restart reads healthy. `MaxLastEvaluatedAt` +is the signal that survives, compared against the most recent +`task_sessions` activity. Both are required. + +**`provider.go`** — `Provide(writer, reader *sqlx.DB, bus events.Bus, log *logger.Logger) +(*Service, func() error, error)`, wired in `internal/backendapp` alongside the +other providers, cleanup calling `Stop()`. + +### Office run outcome (`internal/office`) + +**Schema.** `apps/backend/internal/office/repository/sqlite/base_migrations.go`: + +```go +r.migrate.Apply("runs.outcome", `ALTER TABLE runs ADD COLUMN outcome TEXT`) +``` + +and `outcome TEXT` listed inline in the `runs` `CREATE TABLE` +(`base.go:301-…`) so fresh databases get it. Nullable, no default: pre-activation +rows read `NULL`. Activation point +`telemetry.run_outcome.activated_at` written once via +`persistence.WriteKeyIfAbsent`. **No backfill** — the spec is explicit that the +pre-activation series keeps its own bucket. + +**Writer.** `FinishRun` gains an outcome parameter. The narrowest change that +keeps every existing caller honest: add +`FinishRunWithOutcome(ctx, id string, outcome models.RunOutcome) error` to +`internal/office/service/scheduler_runs.go` and +`internal/office/repository/sqlite`, and leave `FinishRun(ctx, id)` delegating +with `OutcomeProcessed`. The four scheduler call sites in +`internal/office/service/scheduler_integration.go`: + +| Line | Outcome | +|---|---| +| `:218` agent not active | `agent_inactive` | +| `:247` idle skipped | `idle_skipped` | +| `:517` task-tree hold | `task_tree_held` | +| `:829` pre-execution budget block | `budget_blocked` | + +`runs.status` is untouched: a blocked run still reaches `finished`, so every +existing reader of `status` keeps working. The two agent-completion callers +`event_subscribers.go:408` and `:512` keep `processed`. Checkout errors use the +normal retry or failure path, and checkout contention requeues the run; neither +path finishes a run or records completed work. + +**Reader.** `internal/office/repository/sqlite/agent_summary.go:36` — +`RunCountsByDayForAgent` returns five buckets instead of three: + +```sql +SUM(CASE WHEN status = 'finished' AND outcome = 'processed' THEN 1 ELSE 0 END) AS succeeded, +SUM(CASE WHEN status = 'finished' AND outcome IS NOT NULL + AND outcome <> 'processed' THEN 1 ELSE 0 END) AS skipped, +SUM(CASE WHEN status = 'finished' AND outcome IS NULL THEN 1 ELSE 0 END) AS unclassified, +SUM(CASE WHEN status IN ('failed','timed_out') THEN 1 ELSE 0 END) AS failed, +SUM(CASE WHEN status NOT IN ('finished','failed','timed_out') THEN 1 ELSE 0 END) AS other +``` + +`AgentRunDayRow` (`agent_summary.go:13`) gains `Skipped` and `Unclassified`. +Legacy rows land in `unclassified`, never `succeeded` — that is the whole point: +the discontinuity is visible in the data rather than silent. + +**DTO.** `internal/office/dashboard/agent_summary.go` — `AgentRunActivityDay` +(`:44`) gains `skipped` and `unclassified`; `padAgentRunActivity` (`:306`) and +`buildSuccessRate` (`:366`) both change `Total` to +`Succeeded + Skipped + Unclassified + Failed + Other`. `AgentSuccessRateDay` +keeps its `succeeded`/`total` pair, so the success rate now correctly *falls* +when blocked runs stop counting as successes. That is the fix, not a regression. + +--- + +## Frontend + +The ledger adds no frontend surface. The Office run outcome does: two existing +charts read the response shape that changed. + +### `apps/web/lib/state/slices/office/types.ts` + +The run-activity day type (around `:308`) gains `skipped: number` and +`unclassified: number`. + +### `apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx` + +The stacked bar gains two segments between `succeeded` and `failed`: `skipped` +and `unclassified`, with the legend extended to match. `total` already comes +from the backend, so the existing proportional math is unchanged. + +### `apps/web/app/office/agents/[id]/dashboard/components/success-rate-chart.tsx` + +No shape change — it reads `succeeded` and `total`, both of which still exist. +Verify the rendered percentage against the new `total`; no code change is +expected here beyond the test. + +### i18n + +Two new keys in the `office` namespace for the new legend labels, following the +existing `t("office:succeeded")` call at `run-activity-chart.tsx:52`. Both go +through `t()`; neither is compared with `===`; neither is called at module +scope. Add the English and pseudo catalog entries in the same change, and use +plain punctuation (no U+2014). + +--- + +## Tests + +### Backend — shared primitives + +- **What:** `IsMissingTableError` classifies SQLite "no such table" and Postgres + 42P01, and rejects unrelated errors. + **File:** `apps/backend/internal/db/errors_test.go` + **How:** table-driven, alongside the existing duplicate-column cases. +- **What:** `WriteKeyIfAbsent` writes once and is a no-op on replay, returning + `false` and leaving the original value. + **File:** `apps/backend/internal/persistence/meta_test.go` + **How:** real SQLite temp DB; call twice, assert value and returned bool. + +### Backend — ledger + +- **What:** all five classification rules in order, first-match-wins, including + the `default_branch_unknown` fallthrough and the rule-4/rule-5 split + (snapshots-all-empty vs no observations at all). + **File:** `apps/backend/internal/delivery/evaluator_test.go` + **How:** table-driven over `Observation` literals; no DB, no git. One case per + Classification scenario in the spec. +- **What:** a negative ancestry result sets no column and does not change the + outcome; a missing checkout increments the error counter and leaves + `reached_default_at` NULL. + **File:** `apps/backend/internal/delivery/ancestry_test.go` + **How:** real `git init` fixture under `t.TempDir()` for the positive case; + a temp path with no repository for the error case. Channel-based + synchronization, not sleeps. +- **What:** migration applies to a database seeded before the feature, every new + column reads NULL on the pre-existing row, no pre-existing value changes, and + `runMigrations` called **twice** is clean; the activation point retains its + first-boot value across the replay. + **File:** `apps/backend/internal/delivery/store_migration_test.go` + **How:** mirrors + `internal/task/repository/sqlite/task_external_id_migration_test.go` — open a + temp SQLite DB, construct, seed, assert NULLs, construct again, re-assert. +- **What:** the same fresh-then-replay matrix on Postgres, with duplicate-object + errors classified through `internal/db`. + **File:** `apps/backend/internal/delivery/store_postgres_test.go` + **How:** env-gated on `KANDEV_TEST_POSTGRES_DSN`, per ADR 0027. +- **What:** promotion only. A stored `pr_merge` survives an evaluator that + computes `unknown`, and `delivery_ledger_demotions_suppressed_total` + increments. + **File:** `apps/backend/internal/delivery/store_lattice_test.go` + **How:** real SQLite; upsert `pr_merge`, upsert `unknown`, read back. +- **What:** two evaluators racing on one pair converge on `pr_merge` in either + commit order. + **File:** `apps/backend/internal/delivery/store_lattice_test.go` + **How:** run both orders explicitly; assert the same terminal row. The lattice + is in the SQL, so this is a real test of the statement, not of a Go mutex. +- **What:** re-evaluating unchanged inputs leaves all classification and + observation columns byte-identical and `updated_at` unchanged, while + `last_evaluated_at` and `evaluation_seq` advance. + **File:** `apps/backend/internal/delivery/store_idempotency_test.go` + **How:** real SQLite; evaluate twice; compare every column. +- **What:** `reached_default_at` is write-once — a later observation through a + different basis changes neither it nor `reached_default_basis`. + **File:** `apps/backend/internal/delivery/store_idempotency_test.go` +- **What:** `delivery_ref` tiebreaks: earliest `merged_at` wins; on an exact tie + the lower number wins; Azure rows (no `merged_at`) sort last; two + default-branch snapshots sharing `created_at` resolve to the + lexicographically greatest `head_commit`, stably across repeated evaluation. + **File:** `apps/backend/internal/delivery/observe_test.go` + **How:** real SQLite with all three provider tables created. +- **What:** a GitLab merged MR classifies `pr_merge` on the same basis as a + GitHub PR; a detached GitHub PR does not; an absent provider table yields zero + rows instead of an error. + **File:** `apps/backend/internal/delivery/observe_test.go` + **How:** real SQLite; the missing-table case simply omits the `CREATE TABLE`. +- **What:** end-to-end — candidate discovery through classification to a + persisted row, covering a task with no repository writing no row, and a + session with an empty `repository_id` counting as unattributed. + **File:** `apps/backend/internal/delivery/service_test.go` + **How:** integration test over a real SQLite database with task, session, + snapshot and provider rows seeded. +- **What:** the sweep starts, evaluates, and stops without leaking; `Stop` then + `Start` restarts cleanly. + **File:** `apps/backend/internal/delivery/sweep_test.go` + **How:** `testing/synctest` for the ticker; `goleak.VerifyTestMain(m)` in a + package `TestMain`. +- **What:** the stall signal reports the true last write instant with counters at + zero (the post-restart case). + **File:** `apps/backend/internal/delivery/sweep_test.go` + **How:** write a row, construct a fresh `Service` against the same DB, assert + `MaxLastEvaluatedAt` without re-evaluating. + +### Backend — Office run outcome + +- **What:** each of the six active `FinishRun` paths writes its documented outcome + while `status` stays `finished`. + **File:** `apps/backend/internal/office/service/scheduler_integration_test.go` + **How:** drive each path and read the `runs` row. The agent-inactive path is + explicitly included even though it writes no `office_activity_log` row — that + gap is why the activity log cannot carry this. +- **What:** `runs.outcome` migration applies to a pre-existing database, reads + NULL on old rows, and replays cleanly. + **File:** `apps/backend/internal/office/repository/sqlite/runs_outcome_migration_test.go` + **How:** the `task_external_id_migration_test.go` pattern; new file, not an + append, per the 800-effective-line file limit. +- **What:** the five-bucket day rollup — one processed, one budget-blocked, one + failed and one pre-activation NULL row return + `succeeded=1, skipped=1, failed=1, unclassified=1`. + **File:** `apps/backend/internal/office/repository/sqlite/agent_summary_test.go` + **How:** real SQLite, seeded runs; this is the spec's final Office scenario. +- **What:** `padAgentRunActivity` and `buildSuccessRate` compute + `total` over all five buckets. + **File:** `apps/backend/internal/office/dashboard/agent_summary_test.go` + **How:** table-driven over `AgentRunDayRow` literals. + +### Frontend + +- **What:** the run-activity bar renders five segments and the legend lists the + new buckets. + **File:** `apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.test.tsx` + **How:** render with a fixture day; assert segment values and legend labels. + +--- + +## E2E Tests + +- **Scenario:** GIVEN an Office agent with runs on a day, WHEN the agent + dashboard is opened, THEN the Run Activity legend shows the succeeded, + skipped, unclassified, failed and other buckets. + **File:** `apps/web/e2e/tests/office/agent-dashboard.spec.ts` (extend the + existing spec; do not add a new file) + **What to verify:** the legend labels are visible and the chart renders + without error against a seeded agent. + +No E2E is added for the ledger: it has no user-visible surface (spec, Out of +scope). + +--- + +## Verification Results + +Pending. On completion, synchronize this section with each task's `## Results`: +record exact commands and outcomes/counts, generated artifact paths, and +cleanup/teardown evidence. + +--- + +## Implementation Waves And Parallel Candidates + +The default is sequential execution in the primary conversation. Waves expose +possible parallelism only; they do not authorize subagents. + +``` +Wave 1 (parallel candidates — disjoint files — user authorization required): +- [ ] [task-01-shared-primitives](task-01-shared-primitives.md) +- [ ] [task-07-run-outcome-backend](task-07-run-outcome-backend.md) + +Wave 2: +- [ ] [task-02-ledger-store](task-02-ledger-store.md) + +Wave 3 (parallel candidates — disjoint files within internal/delivery): +- [ ] [task-03-evaluator](task-03-evaluator.md) +- [ ] [task-04-observations](task-04-observations.md) + +Wave 4 (parallel candidates — disjoint files): +- [ ] [task-05-ancestry](task-05-ancestry.md) +- [ ] [task-08-run-outcome-frontend](task-08-run-outcome-frontend.md) + +Wave 5: +- [ ] [task-06-sweep-and-health](task-06-sweep-and-health.md) + +Wave 6: +- [ ] [task-09-e2e-agent-dashboard](task-09-e2e-agent-dashboard.md) +``` + +Task 07 touches only `internal/office/**` and task 01 only +`internal/db` + `internal/persistence`, so wave 1 is genuinely disjoint. Tasks 03 +and 04 both live in `internal/delivery` but write different files and both +consume the types task 02 defines; neither edits a shared schema or migration. + +--- + +## Open Questions + +1. **Snapshot-write trigger (D6).** The spec lists "a git snapshot is written" + as an evaluation trigger, but no such event type exists and adding one puts a + publish on the snapshot write path. This plan serves that trigger through the + sweep's freshness predicate, bounding latency at one 5-minute interval. If + sub-interval latency on snapshot writes is actually required, that is a new + `task_session.git_snapshot.created` event type and a task 06 subscriber — + confirm before implementing. diff --git a/docs/plans/task-delivery-ledger/task-01-shared-primitives.md b/docs/plans/task-delivery-ledger/task-01-shared-primitives.md new file mode 100644 index 0000000000..4e4b547b2f --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-01-shared-primitives.md @@ -0,0 +1,102 @@ +--- +id: "01-shared-primitives" +title: "Shared migration and meta primitives" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 01: Shared migration and meta primitives + +Two small primitives the ledger needs and the repository does not yet have. Both +are additions to existing shared packages; neither changes existing behaviour. + +**1. `db.IsMissingTableError`.** The ledger reads three provider tables that may +not exist in a given database (a deployment where the GitLab or Azure store never +initialized). ADR 0027 forbids local `strings.Contains` migration classifiers in +schema-owning packages, so the classifier belongs in `internal/db` alongside +`IsDuplicateColumnError` and `IsAlreadyExistsError`, not in `internal/delivery`. + +```go +// IsMissingTableError reports whether err means the referenced table or +// relation does not exist. Classifies the SQLite "no such table" string and +// Postgres SQLSTATE 42P01 (undefined_table). +func IsMissingTableError(err error) bool +``` + +**2. `persistence.WriteKeyIfAbsent`.** The spec requires both activation points +to be written exactly once and never overwritten on replay. The existing +unexported `writeKey` (`internal/persistence/meta.go:75`) is an upsert and would +rewrite the instant on every boot, which destroys the property the extract +depends on. + +```go +// WriteKeyIfAbsent writes value at key only if key is absent, and reports +// whether this call performed the write. Replay-safe. +func WriteKeyIfAbsent(db *sqlx.DB, key, value string) (bool, error) +``` + +Implement as `INSERT INTO kandev_meta (key, value) VALUES (?, ?) ON CONFLICT +(key) DO NOTHING` through `db.Rebind`, reading `RowsAffected` for the bool. Do +not reuse `metaKeyUpsert`. + +- **Acceptance:** + 1. `db.IsMissingTableError` returns true for a real SQLite missing-table error + and for Postgres SQLSTATE `42P01`, and false for unrelated errors. + 2. `persistence.WriteKeyIfAbsent` returns `(true, nil)` on first write and + `(false, nil)` on every later call, leaving the stored value untouched. + 3. No existing caller of `writeKey` or `WriteVersion` changes behaviour. + +- **Verification:** + `cd apps/backend && go test ./internal/db/... ./internal/persistence/... && make lint` + +- **Files likely touched:** + - `apps/backend/internal/db/errors.go` + - `apps/backend/internal/db/errors_test.go` + - `apps/backend/internal/persistence/meta.go` + - `apps/backend/internal/persistence/meta_test.go` + +- **Dependencies:** None. + +- **Parallelism:** parallel-safe with task 07 — disjoint files + (`internal/db`, `internal/persistence` vs `internal/office`). + +- **Inputs:** Spec **Activation points** and **Persistence guarantees**; plan + **Shared primitives** and decision D5; + `docs/decisions/0027-replayable-schema-migrations.md`; existing classifiers in + `apps/backend/internal/db/errors.go`. + +- **Output contract:** summary, files changed, tests run with counts, blockers, + risks, and task/plan status update in the same conversation. + +## Results + +**Files changed:** `internal/db/errors.go` (+`IsMissingTableError`), +`internal/db/errors_test.go`, `internal/persistence/meta.go` +(+`WriteKeyIfAbsent`). No changes to `writeKey` or `WriteVersion`. + +**Commands run:** +- `cd apps/backend && go test ./internal/db/... ./internal/persistence/...` + → all packages `ok` (`internal/db`, `internal/db/dialect`, + `internal/persistence`); `TestIsMissingTableError` covers SQLite + "no such table" (direct and wrapped), Postgres SQLSTATE `42P01` (direct and + wrapped), a Postgres undefined-column error (must be `false`), an unrelated + error, and `nil` — 7 subtests, all pass. +- `make lint` — clean. + +**Note on acceptance #2:** `WriteKeyIfAbsent` itself has no dedicated +first-write/second-write unit test in `internal/persistence`; its +write-once contract is exercised indirectly through its two production +callers — `internal/delivery/store.go`'s activation write (covered by +`TestPostgresDeliveryLedgerMigration_FreshAndReplay`'s fresh-then-replay +assertion that the activation key is present and non-empty) and +`internal/office/repository/sqlite/run_outcome_activation.go`. No test +directly asserts the `(false, nil)` / unchanged-value behavior on a second +call to `WriteKeyIfAbsent` in isolation — flagged here rather than silently +assumed covered. + +**Security/trust and external side-effects:** None — both primitives are +pure classifier/DB-helper additions to existing packages; no new I/O +surface, no network calls, no secrets handled. diff --git a/docs/plans/task-delivery-ledger/task-02-ledger-store.md b/docs/plans/task-delivery-ledger/task-02-ledger-store.md new file mode 100644 index 0000000000..7f20274471 --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-02-ledger-store.md @@ -0,0 +1,147 @@ +--- +id: "02-ledger-store" +title: "Delivery ledger table, migration and lattice upsert" +status: done +wave: 2 +depends_on: ["01-shared-primitives"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 02: Delivery ledger table, migration and lattice upsert + +Create the `internal/delivery` package and everything that touches persistence: +the vocabularies, the table, its replay-safe migration, its activation point, and +the single upsert statement that enforces the promotion lattice. + +Do **not** implement classification, evidence queries, ancestry or the sweep here +— they are tasks 03-06 and depend on the types this task defines. + +## What to build + +**`models.go`** — the four `Outcome` constants, the seven `delivery_basis` +constants and the four `reached_default_basis` constants exactly as the spec's +**Basis vocabulary** tables, plus `rank(Outcome) int` implementing the lattice +(`"" -> 0, unknown -> 1, no_delivery_observed -> 2, direct_commit -> 3, +pr_merge -> 4`) and a `LedgerRow` struct with pointer/`sql.Null*` fields for +every nullable column so a `NULL` round-trips as `nil`, never as `0` or `""`. + +**`store.go`** — the DDL in the plan's **`store.go`** section, applied through +`db.MigrateLogger`. Every classification and observation column is nullable with +**no** `DEFAULT`. A `DEFAULT 0` on `observed_branch_commits` or a `DEFAULT ''` on +a basis column would make "never evaluated" read identically to "evaluated and +found nothing", which is the exact discontinuity the spec forbids. + +Immediately after the DDL, write the activation point once: + +```go +persistence.WriteKeyIfAbsent(writer, "telemetry.delivery_ledger.activated_at", + time.Now().UTC().Format(time.RFC3339)) +``` + +**The lattice upsert.** One `INSERT ... ON CONFLICT (task_id, repository_id) DO +UPDATE` statement, per the plan. The rank comparison lives in the SQL `SET` +clause, never in a Go read-then-write: two evaluators racing on the same pair +must converge on the higher-ranked outcome regardless of arrival order. + +Points that are easy to get wrong and are individually tested: + +- `delivery_basis` uses `>=` where outcome and ref use `>`, so a re-evaluation at + the same outcome can refine the basis. The spec's fifth Classification + scenario (`branch_commits_unmerged` becoming `reached_default_unattributed` + once ancestry lands) depends on this and on nothing else. +- `reached_default_at` / `_basis` / `_ref` are write-once: `COALESCE` the + timestamp and gate the other two on the *stored* timestamp being NULL. +- `updated_at` advances only when a classification or observation column actually + changes. `last_evaluated_at` and `evaluation_seq` advance on every persisted + evaluation including a no-change one. +- `MAX(a, b)` is SQLite's two-argument scalar form; Postgres needs `GREATEST`. + Select through the existing `internal/db` dialect seam, not a driver-name + string comparison in this package. + +Also expose `MaxLastEvaluatedAt(ctx) (time.Time, error)` for the stall signal +(task 06 consumes it) and an `UpsertResult` carrying whether a demotion was +suppressed, so task 06 can increment the counter without re-reading the row. + +- **Acceptance:** + 1. Against a database seeded before this feature, the table and index exist, + every new column reads `NULL` on a pre-existing row, and no pre-existing + value changes. + 2. Running the migration twice applies cleanly the second time and + `telemetry.delivery_ledger.activated_at` keeps its first-boot value. + 3. A stored `pr_merge` survives an upsert computing `unknown`, in either + commit order, and the result reports a suppressed demotion. + 4. Re-upserting unchanged inputs leaves every classification and observation + column byte-identical and `updated_at` unchanged, while `last_evaluated_at` + and `evaluation_seq` advance. + 5. `reached_default_at` set through one basis is not overwritten by a later + observation through a different basis. + +- **Verification:** + `cd apps/backend && go test ./internal/delivery/... && KANDEV_TEST_POSTGRES_DSN="$KANDEV_TEST_POSTGRES_DSN" go test -run Postgres ./internal/delivery/... && make lint` + + (The Postgres leg is env-gated per ADR 0027; if the DSN is unset, record that + explicitly in Results rather than reporting the leg as passed.) + +- **Files likely touched:** + - `apps/backend/internal/delivery/models.go` + - `apps/backend/internal/delivery/models_test.go` + - `apps/backend/internal/delivery/store.go` + - `apps/backend/internal/delivery/store_migration_test.go` + - `apps/backend/internal/delivery/store_lattice_test.go` + - `apps/backend/internal/delivery/store_idempotency_test.go` + - `apps/backend/internal/delivery/store_postgres_test.go` + +- **Dependencies:** Task 01 (`persistence.WriteKeyIfAbsent`). + +- **Parallelism:** sequential. It creates the package every later ledger task + imports. + +- **Inputs:** Spec **Data model**, **Basis vocabulary**, **Activation points**, + **Persistence guarantees**, **Ordering, idempotency, concurrency**, and the + **Migration and activation** + **Idempotency, ordering and concurrency** + scenarios; plan decisions D1 and D2; + `docs/decisions/0027-replayable-schema-migrations.md`; + `apps/backend/internal/task/repository/sqlite/task_external_id_migration_test.go` + as the migration-test pattern; `apps/backend/CLAUDE.md` "Schema & migrations". + +- **Output contract:** summary, files changed, tests run with counts, whether the + Postgres leg ran, blockers, risks, and task/plan status update in the same + conversation. + +## Results + +**Files changed:** `internal/delivery/models.go`, `internal/delivery/store.go`, +plus migration/lattice/idempotency test files under `internal/delivery/` +(`store_test.go`, `upsert_test.go`, and siblings — organized by concern rather +than exactly the `store_migration_test.go` / `store_lattice_test.go` / +`store_idempotency_test.go` split named in the plan). + +**Commands run:** +- `cd apps/backend && go test ./internal/delivery/...` → `ok`, 99 subtests + pass, 0 fail, 3.9s (full package run, not just this task's scope — the + package now also contains tasks 03-06's code). +- `KANDEV_TEST_POSTGRES_DSN=... go test -run Postgres ./internal/delivery/...` + — **not run**. This sandbox has no reachable Postgres instance: + `psql` against the local Postgres.app rejects trust auth (requires an + interactive GUI permission grant unavailable here), and `docker run + postgres:16` fails with a broken Docker daemon + (`error creating temporary lease: ... input/output error`). The env-gated + tests (`TestPostgresDeliveryLedgerMigration_FreshAndReplay`, + `TestPostgresUpsert_RankGuardedBehavior`) compile and `t.Skip` cleanly + without the DSN; they have not been run against a live Postgres in this + environment. Per ADR 0027 this is recorded as not run, not passed. +- `make lint` — clean. + +**Acceptance verification:** #1 (pre-existing DB, NULL columns) and #2 +(replay-safe migration + activation instant) are covered by the SQLite +migration tests; #3 (suppressed demotion, either commit order) and #4 +(unchanged-input re-upsert leaves classification/observation columns +byte-identical, `updated_at` frozen, `last_evaluated_at`/`evaluation_seq` +advance) and #5 (write-once `reached_default_at` across bases) are each +covered by dedicated `upsert_test.go` cases exercised in the 99-subtest run +above. + +**Security/trust and external side-effects:** None — new table only, no +network calls, no secrets. The rank comparison lives entirely in the SQL +`SET` clause (per the plan), so no read-then-write race exists in Go. diff --git a/docs/plans/task-delivery-ledger/task-03-evaluator.md b/docs/plans/task-delivery-ledger/task-03-evaluator.md new file mode 100644 index 0000000000..90fb83462c --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-03-evaluator.md @@ -0,0 +1,124 @@ +--- +id: "03-evaluator" +title: "Pure classification evaluator" +status: done +wave: 3 +depends_on: ["02-ledger-store"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 03: Pure classification evaluator + +One function, no database and no git: + +```go +func Classify(obs Observation) Classification +``` + +`Observation` is the input struct defined in the plan's **`observe.go`** table +(task 04 populates it; define it here if task 04 has not landed, and task 04 +consumes it as given). `Classification` carries `Outcome`, `DeliveryBasis`, +`DeliveryRef`, `ReachedDefaultAt`, `ReachedDefaultBasis`, `ReachedDefaultRef` and +`ObservedBranchCommits`. + +Keeping this pure is the point: it makes the spec's entire Classification and +Squash-merge scenario set a table-driven unit test with no fixtures, no temp +database and no git. + +## Rules + +The spec's five rules in exactly this order, first match wins: + +1. `pr_merge` — a merged, non-detached provider row exists. Basis + `provider_pr_merged`. +2. `direct_commit` — two or more distinct non-empty head commits on snapshots + whose `branch` equals the repository's `default_branch`. Basis + `default_branch_commit`. +3. `unknown` — commits observed on a non-default branch + (`MaxAhead > 0` or two or more distinct heads) and neither rule 1 nor 2 + matched. Basis `reached_default_unattributed` when `reached_default_at` is + set, `branch_commits_unmerged` otherwise. +4. `no_delivery_observed` — at least one snapshot exists and every one shows + `ahead = 0` with a single distinct head commit. Basis `no_commits_observed`. +5. `unknown` — no snapshot and no provider row. Basis `no_observations`. + +Rule 3 is deliberate, not a gap. The outcome enum is fixed at four values, so +"reached the default branch by a route nobody observed" is recorded in +`reached_default_at` / `reached_default_basis` rather than being forced into +`pr_merge` or `direct_commit`, either of which would assert a route that was not +observed. + +Rules 4 and 5 must not collapse into each other: "we looked and there was +nothing" and "we never looked" are different facts, and `SnapshotCount == 0` is +what separates them. + +## Default-branch observation + +Computed here from three of the four bases: `provider_pr_merged` (a merged +provider row whose base branch equals the default branch), +`default_branch_commit`, and `ancestor_of_default` (task 05 supplies the boolean; +treat an absent probe as no evidence). `push_webhook_default` is in the +vocabulary but is never produced by this card — the webhook subscriber is +explicitly out of scope. + +When `obs.DefaultBranch == ""`, rule 2 and the default-branch observation are +both skipped and the basis records `default_branch_unknown`. Classification still +falls through to rules 3-5; it does not error. + +- **Acceptance:** + 1. Every Classification scenario in the spec has a corresponding table case + and passes, including the rule-4 / rule-5 split and the + `default_branch_unknown` fallthrough. + 2. A negative or absent ancestry result produces no `reached_default_at` and + does not change the outcome. + 3. `Classify` performs no I/O: it takes no `context.Context`, no `*sqlx.DB` and + no `exec.Cmd`. + +- **Verification:** + `cd apps/backend && go test -run TestClassify ./internal/delivery/... && make lint` + +- **Files likely touched:** + - `apps/backend/internal/delivery/evaluator.go` + - `apps/backend/internal/delivery/evaluator_test.go` + - `apps/backend/internal/delivery/models.go` (add `Observation` / + `Classification` if task 04 has not already) + +- **Dependencies:** Task 02 (vocabularies and `rank`). + +- **Parallelism:** parallel-safe with task 04 — different files in the same + package, no shared schema or migration. Coordinate only on the `Observation` + struct: whichever task lands first defines it. + +- **Inputs:** Spec **Classification**, **Default-branch observation**, + **Failure modes**, and the **Classification** + **Squash-merge and negative + evidence** scenarios. + +- **Output contract:** summary, files changed, tests run with counts, blockers, + risks, and task/plan status update in the same conversation. + +## Results + +**Files changed:** `internal/delivery/evaluator.go`, `internal/delivery/evaluator_test.go` +(table-driven `TestClassify*` cases), `internal/delivery/models.go` +(`Observation` / `Classification` types, shared with task 04). + +**Commands run:** +- `cd apps/backend && go test -run TestClassify ./internal/delivery/...` → + `ok`, 25 subtests pass, 0 fail — covers the rule-4/rule-5 split, the + `default_branch_unknown` fallthrough (rules 3-5 still evaluated), and the + squash-merge / negative-evidence scenarios. +- `make lint` — clean. + +**Acceptance verification:** #1 (every spec Classification scenario has a +table case) and #2 (negative/absent ancestry produces no +`reached_default_at` and does not change the outcome) are covered by the +`TestClassify*` table cases above. #3 (`Classify` performs no I/O) verified +by reading `evaluator.go`'s `Classify` signature directly — it takes an +`Observation` value and returns a `Classification` value; no +`context.Context`, `*sqlx.DB`, or `exec.Cmd` appears anywhere in the +function or its call graph within this file. + +**Security/trust and external side-effects:** None — `Classify` is pure +computation over already-fetched data. +applicable, or explicitly state `None`. diff --git a/docs/plans/task-delivery-ledger/task-04-observations.md b/docs/plans/task-delivery-ledger/task-04-observations.md new file mode 100644 index 0000000000..e7c3a22e2f --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-04-observations.md @@ -0,0 +1,164 @@ +--- +id: "04-observations" +title: "Candidate pairs and evidence queries" +status: done +wave: 3 +depends_on: ["02-ledger-store"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 04: Candidate pairs and evidence queries + +Populate the `Observation` the evaluator consumes: discover candidate +`(task, repository)` pairs and gather each pair's evidence from the task tables +and all three provider tables. Every table involved lives in the same database, +so this is plain SQL, not a cross-service call. + +## Candidate pairs + +The union of `task_repositories`, non-empty `task_sessions.repository_id`, and +the `(task_id, repository_id)` of `github_task_prs`, `gitlab_task_mrs` and +`azure_devops_task_prs` — restricted to `repository_id <> ''` and to repositories +with `deleted_at IS NULL`. A soft-deleted repository freezes evaluation for its +pairs: existing rows are retained and `last_evaluated_at` stops advancing. + +A task with no repository yields **no ledger row**. Absence means "not +applicable" and is a different fact from `unknown`; do not synthesize a row with +an empty `repository_id` to make the join tidy. + +Ordering is `tasks.created_at ASC, tasks.id ASC, repositories.id ASC`. It must be +total and deterministic so an interrupted sweep resumes on the same order and +re-evaluates only what it did not reach. + +## Per-pair evidence + +Fill the `Observation` fields from the plan's table. Snapshots reach the pair +through `task_session_git_snapshots.session_id -> task_sessions.id`, matching on +`task_sessions.task_id` **and** `task_sessions.repository_id`. + +Three facts from the reference-store measurement that change the SQL: + +- **`ahead` is base-branch-relative, not upstream-relative** + (`internal/agent/runtime/lifecycle/event_types.go:288-294`). It counts commits + the branch has that its base does not, and it does not fall to zero on push. It + is a valid "this branch carries work" signal and an invalid "this session + authored work" signal. Use it only for `observed_branch_commits` and rule 3. +- **An empty `head_commit` is not a distinct head.** Filter `head_commit <> ''` + in every `COUNT(DISTINCT ...)`, or a pair with two empty snapshots reads as + having authored a commit. +- **A session with an empty `repository_id`** contributes to no pair. Count it + toward `delivery_ledger_sessions_unattributed_total` (task 06 owns the + counter; expose the count on the result here). Three such sessions exist in + the reference store. + +## Provider rows + +```sql +-- GitHub: detachment applies here and only here. +SELECT pr_url, merged_at, pr_number, base_branch FROM github_task_prs + WHERE task_id = ? AND repository_id = ? + AND merged_at IS NOT NULL AND detached_at IS NULL +-- GitLab: no detached_at column exists; the predicate is vacuously true. +SELECT mr_url, merged_at, mr_iid, base_branch FROM gitlab_task_mrs + WHERE task_id = ? AND repository_id = ? AND merged_at IS NOT NULL +-- Azure: no merged_at column exists; status carries merge. +SELECT pull_request_url, NULL, pull_request_id, target_branch + FROM azure_devops_task_prs + WHERE task_id = ? AND repository_id = ? AND status = 'completed' +``` + +Do not add a `detached_at` column to the GitLab or Azure tables, and do not add +`merged_at` to Azure. Those are provider-table changes this card does not own. +An Azure status outside the known vocabulary leaves the pair at `unknown` with +basis `branch_commits_unmerged` rather than being guessed. + +Wrap each provider query so `db.IsMissingTableError` (task 01) yields zero rows +instead of failing the whole evaluation — a database where a provider store never +initialized has no such table. + +## Deterministic reference selection + +- `delivery_ref` for `pr_merge`: earliest `merged_at`; NULL `merged_at` (every + Azure row) sorts **last**; ties broken by number ascending; further ties by + provider name ascending in the order `azure_devops`, `github`, `gitlab`. +- `delivery_ref` for `direct_commit`: among default-branch snapshots sharing the + greatest `created_at`, the lexicographically greatest `head_commit`. +- `observed_branch_commits`: `MAX(ahead)`, no tiebreak needed. + +Both orderings must be total. Anything that falls back to a random identifier or +a dialect-specific row number will make the idempotency test flap. + +- **Acceptance:** + 1. Candidate discovery covers all three sources, excludes empty + `repository_id` and soft-deleted repositories, and produces no row for a + task with no repository. + 2. A merged GitLab MR is found; a detached GitHub PR is not; an absent + provider table yields zero rows rather than an error. + 3. Every `delivery_ref` tiebreak in the spec resolves as specified and is + stable across repeated evaluation. + +- **Verification:** + `cd apps/backend && go test ./internal/delivery/... && make lint` + +- **Files likely touched:** + - `apps/backend/internal/delivery/observe.go` + - `apps/backend/internal/delivery/observe_test.go` + - `apps/backend/internal/delivery/models.go` (add `Observation` if task 03 has + not already) + +- **Dependencies:** Task 02 (package and types); task 01 + (`db.IsMissingTableError`). + +- **Parallelism:** parallel-safe with task 03 — different files in the same + package. Coordinate only on the `Observation` struct. + +- **Inputs:** Spec **Candidate pairs**, **Ordering, idempotency, concurrency**, + **Failure modes**, and **Evidence the classification already works** + findings 1 and 3; plan decisions D3, D4 and D5; table shapes at + `internal/task/repository/sqlite/base_schema.go:762`, + `internal/github/store.go:107`, `internal/gitlab/store.go:70`, + `internal/azuredevops/store.go:37`. + +- **Output contract:** summary, files changed, tests run with counts, blockers, + risks, and task/plan status update in the same conversation. + +## Results + +**Files changed:** `internal/delivery/observe.go`, `internal/delivery/observe_test.go`, +`internal/delivery/provider_seed_test.go` (seed helpers reused by later +tasks' tests), `internal/delivery/models.go` (`Observation`, shared with +task 03). + +**Commands run:** +- `cd apps/backend && go test -run 'TestCandidates|TestSnapshotsForPair|TestProvidersForPair|TestGitHubPRs|TestGitLabMRs|TestAzureDevOpsPRs' ./internal/delivery/...` + → `ok`, 9 subtests pass, 0 fail: + `TestCandidates_ExcludesMissingRepositoryAndTask`, + `TestCandidates_ExcludesMissingTask`, + `TestCandidates_TaskWithNoRepositoryProducesNoRow`, + `TestSnapshotsForPair_JoinsThroughSessions`, + `TestSnapshotsForPair_NullHeadCommitNormalizesToEmpty`, + `TestProvidersForPair_MissingTablesTolerated`, + `TestGitHubPRs_MergedAndDetachedFromColumns`, + `TestGitLabMRs_MergedFromColumnAndScopeIsProjectPath`, + `TestAzureDevOpsPRs_StatusCaseInsensitiveAndUnrecognisedIsNotMerged`. +- `cd apps/backend && go test ./internal/delivery/...` → `ok`, 99 subtests + pass, 0 fail (full-package run, confirms no interaction with tasks 02/03/05/06). +- `make lint` — clean. + +**Acceptance verification:** #1 (candidate discovery, missing-repository +exclusion, no row for a task with no repository) is covered by the three +`TestCandidates_*` cases above. #2 (merged GitLab MR found, detached GitHub +PR not, absent provider table yields zero rows) is covered by +`TestGitHubPRs_MergedAndDetachedFromColumns`, +`TestGitLabMRs_MergedFromColumnAndScopeIsProjectPath`, and +`TestProvidersForPair_MissingTablesTolerated`. #3 (`delivery_ref` tiebreaks) +lives in `evaluator.go`'s `selectProviderRef`, not `observe.go`, and is +covered by the `TestClassify*` table cases (task 03's verification run). + +**Security/trust and external side-effects:** None — read-only queries +against existing provider tables; `db.IsMissingTableError` (task 01) +classifies a missing table as zero rows rather than surfacing a raw driver +error. +teardown evidence. Record security/trust and external side-effect boundaries when +applicable, or explicitly state `None`. diff --git a/docs/plans/task-delivery-ledger/task-05-ancestry.md b/docs/plans/task-delivery-ledger/task-05-ancestry.md new file mode 100644 index 0000000000..6cc19cd019 --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-05-ancestry.md @@ -0,0 +1,128 @@ +--- +id: "05-ancestry" +title: "Default-branch ancestry probe" +status: done +wave: 4 +depends_on: ["02-ledger-store"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 05: Default-branch ancestry probe + +One bounded git call that answers a single question: is this pair's last observed +head commit an ancestor of the repository's default branch in the local checkout? + +```go +subproc.NewGitCommand(ctx, "-C", repo.LocalPath, + "merge-base", "--is-ancestor", headCommit, defaultRef) +// run via subproc.RunGitClass(ctx, subproc.GitWorkClassBackground, cmd) +``` + +Production git must go through `internal/common/subproc` with an explicit work +class (`apps/backend/CLAUDE.md`); `background` is correct for a telemetry sweep +and keeps this out of the interactive and lifecycle admission budgets. At most +one call per `(task, repository)` per evaluation, 10-second timeout. + +## Only exit code 0 is evidence + +This is the crux of the task and the easiest thing to get wrong. + +Exit 0 sets `reached_default_at` with basis `ancestor_of_default`. **Everything +else produces nothing**: `reached_default_at` stays `NULL`, `delivery_outcome` is +untouched, and no column anywhere records `false`. + +That is not defensive coding. Kandev squash-merges by policy, so a merged +branch's head commit never becomes an ancestor of the default branch. The spec's +finding 2 receipts this: PR #2514's branch head `4dfa4d545` is present in the +object store and is **not** an ancestor of `origin/main`; the commit that landed +is the squash `15524de62`. A negative ancestry result is therefore a routine +false negative, and persisting it as a negative fact would be worse than not +looking. + +Increment `delivery_ledger_ancestry_errors_total` (task 06 owns the counter; +surface the error from here) on a git failure, a missing object, or a timeout. +Do **not** increment it on a clean non-zero "not an ancestor" exit — that is a +successful probe with a negative answer, not an error. + +Skip the probe entirely when `repositories.local_path` is empty or +`repositories.default_branch` is empty; both are ordinary states, not failures. + +- **Acceptance:** + 1. A commit genuinely on the default branch of a real local checkout returns a + positive result and sets `reached_default_at` with basis + `ancestor_of_default`. + 2. A commit not on the default branch returns no evidence: no column records + `false` and the outcome is unchanged. + 3. A missing checkout, absent object or timeout increments the error counter, + leaves `reached_default_at` NULL, and does not fail the evaluation. + 4. The git command is constructed via `subproc.NewGitCommand` and run under the + `background` class; no raw `exec.Command("git", ...)` appears. + +- **Verification:** + `cd apps/backend && go test ./internal/delivery/... && make lint` + + The positive case needs a real `git init` fixture under `t.TempDir()`. That + spawns a subprocess, so `testing/synctest` is not applicable — use + channel-based synchronization or plain sequential execution, not `time.Sleep`. + +- **Files likely touched:** + - `apps/backend/internal/delivery/ancestry.go` + - `apps/backend/internal/delivery/ancestry_test.go` + +- **Dependencies:** Task 02 (package and basis vocabulary). + +- **Parallelism:** parallel-safe with task 08 — disjoint trees + (`apps/backend/internal/delivery` vs `apps/web`). + +- **Inputs:** Spec **Default-branch observation**, **Failure modes**, + **Evidence** finding 2, and the **Squash-merge and negative evidence** + scenarios; `apps/backend/CLAUDE.md` git-subprocess rule; + `docs/decisions/2026-08-02-class-aware-git-subprocess-admission.md`; + helper signatures in `apps/backend/internal/common/subproc/shared.go:86`. + +- **Output contract:** summary, files changed, tests run with counts, blockers, + risks, and task/plan status update in the same conversation. + +## Results + +**Files changed:** `internal/delivery/ancestry.go`, `internal/delivery/ancestry_test.go`. +(`TestAncestryChecker_CheckoutResolverErrorIsAnAncestryError` was rewritten +during this feature's Review round 2 to use a valid-looking commit SHA +instead of the literal string `"HEAD"`, which short-circuited +`looksLikeCommitSHA` before ever reaching the checkout-resolver call the +test claimed to cover — the fix also asserts the fake resolver's call count +to prove invocation.) + +**Commands run:** +- `cd apps/backend && go test ./internal/delivery/... -run TestAncestry -v` + → `ok`, 12 subtests pass, 0 fail across the 10 `TestAncestryChecker_*` + functions listed in `ancestry_test.go` (some carry sub-cases via `t.Run`). + `TestAncestryChecker_PositiveResult` and + `TestAncestryChecker_LocalBranchFallbackWhenRemoteTrackingRefMissing` use a + real `git init` fixture under `t.TempDir()`; both clean up automatically + via `t.TempDir()`'s own teardown, no manual removal needed. +- `cd apps/backend && go test ./internal/delivery/...` → `ok`, 99 subtests, + 0 fail (full package). +- `make lint` — clean. + +**Acceptance verification:** #1 (positive ancestor result, +`ancestor_of_default` basis) — `TestAncestryChecker_PositiveResult`. #2 +(commit not on default branch → no evidence, outcome unchanged) — +`TestAncestryChecker_NegativeResultIsNotAnError`. #3 (missing checkout / +absent object / timeout increments the error counter, leaves +`reached_default_at` NULL, does not fail the evaluation) — +`TestAncestryChecker_NeitherRefResolvesIsAnError` and +`TestAncestryChecker_CheckoutResolverErrorIsAnAncestryError`. #4 (git +command via `subproc.NewGitCommand` under `background` class, no raw +`exec.Command("git", ...)`) — confirmed directly in `ancestry.go`: both git +invocations go through `subproc.RunGitAfterAcquire(ctx, subproc.GitBackground, +...)` wrapping `subproc.NewGitCommand`; `grep -n "exec.Command" ancestry.go` +returns no matches. + +**Security/trust and external side-effects:** None beyond the git +subprocess itself, which is read-only (`rev-parse`, `merge-base +--is-ancestor`) and admission-classified per +`docs/decisions/2026-08-02-class-aware-git-subprocess-admission.md`. Test +fixtures are ordinary `t.TempDir()` git repos, cleaned up by the Go test +runner. diff --git a/docs/plans/task-delivery-ledger/task-06-sweep-and-health.md b/docs/plans/task-delivery-ledger/task-06-sweep-and-health.md new file mode 100644 index 0000000000..2275454716 --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-06-sweep-and-health.md @@ -0,0 +1,187 @@ +--- +id: "06-sweep-and-health" +title: "Sweep, event triggers, writer health and wiring" +status: done +wave: 5 +depends_on: ["02-ledger-store", "03-evaluator", "04-observations", "05-ancestry"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 06: Sweep, event triggers, writer health and wiring + +Assemble the pieces into a running writer and make its failure visible. + +## Service + +`service.go` composes observe -> classify -> ancestry -> upsert for one pair, and +drives the whole candidate set for the sweep. Evaluation is idempotent, so a pair +evaluated twice with unchanged inputs is safe and a partially-completed sweep +needs no resume state beyond the deterministic ordering task 04 provides. + +A pair whose task still has running sessions is evaluated normally. There is no +"wait until the task is finished" gate: the lattice makes early classification +safe, because an outcome can only be promoted. + +## Sweep + +`sweep.go` — 5-minute ticker, **no result cap**. It evaluates every candidate +pair whose `last_evaluated_at` is older than its most recent input observation. A +pair is never silently skipped; if the sweep is interrupted, the next one resumes +on the same ordering. + +Follow the repository's goroutine ownership rules exactly +(`apps/backend/CLAUDE.md`): single owner, `Start(ctx)` / `Stop()`, registration +on a `sync.WaitGroup`, `Stop` cancels and waits for drain, idempotent on both +ends, restartable (reset stopped state and create a fresh cancellable context +under the same mutex `Stop` uses). Use `time.NewTicker` inside a `select` that +also watches shutdown — never `time.Sleep` in the loop. Add +`goleak.VerifyTestMain(m)` in a package `TestMain`. + +## Event triggers + +Subscribe to the three events that exist: + +- `events.TaskSessionStateChanged` (`internal/events/types.go:62`) — evaluate the + session's pair when the state is terminal. +- `events.GitHubTaskPRUpdated` (`:291`) +- `events.GitLabTaskMRUpdated` (`:306`) + +The spec's fourth trigger, a git snapshot write, has no event type: +`internal/task/repository/sqlite/git_snapshots.go:101` writes the row and +publishes nothing. Per plan decision D6 it is served by the sweep's freshness +predicate, bounding latency at one 5-minute interval rather than putting a +publish on the snapshot hot path. Azure has no PR-updated event and is covered +the same way. Do not add a new event type without resolving the plan's Open +Question 1 first. + +## Writer health + +Both signals are required, because they fail differently. + +**In-process counters** — `metrics.go`, `expvar` maps published at package init +following `internal/common/subproc/metrics.go`, exposed through the existing +dev-mode `/debug/vars` handler. No new route: + +- `delivery_ledger_evaluations_total` (keyed by outcome) +- `delivery_ledger_rows_written_total` +- `delivery_ledger_demotions_suppressed_total` (from task 02's `UpsertResult`) +- `delivery_ledger_ancestry_errors_total` (from task 05) +- `delivery_ledger_sessions_unattributed_total` (from task 04) + +**A data-side stall signal** — `MaxLastEvaluatedAt` (task 02) compared against +the most recent `task_sessions` activity, with a 15-minute threshold (three +missed sweeps). This is the signal that survives a restart: counters reset to +zero, so a writer that stopped before the last restart reads perfectly healthy by +counters alone and stale by this one. Shipping only the counters would satisfy +the letter of "writer health" and miss the case the spec actually names. + +Because the migration runner swallows errors +(`internal/db/migratelog.go:33`), a failed migration is invisible at boot. With +the table missing, writes fail and the stall detector fires — these health +signals are therefore also the migration-failure signal, and that is the only +detection this card has. + +## Wiring + +`provider.go` — `Provide(writer, reader *sqlx.DB, bus events.Bus, log *logger.Logger) +(*Service, func() error, error)` following the repository's provider pattern, +registered in `internal/backendapp` alongside the other providers, with cleanup +calling `Stop()`. + +- **Acceptance:** + 1. An end-to-end evaluation over a seeded database persists the correct row for + each of a `pr_merge`, `direct_commit`, `no_delivery_observed` and `unknown` + pair, writes no row for a task with no repository, and counts a session with + an empty `repository_id` as unattributed. + 2. The sweep starts, evaluates, and stops with no goroutine leak; `Stop` then + `Start` restarts cleanly. + 3. All five counters appear in `/debug/vars` under dev mode and move as + documented. + 4. After a simulated restart (fresh `Service` over the same database, counters + at zero), the stall signal still reports the true last write instant. + +- **Verification:** + `cd apps/backend && go test -race ./internal/delivery/... ./internal/backendapp/... && make lint` + +- **Files likely touched:** + - `apps/backend/internal/delivery/service.go` + - `apps/backend/internal/delivery/service_test.go` + - `apps/backend/internal/delivery/sweep.go` + - `apps/backend/internal/delivery/sweep_test.go` + - `apps/backend/internal/delivery/metrics.go` + - `apps/backend/internal/delivery/provider.go` + - `apps/backend/internal/delivery/main_test.go` (`goleak.VerifyTestMain`) + - `apps/backend/internal/backendapp/` (provider registration + cleanup) + +- **Dependencies:** Tasks 02, 03, 04, 05. + +- **Parallelism:** sequential. It integrates all four preceding ledger tasks and + touches `internal/backendapp`. + +- **Inputs:** Spec **Evaluation triggers and cadence**, **Writer health**, + **Persistence guarantees**, **Failure modes**, and the **Writer health** + scenarios; plan decision D6; goroutine ownership and leak-testing rules in + `apps/backend/CLAUDE.md`; `internal/common/subproc/metrics.go` as the expvar + precedent. + +- **Output contract:** summary, files changed, tests run with counts, blockers, + risks, and task/plan status update in the same conversation. + +## Results + +**Files changed:** `internal/delivery/sweep.go`, `internal/delivery/sweep_test.go`, +`internal/delivery/sweep_internal_test.go`, `internal/delivery/metrics.go`, +`internal/delivery/metrics_test.go`, `internal/delivery/testmain_test.go` +(`goleak.VerifyTestMain`, added during Review round 2 — see finding #4 +below), `internal/backendapp/` (provider registration). No separate +`service.go`/`provider.go` split — sweep lifecycle and metrics live directly +in `sweep.go` / `metrics.go`. + +**Commands run:** +- `cd apps/backend && go test -race ./internal/delivery/... ./internal/backendapp/...` + → all `ok` (`internal/delivery` 6.2s, `internal/backendapp` 13.1s, + `internal/backendapp/ownershiplock` 4.9s), no race reports, no goroutine + leaks (`TestMain` wraps the `internal/delivery` suite in + `goleak.VerifyTestMain`). +- `make lint` — clean. + +**Acceptance verification:** #1 (skip/write/no-row/unattributed-session +cases) covered by the `TestRunPass_*` table in `sweep_test.go`. #2 (start, +evaluate, stop with no goroutine leak; double-`Start` is a no-op; double- +`Stop` is idempotent) — `TestSweep_StartStopLifecycle`, backed by the +package-wide `goleak.VerifyTestMain`. **Gap noted, not fixed here:** no test +drives `Stop()` then a second `Start()` on the same `*Sweep` and asserts it +evaluates again; `sweep.go`'s `started` flag does reset to `false` in `Stop` +so a restart is structurally supported, but nothing pins it — recording +this honestly rather than claiming acceptance #2's "Stop then Start +restarts cleanly" clause is independently verified. #3 (all five counters +appear in `/debug/vars`) — the counters are `expvar.NewInt`/`NewMap` +package-level vars (`metrics.go`), so the stdlib `/debug/vars` handler +exposes them automatically wherever it's mounted (dev mode); no dedicated +test asserts this beyond `TestSessionsUnattributedGauge_PublishesLatestNotAccumulated` +and the default-branch-persist-error counter test in +`internal/backendapp/orchestrator_default_branch_persist_test.go`. #4 +(stall signal survives a simulated restart) is architecturally guaranteed +rather than directly tested: `ComputeStallSignal` reads only from the +database (`maxLastEvaluatedAt`, `stallComparand`), never from the in-memory +expvar counters, so a fresh `Service` over the same database recomputes the +true last-write instant regardless of process-local counter state; no test +exercises "fresh Service, same DB" explicitly. + +**Security/trust and external side-effects:** None — the sweep loop reads +existing tables and writes only to `task_delivery_ledger` through the +task-02 upsert; no network calls, no secrets. + +**Review round 2 fixes folded in:** finding #3 (the `dueness_unavailable` +and `write_failed` WARN logs discarded their underlying error, contradicting +spec's "carrying the error" requirement — fixed by threading `ledgerErr` +and the last per-pair write error through to both `zap.Error(...)` calls), +finding #4 (added `testmain_test.go`'s `goleak.VerifyTestMain`), finding #7 +(rank-1 degraded-outcome-change counter and `updated_at` assertions), finding +#8 (verified both WARN logs fire exactly once per pass via +`zaptest/observer`), and finding #9 (archived-task stall-comparand +invariant). See the task plan's Review round 2 section for the full finding +list and per-finding verification. +teardown evidence. Record security/trust and external side-effect boundaries when +applicable, or explicitly state `None`. diff --git a/docs/plans/task-delivery-ledger/task-07-run-outcome-backend.md b/docs/plans/task-delivery-ledger/task-07-run-outcome-backend.md new file mode 100644 index 0000000000..29d699e242 --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-07-run-outcome-backend.md @@ -0,0 +1,211 @@ +--- +id: "07-run-outcome-backend" +title: "Office run outcome: column, writers and rollup" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 07: Office run outcome: column, writers and rollup + +The run-grain form of the same defect the ledger fixes: a terminal status is not +an outcome. `FinishRun` writes `status='finished'` on six paths, four of which +did no work, and `RunCountsByDayForAgent` counts every `finished` row as a +success. + +This task is independent of the ledger and shares no files with it. + +## Schema + +`apps/backend/internal/office/repository/sqlite/base_migrations.go`: + +```go +r.migrate.Apply("runs.outcome", `ALTER TABLE runs ADD COLUMN outcome TEXT`) +``` + +Also list `outcome TEXT` inline in the `runs` `CREATE TABLE` +(`internal/office/repository/sqlite/base.go:301`) so fresh databases get it. Per +`apps/backend/CLAUDE.md`, the migration is the source of truth for evolution and +must stand alone; the inline listing is a convenience for fresh databases only. + +Nullable, **no default**. Pre-activation rows read `NULL` and are never +backfilled — the spec is explicit that the pre-activation series keeps its own +bucket so the discontinuity is visible in the data rather than silent. + +Write the activation point once via `persistence.WriteKeyIfAbsent` (task 01): +`telemetry.run_outcome.activated_at`. If task 01 has not landed, note the +dependency and land the column first; do not use the upsert `writeKey`, which +would rewrite the instant on every boot. + +Vocabulary: `processed | budget_blocked | idle_skipped | agent_inactive | +task_tree_held`, as typed constants in +`internal/office/models`. + +## Writers + +Add `FinishRunWithOutcome(ctx, id string, outcome models.RunOutcome) error` to +`internal/office/service/scheduler_runs.go` and the sqlite repository; leave +`FinishRun(ctx, id)` delegating with `OutcomeProcessed` so existing callers are +unchanged. + +The four scheduler call sites in `internal/office/service/scheduler_integration.go`: + +| Line | Path | Outcome | +|---|---|---| +| `:218` | agent not active | `agent_inactive` | +| `:247` | idle skipped | `idle_skipped` | +| `:517` | task-tree hold | `task_tree_held` | +| `:829` | pre-execution budget block | `budget_blocked` | + +The two agent-completion callers (`internal/office/service/event_subscribers.go:408` +and `:512`) keep `processed`. Checkout errors use the normal retry or failure +path, and checkout contention requeues the run; neither path finishes a run or +records completed work. + +`runs.status` is unchanged. A blocked run still reaches `finished`, so every +existing reader of `status` keeps working — that is what keeps this change +contained. + +Note that only three of the six total paths write an `office_activity_log` row +(`run_idle_skipped`, `run_budget_blocked`, `run_processed`). The agent-inactive +and task-tree-hold paths log nothing, which is why the activity log cannot carry +this distinction and a column is needed. Taskless or unlaunchable runs fail +before a finished outcome is written. + +## Rollup + +`internal/office/repository/sqlite/agent_summary.go:36` — +`RunCountsByDayForAgent` returns five buckets instead of three: + +```sql +SUM(CASE WHEN status = 'finished' AND outcome = 'processed' THEN 1 ELSE 0 END) AS succeeded, +SUM(CASE WHEN status = 'finished' AND outcome IS NOT NULL + AND outcome <> 'processed' THEN 1 ELSE 0 END) AS skipped, +SUM(CASE WHEN status = 'finished' AND outcome IS NULL THEN 1 ELSE 0 END) AS unclassified, +SUM(CASE WHEN status IN ('failed','timed_out') THEN 1 ELSE 0 END) AS failed, +SUM(CASE WHEN status NOT IN ('finished','failed','timed_out') THEN 1 ELSE 0 END) AS other +``` + +`AgentRunDayRow` (`:13`) gains `Skipped` and `Unclassified`. Legacy rows land in +`unclassified`, never `succeeded`. + +## DTO + +`internal/office/dashboard/agent_summary.go` — `AgentRunActivityDay` (`:44`) +gains `skipped` and `unclassified`; `padAgentRunActivity` (`:306`) and +`buildSuccessRate` (`:366`) both change `Total` to +`Succeeded + Skipped + Unclassified + Failed + Other`. `AgentSuccessRateDay` +keeps its `succeeded`/`total` pair. + +The success rate will **fall** for agents whose runs were being blocked. That is +the fix landing, not a regression: those runs were never successes. + +- **Acceptance:** + 1. Each of the six `FinishRun` paths writes its documented outcome while + `runs.status` stays `finished`, including the agent-inactive path that + writes no activity-log row. + 2. The migration applies to a pre-existing database, `outcome` reads NULL on + every pre-existing row, and it replays cleanly on a second boot. + 3. A day with one processed, one budget-blocked, one failed and one + pre-activation NULL run reports + `succeeded=1, skipped=1, failed=1, unclassified=1`. + 4. `Total` in both DTO builders sums all five buckets. + +- **Verification:** + `cd apps/backend && go test ./internal/office/... && make lint` + +- **Files likely touched:** + - `apps/backend/internal/office/repository/sqlite/base.go` + - `apps/backend/internal/office/repository/sqlite/base_migrations.go` + - `apps/backend/internal/office/repository/sqlite/runs_outcome_migration_test.go` (new file, not an append — the 800-effective-line file limit applies to test files) + - `apps/backend/internal/office/repository/sqlite/agent_summary.go` + - `apps/backend/internal/office/repository/sqlite/agent_summary_test.go` + - `apps/backend/internal/office/models/models.go` + - `apps/backend/internal/office/service/scheduler_runs.go` + - `apps/backend/internal/office/service/scheduler_integration.go` + - `apps/backend/internal/office/service/scheduler_integration_test.go` + - `apps/backend/internal/office/dashboard/agent_summary.go` + - `apps/backend/internal/office/dashboard/agent_summary_test.go` + +- **Dependencies:** Task 01 for `persistence.WriteKeyIfAbsent` (activation point + only; the column and writers do not depend on it). + +- **Parallelism:** parallel-safe with task 01 — disjoint files + (`internal/office` vs `internal/db` + `internal/persistence`). + +- **Inputs:** Spec **`runs.outcome` (new column)**, **Office run outcome**, and + the **Office run outcome** scenarios; plan decision D2; + `apps/backend/CLAUDE.md` "Schema & migrations" and code-quality limits. + +- **Output contract:** summary, files changed, tests run with counts, blockers, + risks, and task/plan status update in the same conversation. Call out the + expected success-rate drop for affected agents so it is not read as a + regression. + +## Results + +**Files changed:** `internal/office/repository/sqlite/base_migrations.go` +(`outcome` column, nullable, no default), `internal/office/repository/sqlite/run_outcome_migration_test.go`, +`internal/office/repository/sqlite/agent_summary.go` + `agent_summary_test.go`, +`internal/office/models/models.go` (`Run.Outcome *string`), +`internal/office/service/run.go` (the 5-value `RunOutcome*` vocabulary), +`internal/office/service/scheduler_runs.go` (`FinishRun`/`FailRun`/ +`transitionRunTerminal`), `internal/office/service/scheduler_integration.go` +(the four non-subscriber `FinishRun` call sites), `internal/office/service/event_subscribers.go` +(the two `processed` call sites), `internal/office/dashboard/agent_summary.go` ++ `agent_summary_test.go`. + +**Commands run:** +- `cd apps/backend && go test ./internal/office/...` → all 23 test-bearing + packages `ok`, 0 fail (`internal/office/repository/sqlite`, + `internal/office/service`, `internal/office/dashboard`, + `internal/office/scheduler`, and 19 others; 4 sub-packages report + `[no test files]` and are not part of this task's scope). +- `make lint` — clean. + +**Acceptance verification:** #1 (each `FinishRun` path writes its +documented outcome, `runs.status` stays `finished`) — confirmed by reading +every call site directly: `scheduler_integration.go` covers +`agent_inactive`, `idle_skipped`, `task_tree_held` and `budget_blocked` +(4 of the 5 values), `event_subscribers.go` covers `processed` (the 5th, written by the +agent-completed subscriber). Review round 2's `TestFailRun_WritesFailedStatusAndNullOutcome` +(added this Build round, finding #1) additionally pins the failed path +(`status='failed'`, `outcome IS NULL`) at the service level, since +`TestFinishAndFailRun` had never actually called `FailRun`. #2 (migration +applies to a pre-existing DB, `outcome` reads NULL on every pre-existing +row, replays cleanly) — `TestRunOutcomeMigration_AddsNullableColumn`. #3 +(one processed + one budget-blocked + one failed + one pre-activation NULL +run → `succeeded=1, skipped=1, failed=1, unclassified=1`) — +`TestRunCountsByDayForAgent_BucketsStatuses` and +`TestGetAgentSummary_RunActivityBuckets_SkippedAndUnclassified`. #4 +(`Total` sums all five buckets in both DTO builders) — read +`internal/office/dashboard/agent_summary.go` and +`internal/office/repository/sqlite/agent_summary.go` directly: both compute +`Total = Succeeded + Skipped + Unclassified + Failed + Other`. + +**Expected success-rate drop:** confirmed in the task's own framing — agents +whose runs were previously miscounted (budget-blocked, idle-skipped, +agent-inactive, task-tree-held, or never-launched runs silently landing in +`succeeded`) will show a lower `AgentSuccessRateDay` +after this lands. This is the fix taking effect, not a regression: those +runs were never real successes. + +**Security/trust and external side-effects:** None — new nullable column +and read-only aggregation queries; no secrets, no network calls. + +**Lower-priority item folded in (TEST-002, Review round 2):** +`TestTransitionRunTerminal_LastWriterWinsOnStatusAndOutcome` +(`internal/office/service/scheduler_runs_test.go`) drives both orderings of +two terminal callers (`FinishRun` then `FailRun`, and the reverse) on the +same run row and asserts the row always reflects the later call's status +and outcome together — pinning spec:2130-2132's "set in one statement, can +never disagree" contract, which `transitionRunTerminal`'s single +`UPDATE ... SET status = ?, outcome = ?, finished_at = ?` already +guarantees but nothing previously tested directly. + +Pending. Before marking this task done, replace this with every exact command +actually run and its outcome/count, generated artifact paths, and cleanup or +teardown evidence. Record security/trust and external side-effect boundaries when +applicable, or explicitly state `None`. diff --git a/docs/plans/task-delivery-ledger/task-08-run-outcome-frontend.md b/docs/plans/task-delivery-ledger/task-08-run-outcome-frontend.md new file mode 100644 index 0000000000..634e4b354c --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-08-run-outcome-frontend.md @@ -0,0 +1,102 @@ +--- +id: "08-run-outcome-frontend" +title: "Run activity chart: skipped and unclassified buckets" +status: pending +wave: 4 +depends_on: ["07-run-outcome-backend"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 08: Run activity chart: skipped and unclassified buckets + +The only user-visible part of this card. The ledger adds no frontend surface; the +Office run outcome changes a response shape two existing charts read. + +## Types + +`apps/web/lib/state/slices/office/types.ts` — the run-activity day type (around +`:308`, the one carrying `succeeded`) gains `skipped: number` and +`unclassified: number`, matching the backend `AgentRunActivityDay` JSON tags from +task 07. + +## Run activity chart + +`apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx` — +the stacked bar gains two segments between `succeeded` and `failed`: `skipped` +and `unclassified`. Extend the legend to match. `total` comes from the backend +and already sums all five buckets after task 07, so the existing proportional +math needs no change. + +Pick colours that read as distinct from the existing emerald (succeeded), red +(failed) and muted (other) in both light and dark themes. + +## Success rate chart + +`apps/web/app/office/agents/[id]/dashboard/components/success-rate-chart.tsx` — +no shape change: it reads `succeeded` and `total`, both of which still exist. No +code change is expected. Verify the rendered percentage against the new `total` +and stop there. + +The displayed success rate will fall for agents whose runs were being blocked. +That is the correction landing, not a bug to compensate for in the UI. + +## i18n + +Two new keys in the `office` namespace for the new legend labels, following the +existing `t("office:succeeded")` at `run-activity-chart.tsx:52`. Non-negotiable +rules from `apps/web/AGENTS.md` and `docs/i18n.md`: + +- Both labels go through `t()`; no hardcoded literal. The ratchet + (`pnpm run i18n:ratchet`) fails on a hardcoded string in a line you changed. +- Neither is compared with `===`, and neither `t()` call is at module scope. +- Add the English and pseudo catalog entries in the same change. +- Plain punctuation only. No U+2014 em dash in any catalog value. + +A clean lint is not proof the file is done: the rule skips literals assigned to +SCREAMING_CASE identifiers, so review any config-table constants in these two +components by eye. + +## Mobile + +These are existing charts inside an existing dashboard route; the change adds +segments to a bar and entries to a legend. Confirm the wider legend still wraps +rather than overflowing at mobile width, and that no new interaction is +introduced that would need a native mobile pattern. + +- **Acceptance:** + 1. The run-activity bar renders five segments and the legend lists all five + buckets, both labels resolved through `t()`. + 2. `pnpm run i18n:check` and `pnpm run i18n:ratchet` pass. + 3. The success-rate chart renders correctly against the new `total` with no + code change. + 4. The legend wraps rather than overflowing at mobile width. + +- **Verification:** + `cd apps && pnpm install --frozen-lockfile && pnpm --filter @kandev/web lint && pnpm --filter @kandev/web test -- app/office/agents && cd apps/web && pnpm run typecheck && pnpm run i18n:check && pnpm run i18n:ratchet` + +- **Files likely touched:** + - `apps/web/lib/state/slices/office/types.ts` + - `apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.tsx` + - `apps/web/app/office/agents/[id]/dashboard/components/run-activity-chart.test.tsx` + - `apps/web/public/locales/en/office.json` (or the equivalent catalog path) + - the pseudo catalog for the same namespace + +- **Dependencies:** Task 07 (the response shape must exist first). + +- **Parallelism:** parallel-safe with task 05 — disjoint trees (`apps/web` vs + `apps/backend/internal/delivery`). + +- **Inputs:** Spec **Office run outcome** and its scenarios; plan **Frontend**; + `apps/web/AGENTS.md`; `docs/i18n.md`; the existing chart components as the + pattern. + +- **Output contract:** summary, files changed, tests run with counts, i18n check + results, blockers, risks, and task/plan status update in the same conversation. + +## Results + +Pending. Before marking this task done, replace this with every exact command +actually run and its outcome/count, generated artifact paths, and cleanup or +teardown evidence. Record security/trust and external side-effect boundaries when +applicable, or explicitly state `None`. diff --git a/docs/plans/task-delivery-ledger/task-09-e2e-agent-dashboard.md b/docs/plans/task-delivery-ledger/task-09-e2e-agent-dashboard.md new file mode 100644 index 0000000000..be61e74b73 --- /dev/null +++ b/docs/plans/task-delivery-ledger/task-09-e2e-agent-dashboard.md @@ -0,0 +1,58 @@ +--- +id: "09-e2e-agent-dashboard" +title: "E2E: agent dashboard run-activity legend" +status: pending +wave: 6 +depends_on: ["08-run-outcome-frontend"] +plan: "plan.md" +spec: "../../specs/task-delivery-ledger/spec.md" +--- + +# Task 09: E2E: agent dashboard run-activity legend + +One assertion on the only user-visible change in this card. + +**Scenario.** GIVEN an Office agent with runs on a day, WHEN the agent dashboard +is opened, THEN the Run Activity legend shows the succeeded, skipped, +unclassified, failed and other buckets and the chart renders without error. + +Extend the existing `apps/web/e2e/tests/office/agent-dashboard.spec.ts` rather +than adding a new spec file — this is an additional assertion on a route that +already has coverage, not a new user flow. + +The ledger gets no E2E: it has no user-visible surface, which the spec's **Out of +scope** fixes as a contract. Do not add board columns, badges, filters or +dashboard tiles for it here. + +- **Acceptance:** + 1. The extended spec passes against a seeded Office agent. + 2. The legend assertion reads the translated labels the way the rest of the + suite does, not hardcoded English that would break under the pseudo-locale. + 3. No new spec file is added and no unrelated spec is modified. + +- **Verification:** + `cd apps && pnpm install --frozen-lockfile && cd apps/web && pnpm e2e:raw -- e2e/tests/office/agent-dashboard.spec.ts` + +- **Files likely touched:** + - `apps/web/e2e/tests/office/agent-dashboard.spec.ts` + - `apps/web/e2e/fixtures/` or `apps/web/e2e/helpers/` only if the existing + seed does not produce runs with distinct outcomes + +- **Dependencies:** Task 08. + +- **Parallelism:** sequential — it verifies task 08's rendered output. + +- **Inputs:** Spec **Office run outcome** scenarios; plan **E2E Tests**; + `apps/web/e2e/README.md`; the existing `agent-dashboard.spec.ts`. + +- **Output contract:** summary, files changed, exact E2E command and pass/fail + counts, any fixture changes, blockers, risks, and task/plan status update in + the same conversation. + +## Results + +Pending. Before marking this task done, replace this with every exact command +actually run and its outcome/count, generated artifact paths, and cleanup or +teardown evidence (including temporary capture-spec removal and +`git diff --check` when used). Record security/trust and external side-effect +boundaries when applicable, or explicitly state `None`. diff --git a/docs/specs/INDEX.md b/docs/specs/INDEX.md index 100621fa7b..5e6338edde 100644 --- a/docs/specs/INDEX.md +++ b/docs/specs/INDEX.md @@ -32,10 +32,11 @@ The former legacy size exceptions for migrated sources were removed. All canonic ## Unmigrated additions -The following specification was added after this migration and remains in the legacy layout until it moves into an owning system: +The following specifications were added after this migration and remain in the legacy layout until they move into an owning system: - [Task Cost & Token Ledger](task-cost-ledger/spec.md) (draft) - [Startup listener before recovery](startup-listener-before-recovery/spec.md) (draft) - [Workflow on_enter action dispatch](workflow-on-enter-action-dispatch/spec.md) (draft) +- [Task Delivery Ledger](task-delivery-ledger/spec.md) (draft) ## Authoring rule diff --git a/docs/specs/spec-lint.json b/docs/specs/spec-lint.json index ff6c4cedb4..237e7cfce7 100644 --- a/docs/specs/spec-lint.json +++ b/docs/specs/spec-lint.json @@ -12,6 +12,7 @@ "legacy_size_exceptions": { "docs/specs/office-task-content-editing/spec.md": 84103, "docs/specs/task-cost-ledger/spec.md": 91931, + "docs/specs/task-delivery-ledger/spec.md": 157207, "docs/specs/workflow-on-enter-action-dispatch/spec.md": 116869 } } diff --git a/docs/specs/task-delivery-ledger/spec.md b/docs/specs/task-delivery-ledger/spec.md new file mode 100644 index 0000000000..3b28d785c4 --- /dev/null +++ b/docs/specs/task-delivery-ledger/spec.md @@ -0,0 +1,2280 @@ +--- +status: draft +created: 2026-08-12 +updated: 2026-08-13 +revision: 5 +owner: nova28 +decisions: + - ../../decisions/0027-replayable-schema-migrations.md + - ../../decisions/2026-08-02-class-aware-git-subprocess-admission.md +--- + +# Task delivery ledger + +## Why + +A task that never opened a pull request is currently indistinguishable from a task +that produced nothing. In the reference store that is the majority of the board: +of 75 observed `(task, repository)` pairs, only 16 carry a merged pull or merge +request. The remaining 59 are one undifferentiated bucket mixing research cards, +chores, work delivered by committing straight to the default branch, and genuine +failure. + +Kandev already holds enough to split most of that bucket. It does not, and cannot +today, answer the one question that would close it: **did this card's work ever +reach the repository's default branch?** No merge-into-default detection exists +anywhere. `patRepository.DefaultBranch` (`apps/backend/internal/github/pat_client.go:1017`) +is parsed for clone metadata and, at `:1160`, copied onto `pr.BaseDefaultBranch`; +neither use infers a merge. The push webhook — `processPush` in +`apps/backend/internal/github/webhook_service.go` — publishes branch and SHA but +is consumed only by the automation engine +(`apps/backend/internal/automation/github_webhook_subscriber.go`, its +`GitHubPushReceived` subscription), never for merge inference. + +Separately, Kandev's own success metric is wrong in a way that lands in the same +analysis. `FinishRun` writes `status='finished'` on six terminal call sites, +**four** of which did no work, and `agent_summary.go` counts every `finished` row as +a success. Only two of the six — the agent-completed subscribers — follow an agent +actually running. The four scheduler guards record the reason that work did not +run, as spelled out under **Office run outcome**. + +This spec defines a **delivery ledger** that records how, and whether, each +`(task, repository)` pair delivered; and a **run outcome** that stops Office from +reporting non-work as success. + +### Citation convention + +Every `path:line` in this document is a convenience pointer, not the contract. +Where a line number is paired with a symbol name, a function name, or a semantic +label, **the label is authoritative and the line number is not**. If the two +disagree at build time, follow the label and correct the line. This matters most +for the `FinishRun` call-site table under **Office run outcome**, where each site +carries the outcome it must record: a drifted line number must never be allowed +to mis-map an outcome. + +## Evidence the classification already works + +Every number below was measured against the reference store +(`~/.kandev/data/kandev.db`, snapshotted 2026-08-12) before any schema was +proposed, as the card required. They are the input inventory, not estimates. + +| Observation | Value | +|---|---| +| `task_session_git_snapshots` rows | 1,655 | +| ... with `ahead > 0` | 1,390 | +| ... with empty `head_commit` | 0 | +| Sessions with `ahead = 0` and one distinct `head_commit` (produced nothing) | 31 | +| Sessions with `ahead > 0` and a moving `head_commit` (authored commits) | 48 | +| Sessions with `ahead > 0` and a static `head_commit` (inherited a branch) | 8 | +| Sessions with `ahead = 0` and a moving `head_commit` | 0 | +| Sessions working directly on their repository's default branch | 1 (3 distinct heads, `ahead` 2) | + +Applying the precedence in **Classification** to that store, with no Kandev +change, classifies **38 of 75** `(task, repository)` pairs: + +| Outcome | Pairs | +|---|---| +| `pr_merge` | 16 | +| `direct_commit` | 1 | +| `no_delivery_observed` | 21 | +| `unknown` | 37 | + +Three findings from that pass change the design and are therefore normative +context, not commentary: + +1. **`ahead` is base-branch-relative, not upstream-relative** + (`apps/backend/internal/agent/runtime/lifecycle/event_types.go`, the `Ahead` + field). It counts commits the branch has that its base does not, and it does + not fall to zero when the branch is pushed. It is therefore a valid "this + branch carries work" signal and an invalid "this session authored work" + signal — the 8 sessions above with `ahead > 0` and a static head inherited + someone else's branch. **This is why the ancestry precondition in + **Default-branch observation** keys off a moving `head_commit` and not off + `ahead`.** Note the grain: every row in the table above counts **sessions**, not + pairs. Head movement is only evidence of authorship *within one session*, so every + predicate derived from this finding is evaluated per session and never by pooling a + pair's sessions together — see **Classification**, "Head movement is a per-session + fact". +2. **Merge cannot be detected by commit ancestry in this repository.** Kandev + squash-merges by policy, so a merged branch head never becomes an ancestor of + the default branch. Receipt: PR #2514's branch head + `4dfa4d545 "fix(web): key mobile theme toggle off resolvedTheme"` is present + in the object store and is **not** an ancestor of `origin/main`; the commit + that actually landed is the squash + `15524de62 "fix(web): show dark mode toggle in mobile menu (#2514)"`. Ancestry + is a valid positive signal and an invalid negative one. +3. **Pull requests are not a GitHub-only concept here.** Three provider tables + exist — `github_task_prs`, `gitlab_task_mrs`, `azure_devops_task_prs` — and + the reference store's GitLab table holds 4 merged merge requests on a + repository with no GitHub provider at all. A GitHub-only reading of + `pr_merge` undercounts by those 4. The three tables do **not** share a schema, + which the **Provider predicates** table below resolves column by column: + `azure_devops_task_prs` has no `merged_at` and no `detached_at`; + `gitlab_task_mrs` has `merged_at` but no `detached_at`. + +A fourth finding corrects the card's premise about the Office metric: only +**three** of the `FinishRun` paths in `scheduler_integration.go` write an +`office_activity_log` row — `run_idle_skipped`, `run_budget_blocked` and +`run_processed`. The agent-inactive, task-tree-hold, checkout-error and +checkout-unavailable paths log nothing, and the two `event_subscribers.go` +completion paths log a run event rather than an activity row. The activity log +therefore cannot carry this on its own. + +## What + +- Kandev maintains a **delivery ledger**: at most one row per + `(task, repository)`, recording a delivery outcome, the evidence that produced + it, and — separately — whether the pair's work was ever observed reaching the + repository's default branch. +- The ledger is a new table. It is **not** a pull-request state and does not + extend `github_task_prs`, `gitlab_task_mrs`, or `azure_devops_task_prs`. +- `delivery_outcome` takes exactly one of + `pr_merge | direct_commit | no_delivery_observed | unknown`, or `NULL` when the + pair has never been evaluated. +- Reaching the default branch is recorded as its own observation, because the + route by which work reached the default branch is frequently unknowable while + the fact that it arrived is not. +- An Office run records a structured **outcome** alongside its terminal status, + so a budget-blocked, idle-skipped, agent-inactive, tree-held, checkout-blocked or + never-launched run is no longer counted as a success. +- Every column added by this feature is nullable, legacy rows are `NULL` and + never `0` or `''`, and the instant each mechanism became authoritative is + published in the database so a point-in-time extract can tell "not observed" + from "observed as nothing". + +## Delivery slices + +This spec defines two contracts that share no build-order dependency. They are +specified together because they are the same defect at two grains, and they are +**built in this order**, each closing on its own: + +- **Slice A — Office run outcome.** One nullable column, six call sites, **one new + field on `models.Run`**, one reshaped repository query and the two dashboard response + shapes it feeds. Depends on nothing in Slice B. + + **`models.Run` is not optional.** Ten production queries read `SELECT * FROM runs` + and `StructScan` into `office/models.Run` (eight in + `internal/runs/repository/sqlite/runs.go`, plus `office/repository/sqlite/failure.go` + and `run_routing.go`), and no `sqlx` handle in this backend is `Unsafe()`. A column + that exists in the table with no matching `db:` tag on the struct makes every one of + those queries fail at runtime, which would take out the whole Office run pipeline. + `models.Run` therefore gains `Outcome *string` with `json:"outcome,omitempty"` and + `db:"outcome"`, pointer-typed for the same reason the provider-routing columns on that + struct are pointer-typed — so `StructScan` reads the `NULL` that every pre-activation + row carries. + + Adding that field does **not**, by itself, change any response body. `models.Run` is + never serialized to a client: every run surface flattens it through an explicit DTO with + its own field list. **No run-grain DTO gains a field in this card** — specifically + `RunListItem` (via `runToListItem`), `AgentRunSummaryDTO` (via `buildRunSummaryDTO`) and + `RunRuntimeDTO` (via `buildRuntimeDTO`), all in `office/dashboard/`, are all unchanged. + So no per-run response body changes and the "Any user-visible surface" exclusion in + `## Out of scope` stands as written. + + **Two day-grain dashboard shapes DO gain fields, and that is in scope.** `AgentRunActivityDay` + gains `skipped` and `unclassified`, and `AgentSuccessRateDay` gains `unclassified`; the + contract is specified under `## Office run outcome` § Response shapes and summarized in + `## API surface`. All three are additive JSON fields that no frontend renders yet. The + distinction is run-grain versus day-grain, not "nothing changes" — an earlier revision of + this document said flatly "no DTO gains a field", which contradicted the Response shapes + section and could have been read as licence to ship the reshaped query with no wire fields + at all. + + **Additive on the wire is not the same as unchanged on screen.** Both shapes already feed + rendered charts on the agent dashboard, and this card changes the *values* they carry: the + Success Rate chart in particular reports 0% for every pre-activation day once `succeeded` + narrows to `outcome = 'processed'`. No `apps/web/` file changes and no new copy is + introduced, so the E2E path allowlist is unaffected — but nobody should read "no DTO field + is removed" as "no operator sees a difference". The new `unclassified` field is what makes + that difference explainable in-band; see § Response shapes. +- **Slice B — the delivery ledger.** New table, evaluator, sweep, upsert guard, + metrics, ancestry, two-dialect migrations. + +Sequencing is a plan-level decision recorded here so it survives context resets; +it changes no acceptance criterion. Both slices are in scope for this card. + +## Data model + +### `task_delivery_ledger` (new) + +One row per `(task_id, repository_id)`. Every column that carries **evidence** — the +classification triple, the default-branch observation triple, `observed_branch_commits` and +`first_classified_at` — is nullable, so "not observed" is always representable and is never +spelled as `0` or `''`. The columns that are `NOT NULL` are exactly: the identity columns +(`id`, `task_id`, `repository_id`), the denormalized scope column `workspace_id`, the +bookkeeping columns (`last_evaluated_at`, `evaluation_seq`, `created_at`, `updated_at`), and +`evidence_rank`, whose non-nullability is a correctness requirement argued in its own row +below. The table is authoritative; this sentence enumerates rather than generalizes, +because an earlier revision said "all columns other than the identity, sequence and +timestamp columns are nullable" and its own table contradicted it two rows later. + +| Column | Type | Meaning | +|---|---|---| +| `id` | TEXT PK | Row identity. | +| `task_id` | TEXT NOT NULL | FK to `tasks(id)` `ON DELETE CASCADE`. | +| `repository_id` | TEXT NOT NULL | FK to `repositories(id)` `ON DELETE CASCADE`. Never empty. The `ON DELETE` clause is stated rather than left to the implementation, for the same reason it is stated on `task_id`: `repositories` itself declares `FOREIGN KEY (workspace_id) REFERENCES workspaces(id) ON DELETE CASCADE` and `DELETE FROM workspaces` is a live production path, so repository rows **are** removed in ordinary operation even though nothing hard-deletes them directly. A bare FK (dialect default `NO ACTION`) would make workspace deletion begin failing the moment any ledger row referenced one of that workspace's repositories, which is a regression this feature would introduce into an existing user-facing operation. | +| `workspace_id` | TEXT NOT NULL | Denormalized from the task, for extract scoping. **Refreshed on every persisted evaluation** — it is not write-once and not rank-guarded, because it is an identity attribute of the task rather than evidence about delivery. A task moved between workspaces is picked up because `tasks.updated_at` is a due source (see **Sweep selection predicate**). | +| `delivery_outcome` | TEXT NULL | `pr_merge \| direct_commit \| no_delivery_observed \| unknown`. `NULL` means never evaluated. | +| `delivery_basis` | TEXT NULL | The evidence class that produced the outcome. See **Basis vocabulary**. | +| `delivery_ref` | TEXT NULL | The identifying evidence: a provider pull/merge request URL, or a commit SHA. | +| `evidence_rank` | INTEGER NOT NULL DEFAULT 0 | The rank of the `(outcome, basis)` pair that produced this row. See **Evidence rank**. Persisted so the upsert guard needs no lookup table at write time. **NOT NULL by design** — the upsert guard compares this column in SQL, and `x >= NULL` is `NULL` (falsy) on both dialects, so a nullable rank would silently suppress the first real write. This table is new, so it has no legacy rows and the "legacy rows get `NULL`" rule does not apply to it; that rule governs columns added to *existing* tables, i.e. `runs.outcome`. | +| `reached_default_at` | TIMESTAMP NULL | When work for this pair was first *observed* reaching the repository's default branch. Write-once. | +| `reached_default_basis` | TEXT NULL | How that observation was made. See **Basis vocabulary**. | +| `reached_default_ref` | TEXT NULL | The commit SHA or pull/merge request URL that carried the observation. | +| `observed_branch_commits` | INTEGER NULL | Monotonic high-water mark of `ahead`. See **Ordering, idempotency, concurrency**. | +| `first_classified_at` | TIMESTAMP NULL | When a non-`NULL` `delivery_outcome` was first written. Write-once. **Its instant is the one the evaluation began reading its inputs at**, the same source as `last_evaluated_at`, and it is fixed here for the same comparability reason this document gives for `reached_default_at`: a timestamp column whose source varies between implementations cannot be compared across rows. | +| `last_evaluated_at` | TIMESTAMP NOT NULL | Advances on every persisted evaluation, including no-change evaluations. **Its value is the instant the evaluation began reading its inputs, never the instant the upsert committed** — see **Sweep selection predicate**, "Which instant `last_evaluated_at` records". | +| `evaluation_seq` | INTEGER NOT NULL | Monotonic per row; incremented on every persisted evaluation. | +| `created_at` | TIMESTAMP NOT NULL | | +| `updated_at` | TIMESTAMP NOT NULL | Advances only when a classification or observation column changes. | + +`UNIQUE(task_id, repository_id)`. +Index on `(workspace_id, last_evaluated_at)` for the extract and the stall check. + +`reached_default_at` is deliberately **not** derivable from `delivery_outcome` +and does not participate in it. A pair may reach the default branch while its +outcome stays `unknown` (see **Classification**, rule 3). + +### Basis vocabulary + +`delivery_basis`: + +| Value | Meaning | +|---|---| +| `provider_pr_merged` | A non-detached provider pull/merge request for this pair reports merged. | +| `default_branch_commit` | The head commit advanced on snapshots whose branch is the repository's default branch. | +| `reached_default_unattributed` | Work reached the default branch, but by no route Kandev observed. | +| `branch_commits_unmerged` | Commits exist on a non-default branch and no delivery evidence was found. | +| `no_commits_observed` | Snapshots exist for this pair and every one shows no commits. | +| `provider_pr_unmerged` | No snapshot exists, and **no** provider pull/merge request row for this pair reports merged-and-not-detached. This is deliberately weaker than "every row reports not-merged": a row that merged and was later **detached** satisfies this basis, because rule 1 excludes detached rows from counting as delivery. The wording matches rule 5's predicate exactly, so a consumer must not read this basis as "no merged request ever existed" — it asserts that no *attributable* merged request exists now. | +| `no_observations` | Neither a snapshot nor a provider pull/merge request row exists. | +| `default_branch_unknown` | The repository's `default_branch` is empty, so default-branch rules could not be evaluated. Supersedes the matched rule's own basis; see **Degraded evaluation**. | + +`reached_default_basis`: + +| Value | Meaning | +|---|---| +| `provider_pr_merged` | Derived from a merged, **non-detached** provider pull/merge request whose base branch equals the repository's `default_branch`. Non-detached matters and is not inherited by implication: rule 1 excludes detached rows, so without the word here a detached row could offer an observation whose ref-selection set is empty. | +| `default_branch_commit` | The pair's session committed on the default branch itself. | +| `ancestor_of_default` | The pair's selected head commit is an ancestor of the default branch in the local checkout. | +| `push_webhook_default` | A verified push webhook reported this pair's head commit on the default branch ref. Defined for a later card; nothing writes it here (see **Out of scope**). | + +### Evidence rank + +The upsert guard and the monotonicity rule are both defined over a single +integer derived from the `(delivery_outcome, delivery_basis)` pair. **Rank is a +property of the evidence, not of the outcome enum** — that is what makes +`unknown` with observed commits stronger than `no_delivery_observed`, which the +outcome enum alone cannot express. + +| Rank | `delivery_outcome` | `delivery_basis` | +|---|---|---| +| 0 | `NULL` | `NULL` (never evaluated) | +| 1 | any | `default_branch_unknown` | +| 2 | `unknown` | `no_observations` | +| 3 | `unknown` | `provider_pr_unmerged` | +| 4 | `no_delivery_observed` | `no_commits_observed` | +| 5 | `unknown` | `branch_commits_unmerged` | +| 6 | `unknown` | `reached_default_unattributed` | +| 7 | `direct_commit` | `default_branch_commit` | +| 8 | `pr_merge` | `provider_pr_merged` | + +**The rank is injective over ranks 2–8, and deliberately not over ranks 0 and 1.** State +this precisely, because an earlier revision of this document claimed injectivity over the +whole ladder and that claim was false: + +| Rank | Injective? | Why | +|---|---|---| +| 0 | n/a | The never-evaluated sentinel. It is not a classification and no evaluation ever computes it. | +| 1 | **No, by design** | Every degraded evaluation lands here regardless of which rule matched, so rank 1 admits `(pr_merge, default_branch_unknown)`, `(unknown, default_branch_unknown)` and `(no_delivery_observed, default_branch_unknown)`. See **Degraded evaluation**. | +| 2–8 | **Yes** | Each maps to exactly one `(outcome, basis)` pair and each reachable non-degraded pair maps to exactly one rank. | + +Rank 1 is non-injective on purpose and the alternative was considered and rejected: giving +each degraded `(outcome, default_branch_unknown)` pair its own rank would destroy the +property that makes degraded rows safe — that a degraded row is the *weakest* evidence and +is therefore superseded by **any** real evaluation once `default_branch` is populated. A +degraded `pr_merge` at some higher rank would block a later, sighted +`unknown` / `no_observations` at rank 2, which is exactly backwards: the degraded row was +computed blind to the default branch and must never outrank a row that was not. + +What injectivity over 2–8 buys is narrow and is the only thing that rests on it: at ranks +2–8, equal rank implies the same `(outcome, basis)`, which is what makes the equal-rank +rule under **Monotonic evidence** a no-op in outcome and basis. It does **not** imply the +`delivery_ref` is unchanged — see that section. Rank 1 has its own equal-rank rule, stated +under **Degraded evaluation**. + +Do not add a value to ranks 2–8 without giving it its own rank. + +The rank order is the order in which evidence accumulates in practice: a pair +begins unobserved, may acquire an unmerged pull request, then snapshots showing +nothing, then commits, then a default-branch observation, then a route. No +transition in that sequence is a demotion, which is the property **Monotonic +evidence** relies on. + +### `runs.outcome` (new column) + +Nullable TEXT on the existing `runs` table: +`processed | budget_blocked | idle_skipped | agent_inactive | task_tree_held`. + +`NULL` for every row written before activation, for any run that never reaches a terminal +status, and for every run that reaches `status = 'failed'` (see **Office run outcome**, +the `FailRun` bullet). On the **finished** path the writer writes one of the five values +above; it never writes `''`. No database constraint is added — +the reader is total over every possible value (see **Office run outcome**), so +an unrecognised value degrades to `skipped` rather than breaking a query. +`runs.status` semantics are unchanged: a blocked run still reaches `finished`, +so every existing consumer of `status` keeps working. + +### Activation points + +Each mechanism writes its activation instant into the existing `kandev_meta` +key/value table, alongside the `schema_initialized_at` key already written by +`persistence.Provide`: + +- `telemetry.delivery_ledger.activated_at` +- `telemetry.run_outcome.activated_at` + +Both are RFC3339 UTC and never overwritten on replay. + +**An activation instant is written only after its schema is verified present.** This is +the single most important rule in this section, because the migration runner +(`MigrateLogger.Apply`) logs a failure at WARN and **swallows it**, so a migration that +did not apply is invisible at boot. Writing the key "on first application" would therefore +publish an activation instant for a table or column that does not exist — and a consumer +reading that key would conclude the mechanism was live and treat every absent row as +"observed as nothing", which is the exact inversion the activation point exists to +prevent. + +So each key is written only after a positive check, at the same boot, that its schema +actually exists: + +- `telemetry.delivery_ledger.activated_at` — only after a probe confirms + `task_delivery_ledger` is present and queryable. +- `telemetry.run_outcome.activated_at` — only after a probe confirms `runs.outcome` is + present. + +If the probe fails the key is not written, nothing is logged as success, and the next boot +retries both the migration and the probe. An absent activation key therefore means "this +mechanism is not live", which is a true statement a consumer can act on; a present one +means the schema was verified. The two are never ambiguous. + +**The activation key does not gate the writer.** The sweep starts and runs on its normal +cadence whether or not either key was written, and a failed probe never disables it. This +is stated because the opposite is the tempting reading and it would be self-defeating: if +a failed probe stopped the sweep, then a swallowed migration would produce no upsert +attempts, so `delivery_ledger_write_errors_total` would stay at `0` and +`delivery_ledger.write_failed` would never be logged — the migration-failure signals under +**Failure modes** would all be silenced by the very condition they exist to detect. The +key is a published fact *for consumers*, describing when the data became trustworthy. It +is not a feature flag. + +**Reset parity.** `internal/system/database/reset.go` drops every user table but +deliberately preserves `kandev_meta`. Left alone, that would leave an activation +instant pointing at a moment before data that no longer exists — inverting the +guarantee the activation point exists to provide, because a post-reset task with +no ledger row would read as "observed as nothing" rather than "not observed". +The reset path therefore **deletes both `telemetry.*.activated_at` keys** along +with the user tables it drops, so the next boot re-establishes an activation +point that matches the data actually present. This is the only change this spec +makes to the reset path. + +**Ordering is fixed, because the reset drops tables one at a time and is not +transactional.** The keys are deleted **first**, before any user table is dropped. The two +orders are not equivalent under partial failure and only one of them fails safe: + +- **Keys first, then drops.** If a drop then fails, the database holds ledger rows with no + activation key. A consumer reads "this mechanism is not live" and treats the rows as + untrusted — conservative, and recoverable on the next boot, which re-probes the schema + and rewrites the key. +- **Drops first, then keys.** If the key deletion then fails, the database holds an + activation instant pointing at data that no longer exists. A post-reset task with no + ledger row reads as "observed as nothing" rather than "not observed" — the precise + inversion this section exists to prevent, and one nothing later detects, because a + present key is exactly the state a healthy system is in. + +The asymmetry is the whole argument: a missing key is a true statement that costs a boot to +correct, while a stale key is a false statement that survives indefinitely. If the key +deletion itself fails, the reset fails and is reported as such rather than proceeding to +drop tables. + +## Classification + +For each candidate `(task, repository)` pair, rules are evaluated in this exact +order and the **first** match wins. + +**What a predicate may read.** Every predicate below reads the inputs observed on *this* +evaluation, plus the default-branch observation *this* evaluation has just computed. No +rule reads a stored **classification** column (`delivery_outcome`, `delivery_basis`, +`delivery_ref`, `evidence_rank`, `observed_branch_commits`). Rule 3 is the only rule that +reads an observation, and it reads the freshly computed one, never the stored +`reached_default_at`. Evaluation order within a single evaluation is therefore fixed: +**the default-branch observation is computed first, then the rules are matched.** + +The consequence is deliberate and must not be treated as a defect: if a pair's ancestry +check errors or is skipped on a later evaluation, that evaluation computes +`branch_commits_unmerged` (rank 5) rather than `reached_default_unattributed` (rank 6). +That is a demotion, so **Monotonic evidence** suppresses it, the stored row keeps rank 6, +and `delivery_ledger_demotions_suppressed_total` increments. Reading the stored column +instead would hide a real evidence regression behind an equal-rank no-op; counting it is +the point. + +Normalization applied before any rule runs: + +- A `head_commit` that is `NULL`, empty, or whitespace-only is **not** a distinct head + value and is ignored **everywhere in this document**, not merely in this section. It is + never counted toward a distinct-head predicate, never selected as a ref, and never + submitted to git. The scope is stated document-wide because two selection rules that + consume `head_commit` — the ancestry head under **Default-branch observation**, "Which + commit", and the `direct_commit` ref under **Ordering** — live in other sections and + select by greatest `created_at`; scoped to this section alone, a newer empty head would + outrank a real one and reach either `git --is-ancestor` or a write-once column. Both rules + restate the filter locally so neither can be read in isolation and get it wrong. +- An `ahead` that is `NULL` or negative is read as `0`. +- A `repositories.default_branch` that is `NULL` is read as the empty string, and an + empty `default_branch` triggers **Degraded evaluation**. The column is + `TEXT DEFAULT ''` and not `NOT NULL`, so both forms occur. +- A `task_sessions.repository_id` that is `NULL` is read as the empty string, and an + empty value attributes the session to no pair (see **Failure modes**). That column is + also `TEXT DEFAULT ''` and not `NOT NULL`. +- Branch comparison against `repositories.default_branch` is exact and + case-sensitive, and **never matches when `default_branch` is empty** — an empty + `default_branch` is not a branch name, and treating it as one would let a provider row + or snapshot with an equally empty branch column compare equal. See **Degraded + evaluation**. + +**Head movement is a per-session fact.** "Two or more distinct non-empty `head_commit` +values" is evaluated **within a single session's snapshots**, and a pair satisfies such a +predicate when **at least one of its sessions** does. It is never computed by pooling +every session's snapshots together. A task may have many sessions on one repository, and +two sessions that each merely *inherited* a different branch would pool into two distinct +heads and falsely read as authored work — the exact false positive the ancestry +precondition exists to prevent. The Evidence table above was measured per session for the +same reason. + +1. **`pr_merge`** — a provider pull/merge request row exists for this pair, is + not detached, and reports merged (see **Provider predicates**). Basis + `provider_pr_merged`. +2. **`direct_commit`** — at least one of the pair's sessions has two or more distinct + non-empty `head_commit` values among **its own** snapshots whose `branch` equals the + repository's `default_branch`. Basis `default_branch_commit`. +3. **`unknown`** — commits were observed: the greatest `ahead` across the pair's + snapshots is `> 0`, **or** at least one of the pair's sessions has two or more + distinct non-empty `head_commit` values among **its own** snapshots. + - Basis `reached_default_unattributed` when **this evaluation's** default-branch + observation produced a result, by any basis, whether or not that result was the one + persisted (a write-once column already set by an earlier evaluation is not + re-written, but the observation still counts here). + - Basis `branch_commits_unmerged` otherwise — including when the observation was + skipped or errored on this evaluation, which is a suppressed demotion rather than a + reclassification. +4. **`no_delivery_observed`** — at least one snapshot exists for this pair, every + snapshot shows `ahead = 0`, and **no** session of the pair has two or more distinct + non-empty `head_commit` values among its own snapshots. Basis `no_commits_observed`. +5. **`unknown`** — no snapshot exists for this pair, at least one provider + pull/merge request row exists, and none of them reports merged-and-not-detached. + Basis `provider_pr_unmerged`. +6. **`unknown`** — neither a snapshot nor a provider pull/merge request row + exists. Basis `no_observations`. + +**Totality.** The rules are exhaustive over every candidate pair, and this is +checkable by inspection. Rules 2–4 cover every pair with at least one snapshot: +rule 3's predicate and rule 4's predicate are exact complements of each other +(some `ahead > 0` **or** some session has ≥2 distinct heads of its own, versus all +`ahead = 0` **and** no session has ≥2 distinct heads of its own), and rule 2 is a strict +subset of rule 3's domain taken first — a session with ≥2 distinct heads on the default +branch has ≥2 distinct heads. Rules 5 and +6 cover every pair with no snapshot: provider rows exist, or they do not. Rule 1 +crosses both and is taken first. No candidate pair can reach the end of the list +unmatched, and no evaluation ever leaves `delivery_outcome` at `NULL` — `NULL` +means only "no evaluation has been persisted for this row". + +Rule 3 is the deliberate consequence of the outcome enum being fixed at four +values. "Reached the default branch by an unobserved route" is real, and it is +recorded in `reached_default_at` / `reached_default_basis` rather than being +forced into `pr_merge` or `direct_commit`, both of which would assert a route +that was not observed. `delivery_basis` carries the distinction the enum cannot. + +### Provider predicates + +Each provider table is read through the predicates below. There is no shared +schema, so each column is named per table. + +| Concept | `github_task_prs` | `gitlab_task_mrs` | `azure_devops_task_prs` | +|---|---|---|---| +| **Merged** | `merged_at IS NOT NULL` | `merged_at IS NOT NULL` | `LOWER(TRIM(status)) IN ('completed','merged')` | +| **Detached** | `detached_at IS NOT NULL` | never detached (no column; a GitLab detach deletes the row) | never detached (no column) | +| **Merge instant** | `merged_at` | `merged_at` | `updated_at` | +| **Request number** | `pr_number` | `mr_iid` | `pull_request_id` | +| **URL (`delivery_ref`)** | `pr_url` | `mr_url` | `pull_request_url` | +| **Base branch** | `base_branch` | `base_branch` | `target_branch` | + +Azure's merged vocabulary is taken from the terminal-status set this repository +already uses in `internal/azuredevops/watch_cleanup.go` +(`completed`, `abandoned`, `closed`, `merged`); `completed` and `merged` are the +merged states and every other value — including an unrecognised one — is +not-merged. An unrecognised status is **not** an error and is never guessed at: +the pair simply fails rule 1 and falls through. Azure carries no merge +timestamp at all, so `updated_at` is its merge instant; this is a sort key only +and is never written to `reached_default_at`. + +### Candidate pairs + +A pair is a candidate when **all three** hold: + +1. its `repository_id` is non-empty and it appears in any of `task_repositories`, + `task_sessions.repository_id`, or any provider pull/merge request table; **and** +2. that `repository_id` **resolves to a row in `repositories`**; **and** +3. its `task_id` **resolves to a row in `tasks`**. + +A task with no repository produces **no ledger row** — absence means "not applicable" and +is distinct from `unknown`. + +**Why condition 2 is not redundant.** A non-empty `repository_id` is not a guaranteed +foreign key anywhere it is read from: `task_sessions.repository_id` is +`TEXT DEFAULT ''` with **no** FK to `repositories`, and the three provider tables declare +their `repository_id` the same way (`github_task_prs.repository_id` is +`TEXT NOT NULL DEFAULT ''`). A dangling non-empty value is therefore representable in the +data, while `task_delivery_ledger.repository_id` **is** a real FK to `repositories(id)`. + +Without condition 2 such a pair is a candidate every sweep, every input read succeeds, the +upsert fails the foreign key forever, no row is ever written, and — because "no ledger row" +makes a pair unconditionally due — it is retried on every pass for the life of the +database, consuming work and producing nothing. + +So the pair is **excluded from candidacy and counted**, never attempted: +`delivery_ledger_pairs_missing_repository_total` is set per pass to the number of such +pairs. Counting is what keeps this consistent with "a pair is never silently skipped" — +the exclusion is explicit, observable and bounded, rather than being an INNER JOIN that +drops rows invisibly. A repository row that later appears makes the pair a candidate +again with no special handling, because candidacy is recomputed every pass. + +**Why condition 3 exists, and why it is not implied by condition 2.** The hazard is +structurally identical and it is reachable through a different table, so guarding only the +repository side leaves the same failure loop open on the task side. +`github_task_prs.task_id` is `TEXT NOT NULL` with **no** FK to `tasks`, and no code path +deletes `github_task_prs` rows when a task is hard-deleted — the GitHub task-deleted +subscriber prunes PR **watches** and credentials only. (Azure DevOps does clean up, with a +`DELETE FROM azure_devops_task_prs WHERE task_id = ?`; GitHub does not, and the divergence +is why this cannot be reasoned about per provider.) Because the provider tables are a +declared candidacy source in condition 1, an orphaned GitHub pull request row yields a +candidate pair whose task row is gone, while `task_delivery_ledger.task_id` **is** a real +FK to `tasks(id)`. + +Three separate things break if such a pair is admitted, and none of them is recoverable at +build time from the rest of this document: + +- Its upsert fails the `tasks(id)` foreign key on every pass, forever, exactly as in the + repository case — and because every failed upsert increments + `delivery_ledger_write_errors_total`, the counter this spec designates as **the primary + migration-failure signal** would climb continuously on a completely healthy schema. That + does not merely add noise; it destroys the one signal a `/debug/vars` reader is told to + watch, by making its resting value non-zero for an unrelated reason. +- The sweep order under **Ordering** is `tasks.created_at` ascending, then `tasks.id`. A + pair with no task row has neither value. An INNER JOIN would drop it silently, which + contradicts "a pair is never silently skipped"; a LEFT JOIN would sort a `NULL` in a + position neither dialect fixes. +- `workspace_id` is `NOT NULL` and is defined as denormalized *from the task*. With no task + row there is no value to denormalize and no honest default. + +So the pair is **excluded from candidacy and counted**, never attempted, on the same terms +as condition 2: `delivery_ledger_pairs_missing_task_total` is set per pass to the number of +candidate pairs rejected because their `task_id` matched no `tasks` row. It is a separate +gauge from the repository one rather than a combined "unresolvable pairs" count, because +the two have different causes and different owners — a missing repository points at +repository lifecycle, a missing task points at provider-row cleanup — and a single number +would tell an operator neither. A `tasks` row that later appears (or a provider row that is +finally pruned) changes the count on the next pass with no special handling, because +candidacy is recomputed every pass. + +Note that both conditions are different from soft-deletion. A soft-deleted repository +(`repositories.deleted_at` non-`NULL`) still **has** a row, so its pairs remain candidates +and are governed by the freeze under **Persistence guarantees**; a missing row is not a +candidate at all. The two must not be collapsed: one retains its ledger rows, the other +never had any. An **archived** task likewise still has a row and is still evaluated +(**Persistence guarantees**); only a hard-deleted task removes the row, and its ledger rows +go with it by cascade. + +### Degraded evaluation + +When `repositories.default_branch` is empty — or `NULL`, which normalizes to empty — +**every default-branch-dependent step is suspended**: rule 2, the ancestry observation, +*and* the `provider_pr_merged` default-branch observation. All three are suspended, not +just the first two, because the branch comparison is exact: a provider row whose own base +branch column is empty would compare **equal** to an empty `default_branch` and +permanently set the write-once `reached_default_*` triple during an evaluation that is +blind to what the default branch actually is. That is the one outcome degraded mode exists +to prevent, and it is why **Classification**'s normalization states that an empty +`default_branch` never matches any branch. + +Rule 1 itself still runs — whether a pull request merged does not depend on knowing the +default branch — so a degraded pair can still be classified `pr_merge`. Only the +*observation* derived from that row's base branch is suspended. + +Classification still runs and still matches exactly one +rule, but the result is degraded: `delivery_basis` records +`default_branch_unknown` **in place of** the matched rule's own basis, and +`evidence_rank` is `1` regardless of which rule matched. `delivery_outcome` is +whatever the matched rule produced. `delivery_ref` is `NULL`, per **Ordering**. + +Recording the degradation rather than the rule is deliberate: "this pair was +classified blind to the default branch" is the fact a consumer must not miss, +and rank 1 guarantees the degraded row is replaced by any later real evaluation +once `default_branch` is populated. + +**Rank 1 admits more than one outcome, and that has consequences the builder must not +have to derive.** Because every degraded evaluation lands at rank 1 whatever rule matched, +two successive degraded evaluations of the same pair can carry *different* outcomes at the +*same* rank. Worked example, which is reachable and not hypothetical: a degraded pair with +a merged pull request matches rule 1 and stores `pr_merge` / `default_branch_unknown` at +rank 1; the pull request is later detached, so rule 1 no longer matches, every snapshot +reports `ahead = 0`, rule 4 matches, and the pair re-evaluates to `no_delivery_observed` / +`default_branch_unknown` — also rank 1. + +The rules that govern that write are: + +- The write **happens**. The classification columns are written at equal rank, so + `delivery_outcome` becomes `no_delivery_observed`. A degraded row is a blind placeholder + and must track its inputs; freezing it at the first outcome it ever saw would be worse. +- It is **not** a suppressed demotion and + `delivery_ledger_demotions_suppressed_total` does **not** increment — the rank did not + fall, and that counter means what it says. +- It is **not silent**: + `delivery_ledger_degraded_outcome_changed_total` increments whenever an equal-rank write + at rank 1 changes `delivery_outcome`, so a pair thrashing while blind is visible rather + than being invisible in both counters. +- `updated_at` advances, because a classification column changed. + +None of this applies at ranks 2–8, where the rank is injective and an equal-rank write +cannot change the outcome. See **Ordering, idempotency, concurrency**, "Equal-rank +writes". + +### Default-branch observation + +`reached_default_at` may be set by any of the four bases in **Basis vocabulary**. +It is **write-once and monotonic**: once set it is never cleared, moved, or +overwritten by a later observation with a different basis. + +**Basis precedence within one evaluation.** A single evaluation can satisfy more than one +basis at once — a merged pull request whose base branch is the default branch, snapshots +committing on the default branch, and a positive ancestry check are not mutually +exclusive. Because the triple is write-once, whichever basis is chosen is permanent, so +the choice is fixed here and is not the builder's: + +| Precedence | Basis | `reached_default_ref` written | +|---|---|---| +| 1 (strongest) | `provider_pr_merged` | the **URL** column of the row selected by the **default-branch observation ref rule** below — *not* the `pr_merge` ref rule | +| 2 | `default_branch_commit` | the **commit SHA** selected by the `direct_commit` ref rule under **Ordering** | +| 3 | `ancestor_of_default` | the commit SHA submitted to the ancestry check, selected by **Which commit** below | +| 4 (weakest) | `push_webhook_default` | defined for a later card; nothing writes it here | + +The evaluation computes every basis it can, then writes the highest-precedence one that +produced a result. The order is by strength of attribution: a merged provider request +names the route, a default-branch commit names the act, ancestry names only the fact of +arrival. + +**The default-branch observation ref rule.** This is a *different row set* from the +`pr_merge` ref rule under **Ordering**, and conflating the two writes the wrong ref into a +write-once column. The `pr_merge` ref rule selects among the pair's merged, non-detached +provider rows with **no base-branch filter**, because rule 1 asks only whether a pull +request merged. The observation asks a strictly narrower question — whether work reached +*the default branch* — so it selects among the pair's merged, non-detached provider rows +**whose base branch equals the repository's `default_branch`**, and then applies the same +tiebreak keys in the same order (earliest merge instant, then request number ascending, +then provider name ascending in the order `azure_devops`, `github`, `gitlab`, then that +provider's scope column ascending). + +The two rules therefore agree whenever every merged row targets the default branch, and +diverge exactly when they should. Concretely: a pair with a merged pull request into a +release branch and a later merged pull request into `main` selects the release-branch row +for `delivery_ref` (it merged first, and rule 1 does not care where) and the `main` row +for `reached_default_ref`. Selecting the release-branch URL for the observation would +permanently record, in a write-once column, a pull request that never reached the default +branch. + +Because both rules resolve through named columns with total tiebreaks, repeated +evaluations of unchanged inputs select the same ref, and no new tiebreak is introduced. + +This precedence governs only which basis is *offered* to the write-once triple. Whether +it is *stored* is decided by **Monotonic evidence**: if `reached_default_at` is already +non-`NULL`, none of the three columns is written, regardless of precedence. + +Its **value** is the UTC instant at which the observation was first made — the +evaluation's own clock reading — not the snapshot's `created_at`, not the +provider's `merged_at`, and not the commit's author or committer date. Only the +observation instant is available for all four bases, and a column whose source +silently varies by basis cannot be compared across rows. + +The absence of `reached_default_at` never means "did not reach the default +branch". Ancestry in particular produces false negatives for every squash-merged +branch — see **Evidence**, finding 2. + +**Ancestry precondition.** An ancestry check is attempted **only when at least one of the +pair's sessions shows two or more distinct non-empty `head_commit` values among its own +snapshots** — that is, only when some single session's head actually moved and that +session therefore authored commits. The grain is per session, never pooled across the +pair; see **Classification**, "Head movement is a per-session fact". Pooling would let two +sessions that each inherited a different branch satisfy the precondition without either +one authoring anything, which is the precise failure this precondition exists to prevent. +This precondition is load-bearing, not defensive. A session that produced +nothing reports its inherited base commit as its head, and that commit is +already on the default branch's history, so an ungated `--is-ancestor` returns +**true** and would permanently set a write-once column on a pair that delivered +nothing. `ahead` cannot substitute for this test: per **Evidence** finding 1 it +is base-relative, so an inherited branch reports `ahead > 0` with a head that +never moved. + +When the precondition is not met the check is skipped, +`delivery_ledger_ancestry_skipped_total` increments, and nothing is written. + +**Which commit.** The head submitted to the check is selected from the snapshots of the +**session(s) that satisfied the moving-head precondition** — that is, only sessions +showing two or more distinct non-empty `head_commit` values among their own snapshots +contribute candidate heads — and, within that set, **only from snapshots whose +`head_commit` is non-empty** after the normalization in **Classification**. A `NULL`, +empty or whitespace-only head is not a candidate for selection at all, and a snapshot +carrying one is skipped over rather than winning on recency. Among that doubly filtered +set, the head is the `head_commit` of the snapshot with the greatest `created_at`; ties +are broken by the lexicographically greatest `head_commit` among those sharing that +`created_at`. + +The filter is restated here rather than inherited by implication because the ordering key +is `created_at` and the empty value would otherwise win whenever it is newest: a session +that recorded a final snapshot with no head would send an empty string to +`git merge-base --is-ancestor`, whose result is meaningless and whose failure would be +counted as an ancestry error rather than as the selection bug it is. + +**The session restriction is load-bearing, and an earlier revision of this document had it +backwards.** That revision said the selection was "across the pair's snapshots, not +restricted to the session that satisfied the precondition", on the reasoning that the +precondition decides *whether* to check while recency decides *what* to check. That +reasoning established only that a non-empty head exists **somewhere** across the pair. It +never established that pooling every session's snapshots is safe, and it is not. + +The failure it admits is the precondition's own, one step later. A session that authored +nothing reports its inherited base commit as its head, and that commit is already on the +default branch's history. Give a pair one authoring session on an unmerged feature branch +and one idle session sitting on the default branch, and the idle session's snapshot can +carry the **later** `created_at` — so pair-wide recency selects the inherited head, +`git merge-base --is-ancestor` returns **true** trivially (it never diverged, so it is +already an ancestor), and the **write-once** `reached_default_*` triple is permanently set +with a ref naming a commit the pair did not produce. Rule 3 then reads that observation and +promotes the classification from `branch_commits_unmerged` to +`reached_default_unattributed` for the life of the row. + +That is precisely the false positive **Ancestry precondition** above exists to prevent. The +precondition closes it at the *gate*, by refusing to check at all unless some session's head +moved; restricting selection closes the same hole at *selection*, which the gate alone does +not reach. Both are required, because the gate and the selector can otherwise resolve to +different sessions. + +The restricted set is still never empty when a check is attempted: the precondition +requires some session to hold two or more distinct non-empty heads, and that session's +snapshots are exactly what the restriction keeps. + +**Which checkout.** The repository's local checkout is `repositories.local_path`, and it is +**never read from the column directly**. The behaviour this card requires is exactly what +`resolveRepositoryLocalPath` in `internal/task/service/repository_discovery.go` already +implements: reject an empty repository ID, load the repository row, reject an empty +`local_path`, resolve the stored path through the explicit local-repository path +resolution, and reject a stored path that canonicalizes to a different location than the +one saved. This card inherits those canonicalization and containment checks rather than +restating them, because restating them is how they drift apart. + +**The seam, stated because that helper is not directly callable.** +`resolveRepositoryLocalPath` is an **unexported method** on the task service's `Service` +type and depends on that service's repository-entity handle, so it cannot be invoked from +the evaluator as written, and the evaluator does not belong inside the task service. The +contract is therefore stated as a port, not as a function call: + +- The evaluator depends on a **narrow, read-only, single-method port** — repository ID in, + validated absolute checkout path or error out — and on nothing else from the task tier. +- That port is satisfied by the task service's existing resolution behaviour, exposed for + this consumer. Whether it is exposed by exporting the method, by an adapter, or by a + wrapper is an implementation choice, and the choice is free **provided the validation + performed is the existing one and is not reimplemented**. +- The evaluator must **not** take a dependency on the concrete task service type, and must + **not** duplicate the canonicalization or containment logic. Duplicating it is the one + outcome this paragraph exists to prevent: the copy would silently omit the + "saved path resolves elsewhere" rejection, which is the check that stops a moved or + substituted checkout from being read. +- The port returns an error for every rejection above and never a sentinel empty path, so + the caller has exactly one failure shape to handle. + +A session worktree is never used: worktrees are per-session and transient, and the ancestry +question is about the repository. When `local_path` is empty or the port returns an error — +the ordinary case for a provider-cloned repository that has never been materialized locally +— the check is an error under **Failure modes**, never a negative result. + +**Which default branch.** Within that checkout the check resolves the default branch as +`refs/remotes/origin/`, falling back to the local branch +`` when the remote-tracking ref does not exist. If neither resolves, the +check is an error under **Failure modes** — never a negative result. + +## Evaluation triggers and cadence + +**The periodic sweep is the sole guaranteed mechanism.** Every pair is evaluated by the +sweep, on the cadence below, and every acceptance criterion in this document is satisfied +by the sweep alone. An implementation that ships only the sweep is complete against this +spec. + +Event triggers are **permitted accelerators, not requirements**. They reduce latency from +up to one sweep interval to near-immediate; they add no outcome the sweep would not +eventually reach, because evaluation is idempotent and the sweep predicate re-selects any +pair whose inputs moved. Where a trigger is wired it must use an existing bus event: + +| Input that moved | Event available today | +|---|---| +| A task session reached a terminal state | `events.TaskSessionStateChanged` (`task_session.state_changed`) | +| A GitHub pull request row changed | `events.GitHubTaskPRUpdated` (`github.task_pr.updated`) | +| A GitLab merge request row changed | `events.GitLabTaskMRUpdated` (`gitlab.task_mr.updated`) | + +Two inputs have **no** event on the bus today: a git snapshot write, and an Azure DevOps +pull request sync. They are named here so the absence is a recorded fact rather than a gap +a builder discovers and improvises around — this card adds no new event type for them, and +their latency is one sweep interval. + +A trigger, where wired, runs the same evaluation as the sweep, asynchronously, and never +inside the caller's transaction: a telemetry writer must not be able to fail or slow a +session transition or a provider sync. A trigger that is dropped costs latency and nothing +else. + +Every evaluation reads **all** input sources for the pair. There is no partial +evaluator: if any input query fails, the evaluation is abandoned under +**Failure modes** rather than classifying from what it managed to read. + +### Sweep selection predicate + +**The freeze is evaluated first and overrides all three conditions.** A pair whose +repository has a non-`NULL` `repositories.deleted_at` is **never** due, whichever +condition below would otherwise select it. This precedence is stated explicitly because +two of the three conditions would otherwise contradict the freeze promised under +**Persistence guarantees**: a soft-deleted repository's pair with no ledger row would be +unconditionally due forever, and one with `reached_default_at IS NULL` would be selected +by stale refresh every 24 hours. Restoring the repository (clearing `deleted_at`) moves +`repositories.updated_at`, which is itself a due source, so a restored pair resumes +evaluation on the next sweep without special handling. + +A candidate pair is **due** when the freeze does not apply and any of the following three +conditions holds: + +- it has **no ledger row** — a pair with no row is unconditionally due, on every + sweep, until one is written. This is what makes rule 6 reachable: a pair with + no snapshot and no provider row has no input observation to compare against, + and would otherwise never be selected. It also means **the first sweeps after + activation evaluate every pre-existing pair**, once each, until each has a row. That is + intended and is not the historical backfill excluded under **Out of scope** — see the + reconciliation there; or +- its `last_evaluated_at` is older than its **most recent input observation**, + defined as the greatest of the following, ignoring sources with no rows: + +| Source | Column | +|---|---| +| Git snapshots for the pair's sessions | `MAX(task_session_git_snapshots.created_at)` | +| Sessions for the pair | `MAX(task_sessions.updated_at)` | +| Task/repository membership | `MAX(task_repositories.updated_at)` | +| **The task row itself** | **`tasks.updated_at`** | +| **The repository row itself** | **`repositories.updated_at`** | +| GitHub pull requests | `MAX(github_task_prs.updated_at)` | +| GitLab merge requests | `MAX(gitlab_task_mrs.updated_at)` | +| Azure DevOps pull requests | `MAX(azure_devops_task_prs.updated_at)` | + +`repositories.updated_at` is in this table for a specific reason and must not be dropped +as redundant with `task_repositories.updated_at`, which is a **different table** tracking +task/repository membership. `repositories` is where `default_branch` and `deleted_at` +live, so without this row: a **Degraded evaluation** row could never be promoted once +`default_branch` was populated — contradicting the promotion this spec promises and the +scenario that asserts it — and a soft-delete or restore would never re-evaluate the freeze +described under **Persistence guarantees**. + +`tasks.updated_at` is in this table so a task moved between workspaces re-evaluates and +refreshes the denormalized `workspace_id` (see **Data model**). It re-selects pairs more +often than the other sources do, which is accepted: evaluation is idempotent and its +inputs are indexed reads, so a redundant evaluation costs a little work and changes +nothing. The alternative — leaving `workspace_id` to drift — silently mis-scopes the +extract that the column exists to serve. + +Third condition: + +- **Stale refresh.** A pair whose `reached_default_at` is `NULL` is due when + `last_evaluated_at < (sweep pass start instant − 24 hours)`, even when no input moved. + The comparison is strict, and the right-hand side is computed once per pass from the + pass's own start instant rather than per pair, so every pair in one pass is tested + against the same boundary and a long pass cannot let a pair drift past it mid-sweep. + + This exists because the two conditions above are both driven by *Kandev's own tables*, + and the central question this feature asks — did this work reach the default branch? — + can be answered by evidence that never touches them. Two concrete cases, both ordinary: + a branch that merges weeks after its task's last session ends moves no row in any table + above, and a provider-cloned repository whose local checkout did not exist when ancestry + was first attempted moves no row when it is later materialized. Under the first two + conditions alone, neither pair is ever re-examined, and its `reached_default_at` stays + `NULL` forever — which would defeat the card for exactly the finished-card population + the ledger exists to classify. + + It is bounded on both sides. It applies **only** while `reached_default_at` is `NULL`, + so a pair that has been observed reaching the default branch is never re-swept by this + condition — the column is write-once, so there would be nothing to gain. And at 24 hours + it is far longer than the 5-minute sweep interval, so it adds at most one evaluation per + unresolved pair per day and cannot interfere with the "not selected again on the next + sweep" behaviour asserted in **Scenarios**. + + This condition also subsumes ancestry retry, which is why no separate retry state is + persisted: an ancestry check that errored left `reached_default_at` at `NULL`, so the + pair is refreshed a day later and the check is attempted again. **Failure modes** relies + on this — an ancestry error is explicitly not an abandonment, so `last_evaluated_at` + advances and only this condition brings the pair back. + +### When the ledger itself cannot be read + +All three due conditions read `task_delivery_ledger`: condition 1 asks whether a row exists, +conditions 2 and 3 compare `last_evaluated_at` and test `reached_default_at`. So dueness +cannot be computed at all when the ledger table is missing or unreadable — which is +precisely the state a swallowed migration leaves behind. Left unstated, the natural +implementation is a single query joining candidates to the ledger, that query errors, the +pass selects nothing, **no upsert is ever attempted, and +`delivery_ledger_write_errors_total` stays at `0`** — silencing the counter this document +designates as the primary migration-failure signal, and making the missing-table scenario +under **Scenarios** unreachable. That is the same defect as the withdrawn "the stall +detector fires" claim, one signal over, so it is closed here rather than left to be +discovered. + +The contract is therefore a two-stage selection with a defined fallback: + +1. **Candidacy is computed without touching the ledger.** The candidate set comes from + `task_repositories`, `task_sessions`, the three provider tables, `tasks` and + `repositories` — all of which are **Candidate pairs** inputs. This stage never reads + `task_delivery_ledger`, so a missing ledger cannot suppress it. +2. **Dueness is then computed by reading the ledger rows for those candidates.** If that + read **succeeds**, the three due conditions apply exactly as written. If it **fails** — + the table does not exist, or the read errors for any other reason — the pass **treats + every candidate as due**, which is the same answer condition 1 gives for a pair with no + ledger row, and proceeds to evaluate and upsert each one. + +The fallback is what makes the designated signal true: each of those upserts then fails and +increments `delivery_ledger_write_errors_total`, and `delivery_ledger.write_failed` is +logged once for the pass, exactly as **Failure modes** promises. The sweep additionally +logs once per pass at WARN with the message `delivery_ledger.dueness_unavailable`, carrying +the error and the candidate count, so the fallback is distinguishable from a genuine +first-sweep-after-activation pass in which every pair really is new. + +Three properties bound it, and none of them is optional: + +- **A failed dueness read is not an input-query failure.** It does not abandon evaluations + under **Failure modes** and does not increment + `delivery_ledger_evaluation_errors_total`. The ledger is this feature's *output*, not one + of the inputs a classification is computed from, and conflating the two would abandon + every pair and re-silence the counter by a different route. +- **The fallback is idempotent and self-limiting.** When the read failure is transient and + the table does exist, the pass performs one redundant round of evaluations whose upserts + succeed; monotonic evidence and the high-water rules make each of those a no-op in value, + so the only cost is work. The next pass reads dueness normally. +- **The freeze and the candidacy conditions still apply.** The fallback widens *dueness* + only. A soft-deleted repository's pairs are still never due (see the freeze above), and a + pair failing conditions 2 or 3 of **Candidate pairs** is still excluded and counted rather + than attempted — so the fallback cannot turn an unresolvable pair into a write attempt. + +**Which instant `last_evaluated_at` records.** The value written is the UTC instant at +which the evaluation **began reading its inputs**. It is *not* the instant the upsert +committed, and this is not a stylistic preference — the natural implementation is wrong +and fails silently. + +Take an evaluation that reads its inputs at T0, and suppose an input row for the same pair +changes at T1 while the evaluation is still running, with the upsert committing at T2. +Writing `last_evaluated_at = T2` records that the pair was evaluated *after* the T1 change +when in fact it was evaluated before it. The dueness test is +`last_evaluated_at < MAX(input.updated_at)`, so T1 < T2 means the pair is **not** due and +the T1 change is masked permanently. Writing `last_evaluated_at = T0` leaves T1 > T0, so +the pair stays due and the next sweep sees the change. + +This is a lost update, and the case where it bites is the ordinary one rather than a rare +race: the last snapshot a finishing session writes is frequently the last input movement +that pair will ever have, so a masked T1 leaves the pair pinned at whatever the evaluation +computed one moment too early — for example `no_delivery_observed` on a pair whose final +snapshot reported `ahead > 0`. Event triggers cannot rescue it, because they are +explicitly optional accelerators and an implementation that ships only the sweep is +complete against this spec. + +The high-water rule under **Monotonic evidence** still applies on top of this: the stored +`last_evaluated_at` never moves backwards, so a slow evaluation that began before a fast +one cannot rewind the column. + +The sweep evaluates every due pair in the order given under **Ordering**. It has +no result cap: a pair is never silently skipped. If a sweep is interrupted, the +next sweep resumes from the same ordering and re-evaluates whatever it did not +reach; because evaluation is idempotent, partial completion is safe. + +Boundary values, all fixed by this spec so they are not invented downstream: + +| Value | Setting | +|---|---| +| Sweep interval | 5 minutes, **completion-relative**: the next pass starts 5 minutes after the previous pass *finished*, not 5 minutes after it started | +| First sweep | runs at boot, immediately after the schema probe under **Activation points**, with no initial delay | +| Concurrent sweep passes | **never**. A single sweep actor runs at most one pass at a time; a pass that overruns the interval delays the next one rather than overlapping it | +| Refresh interval (stale-refresh due condition) | 24 hours | +| Ancestry call timeout | 10 seconds per `(task, repository)` | +| Ancestry calls per evaluation | at most 1 per pair | +| Ancestry admission class | `subproc.GitBackground` | +| `evaluation_seq` initial value | 1 on insert | +| Stall threshold for the data-side health signal | 15 minutes, i.e. three missed sweeps | +| Sweep result cap | none | + +The three scheduling rows are fixed here rather than left to the implementation because +each one changes observable behaviour. **Completion-relative** with **no overlap** is what +makes the per-pass gauges (`delivery_ledger_sessions_unattributed_total`, +`delivery_ledger_pairs_missing_repository_total`) well defined — "set at the end of every +pass" has no single meaning if two passes can be in flight, and two overlapping passes +would race on the same rows, making `evaluation_seq` advance twice for one round of inputs +and letting the write-once `reached_default_*` triple be decided by scheduling. A +fixed-rate timer would allow exactly that whenever a pass ran long. **First sweep at boot** +matters because the alternative — waiting one interval — leaves a freshly migrated +database with an empty ledger for five minutes while publishing an activation instant that +says the mechanism is live. + +A pair whose task still has running sessions is evaluated normally. Its outcome +may be promoted by any later evaluation; there is no "wait until the task is +finished" gate, because the rank order makes early classification safe. + +## API surface + +No new HTTP route, WebSocket message, or MCP tool, and no new frontend state or +user-visible copy. The ledger is a telemetry table read by the writer-health +signals below and by the external extract described under **Out of scope**. +Existing task, session, and pull-request responses are unchanged. + +Slice A changes two existing day-grain dashboard response shapes additively: +`AgentRunActivityDay` gains `skipped` and `unclassified`, and `AgentSuccessRateDay` gains +`unclassified`. No field is removed or renamed, no run-grain DTO changes, and no new route +is added. That contract is specified under **Office run outcome** § Response shapes. +The values those shapes carry do change for pre-activation days, which is a behavioural +change to an already-rendered chart even though no frontend file changes; the new +`unclassified` field is what makes that change legible in-band. + +## Persistence guarantees + +- The ledger survives backend restarts. `last_evaluated_at` in the table is the + authoritative writer-liveness record; no in-process state is required to detect + a stalled writer. +- `reached_default_at` and `first_classified_at` are write-once for the life of + the row. +- Deleting a task deletes its ledger rows by cascade. Archiving a task does not: + archived tasks continue to be evaluated. +- Soft-deleting a repository (`repositories.deleted_at`) freezes evaluation for + its pairs. Existing rows are retained and `last_evaluated_at` stops advancing. +- Migrations follow ADR 0027: nullable `ADD COLUMN` then idempotent backfill, + replay-safe on SQLite and Postgres, classified through + `db.IsDuplicateColumnError` / `db.IsAlreadyExistsError` rather than a local + string match. + +## Ordering, idempotency, concurrency + +**Ordering.** A sweep evaluates due pairs ordered by `tasks.created_at` +ascending, then `tasks.id` ascending, then `repositories.id` ascending. + +Within a pair, evidence is selected deterministically without relying on a random +identifier or a dialect-specific row number: + +- `pr_merge` ref: among the pair's merged, non-detached provider rows, the one + with the earliest **merge instant** (per **Provider predicates**: `merged_at` + for GitHub and GitLab, `updated_at` for Azure DevOps); ties broken by + **request number** ascending (`pr_number` / `mr_iid` / `pull_request_id`); + further ties by provider name ascending in the order `azure_devops`, `github`, + `gitlab`; and further ties by that provider's **scope column** ascending — + `azure_devops_task_prs.azure_repository_id`, `gitlab_task_mrs.project_path`, and + `github_task_prs.owner` then `github_task_prs.repo`. The ref written is that row's + **URL** column. + + The scope column is not decoration, but it is load-bearing for **two of the three + providers, not all three**, and the difference is stated exactly because the uniqueness + keys are quoted here and a reader can check them: + - `gitlab_task_mrs` is `UNIQUE(task_id, repository_id, project_path, mr_iid)` and + `azure_devops_task_prs` is + `UNIQUE(task_id, repository_id, azure_repository_id, pull_request_id)`. Both keys + *include* the scope column, so one pair can legitimately hold two rows from the same + provider with the same request number and different URLs. For these two the scope + tiebreak is what makes the selection deterministic, and dropping it would leave exactly + the ambiguity this section exists to rule out. + - `github_task_prs` is `UNIQUE(task_id, repository_id, pr_number)`, which does **not** + include `owner` or `repo`. That constraint already makes `(pair, pr_number)` unique for + GitHub, so the collision the scope key resolves cannot occur there and the + `owner`-then-`repo` tiebreak is **redundant for GitHub**. It is retained anyway so the + ordering is total by the same construction for every provider, which keeps the rule + checkable by inspection. + + **No schema change is implied.** The GitHub key is correct as it stands; the redundancy is + in the tiebreak, not in the constraint. Do not "fix" the uniqueness key to match the other + two — widening it would let a pair hold two GitHub rows with the same `pr_number`, which + the current constraint deliberately prevents. +- `direct_commit` ref: restricted to the snapshots of the **session(s) that satisfied rule + 2's own predicate** — sessions holding two or more distinct non-empty `head_commit` + values, among their own snapshots, whose `branch` equals the repository's + `default_branch` — and, within that set, among the pair's default-branch snapshots + **whose `head_commit` is non-empty** after the normalization in **Classification**, those + sharing the greatest `created_at`, and of those the lexicographically greatest + `head_commit`. The non-empty filter is applied **before** the `created_at` comparison, not + after, so a newer empty-headed snapshot cannot displace a real commit SHA and cannot be + written into `delivery_ref` — or, through the `default_branch_commit` basis, into the + write-once `reached_default_ref`. Rule 2 requires two or more distinct non-empty heads on + the default branch for this basis to be reachable at all, so the restricted set is never + empty here. + + **Why the session restriction, stated because the same defect appears twice in this + document.** This is the `direct_commit` twin of the restriction argued at length under + **Default-branch observation**, "Which commit", and it exists for the same reason. Rule 2 + is earned **per session**, but an earlier revision selected the ref pair-wide, so a second + session merely *sitting* on the default branch could supply a newer snapshot whose head is + its inherited tip. That head would then be written as `delivery_ref` — and, because + `default_branch_commit` is basis precedence 2, into the **write-once** `reached_default_ref` + as well — permanently naming a commit that the session which actually earned the + classification did not author. Restricting the candidate set to the session(s) that + satisfied rule 2 is what keeps the ref attributable to the work that produced it. +- **Every other rule writes `delivery_ref` as `NULL`.** Rules 3, 4, 5 and 6 — + `branch_commits_unmerged`, `reached_default_unattributed`, `no_commits_observed`, + `provider_pr_unmerged`, `no_observations` — and any **Degraded evaluation** row identify + no single artifact that carries the delivery, so there is nothing honest to name. In + particular rule 5 does **not** write the unmerged pull request's URL: that row is + evidence the pair did *not* deliver, and putting it in a column defined as "the + identifying evidence" for a delivery would invite a consumer to read it as one. The + head commit is likewise not written, because a branch head is not a delivery. + + This completes the ref rules over all six classification rules, which is what makes the + equal-rank no-op claim under **Monotonic evidence** true for every rule rather than only + for the two that name a ref. +- `observed_branch_commits` incoming value: the greatest `ahead` across the pair's + snapshots after the `NULL`/negative normalization in **Classification**, which needs no + tiebreak. **When the pair has no snapshots at all the incoming value is `NULL`, not + `0`** — the greatest of an empty set is not zero, and `0` would assert "we looked and + there were no commits" for a pair where nothing was ever observed, which is the exact + `NULL`-versus-`0` confusion this feature exists to prevent. Combined with the high-water + rule, a `NULL` incoming value can never lower a stored one: a pair that had snapshots + and then lost them keeps its high-water mark. + +**Idempotency.** Evaluation is a pure function of the observed inputs. Re-running +an evaluation whose inputs have not changed must leave `delivery_outcome`, +`delivery_basis`, `delivery_ref`, `evidence_rank`, `observed_branch_commits`, +`reached_default_at`, `reached_default_basis`, `reached_default_ref` and +`first_classified_at` byte-identical and must not advance `updated_at`. Only +`last_evaluated_at` and `evaluation_seq` advance. + +**Monotonic evidence.** The classification columns move only up the **Evidence +rank** ladder. Comparing the incoming rank to the stored rank gives exactly three +cases, and every column's behaviour is fixed in all three: + +| | incoming rank **>** stored | incoming rank **=** stored | incoming rank **<** stored | +|---|---|---|---| +| `delivery_outcome`, `delivery_basis`, `delivery_ref`, `evidence_rank` | written | written | **not** written | +| `first_classified_at` | written if currently `NULL` | written if currently `NULL` | not written | +| `observed_branch_commits` | high-water (below) | high-water | high-water | +| `reached_default_at` / `_basis` / `_ref` | written **only** if `reached_default_at` is currently `NULL`, as one unit | same | same | +| `last_evaluated_at` | high-water (below) | high-water | high-water | +| `evaluation_seq` | `stored + 1` | `stored + 1` | `stored + 1` | +| `workspace_id` | written | written | written | +| `delivery_ledger_demotions_suppressed_total` | — | — | incremented | +| `delivery_ledger_degraded_outcome_changed_total` | — | incremented **only at rank 1** when `delivery_outcome` changed | — | + +`workspace_id` is written unconditionally in all three columns of that table, including +the demotion case. It is not evidence about delivery and the rank guard has no business +gating it: a pair that moved workspace has moved workspace regardless of what its +classification did. + +**Equal-rank writes, stated separately for ranks 2–8 and for rank 1.** The two cases are +genuinely different and collapsing them is what made an earlier revision of this section +wrong. + +*At ranks 2–8.* Equal rank implies the same `(outcome, basis)`, because the rank is +injective there (**Evidence rank**). So `delivery_outcome` and `delivery_basis` are always +a no-op in value. `delivery_ref` is **not** guaranteed unchanged: the ref names the +currently-selected evidence, and the evidence set can move while the rank does not — a +pair already at rank 8 that acquires a second merged pull request with an earlier merge +instant re-selects to that row's URL. That is intended, not a defect: the ref must name +evidence that exists. When the ref changes, `updated_at` advances and +`delivery_ledger_rows_written_total` increments; when nothing changed, neither does. + +*At rank 1.* Equal rank does **not** imply the same outcome, because rank 1 is +deliberately non-injective. The classification columns are still written, and the outcome +therefore tracks the current inputs — which is the correct behaviour for a degraded row, +since it is a blind placeholder and pinning it to a stale outcome would be worse than +letting it move. But this write can change `delivery_outcome` without any demotion being +recorded, so it is **counted rather than left silent**: an equal-rank write at rank 1 that +changes `delivery_outcome` increments +`delivery_ledger_degraded_outcome_changed_total` (see **Writer health**). + +The concurrency consequence at rank 1 is stated honestly rather than papered over: two +evaluators racing at rank 1 with different outcomes are last-writer-wins, so arrival order +decides which outcome is stored. This is bounded and self-repairing — the next sweep +recomputes from the same inputs and converges, and the whole row is superseded by any +rank ≥ 2 evaluation once `default_branch` is populated. The "converge regardless of +arrival order" guarantee below therefore holds for ranks 2–8 and is explicitly **not** +claimed for rank 1. + +**High-water columns, stated as behaviour and not as a SQL function.** +`observed_branch_commits` and `last_evaluated_at` take the greater of the stored and +incoming values, unconditionally, in all three rank cases, with a **`NULL` stored value +treated as lower than any non-`NULL` incoming value**. `observed_branch_commits` +therefore never decreases when a session — and with it, by `ON DELETE CASCADE`, its +snapshots — is deleted. It is a stored high-water mark and **not** a recomputed +aggregate; this is why rule 3's predicate reads the freshly observed `ahead` values +rather than this column. + +That behaviour is the contract. The obvious SQL spelling does **not** implement it, and +the difference is silent, so it is called out rather than left to be discovered: + +- SQLite's two-argument `max(X, Y)` returns `NULL` if *either* argument is `NULL`, so + `max(observed_branch_commits, excluded.observed_branch_commits)` erases a real value the + first time either side is `NULL` — the opposite of a high-water mark. +- PostgreSQL has no two-argument `max` at all; the equivalent is `GREATEST`, whose + `NULL` handling differs again. + +So the implementation must make the `NULL` handling explicit — coalescing to a floor +before comparing, and using each dialect's own spelling — rather than relying on a bare +`MAX`. This is the one place where **Concurrency**'s "never by a read-then-write in +application code" needs qualifying: the rule stays, and the `NULL` handling belongs +*inside* the statement, not in a Go read-modify-write. + +The same trap applies to the rank guard itself. `x >= NULL` evaluates to `NULL` on both +dialects and a `CASE` on it takes the `ELSE` branch, so a nullable `evidence_rank` would +silently suppress every write against a row that had one. That is why `evidence_rank` is +declared `NOT NULL DEFAULT 0` in the data model. + +Both dialects are in scope for this statement. ADR 0027 governs the migrations, and the +exemption recorded under **Office run outcome** covers only the *pre-existing* +SQLite-only `RunCountsByDayForAgent`; new code added by this card is expected to run on +SQLite and PostgreSQL alike. + +A demotion is never written. If a pull request is detached after it merged, or a +snapshot is deleted, the row keeps its stronger evidence and the suppression is +counted. This is what makes the ledger safe: an evaluation that runs after the +evidence has been removed can never erase what an evaluation that could see it +recorded. + +**Concurrency.** The ledger is written with a single +`INSERT ... ON CONFLICT (task_id, repository_id) DO UPDATE` statement. Every rule +in the table above is expressed inside that statement's `SET` clause — as +`CASE WHEN excluded.evidence_rank >= task_delivery_ledger.evidence_rank THEN ... END` +for the classification columns (sound precisely because `evidence_rank` is `NOT NULL`), +`CASE WHEN task_delivery_ledger.reached_default_at +IS NULL THEN ... END` for the write-once triple, and the explicitly `NULL`-handled, +dialect-appropriate greatest-of expression described under **Monotonic evidence** for the +high-water columns — never by a read-then-write in application code. Consequences, all of +which are contract: + +- Two evaluators racing on the same pair with **different** ranks — say one computing + rank 1 and one rank 8 — converge on rank 8 regardless of arrival order, and neither can + lose the other's write. This is the rank guard doing its job and it holds for every + unequal pair of ranks. +- Two evaluators racing at **equal rank 1** with different outcomes are the one exception + and are **not** order-independent: rank 1 is deliberately non-injective, so the + last writer's outcome is stored. This is bounded and self-repairing — see + **Degraded evaluation** and "Equal-rank writes" above. It is called out here so the + bullet above is not read as a guarantee it does not make. +- Two evaluators supplying a **first** default-branch observation concurrently: + whichever statement commits first writes all three of `reached_default_at`, + `reached_default_basis` and `reached_default_ref`; the second sees a non-`NULL` + `reached_default_at` and writes none of them. The triple is therefore always + internally consistent — it is never assembled from two different observations. +- `evaluation_seq` increments on every persisted evaluation, so a consumer can + tell that the row was re-evaluated even when nothing changed. + +Ancestry checks run through the existing class-aware git subprocess admission +pool at class `subproc.GitBackground` — the sweep is background work and must not +contend with user-facing git — with a bounded timeout, at most one per +`(task, repository)` per evaluation. + +## Failure modes + +- **An input query fails** — a snapshot, session, membership or provider read + returns an error. The evaluation for that pair is **abandoned**: no column is + written, not even `last_evaluated_at`, so the pair stays due and is retried on + the next trigger or sweep. `delivery_ledger_evaluation_errors_total` + increments. A classification is never persisted from a partially-read input + set. +- **The ledger upsert itself fails** — the `INSERT ... ON CONFLICT` returns an error: + the table does not exist because its migration was swallowed, a constraint is violated, + or the write times out. This is a **distinct** failure from an input-query failure and + must not be folded into it, because only this one indicates the ledger's own schema or + storage is broken. No column is written, so the pair stays due and is retried on the + next sweep. `delivery_ledger_write_errors_total` increments — this counter is the + primary migration-failure signal and the one a `/debug/vars` reader should watch. The + sweep logs once per pass at WARN with the message `delivery_ledger.write_failed`, + carrying the error and the count of pairs that failed in that pass; once per pass rather + than once per pair, so a missing table cannot emit one log line per candidate. A failing + upsert **does not abort the pass** — the sweep continues to the next due pair, so one + poisoned pair cannot stop the writer. +- **Ancestry cannot be computed** — the repository has no readable local + checkout, neither default-branch ref resolves, the head commit object is + absent, or the git call fails or times out. No evidence is produced: + `reached_default_at` stays `NULL` and the outcome is unaffected. + `delivery_ledger_ancestry_errors_total` increments. The error is never + persisted as a negative result, and it does not abandon the evaluation — the + rest of the classification still runs. +- **The ancestry precondition is not met** — fewer than two distinct non-empty + `head_commit` values. The check is skipped, + `delivery_ledger_ancestry_skipped_total` increments, and this is not an error. +- **A negative ancestry result** is not evidence. It must not set any column and + must not influence the outcome. Squash-merge makes it a routine false negative. +- **`repositories.default_branch` is empty** — handled by **Degraded + evaluation**: rules still run, `delivery_basis` is `default_branch_unknown`, + `evidence_rank` is 1. +- **A session has an empty or `NULL` `repository_id`** — the column is `TEXT DEFAULT ''` + and not `NOT NULL`, so both forms occur; both normalize to empty. The session + contributes to no pair, is excluded from the stall comparand, and is counted in + `delivery_ledger_sessions_unattributed_total`. Three such sessions exist in the + reference store. +- **`head_commit` is empty, whitespace-only, or `NULL`** — it is not a distinct head value + and is ignored. +- **`repositories.default_branch` is `NULL`** — the column is `TEXT DEFAULT ''` and not + `NOT NULL`. It normalizes to empty and takes the **Degraded evaluation** path, the same + as an empty string. There is no third behaviour. +- **Azure DevOps has no `merged_at`** — resolved by **Provider predicates**: + merged is read from `status`, and `updated_at` is the merge instant used for + ordering only. An unrecognised status is not-merged, so the pair falls through + to a later rule rather than being guessed at. +- **The same pull request is attached to more than one task** — this occurs in + the reference store (PR #2418, two tasks). Each task's pair is classified + independently and both may read `pr_merge`. The ledger does not deduplicate + credit across tasks. +- **A migration fails** — the migration runner logs at WARN and swallows the + error (`apps/backend/internal/db/migratelog.go`, `MigrateLogger.Apply`), so a + failed migration is invisible at boot. Three signals make it visible, and it is worth + being exact about which one does the work, because an earlier revision of this document + claimed "the stall detector fires" and that was **false**: + - `telemetry.delivery_ledger.activated_at` is **absent** from `kandev_meta`, because + the schema probe under **Activation points** found no table. This is the cheapest + check and the one a consumer should make first. + - `delivery_ledger_write_errors_total` climbs on every sweep pass and + `delivery_ledger.write_failed` is logged, per the upsert-failure mode above. This is + the signal that actually fires — **and it fires only because of the dueness fallback** + under **Sweep selection predicate**, "When the ledger itself cannot be read". Without + that fallback the missing table would break dueness computation first, the pass would + select nothing, and this counter would stay at `0`. The two statements are load-bearing + for each other: do not implement one without the other, or this bullet becomes as false + as the withdrawn stall-detector claim below. + - `delivery_ledger.dueness_unavailable` is logged once for the pass, which is what + distinguishes a broken ledger from a legitimate first sweep in which every candidate is + genuinely new. + - `delivery_ledger_last_evaluated_unix` publishes the unavailable sentinel `-1`, per + **Writer health**. + + The stall detector specifically does **not** fire, and relying on it would have been the + mistake: with the table missing there is nothing to take `MAX(last_evaluated_at)` over, + and the empty-table rule would report `delivery_ledger_stall_seconds = 0`, which reads + as healthy. That is why the stall signal is not the migration-failure signal and why + the sentinel above exists. + +## Writer health + +If this writer silently stops, nothing today would notice. Both signals below are +required, because they fail in different ways. + +**In-process counters**, following the `subproc_*` / `routing_*` precedent +(`apps/backend/internal/common/subproc/metrics.go`), published at package init +and exposed through the existing dev-mode `/debug/vars` handler: + +| Counter | Increments | +|---|---| +| `delivery_ledger_evaluations_total` | once per **persisted** evaluation, keyed by the `delivery_outcome` that evaluation computed. An abandoned evaluation persists nothing and is **not** counted here; it is counted only by `delivery_ledger_evaluation_errors_total`. A suppressed demotion **is** counted, keyed by the outcome the evaluation *computed*, not the one that stayed stored — otherwise a pair stuck in demotion looks idle. | +| `delivery_ledger_rows_written_total` | once per upsert that **changed at least one** classification or observation column, i.e. exactly when `updated_at` advances. Not once per evaluation: a re-evaluation with unchanged inputs advances only `last_evaluated_at` and `evaluation_seq`, and counting it would make the counter a duplicate of `evaluations_total`. | +| `delivery_ledger_demotions_suppressed_total` | once per upsert whose incoming rank was strictly lower than the stored rank. | +| `delivery_ledger_evaluation_errors_total` | once per abandoned evaluation (an input query failed). | +| `delivery_ledger_ancestry_errors_total` | once per ancestry check that was attempted and could not produce a result. | +| `delivery_ledger_ancestry_skipped_total` | once per evaluation where the ancestry precondition was not met. | +| `delivery_ledger_write_errors_total` | once per pair whose ledger upsert returned an error (see **Failure modes**). Kept separate from `evaluation_errors_total` because an input-query failure means the *inputs* are unreadable while this means the *ledger's own* schema or storage is broken — the migration-failure case. This is the counter that fires when a swallowed migration left the table missing. | +| `delivery_ledger_degraded_outcome_changed_total` | once per equal-rank write at rank 1 that changed `delivery_outcome` (see **Degraded evaluation**). Rank 1 is deliberately non-injective, so such a write is neither a promotion nor a suppressed demotion and would otherwise be invisible in every other counter here. | +| `delivery_ledger_pairs_missing_repository_total` | **a gauge, not a running total**, set per sweep pass for the same reason as `sessions_unattributed_total`: the count of candidate pairs the pass rejected because their `repository_id` matched no `repositories` row (see **Candidate pairs**). | +| `delivery_ledger_pairs_missing_task_total` | **a gauge, not a running total**, set per sweep pass: the count of candidate pairs the pass rejected because their `task_id` matched no `tasks` row (see **Candidate pairs**, condition 3). Kept separate from the repository gauge because the two have different causes — a missing repository points at repository lifecycle, a missing task at provider-row cleanup — and a combined number would identify neither. A non-zero resting value here is a data-hygiene fact about orphaned provider rows, **not** a ledger fault, and specifically not a migration failure. | +| `delivery_ledger_sessions_unattributed_total` | **a gauge, not a running total.** It is set — not incremented — at the end of every sweep pass to the count of distinct sessions with an empty `repository_id` that the pass encountered. It is the one value here whose natural reading is "how many are there", and incrementing per encounter would add the same three reference-store sessions on every 5-minute pass, turning a fact about the data into a number that only measures uptime. The `_total` suffix is retained for consistency with the `subproc_*` precedent. | + +Every counter is keyed and cadenced above so that two implementations produce the same +numbers from the same inputs. A counter whose meaning is left to the writer is not a +health signal. + +**The data-side stall signal.** Counters reset to zero on restart, so a writer +that stopped before the last restart looks healthy by counters alone. The stall +signal is computed from the database instead, which is what makes it survive a +restart: + +- **Actor and cadence:** the sweep, at the end of every pass, including a pass + that evaluated nothing. +- **What it computes:** `MAX(task_delivery_ledger.last_evaluated_at)` across the + whole table, and as the comparand, `MAX(task_sessions.updated_at)` across only those + sessions that **could have produced work for the writer**: + + | Filter | Column | Why | + |---|---|---| + | session attributes to a pair | `task_sessions.repository_id` non-empty after normalization | A session with no repository contributes to no pair, so it can never advance `last_evaluated_at`. Three such sessions exist in the reference store. | + | repository still live | the joined `repositories.deleted_at IS NULL` | Pairs of a soft-deleted repository have `last_evaluated_at` frozen **by design** under **Persistence guarantees**. | + + Each filter removes a session whose progress the writer is *supposed* to ignore. Without + them the comparand advances on work the ledger is correctly not doing, and the signal + reports a stall on a perfectly healthy writer — a false alarm, which is how a real one + gets ignored. + + **There is deliberately no archived filter**, and the earlier "non-archived sessions" + wording is withdrawn. It named no real column — `task_sessions` has no archived column + in its `CREATE TABLE` or in any migration, and archival lives on `tasks.archived_at` — + but more importantly it was wrong on the merits: **Persistence guarantees** states that + archived tasks continue to be evaluated, so their sessions are work the writer really is + expected to keep up with. Excluding them would have hidden a genuine stall on a board + whose recent activity is mostly archived. The comparand covers every session the writer + is responsible for, and nothing else. +- **What it publishes:** two expvar Ints alongside the counters, + `delivery_ledger_last_evaluated_unix` and `delivery_ledger_stall_seconds`. **Two + independent queries feed them** — the ledger query + (`MAX(task_delivery_ledger.last_evaluated_at)`) and the comparand query (the filtered + `MAX(task_sessions.updated_at)`) — and either can fail or return an empty result on its + own. The table below is total over both, in that order of precedence; states that look + identical without a sentinel are given distinct ones: + + | State | `..._last_evaluated_unix` | `..._stall_seconds` | Meaning | + |---|---|---|---| + | The **ledger** query errored (table missing or unreadable) | `-1` | `-1` | The ledger is **unavailable**. Distinct from healthy-and-empty, which is the whole point of the sentinel. Takes precedence over every row below: if the ledger cannot be read, the comparand's state is irrelevant. | + | The ledger query succeeded and the table is empty | `0` | `0` | No pair has been evaluated yet. Not a stall: an empty table is not an infinite stall. | + | Ledger rows exist and the **comparand** query errored | `MAX(last_evaluated_at)` | `-1` | The stall figure is **unknown**, not zero. The last-evaluated value is real and is still published, because it was read successfully; only the derived figure is unavailable. | + | Ledger rows exist and the comparand set is **empty** (no session survives the filters) | `MAX(last_evaluated_at)` | `0` | Not a stall. There is no work the writer is responsible for keeping up with, so it cannot be behind. This is the ordinary state of a fresh database whose pairs come from `task_repositories` and whose sessions have not run yet. | + | Ledger rows exist and the comparand is present | `MAX(last_evaluated_at)` as Unix seconds | comparand minus it, in seconds, clamped at `0` | The real values. | + + The comparand-empty and comparand-error rows are separated deliberately: an empty + comparand is a true statement about the data and must read healthy, while a failed + comparand read is an absence of information and must not be able to masquerade as + healthy. Publishing `0` for both would let a repeatedly failing comparand query report a + permanently healthy writer. + + When the **ledger** query errors the sweep logs once per pass at WARN with the message + `delivery_ledger.unavailable`; when the **comparand** query errors it logs once per pass + at WARN with `delivery_ledger.comparand_unavailable`. Without the `-1` sentinels a missing + table would publish `0`/`0`, which is the healthy-and-empty state — a broken ledger would + be indistinguishable from a fresh one, which is exactly the inversion this section exists + to prevent. + + A consumer reading `delivery_ledger_stall_seconds` therefore treats `-1` as "no + measurement", never as a magnitude, and never compares it against the 15-minute + threshold. + + Because all of these are recomputed from the database on every pass, the first sweep + after a restart republishes the true values. +- **What "fires" means:** when `delivery_ledger_stall_seconds` exceeds the + 15-minute threshold, the sweep logs once per pass at WARN with the message + `delivery_ledger.stalled`, carrying both values as fields. + +**What these signals do NOT cover, stated as an accepted residual rather than left to be +discovered.** Every signal in this section is published *by the sweep being monitored*. A +sweep goroutine that panics on its first pass, or is never started, publishes nothing at +all — so the counters keep whatever value they last held (typically `0`, which reads as +healthy) and no WARN is ever emitted. This is the literal "the writer silently stops" case +this section opens with, and no in-process signal can close it: a process cannot report its +own death. + +It is accepted rather than solved, for a specific reason that makes the residual small. +The authoritative liveness record is `task_delivery_ledger.last_evaluated_at`, which lives +**in the database**, not in the process. Any consumer — the external extract above all, +which is the reason this table exists — can compute +`now() - MAX(last_evaluated_at)` for itself without the sweep participating at all, and +that computation is correct whether the sweep is running, wedged, or was never started. +The in-process signals are a convenience for a live `/debug/vars` reader; the data-side +record is the ground truth, and this spec's obligation is to keep it accurate. No +acceptance criterion asserts the in-process signals detect a dead sweep, because they +cannot. + +## Office run outcome + +A second, delimited contract in the same card, because it is the run-grain form +of the same defect: a terminal status is not an outcome. + +**Call sites.** Every call site that finishes a run records the outcome that +describes it. The semantic label in the right-hand column is authoritative; the +line number is a pointer (see **Citation convention**). + +| Call site | Outcome | +|---|---| +| `office/service/scheduler_integration.go:218` — agent not active | `agent_inactive` | +| `office/service/scheduler_integration.go:247` — idle skip | `idle_skipped` | +| `office/service/scheduler_integration.go:517` — task-tree hold | `task_tree_held` | +| `office/service/scheduler_integration.go:829` — pre-execution budget block | `budget_blocked` | +| `office/service/event_subscribers.go:408` — agent-completed for a task-bearing run | `processed` | +| `office/service/event_subscribers.go:512` — `handleTasklessAgentCompleted` | `processed` | + +Checkout errors and checkout contention do not finish a run. A checkout error +uses the normal retry and failure path. A contention result requeues the run +and retries it. Neither path reports work that did not run. + +**Reachability of `:512`.** `handleTasklessAgentCompleted` resolves its run through +`GetClaimedTasklessRunForAgent`, which requires `status = 'claimed'` and an empty +`payload.task_id`. Taskless runs fail before they can remain claimed, so the lookup +normally returns `sql.ErrNoRows` and the site is a no-op. It is mapped to `processed` +because an `AgentCompleted` event may still arrive for a legacy claimed run, and it is +retained so the mapping is total over the call sites that exist. + +`:408` is the path where an agent actually ran to completion and is the ordinary source +of `processed`. Leaving any terminal site unset would grow `unclassified` forever after +activation and falsify the guarantee that the activation point tells a consumer when +`unclassified` stops growing. + +**Concurrency, retry and error behaviour for this column.** Slice A does not introduce a +transaction or a guard, because it must not change `runs.status` semantics: + +- `outcome` is written **in the same `UPDATE` statement as `status`**, so a run can never + hold a terminal status with a stale outcome from a different transition. +- **Where that statement actually lives, stated precisely, because the obvious answer is + wrong.** `transitionRunTerminal` (`office/service/scheduler_runs.go:52`) contains **no** + `UPDATE`. It pre-fetches the run, calls `s.repo.FinishRun(ctx, id, status)`, and + publishes `OfficeRunProcessed`. The single `UPDATE runs SET status = ?, finished_at = ? + WHERE id = ?` lives one layer down, in + `internal/runs/repository/sqlite`, `func (r *Repository) FinishRun(ctx context.Context, + id, status string) error` — which is the only implementation of that method in the + repository. Per **Citation convention** the semantic label is authoritative: the statement + to extend is *the one inside the runs repository's `FinishRun`*, not a statement inside + `transitionRunTerminal`, which does not exist. This distinction is not pedantic, because + the statement's real home has a caller the service layer does not. +- **`outcome` becomes a parameter of both layers.** The runs repository's `FinishRun` gains + an outcome parameter alongside `status` and writes both in its one `UPDATE`; + `transitionRunTerminal` gains the same parameter and forwards it. `FinishRun` + (`scheduler_runs.go:38`) takes the value from the call-site table above and `FailRun` + (`:44`) passes `NULL`. `NULL` is correct rather than a placeholder — a run that reached + `status = 'failed'` is bucketed by `RunCountsByDayForAgent` on its status alone and never + reaches the `succeeded` / `skipped` / `unclassified` buckets, so no value from the + five-value vocabulary would ever be read, and inventing one would assert a classification + nothing consumes. This also makes the sentence under `### runs.outcome` exact: the + five-value claim describes the **finished** path; the failed path writes `NULL` in the + same statement. +- **The second caller pair of the shared repository method, and what it passes.** Changing + that signature necessarily reaches + `office/scheduler.SchedulerService.FinishRun` and `.FailRun` + (`internal/office/scheduler/run_processing.go`), which call the same + `repo.FinishRun` **without** going through `transitionRunTerminal` or the six-site table. + Both **pass `NULL`**, and neither is added to the call-site table. The reasoning, so a + builder does not have to guess and does not "improve" it into a vocabulary value: + - `SchedulerService.FailRun` writes `RunStatusFailed`. It is reached in production (via + that package's own retry path), and `NULL` is the same value `FailRun` passes at the + service layer, for the same reason — a failed run is bucketed on status alone. + - `SchedulerService.FinishRun` writes `RunStatusFinished` and has **no production caller + at the time of writing**. It is a dormant parallel copy of the service-layer method. It + passes `NULL` because there is no outcome to describe: no call site means no semantic + label, and inventing `processed` for it would be the exact defect this slice exists to + remove — a run in which nothing is known to have happened counted as a success. `NULL` + routes it to `unclassified`, which is the honest bucket for a path whose meaning is + unestablished. + - Consequently the activation guarantee below is scoped to the six sites and is **not** + weakened by these two: neither is a terminal site this card classifies, and if + `SchedulerService.FinishRun` is ever wired to something real, the card that wires it + owns choosing its outcome and adding it to the table. Deleting the dormant method is + **not** in scope here (see `## Out of scope`). +- That statement is an unconditional `UPDATE runs SET status = ?, outcome = ?, + finished_at = ? WHERE id = ?` with **no status guard**, matching today's behaviour. + If two terminal callers ever reach the same row, **last writer wins** for status and + outcome together. This is the existing contract for `status`, extended unchanged. +- A zero-row update (the run was deleted) is **not** an error and is not retried. +- Four of the six sites discard `FinishRun`'s error (`_ = si.svc.FinishRun(...)`). That + is unchanged by this card. A discarded failure leaves the row non-terminal with + `outcome` `NULL`; if it later reaches `finished` by another path it is bucketed by + whatever that path wrote, and if it never does it is not `finished` and so never + reaches the `succeeded` / `skipped` / `unclassified` buckets at all. +- Consequently the activation guarantee is stated precisely: after activation, + `unclassified` stops growing **for runs that reach `finished` through one of the six + sites above**. It is not a claim that no `finished` row can ever carry a `NULL` + outcome, because this card does not add error handling to paths that discard errors + today. + +`runs.status` is unchanged. Nothing that reads `status` today changes behaviour. + +**Repository query.** `RunCountsByDayForAgent` +(`office/repository/sqlite/agent_summary.go`) reports **five** buckets where it +reports three today. The bucketing is total over every possible column value: + +| Bucket | Predicate | +|---|---| +| `succeeded` | `status = 'finished'` **and** `outcome = 'processed'` | +| `skipped` | `status = 'finished'` **and** `outcome` is any other non-`NULL` value | +| `unclassified` | `status = 'finished'` **and** `outcome IS NULL` | +| `failed` | `status IN ('failed','timed_out')` (unchanged) | +| `other` | every remaining status (unchanged) | + +**Response shapes.** Two shapes in `office/dashboard/agent_summary.go` derive +from those rows, and both change additively: + +- `AgentRunActivityDay` (built by `padAgentRunActivity`) gains `skipped` and + `unclassified` JSON fields. Its documented invariant becomes + `total = succeeded + skipped + unclassified + failed + other`, so `total` is + unchanged in value for any given day. +- `AgentSuccessRateDay` (built by `buildSuccessRate`) gains **one** JSON field, + `unclassified`. `succeeded` now counts processed runs only, while `total` remains the + same full denominator, so the success rate falls to the truth rather than being inflated + by non-work. + + **Why it gains a field rather than keeping its two.** Without one, this shape changes + meaning mid-series with no in-band marker, which is the exact discontinuity the + `NULL`-not-`0` rule in `## What` exists to prevent — and it is worse here than elsewhere, + because the consequence is not a subtle miscount. `succeeded` is today + `status = 'finished'` and becomes `status = 'finished' AND outcome = 'processed'`, while + every run finished before activation carries `outcome IS NULL`. So **every pre-activation + day reports `succeeded = 0` against an unchanged denominator** — the Success Rate chart + reads 0% across all of history the moment this ships. That is not wrong data (those runs + genuinely have no recorded outcome), but presented as two fields it is indistinguishable + from a real collapse in agent success, and a reader has no way to tell without going + outside the response to `kandev_meta`. + + With `unclassified` present the discontinuity is legible in the response itself: a day + where `unclassified` accounts for the whole non-`failed`, non-`other` remainder is a + pre-activation day, and a day where it is `0` is fully classified. This is the same + reasoning, and the same additive treatment, already applied to `AgentRunActivityDay`; the + earlier revision that left this shape at two fields was inconsistent with its own + principle rather than deliberately excepting it. + + `succeeded + unclassified` is deliberately **not** offered as a compatibility value and no + consumer should compute it: it would resurrect exactly the finished-as-success count this + slice removes, because `unclassified` contains pre-activation budget-blocked and + idle-skipped runs alongside genuine work. + +The `Succeeded` field in `office/repository/sqlite/dashboard.go` is sourced from +`task_sessions.state`, not from `runs.status`, and is **not** changed by this +spec. + +No new user-visible copy is introduced: the existing charts keep rendering the +fields they already render, and the two new fields are additive on the wire. +Consequently no `t()` string, no locale catalog entry, and no Playwright spec is +added by this card. A later card that surfaces `skipped` or `unclassified` in the +UI owns that copy and that coverage. + +`RunCountsByDayForAgent` is pre-existing SQLite-only code (it uses `strftime`). +This spec does not make it dialect-portable; the Postgres parity requirement +under **Migration and activation** binds the migrations, which is what ADR 0027 +governs. + +**Legacy rows** land in `unclassified`, not `succeeded`. They are not +reinterpreted and not backfilled: the pre-activation series keeps its own bucket, +so the discontinuity is visible in the data rather than silent. The activation +point tells a consumer when `unclassified` should stop growing. + +"Visible in the data" is meant literally and is satisfied at **both** grains, which is why +`AgentSuccessRateDay` gains `unclassified` above. A consumer reading either response shape +can separate "this day had no successes" from "this day predates the mechanism" without +consulting `kandev_meta` — the activation instant remains the authoritative answer for +*when* the change happened, but no shape requires it to avoid a false reading. + +## Scenarios + +### Classification + +- **GIVEN** a task whose repository has a non-detached merged pull request, + **WHEN** the ledger is evaluated, **THEN** the pair records + `delivery_outcome = pr_merge`, `delivery_basis = provider_pr_merged`, + `delivery_ref` set to that pull request's URL, and `evidence_rank = 8`. +- **GIVEN** a task whose repository is tracked on GitLab and whose merge request + has a non-`NULL` `merged_at`, **WHEN** the ledger is evaluated, **THEN** the + pair records `pr_merge` on the same basis as a GitHub pull request. +- **GIVEN** a task whose only Azure DevOps pull request has + `status = 'Completed'`, **WHEN** the ledger is evaluated, **THEN** the pair + records `pr_merge` — the status match is case-insensitive and whitespace-trimmed. +- **GIVEN** a task whose only Azure DevOps pull request has + `status = 'abandoned'`, and no snapshots, **WHEN** the ledger is evaluated, + **THEN** the pair records `unknown` with `delivery_basis = provider_pr_unmerged` + and `evidence_rank = 3`. +- **GIVEN** a task whose only Azure DevOps pull request has an unrecognised + `status`, and no snapshots, **WHEN** the ledger is evaluated, **THEN** the pair + records `unknown` with `provider_pr_unmerged` — the status is not guessed at and + the evaluation does not error. +- **GIVEN** a task whose only session ran on a branch equal to the repository's + `default_branch` and whose snapshots carry three distinct head commits, + **WHEN** the ledger is evaluated, **THEN** the pair records + `delivery_outcome = direct_commit`, `delivery_basis = default_branch_commit`, + and `reached_default_at` set. +- **GIVEN** a task with snapshots showing `ahead = 12` on a feature branch, no + provider pull request, and no default-branch observation, **WHEN** the ledger + is evaluated, **THEN** the pair records `delivery_outcome = unknown` with + `delivery_basis = branch_commits_unmerged`, `observed_branch_commits = 12` and + `evidence_rank = 5`. +- **GIVEN** that same task, **WHEN** a later evaluation observes its head commit + as an ancestor of the default branch, **THEN** `reached_default_at` and + `reached_default_basis = ancestor_of_default` are set and `delivery_basis` + becomes `reached_default_unattributed`, while `delivery_outcome` remains + `unknown`. +- **GIVEN** a task whose every snapshot shows `ahead = 0` and one distinct head + commit, **WHEN** the ledger is evaluated, **THEN** the pair records + `no_delivery_observed` with basis `no_commits_observed` and `evidence_rank = 4`. +- **GIVEN** a task whose only snapshot shows `ahead = 0` and an **empty** + `head_commit`, **WHEN** the ledger is evaluated, **THEN** the pair records + `no_delivery_observed` — zero distinct non-empty heads satisfies rule 4's + "at most one", so no pair falls through the rule set. +- **GIVEN** a task whose snapshots carry `ahead = NULL` and `ahead = -1`, + **WHEN** the ledger is evaluated, **THEN** both are read as `0` and the pair is + classified as if every snapshot reported `ahead = 0`. +- **GIVEN** a task with a declared repository but no snapshot and no provider + pull request, **WHEN** the ledger is evaluated, **THEN** the pair records + `unknown` with basis `no_observations` — not `no_delivery_observed`. +- **GIVEN** a task with no repository at all, **WHEN** the ledger is evaluated, + **THEN** no ledger row is written for it. +- **GIVEN** a repository whose `default_branch` is empty and a task with + snapshots showing `ahead = 12`, **WHEN** the ledger is evaluated, **THEN** the + pair records `delivery_outcome = unknown`, + `delivery_basis = default_branch_unknown` and `evidence_rank = 1` — the rule's + own basis is superseded — and **WHEN** `default_branch` is later populated and + the pair is re-evaluated, **THEN** it is promoted to `branch_commits_unmerged` + at rank 5. +- **GIVEN** that same degraded pair, **WHEN** `repositories.default_branch` is populated + and nothing else changes, **THEN** the next sweep selects the pair as due — because + `repositories.updated_at` advanced and is one of the named due sources — so the + promotion above is reachable without any other input moving. +- **GIVEN** a repository whose `default_branch` is empty and a merged pull request whose + `base_branch` is also empty, **WHEN** the pair is evaluated, **THEN** + `reached_default_at` stays `NULL` — an empty `default_branch` matches no branch, so the + provider-derived observation is suspended along with rule 2 and ancestry. +- **GIVEN** a repository whose `default_branch` is `NULL` rather than empty, **WHEN** the + pair is evaluated, **THEN** it takes the same degraded path as an empty string, with + `delivery_basis = default_branch_unknown` and `evidence_rank = 1`. +- **GIVEN** a task with two sessions on one repository, each on its own inherited branch, + each reporting a single distinct `head_commit` that differs between the sessions, + **WHEN** the pair is evaluated, **THEN** no session has two or more distinct heads of + its own, so the pair is **not** classified `direct_commit`, no ancestry check is + attempted, and `reached_default_at` stays `NULL` — distinct heads are never pooled + across sessions. +- **GIVEN** a pair classified by rule 3, 4, 5 or 6, or degraded by + **Degraded evaluation**, **WHEN** the row is written, **THEN** `delivery_ref` is + `NULL` — including for `provider_pr_unmerged`, whose provider row URL is evidence of + non-delivery and is deliberately not recorded as a delivery ref. + +### Evidence rank and reclassification + +- **GIVEN** a pair whose first snapshot shows `ahead = 0` with one head commit, + classified `no_delivery_observed` at rank 4, **WHEN** a later snapshot shows + `ahead = 12` and the pair is re-evaluated, **THEN** the row is promoted to + `unknown` / `branch_commits_unmerged` at rank 5, + `delivery_ledger_demotions_suppressed_total` does **not** increment, and + `observed_branch_commits` reads 12. Mid-session evaluation must not pin a + working pair at `no_delivery_observed`. +- **GIVEN** a pair classified `unknown` / `provider_pr_unmerged` at rank 3, + **WHEN** its pull request merges and the pair is re-evaluated, **THEN** the row + reads `pr_merge` at rank 8. +- **GIVEN** a repository whose `default_branch` is empty and a pair with a merged pull + request, stored degraded as `pr_merge` / `default_branch_unknown` at rank 1, **WHEN** + the pull request is detached, every snapshot reports `ahead = 0`, and the pair is + re-evaluated while `default_branch` is still empty, **THEN** the row reads + `no_delivery_observed` / `default_branch_unknown` still at rank 1 — the equal-rank write + happens — **AND** `delivery_ledger_degraded_outcome_changed_total` increments **AND** + `delivery_ledger_demotions_suppressed_total` does **not** increment **AND** `updated_at` + advances. Rank 1 admits more than one outcome, so this is neither a promotion nor a + suppressed demotion, and it must not be silent in both counters. +- **GIVEN** that same degraded pair, **WHEN** `default_branch` is later populated and the + pair is re-evaluated, **THEN** it is promoted to a rank ≥ 2 classification — a degraded + row at rank 1 is superseded by any sighted evaluation, whichever outcome it currently + holds. +- **GIVEN** a pair stored `pr_merge` / `provider_pr_merged` at rank 8 whose `delivery_ref` + names pull request A, **WHEN** a second merged, non-detached pull request B with an + **earlier** merge instant is synced and the pair is re-evaluated, **THEN** the rank is + still 8 and `delivery_outcome` and `delivery_basis` are unchanged, but `delivery_ref` + becomes B's URL, `updated_at` advances and + `delivery_ledger_rows_written_total` increments — equal rank guarantees the same + `(outcome, basis)`, never the same ref. +- **GIVEN** a pair whose `delivery_outcome` is `unknown` and whose `workspace_id` is W1, + **WHEN** its task is moved to workspace W2 and the pair is re-evaluated, **THEN** the + ledger row's `workspace_id` reads W2 — it is refreshed on every persisted evaluation and + is not gated by the rank guard — **AND** the pair was selected as due because + `tasks.updated_at` advanced. + +### Squash-merge, ancestry preconditions and negative evidence + +- **GIVEN** a merged pull request whose branch head is not an ancestor of the + default branch because the merge was squashed, **WHEN** ancestry is evaluated, + **THEN** the negative result is discarded, no column records `false`, and the + pair's `pr_merge` classification from rule 1 is unaffected. +- **GIVEN** a pair whose snapshots all report the same single `head_commit` — + the inherited base commit, which is trivially an ancestor of the default + branch — **WHEN** the pair is evaluated, **THEN** no ancestry check is + attempted, `delivery_ledger_ancestry_skipped_total` increments, and + `reached_default_at` remains `NULL`. +- **GIVEN** a pair reporting `ahead = 4` on an inherited branch whose + `head_commit` never moved across its snapshots, **WHEN** the pair is + evaluated, **THEN** the ancestry precondition is still not met and no ancestry + check is attempted — `ahead > 0` does not satisfy it. +- **GIVEN** a pair one of whose **sessions** carries two distinct head commits among its + own snapshots, and whose most recent two snapshots share the greatest `created_at`, + **WHEN** the ancestry head is selected, **THEN** it is the lexicographically greatest + `head_commit` of those two, and repeated evaluations select the same one. The session + grain is stated in the GIVEN because selection is restricted to the + precondition-satisfying session(s); two heads split across two sessions would satisfy no + session's precondition and no check would be attempted at all. +- **GIVEN** a pair whose session holds two distinct non-empty head commits and whose + **newest** snapshot by `created_at` carries a whitespace-only `head_commit`, **WHEN** the + ancestry head is selected, **THEN** the whitespace-only snapshot is skipped and the newest + **non-empty** head is submitted to the check — an empty value is never sent to git and + never counted as an ancestry error. +- **GIVEN** a pair classified `direct_commit` whose newest default-branch snapshot carries + an empty `head_commit`, **WHEN** `delivery_ref` is selected, **THEN** it is the newest + non-empty `head_commit` among the default-branch snapshots **of the session(s) that + satisfied rule 2**, and the same value is offered as `reached_default_ref` — no empty + string reaches either column. +- **GIVEN** a pair with two sessions on one repository — session A, on a feature branch, + holding two distinct non-empty head commits of its own and therefore satisfying the + moving-head precondition, and session B, which merely sat on the repository's + `default_branch` and wrote a **single** snapshot carrying its inherited head, with a + **later** `created_at` than any of session A's — **WHEN** the ancestry head is selected, + **THEN** it is session **A's** newest non-empty head and **not** session B's inherited + head, even though B's snapshot is the newest on the pair. Selection is restricted to the + precondition-satisfying session(s), so a non-authoring session can never supply the + commit submitted to `git merge-base --is-ancestor`. +- **GIVEN** that same pair, **WHEN** the evaluation completes, **THEN** + `reached_default_at` is **not** set from session B's inherited head — an inherited head + is trivially an ancestor of the default branch, so pair-wide selection would have set the + write-once triple with `reached_default_basis = ancestor_of_default` on a pair whose only + authored work is unmerged, and promoted rule 3's basis to + `reached_default_unattributed`. This is the same false positive the **Ancestry + precondition** prevents at the gate, and it is prevented here at selection. +- **GIVEN** a pair with two sessions on one repository — session C, which committed on the + repository's `default_branch` and holds two distinct non-empty heads there, and session + D, which sat on the same `default_branch` holding a **single** inherited head whose + snapshot has a **later** `created_at` — **WHEN** `delivery_ref` is selected for the + resulting `direct_commit` classification, **THEN** it is session **C's** newest non-empty + head, and the same value is offered as `reached_default_ref`. Only the session(s) + satisfying rule 2's own predicate contribute candidate heads, so session D's inherited + head reaches neither column. +- **GIVEN** a repository whose local checkout is missing, **WHEN** ancestry is + evaluated, **THEN** `delivery_ledger_ancestry_errors_total` increments, + `reached_default_at` stays `NULL`, `delivery_outcome` is unchanged, and the + rest of the classification is still persisted. +- **GIVEN** a repository whose `refs/remotes/origin/` does not + exist but whose local branch `` does, **WHEN** ancestry is + evaluated, **THEN** the local branch is used and the check proceeds. +- **GIVEN** a provider-cloned repository whose `local_path` is empty, **WHEN** ancestry is + evaluated, **THEN** the checkout-resolution port returns an error rather than a path, + `delivery_ledger_ancestry_errors_total` increments, `reached_default_at` stays `NULL`, + and the rest of the classification is still persisted. +- **GIVEN** a repository whose stored `local_path` canonicalizes to a different location + than the one saved, **WHEN** ancestry is evaluated, **THEN** the port returns an error and + no git command is run against the resolved location — the evaluator inherits that + rejection instead of implementing its own path check. +- **GIVEN** a pair with a merged pull request whose base branch is the default branch, + snapshots committing on the default branch, and a head commit that is an ancestor of + the default branch — all three observable in one evaluation — **WHEN** + `reached_default_at` is first written, **THEN** `reached_default_basis` is + `provider_pr_merged` and `reached_default_ref` is that pull request's URL, because + provider merge outranks both other bases; and repeated evaluations of the same inputs + select the same basis and ref. +- **GIVEN** a pair whose only default-branch evidence is a positive ancestry check, + **WHEN** `reached_default_at` is written, **THEN** `reached_default_basis` is + `ancestor_of_default` and `reached_default_ref` is the commit submitted to the check. +- **GIVEN** a repository whose `default_branch` is `main`, and a pair with two merged, + non-detached pull requests — request A into `release/1.2` with the **earlier** merge + instant, and request B into `main` — **WHEN** the pair is evaluated, **THEN** + `delivery_ref` is **A's** URL (the `pr_merge` ref rule does not filter on base branch) + while `reached_default_ref` is **B's** URL and `reached_default_basis` is + `provider_pr_merged` (the observation ref rule selects only rows whose base branch + equals `default_branch`). The two refs are drawn from different row sets and must not be + the same selection. +- **GIVEN** a pair whose only merged pull request is **detached** and whose base branch is + the repository's `default_branch`, plus snapshots showing `ahead > 0`, **WHEN** the pair + is evaluated, **THEN** rule 1 does not match, no `provider_pr_merged` observation is + offered, `reached_default_at` stays `NULL`, and the pair classifies `unknown` / + `branch_commits_unmerged` at rank 5 — a detached row is excluded from the observation + exactly as it is from rule 1. + +### Idempotency, ordering and concurrency + +- **GIVEN** a ledger row classified `pr_merge`, **WHEN** it is evaluated again + with unchanged inputs, **THEN** every classification and observation column is + unchanged, `updated_at` is unchanged, and `last_evaluated_at` and + `evaluation_seq` advance. +- **GIVEN** a pair with two merged pull requests, **WHEN** `delivery_ref` is + selected, **THEN** it is the one with the earliest merge instant, and on an + exact tie the one with the lower request number. +- **GIVEN** a pair with one merged GitHub pull request and one completed Azure + DevOps pull request, **WHEN** `delivery_ref` is selected, **THEN** the GitHub + row's `merged_at` and the Azure row's `updated_at` are compared as merge + instants, and on an exact tie at equal request numbers `azure_devops` sorts + before `github`. +- **GIVEN** a pair with two default-branch snapshots sharing the same + `created_at`, **WHEN** `delivery_ref` is selected, **THEN** it is the + lexicographically greatest `head_commit` of the two, and repeated evaluations + select the same one. +- **GIVEN** a row classified `pr_merge` at rank 8 with + `observed_branch_commits = 12`, **WHEN** the pull request is detached and the + pair is re-evaluated so the computed evidence ranks lower, **THEN** the stored + outcome, basis, ref and rank are all unchanged, + `delivery_ledger_demotions_suppressed_total` increments, and + `observed_branch_commits` still reads 12. +- **GIVEN** a row classified `pr_merge` whose snapshots are all deleted with + their session, **WHEN** the pair is re-evaluated, **THEN** + `observed_branch_commits` does not decrease. +- **GIVEN** two evaluators writing the same pair concurrently, one computing + rank 1 and one computing rank 8, **WHEN** both statements commit in either + order, **THEN** the row reads `pr_merge` at rank 8. +- **GIVEN** a row whose `reached_default_at` is `NULL` and two concurrent + evaluations each supplying a different first observation, **WHEN** both commit, + **THEN** `reached_default_at`, `reached_default_basis` and `reached_default_ref` + all come from the same one of the two — the triple is never mixed. +- **GIVEN** a row whose `reached_default_at` is set, **WHEN** a later evaluation + observes the default branch again through a different basis, **THEN** + `reached_default_at` and `reached_default_basis` are unchanged. +- **GIVEN** a pair with no ledger row and no snapshot and no provider row, + **WHEN** a sweep runs, **THEN** the pair is selected as due and a row is + written with basis `no_observations`; **AND WHEN** the next sweep runs with + nothing changed, **THEN** the pair is not selected again. +- **GIVEN** an evaluation whose snapshot query returns an error, **WHEN** the + pair is evaluated, **THEN** no column is written — including + `last_evaluated_at` — `delivery_ledger_evaluation_errors_total` increments, + `delivery_ledger_evaluations_total` does **not** increment, and + the pair is still due on the next sweep. +- **GIVEN** a row whose stored `observed_branch_commits` is `NULL` and an incoming value + of 7, **WHEN** the upsert runs, **THEN** the column reads 7 — a `NULL` stored value is + lower than any incoming value and must not swallow it. +- **GIVEN** a pair whose inputs have not changed, **WHEN** it is re-evaluated, **THEN** + `delivery_ledger_evaluations_total` increments but `delivery_ledger_rows_written_total` + does **not**, because no classification or observation column changed and `updated_at` + did not advance. +- **GIVEN** a pair with two merged GitLab merge requests carrying the same `mr_iid` under + different `project_path` values, **WHEN** `delivery_ref` is selected, **THEN** the + lower `project_path` wins and repeated evaluations select the same URL. +- **GIVEN** a sweep pass that encounters the same three sessions with empty + `repository_id` on two consecutive passes, **WHEN** each pass publishes, **THEN** + `delivery_ledger_sessions_unattributed_total` reads 3 after both — it is a gauge set per + pass, not a running total. +- **GIVEN** a repository that is soft-deleted, **WHEN** sweeps run, **THEN** its pairs' + `last_evaluated_at` stops advancing and their existing rows are retained unchanged. +- **GIVEN** a soft-deleted repository with a pair whose `reached_default_at` is `NULL` and + whose `last_evaluated_at` is three days old, **WHEN** sweeps run, **THEN** the pair is + **not** selected — the freeze is evaluated before the three due conditions and overrides + stale refresh. +- **GIVEN** a soft-deleted repository with a candidate pair that has **no ledger row**, + **WHEN** sweeps run, **THEN** no row is written and the pair is not selected on any + pass — the freeze also overrides the unconditionally-due rule. +- **GIVEN** that same soft-deleted repository, **WHEN** `deleted_at` is cleared, **THEN** + `repositories.updated_at` advances and the pair is selected as due on the next sweep. +- **GIVEN** an archived task with a live repository, **WHEN** sweeps run, **THEN** its + pairs remain candidates and remain **eligible** for selection under the same three due + conditions as any other pair — archiving a task neither excludes it from candidacy nor + freezes it, and no predicate anywhere reads `tasks.archived_at`. +- **GIVEN** that same archived task, **WHEN** one of its sessions writes a new git snapshot, + **THEN** the pair is selected as due on the next sweep and `last_evaluated_at` advances — + archived pairs are re-evaluated on input movement exactly like unarchived ones. +- **GIVEN** that same archived task with `reached_default_at` already set and no input row + moving, **WHEN** sweeps run, **THEN** the pair is **not** selected and + `last_evaluated_at` does **not** advance — it is due under none of the three conditions, + which is correct and is not a freeze. Eligibility is what archiving preserves; continuous + advance is not something any pair gets. +- **GIVEN** an evaluation that begins reading its inputs at T0 and commits its upsert at + T2, and a snapshot for the same pair written at T1 with T0 < T1 < T2, **WHEN** the + upsert commits, **THEN** the stored `last_evaluated_at` is **T0**, so + `last_evaluated_at < MAX(input.created_at)` still holds and the pair is selected as due + on the next sweep. Recording T2 would mask the T1 snapshot permanently. +- **GIVEN** a pair whose evaluation began at T0 and a concurrent, faster evaluation of the + same pair that began at T3 > T0 and committed first, **WHEN** the slower evaluation + commits, **THEN** `last_evaluated_at` still reads T3 — the high-water rule means a slow + evaluation cannot rewind the column. +- **GIVEN** a pair with no snapshots at all, **WHEN** it is evaluated, **THEN** + `observed_branch_commits` is `NULL` and not `0`. +- **GIVEN** a pair whose stored `observed_branch_commits` is 12 and whose snapshots have + all been deleted, **WHEN** it is re-evaluated so the incoming value is `NULL`, **THEN** + the column still reads 12. +- **GIVEN** a candidate pair whose non-empty `repository_id` matches no row in + `repositories`, **WHEN** a sweep runs, **THEN** no ledger row is written for it, no + upsert is attempted, `delivery_ledger_pairs_missing_repository_total` counts it for that + pass, and `delivery_ledger_write_errors_total` does **not** increment — the pair is + excluded from candidacy rather than attempted and failed. +- **GIVEN** that same pair, **WHEN** the missing `repositories` row is later created, + **THEN** the pair becomes a candidate on the next sweep and a ledger row is written. +- **GIVEN** a `github_task_prs` row whose `task_id` matches no row in `tasks` — the task was + hard-deleted and the provider row was not pruned with it — **WHEN** a sweep runs, **THEN** + no ledger row is written for that pair, no upsert is attempted, + `delivery_ledger_pairs_missing_task_total` counts it for that pass, and + `delivery_ledger_write_errors_total` does **not** increment — so the primary + migration-failure signal keeps its resting value of `0` on a healthy schema. +- **GIVEN** that same orphaned provider row, **WHEN** two consecutive sweep passes run with + nothing changed, **THEN** `delivery_ledger_pairs_missing_task_total` reads the same count + after both — it is a gauge set per pass, not a running total — and neither pass writes a + row or logs `delivery_ledger.write_failed`. +- **GIVEN** a candidate pair rejected for a missing `tasks` row, **WHEN** the sweep orders + its due pairs, **THEN** the pair is absent from the ordering entirely rather than sorting + on a `NULL` `tasks.created_at`, and the exclusion is observable through the gauge rather + than being an invisible join drop. +- **GIVEN** a ledger table that does not exist because its migration was swallowed, + **WHEN** a sweep pass runs, **THEN** candidacy is still computed from the input tables, + the dueness read fails, **every candidate is treated as due**, + `delivery_ledger.dueness_unavailable` is logged exactly **once** for that pass, + `delivery_ledger_write_errors_total` increments once per pair whose upsert failed, + `delivery_ledger.write_failed` is logged exactly **once** for that pass, + `delivery_ledger_evaluation_errors_total` does **not** increment, and the pass still + reaches its last candidate rather than aborting on the first failure. +- **GIVEN** that same missing table and a candidate pair whose repository is soft-deleted, + **WHEN** the pass runs under the dueness fallback, **THEN** that pair is still **not** + evaluated and no upsert is attempted for it — the fallback widens dueness only, and the + freeze is evaluated ahead of it. +- **GIVEN** that same missing table and a candidate pair whose `task_id` matches no `tasks` + row, **WHEN** the pass runs under the dueness fallback, **THEN** the pair is excluded by + **Candidate pairs** condition 3 and counted in + `delivery_ledger_pairs_missing_task_total`, and no upsert is attempted for it — so it + contributes nothing to `delivery_ledger_write_errors_total`. +- **GIVEN** a healthy ledger table and a **transient** failure of the dueness read on one + pass, **WHEN** that pass runs, **THEN** every candidate is evaluated once, every upsert + succeeds, `delivery_ledger_write_errors_total` does **not** increment, no classification + or observation column changes value for a pair whose inputs did not move, and the next + pass computes dueness normally. +- **GIVEN** that same missing table, **WHEN** the sweep publishes at the end of the pass, + **THEN** `delivery_ledger_last_evaluated_unix` reads `-1` and + `delivery_ledger_stall_seconds` reads `-1`, and `delivery_ledger.unavailable` is logged + — **not** `0`/`0`, which is the healthy-and-empty state and would make a broken ledger + indistinguishable from a fresh one. +- **GIVEN** a boot at which the `task_delivery_ledger` migration failed, so + `telemetry.delivery_ledger.activated_at` is absent from `kandev_meta`, **WHEN** the + backend runs, **THEN** the sweep still starts and still attempts its upserts — the + activation key does not gate the writer, and gating it would silence the very counters + that detect the failed migration. +- **GIVEN** a pair whose ancestry check errored and whose `reached_default_at` is + therefore `NULL`, and no input row moves afterwards, **WHEN** 24 hours pass, **THEN** + the pair is selected as due by the stale-refresh condition and the ancestry check is + attempted again. +- **GIVEN** a pair whose `reached_default_at` is `NULL`, whose session authored two or more + distinct non-empty head commits so the ancestry precondition is met, and whose selected + head commit **is** an ancestor of `refs/remotes/origin/` in the + repository's local checkout — a merge that landed long after the task's last session + ended, moving no row in any Kandev table — **WHEN** the next stale refresh evaluates it, + **THEN** `reached_default_at` is set, `reached_default_basis` is `ancestor_of_default`, + and `reached_default_ref` is that commit. The ancestry relation is stated as a + precondition rather than assumed: this repository squash-merges by policy, so a merged + branch head is normally **not** an ancestor (see **Evidence**, finding 2), and a test that + omits the relation is asserting nothing. +- **GIVEN** that same pair where the merge was **squashed**, so the selected head is not an + ancestor of the default branch, **WHEN** the next stale refresh evaluates it, **THEN** the + negative result is discarded, `reached_default_at` stays `NULL`, and the pair is selected + again by stale refresh 24 hours later — the condition retries indefinitely while the + observation is missing, and that is the intended cost of not being able to see a squash. +- **GIVEN** a pair whose `reached_default_at` is already set, **WHEN** 24 hours pass with + no input movement, **THEN** it is **not** selected by the stale-refresh condition — the + condition applies only while the observation is missing. +- **GIVEN** a pair evaluated by a sweep pass, **WHEN** the next sweep runs 5 minutes after + that pass **finished**, **THEN** at most one pass is ever in flight, so a pass that + evaluates that pair advances its `evaluation_seq` by exactly **one, never two**, and each + per-pass gauge is set exactly once for that pass. This says nothing about whether the + pair is evaluated at all on a given pass — a pair that is not due is not evaluated and + its `evaluation_seq` does not move, per the dueness scenarios above. The guarantee is + against double-counting within a pass, not a promise of one increment per pass. + +### Migration and activation + +- **GIVEN** a database created before this feature, **WHEN** the backend boots, + **THEN** `task_delivery_ledger` exists, `runs.outcome` exists and is `NULL` on + every pre-existing row, and no pre-existing row's values change. +- **GIVEN** the backend boots twice against the same database, **WHEN** + migrations replay, **THEN** they apply cleanly the second time and + `telemetry.delivery_ledger.activated_at` retains the value written on the + first boot. +- **GIVEN** a Postgres deployment, **WHEN** the same migrations run fresh and + then replay, **THEN** both succeed and duplicate-object errors are classified + through `internal/db`. +- **GIVEN** a boot in which the `task_delivery_ledger` migration fails and the runner + swallows the error at WARN, **WHEN** the boot completes, **THEN** + `telemetry.delivery_ledger.activated_at` is **absent** from `kandev_meta`, because the + schema probe found no table; **AND WHEN** a later boot applies the migration + successfully, **THEN** the key is written with that boot's instant. +- **GIVEN** a boot in which the `runs.outcome` migration fails, **WHEN** the boot + completes, **THEN** `telemetry.run_outcome.activated_at` is absent for the same reason, + so no consumer reads the mechanism as live while the column is missing. +- **GIVEN** an activated database, **WHEN** a database reset runs, **THEN** + `telemetry.delivery_ledger.activated_at` and `telemetry.run_outcome.activated_at` + are absent from `kandev_meta` afterwards, and **WHEN** the backend next boots, + **THEN** both are rewritten with the new instant. +- **GIVEN** an activated database, **WHEN** a reset deletes both activation keys and then + fails partway through dropping user tables, **THEN** `task_delivery_ledger` rows may + remain but both keys are absent — so no consumer reads the mechanism as live over data + it cannot trust — and **WHEN** the backend next boots and the schema probe succeeds, + **THEN** the keys are rewritten with that boot's instant. +- **GIVEN** a reset in which the activation-key deletion itself fails, **WHEN** the reset + runs, **THEN** it fails and reports the failure, and **no** user table is dropped — the + keys are never left pointing at data that a later step removed. +- **GIVEN** a task created before `telemetry.delivery_ledger.activated_at` that + has no ledger row, **WHEN** the database is read, **THEN** the two facts are + separately observable — the activation instant from `kandev_meta` and the + absence of a row from `task_delivery_ledger` — so "not observed by this + mechanism" is distinguishable from `no_delivery_observed` without inspecting + any consumer. + +### Office run outcome + +- **GIVEN** a run blocked by the pre-execution budget check, **WHEN** it + finishes, **THEN** `runs.status = 'finished'` and `runs.outcome = + 'budget_blocked'`. +- **GIVEN** a run skipped because the agent is not active, **WHEN** it finishes, + **THEN** `runs.outcome = 'agent_inactive'` — even though that path writes no + `office_activity_log` row. +- **GIVEN** a run held by an active task-tree hold, **WHEN** it finishes, + **THEN** its outcome is `task_tree_held`. +- **GIVEN** a run whose task checkout fails transiently, **WHEN** the scheduler + handles it, **THEN** it is retried or escalated after the retry budget and is + not finished as completed work. +- **GIVEN** a run whose task is already checked out, **WHEN** the scheduler + handles it, **THEN** it is requeued for retry and is not finished as completed + work. +- **GIVEN** a task-bearing run whose agent was launched by the task starter, **WHEN** the + agent-completed subscriber finishes it, **THEN** `runs.outcome = 'processed'` and it is + not left `NULL` after activation. +- **GIVEN** a taskless run, **WHEN** the scheduler cannot launch it, **THEN** it is + marked `failed` with a diagnostic error and does not report completed work. +- **GIVEN** a task-bearing run in a deployment with no task starter configured, **WHEN** + the scheduler handles it, **THEN** it is marked `failed` with a diagnostic error, + its checkout is released, and the agent is not launched. +- **GIVEN** two terminal callers reaching the same run row, **WHEN** both write, **THEN** + the row's `status` and `outcome` both come from the later write — they are set in one + statement and can never disagree. +- **GIVEN** a run that fails, **WHEN** `FailRun` transitions it through the shared + `transitionRunTerminal` statement, **THEN** `runs.status = 'failed'` and + `runs.outcome IS NULL` — the failed path writes `NULL` in the same statement, and no + value from the five-value vocabulary is invented for it. +- **GIVEN** a day containing one failed run written that way, **WHEN** + `RunCountsByDayForAgent` reports that day, **THEN** it counts in `failed` and in + neither `unclassified` nor `skipped` — the `NULL` outcome is never read, because the + three outcome-derived buckets are all conditioned on `status = 'finished'`. +- **GIVEN** a run whose `FinishRun` call fails and whose error the call site discards, + **WHEN** the run is later inspected, **THEN** it is **not** `finished`, so it counts in + neither `succeeded` nor `skipped` nor `unclassified`. +- **GIVEN** a run transitioned to a terminal status through + `office/scheduler.SchedulerService.FailRun`, which reaches the runs repository's + `FinishRun` without passing through `transitionRunTerminal`, **WHEN** the row is + inspected, **THEN** `runs.status = 'failed'` and `runs.outcome IS NULL`, and the row + counts in `failed` and in neither `unclassified` nor `skipped`. +- **GIVEN** a run transitioned through `office/scheduler.SchedulerService.FinishRun`, + **WHEN** the row is inspected, **THEN** `runs.status = 'finished'` and + `runs.outcome IS NULL`, so it counts in `unclassified` and **not** in `succeeded` — a + terminal path with no established semantic label is never counted as a success. +- **GIVEN** a day containing one processed run, one budget-blocked run, one + failed run and one pre-activation `finished` run with `outcome IS NULL`, + **WHEN** `RunCountsByDayForAgent` reports that day, **THEN** it returns + `succeeded = 1`, `skipped = 1`, `unclassified = 1`, `failed = 1` and + `other = 0`. +- **GIVEN** a `finished` run whose `outcome` holds a value outside the five + named ones, **WHEN** that day is reported, **THEN** it counts in `skipped` — + the bucketing is total and no query errors. +- **GIVEN** that same day, **WHEN** the agent dashboard renders it, **THEN** + `AgentRunActivityDay.total` equals + `succeeded + skipped + unclassified + failed + other`, and + `AgentSuccessRateDay` reports `succeeded = 1` and `unclassified = 1` against the same + total. +- **GIVEN** a day made up **entirely** of pre-activation `finished` runs with + `outcome IS NULL` — three of them, and nothing else — **WHEN** + `RunCountsByDayForAgent` reports that day, **THEN** `succeeded = 0`, `skipped = 0`, + `unclassified = 3`, `failed = 0`, `other = 0`; **AND WHEN** the dashboard renders it, + **THEN** `AgentSuccessRateDay` reports `succeeded = 0`, `unclassified = 3` and + `total = 3`, so a reader can tell from the response alone that the day predates the + mechanism rather than recording three failures to succeed. +- **GIVEN** a day made up entirely of post-activation runs, **WHEN** the dashboard renders + it, **THEN** `AgentSuccessRateDay.unclassified` is `0` — the field is the marker for the + pre-activation series and reads zero once every terminal run carries an outcome. + +### Writer health + +- **GIVEN** the ledger writer has not evaluated any pair for longer than its + 15-minute stall threshold while task sessions continue to complete, **WHEN** + the sweep completes a pass, **THEN** `delivery_ledger_stall_seconds` exceeds + 900 and a WARN log with the message `delivery_ledger.stalled` is emitted once + for that pass. +- **GIVEN** the backend restarts, **WHEN** the first sweep after the restart + completes, **THEN** `delivery_ledger_last_evaluated_unix` reports the true + `MAX(last_evaluated_at)` from the database even though every counter has reset + to zero. +- **GIVEN** an empty ledger table, **WHEN** the sweep publishes, + **THEN** `delivery_ledger_last_evaluated_unix` is `0` and + `delivery_ledger_stall_seconds` is `0` — an empty table is not reported as an + infinite stall. +- **GIVEN** a ledger holding evaluated rows and a database in which **no** session survives + the comparand filters — every session has an empty `repository_id`, or belongs to a + soft-deleted repository, or there are no sessions at all — **WHEN** the sweep publishes, + **THEN** `delivery_ledger_last_evaluated_unix` reports the true + `MAX(last_evaluated_at)` and `delivery_ledger_stall_seconds` is `0`, and no + `delivery_ledger.stalled` WARN is emitted — an empty comparand means there is no work to + be behind on, not that the writer is stalled. +- **GIVEN** a ledger holding evaluated rows and a comparand query that errors while the + ledger query succeeds, **WHEN** the sweep publishes, **THEN** + `delivery_ledger_last_evaluated_unix` reports the true `MAX(last_evaluated_at)`, + `delivery_ledger_stall_seconds` reads `-1`, and + `delivery_ledger.comparand_unavailable` is logged once for that pass — the stall figure + is unknown rather than zero, so a repeatedly failing comparand read can never report a + healthy writer. +- **GIVEN** a healthy writer that has evaluated every due pair, and a session with an + empty `repository_id` that updated one hour ago, **WHEN** the sweep publishes, **THEN** + `delivery_ledger_stall_seconds` is `0` and no `delivery_ledger.stalled` WARN is emitted + — an unattributed session contributes to no pair and must not drive the comparand. +- **GIVEN** a healthy writer and a session belonging to a soft-deleted repository that + updated one hour ago, **WHEN** the sweep publishes, **THEN** + `delivery_ledger_stall_seconds` is `0` — that pair's evaluation is frozen by design. +- **GIVEN** a writer that has stopped, and an **archived** task whose session updated + twenty minutes ago, **WHEN** the sweep publishes, **THEN** + `delivery_ledger_stall_seconds` exceeds 900 and `delivery_ledger.stalled` is emitted — + archived tasks are still evaluated, so their sessions still count toward the comparand + and a stall on them is a real stall. + +## Out of scope + +Each exclusion below is a contract, not an omission. + +- **The extract itself.** The downstream point-in-time extract that reads this + table is external to this repository; no extract code exists here and none is + added. This spec's obligation is to make the ledger *readable without + ambiguity* — the activation instant in `kandev_meta`, `NULL` rather than `0` + for legacy rows, and `evaluation_seq` — and every scenario above is stated as + an observable property of the database, never of a consumer. +- **A structured stop reason for a task or session.** `task_sessions` carries + free-text `error_message` ("task archived", "task tree archived", raw + rate-limit JSON) and a `state` enum, and nothing else. This is a different + grain (session, not run), a different table, and its enum cannot be derived + without enumerating every writer of `error_message` and every `state` + transition. It gets its own card. The `runs.outcome` contract in this spec does + **not** cover it and must not be extended to `task_sessions`. +- **Consuming the GitHub push webhook for merge inference.** The + `push_webhook_default` basis is defined in the vocabulary so a later card can + populate it without a schema change, but no subscriber is added here. The + webhook fires only for GitHub App installations; 10 of the 12 repositories in + the reference store have no provider at all, so it can never be the primary + mechanism. +- **Reconstructing history.** No ledger column is ever backdated or inferred from + evidence the mechanism did not itself observe. `first_classified_at` and + `reached_default_at` are the instants *this* mechanism made the observation, never the + instant the underlying work happened, so a pair whose branch merged months before + activation records the observation instant and not the merge. There is no import, no + reconstruction from `office_activity_log`, and no derivation from a provider's history + API. + + This is **not** a claim that old pairs go unevaluated. Every pre-existing + `(task, repository)` pair is a candidate, has no ledger row, and is therefore due on + the first sweep after activation and classified from whatever evidence is still + present — see **Sweep selection predicate**. Those two statements were previously in + tension in this document; the distinction that resolves them is *reconstruction* versus + *evaluation*. Classifying an old pair from evidence visible today is ordinary + evaluation. Asserting when that pair delivered, from evidence the ledger never saw, is + reconstruction, and only reconstruction is excluded. The activation instant in + `kandev_meta` is what lets a consumer tell the two apart: every row is timestamped at or + after it. +- **Making `RunCountsByDayForAgent` dialect-portable.** It is pre-existing + SQLite-only code; this card reshapes its buckets and nothing else. +- **Retiring the dormant `office/scheduler.SchedulerService.FinishRun` / `.FailRun` + pair.** They call the runs repository's `FinishRun` directly, bypassing + `transitionRunTerminal` and therefore also bypassing the `OfficeRunProcessed` publish + that the service-layer path performs. This card threads `outcome` through them (both pass + `NULL`, see **Office run outcome**) because the shared signature change forces the + compiler to reach them, and stops there. Deciding whether a second terminal-transition + path should exist at all — and whether the missing bus publish is a defect — is a + behavioural question about run lifecycle, not about telemetry, and it gets its own card. +- **The second `status = 'finished'`-as-success consumer.** + `apps/backend/internal/office/summary/inputs.go` counts `r.status = 'finished'` as + `completed` when it assembles continuation summaries for an agent. It has exactly the + defect this card fixes in `RunCountsByDayForAgent` — a budget-blocked or idle-skipped + run reads as completed work — and it is **deliberately not changed here**. It is a + different consumer at a different grain, feeding agent prompt context rather than a + dashboard, and changing what an agent is told about its own history is a behavioural + change that deserves its own card and its own review. It is named here rather than left + silent so that after this card ships, nobody concludes from `## Why` that every + finished-as-success reader has been corrected. It gets its own card. +- **Distinguishing a research card from a chore.** The ledger separates delivery + from non-delivery. It does not classify intent, and + `no_delivery_observed` is not a judgement about whether the card should have + delivered. +- **Attributing delivery to a session, a turn, or a workflow step.** The ledger's + grain is `(task, repository)`. A session's snapshots attribute to that + session's single `task_sessions.repository_id`; a session spanning more than + one repository is not modelled. +- **Deduplicating delivery credit when one pull request serves several tasks.** + Both tasks read `pr_merge`. +- **Recording a closed pull request's disposition** (`superseded`, `duplicate`, + `withdrawn`). That is a separate card against the provider tables. +- **Any user-visible surface.** No board column, badge, filter, dashboard tile, + or settings control is added, and no new copy is rendered. If a later card + surfaces the ledger or the new run-outcome buckets in the web UI, that copy + goes through `t()` and that card owns the E2E coverage. +- **Changing `runs.status` semantics**, or changing what any existing consumer of + `runs.status` reports. +- **Populating `task_session_commits`, `pre_commit_snapshot_id`, or + `post_commit_snapshot_id`.** Those are separate cards and this spec does not + depend on them.