From 56bc954001cfbf0c1cec5bd815e26885015b5e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Sat, 12 Sep 2026 20:44:13 +0100 Subject: [PATCH] ci: enforce pull request documentation coverage (#3586) --- .github/AGENTS.md | 9 + .github/pull_request_template.md | 7 + .../scripts/pr-docs-workflow-contract_test.py | 100 + .github/scripts/pr-docs.cjs | 1778 +++++++++++++++++ .github/scripts/pr-docs.test.cjs | 1209 +++++++++++ .github/workflows/lint-action-pinning.yml | 6 + .github/workflows/pr-docs.yml | 63 + CONTRIBUTING.md | 6 + Makefile | 1 + docs/ci-merge-queue.md | 33 + .../2026-09-10-pr-documentation-coverage.md | 44 + docs/plans/pr-documentation-coverage/plan.md | 105 + .../task-01-coverage-validator.md | 87 + .../task-02-pr-workflow.md | 111 + .../task-03-queue-coverage.md | 115 ++ docs/specs/ci/README.md | 1 + .../pull-request-documentation-coverage.md | 64 + .../pull-request-documentation-coverage.md | 149 ++ 18 files changed, 3888 insertions(+) create mode 100644 .github/scripts/pr-docs-workflow-contract_test.py create mode 100644 .github/scripts/pr-docs.cjs create mode 100644 .github/scripts/pr-docs.test.cjs create mode 100644 .github/workflows/pr-docs.yml create mode 100644 docs/decisions/2026-09-10-pr-documentation-coverage.md create mode 100644 docs/plans/pr-documentation-coverage/plan.md create mode 100644 docs/plans/pr-documentation-coverage/task-01-coverage-validator.md create mode 100644 docs/plans/pr-documentation-coverage/task-02-pr-workflow.md create mode 100644 docs/plans/pr-documentation-coverage/task-03-queue-coverage.md create mode 100644 docs/specs/ci/requirements/pull-request-documentation-coverage.md create mode 100644 docs/specs/ci/system-design/pull-request-documentation-coverage.md diff --git a/.github/AGENTS.md b/.github/AGENTS.md index 45da19d5de..75b4a01810 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -34,3 +34,12 @@ Apply this guidance whenever editing `.github/**`. - For workflow security changes, run the relevant raw workflow-contract tests, `python3 .github/scripts/lint-action-pinning_test.py`, `zizmor .github/workflows`, and `git diff --check`. +- The `pr-docs.yml` workflow is a base-controlled, metadata-only check. It reads + pull-request files through the bounded `.github/scripts/pr-docs.cjs` adapter; + it must never check out or execute a pull-request head. +- Its `PR documentation coverage` status is revision-specific. The exact + `no-docs-allow` label is the only policy override, and merge-group evaluation + must resolve and validate every member independently against the group's + entry boundaries. All event types share a non-cancelling target-branch lock + with `queue: max` so queued label reevaluations cannot race with merge-group + status writes or replace the single pending run. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5a9dae6419..17e62ee408 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -35,6 +35,13 @@ DIAGRAM (optional) POSSIBLE IMPROVEMENTS (optional) One line: risk level + what could go wrong or be improved. Skip if negligible. +DOCUMENTATION COVERAGE + The `PR documentation coverage` status uses a deterministic path and artifact + check. Runtime, workflow, configuration, and other non-exempt changes need a + changed work order linked to its plan, requirements, acceptance criteria, and + system design. The exact `no-docs-allow` label is a maintainer exception. The + check does not claim semantic completeness or require planning before coding. + RELATED ISSUES Use "Closes #N" if this resolves an issue. Remove the line if there is no related issue. diff --git a/.github/scripts/pr-docs-workflow-contract_test.py b/.github/scripts/pr-docs-workflow-contract_test.py new file mode 100644 index 0000000000..507dc317a7 --- /dev/null +++ b/.github/scripts/pr-docs-workflow-contract_test.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Contract tests for the pull request documentation coverage workflow.""" + +from __future__ import annotations + +from pathlib import Path +import re +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pr-docs.yml" +LINT_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "lint-action-pinning.yml" + + +class PullRequestDocumentationWorkflowContractTest(unittest.TestCase): + def setUp(self) -> None: + self.assertTrue(WORKFLOW.is_file(), "Documentation coverage workflow is missing") + self.workflow = WORKFLOW.read_text(encoding="utf-8") + + # @covers AC-CI-PR-DOCS-001.1, AC-CI-PR-DOCS-002.2 + def test_runs_for_all_pr_and_manual_retry_events(self) -> None: + trigger = self.workflow.partition("on:\n")[2].partition("\nconcurrency:")[0] + self.assertIn("pull_request_target:", trigger) + for event in ( + "opened", + "reopened", + "synchronize", + "edited", + "labeled", + "unlabeled", + "ready_for_review", + ): + self.assertIn(event, trigger) + self.assertIn("merge_group:", trigger) + self.assertIn("checks_requested", trigger) + self.assertIn("workflow_dispatch:", trigger) + self.assertIn("pr_number:", trigger) + self.assertNotIn("paths:", trigger) + + # @covers AC-CI-PR-DOCS-003.1, AC-CI-PR-DOCS-003.4 + def test_serializes_prs_without_cancelling_merge_group_runs(self) -> None: + self.assertIn("group: >-", self.workflow) + self.assertIn("github.event.merge_group.base_ref", self.workflow) + self.assertIn("github.event.pull_request.base.ref", self.workflow) + self.assertIn("queue: max", self.workflow) + self.assertIn("cancel-in-progress: false", self.workflow) + + # @covers AC-CI-PR-DOCS-003.3 + def test_uses_trusted_least_privilege_checkout(self) -> None: + permissions = self.workflow.partition("permissions:\n")[2].partition("\njobs:")[0] + self.assertEqual( + permissions.strip(), + "contents: read\n pull-requests: read\n statuses: write", + ) + self.assertRegex( + self.workflow, + r"uses: actions/checkout@[0-9a-f]{40} # v[0-9]+", + ) + self.assertIn("persist-credentials: false", self.workflow) + self.assertIn("github.event.merge_group.base_sha", self.workflow) + self.assertNotIn("github.event.pull_request.head.sha", self.workflow) + self.assertIn("ref: ${{ env.CHECKOUT_REF }}", self.workflow) + self.assertNotIn("pull_request:\n", self.workflow) + self.assertNotIn("pnpm install", self.workflow) + self.assertNotIn("npm install", self.workflow) + self.assertIn("node .github/scripts/pr-docs.cjs", self.workflow) + self.assertIn("name: Publish PR documentation coverage status", self.workflow) + self.assertNotIn("name: PR documentation coverage\n", self.workflow) + + # @covers AC-CI-PR-DOCS-001.1, AC-CI-PR-DOCS-002.1, AC-CI-PR-DOCS-003.2 + def test_dispatch_is_restricted_to_the_default_branch_and_script_owns_statuses(self) -> None: + self.assertRegex( + self.workflow, + re.compile( + r"github\.event_name != 'workflow_dispatch'.*github\.ref.*default_branch", + re.DOTALL, + ), + ) + self.assertIn("PR documentation coverage", (REPO_ROOT / ".github" / "scripts" / "pr-docs.cjs").read_text(encoding="utf-8")) + self.assertIn("statuses: write", self.workflow) + + def test_contract_is_registered_in_the_required_lint_workflow(self) -> None: + lint_workflow = LINT_WORKFLOW.read_text(encoding="utf-8") + self.assertRegex( + lint_workflow, + r"(?m)^ - name: Test pull request documentation workflow contract$\n" + r"^ run: python3 .github/scripts/pr-docs-workflow-contract_test.py$", + ) + self.assertRegex( + lint_workflow, + r"(?m)^ - name: Test pull request documentation validator$\n" + r"^ run: node --test .github/scripts/pr-docs.test.cjs$", + ) + makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8") + self.assertIn("@node --test .github/scripts/pr-docs.test.cjs", makefile) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/pr-docs.cjs b/.github/scripts/pr-docs.cjs new file mode 100644 index 0000000000..b8386d9620 --- /dev/null +++ b/.github/scripts/pr-docs.cjs @@ -0,0 +1,1778 @@ +'use strict'; + +const path = require('node:path'); +const fs = require('node:fs'); +const { TextDecoder } = require('node:util'); + +const POSIX_PATH = path.posix; +const WORK_ORDER_PATTERN = /^docs\/plans\/[^/]+\/task-\d{2}-[^/]+\.md$/; +const LOCK_BASENAMES = new Set([ + 'pnpm-lock.yaml', + 'package-lock.json', + 'yarn.lock', + 'go.sum', + 'Cargo.lock', +]); +const SCRIPT_TEST_EXTENSIONS = new Set([ + 'cjs', + 'cts', + 'js', + 'jsx', + 'mjs', + 'mts', + 'ts', + 'tsx', +]); +const FRONTMATTER_FIELDS = new Set([ + 'acceptance_criteria', + 'created', + 'depends_on', + 'id', + 'legacy_specs', + 'migration', + 'owners', + 'plan', + 'requirements', + 'status', + 'system', + 'system_design', + 'title', + 'updated', + 'last_updated', + 'specification_version', + 'wave', +]); +const ARRAY_FIELDS = new Set([ + 'acceptance_criteria', + 'depends_on', + 'legacy_specs', + 'owners', + 'requirements', + 'system_design', +]); +const WORK_ORDER_REQUIRED_FIELDS = [ + 'depends_on', + 'id', + 'title', + 'status', + 'wave', + 'plan', + 'requirements', + 'acceptance_criteria', + 'system_design', +]; +const MAX_DOCUMENT_BYTES = 256 * 1024; +const MAX_DOCUMENTS = 100; +const MAX_TOTAL_DOCUMENT_BYTES = 4 * 1024 * 1024; +const MAX_RESPONSE_BYTES = 8 * 1024 * 1024; +const MAX_CHANGED_FILES = 3000; +const REQUEST_TIMEOUT_MS = 30_000; +const NO_DOCS_LABEL = 'no-docs-allow'; +const STATUS_CONTEXT = 'PR documentation coverage'; +const GITHUB_API = 'https://api.github.com'; + +function normalizeRepoPath(value) { + if (typeof value !== 'string' || value.length === 0) { + throw new Error('repository path must be a non-empty string'); + } + if (value.includes('\0') || value.includes('\\') || value.startsWith('/')) { + throw new Error(`unsafe repository path: ${value}`); + } + + const normalized = POSIX_PATH.normalize(value); + if ( + normalized === '.' || + normalized === '..' || + normalized.startsWith('../') || + normalized.includes('/../') + ) { + throw new Error(`repository path escapes the repository: ${value}`); + } + return normalized; +} + +function pathExemption(pathname) { + if (pathname === 'docs' || pathname.startsWith('docs/')) { + return 'documentation tree'; + } + if (/\.(?:md|mdx|markdown)$/i.test(pathname)) { + return 'Markdown file'; + } + if (/_test\.go$/i.test(pathname)) { + return 'Go test file'; + } + if (pathname.startsWith('apps/web/e2e/')) { + return 'web end-to-end test'; + } + + const basename = POSIX_PATH.basename(pathname); + const extension = POSIX_PATH.extname(basename).slice(1).toLowerCase(); + if ( + /\.(?:test|spec)\.[^.]+$/i.test(basename) && + SCRIPT_TEST_EXTENSIONS.has(extension) + ) { + return 'JavaScript or TypeScript test file'; + } + if (/^apps\/web\/src\/locales\/[^/]+\/[^/]+\.json$/.test(pathname)) { + return 'web translation catalog'; + } + if (LOCK_BASENAMES.has(basename)) { + return 'dependency lock file'; + } + return null; +} + +function changedPaths(changedFiles) { + const paths = []; + const invalidPaths = []; + const seen = new Set(); + for (const change of changedFiles ?? []) { + if (typeof change !== 'string' && typeof change?.filename !== 'string') { + invalidPaths.push('[missing filename]'); + continue; + } + const candidates = typeof change === 'string' + ? [change] + : [change.filename, change.previous_filename]; + for (const candidate of candidates) { + if (candidate == null) { + continue; + } + try { + const normalized = normalizeRepoPath(candidate); + if (!seen.has(normalized)) { + seen.add(normalized); + paths.push(normalized); + } + } catch (error) { + invalidPaths.push(String(candidate)); + } + } + } + return { paths, invalidPaths }; +} + +function classifyChangedFiles(changedFiles) { + const { paths, invalidPaths } = changedPaths(changedFiles); + const exemptPaths = []; + const triggeringPaths = [...invalidPaths]; + const reasons = invalidPaths.map(pathname => ({ + path: pathname, + reason: pathname === '[missing filename]' ? 'invalid changed-file record' : 'invalid repository path', + })); + + for (const pathname of paths) { + const reason = pathExemption(pathname); + if (reason == null) { + triggeringPaths.push(pathname); + reasons.push({ path: pathname, reason: 'not in the exemption list' }); + } else { + exemptPaths.push(pathname); + } + } + + return { + changedPaths: paths, + exemptPaths, + invalidPaths, + reasons, + requiresCoverage: triggeringPaths.length > 0, + triggeringPaths, + }; +} + +function hasContentChanges(change) { + if (change?.status !== 'renamed') { + return change?.status !== 'removed'; + } + if (Number.isFinite(change?.changes)) { + return change.changes > 0; + } + return Number(change?.additions ?? 0) + Number(change?.deletions ?? 0) > 0; +} + +function selectChangedWorkOrders(changedFiles) { + const selected = []; + const seen = new Set(); + for (const change of changedFiles ?? []) { + const pathname = change?.filename ?? change; + if (typeof pathname !== 'string') { + continue; + } + let normalized; + try { + normalized = normalizeRepoPath(pathname); + } catch { + continue; + } + if (WORK_ORDER_PATTERN.test(normalized) && hasContentChanges(change)) { + if (!seen.has(normalized)) { + seen.add(normalized); + selected.push(normalized); + } + } + } + return selected; +} + +function parseScalar(value, field) { + const trimmed = value.trim(); + if (trimmed.length === 0) { + throw new Error(`frontmatter field ${field} must have a value`); + } + if (trimmed.startsWith('!') || trimmed.startsWith('&') || trimmed.startsWith('*')) { + throw new Error(`frontmatter field ${field} uses unsupported YAML syntax`); + } + if (trimmed.startsWith('"')) { + if (!trimmed.endsWith('"')) { + throw new Error(`frontmatter field ${field} has an unterminated string`); + } + const parsed = JSON.parse(trimmed); + if (typeof parsed !== 'string') { + throw new Error(`frontmatter field ${field} must be a string`); + } + return parsed; + } + if (trimmed.startsWith("'")) { + if (!trimmed.endsWith("'")) { + throw new Error(`frontmatter field ${field} has an unterminated string`); + } + return trimmed.slice(1, -1).replaceAll("''", "'"); + } + if (/[[\]{}|>]/.test(trimmed)) { + throw new Error(`frontmatter field ${field} uses unsupported YAML syntax`); + } + if (/^-?\d+$/.test(trimmed)) { + return Number(trimmed); + } + return trimmed; +} + +function parseInlineArray(value, field) { + const trimmed = value.trim(); + if (trimmed === '[]') { + return []; + } + if (!trimmed.startsWith('[') || !trimmed.endsWith(']')) { + throw new Error(`frontmatter field ${field} must be a list`); + } + const inner = trimmed.slice(1, -1).trim(); + if (inner.length === 0) { + return []; + } + const values = []; + let current = ''; + let quote = null; + let escaped = false; + for (const character of inner) { + if (quote !== null) { + current += character; + if (escaped) { + escaped = false; + } else if (character === '\\' && quote === '"') { + escaped = true; + } else if (character === quote) { + quote = null; + } + } else if (character === '"' || character === "'") { + quote = character; + current += character; + } else if (character === ',') { + values.push(parseScalar(current, field)); + current = ''; + } else { + current += character; + } + } + if (quote !== null) { + throw new Error(`frontmatter field ${field} has an unterminated string`); + } + values.push(parseScalar(current, field)); + return values; +} + +function parseFrontmatter(source) { + if (typeof source !== 'string' || source.length === 0) { + throw new Error('document is empty'); + } + const lines = source.split(/\r?\n/); + if (lines[0] !== '---') { + throw new Error('frontmatter is missing'); + } + + const values = {}; + let currentList = null; + let closingIndex = -1; + for (let index = 1; index < lines.length; index += 1) { + const line = lines[index]; + if (line === '---') { + closingIndex = index; + break; + } + if (line.trim().length === 0 || line.trim().startsWith('#')) { + continue; + } + if (/^\s/.test(line)) { + if (!currentList || !/^\s+-\s+/.test(line)) { + throw new Error(`frontmatter line ${index + 1} is not a supported list item`); + } + const item = line.replace(/^\s+-\s+/, ''); + values[currentList].push(parseScalar(item, currentList)); + continue; + } + + const match = /^([A-Za-z_][A-Za-z0-9_-]*):(?:[ \t]*(.*))?$/.exec(line); + if (!match) { + throw new Error(`frontmatter line ${index + 1} is invalid`); + } + const [, field, rawValue = ''] = match; + if (!FRONTMATTER_FIELDS.has(field)) { + throw new Error(`frontmatter field ${field} is unsupported`); + } + if (Object.hasOwn(values, field)) { + throw new Error(`frontmatter field ${field} is duplicated`); + } + if (ARRAY_FIELDS.has(field)) { + if (rawValue.trim().length === 0) { + values[field] = []; + currentList = field; + } else { + values[field] = parseInlineArray(rawValue, field); + currentList = null; + } + } else { + values[field] = parseScalar(rawValue, field); + currentList = null; + } + } + if (closingIndex < 0) { + throw new Error('frontmatter closing delimiter is missing'); + } + return { + body: lines.slice(closingIndex + 1).join('\n'), + data: values, + frontmatter: lines.slice(1, closingIndex).join('\n'), + }; +} + +function contentEntries(fileContents) { + if (fileContents instanceof Map) { + return [...fileContents.entries()]; + } + if (fileContents && typeof fileContents === 'object') { + return Object.entries(fileContents); + } + return []; +} + +function contentFor(fileContents, pathname, deletedPaths) { + if (deletedPaths.has(pathname)) { + return undefined; + } + if (fileContents instanceof Map) { + return fileContents.has(pathname) ? fileContents.get(pathname) : undefined; + } + return Object.hasOwn(fileContents ?? {}, pathname) ? fileContents[pathname] : undefined; +} + +function resolveReference(basePath, reference) { + if (typeof reference !== 'string' || reference.trim().length === 0) { + throw new Error(`reference from ${basePath} is empty`); + } + if (reference.includes('\\') || reference.startsWith('/')) { + throw new Error(`reference from ${basePath} is not repository-relative: ${reference}`); + } + const resolved = normalizeRepoPath(POSIX_PATH.join(POSIX_PATH.dirname(basePath), reference)); + return resolved; +} + +function hasMarkdownLinkTo(planPath, body, targetPath) { + const linkPattern = /\[[^\]]+\]\(\s*(?:<([^>]+)>|([^\s)]+))/g; + for (const match of body.matchAll(linkPattern)) { + const destination = (match[1] ?? match[2]).split(/[?#]/, 1)[0]; + if (destination.length === 0) { + continue; + } + try { + if (resolveReference(planPath, destination) === targetPath) { + return true; + } + } catch { + // Invalid links are not links to the selected work order. + } + } + return false; +} + +function asNonEmptyString(value, field, pathname) { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`${pathname} frontmatter field ${field} must be a non-empty string`); + } + return value; +} + +function asNonEmptyStringList(value, field, pathname) { + if (!Array.isArray(value) || value.length === 0) { + throw new Error(`${pathname} frontmatter field ${field} must be a non-empty list`); + } + if (value.some(item => typeof item !== 'string' || item.trim().length === 0)) { + throw new Error(`${pathname} frontmatter field ${field} contains an invalid item`); + } + return value; +} + +function acceptanceCriteriaByRequirement(criteria, requirements, pathname, errors) { + const referencedRequirements = new Set(requirements); + const criteriaByRequirement = new Map(); + for (const criterion of criteria) { + const match = /^AC-(.+)\.\d+$/.exec(criterion); + const requirementId = match ? `REQ-${match[1]}` : null; + if (!requirementId || !referencedRequirements.has(requirementId)) { + errors.push(`${criterion} does not belong to a referenced requirement in ${pathname}`); + continue; + } + const ownedCriteria = criteriaByRequirement.get(requirementId) ?? []; + ownedCriteria.push(criterion); + criteriaByRequirement.set(requirementId, ownedCriteria); + } + return criteriaByRequirement; +} + +function requirementHeadingPattern(requirementId) { + return new RegExp(`^#{1,6}\\s+${escapeRegExp(requirementId)}(?:\\s|:|$)`, 'm'); +} + +function escapeRegExp(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +function requirementDefinitions(fileContents, requirementId, system) { + const prefix = `docs/specs/${system}/requirements/`; + return contentEntries(fileContents) + .filter(([pathname, content]) => { + return ( + pathname.startsWith(prefix) && + pathname.endsWith('.md') && + typeof content === 'string' && + requirementHeadingPattern(requirementId).test(content) + ); + }) + .map(([pathname, content]) => ({ pathname, content })); +} + +function acceptanceCriteriaForRequirement(content, requirementId) { + const lines = content.split(/\r?\n/); + const heading = new RegExp(`^#{1,6}\\s+${escapeRegExp(requirementId)}(?:\\s|:|$)`); + const start = lines.findIndex(line => heading.test(line)); + if (start < 0) { + return []; + } + const level = (lines[start].match(/^#+/) ?? [''])[0].length; + const section = []; + for (let index = start + 1; index < lines.length; index += 1) { + const nextHeading = lines[index].match(/^(#+)\s/); + if (nextHeading && nextHeading[1].length <= level) { + break; + } + section.push(lines[index]); + } + return [...new Set((section.join('\n').match(/\bAC-[A-Z0-9-]+\.\d+\b/g) ?? []))]; +} + +function deletedRepositoryPaths(changedFiles) { + const deleted = new Set(); + for (const change of changedFiles ?? []) { + if (change?.status === 'removed' || change?.status === 'renamed') { + const pathname = change?.previous_filename; + if (typeof pathname === 'string') { + try { + deleted.add(normalizeRepoPath(pathname)); + } catch { + // Invalid changed paths are reported by classifyChangedFiles. + } + } + } + } + return deleted; +} + +function validateWorkOrder(workOrderPath, fileContents, deletedPaths) { + const errors = []; + const acceptedReferences = []; + const workOrderContent = contentFor(fileContents, workOrderPath, deletedPaths); + if (typeof workOrderContent !== 'string' || workOrderContent.trim().length === 0) { + return { + acceptedReferences, + errors: [`${workOrderPath} is missing or empty`], + }; + } + if (Buffer.byteLength(workOrderContent, 'utf8') > MAX_DOCUMENT_BYTES) { + return { + acceptedReferences, + errors: [`${workOrderPath} exceeds the ${MAX_DOCUMENT_BYTES}-byte document limit`], + }; + } + + let workOrder; + try { + workOrder = parseFrontmatter(workOrderContent); + for (const field of WORK_ORDER_REQUIRED_FIELDS) { + if (!Object.hasOwn(workOrder.data, field)) { + throw new Error(`${workOrderPath} is missing frontmatter field ${field}`); + } + } + asNonEmptyString(workOrder.data.id, 'id', workOrderPath); + asNonEmptyString(workOrder.data.title, 'title', workOrderPath); + asNonEmptyString(workOrder.data.status, 'status', workOrderPath); + if (!Number.isInteger(workOrder.data.wave) || workOrder.data.wave < 0) { + throw new Error(`${workOrderPath} frontmatter field wave must be a non-negative integer`); + } + if (!Array.isArray(workOrder.data.depends_on)) { + throw new Error(`${workOrderPath} frontmatter field depends_on must be a list`); + } + asNonEmptyStringList(workOrder.data.requirements, 'requirements', workOrderPath); + asNonEmptyStringList( + workOrder.data.acceptance_criteria, + 'acceptance_criteria', + workOrderPath, + ); + asNonEmptyStringList(workOrder.data.system_design, 'system_design', workOrderPath); + } catch (error) { + return { acceptedReferences, errors: [error.message] }; + } + const criteriaByRequirement = acceptanceCriteriaByRequirement( + workOrder.data.acceptance_criteria, + workOrder.data.requirements, + workOrderPath, + errors, + ); + + let planPath; + try { + planPath = resolveReference(workOrderPath, workOrder.data.plan); + const expectedPlanPath = `${POSIX_PATH.dirname(workOrderPath)}/plan.md`; + if (planPath !== expectedPlanPath) { + throw new Error( + `${workOrderPath} plan reference resolves to ${planPath}; expected sibling plan.md`, + ); + } + } catch (error) { + errors.push(error.message); + } + + if (planPath) { + const planContent = contentFor(fileContents, planPath, deletedPaths); + if (typeof planContent !== 'string' || planContent.trim().length === 0) { + errors.push(`${workOrderPath} references missing or empty ${planPath}`); + } else if (Buffer.byteLength(planContent, 'utf8') > MAX_DOCUMENT_BYTES) { + errors.push(`${planPath} exceeds the ${MAX_DOCUMENT_BYTES}-byte document limit`); + } else { + try { + const plan = parseFrontmatter(planContent); + if (!hasMarkdownLinkTo(planPath, plan.body, workOrderPath)) { + errors.push(`${planPath} does not link back to ${workOrderPath}`); + } + const planRequirements = Array.isArray(plan.data.requirements) + ? plan.data.requirements + : []; + for (const requirementId of workOrder.data.requirements) { + if (!planRequirements.includes(requirementId)) { + errors.push(`${planPath} does not cover requirement ${requirementId}`); + } + } + const planDesignPaths = new Set(); + for (const reference of Array.isArray(plan.data.system_design) + ? plan.data.system_design + : []) { + try { + planDesignPaths.add(resolveReference(planPath, reference)); + } catch (error) { + errors.push(error.message); + } + } + const declaredRequirements = new Set(); + for (const reference of workOrder.data.system_design) { + try { + const designPath = resolveReference(workOrderPath, reference); + if (!planDesignPaths.has(designPath)) { + errors.push(`${planPath} does not cover system design ${designPath}`); + } + const designContent = contentFor(fileContents, designPath, deletedPaths); + if (typeof designContent !== 'string' || designContent.trim().length === 0) { + errors.push(`${workOrderPath} references missing or empty ${designPath}`); + continue; + } + if (Buffer.byteLength(designContent, 'utf8') > MAX_DOCUMENT_BYTES) { + errors.push(`${designPath} exceeds the ${MAX_DOCUMENT_BYTES}-byte document limit`); + continue; + } + const design = parseFrontmatter(designContent); + const designMatch = /^docs\/specs\/([^/]+)\/system-design\/[^/]+\.md$/.exec( + designPath, + ); + if (!designMatch) { + errors.push(`${designPath} is outside the supported system-design directory`); + continue; + } + const system = designMatch[1]; + const designRequirements = Array.isArray(design.data.requirements) + ? design.data.requirements + : []; + const ownedRequirements = workOrder.data.requirements.filter(requirementId => + designRequirements.includes(requirementId) + ); + for (const requirementId of ownedRequirements) { + declaredRequirements.add(requirementId); + const definitions = requirementDefinitions(fileContents, requirementId, system); + if (definitions.length === 0) { + errors.push(`${requirementId} is not defined in ${system} requirements`); + } else if (definitions.length > 1) { + errors.push(`${requirementId} has ambiguous definitions in ${system} requirements`); + } else { + const definedCriteria = acceptanceCriteriaForRequirement( + definitions[0].content, + requirementId, + ); + for (const criterion of criteriaByRequirement.get(requirementId) ?? []) { + if (!definedCriteria.includes(criterion)) { + errors.push( + `${criterion} is not defined under ${requirementId} in ${definitions[0].pathname}`, + ); + } + } + } + } + acceptedReferences.push({ designPath, requirements: ownedRequirements }); + } catch (error) { + errors.push(error.message); + } + } + for (const requirementId of workOrder.data.requirements) { + if (!declaredRequirements.has(requirementId)) { + errors.push( + `${workOrderPath} requirement ${requirementId} is not declared by any referenced system design`, + ); + } + } + } catch (error) { + errors.push(`${planPath}: ${error.message}`); + } + } + } + + return { acceptedReferences, errors }; +} + +function validateCoverage({ changedFiles = [], fileContents = {} } = {}) { + const classification = classifyChangedFiles(changedFiles); + const result = { + acceptedReferences: [], + changedPaths: classification.changedPaths, + errors: [], + exemptPaths: classification.exemptPaths, + ok: false, + remediation: [], + requiresCoverage: classification.requiresCoverage, + status: 'error', + triggeringPaths: classification.triggeringPaths, + workOrders: [], + }; + + if (!classification.requiresCoverage) { + result.ok = true; + result.status = 'exempt'; + result.remediation = ['No delivery package is required for these changed paths.']; + return result; + } + for (const pathname of classification.invalidPaths) { + result.errors.push(`Changed path ${pathname} is invalid`); + } + + result.workOrders = selectChangedWorkOrders(changedFiles); + if (result.workOrders.length > MAX_DOCUMENTS) { + result.errors.push(`More than ${MAX_DOCUMENTS} changed work orders were supplied`); + } + if (result.workOrders.length === 0) { + result.status = 'missing'; + result.errors.push( + 'No changed work order covers the triggering paths. Add docs/plans//task-NN-.md with linked plan, requirements, acceptance criteria, and system design.', + ); + } + + const deletedPaths = deletedRepositoryPaths(changedFiles); + let totalBytes = 0; + for (const [pathname] of contentEntries(fileContents)) { + const content = contentFor(fileContents, pathname, deletedPaths); + if (typeof content === 'string') { + totalBytes += Buffer.byteLength(content, 'utf8'); + } + } + if (totalBytes > MAX_TOTAL_DOCUMENT_BYTES) { + result.errors.push(`Referenced documents exceed the ${MAX_TOTAL_DOCUMENT_BYTES}-byte total limit`); + } + for (const workOrderPath of result.workOrders) { + const validation = validateWorkOrder( + workOrderPath, + fileContents, + deletedPaths, + ); + result.acceptedReferences.push(...validation.acceptedReferences); + result.errors.push(...validation.errors); + } + if (result.errors.length === 0) { + result.ok = true; + result.status = 'covered'; + result.remediation = ['Keep the linked work order and referenced contracts available at this revision.']; + } else { + result.status = result.status === 'missing' ? 'missing' : 'invalid'; + result.remediation = [ + 'Add or correct a changed work order and its complete plan, requirement, acceptance, and system-design references.', + ]; + } + return result; +} + +function requireCommitSha(value, description) { + if (typeof value !== 'string' || !/^[0-9a-f]{40}$/i.test(value)) { + throw new Error(`${description} is missing or invalid`); + } + return value; +} + +function requireBranchName(value, description) { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`${description} is missing or invalid`); + } + const branch = value.startsWith('refs/heads/') ? value.slice('refs/heads/'.length) : value; + if ( + branch.length === 0 + || branch.includes('\0') + || /[\r\n]/.test(branch) + || branch.startsWith('refs/') + ) { + throw new Error(`${description} is missing or invalid`); + } + return branch; +} + +function requirePullRequestNumber(value) { + if (!Number.isInteger(value) || value < 1) { + throw new Error('pull-request number is missing or invalid'); + } + return value; +} + +class GitHubClient { + constructor({ owner, repo, token, fetchImpl = globalThis.fetch } = {}) { + if (typeof owner !== 'string' || typeof repo !== 'string' || owner === '' || repo === '') { + throw new Error('GitHub repository owner and name are required'); + } + if (typeof token !== 'string' || token.length === 0) { + throw new Error('GitHub token is required'); + } + if (typeof fetchImpl !== 'function') { + throw new Error('fetch implementation is required'); + } + this.owner = owner; + this.repo = repo; + this.token = token; + this.fetchImpl = fetchImpl; + } + + async request(endpoint, { method = 'GET', body } = {}) { + if (typeof endpoint !== 'string' || !endpoint.startsWith('/')) { + throw new Error('GitHub endpoint must be an absolute API path'); + } + const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${this.token}`, + 'X-GitHub-Api-Version': '2022-11-28', + }; + if (body !== undefined) { + headers['Content-Type'] = 'application/json'; + } + const timeoutSignal = typeof AbortSignal === 'function' + && typeof AbortSignal.timeout === 'function' + ? AbortSignal.timeout(REQUEST_TIMEOUT_MS) + : undefined; + let response; + try { + response = await this.fetchImpl(`${GITHUB_API}${endpoint}`, { + method, + headers, + body: body === undefined ? undefined : JSON.stringify(body), + ...(timeoutSignal ? { signal: timeoutSignal } : {}), + }); + } catch (error) { + if (error?.name === 'TimeoutError' || error?.name === 'AbortError') { + throw new Error(`GitHub API request timed out after ${REQUEST_TIMEOUT_MS}ms`); + } + throw error; + } + if (!response || typeof response.text !== 'function') { + throw new Error('GitHub API returned an invalid response'); + } + const text = await response.text(); + if (Buffer.byteLength(text, 'utf8') > MAX_RESPONSE_BYTES) { + throw new Error(`GitHub API response exceeds the ${MAX_RESPONSE_BYTES}-byte limit`); + } + let payload = null; + if (text.length > 0) { + try { + payload = JSON.parse(text); + } catch { + throw new Error('GitHub API returned invalid JSON'); + } + } + if (!response.ok) { + const detail = + payload && typeof payload.message === 'string' ? `: ${payload.message.slice(0, 200)}` : ''; + throw new Error(`GitHub API request failed with HTTP ${response.status}${detail}`); + } + return payload; + } + + async getPullRequest(number) { + requirePullRequestNumber(number); + const pullRequest = await this.request( + `/repos/${encodeURIComponent(this.owner)}/${encodeURIComponent(this.repo)}/pulls/${number}`, + ); + if (!pullRequest || pullRequest.number !== number) { + throw new Error('GitHub returned a pull request with the wrong number'); + } + if (typeof pullRequest.head?.sha !== 'string' || typeof pullRequest.base?.sha !== 'string') { + throw new Error(`pull request #${number} has incomplete revision metadata`); + } + requireCommitSha(pullRequest.head.sha, `pull request #${number} head revision`); + requireCommitSha(pullRequest.base.sha, `pull request #${number} base revision`); + if ( + !Number.isInteger(pullRequest.changed_files) || + pullRequest.changed_files < 0 || + pullRequest.changed_files > MAX_CHANGED_FILES + ) { + throw new Error(`pull request #${number} has an invalid changed-file count`); + } + if (!Array.isArray(pullRequest.labels)) { + throw new Error(`pull request #${number} has incomplete label metadata`); + } + return pullRequest; + } + + async listFiles(number, expectedCount) { + requirePullRequestNumber(number); + const files = []; + for (let page = 1; page <= 30; page += 1) { + const pageFiles = await this.request( + `/repos/${encodeURIComponent(this.owner)}/${encodeURIComponent(this.repo)}/pulls/${number}/files?per_page=100&page=${page}`, + ); + if (!Array.isArray(pageFiles)) { + throw new Error('GitHub changed-file response is not a list'); + } + for (const file of pageFiles) { + if (!file || typeof file.filename !== 'string' || file.filename.length === 0) { + throw new Error('GitHub changed-file entry has no filename'); + } + normalizeRepoPath(file.filename); + if (!['added', 'changed', 'copied', 'deleted', 'modified', 'removed', 'renamed', 'unchanged'].includes(file.status)) { + throw new Error(`GitHub changed-file entry for ${file.filename} has no supported status`); + } + if (file.status === 'renamed') { + if (typeof file.previous_filename !== 'string' || file.previous_filename.length === 0) { + throw new Error(`GitHub renamed file ${file.filename} has no previous filename`); + } + normalizeRepoPath(file.previous_filename); + } + } + files.push(...pageFiles); + if (files.length >= MAX_CHANGED_FILES) { + throw new Error("GitHub changed-file response reaches the 3,000-file limit"); + } + if (pageFiles.length < 100) { + if (expectedCount !== undefined && expectedCount !== files.length) { + throw new Error( + `GitHub changed-file count ${files.length} does not match pull request metadata ${expectedCount}`, + ); + } + return files; + } + } + throw new Error('GitHub changed-file pagination exceeded the supported limit'); + } + + async getFile(pathname, ref) { + const normalized = normalizeRepoPath(pathname); + if (typeof ref !== 'string' || ref.length === 0) { + throw new Error(`content reference for ${normalized} is missing`); + } + const encodedPath = normalized.split('/').map(encodeURIComponent).join('/'); + const response = await this.request( + `/repos/${encodeURIComponent(this.owner)}/${encodeURIComponent(this.repo)}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + if (typeof response?.path !== 'string' || normalizeRepoPath(response.path) !== normalized) { + throw new Error(`${normalized} response has a mismatched returned path`); + } + if (!response || response.type !== 'file' || response.encoding !== 'base64') { + throw new Error(`${normalized} is not a regular base64-encoded file`); + } + if (typeof response.content !== 'string') { + throw new Error(`${normalized} has no file content`); + } + const encoded = response.content.replace(/\s/g, ''); + if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded)) { + throw new Error(`${normalized} has invalid base64 content`); + } + const bytes = Buffer.from(encoded, 'base64'); + if (bytes.length > MAX_DOCUMENT_BYTES) { + throw new Error(`${normalized} exceeds the ${MAX_DOCUMENT_BYTES}-byte document limit`); + } + try { + return new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + throw new Error(`${normalized} is not valid UTF-8 text`); + } + } + + async listDirectory(pathname, ref) { + const normalized = normalizeRepoPath(pathname); + const encodedPath = normalized.split('/').map(encodeURIComponent).join('/'); + const response = await this.request( + `/repos/${encodeURIComponent(this.owner)}/${encodeURIComponent(this.repo)}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + if (!Array.isArray(response)) { + throw new Error(`${normalized} is not a directory listing`); + } + return response.map(entry => { + if (!entry || typeof entry.path !== 'string') { + throw new Error(`${normalized} contains an invalid directory entry`); + } + const entryPath = normalizeRepoPath(entry.path); + if (!['file', 'directory'].includes(entry.type)) { + throw new Error(`${entryPath} is not a regular file or directory`); + } + return { path: entryPath, type: entry.type }; + }); + } + + async searchCode(text, directory) { + if (typeof text !== 'string' || text.trim().length === 0) { + throw new Error('GitHub code-search text is missing'); + } + const normalizedDirectory = normalizeRepoPath(directory); + const query = `"${text}" repo:${this.owner}/${this.repo} path:${normalizedDirectory}`; + const response = await this.request( + `/search/code?q=${encodeURIComponent(query)}&per_page=100`, + ); + if ( + !response + || !Array.isArray(response.items) + || response.incomplete_results === true + || !Number.isInteger(response.total_count) + || response.total_count < response.items.length + || response.total_count > 100 + ) { + throw new Error('GitHub code-search response is incomplete or exceeds the supported limit'); + } + const prefix = `${normalizedDirectory}/`; + const paths = new Set(); + for (const item of response.items) { + if (typeof item?.path !== 'string') { + throw new Error('GitHub code-search entry has no path'); + } + const itemPath = normalizeRepoPath(item.path); + if (!itemPath.startsWith(prefix) || !itemPath.endsWith('.md')) { + throw new Error(`GitHub code-search returned an unsupported path: ${itemPath}`); + } + paths.add(itemPath); + } + return [...paths]; + } + + async createCommitStatus(sha, status) { + requireCommitSha(sha, 'status revision'); + if (!status || !['pending', 'success', 'failure', 'error'].includes(status.state)) { + throw new Error('commit status has an invalid state'); + } + return this.request( + `/repos/${encodeURIComponent(this.owner)}/${encodeURIComponent(this.repo)}/statuses/${sha}`, + { + method: 'POST', + body: { + state: status.state, + context: STATUS_CONTEXT, + description: String(status.description ?? '').slice(0, 140), + target_url: status.targetUrl, + }, + }, + ); + } + + async graphql(query, variables) { + const response = await this.request('/graphql', { + method: 'POST', + body: { query, variables }, + }); + if (Array.isArray(response?.errors) && response.errors.length > 0) { + throw new Error(`GitHub GraphQL request failed: ${String(response.errors[0].message ?? 'unknown error')}`); + } + return response?.data; + } + + async listMergeQueueEntries(branch) { + const targetBranch = requireBranchName(branch, 'merge-queue target branch'); + const query = ` + query($owner: String!, $repo: String!, $branch: String!, $after: String) { + repository(owner: $owner, name: $repo) { + mergeQueue(branch: $branch) { + entries(first: 100, after: $after) { + nodes { + baseCommit { oid } + headCommit { oid } + pullRequest { number headRefOid } + } + pageInfo { hasNextPage endCursor } + } + } + } + } + `; + const entries = []; + let after = null; + for (let page = 0; page < 10; page += 1) { + const data = await this.graphql(query, { + branch: targetBranch, + owner: this.owner, + repo: this.repo, + after, + }); + const connection = data?.repository?.mergeQueue?.entries; + if (!connection || !Array.isArray(connection.nodes) || !connection.pageInfo) { + throw new Error('GitHub merge queue response is incomplete'); + } + entries.push(...connection.nodes); + if (!connection.pageInfo.hasNextPage) { + return entries; + } + if (typeof connection.pageInfo.endCursor !== 'string') { + throw new Error('GitHub merge queue pagination cursor is missing'); + } + after = connection.pageInfo.endCursor; + } + throw new Error('GitHub merge queue pagination exceeded the supported limit'); + } +} + +function isNoDocsOverride(pullRequest) { + return Array.isArray(pullRequest?.labels) + && pullRequest.labels.some(label => label?.name === NO_DOCS_LABEL); +} + +function resultMetadata(pullRequest) { + return { + baseSha: pullRequest.base.sha, + baseRef: pullRequest.base.ref, + headSha: pullRequest.head.sha, + labels: pullRequest.labels.map(label => label?.name).filter(name => typeof name === 'string').sort(), + }; +} + +function sameMetadata(left, right) { + return ( + left.headSha === right.headSha && + left.baseSha === right.baseSha && + left.baseRef === right.baseRef && + JSON.stringify(left.labels) === JSON.stringify(right.labels) + ); +} + +function errorCoverageResult(message, metadata = {}) { + return { + acceptedReferences: [], + changedPaths: [], + errors: [message], + exemptPaths: [], + baseRef: metadata.baseRef, + headSha: metadata.headSha, + ok: false, + remediation: ['Retry the workflow after GitHub returns complete, stable revision and label data.'], + requiresCoverage: true, + status: 'error', + triggeringPaths: [], + workOrders: [], + }; +} + +function isMissingResourceError(error) { + return /\bHTTP 404\b/.test(String(error?.message ?? error)); +} + +async function loadCoverageContents({ client, changedFiles, headSha }) { + if (!classifyChangedFiles(changedFiles).requiresCoverage) { + return {}; + } + const contents = {}; + const loaded = new Set(); + let documentCount = 0; + let totalBytes = 0; + async function load(pathname) { + const normalized = normalizeRepoPath(pathname); + if (loaded.has(normalized)) { + return contents[normalized]; + } + if (documentCount >= MAX_DOCUMENTS) { + throw new Error(`Referenced documents exceed the ${MAX_DOCUMENTS}-document limit`); + } + let content; + try { + content = await client.getFile(normalized, headSha); + } catch (error) { + if (!isMissingResourceError(error)) { + throw error; + } + } + loaded.add(normalized); + documentCount += 1; + if (typeof content !== 'string' || content.trim().length === 0) { + contents[normalized] = undefined; + return undefined; + } + const size = Buffer.byteLength(content, 'utf8'); + if (size > MAX_DOCUMENT_BYTES) { + throw new Error(`${normalized} exceeds the ${MAX_DOCUMENT_BYTES}-byte document limit`); + } + totalBytes += size; + if (totalBytes > MAX_TOTAL_DOCUMENT_BYTES) { + throw new Error(`Referenced documents exceed the ${MAX_TOTAL_DOCUMENT_BYTES}-byte total limit`); + } + contents[normalized] = content; + return content; + } + + for (const workOrderPath of selectChangedWorkOrders(changedFiles)) { + const workOrderContent = await load(workOrderPath); + if (workOrderContent === undefined) { + continue; + } + let workOrder; + try { + workOrder = parseFrontmatter(workOrderContent).data; + } catch { + continue; + } + let planPath; + try { + planPath = resolveReference(workOrderPath, workOrder.plan); + } catch { + continue; + } + const planContent = await load(planPath); + if (planContent === undefined) { + continue; + } + let plan; + try { + plan = parseFrontmatter(planContent).data; + } catch { + continue; + } + const designReferences = [ + ...(Array.isArray(workOrder.system_design) ? workOrder.system_design : []), + ...(Array.isArray(plan.system_design) ? plan.system_design : []), + ]; + const designPaths = new Set(); + for (const reference of designReferences) { + try { + designPaths.add(resolveReference(workOrderPath, reference)); + } catch { + // The validator reports the invalid reference; do not read outside the root. + } + } + for (const designPath of designPaths) { + const designContent = await load(designPath); + if (designContent === undefined) { + continue; + } + let system; + try { + parseFrontmatter(designContent); + const match = /^docs\/specs\/([^/]+)\/system-design\/[^/]+\.md$/.exec(designPath); + if (!match) { + continue; + } + system = match[1]; + } catch { + continue; + } + const requirementDirectory = `docs/specs/${system}/requirements`; + const requirementPaths = new Set(); + const changedRepositoryPaths = changedPaths(changedFiles).paths; + for (const pathname of changedRepositoryPaths) { + if (pathname.startsWith(`${requirementDirectory}/`) && pathname.endsWith('.md')) { + requirementPaths.add(pathname); + } + } + const unresolvedRequirementIds = []; + if (typeof client.searchCode === 'function') { + for (const requirementId of workOrder.requirements ?? []) { + const matches = await client.searchCode(requirementId, requirementDirectory); + if (matches.length === 0) { + unresolvedRequirementIds.push(requirementId); + } + for (const pathname of matches) { + requirementPaths.add(pathname); + } + } + } else { + unresolvedRequirementIds.push(...(workOrder.requirements ?? [])); + } + if (unresolvedRequirementIds.length > 0 && typeof client.listDirectory === 'function') { + let entries = []; + try { + entries = await client.listDirectory(requirementDirectory, headSha); + } catch (error) { + if (!isMissingResourceError(error)) { + throw error; + } + } + const candidateNames = new Set(unresolvedRequirementIds.map(requirementId => { + const parts = requirementId.split('-'); + return `${parts.slice(2).join('-').replace(/-\d+$/, '').toLowerCase()}.md`; + })); + for (const entry of entries) { + if (entry.type === 'file' && candidateNames.has(POSIX_PATH.basename(entry.path))) { + requirementPaths.add(entry.path); + } + } + } + for (const pathname of requirementPaths) { + const requirementContent = await load(pathname); + if (requirementContent === undefined) { + continue; + } + } + } + } + return contents; +} + +async function evaluatePullRequestSnapshot({ client, pullRequest, expectedHeadSha }) { + if (expectedHeadSha && pullRequest.head.sha !== expectedHeadSha) { + return errorCoverageResult( + `pull request #${pullRequest.number} head ${pullRequest.head.sha} does not match expected queue head ${expectedHeadSha}`, + { headSha: pullRequest.head.sha }, + ); + } + const metadata = { + baseSha: pullRequest.base.sha, + baseRef: pullRequest.base.ref, + headSha: pullRequest.head.sha, + pullRequestNumber: pullRequest.number, + }; + if (isNoDocsOverride(pullRequest)) { + return { + acceptedReferences: [], + changedPaths: [], + errors: [], + exemptPaths: [], + ...metadata, + ok: true, + override: NO_DOCS_LABEL, + remediation: ['The no-docs-allow label is present. Remove it to restore normal evaluation.'], + requiresCoverage: false, + status: 'override', + triggeringPaths: [], + workOrders: [], + }; + } + const changedFiles = await client.listFiles(pullRequest.number, pullRequest.changed_files); + const fileContents = await loadCoverageContents({ + changedFiles, + client, + headSha: pullRequest.head.sha, + }); + return { + ...validateCoverage({ changedFiles, fileContents }), + ...metadata, + }; +} + +async function evaluatePullRequest({ client, pullNumber, expectedHeadSha, maxAttempts = 3 } = {}) { + requirePullRequestNumber(pullNumber); + if (!Number.isInteger(maxAttempts) || maxAttempts < 1 || maxAttempts > 5) { + throw new Error('maxAttempts must be an integer from 1 through 5'); + } + let lastMetadata; + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + let pullRequest; + try { + pullRequest = await client.getPullRequest(pullNumber); + const before = resultMetadata(pullRequest); + if (expectedHeadSha && pullRequest.head.sha !== expectedHeadSha) { + return errorCoverageResult( + `pull request #${pullNumber} head does not match the expected revision`, + before, + ); + } + const result = await evaluatePullRequestSnapshot({ + client, + expectedHeadSha, + pullRequest, + }); + const latest = await client.getPullRequest(pullNumber); + const after = resultMetadata(latest); + if (sameMetadata(before, after)) { + return result; + } + lastMetadata = after; + } catch (error) { + return errorCoverageResult(error.message, lastMetadata ?? {}); + } + } + return errorCoverageResult( + `pull request #${pullNumber} changed during evaluation; retry limit reached`, + lastMetadata ?? {}, + ); +} + +function validateMergeQueueEntries(entries) { + if (!Array.isArray(entries) || entries.length === 0) { + throw new Error('merge-group membership is missing'); + } + return entries.map((entry, index) => { + const entryBaseSha = requireCommitSha(entry?.baseCommit?.oid, `merge-queue entry ${index} base`); + const entryHeadSha = requireCommitSha(entry?.headCommit?.oid, `merge-queue entry ${index} head`); + const number = requirePullRequestNumber(entry?.pullRequest?.number); + if (entryBaseSha === entryHeadSha) { + throw new Error(`merge-queue entry ${number} has identical base and head revisions`); + } + return { + baseSha: entryBaseSha, + headSha: entryHeadSha, + number, + pullRequest: entry.pullRequest, + raw: entry, + }; + }); +} + +function resolveMergeGroupMembers({ baseSha, headSha, entries } = {}) { + requireCommitSha(baseSha, 'merge-group base revision'); + requireCommitSha(headSha, 'merge-group head revision'); + const normalizedEntries = validateMergeQueueEntries(entries); + + const members = []; + const usedEntries = new Set(); + const usedPullRequests = new Set(); + let current = headSha; + while (current !== baseSha) { + const candidates = normalizedEntries.filter(entry => entry.headSha === current); + if (candidates.length === 0) { + throw new Error(`merge-group boundary ${current} is not connected to its base`); + } + if (candidates.length > 1) { + throw new Error(`merge-group boundary ${current} has ambiguous membership`); + } + const entry = candidates[0]; + if (usedEntries.has(entry) || usedPullRequests.has(entry.number)) { + throw new Error('merge-group membership contains a cycle or duplicate pull request'); + } + usedEntries.add(entry); + usedPullRequests.add(entry.number); + members.push(entry); + current = entry.baseSha; + if (members.length > normalizedEntries.length) { + throw new Error('merge-group membership exceeds the available queue entries'); + } + } + if (members.length === 0) { + throw new Error('merge-group contains no pull requests'); + } + members.reverse(); + return { baseSha, headSha, members }; +} + +async function evaluateMergeGroup({ client, baseSha, headSha, entries, maxAttempts = 3 } = {}) { + const group = resolveMergeGroupMembers({ baseSha, entries, headSha }); + const memberResults = []; + for (const member of group.members) { + const expectedHeadSha = requireCommitSha( + member.pullRequest?.headRefOid, + `merge-queue member ${member.number} pull-request head`, + ); + const result = await evaluatePullRequest({ + client, + expectedHeadSha, + maxAttempts, + pullNumber: member.number, + }); + memberResults.push({ + ...result, + baseSha: member.baseSha, + expectedHeadSha, + number: member.number, + }); + } + const errors = memberResults.flatMap(result => result.errors ?? []); + return { + baseSha, + errors, + headSha, + memberResults, + ok: memberResults.every(result => result.ok), + status: memberResults.some(result => result.status === 'error') + ? 'error' + : memberResults.every(result => result.ok) + ? 'success' + : 'failure', + }; +} + +function findAffectedMergeGroups({ entries, pullRequestNumber } = {}) { + if (!Array.isArray(entries) || entries.length === 0) { + return []; + } + if (pullRequestNumber !== undefined) { + requirePullRequestNumber(pullRequestNumber); + } + const normalizedEntries = validateMergeQueueEntries(entries); + const headRevisions = new Set(normalizedEntries.map(entry => entry.headSha)); + const terminalEntries = normalizedEntries.filter(entry => !normalizedEntries.some( + other => other.baseSha === entry.headSha, + )); + if (terminalEntries.length === 0) { + throw new Error('merge-queue entries do not have a terminal group boundary'); + } + + const groups = []; + const consumed = new Set(); + for (const terminal of terminalEntries) { + const chain = []; + const chainEntries = new Set(); + let current = terminal.headSha; + while (true) { + const candidates = normalizedEntries.filter(entry => entry.headSha === current); + if (candidates.length === 0) { + throw new Error(`merge-group boundary ${current} is not connected to any queue entry`); + } + if (candidates.length > 1) { + throw new Error(`merge-group boundary ${current} has ambiguous membership`); + } + const entry = candidates[0]; + if (chainEntries.has(entry) || consumed.has(entry)) { + throw new Error('merge-queue entries overlap or contain a cycle'); + } + chainEntries.add(entry); + chain.unshift(entry); + if (!headRevisions.has(entry.baseSha)) { + break; + } + current = entry.baseSha; + } + for (const entry of chain) { + consumed.add(entry); + } + for (let prefixLength = 1; prefixLength <= chain.length; prefixLength += 1) { + const prefix = chain.slice(0, prefixLength); + if ( + pullRequestNumber !== undefined && + !prefix.some(entry => entry.number === pullRequestNumber) + ) { + continue; + } + groups.push({ + baseSha: prefix[0].baseSha, + entries: prefix.map(entry => entry.raw), + headSha: prefix[prefix.length - 1].headSha, + }); + } + } + if (consumed.size !== normalizedEntries.length) { + throw new Error('merge-queue membership contains an unconnected entry'); + } + return groups; +} + +function statusState(result) { + if (result?.status === 'error') { + return 'error'; + } + return result?.ok ? 'success' : 'failure'; +} + +function markdownValue(value) { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", ''') + .replaceAll('`', '`') + .replace(/[\r\n]/g, ' '); +} + +function resultSummary(result) { + const lines = [`## PR documentation coverage`, `Result: **${markdownValue(result.status)}**`]; + if (result.override) { + lines.push(`Override: \`${markdownValue(result.override)}\``); + } + const reportedPaths = result.triggeringPaths?.length > 0 + ? result.triggeringPaths + : result.changedPaths ?? []; + if (reportedPaths.length > 0) { + lines.push('', result.triggeringPaths?.length > 0 ? 'Triggering paths:' : 'Changed paths:'); + for (const pathname of reportedPaths) { + lines.push(`- \`${markdownValue(pathname)}\``); + } + } + if (result.workOrders?.length > 0) { + lines.push('', 'Accepted work orders:'); + for (const pathname of result.workOrders) { + lines.push(`- \`${markdownValue(pathname)}\``); + } + } + if (result.acceptedReferences?.length > 0) { + lines.push('', 'Accepted references:'); + for (const reference of result.acceptedReferences) { + lines.push(`- \`${markdownValue(reference.designPath)}\``); + } + } + if (result.errors?.length > 0) { + lines.push('', 'Problems:'); + for (const error of result.errors) { + lines.push(`- ${markdownValue(error)}`); + } + } + if (result.remediation?.length > 0) { + lines.push('', 'Next steps:'); + for (const remediation of result.remediation) { + lines.push(`- ${markdownValue(remediation)}`); + } + } + if (result.memberResults?.length > 0) { + lines.push('', 'Merge-group members:'); + for (const member of result.memberResults) { + lines.push(`- #${member.number}: **${markdownValue(member.status)}**`); + } + } + return `${lines.join('\n')}\n`; +} + +function statusDescription(result) { + if (result.override) { + return `Override: ${NO_DOCS_LABEL}`; + } + if (result.status === 'exempt') { + return 'All changed paths are exempt'; + } + if (result.status === 'covered') { + return 'Linked delivery package found'; + } + if (result.status === 'missing') { + return 'Linked delivery package is missing'; + } + if (result.status === 'invalid') { + return 'Linked delivery package is incomplete'; + } + if (result.status === 'success') { + return 'Every merge-group member satisfies documentation coverage'; + } + if (result.status === 'failure') { + return 'A merge-group member does not satisfy documentation coverage'; + } + return 'Coverage evaluation failed'; +} + +function runUrl(env) { + if (!env.GITHUB_SERVER_URL || !env.GITHUB_REPOSITORY || !env.GITHUB_RUN_ID) { + return undefined; + } + return `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${encodeURIComponent(env.GITHUB_RUN_ID)}`; +} + +function eventPullRequestNumber(event) { + const value = event?.pull_request?.number ?? event?.inputs?.pr_number; + const number = typeof value === 'string' ? Number(value) : value; + return requirePullRequestNumber(number); +} + +async function writeRunSummary(summary, env, writeSummary) { + if (typeof writeSummary === 'function') { + await writeSummary(summary); + return; + } + if (typeof env.GITHUB_STEP_SUMMARY === 'string' && env.GITHUB_STEP_SUMMARY.length > 0) { + fs.appendFileSync(env.GITHUB_STEP_SUMMARY, summary, 'utf8'); + } +} + +async function publishResult(client, sha, result, targetUrl) { + await client.createCommitStatus(sha, { + description: statusDescription(result), + state: statusState(result), + targetUrl, + }); +} + +async function evaluateAffectedGroups({ client, pullNumber, targetUrl, entries }) { + const groups = findAffectedMergeGroups({ entries, pullRequestNumber: pullNumber }); + const groupResults = []; + for (const group of groups) { + await client.createCommitStatus(group.headSha, { + description: 'Evaluating merge-group documentation coverage', + state: 'pending', + targetUrl, + }); + const result = await evaluateMergeGroup({ + baseSha: group.baseSha, + client, + entries: group.entries, + headSha: group.headSha, + }); + await publishResult(client, group.headSha, result, targetUrl); + groupResults.push(result); + } + return groupResults; +} + +async function run({ client, env = process.env, event, eventName, writeSummary } = {}) { + const effectiveEventName = eventName ?? env.GITHUB_EVENT_NAME; + const effectiveEvent = event ?? (() => { + if (typeof env.GITHUB_EVENT_PATH !== 'string' || env.GITHUB_EVENT_PATH.length === 0) { + throw new Error('GitHub event payload path is missing'); + } + return JSON.parse(fs.readFileSync(env.GITHUB_EVENT_PATH, 'utf8')); + })(); + if (!['pull_request_target', 'workflow_dispatch', 'merge_group'].includes(effectiveEventName)) { + throw new Error(`unsupported GitHub event: ${effectiveEventName}`); + } + + let apiClient = client; + if (!apiClient) { + const repository = /^([^/]+)\/([^/]+)$/.exec(env.GITHUB_REPOSITORY ?? ''); + if (!repository) { + throw new Error('GITHUB_REPOSITORY is missing or invalid'); + } + apiClient = new GitHubClient({ + fetchImpl: globalThis.fetch, + owner: repository[1], + repo: repository[2], + token: env.GITHUB_TOKEN, + }); + } + + const targetUrl = runUrl(env); + let pendingSha; + let result; + try { + if (effectiveEventName === 'merge_group') { + const targetBranch = requireBranchName( + effectiveEvent.merge_group?.base_ref, + 'merge-group target branch', + ); + const baseSha = requireCommitSha( + effectiveEvent.merge_group?.base_sha, + 'merge-group base revision', + ); + const headSha = requireCommitSha( + effectiveEvent.merge_group?.head_sha, + 'merge-group head revision', + ); + pendingSha = headSha; + await apiClient.createCommitStatus(headSha, { + description: 'Evaluating merge-group documentation coverage', + state: 'pending', + targetUrl, + }); + const entries = await apiClient.listMergeQueueEntries(targetBranch); + result = await evaluateMergeGroup({ + baseSha, + client: apiClient, + entries, + headSha, + }); + await publishResult(apiClient, headSha, result, targetUrl); + } else { + const pullNumber = eventPullRequestNumber(effectiveEvent); + const current = await apiClient.getPullRequest(pullNumber); + pendingSha = requireCommitSha(current.head.sha, 'pull-request head revision'); + await apiClient.createCommitStatus(pendingSha, { + description: 'Evaluating pull-request documentation coverage', + state: 'pending', + targetUrl, + }); + result = await evaluatePullRequest({ + client: apiClient, + pullNumber, + }); + await publishResult(apiClient, result.headSha ?? pendingSha, result, targetUrl); + + if (effectiveEventName !== 'workflow_dispatch') { + const action = effectiveEvent.action; + if (action === 'labeled' || action === 'unlabeled') { + const targetBranch = requireBranchName( + result.baseRef ?? current.base?.ref, + 'pull-request target branch', + ); + const entries = await apiClient.listMergeQueueEntries(targetBranch); + const groupResults = await evaluateAffectedGroups({ + client: apiClient, + entries, + pullNumber, + targetUrl, + }); + if (groupResults.some(groupResult => !groupResult.ok)) { + result = { + ...result, + affectedGroups: groupResults, + errors: [ + ...(result.errors ?? []), + 'An affected merge group does not satisfy documentation coverage.', + ], + ok: false, + status: 'failure', + }; + } + } + } + } + } catch (error) { + result = errorCoverageResult(error.message, { headSha: pendingSha }); + if (pendingSha) { + try { + await publishResult(apiClient, pendingSha, result, targetUrl); + } catch { + // The original error remains the authoritative failure for the job. + } + } + } + + await writeRunSummary(resultSummary(result), env, writeSummary); + return { exitCode: result.ok ? 0 : 1, result }; +} + +if (require.main === module) { + run() + .then(({ exitCode }) => { + process.exitCode = exitCode; + }) + .catch(error => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; + }); +} + +module.exports = { + classifyChangedFiles, + evaluateMergeGroup, + evaluatePullRequest, + findAffectedMergeGroups, + GitHubClient, + isNoDocsOverride, + normalizeRepoPath, + parseFrontmatter, + resolveMergeGroupMembers, + resultSummary, + run, + selectChangedWorkOrders, + statusState, + validateCoverage, +}; diff --git a/.github/scripts/pr-docs.test.cjs b/.github/scripts/pr-docs.test.cjs new file mode 100644 index 0000000000..603eb6c968 --- /dev/null +++ b/.github/scripts/pr-docs.test.cjs @@ -0,0 +1,1209 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); + +const MODULE_PATH = path.join(__dirname, 'pr-docs.cjs'); +const validator = require(MODULE_PATH); + +test('documentation coverage validator exposes the path and artifact APIs', () => { + assert.equal(fs.existsSync(MODULE_PATH), true); + assert.equal(typeof validator.classifyChangedFiles, 'function'); + assert.equal(typeof validator.validateCoverage, 'function'); + assert.equal(typeof validator.parseFrontmatter, 'function'); + assert.equal(typeof validator.evaluatePullRequest, 'function'); + assert.equal(typeof validator.GitHubClient, 'function'); + assert.equal(typeof validator.resolveMergeGroupMembers, 'function'); + assert.equal(typeof validator.findAffectedMergeGroups, 'function'); + assert.equal(typeof validator.run, 'function'); +}); + +// @covers AC-CI-PR-DOCS-001.2 +test('recognized documentation-only paths are exempt', () => { + const result = validator.classifyChangedFiles([ + { filename: 'docs/guide.txt', status: 'modified' }, + { filename: 'README.md', status: 'modified' }, + { filename: 'notes/guide.markdown', status: 'modified' }, + { filename: 'apps/backend/worker_test.go', status: 'modified' }, + { filename: 'apps/web/lib/worker.test.ts', status: 'modified' }, + { filename: 'apps/web/lib/worker.spec.jsx', status: 'modified' }, + { filename: 'apps/web/e2e/tasks/example.ts', status: 'modified' }, + { filename: 'apps/web/src/locales/en/common.json', status: 'modified' }, + { filename: 'vendor/pnpm-lock.yaml', status: 'modified' }, + ]); + + assert.equal(result.requiresCoverage, false); + assert.deepEqual(result.triggeringPaths, []); + assert.equal(result.exemptPaths.length, 9); +}); + +// @covers AC-CI-PR-DOCS-001.3 +test('shipped files and unsupported test-like paths require coverage', () => { + const result = validator.classifyChangedFiles([ + { filename: '.github/workflows/release.yml', status: 'modified' }, + { filename: 'apps/web/package.json', status: 'modified' }, + { filename: 'apps/web/lib/fixture.test.json', status: 'modified' }, + { filename: 'apps/backend/worker_test.go.txt', status: 'modified' }, + { filename: 'apps/web/src/locales/en/common.yaml', status: 'modified' }, + { filename: 'apps/web/src/locales/en/nested/common.json', status: 'modified' }, + { filename: 'apps/web/src/runtime.ts', status: 'modified' }, + ]); + + assert.equal(result.requiresCoverage, true); + assert.deepEqual(result.triggeringPaths, [ + '.github/workflows/release.yml', + 'apps/web/package.json', + 'apps/web/lib/fixture.test.json', + 'apps/backend/worker_test.go.txt', + 'apps/web/src/locales/en/common.yaml', + 'apps/web/src/locales/en/nested/common.json', + 'apps/web/src/runtime.ts', + ]); +}); + +test('malformed changed-file records fail closed', () => { + const result = validator.classifyChangedFiles([{}]); + assert.equal(result.requiresCoverage, true); + assert.deepEqual(result.invalidPaths, ['[missing filename]']); + assert.equal(result.reasons[0].reason, 'invalid changed-file record'); +}); + +test('renames classify both old and new paths and pure work-order renames do not qualify', () => { + const result = validator.classifyChangedFiles([ + { + filename: 'docs/runtime-recovery.md', + previous_filename: 'apps/backend/runtime-recovery.go', + status: 'renamed', + additions: 0, + deletions: 0, + changes: 0, + }, + ]); + assert.deepEqual(result.changedPaths, [ + 'docs/runtime-recovery.md', + 'apps/backend/runtime-recovery.go', + ]); + assert.deepEqual(result.triggeringPaths, ['apps/backend/runtime-recovery.go']); + + assert.deepEqual( + validator.selectChangedWorkOrders([ + { + filename: 'docs/plans/recovery/task-01-recovery.md', + previous_filename: 'docs/plans/old/task-01-recovery.md', + status: 'renamed', + additions: 0, + deletions: 0, + changes: 0, + }, + ]), + [], + ); + assert.deepEqual( + validator.selectChangedWorkOrders([ + { + filename: 'docs/plans/recovery/task-01-recovery.md', + previous_filename: 'docs/plans/old/task-01-recovery.md', + status: 'renamed', + additions: 1, + deletions: 1, + changes: 2, + }, + ]), + ['docs/plans/recovery/task-01-recovery.md'], + ); +}); + +test('frontmatter parser accepts the documented subset and rejects unsafe syntax', () => { + const parsed = validator.parseFrontmatter(`--- +id: "01-recovery" +title: "Recover worktrees" +status: pending +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-CI-PR-DOCS-001 +acceptance_criteria: + - AC-CI-PR-DOCS-001.3 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +--- + +# Work order +`); + assert.deepEqual(parsed.data.requirements, ['REQ-CI-PR-DOCS-001']); + assert.equal(parsed.data.wave, 1); + + const singleQuotedBackslash = validator.parseFrontmatter( + `---\nrequirements: ['a${String.fromCharCode(92)}']\n---\n`, + ); + assert.deepEqual(singleQuotedBackslash.data.requirements, [`a${String.fromCharCode(92)}`]); + + for (const source of [ + '---\nunknown: value\n---\n', + '---\nid: first\nid: second\n---\n', + '---\nrequirements: !!js/function evil\n---\n', + 'not-frontmatter', + ]) { + assert.throws(() => validator.parseFrontmatter(source)); + } +}); + +function fixtureContents(overrides = {}) { + const files = { + 'docs/plans/recovery/plan.md': `--- +status: draft +requirements: + - REQ-CI-PR-DOCS-001 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +--- + +# Recovery plan + +- [ ] [Task 01: Recover worktrees](task-01-recovery.md) +`, + 'docs/plans/recovery/task-01-recovery.md': `--- +id: "01-recovery" +title: "Recover worktrees" +status: pending +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-CI-PR-DOCS-001 +acceptance_criteria: + - AC-CI-PR-DOCS-001.3 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +--- + +# Task 01 + +## Scope + +- \`apps/backend/runtime-recovery.go\` +`, + 'docs/specs/ci/system-design/pull-request-documentation-coverage.md': `--- +status: draft +system: ci +requirements: + - REQ-CI-PR-DOCS-001 +--- + +# Design + +REQ-CI-PR-DOCS-001 +`, + 'docs/specs/ci/requirements/pull-request-documentation-coverage.md': `--- +status: draft +system: ci +--- + +### REQ-CI-PR-DOCS-001 + +- **AC-CI-PR-DOCS-001.3:** Runtime changes have a work order. +`, + 'apps/backend/runtime-recovery.go': 'package runtime\n', + }; + return { ...files, ...overrides }; +} + +function runtimeAndWorkOrderDiff(overrides = []) { + return [ + { filename: 'apps/backend/runtime-recovery.go', status: 'modified' }, + { + filename: 'docs/plans/recovery/task-01-recovery.md', + status: 'modified', + additions: 3, + changes: 3, + }, + ...overrides, + ]; +} + +function uiFixtureContents() { + const contents = fixtureContents(); + contents['docs/plans/recovery/plan.md'] = contents['docs/plans/recovery/plan.md'] + .replaceAll('REQ-CI-PR-DOCS-001', 'REQ-UI-COVERAGE-001') + .replaceAll( + '../../specs/ci/system-design/pull-request-documentation-coverage.md', + '../../specs/ui/system-design/ui-coverage.md', + ); + contents['docs/plans/recovery/task-01-recovery.md'] = contents[ + 'docs/plans/recovery/task-01-recovery.md' + ] + .replaceAll('REQ-CI-PR-DOCS-001', 'REQ-UI-COVERAGE-001') + .replaceAll('AC-CI-PR-DOCS-001.3', 'AC-UI-COVERAGE-001.3') + .replaceAll( + '../../specs/ci/system-design/pull-request-documentation-coverage.md', + '../../specs/ui/system-design/ui-coverage.md', + ); + delete contents['docs/specs/ci/system-design/pull-request-documentation-coverage.md']; + delete contents['docs/specs/ci/requirements/pull-request-documentation-coverage.md']; + contents['docs/specs/ui/system-design/ui-coverage.md'] = `--- +status: current +system: ui +requirements: + - REQ-UI-COVERAGE-001 +--- + +# UI coverage design +`; + contents['docs/specs/ui/requirements/ui-coverage.md'] = `--- +status: current +system: ui +--- + +### REQ-UI-COVERAGE-001 + +- **AC-UI-COVERAGE-001.3:** Runtime changes have a work order. +`; + return contents; +} + +function multiDesignFixtureContents() { + const contents = fixtureContents(); + contents['docs/plans/recovery/plan.md'] = `--- +status: draft +requirements: + - REQ-PLATFORM-DIAGNOSTIC-LOGGING-001 + - REQ-WEB-BROWSER-RETENTION-001 +system_design: + - ../../specs/platform/system-design/logging.md + - ../../specs/web/system-design/retention.md +--- + +# Recovery plan + +- [ ] [Task 01: Recover worktrees](task-01-recovery.md) +`; + contents['docs/plans/recovery/task-01-recovery.md'] = `--- +id: "01-recovery" +title: "Recover worktrees" +status: pending +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLATFORM-DIAGNOSTIC-LOGGING-001 + - REQ-WEB-BROWSER-RETENTION-001 +acceptance_criteria: + - AC-PLATFORM-DIAGNOSTIC-LOGGING-001.1 + - AC-WEB-BROWSER-RETENTION-001.1 +system_design: + - ../../specs/platform/system-design/logging.md + - ../../specs/web/system-design/retention.md +--- + +# Task 01 +`; + delete contents['docs/specs/ci/system-design/pull-request-documentation-coverage.md']; + delete contents['docs/specs/ci/requirements/pull-request-documentation-coverage.md']; + contents['docs/specs/platform/system-design/logging.md'] = `--- +status: current +system: platform +requirements: + - REQ-PLATFORM-DIAGNOSTIC-LOGGING-001 +--- + +# Diagnostic logging design +`; + contents['docs/specs/web/system-design/retention.md'] = `--- +status: current +system: web +requirements: + - REQ-WEB-BROWSER-RETENTION-001 +--- + +# Browser retention design +`; + contents['docs/specs/platform/requirements/diagnostic-logging.md'] = `### REQ-PLATFORM-DIAGNOSTIC-LOGGING-001 + +- **AC-PLATFORM-DIAGNOSTIC-LOGGING-001.1:** Runtime changes have a work order. +`; + contents['docs/specs/web/requirements/browser-retention.md'] = `### REQ-WEB-BROWSER-RETENTION-001 + +- **AC-WEB-BROWSER-RETENTION-001.1:** Runtime changes have a work order. +`; + return contents; +} + +// @covers AC-CI-PR-DOCS-001.3, AC-CI-PR-DOCS-001.4 +test('valid linked work order covers a runtime change without editing existing contracts', () => { + const result = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: fixtureContents(), + }); + + assert.equal(result.ok, true); + assert.equal(result.status, 'covered'); + assert.deepEqual(result.workOrders, ['docs/plans/recovery/task-01-recovery.md']); + assert.deepEqual(result.errors, []); +}); + +test('multi-requirement work orders map acceptance criteria to their owning requirement', () => { + const contents = fixtureContents(); + contents['docs/plans/recovery/plan.md'] = contents['docs/plans/recovery/plan.md'].replace( + ' - REQ-CI-PR-DOCS-001\nsystem_design:', + ' - REQ-CI-PR-DOCS-001\n - REQ-CI-PR-DOCS-002\nsystem_design:', + ); + contents['docs/plans/recovery/task-01-recovery.md'] = contents[ + 'docs/plans/recovery/task-01-recovery.md' + ].replace( + ' - REQ-CI-PR-DOCS-001\nacceptance_criteria:\n - AC-CI-PR-DOCS-001.3', + ' - REQ-CI-PR-DOCS-001\n - REQ-CI-PR-DOCS-002\nacceptance_criteria:\n - AC-CI-PR-DOCS-001.3\n - AC-CI-PR-DOCS-002.1', + ); + contents['docs/specs/ci/system-design/pull-request-documentation-coverage.md'] = contents[ + 'docs/specs/ci/system-design/pull-request-documentation-coverage.md' + ].replace( + ' - REQ-CI-PR-DOCS-001\n---', + ' - REQ-CI-PR-DOCS-001\n - REQ-CI-PR-DOCS-002\n---', + ); + contents['docs/specs/ci/requirements/pull-request-documentation-coverage.md'] += ` + +### REQ-CI-PR-DOCS-002: Explicit documentation exception + +- **AC-CI-PR-DOCS-002.1:** The exact exception label passes coverage. +`; + + const result = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: contents, + }); + + assert.equal(result.ok, true, result.errors.join('; ')); + assert.deepEqual(result.errors, []); +}); + +test('multi-design work orders validate each requirement against its owning design', () => { + const result = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: multiDesignFixtureContents(), + }); + + assert.equal(result.ok, true, result.errors.join('; ')); + assert.deepEqual(result.errors, []); +}); + +test('multi-design work orders fail when a requirement has no owning design', () => { + const contents = multiDesignFixtureContents(); + contents['docs/specs/web/system-design/retention.md'] = contents[ + 'docs/specs/web/system-design/retention.md' + ].replace(' - REQ-WEB-BROWSER-RETENTION-001\n', ''); + + const result = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: contents, + }); + + assert.equal(result.ok, false); + assert.equal( + result.errors.some(error => + error.includes('REQ-WEB-BROWSER-RETENTION-001') + && error.includes('not declared by any referenced system design') + ), + true, + result.errors.join('; '), + ); +}); + +test('linked plans and designs accept existing update metadata', () => { + const contents = fixtureContents(); + contents['docs/plans/recovery/plan.md'] = contents['docs/plans/recovery/plan.md'].replace( + 'status: draft', + 'status: draft\nupdated: 2026-09-09', + ); + contents['docs/specs/ci/system-design/pull-request-documentation-coverage.md'] = contents[ + 'docs/specs/ci/system-design/pull-request-documentation-coverage.md' + ].replace( + 'status: draft', + 'status: draft\nlast_updated: 2026-09-09', + ); + + const result = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: contents, + }); + + assert.equal(result.ok, true, result.errors.join('; ')); + assert.deepEqual(result.errors, []); +}); + +// @covers AC-CI-PR-DOCS-001.5, AC-CI-PR-DOCS-001.6 +test('missing, empty, deleted, escaping, and mismatched references fail precisely', () => { + const baseContents = fixtureContents(); + const cases = [ + { + name: 'missing plan', + overrides: { 'docs/plans/recovery/plan.md': undefined }, + expected: 'plan.md', + }, + { + name: 'empty work order', + overrides: { 'docs/plans/recovery/task-01-recovery.md': '' }, + expected: 'empty', + }, + { + name: 'missing dependency list', + overrides: { + 'docs/plans/recovery/task-01-recovery.md': baseContents[ + 'docs/plans/recovery/task-01-recovery.md' + ].replace('depends_on: []\n', ''), + }, + expected: 'depends_on', + }, + { + name: 'deleted design', + overrides: { + 'docs/specs/ci/system-design/pull-request-documentation-coverage.md': undefined, + }, + expected: 'system-design', + }, + { + name: 'escaping plan reference', + overrides: { + 'docs/plans/recovery/task-01-recovery.md': baseContents[ + 'docs/plans/recovery/task-01-recovery.md' + ].replace('plan: "plan.md"', 'plan: "../../outside.md"'), + }, + expected: 'outside.md', + }, + { + name: 'acceptance criterion belongs to another requirement', + overrides: { + 'docs/specs/ci/requirements/pull-request-documentation-coverage.md': baseContents[ + 'docs/specs/ci/requirements/pull-request-documentation-coverage.md' + ].replace('REQ-CI-PR-DOCS-001', 'REQ-CI-PR-DOCS-999'), + }, + expected: 'REQ-CI-PR-DOCS-001', + }, + { + name: 'work-order basename without a Markdown link', + overrides: { + 'docs/plans/recovery/plan.md': baseContents['docs/plans/recovery/plan.md'].replace( + '- [ ] [Task 01: Recover worktrees](task-01-recovery.md)', + 'The task-01-recovery.md file is tracked here.', + ), + }, + expected: 'does not link back', + }, + ]; + + for (const { name, overrides, expected } of cases) { + const result = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: fixtureContents(overrides), + }); + assert.equal(result.ok, false, name); + assert.equal( + result.errors.some(error => error.includes(expected)), + true, + `${name}: ${result.errors.join('; ')}`, + ); + } +}); + +test('ambiguous requirement definitions and missing work orders fail closed', () => { + const ambiguous = validator.validateCoverage({ + changedFiles: runtimeAndWorkOrderDiff(), + fileContents: fixtureContents({ + 'docs/specs/ci/requirements/duplicate.md': + '### REQ-CI-PR-DOCS-001\n\n- **AC-CI-PR-DOCS-001.3:** duplicate\n', + }), + }); + assert.equal(ambiguous.ok, false); + assert.equal(ambiguous.errors.some(error => error.includes('ambiguous')), true); + + const noWorkOrder = validator.validateCoverage({ + changedFiles: [{ filename: 'apps/backend/runtime-recovery.go', status: 'modified' }], + fileContents: fixtureContents(), + }); + assert.equal(noWorkOrder.ok, false); + assert.equal(noWorkOrder.status, 'missing'); + assert.equal(noWorkOrder.errors.some(error => error.includes('work order')), true); +}); + +const SHA_A = 'a'.repeat(40); +const SHA_B = 'b'.repeat(40); +const SHA_C = 'c'.repeat(40); +const SHA_D = 'd'.repeat(40); +const SHA_E = 'e'.repeat(40); + +function pullRequest(number, headSha, labels = [], changedFiles = 1) { + return { + number, + state: 'open', + draft: false, + changed_files: changedFiles, + head: { sha: headSha }, + base: { sha: SHA_A, ref: 'main' }, + labels: labels.map(name => ({ name })), + }; +} + +// @covers AC-CI-PR-DOCS-002.1, AC-CI-PR-DOCS-002.4 +test('exact no-docs-allow label passes before changed-file reads', async () => { + let listFilesCalls = 0; + const client = { + async getPullRequest() { + return pullRequest(42, SHA_B, ['no-docs-allow']); + }, + async listFiles() { + listFilesCalls += 1; + throw new Error('changed files should not be read for an override'); + }, + }; + + const result = await validator.evaluatePullRequest({ client, pullNumber: 42 }); + assert.equal(result.ok, true); + assert.equal(result.status, 'override'); + assert.equal(result.override, 'no-docs-allow'); + assert.equal(listFilesCalls, 0); +}); + +test('documentation-only changes do not load delivery artifacts', async () => { + let getFileCalls = 0; + const client = { + async getPullRequest() { + return pullRequest(42, SHA_B); + }, + async listFiles() { + return [{ + filename: 'docs/plans/recovery/task-01-recovery.md', + status: 'modified', + }]; + }, + async getFile() { + getFileCalls += 1; + throw new Error('documentation-only changes should not load artifacts'); + }, + }; + + const result = await validator.evaluatePullRequest({ client, pullNumber: 42 }); + assert.equal(result.ok, true); + assert.equal(result.status, 'exempt'); + assert.equal(getFileCalls, 0); +}); + +// @covers AC-CI-PR-DOCS-002.2, AC-CI-PR-DOCS-002.3 +test('label removal is reevaluated and does not leave an obsolete override', async () => { + const metadata = [pullRequest(42, SHA_B, ['no-docs-allow']), pullRequest(42, SHA_B)]; + let metadataCalls = 0; + const client = { + async getPullRequest() { + return metadata[Math.min(metadataCalls++, metadata.length - 1)]; + }, + async listFiles() { + return [{ filename: 'apps/backend/runtime.go', status: 'modified' }]; + }, + }; + + const result = await validator.evaluatePullRequest({ client, pullNumber: 42 }); + assert.equal(result.ok, false); + assert.equal(result.status, 'missing'); + assert.equal(metadataCalls >= 2, true); +}); + +// @covers AC-CI-PR-DOCS-003.1, AC-CI-PR-DOCS-003.2 +test('head changes during evaluation fail closed after bounded retries', async () => { + let metadataCalls = 0; + const client = { + async getPullRequest() { + const sha = metadataCalls++ % 2 === 0 ? SHA_B : SHA_C; + return pullRequest(42, sha); + }, + async listFiles() { + return [{ filename: 'apps/backend/runtime.go', status: 'modified' }]; + }, + }; + + const result = await validator.evaluatePullRequest({ client, pullNumber: 42, maxAttempts: 2 }); + assert.equal(result.ok, false); + assert.equal(result.status, 'error'); + assert.equal(result.errors.some(error => error.includes('changed during evaluation')), true); + assert.equal(metadataCalls, 4); +}); + +test('GitHub client rejects incomplete pages and decodes bounded file contents', async () => { + const requests = []; + const responses = [ + { + ok: true, + status: 200, + async text() { + return JSON.stringify([ + { filename: 'apps/backend/runtime.go', status: 'modified' }, + ]); + }, + }, + { + ok: true, + status: 200, + async text() { + return JSON.stringify({ + path: 'docs/plan.md', + type: 'file', + encoding: 'base64', + content: Buffer.from('hello').toString('base64'), + }); + }, + }, + ]; + const client = new validator.GitHubClient({ + owner: 'kdlbs', + repo: 'kandev', + token: 'token-is-not-logged', + fetchImpl: async (url, options) => { + requests.push({ url, options }); + return responses.shift(); + }, + }); + + const files = await client.listFiles(42, 1); + assert.equal(files.length, 1); + assert.equal(await client.getFile('docs/plan.md', SHA_B), 'hello'); + assert.equal(requests[1].options.method, 'GET'); + assert.equal(requests[1].options.headers.Authorization, 'Bearer token-is-not-logged'); +}); + +test('GitHub client rejects malformed changed-file entries and mismatched content paths', async () => { + const responses = [ + { + ok: true, + status: 200, + async text() { + return JSON.stringify([{}]); + }, + }, + { + ok: true, + status: 200, + async text() { + return JSON.stringify({ + path: 'docs/other.md', + type: 'file', + encoding: 'base64', + content: Buffer.from('hello').toString('base64'), + }); + }, + }, + ]; + const client = new validator.GitHubClient({ + owner: 'kdlbs', + repo: 'kandev', + token: 'token', + fetchImpl: async () => responses.shift(), + }); + + await assert.rejects(client.listFiles(42, 1), /filename/); + await assert.rejects(client.getFile('docs/plan.md', SHA_B), /returned path/); +}); + +test('GitHub client scopes merge-queue lookup to the target branch', async () => { + const requests = []; + const client = new validator.GitHubClient({ + owner: 'kdlbs', + repo: 'kandev', + token: 'token', + fetchImpl: async (url, options) => { + requests.push({ url, options }); + return { + ok: true, + status: 200, + async text() { + return JSON.stringify({ + data: { + repository: { + mergeQueue: { + entries: { + nodes: [], + pageInfo: { hasNextPage: false, endCursor: null }, + }, + }, + }, + }, + }); + }, + }; + }, + }); + + assert.deepEqual(await client.listMergeQueueEntries('release/next'), []); + const request = JSON.parse(requests[0].options.body); + assert.match(request.query, /mergeQueue\(branch: \$branch\)/); + assert.equal(request.variables.branch, 'release/next'); +}); + +test('GitHub client converts a request timeout into a bounded error', async () => { + let signal; + const client = new validator.GitHubClient({ + owner: 'kdlbs', + repo: 'kandev', + token: 'token', + fetchImpl: async (_url, options) => { + signal = options.signal; + const error = new Error('request timed out'); + error.name = 'TimeoutError'; + throw error; + }, + }); + + await assert.rejects(client.getPullRequest(42), /timed out/i); + assert.equal(typeof signal?.aborted, 'boolean'); +}); + +test('coverage loading searches only the referenced requirement files', async () => { + const contents = uiFixtureContents(); + const loaded = []; + const searches = []; + const workOrderPath = 'docs/plans/recovery/task-01-recovery.md'; + const changed = [ + { filename: 'apps/web/lib/runtime.ts', status: 'modified' }, + { filename: workOrderPath, status: 'modified', additions: 1, changes: 1 }, + ]; + const client = { + async getPullRequest() { + return pullRequest(42, SHA_B, [], changed.length); + }, + async listFiles() { + return changed; + }, + async getFile(pathname) { + loaded.push(pathname); + if (!Object.hasOwn(contents, pathname)) { + throw new Error('GitHub API request failed with HTTP 404: Not Found'); + } + return contents[pathname]; + }, + async searchCode(requirementId, directory) { + searches.push({ requirementId, directory }); + return ['docs/specs/ui/requirements/ui-coverage.md']; + }, + }; + + const result = await validator.evaluatePullRequest({ client, pullNumber: 42 }); + assert.equal(result.ok, true, result.errors.join('; ')); + assert.deepEqual(searches, [{ + requirementId: 'REQ-UI-COVERAGE-001', + directory: 'docs/specs/ui/requirements', + }]); + assert.deepEqual(loaded, [ + workOrderPath, + 'docs/plans/recovery/plan.md', + 'docs/specs/ui/system-design/ui-coverage.md', + 'docs/specs/ui/requirements/ui-coverage.md', + ]); +}); + +test('missing referenced artifacts become invalid coverage through the API adapter', async () => { + const contents = fixtureContents(); + const workOrderPath = 'docs/plans/recovery/task-01-recovery.md'; + const changed = runtimeAndWorkOrderDiff(); + const client = { + async getPullRequest() { + return pullRequest(42, SHA_B, [], changed.length); + }, + async listFiles() { + return changed; + }, + async getFile(pathname) { + if (pathname === workOrderPath) { + return contents[pathname]; + } + throw new Error('GitHub API request failed with HTTP 404: Not Found'); + }, + }; + + const result = await validator.evaluatePullRequest({ client, pullNumber: 42 }); + assert.equal(result.ok, false); + assert.equal(result.status, 'invalid'); + assert.equal(result.errors.some(error => error.includes('plan.md')), true); +}); + +test('GitHub client rejects a changed-file count at the API cap', async () => { + const client = new validator.GitHubClient({ + owner: 'kdlbs', + repo: 'kandev', + token: 'token', + fetchImpl: async () => ({ + ok: true, + status: 200, + async text() { + return JSON.stringify(Array.from({ length: 100 }, (_, index) => ({ + filename: `apps/runtime-${index}.go`, + status: 'modified', + }))); + }, + }), + }); + + await assert.rejects( + client.listFiles(42, 3000), + /3,000-file limit/, + ); +}); + +test('GitHub pull-request metadata requires a bounded changed-file count', async () => { + const client = new validator.GitHubClient({ + owner: 'kdlbs', + repo: 'kandev', + token: 'token', + fetchImpl: async () => ({ + ok: true, + status: 200, + async text() { + return JSON.stringify({ + number: 42, + head: { sha: SHA_B }, + base: { sha: SHA_A }, + labels: [], + changed_files: 3001, + }); + }, + }), + }); + + await assert.rejects(client.getPullRequest(42), /changed-file count/); +}); + +// @covers AC-CI-PR-DOCS-003.4 +test('merge-group members are resolved from exact entry boundaries', () => { + const entries = [ + { + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_B }, + pullRequest: { number: 1, headRefOid: SHA_B }, + }, + { + baseCommit: { oid: SHA_B }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 2, headRefOid: SHA_C }, + }, + ]; + const result = validator.resolveMergeGroupMembers({ + baseSha: SHA_A, + headSha: SHA_C, + entries, + }); + assert.deepEqual(result.members.map(member => member.number), [1, 2]); + assert.equal(result.baseSha, SHA_A); + assert.equal(result.headSha, SHA_C); + + assert.throws( + () => validator.resolveMergeGroupMembers({ + baseSha: SHA_A, + headSha: SHA_C, + entries: [...entries, { ...entries[1] }], + }), + /ambiguous/, + ); +}); + +// @covers AC-CI-PR-DOCS-003.4 +test('merge-group evaluation keeps each member policy independent', async () => { + const entries = [ + { + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_B }, + pullRequest: { number: 1, headRefOid: SHA_D }, + }, + { + baseCommit: { oid: SHA_B }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 2, headRefOid: SHA_E }, + }, + ]; + const client = { + async getPullRequest(number) { + return number === 1 + ? pullRequest(1, SHA_D, ['no-docs-allow']) + : pullRequest(2, SHA_E); + }, + async listFiles(number) { + return [{ + filename: `apps/backend/member-${number}.go`, + status: 'modified', + }]; + }, + }; + + const result = await validator.evaluateMergeGroup({ + client, + baseSha: SHA_A, + headSha: SHA_C, + entries, + }); + assert.equal(result.ok, false); + assert.deepEqual(result.memberResults.map(member => member.status), ['override', 'missing']); + assert.deepEqual( + result.memberResults.map(member => member.expectedHeadSha), + [SHA_D, SHA_E], + ); +}); + +test('affected merge groups can be found for label-triggered reevaluation', () => { + const entries = [ + { + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_B }, + pullRequest: { number: 1, headRefOid: SHA_B }, + }, + { + baseCommit: { oid: SHA_B }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 2, headRefOid: SHA_C }, + }, + { + baseCommit: { oid: SHA_A }, + headCommit: { oid: 'd'.repeat(40) }, + pullRequest: { number: 3, headRefOid: 'd'.repeat(40) }, + }, + ]; + const groups = validator.findAffectedMergeGroups({ entries, pullRequestNumber: 2 }); + assert.equal(groups.length, 1); + assert.equal(groups[0].baseSha, SHA_A); + assert.equal(groups[0].headSha, SHA_C); + assert.deepEqual(groups[0].entries.map(entry => entry.pullRequest.number), [1, 2]); +}); + +test('label reevaluation includes every queued prefix containing the changed first member', () => { + const entries = [ + { + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_B }, + pullRequest: { number: 1, headRefOid: SHA_B }, + }, + { + baseCommit: { oid: SHA_B }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 2, headRefOid: SHA_C }, + }, + ]; + + const groups = validator.findAffectedMergeGroups({ entries, pullRequestNumber: 1 }); + + assert.deepEqual( + groups.map(group => ({ + baseSha: group.baseSha, + headSha: group.headSha, + members: group.entries.map(entry => entry.pullRequest.number), + })), + [ + { baseSha: SHA_A, headSha: SHA_B, members: [1] }, + { baseSha: SHA_A, headSha: SHA_C, members: [1, 2] }, + ], + ); +}); + +test('label removal reevaluates both prefix and full groups for the first queued member', async () => { + const statuses = []; + const entries = [ + { + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_B }, + pullRequest: { number: 42, headRefOid: SHA_B }, + }, + { + baseCommit: { oid: SHA_B }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 43, headRefOid: SHA_C }, + }, + ]; + const client = { + async getPullRequest(number) { + return number === 42 ? pullRequest(42, SHA_B) : pullRequest(43, SHA_C); + }, + async listFiles(number) { + return [{ + filename: number === 42 ? 'apps/backend/runtime.go' : 'docs/guide.md', + status: 'modified', + }]; + }, + async listMergeQueueEntries(branch) { + assert.equal(branch, 'main'); + return entries; + }, + async createCommitStatus(sha, status) { + statuses.push({ sha, state: status.state }); + }, + }; + + const result = await validator.run({ + client, + env: {}, + event: { action: 'unlabeled', pull_request: { number: 42 } }, + eventName: 'pull_request_target', + writeSummary: () => {}, + }); + + assert.equal(result.exitCode, 1); + assert.deepEqual(result.result.affectedGroups.map(group => group.headSha), [SHA_B, SHA_C]); + assert.deepEqual(statuses.map(status => status.sha), [SHA_B, SHA_B, SHA_B, SHA_B, SHA_C, SHA_C]); +}); + +// @covers AC-CI-PR-DOCS-001.1, AC-CI-PR-DOCS-003.1 +test('run publishes pending and final status for the stable current pull-request head', async () => { + const statuses = []; + const summaries = []; + const client = { + async getPullRequest() { + return pullRequest(42, SHA_B); + }, + async listFiles() { + return [{ filename: 'docs/guide.md', status: 'modified' }]; + }, + async createCommitStatus(sha, status) { + statuses.push({ sha, ...status }); + }, + }; + + const result = await validator.run({ + client, + env: { + GITHUB_REPOSITORY: 'kdlbs/kandev', + GITHUB_RUN_ID: '99', + GITHUB_SERVER_URL: 'https://github.com', + }, + event: { pull_request: { number: 42 } }, + eventName: 'pull_request_target', + writeSummary: summary => summaries.push(summary), + }); + + assert.equal(result.exitCode, 0); + assert.deepEqual(statuses.map(status => status.state), ['pending', 'success']); + assert.deepEqual(statuses.map(status => status.sha), [SHA_B, SHA_B]); + assert.equal(summaries.length, 1); + assert.match(summaries[0], /docs\/guide\.md/); +}); + +test('workflow dispatch reads the pull-request number from its input', async () => { + const statuses = []; + const client = { + async getPullRequest(number) { + assert.equal(number, 99); + return pullRequest(99, SHA_B); + }, + async listFiles() { + return [{ filename: 'docs/guide.md', status: 'modified' }]; + }, + async createCommitStatus(sha, status) { + statuses.push({ sha, state: status.state }); + }, + }; + + const result = await validator.run({ + client, + env: { + GITHUB_REPOSITORY: 'kdlbs/kandev', + GITHUB_RUN_ID: '100', + GITHUB_SERVER_URL: 'https://github.com', + }, + event: { inputs: { pr_number: '99' } }, + eventName: 'workflow_dispatch', + writeSummary: () => {}, + }); + + assert.equal(result.exitCode, 0); + assert.deepEqual(statuses, [ + { sha: SHA_B, state: 'pending' }, + { sha: SHA_B, state: 'success' }, + ]); +}); + +test('merge-group runs publish one status on the synthetic group head', async () => { + const statuses = []; + const client = { + async listMergeQueueEntries() { + return [{ + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 1, headRefOid: SHA_C }, + }]; + }, + async getPullRequest() { + return pullRequest(1, SHA_C); + }, + async listFiles() { + return [{ filename: 'docs/guide.md', status: 'modified' }]; + }, + async createCommitStatus(sha, status) { + statuses.push({ sha, ...status }); + }, + }; + + const result = await validator.run({ + client, + env: {}, + event: { + merge_group: { + base_ref: 'refs/heads/main', + base_sha: SHA_A, + head_sha: SHA_C, + }, + }, + eventName: 'merge_group', + writeSummary: () => {}, + }); + + assert.equal(result.exitCode, 0); + assert.deepEqual(statuses.map(status => status.state), ['pending', 'success']); + assert.deepEqual(statuses.map(status => status.sha), [SHA_C, SHA_C]); + assert.equal(statuses[1].description, 'Every merge-group member satisfies documentation coverage'); +}); + +// @covers AC-CI-PR-DOCS-002.2, AC-CI-PR-DOCS-003.4 +test('label-triggered runs reevaluate affected merge groups independently', async () => { + const statuses = []; + const client = { + async getPullRequest() { + return pullRequest(42, SHA_C); + }, + async listFiles() { + return [{ filename: 'apps/backend/runtime.go', status: 'modified' }]; + }, + async listMergeQueueEntries() { + return [{ + baseCommit: { oid: SHA_A }, + headCommit: { oid: SHA_C }, + pullRequest: { number: 42, headRefOid: SHA_C }, + }]; + }, + async createCommitStatus(sha, status) { + statuses.push({ sha, ...status }); + }, + }; + + const result = await validator.run({ + client, + env: {}, + event: { action: 'unlabeled', pull_request: { number: 42 } }, + eventName: 'pull_request_target', + writeSummary: () => {}, + }); + + assert.equal(result.exitCode, 1); + assert.equal(result.result.affectedGroups.length, 1); + assert.deepEqual(statuses.map(status => status.state), [ + 'pending', + 'failure', + 'pending', + 'failure', + ]); +}); + +test('run summaries escape untrusted paths and error text', () => { + const summary = validator.resultSummary({ + changedPaths: ['bad`path\n- forged item'], + errors: ['bad '], + ok: false, + status: 'invalid', + triggeringPaths: ['bad`path\n- forged item'], + }); + assert.equal(summary.includes('bad`path'), false); + assert.equal(summary.includes('`'), true); + assert.equal(summary.includes('<input>'), true); + assert.equal(summary.includes('\n- forged item'), false); +}); diff --git a/.github/workflows/lint-action-pinning.yml b/.github/workflows/lint-action-pinning.yml index 118ecb0683..206039958f 100644 --- a/.github/workflows/lint-action-pinning.yml +++ b/.github/workflows/lint-action-pinning.yml @@ -93,6 +93,12 @@ jobs: - name: Test pull request size label workflow contract run: python3 .github/scripts/pr-size-label-workflow-contract_test.py + - name: Test pull request documentation workflow contract + run: python3 .github/scripts/pr-docs-workflow-contract_test.py + + - name: Test pull request documentation validator + run: node --test .github/scripts/pr-docs.test.cjs + - name: Test managed runtime pin update workflow contract run: python3 .github/scripts/update-agent-runtime-pins-workflow-contract_test.py diff --git a/.github/workflows/pr-docs.yml b/.github/workflows/pr-docs.yml new file mode 100644 index 0000000000..04f264cd24 --- /dev/null +++ b/.github/workflows/pr-docs.yml @@ -0,0 +1,63 @@ +name: Pull request documentation coverage + +# This workflow reads untrusted pull-request metadata and files as data only. +# The checked-out validator is always from a trusted workflow/base revision. +on: + pull_request_target: # zizmor: ignore[dangerous-triggers] base-controlled metadata-only workflow + types: [opened, reopened, synchronize, edited, labeled, unlabeled, ready_for_review] + merge_group: + types: [checks_requested] + workflow_dispatch: + inputs: + pr_number: + description: Pull request number to reevaluate + required: true + type: number + +concurrency: + # Use the target branch as the shared lock. Label events have only PR + # metadata, while merge-group events have the synthetic group SHA; the + # branch lock prevents either event from publishing over the other. + group: >- + pr-docs-${{ + github.event_name == 'merge_group' && github.event.merge_group.base_ref || + format('refs/heads/{0}', github.event.pull_request.base.ref || github.event.repository.default_branch) + }} + # Keep up to 100 pending events instead of replacing the single pending run. + queue: max + cancel-in-progress: false + +permissions: + contents: read + pull-requests: read + statuses: write + +jobs: + coverage: + name: Publish PR documentation coverage status + if: >- + github.event_name != 'workflow_dispatch' || + github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + CHECKOUT_REF: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.workflow_sha }} + + steps: + - name: Checkout trusted coverage validator + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + ref: ${{ env.CHECKOUT_REF }} + fetch-depth: 1 + persist-credentials: false + + - name: Run documentation coverage evaluator + env: + GITHUB_TOKEN: ${{ github.token }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_EVENT_PATH: ${{ github.event_path }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_STEP_SUMMARY: ${{ github.step_summary }} + run: node .github/scripts/pr-docs.cjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b228ddbc1..94244b77b2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,12 @@ discussed the direction before opening the PR. Link the issue from the PR. If an agent is preparing the change, it must stop and report missing discussion instead of opening the PR. +### Documentation coverage + +When implementing a feature or fix, update the relevant requirements, system +design, plan, and work-order files. CI runs a linter to check PR documentation +coverage. + ## How to Contribute 1. **Fork and branch.** Create a feature branch from `main`. diff --git a/Makefile b/Makefile index 057564bf74..5571441b0f 100644 --- a/Makefile +++ b/Makefile @@ -559,6 +559,7 @@ test-cli: test-scripts: @printf "$(CYAN)Running script tests...$(RESET)\n" @python3 .github/scripts/lint-action-pinning_test.py + @node --test .github/scripts/pr-docs.test.cjs @bash scripts/pr-state.test.sh @bash scripts/pr-await.test.sh @bash scripts/run-quiet.test.sh diff --git a/docs/ci-merge-queue.md b/docs/ci-merge-queue.md index b9a2e853e0..f5dec2c4a7 100644 --- a/docs/ci-merge-queue.md +++ b/docs/ci-merge-queue.md @@ -222,6 +222,39 @@ in the table above. It does not require per-shard names. Do not require `lint` from `pr-title.yml`, `Validate public docs`, review checks, or deployment checks. These checks do not run in every merge group. +### PR documentation coverage rollout + +`pr-docs.yml` also listens for `merge_group: checks_requested` and publishes the +`PR documentation coverage` status on the synthetic merge-group head. It maps +the queue entry boundaries back to the event base and evaluates every included +pull request with its own changed files, work-order references, and labels. A +member's `no-docs-allow` label cannot exempt another member. + +All PR and merge-group events use the target branch as one non-cancelling +concurrency key with `queue: max`. GitHub retains up to 100 pending runs in +that key, so ordinary event bursts do not replace the one pending run; events +that arrive after the bound is full can still be canceled. This keeps a queued +label reevaluation from racing with a merge-group status write while the +published coverage status remains attached to the evaluated PR or synthetic +group revision. A label removal reevaluates every active queue-group prefix +that contains the PR, not only the longest group. + +The status is not in the active required-check list above until live merge-group +evidence is collected. Roll out the requirement in two stages: + +1. Observe ordinary, draft, fork, label-add, label-remove, manual-retry, and + merge-group runs. Confirm that covered, missing, override, and incomplete + responses are attached to the evaluated revision and that all affected queue + group prefixes reevaluate after label removal. +2. Add `PR documentation coverage` as a separate required status through an + administrator ruleset change. Preserve the existing six checks and bypass + rules, then verify a covered PR, an uncovered PR, an override, and a mixed + merge group. + +Until stage two is complete, a failing result is visible but does not itself +block merging. The workflow does not add or remove labels, dequeue PRs, or +change rulesets. + ### Stable release exception The Stable release workflow creates a mechanical release PR with `GITHUB_TOKEN`. diff --git a/docs/decisions/2026-09-10-pr-documentation-coverage.md b/docs/decisions/2026-09-10-pr-documentation-coverage.md new file mode 100644 index 0000000000..5ddb94fd2c --- /dev/null +++ b/docs/decisions/2026-09-10-pr-documentation-coverage.md @@ -0,0 +1,44 @@ +# ADR-2026-09-10-pr-documentation-coverage: Require linked delivery context for pull requests + +**Status:** proposed +**Date:** 2026-09-10 +**Area:** workflow + +## Context + +PR #3137 adds worktree recovery behavior with no changed specifications or delivery records. +Its `fix:` title illustrates why feature-title checks cannot identify all behavior changes. +The diff does not establish which harness the contributor used. + +The file-based knowledge system permits manually authored artifacts. +Enforcement should validate useful context while allowing small corrections without artificial documentation churn. + +## Decision + +Propose deterministic structural coverage with narrow path exemptions and the explicit `no-docs-allow` label. +Other changes require a changed work order linked to a plan, requirements, acceptance criteria, and system design. +Previously merged specifications can be referenced without cosmetic edits. Reviewers judge whether changed behavior needs updated contracts. +The override persists while the label is present and affects only this check. + +CI owns this policy. It does not require use of a particular agent or prove that plans were committed before code. +The check reports failures before ruleset activation; mandatory merge enforcement requires a separately verified administrator rollout. + +## Consequences + +Results are reproducible and need no model credentials. Small code fixes and refactors can require a maintainer exception. +Structural links cannot prove semantic relevance. Review remains responsible for incomplete or misleading artifacts. +PR-wide exceptions can become stale as scope grows; maintainers must remove the label when it no longer applies. + +## Alternatives considered + +- Require documents for every PR: adds unnecessary work to tests, translations, and documentation corrections. +- Gate only `feat:` titles: misses behavior-changing fixes and can be bypassed by renaming a PR. +- Classify by line counts: a small protocol change can matter more than a large mechanical edit. +- Let a PR-body declaration exempt runtime changes: makes the policy self-waivable without the requested label. +- Use an AI classifier as the required gate: introduces cost, nondeterminism, and untrusted prompt inputs before a simple structural policy has been evaluated. +- Require changes to every linked document: encourages meaningless edits when an implementation follows an already merged design. + +## Related artifacts + +- [Requirements](../specs/ci/requirements/pull-request-documentation-coverage.md) +- [System design](../specs/ci/system-design/pull-request-documentation-coverage.md) diff --git a/docs/plans/pr-documentation-coverage/plan.md b/docs/plans/pr-documentation-coverage/plan.md new file mode 100644 index 0000000000..9e4be6e387 --- /dev/null +++ b/docs/plans/pr-documentation-coverage/plan.md @@ -0,0 +1,105 @@ +--- +created: 2026-09-10 +status: done +requirements: + - REQ-CI-PR-DOCS-001 + - REQ-CI-PR-DOCS-002 + - REQ-CI-PR-DOCS-003 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +legacy_specs: [] +--- + +# Implementation plan: PR documentation coverage + +## Overview + +Build a deterministic coverage validator, integrate PR status reporting, then add merge-queue evaluation and contributor guidance. +The CI system owns this package because it controls repository checks and label exceptions. + +## Scope + +In scope: narrow exemptions, linked artifact validation, the exact `no-docs-allow` override, current-revision statuses, queue compatibility, and actionable summaries. + +Out of scope: AI classification, application UI, automatic label management, publishing a PR, and live ruleset mutations. +Public user documentation remains governed by the existing contribution checklist. + +## Technical approach + +Use the proposed `.github/scripts/pr-docs.cjs` module for pure policy functions and bounded GitHub reads. +Use `.github/workflows/pr-docs.yml` for base-controlled execution and the `PR documentation coverage` commit status. +Reuse the existing Node runtime from GitHub-hosted runners and the built-in test runner; no pnpm dependency installation is needed. +Register Node and workflow contract tests in `.github/workflows/lint-action-pinning.yml`. + +Existing patterns: `.github/workflows/pr-size-label.yml` for metadata pagination, +`.github/scripts/pr-size-label-workflow-contract_test.py` for workflow checks, and +`docs/ci-merge-queue.md` for required-check rollout constraints. + +## Tests + +All named test suites below are proposed files. + +| Criteria | Evidence | +| --- | --- | +| AC-CI-PR-DOCS-001.2 through .6 | `pr-docs.test.cjs`: classification and artifact graph fixtures | +| AC-CI-PR-DOCS-001.1; AC-CI-PR-DOCS-002.1 through .4 | Fake GitHub adapter exercises draft/fork events and label transitions | +| AC-CI-PR-DOCS-003.1 through .3 | API failures, pagination caps, stale head/label reads, path and content restrictions | +| AC-CI-PR-DOCS-003.4 through .5 | Per-member queue evaluation, active prefix reevaluation, mismatched queue boundaries, override removal, and rollout checklist | + +Use a representative #3137 fixture: `fix(worktree)` with runtime recovery paths and no artifacts fails. +Do not fetch the live PR in unit tests. + +## End-to-end evidence + +No application browser flow changes. The workflow contract and mocked API tests run locally. +After deployment, record real GitHub run URLs for an uncovered PR, a covered PR, an exempt PR, a fork, override addition/removal without pushes, and a merge group. +Verify one member's documents or override cannot satisfy another member. +Live smoke tests and required-check activation are deployment steps, not completed local evidence. + +## Work orders + +- [x] [Task 01: Validate documentation coverage](task-01-coverage-validator.md) +- [x] [Task 02: Report pull request coverage](task-02-pr-workflow.md) +- [x] [Task 03: Support merge queue coverage](task-03-queue-coverage.md) + +Execute sequentially. No subagents are authorized. + +## Verification results + +Design validation on 2026-09-10: + +- `python3 scripts/lint-spec-files.test.py`: 36 tests passed. +- `python3 scripts/lint-spec-files.py --all`: passed. +- `git diff --check -- docs/specs docs/decisions docs/plans/pr-documentation-coverage`: passed. +- `git status --short -- docs/plans/pr-documentation-coverage`: confirmed the new package is present and untracked. + +Implementation is complete. Live GitHub smoke-test evidence and required-check +activation remain deployment steps and are not claimed by this change. + +Implementation verification on 2026-09-10: + +- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed. +- `python3 .github/scripts/pr-docs-workflow-contract_test.py`: 5 tests passed. +- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed. +- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows passed. +- `python3 scripts/lint-spec-files.py --all`: passed. +- `zizmor .github/workflows/pr-docs.yml`: no findings. +- `git diff --check`: passed. + +Repository-wide `zizmor .github/workflows` still reports existing findings in +unrelated workflows; the new workflow has no reported findings. + +## Risks + +- Conservative defaults require labels for some small fixes, refactors, and dependency manifest changes. +- The target-branch event queue retains up to 100 pending runs; higher bursts can still cancel new runs. +- Structural validation cannot establish semantic relevance or planning chronology. +- Merge queue boundary resolution needs real payload validation before mandatory rollout. +- GitHub label/status publication is eventually consistent. +- Adding a required status is an administrator operation separate from merging workflow files. + +## References + +- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md) +- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md) +- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md) diff --git a/docs/plans/pr-documentation-coverage/task-01-coverage-validator.md b/docs/plans/pr-documentation-coverage/task-01-coverage-validator.md new file mode 100644 index 0000000000..ca066b4960 --- /dev/null +++ b/docs/plans/pr-documentation-coverage/task-01-coverage-validator.md @@ -0,0 +1,87 @@ +--- +id: "01-coverage-validator" +title: "Validate documentation coverage" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-CI-PR-DOCS-001 +acceptance_criteria: + - AC-CI-PR-DOCS-001.2 + - AC-CI-PR-DOCS-001.3 + - AC-CI-PR-DOCS-001.4 + - AC-CI-PR-DOCS-001.5 + - AC-CI-PR-DOCS-001.6 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +--- + +# Task 01: Validate documentation coverage + +## Summary + +Implement pure path classification and artifact graph validation with TDD. Cover existing artifacts, new packages, deletions, renames, unknown paths, malformed metadata, ambiguous IDs, and incomplete references. + +## In scope + +- `.github/scripts/pr-docs.cjs` +- `.github/scripts/pr-docs.test.cjs` + +## Out of scope + +Application code, unrelated workflow cleanup, live label changes, and live ruleset changes. + +## Acceptance + +- Only explicitly exempt changes pass without a work order; the #3137 runtime-change fixture fails. +- A changed work order and valid referenced package pass without cosmetic edits to existing specifications. +- Missing, empty, deleted, escaping, or mismatched references produce precise failures. + +## Verification + +Run from the repository root: + +```bash +node --test .github/scripts/pr-docs.test.cjs +git diff --check +``` + +## Files likely touched + +- `.github/scripts/pr-docs.cjs` +- `.github/scripts/pr-docs.test.cjs` + +## Dependencies + +None. + +## Risks + +Do not confuse the size-label file filter with this policy; their exclusions differ. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md) +- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md) +- [Plan](plan.md) +- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md) +- `.github/AGENTS.md` and existing PR size workflow contract tests. + +## Results + +Implemented `.github/scripts/pr-docs.cjs` path classification, bounded +frontmatter parsing, and linked artifact validation. The validator handles +renames, deleted references, malformed metadata, multi-requirement acceptance +criteria, ambiguous requirement IDs, bounded requirement search, disjoint +multi-design ownership, document limits, and the representative runtime-change +fixture. + +Verification: + +- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed. +- `git diff --check`: passed. diff --git a/docs/plans/pr-documentation-coverage/task-02-pr-workflow.md b/docs/plans/pr-documentation-coverage/task-02-pr-workflow.md new file mode 100644 index 0000000000..17459601ff --- /dev/null +++ b/docs/plans/pr-documentation-coverage/task-02-pr-workflow.md @@ -0,0 +1,111 @@ +--- +id: "02-pr-workflow" +title: "Report pull request coverage" +status: done +wave: 2 +depends_on: + - "01-coverage-validator" +plan: "plan.md" +requirements: + - REQ-CI-PR-DOCS-001 + - REQ-CI-PR-DOCS-002 + - REQ-CI-PR-DOCS-003 +acceptance_criteria: + - AC-CI-PR-DOCS-001.1 + - AC-CI-PR-DOCS-002.1 + - AC-CI-PR-DOCS-002.2 + - AC-CI-PR-DOCS-002.3 + - AC-CI-PR-DOCS-002.4 + - AC-CI-PR-DOCS-003.1 + - AC-CI-PR-DOCS-003.2 + - AC-CI-PR-DOCS-003.3 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +--- + +# Task 02: Report pull request coverage + +## Summary + +Integrate the validator with bounded, revision-bound API reads and a trusted PR workflow. Add status summaries, label reevaluation, retry dispatch, and tests registered in the existing lint workflow. + +## In scope + +- `.github/workflows/pr-docs.yml` +- `.github/scripts/pr-docs.cjs` +- `.github/scripts/pr-docs.test.cjs` +- `.github/scripts/pr-docs-workflow-contract_test.py` +- `.github/workflows/lint-action-pinning.yml` +- `Makefile` + +## Out of scope + +Application code, unrelated workflow cleanup, live label changes, and live ruleset changes. + +## Acceptance + +- Open, draft, fork, push, edited, and label events publish the correct status on the evaluated head. +- Override addition passes and removal restores validation without a push; concurrent state changes cannot be silently accepted. +- Trusted execution, bounded reads, errors, parser limits, and test registration have passing contract coverage. + +## Verification + +Run from the repository root: + +```bash +node --test .github/scripts/pr-docs.test.cjs +python3 .github/scripts/pr-docs-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +python3 .github/scripts/lint-action-pinning.py +zizmor .github/workflows +git diff --check +``` + +## Files likely touched + +- `.github/workflows/pr-docs.yml` +- `.github/scripts/pr-docs.cjs` +- `.github/scripts/pr-docs.test.cjs` +- `.github/scripts/pr-docs-workflow-contract_test.py` +- `.github/workflows/lint-action-pinning.yml` + +## Dependencies + +Task 01. + +## Risks + +A pull_request_target native job check is not sufficient evidence on the PR head. Do not activate required checks at this intermediate task. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md) +- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md) +- [Plan](plan.md) +- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md) +- `.github/AGENTS.md` and existing PR size workflow contract tests. + +## Results + +Implemented the bounded GitHub API adapter, revision and label consistency +checks, status reporting, manual retry handling, trusted workflow, bounded +pending event queue, and workflow contract coverage. The validator suite is +registered in the lint workflow and the repository script test target. The +workflow uses only read access to pull requests and contents plus commit-status +writes. + +Verification: + +- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed. +- `python3 .github/scripts/pr-docs-workflow-contract_test.py`: 5 tests passed. +- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed. +- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows passed. +- `zizmor .github/workflows/pr-docs.yml`: no findings. +- `git diff --check`: passed. + +The repository-wide `zizmor .github/workflows` command still reports existing +findings in unrelated workflows. No new finding was reported for `pr-docs.yml`. diff --git a/docs/plans/pr-documentation-coverage/task-03-queue-coverage.md b/docs/plans/pr-documentation-coverage/task-03-queue-coverage.md new file mode 100644 index 0000000000..413b99500e --- /dev/null +++ b/docs/plans/pr-documentation-coverage/task-03-queue-coverage.md @@ -0,0 +1,115 @@ +--- +id: "03-queue-coverage" +title: "Support merge queue coverage" +status: done +wave: 3 +depends_on: + - "02-pr-workflow" +plan: "plan.md" +requirements: + - REQ-CI-PR-DOCS-001 + - REQ-CI-PR-DOCS-002 + - REQ-CI-PR-DOCS-003 +acceptance_criteria: + - AC-CI-PR-DOCS-001.6 + - AC-CI-PR-DOCS-002.2 + - AC-CI-PR-DOCS-003.1 + - AC-CI-PR-DOCS-003.4 + - AC-CI-PR-DOCS-003.5 +system_design: + - ../../specs/ci/system-design/pull-request-documentation-coverage.md +--- + +# Task 03: Support merge queue coverage + +## Summary + +Add merge-group membership resolution and per-member evaluation. Reconcile live group status after label changes. Document contribution examples, exact exemptions, label persistence, structural limits, dispatch retries, and administrator rollout. + +## In scope + +- `.github/workflows/pr-docs.yml` +- `.github/scripts/pr-docs.cjs` +- `.github/scripts/pr-docs.test.cjs` +- `.github/scripts/pr-docs-workflow-contract_test.py` +- `.github/pull_request_template.md` +- `.github/AGENTS.md` +- `CONTRIBUTING.md` +- `docs/ci-merge-queue.md` + +## Out of scope + +Application code, unrelated workflow cleanup, live label changes, and live ruleset changes. + +## Acceptance + +- Queue groups evaluate every member independently and publish the same context on the group head; missing or ambiguous membership fails. +- Removing an override reevaluates affected current groups without reusing an obsolete success. +- Contributor instructions and rollout checks distinguish deployed failures from ruleset-enforced blocking, with no live ruleset edits. + +## Verification + +Run from the repository root: + +```bash +node --test .github/scripts/pr-docs.test.cjs +python3 .github/scripts/pr-docs-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +python3 .github/scripts/lint-action-pinning.py +zizmor .github/workflows +python3 scripts/lint-spec-files.py --all +git diff --check +``` + +## Files likely touched + +- `.github/workflows/pr-docs.yml` +- `.github/scripts/pr-docs.cjs` +- `.github/scripts/pr-docs.test.cjs` +- `.github/scripts/pr-docs-workflow-contract_test.py` +- `.github/pull_request_template.md` +- `.github/AGENTS.md` +- `CONTRIBUTING.md` +- `docs/ci-merge-queue.md` + +## Dependencies + +Task 02. + +## Risks + +Queue head/base mapping is a GitHub integration assumption. Validate with captured real payloads before adding the required context; do not pass unknown groups. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md) +- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md) +- [Plan](plan.md) +- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md) +- `.github/AGENTS.md` and existing PR size workflow contract tests. + +## Results + +Implemented exact merge-group boundary resolution, independent member +evaluation, all active prefix reevaluation after label removal, target-branch +serialization with a bounded pending queue, contributor guidance, and the +administrator rollout notes. The workflow publishes the shared status on the +synthetic group head without changing labels, queue membership, or rulesets. + +Verification: + +- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed. +- `python3 .github/scripts/pr-docs-workflow-contract_test.py`: 5 tests passed. +- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed. +- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows passed. +- `python3 scripts/lint-spec-files.py --all`: passed. +- `zizmor .github/workflows/pr-docs.yml`: no findings. +- `git diff --check`: passed. + +The repository-wide `zizmor .github/workflows` command still reports existing +findings in unrelated workflows. Live merge-group payload validation remains a +deployment step before the status becomes required. diff --git a/docs/specs/ci/README.md b/docs/specs/ci/README.md index 5134738b11..dd62a4132d 100644 --- a/docs/specs/ci/README.md +++ b/docs/specs/ci/README.md @@ -23,6 +23,7 @@ deployment, and pull request walkthrough generation. - Permissions and credential boundaries for review, preview, walkthrough, and publication jobs. - Workflow contract tests for these boundaries. +- Pull request documentation coverage and explicit exceptions. ## Exclusions diff --git a/docs/specs/ci/requirements/pull-request-documentation-coverage.md b/docs/specs/ci/requirements/pull-request-documentation-coverage.md new file mode 100644 index 0000000000..39e4baa38e --- /dev/null +++ b/docs/specs/ci/requirements/pull-request-documentation-coverage.md @@ -0,0 +1,64 @@ +--- +status: draft +system: ci +created: 2026-09-10 +owners: + - kandev +--- + +# Pull request documentation coverage requirements + +## Overview + +Contributors must include the delivery record for changes that need design context. +The CI system owns the coverage check, automatic exemptions, and label override. +Contributors can write the artifacts manually; use of the repository harness is optional. + +## Requirements + +### REQ-CI-PR-DOCS-001: Predictable artifact coverage + +**Intent:** Require reviewable context without requiring new specifications for every correction. + +#### Acceptance criteria + +- **AC-CI-PR-DOCS-001.1:** Every open pull request shall receive a documentation coverage result, including drafts and forks. +- **AC-CI-PR-DOCS-001.2:** A pull request containing only recognized documentation, tests, translation catalogs, or dependency lock changes shall pass without a delivery package. +- **AC-CI-PR-DOCS-001.3:** Other changes shall require an added or modified work order, its plan, and linked requirements and system designs. Changing the title to `fix`, `chore`, or `refactor` shall not exempt the change. +- **AC-CI-PR-DOCS-001.4:** Existing plans and specifications shall qualify when the work order references them and they exist in the proposed revision. Contributors shall update contracts when behavior changes, but shall not need meaningless edits to unchanged contracts. +- **AC-CI-PR-DOCS-001.5:** A deleted artifact, empty file, unresolved reference, unrelated unlinked document, or work order without requirement and acceptance references shall not satisfy coverage. +- **AC-CI-PR-DOCS-001.6:** The result shall identify triggering paths, accepted references, missing artifacts, and corrective steps. Structural coverage shall not claim semantic completeness or prove that planning preceded coding. + +### REQ-CI-PR-DOCS-002: Explicit documentation exception + +**Intent:** Let repository label managers waive unnecessary documentation work. + +#### Acceptance criteria + +- **AC-CI-PR-DOCS-002.1:** When the exact label `no-docs-allow` is present, the coverage policy shall pass and identify the override in its result. +- **AC-CI-PR-DOCS-002.2:** Adding or removing the label shall reevaluate the current pull request without requiring a new commit. Removal shall restore normal evaluation. +- **AC-CI-PR-DOCS-002.3:** The label shall remain effective across pushes until removed. The workflow shall not add, remove, or grant permission to apply it. +- **AC-CI-PR-DOCS-002.4:** The exception shall affect only documentation coverage. Other validation and merge requirements shall remain independent. + +### REQ-CI-PR-DOCS-003: Reliable enforcement + +**Intent:** Make the result usable as a required check without trusting contributor execution. + +#### Acceptance criteria + +- **AC-CI-PR-DOCS-003.1:** The result shall belong to the evaluated revision. A stale event shall not publish a success for a newer revision. +- **AC-CI-PR-DOCS-003.2:** Missing or incomplete evaluation data shall produce an error, never an automatic exemption. +- **AC-CI-PR-DOCS-003.3:** The check shall evaluate contributor content as data and shall not execute contributor code with repository write credentials. +- **AC-CI-PR-DOCS-003.4:** A merge group shall pass only when every included pull request meets its own coverage policy or has its own override. One pull request's artifacts or label shall not exempt another. +- **AC-CI-PR-DOCS-003.5:** Required-check rollout shall include evidence that ordinary PRs, label changes, forks, and merge groups report the expected result. + +## Out of scope + +- AI classification of feature intent or documentation quality. +- Enforcing the historical order of planning and implementation commits. +- Requiring public user documentation for every code change. Existing public-docs review obligations continue. +- Automatically changing repository rulesets, labels, merge queue membership, or PR #3137. + +## Implementation plans + +- [PR documentation coverage](../../../plans/pr-documentation-coverage/plan.md) diff --git a/docs/specs/ci/system-design/pull-request-documentation-coverage.md b/docs/specs/ci/system-design/pull-request-documentation-coverage.md new file mode 100644 index 0000000000..e45c040e0e --- /dev/null +++ b/docs/specs/ci/system-design/pull-request-documentation-coverage.md @@ -0,0 +1,149 @@ +--- +status: draft +system: ci +requirements: + - REQ-CI-PR-DOCS-001 + - REQ-CI-PR-DOCS-002 + - REQ-CI-PR-DOCS-003 +--- + +# Pull request documentation coverage system design + +## Ownership and mapping + +CI owns the contributor coverage policy and its trusted execution. Application systems continue to own their specifications. + +| Requirement | Design sections | +| --- | --- | +| REQ-CI-PR-DOCS-001 | Classification; Artifact contract | +| REQ-CI-PR-DOCS-002 | Events and overrides | +| REQ-CI-PR-DOCS-003 | Reporting and consistency; Merge queue; Security | + +## Components + +Proposed files: + +- `.github/workflows/pr-docs.yml`: trusted orchestration, PR and merge-group entry points. +- `.github/scripts/pr-docs.cjs`: pure classification and reference validation, plus a bounded GitHub API adapter. +- `.github/scripts/pr-docs.test.cjs`: Node built-in test runner, with fake GitHub responses. +- `.github/scripts/pr-docs-workflow-contract_test.py`: workflow event, permission, and registration contract tests. + +Register tests in `.github/workflows/lint-action-pinning.yml`, following the existing PR size workflow contract test pattern. + +## Classification + +Use an explicit exemption list. A changed path outside it requires coverage, including unknown roots and runtime configuration. +Evaluate both old and new paths for renames; deleting runtime code still requires coverage. + +Initial exemptions: + +- `docs/**` and Markdown files, including `AGENTS.md` and `CONTRIBUTING.md`. +- Go `*_test.go`; JS/TS `*.test.*` and `*.spec.*` restricted to JS/TS source extensions; `apps/web/e2e/**`. +- `apps/web/src/locales//.json`. +- Exact lock basenames `pnpm-lock.yaml`, `package-lock.json`, `yarn.lock`, `go.sum`, and `Cargo.lock`. + +Do not exempt all JSON, YAML, assets, scripts, package manifests, Rust files, workflows, generated directories, or files containing the word `test`. +These can change shipped behavior or repository contracts. Add exemptions later only with concrete fixtures. +This conservative policy creates false positives for small runtime fixes and refactors. The explicit label is their escape hatch. + +## Artifact contract + +Select added or modified `docs/plans//task--.md` files from the complete PR diff. +A pure rename with no content change does not qualify. Require at least one qualifying work order. +Read selected work orders and references at the exact PR head through the content API, as bounded text data. + +Validate the repository's existing frontmatter fields: `id`, `title`, `status`, `wave`, `depends_on`, `plan`, `requirements`, `acceptance_criteria`, and `system_design`. +Require nonempty requirement, acceptance, and design lists. Resolve `plan` to the sibling `plan.md`; require a link back to the selected work order. +The plan's requirement/design references must cover the selected work order's references. +Resolve each design under `docs/specs//system-design/`. Each design +owns the subset of work-order requirements that it declares, and the union of +the referenced designs must cover every work-order requirement. Find each +owned requirement and acceptance ID in that design system's requirements +directory. +Check that AC IDs belong to referenced REQ IDs and that the design declares those REQ IDs. +Use a bounded parser for the documented frontmatter subset, not executable YAML tags or PR-supplied parsing code. + +Referenced plans and specifications may already exist on the base branch. Require the work order update to record the current delivery scope and results. +Reject empty documents and references to deleted files. Allow pending work during PR development; do not require a completed implementation before a draft PR can pass structural coverage. +Semantic review determines whether changed behavior requires specification updates and whether the selected work order actually covers all changed code. +A handful of unrelated documentation changes does not establish the reference chain. + +No PR-body schema is needed. The existing repository frontmatter supplies machine-readable links. +Update the PR template and contributor guide with examples and the distinction between structural coverage and semantic review. + +## Events and overrides + +Use `pull_request_target` events `opened`, `reopened`, `synchronize`, `edited`, `labeled`, `unlabeled`, and `ready_for_review` without path filters. +Read current PR metadata and labels, not just the event snapshot. Drafts follow the same policy. +The exact current label `no-docs-allow` returns an override success before expensive file reads. +Failure to read current metadata or labels is still an infrastructure error. + +GitHub repository permissions control who applies the label. No separate author allowlist or title-based exception exists. +The override is PR-wide and persists across pushes. This is distinct from the revision-specific `ready-to-merge` contract. +Document that label actions made with `GITHUB_TOKEN` may not trigger another workflow; such automation must explicitly arrange a reevaluation. +Provide a `workflow_dispatch` PR-number input for retries, read current metadata, and validate that the PR belongs to this repository. + +## Reporting and consistency + +Publish one commit status context, `PR documentation coverage`, on the current PR head using `statuses: write`. +Do not rely on the native `pull_request_target` job check, whose execution identity is the base revision. +Use a distinct job name to avoid a status/check-name collision. +Set pending before evaluation; publish success, failure for missing coverage, or error for incomplete data. +The status target URL points to the run summary, which lists reasons, paths, references, and remediation. +Also fail the workflow job on policy failure or infrastructure error. Do not post PR comments. + +Serialize all events by the normalized target branch with `queue: max` and +`cancel-in-progress: false`. GitHub retains up to 100 pending runs in this +shared lock; additional runs can be canceled when that bound is full. The +lock is intentionally broader than a PR-number or group-SHA lock so queued +label reevaluations cannot race with a merge-group evaluation for the same +target branch. Every surviving run reads current state. +Before publishing, reread the head, base, and label set. If changed, reevaluate with a bounded retry, then fail if state remains unstable. +Do not report stale successes for a new head. Metadata publication is eventually consistent; GitHub does not provide an atomic label-read/status-write transaction. + +Paginate file lists and compare the count to current `changed_files`. Reject results at GitHub's 3,000-file cap and mismatched counts. +Reject truncated responses, invalid encodings, oversized documents, unsupported frontmatter, symlinks, submodules, and ambiguous IDs. +Bound artifact reads to 100 documents, 256 KiB each, and 4 MiB total. Report limits explicitly; never silently discard documents. + +## Merge queue + +Handle `merge_group: checks_requested` and publish the same status on `merge_group.head_sha`. +Load only trusted base code. Never evaluate coverage against the combined diff, where one PR could supply another's artifacts. + +Resolve members using the paginated GraphQL merge queue entries, including each entry's `baseCommit`, `headCommit`, and `pullRequest`. +Find the entry for the event head and trace the entry commit boundaries back to the event base. +Require a complete, unambiguous chain; unknown membership produces an error, not success. +Evaluate each member's own current PR diff, artifact chain, and labels. Confirm queue identities and member heads again before publication. +Use the target-branch concurrency key shared with queued label reevaluation and +never cancel active group evaluation. The group status itself remains on the +synthetic group SHA. + +Integration must validate the entry-boundary mapping against real GitHub merge-group payloads before requiring this status. +If GitHub cannot supply the expected chain for a supported queue mode, revise the mapping and fixtures before rollout. +Label removal while queued must also reevaluate every active group prefix +containing that PR, through the same group evaluator and serialization key. +Do not dequeue, requeue, or automatically merge PRs. + +## Security + +Permissions are `contents: read`, `pull-requests: read`, and `statuses: write` only. +Use SHA-pinned actions, GitHub-hosted runners, a trusted base checkout, and `persist-credentials: false`. +For dispatch, restrict the workflow definition to the default branch. For queue runs, pin consumed scripts to the event base SHA. +Do not install PR dependencies, execute PR files, or interpolate PR text into shell or JavaScript source. +Normalize artifact paths inside their allowed roots, bind content requests to exact repository/head identities, and escape summary text. + +## Rollout and persistence + +GitHub stores labels, statuses, and run summaries. No application storage or feature flag is needed. +First merge the workflow and contributor instructions. Exercise pass, failure, override, removal, fork, and merge-group cases. +Then add the status context to the `main` ruleset as a separate administrator action, preserving all existing checks and bypass rules. +Until required, it is a failing PR check but does not itself prohibit merge. Do not report ruleset enforcement as deployed before verification. +Existing open PRs need a supported event or dispatch to receive their first result. + +## Related decisions and references + +- [Documentation coverage policy](../../../decisions/2026-09-10-pr-documentation-coverage.md) +- [Existing merge queue contract](../../../ci-merge-queue.md) +- [GitHub workflow events](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows) +- [GitHub required status checks](https://docs.github.com/en/pull-requests/how-tos/merge-and-close-pull-requests/troubleshooting-required-status-checks) +- [GitHub merge queue API fields](https://docs.github.com/en/graphql/reference/pulls)