ci: enforce pull request documentation coverage (#3586)

This commit is contained in:
Carlos Florêncio
2026-09-12 20:44:13 +01:00
committed by GitHub
parent 0483c054ae
commit 56bc954001
18 changed files with 3888 additions and 0 deletions
+9
View File
@@ -34,3 +34,12 @@ Apply this guidance whenever editing `.github/**`.
- For workflow security changes, run the relevant raw workflow-contract tests,
`python3 .github/scripts/lint-action-pinning_test.py`, `zizmor .github/workflows`,
and `git diff --check`.
- The `pr-docs.yml` workflow is a base-controlled, metadata-only check. It reads
pull-request files through the bounded `.github/scripts/pr-docs.cjs` adapter;
it must never check out or execute a pull-request head.
- Its `PR documentation coverage` status is revision-specific. The exact
`no-docs-allow` label is the only policy override, and merge-group evaluation
must resolve and validate every member independently against the group's
entry boundaries. All event types share a non-cancelling target-branch lock
with `queue: max` so queued label reevaluations cannot race with merge-group
status writes or replace the single pending run.
+7
View File
@@ -35,6 +35,13 @@ DIAGRAM (optional)
POSSIBLE IMPROVEMENTS (optional)
One line: risk level + what could go wrong or be improved. Skip if negligible.
DOCUMENTATION COVERAGE
The `PR documentation coverage` status uses a deterministic path and artifact
check. Runtime, workflow, configuration, and other non-exempt changes need a
changed work order linked to its plan, requirements, acceptance criteria, and
system design. The exact `no-docs-allow` label is a maintainer exception. The
check does not claim semantic completeness or require planning before coding.
RELATED ISSUES
Use "Closes #N" if this resolves an issue. Remove the line if there is no related issue.
@@ -0,0 +1,100 @@
#!/usr/bin/env python3
"""Contract tests for the pull request documentation coverage workflow."""
from __future__ import annotations
from pathlib import Path
import re
import unittest
REPO_ROOT = Path(__file__).resolve().parents[2]
WORKFLOW = REPO_ROOT / ".github" / "workflows" / "pr-docs.yml"
LINT_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "lint-action-pinning.yml"
class PullRequestDocumentationWorkflowContractTest(unittest.TestCase):
def setUp(self) -> None:
self.assertTrue(WORKFLOW.is_file(), "Documentation coverage workflow is missing")
self.workflow = WORKFLOW.read_text(encoding="utf-8")
# @covers AC-CI-PR-DOCS-001.1, AC-CI-PR-DOCS-002.2
def test_runs_for_all_pr_and_manual_retry_events(self) -> None:
trigger = self.workflow.partition("on:\n")[2].partition("\nconcurrency:")[0]
self.assertIn("pull_request_target:", trigger)
for event in (
"opened",
"reopened",
"synchronize",
"edited",
"labeled",
"unlabeled",
"ready_for_review",
):
self.assertIn(event, trigger)
self.assertIn("merge_group:", trigger)
self.assertIn("checks_requested", trigger)
self.assertIn("workflow_dispatch:", trigger)
self.assertIn("pr_number:", trigger)
self.assertNotIn("paths:", trigger)
# @covers AC-CI-PR-DOCS-003.1, AC-CI-PR-DOCS-003.4
def test_serializes_prs_without_cancelling_merge_group_runs(self) -> None:
self.assertIn("group: >-", self.workflow)
self.assertIn("github.event.merge_group.base_ref", self.workflow)
self.assertIn("github.event.pull_request.base.ref", self.workflow)
self.assertIn("queue: max", self.workflow)
self.assertIn("cancel-in-progress: false", self.workflow)
# @covers AC-CI-PR-DOCS-003.3
def test_uses_trusted_least_privilege_checkout(self) -> None:
permissions = self.workflow.partition("permissions:\n")[2].partition("\njobs:")[0]
self.assertEqual(
permissions.strip(),
"contents: read\n pull-requests: read\n statuses: write",
)
self.assertRegex(
self.workflow,
r"uses: actions/checkout@[0-9a-f]{40} # v[0-9]+",
)
self.assertIn("persist-credentials: false", self.workflow)
self.assertIn("github.event.merge_group.base_sha", self.workflow)
self.assertNotIn("github.event.pull_request.head.sha", self.workflow)
self.assertIn("ref: ${{ env.CHECKOUT_REF }}", self.workflow)
self.assertNotIn("pull_request:\n", self.workflow)
self.assertNotIn("pnpm install", self.workflow)
self.assertNotIn("npm install", self.workflow)
self.assertIn("node .github/scripts/pr-docs.cjs", self.workflow)
self.assertIn("name: Publish PR documentation coverage status", self.workflow)
self.assertNotIn("name: PR documentation coverage\n", self.workflow)
# @covers AC-CI-PR-DOCS-001.1, AC-CI-PR-DOCS-002.1, AC-CI-PR-DOCS-003.2
def test_dispatch_is_restricted_to_the_default_branch_and_script_owns_statuses(self) -> None:
self.assertRegex(
self.workflow,
re.compile(
r"github\.event_name != 'workflow_dispatch'.*github\.ref.*default_branch",
re.DOTALL,
),
)
self.assertIn("PR documentation coverage", (REPO_ROOT / ".github" / "scripts" / "pr-docs.cjs").read_text(encoding="utf-8"))
self.assertIn("statuses: write", self.workflow)
def test_contract_is_registered_in_the_required_lint_workflow(self) -> None:
lint_workflow = LINT_WORKFLOW.read_text(encoding="utf-8")
self.assertRegex(
lint_workflow,
r"(?m)^ - name: Test pull request documentation workflow contract$\n"
r"^ run: python3 .github/scripts/pr-docs-workflow-contract_test.py$",
)
self.assertRegex(
lint_workflow,
r"(?m)^ - name: Test pull request documentation validator$\n"
r"^ run: node --test .github/scripts/pr-docs.test.cjs$",
)
makefile = (REPO_ROOT / "Makefile").read_text(encoding="utf-8")
self.assertIn("@node --test .github/scripts/pr-docs.test.cjs", makefile)
if __name__ == "__main__":
unittest.main()
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -93,6 +93,12 @@ jobs:
- name: Test pull request size label workflow contract
run: python3 .github/scripts/pr-size-label-workflow-contract_test.py
- name: Test pull request documentation workflow contract
run: python3 .github/scripts/pr-docs-workflow-contract_test.py
- name: Test pull request documentation validator
run: node --test .github/scripts/pr-docs.test.cjs
- name: Test managed runtime pin update workflow contract
run: python3 .github/scripts/update-agent-runtime-pins-workflow-contract_test.py
+63
View File
@@ -0,0 +1,63 @@
name: Pull request documentation coverage
# This workflow reads untrusted pull-request metadata and files as data only.
# The checked-out validator is always from a trusted workflow/base revision.
on:
pull_request_target: # zizmor: ignore[dangerous-triggers] base-controlled metadata-only workflow
types: [opened, reopened, synchronize, edited, labeled, unlabeled, ready_for_review]
merge_group:
types: [checks_requested]
workflow_dispatch:
inputs:
pr_number:
description: Pull request number to reevaluate
required: true
type: number
concurrency:
# Use the target branch as the shared lock. Label events have only PR
# metadata, while merge-group events have the synthetic group SHA; the
# branch lock prevents either event from publishing over the other.
group: >-
pr-docs-${{
github.event_name == 'merge_group' && github.event.merge_group.base_ref ||
format('refs/heads/{0}', github.event.pull_request.base.ref || github.event.repository.default_branch)
}}
# Keep up to 100 pending events instead of replacing the single pending run.
queue: max
cancel-in-progress: false
permissions:
contents: read
pull-requests: read
statuses: write
jobs:
coverage:
name: Publish PR documentation coverage status
if: >-
github.event_name != 'workflow_dispatch' ||
github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
runs-on: ubuntu-latest
timeout-minutes: 10
env:
CHECKOUT_REF: ${{ github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.workflow_sha }}
steps:
- name: Checkout trusted coverage validator
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
with:
ref: ${{ env.CHECKOUT_REF }}
fetch-depth: 1
persist-credentials: false
- name: Run documentation coverage evaluator
env:
GITHUB_TOKEN: ${{ github.token }}
GITHUB_EVENT_NAME: ${{ github.event_name }}
GITHUB_EVENT_PATH: ${{ github.event_path }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_SERVER_URL: ${{ github.server_url }}
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_STEP_SUMMARY: ${{ github.step_summary }}
run: node .github/scripts/pr-docs.cjs
+6
View File
@@ -30,6 +30,12 @@ discussed the direction before opening the PR. Link the issue from the PR. If
an agent is preparing the change, it must stop and report missing discussion
instead of opening the PR.
### Documentation coverage
When implementing a feature or fix, update the relevant requirements, system
design, plan, and work-order files. CI runs a linter to check PR documentation
coverage.
## How to Contribute
1. **Fork and branch.** Create a feature branch from `main`.
+1
View File
@@ -559,6 +559,7 @@ test-cli:
test-scripts:
@printf "$(CYAN)Running script tests...$(RESET)\n"
@python3 .github/scripts/lint-action-pinning_test.py
@node --test .github/scripts/pr-docs.test.cjs
@bash scripts/pr-state.test.sh
@bash scripts/pr-await.test.sh
@bash scripts/run-quiet.test.sh
+33
View File
@@ -222,6 +222,39 @@ in the table above. It does not require per-shard names.
Do not require `lint` from `pr-title.yml`, `Validate public docs`, review checks,
or deployment checks. These checks do not run in every merge group.
### PR documentation coverage rollout
`pr-docs.yml` also listens for `merge_group: checks_requested` and publishes the
`PR documentation coverage` status on the synthetic merge-group head. It maps
the queue entry boundaries back to the event base and evaluates every included
pull request with its own changed files, work-order references, and labels. A
member's `no-docs-allow` label cannot exempt another member.
All PR and merge-group events use the target branch as one non-cancelling
concurrency key with `queue: max`. GitHub retains up to 100 pending runs in
that key, so ordinary event bursts do not replace the one pending run; events
that arrive after the bound is full can still be canceled. This keeps a queued
label reevaluation from racing with a merge-group status write while the
published coverage status remains attached to the evaluated PR or synthetic
group revision. A label removal reevaluates every active queue-group prefix
that contains the PR, not only the longest group.
The status is not in the active required-check list above until live merge-group
evidence is collected. Roll out the requirement in two stages:
1. Observe ordinary, draft, fork, label-add, label-remove, manual-retry, and
merge-group runs. Confirm that covered, missing, override, and incomplete
responses are attached to the evaluated revision and that all affected queue
group prefixes reevaluate after label removal.
2. Add `PR documentation coverage` as a separate required status through an
administrator ruleset change. Preserve the existing six checks and bypass
rules, then verify a covered PR, an uncovered PR, an override, and a mixed
merge group.
Until stage two is complete, a failing result is visible but does not itself
block merging. The workflow does not add or remove labels, dequeue PRs, or
change rulesets.
### Stable release exception
The Stable release workflow creates a mechanical release PR with `GITHUB_TOKEN`.
@@ -0,0 +1,44 @@
# ADR-2026-09-10-pr-documentation-coverage: Require linked delivery context for pull requests
**Status:** proposed
**Date:** 2026-09-10
**Area:** workflow
## Context
PR #3137 adds worktree recovery behavior with no changed specifications or delivery records.
Its `fix:` title illustrates why feature-title checks cannot identify all behavior changes.
The diff does not establish which harness the contributor used.
The file-based knowledge system permits manually authored artifacts.
Enforcement should validate useful context while allowing small corrections without artificial documentation churn.
## Decision
Propose deterministic structural coverage with narrow path exemptions and the explicit `no-docs-allow` label.
Other changes require a changed work order linked to a plan, requirements, acceptance criteria, and system design.
Previously merged specifications can be referenced without cosmetic edits. Reviewers judge whether changed behavior needs updated contracts.
The override persists while the label is present and affects only this check.
CI owns this policy. It does not require use of a particular agent or prove that plans were committed before code.
The check reports failures before ruleset activation; mandatory merge enforcement requires a separately verified administrator rollout.
## Consequences
Results are reproducible and need no model credentials. Small code fixes and refactors can require a maintainer exception.
Structural links cannot prove semantic relevance. Review remains responsible for incomplete or misleading artifacts.
PR-wide exceptions can become stale as scope grows; maintainers must remove the label when it no longer applies.
## Alternatives considered
- Require documents for every PR: adds unnecessary work to tests, translations, and documentation corrections.
- Gate only `feat:` titles: misses behavior-changing fixes and can be bypassed by renaming a PR.
- Classify by line counts: a small protocol change can matter more than a large mechanical edit.
- Let a PR-body declaration exempt runtime changes: makes the policy self-waivable without the requested label.
- Use an AI classifier as the required gate: introduces cost, nondeterminism, and untrusted prompt inputs before a simple structural policy has been evaluated.
- Require changes to every linked document: encourages meaningless edits when an implementation follows an already merged design.
## Related artifacts
- [Requirements](../specs/ci/requirements/pull-request-documentation-coverage.md)
- [System design](../specs/ci/system-design/pull-request-documentation-coverage.md)
@@ -0,0 +1,105 @@
---
created: 2026-09-10
status: done
requirements:
- REQ-CI-PR-DOCS-001
- REQ-CI-PR-DOCS-002
- REQ-CI-PR-DOCS-003
system_design:
- ../../specs/ci/system-design/pull-request-documentation-coverage.md
legacy_specs: []
---
# Implementation plan: PR documentation coverage
## Overview
Build a deterministic coverage validator, integrate PR status reporting, then add merge-queue evaluation and contributor guidance.
The CI system owns this package because it controls repository checks and label exceptions.
## Scope
In scope: narrow exemptions, linked artifact validation, the exact `no-docs-allow` override, current-revision statuses, queue compatibility, and actionable summaries.
Out of scope: AI classification, application UI, automatic label management, publishing a PR, and live ruleset mutations.
Public user documentation remains governed by the existing contribution checklist.
## Technical approach
Use the proposed `.github/scripts/pr-docs.cjs` module for pure policy functions and bounded GitHub reads.
Use `.github/workflows/pr-docs.yml` for base-controlled execution and the `PR documentation coverage` commit status.
Reuse the existing Node runtime from GitHub-hosted runners and the built-in test runner; no pnpm dependency installation is needed.
Register Node and workflow contract tests in `.github/workflows/lint-action-pinning.yml`.
Existing patterns: `.github/workflows/pr-size-label.yml` for metadata pagination,
`.github/scripts/pr-size-label-workflow-contract_test.py` for workflow checks, and
`docs/ci-merge-queue.md` for required-check rollout constraints.
## Tests
All named test suites below are proposed files.
| Criteria | Evidence |
| --- | --- |
| AC-CI-PR-DOCS-001.2 through .6 | `pr-docs.test.cjs`: classification and artifact graph fixtures |
| AC-CI-PR-DOCS-001.1; AC-CI-PR-DOCS-002.1 through .4 | Fake GitHub adapter exercises draft/fork events and label transitions |
| AC-CI-PR-DOCS-003.1 through .3 | API failures, pagination caps, stale head/label reads, path and content restrictions |
| AC-CI-PR-DOCS-003.4 through .5 | Per-member queue evaluation, active prefix reevaluation, mismatched queue boundaries, override removal, and rollout checklist |
Use a representative #3137 fixture: `fix(worktree)` with runtime recovery paths and no artifacts fails.
Do not fetch the live PR in unit tests.
## End-to-end evidence
No application browser flow changes. The workflow contract and mocked API tests run locally.
After deployment, record real GitHub run URLs for an uncovered PR, a covered PR, an exempt PR, a fork, override addition/removal without pushes, and a merge group.
Verify one member's documents or override cannot satisfy another member.
Live smoke tests and required-check activation are deployment steps, not completed local evidence.
## Work orders
- [x] [Task 01: Validate documentation coverage](task-01-coverage-validator.md)
- [x] [Task 02: Report pull request coverage](task-02-pr-workflow.md)
- [x] [Task 03: Support merge queue coverage](task-03-queue-coverage.md)
Execute sequentially. No subagents are authorized.
## Verification results
Design validation on 2026-09-10:
- `python3 scripts/lint-spec-files.test.py`: 36 tests passed.
- `python3 scripts/lint-spec-files.py --all`: passed.
- `git diff --check -- docs/specs docs/decisions docs/plans/pr-documentation-coverage`: passed.
- `git status --short -- docs/plans/pr-documentation-coverage`: confirmed the new package is present and untracked.
Implementation is complete. Live GitHub smoke-test evidence and required-check
activation remain deployment steps and are not claimed by this change.
Implementation verification on 2026-09-10:
- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed.
- `python3 .github/scripts/pr-docs-workflow-contract_test.py`: 5 tests passed.
- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed.
- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows passed.
- `python3 scripts/lint-spec-files.py --all`: passed.
- `zizmor .github/workflows/pr-docs.yml`: no findings.
- `git diff --check`: passed.
Repository-wide `zizmor .github/workflows` still reports existing findings in
unrelated workflows; the new workflow has no reported findings.
## Risks
- Conservative defaults require labels for some small fixes, refactors, and dependency manifest changes.
- The target-branch event queue retains up to 100 pending runs; higher bursts can still cancel new runs.
- Structural validation cannot establish semantic relevance or planning chronology.
- Merge queue boundary resolution needs real payload validation before mandatory rollout.
- GitHub label/status publication is eventually consistent.
- Adding a required status is an administrator operation separate from merging workflow files.
## References
- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md)
- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md)
- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md)
@@ -0,0 +1,87 @@
---
id: "01-coverage-validator"
title: "Validate documentation coverage"
status: done
wave: 1
depends_on: []
plan: "plan.md"
requirements:
- REQ-CI-PR-DOCS-001
acceptance_criteria:
- AC-CI-PR-DOCS-001.2
- AC-CI-PR-DOCS-001.3
- AC-CI-PR-DOCS-001.4
- AC-CI-PR-DOCS-001.5
- AC-CI-PR-DOCS-001.6
system_design:
- ../../specs/ci/system-design/pull-request-documentation-coverage.md
---
# Task 01: Validate documentation coverage
## Summary
Implement pure path classification and artifact graph validation with TDD. Cover existing artifacts, new packages, deletions, renames, unknown paths, malformed metadata, ambiguous IDs, and incomplete references.
## In scope
- `.github/scripts/pr-docs.cjs`
- `.github/scripts/pr-docs.test.cjs`
## Out of scope
Application code, unrelated workflow cleanup, live label changes, and live ruleset changes.
## Acceptance
- Only explicitly exempt changes pass without a work order; the #3137 runtime-change fixture fails.
- A changed work order and valid referenced package pass without cosmetic edits to existing specifications.
- Missing, empty, deleted, escaping, or mismatched references produce precise failures.
## Verification
Run from the repository root:
```bash
node --test .github/scripts/pr-docs.test.cjs
git diff --check
```
## Files likely touched
- `.github/scripts/pr-docs.cjs`
- `.github/scripts/pr-docs.test.cjs`
## Dependencies
None.
## Risks
Do not confuse the size-label file filter with this policy; their exclusions differ.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md)
- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md)
- [Plan](plan.md)
- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md)
- `.github/AGENTS.md` and existing PR size workflow contract tests.
## Results
Implemented `.github/scripts/pr-docs.cjs` path classification, bounded
frontmatter parsing, and linked artifact validation. The validator handles
renames, deleted references, malformed metadata, multi-requirement acceptance
criteria, ambiguous requirement IDs, bounded requirement search, disjoint
multi-design ownership, document limits, and the representative runtime-change
fixture.
Verification:
- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed.
- `git diff --check`: passed.
@@ -0,0 +1,111 @@
---
id: "02-pr-workflow"
title: "Report pull request coverage"
status: done
wave: 2
depends_on:
- "01-coverage-validator"
plan: "plan.md"
requirements:
- REQ-CI-PR-DOCS-001
- REQ-CI-PR-DOCS-002
- REQ-CI-PR-DOCS-003
acceptance_criteria:
- AC-CI-PR-DOCS-001.1
- AC-CI-PR-DOCS-002.1
- AC-CI-PR-DOCS-002.2
- AC-CI-PR-DOCS-002.3
- AC-CI-PR-DOCS-002.4
- AC-CI-PR-DOCS-003.1
- AC-CI-PR-DOCS-003.2
- AC-CI-PR-DOCS-003.3
system_design:
- ../../specs/ci/system-design/pull-request-documentation-coverage.md
---
# Task 02: Report pull request coverage
## Summary
Integrate the validator with bounded, revision-bound API reads and a trusted PR workflow. Add status summaries, label reevaluation, retry dispatch, and tests registered in the existing lint workflow.
## In scope
- `.github/workflows/pr-docs.yml`
- `.github/scripts/pr-docs.cjs`
- `.github/scripts/pr-docs.test.cjs`
- `.github/scripts/pr-docs-workflow-contract_test.py`
- `.github/workflows/lint-action-pinning.yml`
- `Makefile`
## Out of scope
Application code, unrelated workflow cleanup, live label changes, and live ruleset changes.
## Acceptance
- Open, draft, fork, push, edited, and label events publish the correct status on the evaluated head.
- Override addition passes and removal restores validation without a push; concurrent state changes cannot be silently accepted.
- Trusted execution, bounded reads, errors, parser limits, and test registration have passing contract coverage.
## Verification
Run from the repository root:
```bash
node --test .github/scripts/pr-docs.test.cjs
python3 .github/scripts/pr-docs-workflow-contract_test.py
python3 .github/scripts/lint-action-pinning_test.py
python3 .github/scripts/lint-action-pinning.py
zizmor .github/workflows
git diff --check
```
## Files likely touched
- `.github/workflows/pr-docs.yml`
- `.github/scripts/pr-docs.cjs`
- `.github/scripts/pr-docs.test.cjs`
- `.github/scripts/pr-docs-workflow-contract_test.py`
- `.github/workflows/lint-action-pinning.yml`
## Dependencies
Task 01.
## Risks
A pull_request_target native job check is not sufficient evidence on the PR head. Do not activate required checks at this intermediate task.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md)
- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md)
- [Plan](plan.md)
- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md)
- `.github/AGENTS.md` and existing PR size workflow contract tests.
## Results
Implemented the bounded GitHub API adapter, revision and label consistency
checks, status reporting, manual retry handling, trusted workflow, bounded
pending event queue, and workflow contract coverage. The validator suite is
registered in the lint workflow and the repository script test target. The
workflow uses only read access to pull requests and contents plus commit-status
writes.
Verification:
- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed.
- `python3 .github/scripts/pr-docs-workflow-contract_test.py`: 5 tests passed.
- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed.
- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows passed.
- `zizmor .github/workflows/pr-docs.yml`: no findings.
- `git diff --check`: passed.
The repository-wide `zizmor .github/workflows` command still reports existing
findings in unrelated workflows. No new finding was reported for `pr-docs.yml`.
@@ -0,0 +1,115 @@
---
id: "03-queue-coverage"
title: "Support merge queue coverage"
status: done
wave: 3
depends_on:
- "02-pr-workflow"
plan: "plan.md"
requirements:
- REQ-CI-PR-DOCS-001
- REQ-CI-PR-DOCS-002
- REQ-CI-PR-DOCS-003
acceptance_criteria:
- AC-CI-PR-DOCS-001.6
- AC-CI-PR-DOCS-002.2
- AC-CI-PR-DOCS-003.1
- AC-CI-PR-DOCS-003.4
- AC-CI-PR-DOCS-003.5
system_design:
- ../../specs/ci/system-design/pull-request-documentation-coverage.md
---
# Task 03: Support merge queue coverage
## Summary
Add merge-group membership resolution and per-member evaluation. Reconcile live group status after label changes. Document contribution examples, exact exemptions, label persistence, structural limits, dispatch retries, and administrator rollout.
## In scope
- `.github/workflows/pr-docs.yml`
- `.github/scripts/pr-docs.cjs`
- `.github/scripts/pr-docs.test.cjs`
- `.github/scripts/pr-docs-workflow-contract_test.py`
- `.github/pull_request_template.md`
- `.github/AGENTS.md`
- `CONTRIBUTING.md`
- `docs/ci-merge-queue.md`
## Out of scope
Application code, unrelated workflow cleanup, live label changes, and live ruleset changes.
## Acceptance
- Queue groups evaluate every member independently and publish the same context on the group head; missing or ambiguous membership fails.
- Removing an override reevaluates affected current groups without reusing an obsolete success.
- Contributor instructions and rollout checks distinguish deployed failures from ruleset-enforced blocking, with no live ruleset edits.
## Verification
Run from the repository root:
```bash
node --test .github/scripts/pr-docs.test.cjs
python3 .github/scripts/pr-docs-workflow-contract_test.py
python3 .github/scripts/lint-action-pinning_test.py
python3 .github/scripts/lint-action-pinning.py
zizmor .github/workflows
python3 scripts/lint-spec-files.py --all
git diff --check
```
## Files likely touched
- `.github/workflows/pr-docs.yml`
- `.github/scripts/pr-docs.cjs`
- `.github/scripts/pr-docs.test.cjs`
- `.github/scripts/pr-docs-workflow-contract_test.py`
- `.github/pull_request_template.md`
- `.github/AGENTS.md`
- `CONTRIBUTING.md`
- `docs/ci-merge-queue.md`
## Dependencies
Task 02.
## Risks
Queue head/base mapping is a GitHub integration assumption. Validate with captured real payloads before adding the required context; do not pass unknown groups.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/ci/requirements/pull-request-documentation-coverage.md)
- [System design](../../specs/ci/system-design/pull-request-documentation-coverage.md)
- [Plan](plan.md)
- [Decision](../../decisions/2026-09-10-pr-documentation-coverage.md)
- `.github/AGENTS.md` and existing PR size workflow contract tests.
## Results
Implemented exact merge-group boundary resolution, independent member
evaluation, all active prefix reevaluation after label removal, target-branch
serialization with a bounded pending queue, contributor guidance, and the
administrator rollout notes. The workflow publishes the shared status on the
synthetic group head without changing labels, queue membership, or rulesets.
Verification:
- `node --test .github/scripts/pr-docs.test.cjs`: 35 tests passed.
- `python3 .github/scripts/pr-docs-workflow-contract_test.py`: 5 tests passed.
- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed.
- `python3 .github/scripts/lint-action-pinning.py`: 24 workflows passed.
- `python3 scripts/lint-spec-files.py --all`: passed.
- `zizmor .github/workflows/pr-docs.yml`: no findings.
- `git diff --check`: passed.
The repository-wide `zizmor .github/workflows` command still reports existing
findings in unrelated workflows. Live merge-group payload validation remains a
deployment step before the status becomes required.
+1
View File
@@ -23,6 +23,7 @@ deployment, and pull request walkthrough generation.
- Permissions and credential boundaries for review, preview, walkthrough, and
publication jobs.
- Workflow contract tests for these boundaries.
- Pull request documentation coverage and explicit exceptions.
## Exclusions
@@ -0,0 +1,64 @@
---
status: draft
system: ci
created: 2026-09-10
owners:
- kandev
---
# Pull request documentation coverage requirements
## Overview
Contributors must include the delivery record for changes that need design context.
The CI system owns the coverage check, automatic exemptions, and label override.
Contributors can write the artifacts manually; use of the repository harness is optional.
## Requirements
### REQ-CI-PR-DOCS-001: Predictable artifact coverage
**Intent:** Require reviewable context without requiring new specifications for every correction.
#### Acceptance criteria
- **AC-CI-PR-DOCS-001.1:** Every open pull request shall receive a documentation coverage result, including drafts and forks.
- **AC-CI-PR-DOCS-001.2:** A pull request containing only recognized documentation, tests, translation catalogs, or dependency lock changes shall pass without a delivery package.
- **AC-CI-PR-DOCS-001.3:** Other changes shall require an added or modified work order, its plan, and linked requirements and system designs. Changing the title to `fix`, `chore`, or `refactor` shall not exempt the change.
- **AC-CI-PR-DOCS-001.4:** Existing plans and specifications shall qualify when the work order references them and they exist in the proposed revision. Contributors shall update contracts when behavior changes, but shall not need meaningless edits to unchanged contracts.
- **AC-CI-PR-DOCS-001.5:** A deleted artifact, empty file, unresolved reference, unrelated unlinked document, or work order without requirement and acceptance references shall not satisfy coverage.
- **AC-CI-PR-DOCS-001.6:** The result shall identify triggering paths, accepted references, missing artifacts, and corrective steps. Structural coverage shall not claim semantic completeness or prove that planning preceded coding.
### REQ-CI-PR-DOCS-002: Explicit documentation exception
**Intent:** Let repository label managers waive unnecessary documentation work.
#### Acceptance criteria
- **AC-CI-PR-DOCS-002.1:** When the exact label `no-docs-allow` is present, the coverage policy shall pass and identify the override in its result.
- **AC-CI-PR-DOCS-002.2:** Adding or removing the label shall reevaluate the current pull request without requiring a new commit. Removal shall restore normal evaluation.
- **AC-CI-PR-DOCS-002.3:** The label shall remain effective across pushes until removed. The workflow shall not add, remove, or grant permission to apply it.
- **AC-CI-PR-DOCS-002.4:** The exception shall affect only documentation coverage. Other validation and merge requirements shall remain independent.
### REQ-CI-PR-DOCS-003: Reliable enforcement
**Intent:** Make the result usable as a required check without trusting contributor execution.
#### Acceptance criteria
- **AC-CI-PR-DOCS-003.1:** The result shall belong to the evaluated revision. A stale event shall not publish a success for a newer revision.
- **AC-CI-PR-DOCS-003.2:** Missing or incomplete evaluation data shall produce an error, never an automatic exemption.
- **AC-CI-PR-DOCS-003.3:** The check shall evaluate contributor content as data and shall not execute contributor code with repository write credentials.
- **AC-CI-PR-DOCS-003.4:** A merge group shall pass only when every included pull request meets its own coverage policy or has its own override. One pull request's artifacts or label shall not exempt another.
- **AC-CI-PR-DOCS-003.5:** Required-check rollout shall include evidence that ordinary PRs, label changes, forks, and merge groups report the expected result.
## Out of scope
- AI classification of feature intent or documentation quality.
- Enforcing the historical order of planning and implementation commits.
- Requiring public user documentation for every code change. Existing public-docs review obligations continue.
- Automatically changing repository rulesets, labels, merge queue membership, or PR #3137.
## Implementation plans
- [PR documentation coverage](../../../plans/pr-documentation-coverage/plan.md)
@@ -0,0 +1,149 @@
---
status: draft
system: ci
requirements:
- REQ-CI-PR-DOCS-001
- REQ-CI-PR-DOCS-002
- REQ-CI-PR-DOCS-003
---
# Pull request documentation coverage system design
## Ownership and mapping
CI owns the contributor coverage policy and its trusted execution. Application systems continue to own their specifications.
| Requirement | Design sections |
| --- | --- |
| REQ-CI-PR-DOCS-001 | Classification; Artifact contract |
| REQ-CI-PR-DOCS-002 | Events and overrides |
| REQ-CI-PR-DOCS-003 | Reporting and consistency; Merge queue; Security |
## Components
Proposed files:
- `.github/workflows/pr-docs.yml`: trusted orchestration, PR and merge-group entry points.
- `.github/scripts/pr-docs.cjs`: pure classification and reference validation, plus a bounded GitHub API adapter.
- `.github/scripts/pr-docs.test.cjs`: Node built-in test runner, with fake GitHub responses.
- `.github/scripts/pr-docs-workflow-contract_test.py`: workflow event, permission, and registration contract tests.
Register tests in `.github/workflows/lint-action-pinning.yml`, following the existing PR size workflow contract test pattern.
## Classification
Use an explicit exemption list. A changed path outside it requires coverage, including unknown roots and runtime configuration.
Evaluate both old and new paths for renames; deleting runtime code still requires coverage.
Initial exemptions:
- `docs/**` and Markdown files, including `AGENTS.md` and `CONTRIBUTING.md`.
- Go `*_test.go`; JS/TS `*.test.*` and `*.spec.*` restricted to JS/TS source extensions; `apps/web/e2e/**`.
- `apps/web/src/locales/<locale>/<namespace>.json`.
- Exact lock basenames `pnpm-lock.yaml`, `package-lock.json`, `yarn.lock`, `go.sum`, and `Cargo.lock`.
Do not exempt all JSON, YAML, assets, scripts, package manifests, Rust files, workflows, generated directories, or files containing the word `test`.
These can change shipped behavior or repository contracts. Add exemptions later only with concrete fixtures.
This conservative policy creates false positives for small runtime fixes and refactors. The explicit label is their escape hatch.
## Artifact contract
Select added or modified `docs/plans/<initiative>/task-<NN>-<slug>.md` files from the complete PR diff.
A pure rename with no content change does not qualify. Require at least one qualifying work order.
Read selected work orders and references at the exact PR head through the content API, as bounded text data.
Validate the repository's existing frontmatter fields: `id`, `title`, `status`, `wave`, `depends_on`, `plan`, `requirements`, `acceptance_criteria`, and `system_design`.
Require nonempty requirement, acceptance, and design lists. Resolve `plan` to the sibling `plan.md`; require a link back to the selected work order.
The plan's requirement/design references must cover the selected work order's references.
Resolve each design under `docs/specs/<system>/system-design/`. Each design
owns the subset of work-order requirements that it declares, and the union of
the referenced designs must cover every work-order requirement. Find each
owned requirement and acceptance ID in that design system's requirements
directory.
Check that AC IDs belong to referenced REQ IDs and that the design declares those REQ IDs.
Use a bounded parser for the documented frontmatter subset, not executable YAML tags or PR-supplied parsing code.
Referenced plans and specifications may already exist on the base branch. Require the work order update to record the current delivery scope and results.
Reject empty documents and references to deleted files. Allow pending work during PR development; do not require a completed implementation before a draft PR can pass structural coverage.
Semantic review determines whether changed behavior requires specification updates and whether the selected work order actually covers all changed code.
A handful of unrelated documentation changes does not establish the reference chain.
No PR-body schema is needed. The existing repository frontmatter supplies machine-readable links.
Update the PR template and contributor guide with examples and the distinction between structural coverage and semantic review.
## Events and overrides
Use `pull_request_target` events `opened`, `reopened`, `synchronize`, `edited`, `labeled`, `unlabeled`, and `ready_for_review` without path filters.
Read current PR metadata and labels, not just the event snapshot. Drafts follow the same policy.
The exact current label `no-docs-allow` returns an override success before expensive file reads.
Failure to read current metadata or labels is still an infrastructure error.
GitHub repository permissions control who applies the label. No separate author allowlist or title-based exception exists.
The override is PR-wide and persists across pushes. This is distinct from the revision-specific `ready-to-merge` contract.
Document that label actions made with `GITHUB_TOKEN` may not trigger another workflow; such automation must explicitly arrange a reevaluation.
Provide a `workflow_dispatch` PR-number input for retries, read current metadata, and validate that the PR belongs to this repository.
## Reporting and consistency
Publish one commit status context, `PR documentation coverage`, on the current PR head using `statuses: write`.
Do not rely on the native `pull_request_target` job check, whose execution identity is the base revision.
Use a distinct job name to avoid a status/check-name collision.
Set pending before evaluation; publish success, failure for missing coverage, or error for incomplete data.
The status target URL points to the run summary, which lists reasons, paths, references, and remediation.
Also fail the workflow job on policy failure or infrastructure error. Do not post PR comments.
Serialize all events by the normalized target branch with `queue: max` and
`cancel-in-progress: false`. GitHub retains up to 100 pending runs in this
shared lock; additional runs can be canceled when that bound is full. The
lock is intentionally broader than a PR-number or group-SHA lock so queued
label reevaluations cannot race with a merge-group evaluation for the same
target branch. Every surviving run reads current state.
Before publishing, reread the head, base, and label set. If changed, reevaluate with a bounded retry, then fail if state remains unstable.
Do not report stale successes for a new head. Metadata publication is eventually consistent; GitHub does not provide an atomic label-read/status-write transaction.
Paginate file lists and compare the count to current `changed_files`. Reject results at GitHub's 3,000-file cap and mismatched counts.
Reject truncated responses, invalid encodings, oversized documents, unsupported frontmatter, symlinks, submodules, and ambiguous IDs.
Bound artifact reads to 100 documents, 256 KiB each, and 4 MiB total. Report limits explicitly; never silently discard documents.
## Merge queue
Handle `merge_group: checks_requested` and publish the same status on `merge_group.head_sha`.
Load only trusted base code. Never evaluate coverage against the combined diff, where one PR could supply another's artifacts.
Resolve members using the paginated GraphQL merge queue entries, including each entry's `baseCommit`, `headCommit`, and `pullRequest`.
Find the entry for the event head and trace the entry commit boundaries back to the event base.
Require a complete, unambiguous chain; unknown membership produces an error, not success.
Evaluate each member's own current PR diff, artifact chain, and labels. Confirm queue identities and member heads again before publication.
Use the target-branch concurrency key shared with queued label reevaluation and
never cancel active group evaluation. The group status itself remains on the
synthetic group SHA.
Integration must validate the entry-boundary mapping against real GitHub merge-group payloads before requiring this status.
If GitHub cannot supply the expected chain for a supported queue mode, revise the mapping and fixtures before rollout.
Label removal while queued must also reevaluate every active group prefix
containing that PR, through the same group evaluator and serialization key.
Do not dequeue, requeue, or automatically merge PRs.
## Security
Permissions are `contents: read`, `pull-requests: read`, and `statuses: write` only.
Use SHA-pinned actions, GitHub-hosted runners, a trusted base checkout, and `persist-credentials: false`.
For dispatch, restrict the workflow definition to the default branch. For queue runs, pin consumed scripts to the event base SHA.
Do not install PR dependencies, execute PR files, or interpolate PR text into shell or JavaScript source.
Normalize artifact paths inside their allowed roots, bind content requests to exact repository/head identities, and escape summary text.
## Rollout and persistence
GitHub stores labels, statuses, and run summaries. No application storage or feature flag is needed.
First merge the workflow and contributor instructions. Exercise pass, failure, override, removal, fork, and merge-group cases.
Then add the status context to the `main` ruleset as a separate administrator action, preserving all existing checks and bypass rules.
Until required, it is a failing PR check but does not itself prohibit merge. Do not report ruleset enforcement as deployed before verification.
Existing open PRs need a supported event or dispatch to receive their first result.
## Related decisions and references
- [Documentation coverage policy](../../../decisions/2026-09-10-pr-documentation-coverage.md)
- [Existing merge queue contract](../../../ci-merge-queue.md)
- [GitHub workflow events](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows)
- [GitHub required status checks](https://docs.github.com/en/pull-requests/how-tos/merge-and-close-pull-requests/troubleshooting-required-status-checks)
- [GitHub merge queue API fields](https://docs.github.com/en/graphql/reference/pulls)