chore: record harness learnings (#3131)

This commit is contained in:
Carlos Florêncio
2026-09-10 14:30:58 +01:00
committed by GitHub
parent b1b28d72a3
commit 39864c036c
58 changed files with 2688 additions and 548 deletions
+91 -8
View File
@@ -26,7 +26,28 @@ SIZE_EXCEPTIONS_PATH = "docs/specs/spec-lint-exceptions.tsv"
REQ_ID_PATTERN = r"REQ-[A-Z0-9]+(?:-[A-Z0-9]+)*-\d{3}"
AC_ID_PATTERN = r"AC-[A-Z0-9]+(?:-[A-Z0-9]+)*-\d{3}\.\d+"
REQ_HEADING = re.compile(rf"^###\s+(?P<id>{REQ_ID_PATTERN}):\s+\S")
AC_DEFINITION = re.compile(rf"^\s*-\s+\*\*(?P<id>{AC_ID_PATTERN}):\*\*\s+\S")
AC_DEFINITION = re.compile(
rf"^\s*-\s+\*\*(?P<id>{AC_ID_PATTERN}):\*\*\s+(?P<body>\S.*)$"
)
MIGRATED_SOURCE_HEADING = re.compile(r"^\s*##\s+Migrated source detail\s*$", re.IGNORECASE)
GENERIC_AC_PATTERNS = (
re.compile(
r"\b(?:all|any|every|remaining|other)\s+(?:the\s+)?"
r"(?:behavior|behaviour|requirements?|details?|scenarios?)\b.*"
r"\b(?:legacy|original|source|copied|elsewhere)\b",
re.IGNORECASE,
),
re.compile(
r"\b(?:as|per)\s+(?:described|specified|documented)\b.*"
r"\b(?:legacy|original|source|elsewhere|above)\b",
re.IGNORECASE,
),
re.compile(
r"\b(?:see|refer(?:s)?|defer(?:s)?)\b.*"
r"\b(?:legacy|original|source|copied|elsewhere)\b",
re.IGNORECASE,
),
)
def load_config(root: Path) -> dict:
@@ -38,6 +59,7 @@ def lint_specs(
root: Path,
config: dict | None = None,
exceptions: dict[str, int] | None = None,
migration_audit: bool = False,
) -> list[Violation]:
config = config or load_config(root)
validate_config(config)
@@ -53,7 +75,7 @@ def lint_specs(
return exception_violations
paths = sorted(path for path in specs_root.rglob("*.md") if path.is_file())
migrated_systems = set()
system_migrations: dict[str, str] = {}
for path in paths:
inside = path.relative_to(specs_root).parts
if (
@@ -67,9 +89,9 @@ def lint_specs(
error is None
and metadata is not None
and metadata.get("specification_version") == "1"
and metadata.get("migration") == "complete"
and metadata.get("migration") in VALID_MIGRATION_STATUSES
):
migrated_systems.add(inside[0])
system_migrations[inside[0]] = str(metadata["migration"])
violations: list[Violation] = []
requirement_definitions: dict[str, list[tuple[Path, int]]] = {}
acceptance_definitions: dict[str, list[tuple[Path, int]]] = {}
@@ -82,7 +104,7 @@ def lint_specs(
kind, system = classify_path(relative)
violations.extend(check_size(path, relative, kind, config, exceptions))
inside = relative.relative_to("docs/specs").parts
if inside and inside[0] in migrated_systems and kind == "legacy":
if inside and system_migrations.get(inside[0]) == "complete" and kind == "legacy":
violations.append(
Violation(
"system-layout",
@@ -108,6 +130,11 @@ def lint_specs(
systems_with_new_artifacts.add(system)
text = path.read_text(encoding="utf-8")
migration = system_migrations.get(system) if system is not None else None
if migration is not None:
violations.extend(
check_migration_content(path, text, migration, migration_audit)
)
metadata, metadata_line, metadata_error = parse_frontmatter(text)
if metadata_error:
violations.append(Violation("frontmatter", path, metadata_line, metadata_error))
@@ -213,6 +240,50 @@ def lint_specs(
return sorted(violations, key=lambda item: (item.path.as_posix(), item.line, item.rule))
def check_migration_content(
path: Path, text: str, migration: str, migration_audit: bool
) -> list[Violation]:
"""Check canonical artifacts for migration-only content debt.
In-progress systems are intentionally opt-in so existing migrations can be
audited without making the normal specification lint fail. Completed
systems treat the same findings as blocking content violations.
"""
if migration == "in_progress" and not migration_audit:
return []
if migration not in VALID_MIGRATION_STATUSES:
return []
rule = "migration-audit" if migration == "in_progress" else "migration-content"
violations: list[Violation] = []
for line_number, line in enumerate(text.splitlines(), start=1):
if MIGRATED_SOURCE_HEADING.match(line):
violations.append(
Violation(
rule,
path,
line_number,
"remove the `Migrated source detail` wrapper after migration; "
"keep authoritative content in named requirement or design sections",
)
)
acceptance_match = AC_DEFINITION.match(line)
if acceptance_match and any(
pattern.search(acceptance_match.group("body"))
for pattern in GENERIC_AC_PATTERNS
):
violations.append(
Violation(
rule,
path,
line_number,
f"{acceptance_match.group('id')} must state observable behavior "
"directly; do not defer to generic or copied legacy prose",
)
)
return violations
def validate_config(config: dict) -> None:
if config.get("version") != 1:
raise ValueError("specification lint configuration must use version 1")
@@ -660,19 +731,31 @@ def check_duplicate_ids(kind: str, definitions: dict[str, list[tuple[Path, int]]
def main() -> int:
parser = argparse.ArgumentParser(description="Lint Kandev specification files.")
parser.add_argument("--all", action="store_true", help="Lint all specification files")
parser.add_argument(
"--migration-audit",
action="store_true",
help="Report non-blocking content debt in in-progress migrations",
)
args = parser.parse_args()
if not args.all:
parser.print_usage(sys.stderr)
return 2
root = Path.cwd()
violations = lint_specs(root)
violations = lint_specs(root, migration_audit=args.migration_audit)
for violation in violations:
relative = violation.path.relative_to(root)
print(f"{relative}:{violation.line}: [{violation.rule}] {violation.message}")
if violations:
print(f"\n{len(violations)} specification lint violation(s) found.", file=sys.stderr)
blocking = [violation for violation in violations if violation.rule != "migration-audit"]
if blocking:
print(f"\n{len(blocking)} specification lint violation(s) found.", file=sys.stderr)
return 1
if violations:
print(
f"Specification files passed; {len(violations)} non-blocking migration "
"audit finding(s) reported."
)
return 0
print("All specification files passed.")
return 0
+110 -5
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
import importlib.util
import json
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path
@@ -78,15 +80,15 @@ requirements:
# Dependencies System Design
"""
def valid_system_index(self) -> str:
def valid_system_index(self, migration: str = "in_progress") -> str:
return """---
status: draft
system: task-system
specification_version: 1
migration: in_progress
migration: {migration}
---
# Task System
"""
""".format(migration=migration)
def rules(self, violations) -> list[str]:
return [violation.rule for violation in violations]
@@ -374,7 +376,7 @@ migration: in_progress
def test_rejects_a_stray_markdown_file_in_a_migrated_system(self) -> None:
self.write(
"docs/specs/task-system/README.md",
"---\nstatus: active\nsystem: task-system\nspecification_version: 1\nmigration: complete\n---\n# Task System\n",
self.valid_system_index("complete"),
)
self.write("docs/specs/task-system/spec.md", "# Generic spec\n")
@@ -385,7 +387,7 @@ migration: in_progress
def test_allows_legacy_files_while_a_system_migration_is_in_progress(self) -> None:
self.write(
"docs/specs/task-system/README.md",
"---\nstatus: active\nsystem: task-system\nspecification_version: 1\nmigration: in_progress\n---\n# Task System\n",
self.valid_system_index(),
)
self.write("docs/specs/task-system/spec.md", "# Legacy spec\n")
@@ -401,6 +403,109 @@ migration: in_progress
self.assertIn("system-index-migration", self.rules(violations))
def test_completed_migration_accepts_real_requirement_and_design_content(self) -> None:
self.write("docs/specs/task-system/README.md", self.valid_system_index("complete"))
self.write("docs/specs/task-system/requirements/dependencies.md", self.valid_requirement())
self.write("docs/specs/task-system/system-design/dependencies.md", self.valid_design())
self.assertNotIn(
"migration-content",
self.rules(load_linter().lint_specs(self.root, self.config)),
)
def test_completed_migration_allows_observable_quantifier_content(self) -> None:
self.write("docs/specs/task-system/README.md", self.valid_system_index("complete"))
requirement = self.valid_requirement().replace(
"When a predecessor is pending, the system shall wait.",
"The response includes all details returned by the provider.",
)
self.write("docs/specs/task-system/requirements/dependencies.md", requirement)
self.write("docs/specs/task-system/system-design/dependencies.md", self.valid_design())
self.assertNotIn(
"migration-content",
self.rules(load_linter().lint_specs(self.root, self.config)),
)
def test_completed_migration_rejects_generic_criteria_and_source_wrapper(self) -> None:
self.write("docs/specs/task-system/README.md", self.valid_system_index("complete"))
requirement = self.valid_requirement().replace(
"When a predecessor is pending, the system shall wait.",
"The system shall preserve all behavior described in the legacy source.",
)
self.write("docs/specs/task-system/requirements/dependencies.md", requirement)
self.write(
"docs/specs/task-system/system-design/dependencies.md",
self.valid_design() + "\n## Migrated source detail\n\nCopied source.\n",
)
violations = load_linter().lint_specs(self.root, self.config)
migration_violations = [
violation for violation in violations if violation.rule == "migration-content"
]
self.assertEqual(len(migration_violations), 2)
self.assertEqual(migration_violations[0].line, 12)
self.assertEqual(migration_violations[1].line, 10)
self.assertTrue(
all(
"legacy" in violation.message or "wrapper" in violation.message
for violation in migration_violations
)
)
def test_migration_audit_reports_in_progress_debt_without_blocking_normal_lint(self) -> None:
self.write("docs/specs/task-system/README.md", self.valid_system_index())
requirement = self.valid_requirement().replace(
"When a predecessor is pending, the system shall wait.",
"The system shall preserve all behavior described in the legacy source.",
)
self.write("docs/specs/task-system/requirements/dependencies.md", requirement)
self.write(
"docs/specs/task-system/system-design/dependencies.md",
self.valid_design() + "\n## Migrated source detail\n",
)
linter = load_linter()
self.assertEqual(linter.lint_specs(self.root, self.config), [])
audit_rules = self.rules(linter.lint_specs(self.root, self.config, migration_audit=True))
self.assertEqual(audit_rules, ["migration-audit", "migration-audit"])
def test_templates_and_historical_docs_are_outside_the_migration_content_gate(self) -> None:
self.write("docs/specs/task-system/README.md", self.valid_system_index("complete"))
self.write(
"docs/specs/templates/old-template.md",
"## Migrated source detail\n- **AC-TASK-DEP-001.1:** see the legacy source.\n",
)
self.write(
"docs/specs/legacy/historical.md",
"## Migrated source detail\n- **AC-TASK-DEP-001.1:** see the legacy source.\n",
)
violations = load_linter().lint_specs(self.root, self.config)
self.assertNotIn("migration-content", self.rules(violations))
def test_migration_audit_cli_keeps_audit_findings_non_blocking(self) -> None:
self.write("docs/specs/task-system/README.md", self.valid_system_index())
self.write("docs/specs/task-system/requirements/dependencies.md", self.valid_requirement())
self.write(
"docs/specs/task-system/system-design/dependencies.md",
self.valid_design() + "\n## Migrated source detail\n",
)
self.write("docs/specs/spec-lint.json", json.dumps(self.config))
result = subprocess.run(
[sys.executable, str(SCRIPT), "--all", "--migration-audit"],
cwd=self.root,
capture_output=True,
text=True,
check=False,
)
self.assertEqual(result.returncode, 0)
self.assertIn("non-blocking migration audit finding", result.stdout)
if __name__ == "__main__":
unittest.main()
+151 -11
View File
@@ -5,12 +5,15 @@
# scripts/pr-await <PR> [options]
#
# Options:
# --mode all-terminal | first-failure
# --mode all-terminal | strict-deadline | first-failure
# all-terminal (default) waits until no check is pending,
# so every failure lands in a single report and a single
# fix pass. first-failure returns as soon as one check
# fails; it exists for the interactive case and costs an
# extra full CI cycle per failure.
# extra full CI cycle per failure. strict-deadline keeps
# collecting state until the deadline, even after checks
# become terminal; it is for an explicit fixed-duration
# wait.
# --deadline-min N Hard stop, in minutes (default 45).
# --interval-sec N Seconds between polls (default 60).
# --format text | json Report format (default text).
@@ -27,7 +30,7 @@
# Exit codes:
# 0 terminal and clean - no failed checks, no unresolved threads
# 1 terminal with findings - failures and/or review threads to act on
# 2 deadline reached - checks still pending; names reported
# 2 deadline reached - checks still pending or terminal rollup not confirmed
# 3 blocked - merged/closed, approval required, or access lost
#
# Requires: gh (authenticated), jq, and scripts/pr-state alongside this script.
@@ -103,7 +106,7 @@ while [[ $# -gt 0 ]]; do
shift
[[ $# -gt 0 ]] || usage 1
case "$1" in
all-terminal | first-failure) MODE=$1 ;;
all-terminal | strict-deadline | first-failure) MODE=$1 ;;
*) usage 1 ;;
esac
shift
@@ -157,6 +160,55 @@ progress() {
[[ "$QUIET" -eq 1 ]] || printf '%s\n' "$*" >&2
}
# A current-head check rollup can briefly contain only the first materialized
# workflows while reporting no failed or pending checks. Require two identical
# terminal snapshots so a later workflow registration is observed before the
# caller receives a clean verdict.
TERMINAL_ROLLUP_FINGERPRINT=''
TERMINAL_ROLLUP_STREAK=0
TERMINAL_ROLLUP_PROVISIONAL=0
terminal_rollup_fingerprint() {
jq -c '
def stable_checks:
sort_by([
(.workflow // ""),
(.name // ""),
(.status // ""),
(.conclusion // ""),
((.run_id // "") | tostring),
((.job_id // "") | tostring)
]);
{
check_count: (.check_count // 0),
passed_check_count: (.passed_check_count // 0),
successful_checks: ((.successful_checks // []) | stable_checks),
terminal_checks: ((.terminal_checks // []) | stable_checks),
failed_checks: ((.failed_checks // []) | stable_checks),
pending_checks: ((.pending_checks // []) | stable_checks)
}' <<<"$1"
}
augment_required_status_checks() {
# pr-state performs this augmentation in production. Keep this defensive
# fallback for raw snapshots supplied by tests or another snapshot producer.
jq -c '
if (.required_status_checks_known // false) != true then
.
else
([.successful_checks[]?, .terminal_checks[]?, .failed_checks[]?, .pending_checks[]? | .name] | unique) as $reported
| ([.required_status_checks[]?
| . as $required
| select(($reported | index($required)) == null)
| {name: $required, workflow: null, status: "missing", run_id: null,
job_id: null, details_url: null, target_url: null}
]) as $missing
| .pending_checks = ((.pending_checks // []) + $missing)
| .check_count = ((.check_count // 0) + ($missing | length))
end
'
}
# Mergeability is deliberately absent from `pr-state --summary`, so it is
# queried separately below rather than letting a caller assume a clean merge
# state. There is no `|| {}` fallback: a failed query is unknown state, and the
@@ -350,6 +402,21 @@ trap 'handle_signal 130' INT
trap 'handle_signal 143' TERM
DEADLINE_SEC=$((DEADLINE_MIN * 60))
# Tests can use a short deterministic deadline without making the public
# minute-based interface less clear. This is intentionally not documented as a
# user-facing option.
if [[ -n "${PR_AWAIT_DEADLINE_SEC:-}" ]]; then
[[ "${PR_AWAIT_DEADLINE_SEC}" =~ ^[0-9]+$ ]] || usage 1
DEADLINE_SEC=$((10#${PR_AWAIT_DEADLINE_SEC}))
fi
strict_deadline_reached() {
if [[ "$MODE" == 'strict-deadline' && "$TERMINAL_ROLLUP_STREAK" -ge 2 ]]; then
OUTCOME='strict-terminal'
else
OUTCOME='deadline'
fi
}
# Sleep until the next poll unless the deadline arrives first, capping the last
# sleep to whatever time is left. Returns 1 when the caller must stop, so a
@@ -370,6 +437,7 @@ MERGE_ERRORS=0
MERGE_TIMED_OUT=0
EVIDENCE_ERRORS=0
MERGE_UNKNOWN=0
REQUIRED_STATUS_UNKNOWN=0
HEAD_CHANGES=0
FIRST_HEAD=''
SUMMARY_JSON=''
@@ -387,7 +455,7 @@ while :; do
budget="$(remaining_budget)"
if [[ "$budget" -le 0 ]]; then
if [[ -n "$SUMMARY_JSON" ]]; then
OUTCOME='deadline'
strict_deadline_reached
break
fi
floor_start=$SECONDS
@@ -425,6 +493,7 @@ while :; do
fi
continue
fi
summary_body="$(augment_required_status_checks <<<"$summary_body")"
SUMMARY_JSON="$summary_body"
CONSECUTIVE_ERRORS=0
@@ -447,7 +516,7 @@ while :; do
read_rc=0
run_bounded_capture "$budget" "$READ_TMP" \
"$GH" pr view "$PR" --json state,mergeable,mergeStateStatus,headRefOid || read_rc=$?
"$GH" pr view "$PR" --json state,mergeable,mergeStateStatus,reviewDecision,headRefOid || read_rc=$?
if [[ "$read_rc" -eq 124 ]]; then
progress "poll ${POLLS}: gh pr view exceeded its ${budget}s bound"
OUTCOME='blocked-merge-state'
@@ -492,6 +561,9 @@ while :; do
# deadline reached before the next read print those checks under the new
# SHA; with no snapshot the run reports blocked, which is the truth.
SUMMARY_JSON=''
TERMINAL_ROLLUP_FINGERPRINT=''
TERMINAL_ROLLUP_STREAK=0
TERMINAL_ROLLUP_PROVISIONAL=0
if ! sleep_or_stop; then
OUTCOME='deadline'
break
@@ -529,6 +601,22 @@ while :; do
continue
fi
required_status_known="$(jq -r 'if .required_status_checks_known == true then "yes" else "no" end' <<<"$SUMMARY_JSON")"
if [[ "$required_status_known" != 'yes' ]]; then
REQUIRED_STATUS_UNKNOWN=$((REQUIRED_STATUS_UNKNOWN + 1))
progress "poll ${POLLS}: required status policy is unavailable (${REQUIRED_STATUS_UNKNOWN} in a row)"
if [[ "$REQUIRED_STATUS_UNKNOWN" -ge 3 ]]; then
OUTCOME='blocked-required-statuses'
break
fi
if ! sleep_or_stop; then
OUTCOME='blocked-required-statuses'
break
fi
continue
fi
REQUIRED_STATUS_UNKNOWN=0
if [[ "$(jq -r '.approval_required_runs | length' <<<"$SUMMARY_JSON")" -gt 0 ]]; then
# action_required reads as green-or-skipped in a naive check count. It is
# blocked verification: waiting for it to finish would never terminate.
@@ -540,6 +628,9 @@ while :; do
# empty rollup satisfies "nothing pending" without any evidence behind it;
# in this repository the required jobs always run, so zero means not yet.
if [[ "$check_count" -eq 0 ]]; then
TERMINAL_ROLLUP_FINGERPRINT=''
TERMINAL_ROLLUP_STREAK=0
TERMINAL_ROLLUP_PROVISIONAL=0
progress "poll ${POLLS}: no checks registered at this head yet; waiting"
if ! sleep_or_stop; then
OUTCOME='deadline'
@@ -548,6 +639,12 @@ while :; do
continue
fi
if [[ "$pending" -gt 0 || "$complete" != 'true' ]]; then
TERMINAL_ROLLUP_FINGERPRINT=''
TERMINAL_ROLLUP_STREAK=0
TERMINAL_ROLLUP_PROVISIONAL=0
fi
# A failed job can be visible while its parent workflow is still running, so
# in all-terminal mode a failure is never on its own a reason to stop.
if [[ "$MODE" == 'first-failure' && "$failed" -gt 0 ]]; then
@@ -578,6 +675,9 @@ while :; do
snapshot_errors="$(jq -r '.errors // [] | length' <<<"$SUMMARY_JSON")"
threads_known="$(jq -r 'if .unresolved_review_thread_count == null then "no" else "yes" end' <<<"$SUMMARY_JSON")"
if [[ "$snapshot_errors" -gt 0 || "$threads_known" == 'no' ]]; then
TERMINAL_ROLLUP_FINGERPRINT=''
TERMINAL_ROLLUP_STREAK=0
TERMINAL_ROLLUP_PROVISIONAL=0
EVIDENCE_ERRORS=$((EVIDENCE_ERRORS + 1))
progress "poll ${POLLS}: checks terminal but evidence incomplete (${EVIDENCE_ERRORS} in a row)"
if [[ "$EVIDENCE_ERRORS" -ge 3 ]]; then
@@ -595,6 +695,9 @@ while :; do
# GitHub reports UNKNOWN while it is still computing mergeability. Deciding
# on it would call a PR clean before the answer exists.
if [[ "$(jq -r '.mergeable // "UNKNOWN"' <<<"$MERGE_JSON")" == 'UNKNOWN' ]]; then
TERMINAL_ROLLUP_FINGERPRINT=''
TERMINAL_ROLLUP_STREAK=0
TERMINAL_ROLLUP_PROVISIONAL=0
MERGE_UNKNOWN=$((MERGE_UNKNOWN + 1))
progress "poll ${POLLS}: mergeability still UNKNOWN (${MERGE_UNKNOWN} in a row)"
if [[ "$MERGE_UNKNOWN" -ge 3 ]]; then
@@ -609,8 +712,34 @@ while :; do
fi
MERGE_UNKNOWN=0
OUTCOME='terminal'
break
terminal_fingerprint="$(terminal_rollup_fingerprint "$SUMMARY_JSON")"
if [[ "$terminal_fingerprint" == "$TERMINAL_ROLLUP_FINGERPRINT" ]]; then
TERMINAL_ROLLUP_STREAK=$((TERMINAL_ROLLUP_STREAK + 1))
else
TERMINAL_ROLLUP_FINGERPRINT="$terminal_fingerprint"
TERMINAL_ROLLUP_STREAK=1
fi
if [[ "$TERMINAL_ROLLUP_STREAK" -lt 2 ]]; then
TERMINAL_ROLLUP_PROVISIONAL=1
progress "poll ${POLLS}: terminal check rollup is provisional; waiting for a matching snapshot"
if ! sleep_or_stop; then
OUTCOME='deadline'
break
fi
continue
fi
TERMINAL_ROLLUP_PROVISIONAL=0
if [[ "$MODE" != 'strict-deadline' ]]; then
OUTCOME='terminal'
break
fi
progress "poll ${POLLS}: terminal evidence collected; continuing through the strict deadline"
if ! sleep_or_stop; then
strict_deadline_reached
break
fi
continue
fi
if ! sleep_or_stop; then
@@ -665,6 +794,7 @@ threads_known_final="$(jq -r 'if .unresolved_review_thread_count == null then "n
base_advanced="$(jq -r 'if .pr.base_advanced_since_head == true then 1 else 0 end' <<<"$SUMMARY_JSON")"
mergeable="$(jq -r '.mergeable // "UNKNOWN"' <<<"$MERGE_JSON")"
merge_status="$(jq -r '.mergeStateStatus // "UNKNOWN"' <<<"$MERGE_JSON")"
review_decision="$(jq -r '.reviewDecision // "UNKNOWN"' <<<"$MERGE_JSON")"
conflicted=0
[[ "$mergeable" == 'CONFLICTING' || "$merge_status" == 'DIRTY' ]] && conflicted=1
@@ -675,8 +805,9 @@ conflicted=0
findings=$((failed_n + threads_total_n + comments_n + conflicted + changes_req_n + blocked_rev_n + base_advanced))
case "$OUTCOME" in
blocked-pr-state | blocked-approval | blocked-evidence | blocked-merge-unknown) code=3 ;;
blocked-pr-state | blocked-approval | blocked-evidence | blocked-merge-unknown | blocked-required-statuses) code=3 ;;
deadline) code=2 ;;
strict-terminal) [[ "$findings" -gt 0 ]] && code=1 || code=0 ;;
*) [[ "$findings" -gt 0 ]] && code=1 || code=0 ;;
esac
@@ -712,7 +843,7 @@ printf 'gate: %s | outcome: %s | waited %dm%02ds over %d poll(s)\n' \
"$MODE" "$OUTCOME" "$((WAITED / 60))" "$((WAITED % 60))" "$POLLS"
[[ "$HEAD_CHANGES" -gt 0 ]] && printf 'note: head moved %d time(s) during the wait\n' "$HEAD_CHANGES"
printf 'checks: %s passed, %s failed, %s pending\n' "$passed_n" "$failed_n" "$pending_n"
printf 'merge: %s / %s\n' "$mergeable" "$merge_status"
printf 'merge: %s / %s | review: %s\n' "$mergeable" "$merge_status" "$review_decision"
printf 'toolchain: %s\n' "$TOOLCHAIN"
if [[ "$failed_n" -gt 0 ]]; then
@@ -761,16 +892,25 @@ case "$OUTCOME" in
"${error_sources:-null unresolved thread count}" ;;
blocked-merge-unknown)
printf 'GitHub still reports mergeability as UNKNOWN. Re-run before treating this head as clean.\n' ;;
blocked-required-statuses)
printf 'required status check policy is unavailable. Treat the check rollup as unknown, not clean.\n' ;;
deadline)
if [[ "$(jq -r '.check_count // 0' <<<"$SUMMARY_JSON")" -eq 0 ]]; then
if [[ "$TERMINAL_ROLLUP_PROVISIONAL" -eq 1 ]]; then
printf 'deadline hit while confirming the terminal check rollup. Re-run or raise --deadline-min.\n'
elif [[ "$(jq -r '.check_count // 0' <<<"$SUMMARY_JSON")" -eq 0 ]]; then
printf 'deadline hit with no checks registered yet at this head. Re-run or raise --deadline-min.\n'
else
printf 'deadline hit with %s check(s) pending. Re-run or raise --deadline-min.\n' "$pending_n"
fi
;;
strict-terminal)
printf 'fixed-duration wait ended with terminal evidence; review the findings above.\n'
;;
*)
if [[ "$conflicted" -eq 1 ]]; then
printf 'resolve the merge conflict before triaging checks or threads.\n'
elif [[ "$findings" -eq 0 && "$review_decision" == 'REVIEW_REQUIRED' && "$merge_status" == 'BLOCKED' ]]; then
printf 'checks and reviews are clean, but human approval is required before merge.\n'
elif [[ "$findings" -eq 0 ]]; then
printf 'clean at this head.\n'
elif [[ "$failed_n" -eq 0 && "$threads_total_n" -eq 0 ]]; then
+134 -8
View File
@@ -88,6 +88,8 @@ snapshot() {
failed_checks: $f, pending_checks: $p, passed_check_count: $passed,
check_count: ($passed + $failed + $pending),
checks_head_sha: $head, checks_snapshot_complete: $complete,
terminal_checks: [],
required_status_checks: [], required_status_checks_known: true,
approval_required_runs: [], unresolved_review_thread_count: 0,
unresolved_threads: [], hidden_unresolved_threads: [],
actionable_issue_comment_count: 0, errors: [],
@@ -98,7 +100,7 @@ snapshot() {
run_await() {
local dir=$1; shift
FAKE_DIR="$dir" PR_AWAIT_PR_STATE="$dir/pr-state" PR_AWAIT_GH="$dir/gh" \
PR_AWAIT_SLEEP="$dir/sleep" \
PR_AWAIT_SLEEP="${PR_AWAIT_SLEEP_OVERRIDE:-$dir/sleep}" \
PR_AWAIT_JQ="${PROBE_JQ:-$dir/jq-ok}" PR_AWAIT_PROBE_BASH="${PROBE_BASH:-$dir/bash-ok}" \
"$SCRIPT" "$@"
}
@@ -175,8 +177,8 @@ snapshot "$d" 1 20 0 0 false
snapshot "$d" 2 20 0 0 true
out="$(run_await "$d" 12 --interval-sec 1 2>"$d/err")" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "expected 0 after snapshot completed, got $rc" "$out"
grep -q '2 poll' <<<"$out" || fail "should have polled twice, not stopped on the incomplete snapshot" "$out"
pass "checks_snapshot_complete=false keeps waiting"
grep -q '3 poll' <<<"$out" || fail "should confirm the terminal rollup after the incomplete snapshot" "$out"
pass "checks_snapshot_complete=false and the first terminal snapshot keep waiting"
# --- deadline ---------------------------------------------------------------
d="$(make_tmp_dir)"; setup_fake "$d"
@@ -187,6 +189,50 @@ grep -q 'STILL PENDING' <<<"$out" || fail "deadline should name pending checks"
grep -q 'Pending 0' <<<"$out" || fail "deadline should list check names" "$out"
pass "deadline exits 2 and names the pending checks"
# --- strict-deadline holds a fixed-duration request after terminal CI --------
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 20 0 0
cat >"$d/slow-sleep" <<'SLOWSLEEP'
#!/usr/bin/env bash
printf 'slept %s\n' "$1" >> "$FAKE_DIR/sleeps"
/usr/bin/sleep "$1"
SLOWSLEEP
chmod +x "$d/slow-sleep"
out="$(PR_AWAIT_DEADLINE_SEC=5 PR_AWAIT_SLEEP_OVERRIDE="$d/slow-sleep" \
run_await "$d" 12 --mode strict-deadline --interval-sec 1 --quiet --format json 2>/dev/null)" \
&& rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "strict-deadline clean terminal should exit 0, got $rc" "$out"
[[ "$(jq -r '.outcome' <<<"$out")" == 'strict-terminal' ]] \
|| fail "strict-deadline should report its fixed-duration terminal outcome" "$out"
[[ "$(jq -r '.mode' <<<"$out")" == 'strict-deadline' ]] \
|| fail "strict-deadline mode should be recorded" "$out"
[[ "$(wc -l < "$d/sleeps" | tr -d ' ')" -ge 1 ]] \
|| fail "strict-deadline should wait after terminal evidence" "$out"
pass "strict-deadline holds a terminal-clean wait through its requested duration"
# --- strict-deadline returns findings or pending work with distinct statuses -
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 20 1 0
cat >"$d/slow-sleep" <<'SLOWSLEEP'
#!/usr/bin/env bash
/usr/bin/sleep "$1"
SLOWSLEEP
chmod +x "$d/slow-sleep"
out="$(PR_AWAIT_DEADLINE_SEC=5 PR_AWAIT_SLEEP_OVERRIDE="$d/slow-sleep" \
run_await "$d" 12 --mode strict-deadline \
--interval-sec 1 --quiet --format json 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 1 ]] || fail "strict-deadline terminal findings should exit 1, got $rc" "$out"
[[ "$(jq -r '.summary.failed_checks | length' <<<"$out")" == '1' ]] \
|| fail "strict-deadline should preserve terminal findings" "$out"
pass "strict-deadline reports terminal findings as exit 1"
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 10 0 1
out="$(PR_AWAIT_DEADLINE_SEC=0 run_await "$d" 12 --mode strict-deadline \
--interval-sec 1 --quiet --format json 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 2 ]] || fail "strict-deadline pending work should exit 2, got $rc" "$out"
pass "strict-deadline reports pending work as exit 2"
# --- approval-required is blocked, not green --------------------------------
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 0 0 0 true aaaaaaaaaaaa \
@@ -214,6 +260,16 @@ out="$(run_await "$d" 12 --interval-sec 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
grep -q 'resolve the merge conflict' <<<"$out" || fail "conflict should lead the NEXT line" "$out"
pass "a merge conflict is surfaced even when every check passed"
# --- clean checks can still require human approval --------------------------
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 20 0 0
printf '{"state":"OPEN","mergeable":"MERGEABLE","mergeStateStatus":"BLOCKED","reviewDecision":"REVIEW_REQUIRED","headRefOid":"aaaaaaaaaaaa"}' > "$d/gh.json"
out="$(run_await "$d" 12 --interval-sec 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "approval-gated clean checks should exit 0, got $rc" "$out"
grep -q 'review: REVIEW_REQUIRED' <<<"$out" || fail "wait output should expose the review decision" "$out"
grep -qi 'human approval is required' <<<"$out" || fail "wait output should distinguish the approval gate" "$out"
pass "clean checks report the human approval gate without a code failure"
# --- transient pr-state failures recover; sustained ones block --------------
d="$(make_tmp_dir)"; setup_fake "$d"
: > "$d/seq/1.fail"; snapshot "$d" 1 0 0 0
@@ -282,8 +338,8 @@ d="$(make_tmp_dir)"; setup_fake "$d"
for i in 1 2 3 4 5 6 7; do snapshot "$d" "$i" "$i" 0 $((8 - i)); done
snapshot "$d" 8 20 0 0
run_await "$d" 12 --interval-sec 1 --quiet >/dev/null 2>&1 || true
[[ "$(wc -l < "$d/sleeps" | tr -d ' ')" -eq 7 ]] || fail "expected 7 internal sleeps" "$(cat "$d/sleeps")"
pass "eight polls happen inside a single invocation (7 internal sleeps, 0 caller round-trips)"
[[ "$(wc -l < "$d/sleeps" | tr -d ' ')" -eq 8 ]] || fail "expected 8 internal sleeps" "$(cat "$d/sleeps")"
pass "nine polls happen inside a single invocation (8 internal sleeps, 0 caller round-trips)"
# --- P1: a snapshot carrying errors is never clean ---------------------------
d="$(make_tmp_dir)"; setup_fake "$d"
@@ -605,8 +661,8 @@ for i in 1 2 3; do snapshot "$d" "$i" 0 0 0; done
snapshot "$d" 4 20 0 0
out="$(run_await "$d" 12 --interval-sec 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "expected clean once checks appeared, got $rc" "$out"
grep -q '4 poll' <<<"$out" || fail "must wait for checks to appear, not stop at the empty rollup" "$out"
pass "an empty check rollup keeps waiting instead of reporting clean"
grep -q '5 poll' <<<"$out" || fail "must wait for checks to appear and stabilize, not stop at the empty rollup" "$out"
pass "an empty check rollup keeps waiting until the terminal rollup stabilizes"
d="$(make_tmp_dir)"; setup_fake "$d"
for i in 1 2 3 4 5; do snapshot "$d" "$i" 0 0 0; done
@@ -625,12 +681,82 @@ pass "a zero-check deadline is distinguished from a deadline with pending checks
for conclusion in skipped neutral; do
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 0 0 0 true aaaaaaaaaaaa "{\"check_count\":1,\"terminal_conclusions\":[\"$conclusion\"]}"
out="$(run_await "$d" 12 --interval-sec 1 --deadline-min 0 --quiet --format json 2>/dev/null)" && rc=0 || rc=$?
out="$(run_await "$d" 12 --interval-sec 1 --deadline-min 1 --quiet --format json 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "$conclusion-only check rollup should be terminal, got $rc" "$out"
[[ "$(jq -r '.outcome' <<<"$out")" == 'terminal' ]] || fail "$conclusion-only rollup should emit terminal" "$out"
done
pass "skipped and neutral terminal checks do not look like an empty rollup"
# --- a sparse non-empty rollup is provisional -------------------------------
# Immediately after a push, GitHub can report a couple of terminal checks
# before the rest of the required workflows materialize. A non-zero check count
# must not be treated as proof that the rollup is complete.
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 2 0 0 true aaaaaaaaaaaa
snapshot "$d" 2 45 0 0 true aaaaaaaaaaaa
out="$(run_await "$d" 12 --interval-sec 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "sparse rollup should settle to clean, got $rc" "$out"
grep -q '45 passed' <<<"$out" || fail "must report the materialized full rollup, not the sparse one" "$out"
grep -q '3 poll' <<<"$out" || fail "must confirm the expanded rollup before returning" "$out"
pass "a sparse non-empty rollup stays provisional until the full set stabilizes"
# The set of checks is logical state, not an API-order-sensitive sequence. A
# reordered response must not reset the terminal confirmation streak.
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 2 0 0 true aaaaaaaaaaaa \
'{"successful_checks":[{"name":"check-b","workflow":"ci","status":"completed","conclusion":"success"},{"name":"check-a","workflow":"ci","status":"completed","conclusion":"success"}]}'
snapshot "$d" 2 2 0 0 true aaaaaaaaaaaa \
'{"successful_checks":[{"name":"check-a","workflow":"ci","status":"completed","conclusion":"success"},{"name":"check-b","workflow":"ci","status":"completed","conclusion":"success"}]}'
out="$(run_await "$d" 12 --interval-sec 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "reordered terminal checks should settle clean, got $rc" "$out"
grep -q '2 poll' <<<"$out" || fail "reordered terminal checks should not reset the confirmation streak" "$out"
pass "terminal rollup fingerprint ignores API ordering"
# --- required contexts prevent an optional-only rollup from becoming clean ---
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 1 0 0 true aaaaaaaaaaaa \
'{"required_status_checks":["E2E Tests Passed"],"required_status_checks_known":true,
"successful_checks":[{"name":"CodeRabbit","workflow":"review","status":"completed","conclusion":"success"}]}'
out="$(run_await "$d" 12 --interval-sec 1 --deadline-min 0 --quiet --format json 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 2 ]] || fail "an optional-only rollup must hit the deadline, got $rc" "$out"
[[ "$(jq -r '.outcome' <<<"$out")" == 'deadline' ]] || fail "optional-only rollup outcome missing" "$out"
grep -q 'E2E Tests Passed' <<<"$(jq -r '.summary.pending_checks[].name' <<<"$out")" \
|| fail "missing required context should remain pending" "$out"
pass "an optional-only rollup does not become clean while required contexts are absent"
d="$(make_tmp_dir)"; setup_fake "$d"
for i in 1 2 3; do
snapshot "$d" "$i" 1 0 0 true aaaaaaaaaaaa \
'{"required_status_checks_known":false}'
done
out="$(run_await "$d" 12 --interval-sec 1 --deadline-min 1 --quiet --format json 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 3 ]] || fail "an unavailable required-status policy should block, got $rc" "$out"
[[ "$(jq -r '.outcome' <<<"$out")" == 'blocked-required-statuses' ]] \
|| fail "an unavailable required-status policy should report its blocked outcome" "$out"
pass "an unavailable required-status policy never becomes clean"
d="$(make_tmp_dir)"; setup_fake "$d"
terminal_required='{"check_count":1,"required_status_checks":["Skipped required"],"required_status_checks_known":true,"terminal_checks":[{"name":"Skipped required","workflow":"ci","status":"completed","conclusion":"skipped"}]}'
snapshot "$d" 1 0 0 0 true aaaaaaaaaaaa "$terminal_required"
snapshot "$d" 2 0 0 0 true aaaaaaaaaaaa "$terminal_required"
out="$(run_await "$d" 12 --interval-sec 1 --deadline-min 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "a skipped required context should be terminal, got $rc" "$out"
grep -q '0 pending' <<<"$out" || fail "a skipped required context must not be synthesized as pending" "$out"
pass "skipped required contexts remain terminal instead of pending"
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 1 0 0 true aaaaaaaaaaaa \
'{"required_status_checks":["E2E Tests Passed"],"required_status_checks_known":true,
"successful_checks":[{"name":"CodeRabbit","workflow":"review","status":"completed","conclusion":"success"}]}'
snapshot "$d" 2 2 0 0 true aaaaaaaaaaaa \
'{"required_status_checks":["E2E Tests Passed"],"required_status_checks_known":true,
"successful_checks":[{"name":"CodeRabbit","workflow":"review","status":"completed","conclusion":"success"},
{"name":"E2E Tests Passed","workflow":"e2e","status":"completed","conclusion":"success"}]}'
out="$(run_await "$d" 12 --interval-sec 1 --quiet 2>/dev/null)" && rc=0 || rc=$?
[[ "$rc" -eq 0 ]] || fail "a materialized required context should allow clean, got $rc" "$out"
grep -q '3 poll' <<<"$out" || fail "required context should settle after it materializes" "$out"
pass "a required context that materializes allows the terminal verdict"
# --- a base that advanced leaves the merge result untested -----------------
d="$(make_tmp_dir)"; setup_fake "$d"
snapshot "$d" 1 20 0 0 true aaaaaaaaaaaa '{"pr":{"base_advanced_since_head":true}}'
+52
View File
@@ -10,6 +10,10 @@
# Print the full thread/comment body plus path, line, author, commit id,
# and URL. Use before deciding how to answer a preview from list.
#
# scripts/pr-resolve reopen <PR> <thread_id>
# Reopen an already-resolved review thread after an authorized later
# change invalidates its resolution rationale.
#
# scripts/pr-resolve reply <PR> <comment_id> <thread_id> <body>
# scripts/pr-resolve reply <PR> <comment_id> <thread_id> --body-file <path>
# Reply to the thread starter, resolve the thread, and add a +1
@@ -58,6 +62,14 @@ if [[ "$cmd" == "reply" && ( $# -eq 4 || $# -eq 5 ) ]]; then
fi
fi
if [[ "$cmd" == "reopen" ]]; then
[ $# -eq 2 ] || usage
[[ "$2" == PRRT_* ]] || {
echo "reopen expects a review thread ID as its second argument" >&2
exit 1
}
fi
read -r OWNER REPO < <(gh repo view --json owner,name --jq '"\(.owner.login) \(.name)"')
case "$cmd" in
@@ -159,6 +171,46 @@ case "$cmd" in
esac
;;
reopen)
PR=$1; THREAD_ID=$2
if ! THREAD_JSON=$(gh api graphql -f query='
query($id: ID!) {
node(id: $id) {
... on PullRequestReviewThread {
id
isResolved
pullRequest { number repository { nameWithOwner } }
}
}
}' -f id="$THREAD_ID"); then
echo "failed to fetch review thread $THREAD_ID for PR #$PR" >&2
exit 1
fi
if ! jq -e --arg pr "$PR" --arg thread_id "$THREAD_ID" --arg repository "$OWNER/$REPO" '
.data.node as $thread
| if $thread == null then error("review thread \($thread_id) not found on PR #\($pr)")
elif (($thread.pullRequest.number // -1) | tostring) != $pr then
error("thread belongs to PR #\($thread.pullRequest.number), not #\($pr)")
elif (($thread.pullRequest.repository.nameWithOwner // "") != $repository) then
($thread.pullRequest.repository.nameWithOwner // "unknown") as $thread_repository
| error("thread belongs to repository \($thread_repository), not \($repository)")
elif $thread.isResolved != true then error("thread is not resolved")
else true
end' <<<"$THREAD_JSON" >/dev/null; then
exit 1
fi
if ! UNRESOLVE_JSON=$(gh api graphql -f query='mutation($t: ID!) { unresolveReviewThread(input: {threadId: $t}) { thread { isResolved } } }' \
-f t="$THREAD_ID"); then
echo "failed to reopen review thread $THREAD_ID on PR #$PR" >&2
exit 1
fi
if ! jq -e '.data.unresolveReviewThread.thread.isResolved == false' <<<"$UNRESOLVE_JSON" >/dev/null; then
echo "GitHub returned an invalid reopen response for review thread $THREAD_ID on PR #$PR" >&2
exit 1
fi
echo "ok $THREAD_ID"
;;
reply)
[[ $# -eq 4 || $# -eq 5 ]] || usage
PR=$1; COMMENT_ID=$2; THREAD_ID=$3
+129
View File
@@ -27,6 +27,10 @@ run_invalid_reply() {
PATH="$TMP_DIR/bin:$PATH" "$SCRIPT" reply 123 "$@" 2>"$TMP_DIR/stderr"
}
run_invalid_reopen() {
PATH="$TMP_DIR/bin:$PATH" "$SCRIPT" reopen 123 "$@" 2>"$TMP_DIR/stderr"
}
if PATH="$TMP_DIR/bin:$PATH" "$SCRIPT" --help >"$TMP_DIR/help" 2>"$TMP_DIR/stderr"; then
if grep -q "scripts/pr-resolve list <PR>" "$TMP_DIR/help"; then
pass "--help prints usage without gh"
@@ -37,6 +41,25 @@ else
fail "--help exits zero"
fi
if PATH="$TMP_DIR/bin:$PATH" "$SCRIPT" --help >"$TMP_DIR/help" 2>"$TMP_DIR/stderr"; then
if grep -q "scripts/pr-resolve reopen <PR> <thread_id>" "$TMP_DIR/help"; then
pass "--help documents authorized thread reopening"
else
fail "--help documents authorized thread reopening"
fi
else
fail "--help exits zero for reopen documentation"
fi
if run_invalid_reopen 456; then
fail "non-thread id in reopen position fails"
fi
if grep -q "review thread ID as its second argument" "$TMP_DIR/stderr"; then
pass "reopen validates the thread id"
else
fail "reopen validates the thread id"
fi
if run_invalid_reply PRRT_bad 456 body; then
fail "thread id in comment position fails"
fi
@@ -66,6 +89,112 @@ else
fail "empty body file has clear error"
fi
NULL_GH_DIR="$TMP_DIR/null-gh"
mkdir -p "$NULL_GH_DIR"
cat >"$NULL_GH_DIR/gh" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
if [[ "$1" == "repo" && "$2" == "view" ]]; then
printf 'kdlbs kandev\n'
exit 0
fi
if [[ "$1" == "api" && "$2" == "graphql" ]]; then
printf '%s\n' '{"data":{"node":null}}'
exit 0
fi
echo "unexpected gh arguments: $*" >&2
exit 1
EOF
chmod +x "$NULL_GH_DIR/gh"
if PATH="$NULL_GH_DIR:$PATH" "$SCRIPT" reopen 123 PRRT_missing 2>"$TMP_DIR/stderr"; then
fail "missing review thread fails clearly"
fi
if grep -q "review thread PRRT_missing not found on PR #123" "$TMP_DIR/stderr"; then
pass "missing review thread includes its id and PR"
else
fail "missing review thread includes its id and PR"
fi
REOPEN_GH_DIR="$TMP_DIR/reopen-gh"
mkdir -p "$REOPEN_GH_DIR"
cat >"$REOPEN_GH_DIR/gh" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
if [[ "$1" == "repo" && "$2" == "view" ]]; then
printf 'kdlbs kandev\n'
exit 0
fi
if [[ "$1" == "api" && "$2" == "graphql" ]]; then
if [[ "$*" == *"unresolveReviewThread"* ]]; then
case "${REOPEN_SCENARIO:-resolved}" in
mutation-failed)
echo "mutation failed" >&2
exit 1
;;
mutation-malformed)
printf '%s\n' '{"data":{"unresolveReviewThread":{"thread":null}}}'
exit 0
;;
*)
printf '%s\n' '{"data":{"unresolveReviewThread":{"thread":{"isResolved":false}}}}'
exit 0
;;
esac
fi
case "${REOPEN_SCENARIO:-resolved}" in
unresolved)
printf '%s\n' '{"data":{"node":{"id":"PRRT_target","isResolved":false,"pullRequest":{"number":123,"repository":{"nameWithOwner":"kdlbs/kandev"}}}}}'
;;
foreign)
printf '%s\n' '{"data":{"node":{"id":"PRRT_target","isResolved":true,"pullRequest":{"number":123,"repository":{"nameWithOwner":"other/kandev"}}}}}'
;;
*)
printf '%s\n' '{"data":{"node":{"id":"PRRT_target","isResolved":true,"pullRequest":{"number":123,"repository":{"nameWithOwner":"kdlbs/kandev"}}}}}'
;;
esac
exit 0
fi
echo "unexpected gh arguments: $*" >&2
exit 1
EOF
chmod +x "$REOPEN_GH_DIR/gh"
if REOPEN_SCENARIO=resolved PATH="$REOPEN_GH_DIR:$PATH" "$SCRIPT" reopen 123 PRRT_target >"$TMP_DIR/out" 2>"$TMP_DIR/stderr" \
&& grep -q "ok PRRT_target" "$TMP_DIR/out"; then
pass "reopen succeeds for a resolved same-repository thread"
else
fail "reopen succeeds for a resolved same-repository thread"
fi
if REOPEN_SCENARIO=unresolved PATH="$REOPEN_GH_DIR:$PATH" "$SCRIPT" reopen 123 PRRT_target >"$TMP_DIR/out" 2>"$TMP_DIR/stderr"; then
fail "reopen rejects an unresolved thread"
fi
grep -q "thread is not resolved" "$TMP_DIR/stderr" \
&& pass "reopen rejects an unresolved thread" \
|| fail "reopen rejects an unresolved thread"
if REOPEN_SCENARIO=foreign PATH="$REOPEN_GH_DIR:$PATH" "$SCRIPT" reopen 123 PRRT_target >"$TMP_DIR/out" 2>"$TMP_DIR/stderr"; then
fail "reopen rejects a same-number thread from another repository"
fi
grep -q "thread belongs to repository other/kandev" "$TMP_DIR/stderr" \
&& pass "reopen rejects a same-number thread from another repository" \
|| fail "reopen rejects a same-number thread from another repository"
if REOPEN_SCENARIO=mutation-failed PATH="$REOPEN_GH_DIR:$PATH" "$SCRIPT" reopen 123 PRRT_target >"$TMP_DIR/out" 2>"$TMP_DIR/stderr"; then
fail "reopen reports a failed mutation"
fi
grep -q "failed to reopen review thread PRRT_target on PR #123" "$TMP_DIR/stderr" \
&& pass "reopen reports a failed mutation" \
|| fail "reopen reports a failed mutation"
if REOPEN_SCENARIO=mutation-malformed PATH="$REOPEN_GH_DIR:$PATH" "$SCRIPT" reopen 123 PRRT_target >"$TMP_DIR/out" 2>"$TMP_DIR/stderr"; then
fail "reopen reports a malformed mutation response"
fi
grep -q "invalid reopen response for review thread PRRT_target on PR #123" "$TMP_DIR/stderr" \
&& pass "reopen reports a malformed mutation response" \
|| fail "reopen reports a malformed mutation response"
if run_invalid_reply 456 PRRT_xyz --body-file; then
fail "missing body file path fails"
fi
+155 -14
View File
@@ -3,7 +3,7 @@
#
# Usage:
# scripts/pr-state [--summary] [--all] [--trusted-reviewer <login>] <PR>
# scripts/pr-state --comment <comment_id>
# scripts/pr-state --comment <comment_id> # review or top-level issue comment
# scripts/pr-state --job-log <job_id>
#
# Raw output:
@@ -15,6 +15,8 @@
# - checks: [{ type, name, workflow, status, conclusion, details_url, target_url, run_id, job_id }]
# - checks_head_sha: <SHA or null>
# - checks_snapshot_complete: <boolean>
# - required_status_checks: [<status context>]
# - required_status_checks_known: <boolean>
# - approval_required_runs: [{ run_id, name, workflow, status, conclusion, head_sha, url }]
# - review_threads: [{ thread_id, is_resolved, path, comment_id, comment_created_at, author, body_preview }]
# - hidden_unresolved_threads: [{ thread_id, comment_id, author, path, body_preview }]
@@ -38,9 +40,12 @@
# - failed_checks: [{ name, workflow, status, conclusion, run_id, job_id, details_url, target_url }]
# - pending_checks: [{ name, workflow, status, run_id, job_id, details_url, target_url }]
# - successful_checks: [{ name, workflow, status, conclusion, run_id, job_id, details_url, target_url }]
# - terminal_checks: [{ name, workflow, status, conclusion, run_id, job_id, details_url, target_url }]
# - passed_check_count: <N>
# - checks_head_sha: <SHA or null>
# - checks_snapshot_complete: <boolean>
# - required_status_checks: [<status context>]
# - required_status_checks_known: <boolean>
# - approval_required_runs: [{ run_id, name, workflow, status, conclusion, head_sha, url }]
# - unresolved_review_thread_count: <N>
# - filtered_review_thread_count: <N>
@@ -56,8 +61,9 @@
# - errors: [{ source, message }]
#
# Comment output (--comment):
# JSON object for one PR review comment:
# - comment_id, author, body, path, line, original_line, commit_id, html_url, created_at, updated_at
# JSON object for one PR review or top-level issue comment:
# - comment_id, comment_type, author, body, path, line, original_line,
# commit_id, html_url, created_at, updated_at
#
# Job-log output (--job-log):
# Bounded context around the most likely failing test or step. The GitHub
@@ -154,6 +160,8 @@ pr_view_succeeded=false
checks_json='[]'
checks_head_sha=''
checks_snapshot_complete=false
required_status_checks_json='[]'
required_status_checks_known=false
base_ref_name=''
base_head_oid=''
merge_base_oid=''
@@ -188,6 +196,7 @@ emit_comment_json() {
local owner
local repo
local comment_json
local comment_type
repo_json="$(gh_json repo view --json owner,name)"
owner="$(jq -r '.owner.login // ""' <<<"$repo_json")"
@@ -197,13 +206,18 @@ emit_comment_json() {
return 1
fi
if ! comment_json="$(gh_json api "repos/$owner/$repo/pulls/comments/$comment_id")"; then
if comment_json="$(gh_json api "repos/$owner/$repo/pulls/comments/$comment_id")"; then
comment_type='review'
elif comment_json="$(gh_json api "repos/$owner/$repo/issues/comments/$comment_id")"; then
comment_type='issue'
else
printf 'failed to fetch PR comment %s\n' "$comment_id" >&2
return 1
fi
jq -c '{
jq -c --arg comment_type "$comment_type" '{
comment_id: (.id // null),
comment_type: $comment_type,
author: (.user.login // "unknown"),
body: (.body // ""),
path: (.path // null),
@@ -328,6 +342,8 @@ extract_issue_comments_json() {
actionable: (
(((.author.login // "unknown") == "github-actions") or ((.author.login // "unknown") == "github-actions[bot]"))
and (
(((.body // "") | contains("<!-- kandev-docs-cloudflare-preview -->")) | not)
and
(((.body // "") | gsub("^[[:space:]]+|[[:space:]]+$"; ""))
| test("^<!-- opencode-review:no-findings -->\\n\\*\\*OpenCode review complete\\*\\*\\n\\nOpenCode model: `[^`\\n]+`\\n\\nNo suggestions found for commit `[0-9A-Fa-f]{7,40}`\\.$")
| not)
@@ -562,29 +578,45 @@ summarize_state_json() {
(.status == "completed" and .conclusion == "success")
or (.status == "success" and (.conclusion == null or .conclusion == "success"));
{
(effective_checks) as $effective
| (if .required_status_checks_known == true then
[
(.required_status_checks // [])[] as $required
| select(any($effective[]?; .name == $required) | not)
| {name: $required, workflow: null, status: "missing", run_id: null,
job_id: null, details_url: null, target_url: null}
]
else [] end) as $missing_required
| {
pr,
since,
checks_head_sha,
checks_snapshot_complete,
required_status_checks: (.required_status_checks // []),
required_status_checks_known: (.required_status_checks_known == true),
approval_required_runs: (.approval_required_runs // []),
failed_checks: [
effective_checks[]
$effective[]
| select(terminal_failure_status and bad_conclusion)
| {name, workflow, status, conclusion, run_id, job_id, details_url, target_url}
],
pending_checks: [
effective_checks[]
pending_checks: ([
$effective[]
| select(pending_status and .conclusion == null)
| {name, workflow, status, run_id, job_id, details_url, target_url}
],
] + $missing_required),
successful_checks: [
effective_checks[]
$effective[]
| select(successful_check)
| {name, workflow, status, conclusion, run_id, job_id, details_url, target_url}
],
check_count: ([effective_checks[]] | length),
passed_check_count: ([effective_checks[] | select(successful_check)] | length),
terminal_checks: [
$effective[]
| select(.conclusion == "skipped" or .conclusion == "neutral")
| {name, workflow, status, conclusion, run_id, job_id, details_url, target_url}
],
check_count: (($effective | length) + ($missing_required | length)),
passed_check_count: ([$effective[] | select(successful_check)] | length),
unresolved_review_thread_count,
filtered_review_thread_count,
review_evidence,
@@ -597,6 +629,11 @@ summarize_state_json() {
hidden_unresolved_threads: (.hidden_unresolved_threads // []),
summary_notes: (
[
if .required_status_checks_known != true then
"required status check policy is unavailable; terminal check counts are not sufficient"
else
empty
end,
if (
(.unresolved_review_thread_count | type) == "number"
and .unresolved_review_thread_count > 0
@@ -700,6 +737,95 @@ fetch_base_ref_json() {
}'
}
fetch_legacy_required_status_checks_json() {
local response status body succeeded=false
if response="$(gh_json api "$1" --include)"; then
succeeded=true
fi
status="$(awk '/^HTTP\// {code=$2} END {print code}' <<<"$response")"
body="$(awk 'body {print} !body && /^\r?$/ {body=1}' <<<"$response")"
# A generic 404 can hide inaccessible policy. Only the explicit absence
# response establishes that this branch has no legacy status requirements.
if [[ "$status" == '404' ]] && jq -e '.message == "Branch not protected"' >/dev/null 2>&1 <<<"$body"; then
printf '[]\n'
elif [[ "$succeeded" == true && "$status" == '200' ]]; then
jq -ce 'select(type == "object" and (.contexts | type) == "array" and (.checks | type) == "array")
| [.checks[]?.context, .contexts[]?] | map(select(type == "string" and length > 0)) | unique' <<<"$body"
else
return 1
fi
}
fetch_required_status_checks_json() {
local owner=$1
local repo=$2
local base_ref=$3
local default_ref=$4
local rulesets_pages rulesets_json details='[]' ruleset_id detail
local ruleset_required='[]' legacy_required='[]'
[[ -n "$owner" && -n "$repo" && -n "$base_ref" ]] || return 1
# Rulesets expose the required status contexts, while the list endpoint only
# exposes their IDs. Fetch the active rule definitions before deciding that a
# small rollup is complete. Legacy branch protection can coexist with
# rulesets, so query it as well and union both policy sources.
if rulesets_pages="$(gh_json api --paginate "repos/$owner/$repo/rulesets?per_page=100")"; then
rulesets_json="$(jq -cs '[.[][]?]' <<<"$rulesets_pages")" || return 1
while IFS= read -r ruleset_id; do
[[ -n "$ruleset_id" ]] || continue
if ! detail="$(gh_json api "repos/$owner/$repo/rulesets/$ruleset_id")"; then
return 1
fi
jq -e . >/dev/null 2>&1 <<<"$detail" || return 1
details="$(jq -cn --argjson prev "$details" --argjson detail "$detail" '$prev + [$detail]')" || return 1
done < <(jq -r '.[] | select(.target == "branch" and .enforcement == "active") | .id // empty' <<<"$rulesets_json")
# GitHub uses fnmatch semantics for ref conditions. Exact names and the
# documented special values are evaluated below; unsupported glob syntax
# must make policy discovery unknown instead of silently dropping a rule.
ruleset_required="$(jq -c --arg base "$base_ref" --arg default "$default_ref" '
def matches_ref($candidate):
$candidate == "~ALL"
or $candidate == $base
or $candidate == ("refs/heads/" + $base)
or ($candidate == "~DEFAULT_BRANCH" and $default != "" and $base == $default);
def unsupported_ref_pattern:
contains("*") or contains("?") or contains("[") or contains("]") or contains("\\");
if any(
.[]
| select(.enforcement == "active")
| ((.conditions.ref_name.include // []) + (.conditions.ref_name.exclude // []))[]?
; unsupported_ref_pattern
) then
error("ruleset ref pattern cannot be evaluated safely")
else
[
.[]
| select(.enforcement == "active")
| select(any((.conditions.ref_name.include // [])[]?; matches_ref(.)))
| select(any((.conditions.ref_name.exclude // [])[]?; matches_ref(.)) | not)
| .rules[]?
| select(.type == "required_status_checks")
| .parameters.required_status_checks[]?
] as $required_checks
| if any($required_checks[]?; (.integration_id? // null) != null) then
error("ruleset required check integration source cannot be verified")
else
[$required_checks[]? | .context // empty] | unique
end
end
' <<<"$details" 2>/dev/null)" || return 1
legacy_required="$(fetch_legacy_required_status_checks_json "repos/$owner/$repo/branches/$base_ref/protection/required_status_checks")" || return 1
jq -cn --argjson ruleset "$ruleset_required" --argjson legacy "$legacy_required" '$ruleset + $legacy | unique'
return
fi
return 1
}
fetch_merge_base_json() {
local owner=$1
local repo=$2
@@ -790,9 +916,11 @@ fetch_review_threads_nodes_json() {
repo_owner=''
repo_name=''
if repo_json="$(gh_json repo view --json owner,name)"; then
default_branch_name=''
if repo_json="$(gh_json repo view --json owner,name,defaultBranchRef)"; then
repo_owner="$(jq -r '.owner.login // ""' <<<"$repo_json")"
repo_name="$(jq -r '.name // ""' <<<"$repo_json")"
default_branch_name="$(jq -r '.defaultBranchRef.name // ""' <<<"$repo_json")"
else
add_error "repo" "gh repo view failed"
fi
@@ -844,6 +972,14 @@ if [[ "$pr_view_succeeded" == "true" && -n "$repo_owner" && -n "$repo_name" && -
add_error "base" "gh api graphql base ref lookup failed"
fi
if [[ -n "$base_ref_name" ]]; then
if required_status_checks_json="$(fetch_required_status_checks_json "$repo_owner" "$repo_name" "$base_ref_name" "$default_branch_name")"; then
required_status_checks_known=true
else
add_error "required_status_checks" "required status check policy lookup failed"
fi
fi
if [[ -n "$base_head_oid" && -n "$checks_head_sha" ]]; then
if merge_base_json="$(fetch_merge_base_json "$repo_owner" "$repo_name" "$base_head_oid" "$checks_head_sha")"; then
merge_base_oid="$(jq -r '.merge_base_oid // ""' <<<"$merge_base_json")"
@@ -1013,6 +1149,7 @@ trap cleanup_state_tmp EXIT
printf '%s\n' "$pr_json" >"$state_tmp_dir/pr.json"
printf '%s\n' "$since_json" >"$state_tmp_dir/since.json"
printf '%s\n' "$checks_json" >"$state_tmp_dir/checks.json"
printf '%s\n' "$required_status_checks_json" >"$state_tmp_dir/required_status_checks.json"
printf '%s\n' "$approval_required_runs_json" >"$state_tmp_dir/approval_required_runs.json"
printf '%s\n' "$review_threads_json" >"$state_tmp_dir/review_threads.json"
printf '%s\n' "$hidden_unresolved_threads_json" >"$state_tmp_dir/hidden_unresolved_threads.json"
@@ -1025,6 +1162,7 @@ state_json="$(jq -cn \
--slurpfile pr "$state_tmp_dir/pr.json" \
--slurpfile since "$state_tmp_dir/since.json" \
--slurpfile checks "$state_tmp_dir/checks.json" \
--slurpfile required_status_checks "$state_tmp_dir/required_status_checks.json" \
--slurpfile approval_required_runs "$state_tmp_dir/approval_required_runs.json" \
--slurpfile review_threads "$state_tmp_dir/review_threads.json" \
--slurpfile hidden_unresolved_threads "$state_tmp_dir/hidden_unresolved_threads.json" \
@@ -1033,6 +1171,7 @@ state_json="$(jq -cn \
--slurpfile issue_comments "$state_tmp_dir/issue_comments.json" \
--arg checks_head_sha "$checks_head_sha" \
--argjson checks_snapshot_complete "$checks_snapshot_complete" \
--argjson required_status_checks_known "$required_status_checks_known" \
--argjson unresolved_review_thread_count "$unresolved_review_thread_count" \
--argjson filtered_review_thread_count "$filtered_review_thread_count" \
--slurpfile errors "$state_tmp_dir/errors.json" \
@@ -1042,6 +1181,8 @@ state_json="$(jq -cn \
checks: $checks[0],
checks_head_sha: (if $checks_head_sha == "" then null else $checks_head_sha end),
checks_snapshot_complete: $checks_snapshot_complete,
required_status_checks: $required_status_checks[0],
required_status_checks_known: $required_status_checks_known,
approval_required_runs: $approval_required_runs[0],
review_threads: $review_threads[0],
hidden_unresolved_threads: $hidden_unresolved_threads[0],
+229 -8
View File
@@ -64,7 +64,7 @@ if [[ "${GH_FAIL_REPO:-0}" == "1" && "$1" == "repo" && "$2" == "view" ]]; then
exit 1
fi
if [[ "${GH_FAIL_COMMENT:-0}" == "1" && "$1" == "api" && "$2" == repos/kdlbs/kandev/pulls/comments/* ]]; then
if [[ "${GH_FAIL_COMMENT:-0}" == "1" && "$1" == "api" && ("$2" == repos/kdlbs/kandev/pulls/comments/* || "$2" == repos/kdlbs/kandev/issues/comments/*) ]]; then
echo "comment api failed" >&2
exit 1
fi
@@ -75,10 +75,53 @@ if [[ "${GH_FAIL_APPROVAL_RUNS:-0}" == "1" && "$*" == *"repos/kdlbs/kandev/actio
fi
if [[ "$1" == "repo" && "$2" == "view" ]]; then
printf '{"owner":{"login":"kdlbs"},"name":"kandev"}\n'
printf '{"owner":{"login":"kdlbs"},"name":"kandev","defaultBranchRef":{"name":"main"}}\n'
exit 0
fi
if [[ "$*" == *"repos/kdlbs/kandev/rulesets?per_page=100"* ]]; then
if [[ "${GH_FAIL_RULESETS:-0}" == "1" ]]; then
exit 1
fi
if [[ "${GH_REQUIRED_RULESET:-0}" == "1" ]]; then
printf '%s\n' '[{"id":7,"target":"branch","enforcement":"active"}]'
else
printf '%s\n' '[]'
fi
exit 0
fi
if [[ "$1" == "api" && "$2" == "repos/kdlbs/kandev/rulesets/7" ]]; then
if [[ "${GH_RULESET_PATTERN:-0}" == "1" ]]; then
printf '%s\n' '{"id":7,"target":"branch","enforcement":"active","conditions":{"ref_name":{"include":["release/*"]}},"rules":[{"type":"required_status_checks","parameters":{"required_status_checks":[{"context":"Pattern required check"}]}}]}'
elif [[ "${GH_RULESET_INTEGRATION:-0}" == "1" ]]; then
printf '%s\n' '{"id":7,"target":"branch","enforcement":"active","conditions":{"ref_name":{"include":["~ALL"]}},"rules":[{"type":"required_status_checks","parameters":{"required_status_checks":[{"context":"App required check","integration_id":42}]}}]}'
elif [[ "${GH_RULESET_EXCLUDES_MAIN:-0}" == "1" ]]; then
printf '%s\n' '{"id":7,"target":"branch","enforcement":"active","conditions":{"ref_name":{"include":["~ALL"],"exclude":["main"]}},"rules":[{"type":"required_status_checks","parameters":{"required_status_checks":[{"context":"Excluded required check"}]}}]}'
else
printf '%s\n' '{"id":7,"target":"branch","enforcement":"active","conditions":{"ref_name":{"include":["~DEFAULT_BRANCH"]}},"rules":[{"type":"required_status_checks","parameters":{"required_status_checks":[{"context":"E2E Tests Passed"},{"context":"Run Backend Tests"}]}}]}'
fi
exit 0
fi
if [[ "$1" == "api" && "$2" == "repos/kdlbs/kandev/branches/main/protection/required_status_checks" ]]; then
case "${GH_LEGACY_FAILURE:-}" in
forbidden) printf 'HTTP/2.0 403 Forbidden\r\n\r\n{"message":"Forbidden"}\n'; exit 1 ;;
unavailable) printf 'HTTP/2.0 503 Unavailable\r\n\r\n{"message":"Unavailable"}\n'; exit 1 ;;
hidden) printf 'HTTP/2.0 404 Not Found\r\n\r\n{"message":"Not Found"}\n'; exit 1 ;;
network) exit 1 ;;
esac
if [[ "${GH_LEGACY_REQUIRED:-0}" == "1" ]]; then
if [[ "$*" == *"--include"* ]]; then
printf 'HTTP/2.0 200 OK\r\nContent-Type: application/json\r\n\r\n'
fi
printf '%s\n' '{"contexts":["Legacy required check"],"checks":[{"context":"Legacy checked required"}]}'
exit 0
fi
printf 'HTTP/2.0 404 Not Found\r\n\r\n{"message":"Branch not protected"}\n'
exit 1
fi
if [[ "$1" == "api" && "$2" == repos/kdlbs/kandev/compare/* ]]; then
merge_base="${GH_MERGE_BASE:-base-branch-sha}"
if [[ "${GH_BASE_SEQUENCE:-stable}" == "race" && "$2" == *"late-base-head-sha..."* ]]; then
@@ -111,6 +154,20 @@ JSON
exit 0
fi
if [[ "$1" == "api" && "$2" == "repos/kdlbs/kandev/issues/comments/222" ]]; then
cat <<'JSON'
{
"id": 222,
"user": { "login": "github-actions[bot]" },
"body": "<!-- kandev-docs-cloudflare-preview -->\nPreview is ready.",
"html_url": "https://github.com/kdlbs/kandev/issues/123#issuecomment-222",
"created_at": "2026-06-01T11:00:00Z",
"updated_at": "2026-06-01T11:01:00Z"
}
JSON
exit 0
fi
if [[ "$*" == *"repos/kdlbs/kandev/actions/runs?event=pull_request&head_sha=abc123&per_page=100"* ]]; then
if [[ "${GH_APPROVAL_REQUIRED:-0}" == "1" ]]; then
cat <<'JSON'
@@ -175,6 +232,34 @@ JSON
exit 0
fi
if [[ "${GH_TERMINAL_CHECKS:-0}" == "1" && "$1" == "pr" && "$2" == "view" && "$4" == "--json" ]]; then
cat <<'JSON'
{
"number": 123,
"baseRefName": "main",
"headRefName": "feat/pr-state",
"headRefOid": "abc123",
"headRepositoryOwner": { "login": "kdlbs" },
"headRepository": { "name": "kandev" },
"maintainerCanModify": true,
"isCrossRepository": false,
"url": "https://github.com/kdlbs/kandev/pull/123",
"comments": [],
"statusCheckRollup": [
{
"__typename": "CheckRun",
"name": "skipped required check",
"workflowName": "CI",
"status": "COMPLETED",
"conclusion": "SKIPPED",
"detailsUrl": "https://github.com/kdlbs/kandev/actions/runs/27340000010/job/55150000010"
}
]
}
JSON
exit 0
fi
if [[ "$1" == "pr" && "$2" == "view" && "$4" == "--json" ]]; then
if [[ "${GH_NO_PR_URL:-0}" == "1" ]]; then
pr_url='null'
@@ -234,6 +319,11 @@ if [[ "$1" == "pr" && "$2" == "view" && "$4" == "--json" ]]; then
"author": { "login": "github-actions" },
"body": "<!-- opencode-review:fallback-findings --> finding",
"createdAt": "2026-06-01T13:06:00Z"
},
{
"author": { "login": "github-actions" },
"body": "<!-- kandev-docs-cloudflare-preview -->\\n## Cloudflare Pages docs preview\\n\\n[Open the docs preview](https://preview.example/docs)\\n\\nBuilt from docs commit `abc1234`.",
"createdAt": "2026-06-01T13:07:00Z"
}
],
"statusCheckRollup": [
@@ -553,7 +643,7 @@ if [[ "$1" == "api" && "$2" == "graphql" ]]; then
"data": {
"repository": {
"pullRequest": {
"baseRefName": "main",
"baseRefName": "'"${GH_BASE_REF_NAME:-main}"'",
"baseRefOid": "'"$base_head"'"
}
}
@@ -755,9 +845,10 @@ test_snapshot_happy_path() {
assert_jq "thread comment timestamp" '.review_threads[] | select(.thread_id == "PRRT_1") | .comment_created_at == "2026-06-01T13:00:00Z"' "$json"
assert_jq "reviews count" '.reviews | length == 1' "$json"
assert_jq "review author" '.reviews[] | select(.author == "cubic-dev-ai[bot]") | .author == "cubic-dev-ai[bot]"' "$json"
assert_jq "issue comments count" '.issue_comments | length == 7' "$json"
assert_jq "issue comments count" '.issue_comments | length == 8' "$json"
assert_jq "issue comment author" 'any(.issue_comments[]; .author == "github-actions" and (.body | contains("Verdict")))' "$json"
assert_jq "only GitHub Actions noncanonical output is actionable" '([.issue_comments[] | select(.actionable == false)] | length) == 2 and .actionable_issue_comment_count == 5' "$json"
assert_jq "only GitHub Actions noncanonical output is actionable" '([.issue_comments[] | select(.actionable == false)] | length) == 3 and .actionable_issue_comment_count == 5' "$json"
assert_jq "Cloudflare preview marker is informational" 'any(.issue_comments[]; (.body | contains("<!-- kandev-docs-cloudflare-preview -->")) and .actionable == false)' "$json"
assert_jq "non-GitHub-Actions canonical marker is nonactionable while workflow diagnostic and fallback remain actionable" 'any(.issue_comments[]; .author == "other-bot[bot]" and .actionable == false) and all(.issue_comments[] | select(.body | test("Blocker: real|diagnostic|fallback"; "i")); .actionable == true)' "$json"
assert_jq "no errors" '.errors == []' "$json"
assert_jq "raw review evidence is complete" '.review_evidence.complete == true and .review_evidence.current_head_sha == "abc123"' "$json"
@@ -1129,7 +1220,7 @@ test_partial_failure_records_error_but_keeps_other_data() {
assert_jq "reviews empty on failure" '.reviews == []' "$json"
assert_jq "checks still present" '.checks | length < 9' "$json"
assert_jq "new issue comments still present" '.issue_comments | length == 7' "$json"
assert_jq "new issue comments still present" '.issue_comments | length == 8' "$json"
assert_jq "partial failure recorded" '.errors | length == 1' "$json"
assert_jq "partial failure source" '.errors[0].source == "reviews"' "$json"
pass "partial failure records error but keeps other data"
@@ -1186,7 +1277,7 @@ test_graphql_failure_records_error_but_keeps_other_data() {
assert_jq "graphql failure records review_threads error" '.errors[] | select(.source == "review_threads") | .message == "gh api graphql reviewThreads failed"' "$json"
assert_jq "graphql failure records closing head error" '.errors[] | select(.source == "closing_head") | .message == "gh api graphql closing head commit failed"' "$json"
assert_jq "graphql failure records closing base error" '.errors[] | select(.source == "closing_base") | .message == "gh api graphql base ref lookup failed after evidence collection"' "$json"
assert_jq "since fallback includes all historical comments" '.issue_comments | length == 8' "$json"
assert_jq "since fallback includes all historical comments" '.issue_comments | length == 9' "$json"
pass "graphql failure records error but keeps other data"
}
@@ -1218,7 +1309,7 @@ test_all_flag_includes_historical_comments_and_reviews() {
json="$(<"$tmp/out.json")"
assert_jq "since omitted in all mode" '.since == null' "$json"
assert_jq "all issue comments present" '.issue_comments | length == 8' "$json"
assert_jq "all issue comments present" '.issue_comments | length == 9' "$json"
assert_jq "all reviews present" '.reviews | length == 2' "$json"
assert_jq "all review threads present" '.review_threads | length == 2' "$json"
assert_jq "all mode keeps historical thread comment" '.review_threads[] | select(.thread_id == "PRRT_1") | .comment_id == 111' "$json"
@@ -1261,6 +1352,110 @@ test_summary_mode_returns_compact_fixup_state() {
pass "--summary returns compact fixup state"
}
test_summary_keeps_failed_policy_sources_unknown() {
local tmp json failure
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
for failure in forbidden unavailable hidden network; do
GH_LEGACY_FAILURE="$failure" PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "legacy $failure keeps policy unknown" '.required_status_checks_known == false' "$json"
assert_jq "legacy $failure records missing evidence" 'any(.errors[]; .source == "required_status_checks")' "$json"
done
GH_FAIL_RULESETS=1 GH_LEGACY_REQUIRED=1 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
assert_jq "legacy success cannot hide failed rulesets" '.required_status_checks_known == false' "$(<"$tmp/out.json")"
PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
assert_jq "explicitly absent legacy protection is known" '.required_status_checks_known == true and .required_status_checks == []' "$(<"$tmp/out.json")"
pass "policy discovery distinguishes absent protection from unavailable evidence"
}
test_summary_reports_required_status_contexts_from_rulesets() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
GH_REQUIRED_RULESET=1 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "required status policy is known" '.required_status_checks_known == true' "$json"
assert_jq "required status contexts are listed" '.required_status_checks == ["E2E Tests Passed", "Run Backend Tests"]' "$json"
assert_jq "missing required contexts remain pending" '.pending_checks | map(.name) | index("E2E Tests Passed") != null' "$json"
pass "summary reports required status contexts from the active ruleset"
}
test_summary_unions_legacy_and_ruleset_required_status_contexts() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
GH_REQUIRED_RULESET=1 GH_LEGACY_REQUIRED=1 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "legacy and ruleset policies are both known" '.required_status_checks_known == true' "$json"
assert_jq "legacy and ruleset contexts are unioned" '.required_status_checks == ["E2E Tests Passed", "Legacy checked required", "Legacy required check", "Run Backend Tests"]' "$json"
pass "summary unions legacy and ruleset required status contexts"
}
test_summary_ignores_excluded_ruleset_branch() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
GH_REQUIRED_RULESET=1 GH_RULESET_EXCLUDES_MAIN=1 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "excluded ruleset policy is known" '.required_status_checks_known == true' "$json"
assert_jq "excluded ruleset contributes no required context" '.required_status_checks == []' "$json"
pass "summary honors ruleset branch exclusions"
}
test_summary_marks_unsupported_ruleset_patterns_unknown() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
GH_REQUIRED_RULESET=1 GH_RULESET_PATTERN=1 GH_BASE_REF_NAME=release/1.2 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "unsupported ruleset patterns make policy unknown" '.required_status_checks_known == false' "$json"
assert_jq "unsupported ruleset pattern is reported" '.errors[] | select(.source == "required_status_checks") | .message == "required status check policy lookup failed"' "$json"
pass "summary does not guess at unsupported ruleset patterns"
}
test_summary_marks_integration_scoped_rules_unknown() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
GH_REQUIRED_RULESET=1 GH_RULESET_INTEGRATION=1 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "integration-scoped rules make policy unknown" '.required_status_checks_known == false' "$json"
assert_jq "integration-scoped policy failure is reported" '.errors[] | select(.source == "required_status_checks") | .message == "required status check policy lookup failed"' "$json"
pass "summary does not accept an unverified required-check integration"
}
test_summary_preserves_terminal_skipped_contexts() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
GH_TERMINAL_CHECKS=1 PATH="$tmp/bin:$PATH" "$SCRIPT" --summary 123 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "skipped check remains terminal evidence" '.terminal_checks | length == 1 and .[0].name == "skipped required check" and .[0].conclusion == "skipped"' "$json"
pass "summary preserves skipped and neutral terminal checks"
}
test_summary_reports_current_head_fork_approval_runs() {
local tmp
make_tmp_dir tmp
@@ -1375,6 +1570,7 @@ test_comment_mode_returns_full_review_comment() {
json="$(<"$tmp/out.json")"
assert_jq "comment id" '.comment_id == 111' "$json"
assert_jq "comment type" '.comment_type == "review"' "$json"
assert_jq "comment body is full" '.body | contains("Full rationale here.")' "$json"
assert_jq "comment path" '.path == "apps/web/file.ts"' "$json"
assert_jq "comment line" '.line == 42' "$json"
@@ -1382,6 +1578,23 @@ test_comment_mode_returns_full_review_comment() {
pass "--comment returns full review comment"
}
test_comment_mode_falls_back_to_top_level_issue_comment() {
local tmp
make_tmp_dir tmp
make_mock_gh "$tmp/bin"
local json
PATH="$tmp/bin:$PATH" "$SCRIPT" --comment 222 >"$tmp/out.json"
json="$(<"$tmp/out.json")"
assert_jq "issue comment id" '.comment_id == 222' "$json"
assert_jq "issue comment type" '.comment_type == "issue"' "$json"
assert_jq "issue comment body is full" '.body | contains("Preview is ready.")' "$json"
assert_jq "issue comment has no review path" '.path == null and .line == null' "$json"
assert_jq "issue comment author" '.author == "github-actions[bot]"' "$json"
pass "--comment falls back to top-level issue comments"
}
test_comment_mode_reports_fetch_failure() {
local tmp
make_tmp_dir tmp
@@ -1594,6 +1807,13 @@ test_graphql_failure_records_error_but_keeps_other_data
test_graphql_pagination_collects_all_threads
test_all_flag_includes_historical_comments_and_reviews
test_summary_mode_returns_compact_fixup_state
test_summary_keeps_failed_policy_sources_unknown
test_summary_reports_required_status_contexts_from_rulesets
test_summary_unions_legacy_and_ruleset_required_status_contexts
test_summary_ignores_excluded_ruleset_branch
test_summary_marks_unsupported_ruleset_patterns_unknown
test_summary_marks_integration_scoped_rules_unknown
test_summary_preserves_terminal_skipped_contexts
test_summary_reports_current_head_fork_approval_runs
test_summary_reports_base_not_advanced_when_head_matches_merge_base
test_summary_revalidates_base_at_closing_head
@@ -1602,6 +1822,7 @@ test_summary_all_flag_includes_historical_unresolved_threads
test_job_log_mode_emits_bounded_failure_context
test_job_log_mode_unpacks_zip_responses
test_comment_mode_returns_full_review_comment
test_comment_mode_falls_back_to_top_level_issue_comment
test_comment_mode_reports_fetch_failure
test_comment_mode_rejects_incompatible_flags
test_array_expansions_are_safe_under_set_u
+20
View File
@@ -912,6 +912,7 @@ function assertExperimentalCalloutPlacement(file, markdown) {
*/
async function assertLocalLinks(docsDir, file, markdown) {
const source = stripMarkdownCode(markdown);
assertNoNestedLocalImages(file, source);
const definitionPattern = /^\s{0,3}\[([^\]\n]+)\]:\s*(\S.*)$/gm;
const referencePattern = /!?\[([^\]\n]+)\]\[([^\]\n]*)\]/g;
const shortcutReferencePattern = /(?<![!\\\[\]])\[([^\]\n]+)\](?![\[(:])/g;
@@ -992,6 +993,25 @@ async function assertLocalLinks(docsDir, file, markdown) {
}
}
/**
* Reject nested local image links unsupported by the public-doc publisher.
*
* @param {string} file Relative source path used in validation errors.
* @param {string} markdown Markdown with code regions removed.
* @returns {void}
*/
function assertNoNestedLocalImages(file, markdown) {
const nestedImagePattern =
/\[[^\]\r\n]*!\[[^\]\r\n]*\]\(([^)\r\n]+)\)\]\(([^)\r\n]+)\)/g;
for (const match of markdown.matchAll(nestedImagePattern)) {
const imageHref = parseMarkdownDestination(match[1]);
if (!imageHref || isExternalDestination(imageHref)) continue;
throw new Error(
`${file} uses a nested local image link: ${imageHref}; Landing copies only a direct plain Markdown image`,
);
}
}
/**
* Require a local Markdown fragment to match the published heading identifier.
*
+20 -3
View File
@@ -566,13 +566,14 @@ test("rejects a DocsVideo attribute that points to missing media", async () => {
);
});
test("rejects a broken local image nested inside a link", async () => {
test("rejects a local image nested inside a link", async () => {
const dir = await createDocs(
{
"index.md": validPage.replace(
"Page body.",
"[![Missing](assets/missing.png)](guide.md)",
"[![Screenshot](assets/screenshot.png)](guide.md)",
),
"assets/screenshot.png": "image",
"guide.md": validPage,
},
{ pages: ["index", "guide"] },
@@ -580,10 +581,26 @@ test("rejects a broken local image nested inside a link", async () => {
await assert.rejects(
validatePublicDocs(dir),
/index.md links to missing local target: assets\/missing.png/,
/index.md uses a nested local image link: assets\/screenshot.png; Landing copies only a direct plain Markdown image/,
);
});
test("accepts a direct local image with a separate full-size link", async () => {
const dir = await createDocs(
{
"index.md": validPage.replace(
"Page body.",
"![Screenshot](assets/screenshot.png)\n\n[Open full size](guide.md)",
),
"assets/screenshot.png": "image",
"guide.md": validPage,
},
{ pages: ["index", "guide"] },
);
await assert.doesNotReject(validatePublicDocs(dir));
});
test("accepts external, existing heading, and fenced-code links", async () => {
const dir = await createDocs(
{