From 89cddc2239956009f289ee413db1d1d1336a6dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Sun, 6 Sep 2026 14:45:59 +0100 Subject: [PATCH] ci: add selectable external CI runners (#3451) --- .../actions/plan-external-runners/action.yml | 59 +++++ .../backend-tests-workflow-contract_test.py | 9 +- .../e2e-tests-workflow-contract_test.py | 52 ++++ .../external-runner-workflow-contract_test.py | 157 ++++++++++++ .github/scripts/runner-plan.py | 226 +++++++++++++++++ .github/scripts/runner-plan_test.py | 223 ++++++++++++++++ .github/workflows/architecture-lint.yml | 2 + .github/workflows/backend-tests.yml | 59 ++++- .github/workflows/e2e-tests.yml | 63 ++++- .github/workflows/frontend-tests.yml | 47 +++- .github/workflows/lint-action-pinning.yml | 8 + .github/workflows/lint-harness-files.yml | 2 + docs/ci-merge-queue.md | 73 ++++++ .../2026-09-06-opt-in-external-e2e-runners.md | 143 +++++++++++ docs/decisions/INDEX.md | 1 + .../external-e2e-runner-capacity/plan.md | 175 +++++++++++++ ...task-01-add-opt-in-e2e-runner-selection.md | 116 +++++++++ ...sk-02-extend-tiered-runners-to-linux-ci.md | 119 +++++++++ ...ask-03-deterministic-percentage-rollout.md | 116 +++++++++ docs/specs/platform/README.md | 2 + .../external-e2e-runner-capacity.md | 135 ++++++++++ .../external-e2e-runner-capacity.md | 237 ++++++++++++++++++ 22 files changed, 2002 insertions(+), 22 deletions(-) create mode 100644 .github/actions/plan-external-runners/action.yml create mode 100644 .github/scripts/external-runner-workflow-contract_test.py create mode 100644 .github/scripts/runner-plan.py create mode 100644 .github/scripts/runner-plan_test.py create mode 100644 docs/decisions/2026-09-06-opt-in-external-e2e-runners.md create mode 100644 docs/plans/external-e2e-runner-capacity/plan.md create mode 100644 docs/plans/external-e2e-runner-capacity/task-01-add-opt-in-e2e-runner-selection.md create mode 100644 docs/plans/external-e2e-runner-capacity/task-02-extend-tiered-runners-to-linux-ci.md create mode 100644 docs/plans/external-e2e-runner-capacity/task-03-deterministic-percentage-rollout.md create mode 100644 docs/specs/platform/requirements/external-e2e-runner-capacity.md create mode 100644 docs/specs/platform/system-design/external-e2e-runner-capacity.md diff --git a/.github/actions/plan-external-runners/action.yml b/.github/actions/plan-external-runners/action.yml new file mode 100644 index 0000000000..3a5696884b --- /dev/null +++ b/.github/actions/plan-external-runners/action.yml @@ -0,0 +1,59 @@ +name: Plan external runner allocation +description: Compute deterministic external-runner assignments for a workflow + +inputs: + workflow: + description: Runner-plan workflow family key + required: true + run-id: + description: Stable workflow run identifier used for allocation + required: true + families: + description: JSON job-family definitions owned by the calling workflow + required: true + burst: + description: Exact true value enables external allocation + required: false + default: "" + percent: + description: External allocation percentage from 0 to 100 + required: false + default: "" + light-label: + description: Configured light-tier runner label + required: false + default: "" + standard-label: + description: Configured standard-tier runner label + required: false + default: "" + +outputs: + plan: + description: JSON object containing runner and matrix outputs + value: ${{ steps.plan.outputs.plan }} + +runs: + using: composite + steps: + - name: Plan runner allocation + id: plan + shell: bash + env: + RUNNER_PLAN_WORKFLOW: ${{ inputs.workflow }} + RUNNER_PLAN_RUN_ID: ${{ inputs.run-id }} + RUNNER_PLAN_FAMILIES: ${{ inputs.families }} + KANDEV_CI_EXTERNAL_ENABLED: ${{ inputs.burst }} + KANDEV_CI_EXTERNAL_PERCENT: ${{ inputs.percent }} + KANDEV_CI_RUNNER_LIGHT: ${{ inputs.light-label }} + KANDEV_CI_RUNNER_STANDARD: ${{ inputs.standard-label }} + run: | + python3 "$GITHUB_WORKSPACE/.github/scripts/runner-plan.py" \ + --workflow "$RUNNER_PLAN_WORKFLOW" \ + --run-id "$RUNNER_PLAN_RUN_ID" \ + --families "$RUNNER_PLAN_FAMILIES" \ + --burst "$KANDEV_CI_EXTERNAL_ENABLED" \ + --percent "$KANDEV_CI_EXTERNAL_PERCENT" \ + --light-label "$KANDEV_CI_RUNNER_LIGHT" \ + --standard-label "$KANDEV_CI_RUNNER_STANDARD" \ + --output "$GITHUB_OUTPUT" diff --git a/.github/scripts/backend-tests-workflow-contract_test.py b/.github/scripts/backend-tests-workflow-contract_test.py index 7e2163122a..60682cce58 100644 --- a/.github/scripts/backend-tests-workflow-contract_test.py +++ b/.github/scripts/backend-tests-workflow-contract_test.py @@ -25,7 +25,7 @@ class BackendTestsWorkflowContractTest(unittest.TestCase): generate = step_block(workflow, "Generate test report") publish = step_block(workflow, "Publish bounded test report summary") - temporary_summary = "$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md" + temporary_summary = "$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md" self.assertIn( "uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10", checkout, @@ -37,12 +37,13 @@ class BackendTestsWorkflowContractTest(unittest.TestCase): ) self.assertIn("path: .github/vendor/go-test-action", checkout) self.assertIn(f'report_summary="{temporary_summary}"', generate) + self.assertIn("BACKEND_TEST_SHARD: ${{ matrix.shard }}", generate) self.assertNotIn("${{ runner.temp }}", generate) self.assertIn(': > "$report_summary"', generate) self.assertIn('GITHUB_STEP_SUMMARY="$report_summary"', generate) self.assertIn( 'INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/' - 'test-results-${{ matrix.shard }}.json"', + 'test-results-${BACKEND_TEST_SHARD}.json"', generate, ) self.assertIn( @@ -60,8 +61,8 @@ class BackendTestsWorkflowContractTest(unittest.TestCase): self.assertIn(f'--input "{temporary_summary}"', publish) self.assertIn('--output "$GITHUB_STEP_SUMMARY"', publish) self.assertIn("--diagnostics-label", publish) - self.assertIn("backend-test-results-${{ matrix.shard }}", publish) - self.assertIn("${{ github.run_id }}#artifacts", publish) + self.assertIn("backend-test-results-${BACKEND_TEST_SHARD}", publish) + self.assertIn("$GITHUB_RUN_ID#artifacts", publish) def test_full_json_diagnostics_remain_in_the_named_artifact(self) -> None: workflow = BACKEND_WORKFLOW.read_text(encoding="utf-8") diff --git a/.github/scripts/e2e-tests-workflow-contract_test.py b/.github/scripts/e2e-tests-workflow-contract_test.py index c5ff61cb3b..047bd2e6ad 100644 --- a/.github/scripts/e2e-tests-workflow-contract_test.py +++ b/.github/scripts/e2e-tests-workflow-contract_test.py @@ -193,6 +193,58 @@ cat "${FAKE_DOCKER_MANIFEST}" self.assertIn("timeout-minutes: 35", normal_job) self.assertNotIn("timeout-minutes: 25", normal_job) + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1 + # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2 + def test_external_runner_tiers_are_toggleable_for_eligible_jobs(self) -> None: + workflow = E2E_WORKFLOW.read_text(encoding="utf-8") + + light_jobs = { + "changes": ("build", "changes_runner"), + "e2e-gate": (None, "e2e_gate_runner"), + } + + for job, (next_job, output_name) in light_jobs.items(): + if next_job is None: + job_text = workflow.partition(" e2e-gate:\n")[2] + else: + job_text = job_block(workflow, job, next_job) + expected = ( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan)." + f"{output_name} }}}}" + ) + self.assertIn( + expected, + job_text, + f"{job} must select its configured tier", + ) + e2e_job = job_block(workflow, "e2e", "playwright_image") + self.assertIn("runs-on: ${{ matrix.runner }}", e2e_job) + self.assertIn( + "matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}", + e2e_job, + ) + + protected_jobs = { + "build": "e2e", + "e2e-report": "e2e-gate", + "playwright_image": "e2e-containers", + "e2e-containers": "e2e-kubernetes-compatibility", + "e2e-kubernetes-compatibility": "desktop-e2e", + "desktop-e2e": "e2e-report", + } + for job, next_job in protected_jobs.items(): + protected_text = job_block(workflow, job, next_job) + self.assertIn("runs-on: ubuntu-latest", protected_text) + self.assertNotIn("runner_plan", protected_text) + self.assertNotIn("KANDEV_CI_EXTERNAL", protected_text) + self.assertNotIn("KANDEV_CI_RUNNER_", protected_text) + def test_contract_runs_in_the_unfiltered_required_workflow(self) -> None: workflow = LINT_WORKFLOW.read_text(encoding="utf-8") diff --git a/.github/scripts/external-runner-workflow-contract_test.py b/.github/scripts/external-runner-workflow-contract_test.py new file mode 100644 index 0000000000..14ee5177db --- /dev/null +++ b/.github/scripts/external-runner-workflow-contract_test.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Contract tests for external runner planner wiring and protected jobs.""" + +from pathlib import Path +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW_ROOT = REPO_ROOT / ".github" / "workflows" +PLAN_SCRIPT = REPO_ROOT / ".github" / "scripts" / "runner-plan.py" + + +def job_block(workflow: str, job: str, next_job: str | None) -> str: + marker = f" {job}:\n" + _, separator, remainder = workflow.partition(marker) + if not separator: + raise AssertionError(f"Workflow has no {job} job") + if next_job is None: + return remainder + return remainder.partition(f"\n {next_job}:\n")[0] + + +class ExternalRunnerWorkflowContractTest(unittest.TestCase): + def assert_planner( + self, workflow_name: str, workflow_key: str, families: tuple[str, ...] + ) -> str: + workflow = (WORKFLOW_ROOT / workflow_name).read_text(encoding="utf-8") + planner = job_block(workflow, "runner_plan", "changes") + self.assertIn("runs-on: ubuntu-latest", planner) + self.assertIn("plan: ${{ steps.plan.outputs.plan }}", planner) + self.assertIn("uses: ./.github/actions/plan-external-runners", planner) + self.assertIn(f"workflow: {workflow_key}", planner) + self.assertIn("families: >-", planner) + self.assertIn("run-id: ${{ github.run_id }}", planner) + self.assertIn("burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}", planner) + self.assertIn("percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}", planner) + self.assertIn("light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}", planner) + self.assertIn("standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}", planner) + for family in families: + self.assertIn(f'"name":"{family}"', planner) + return workflow + + def test_e2e_uses_planner_for_eligible_jobs_and_keeps_protected_jobs_hosted(self) -> None: + workflow = self.assert_planner( + "e2e-tests.yml", + "e2e", + ("changes", "e2e", "e2e_gate"), + ) + self.assertIn( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}", + job_block(workflow, "changes", "build"), + ) + build = job_block(workflow, "build", "e2e") + self.assertIn("runs-on: ubuntu-latest", build) + self.assertNotIn("runner_plan", build) + e2e = job_block(workflow, "e2e", "playwright_image") + self.assertIn( + "matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}", + e2e, + ) + self.assertIn("runs-on: ${{ matrix.runner }}", e2e) + report = job_block(workflow, "e2e-report", "e2e-gate") + self.assertIn("runs-on: ubuntu-latest", report) + self.assertNotIn("runner_plan", report) + self.assertIn( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }}", + job_block(workflow, "e2e-gate", None), + ) + for job, next_job in ( + ("playwright_image", "e2e-containers"), + ("e2e-containers", "e2e-kubernetes-compatibility"), + ("e2e-kubernetes-compatibility", "desktop-e2e"), + ("desktop-e2e", "e2e-report"), + ): + protected = job_block(workflow, job, next_job) + self.assertIn("runs-on: ubuntu-latest", protected) + self.assertNotIn("runner_plan", protected) + self.assertNotIn("KANDEV_CI_EXTERNAL", protected) + + def test_backend_uses_planner_and_keeps_services_and_windows_hosted(self) -> None: + workflow = self.assert_planner( + "backend-tests.yml", + "backend", + ("test",), + ) + for job, next_job in ( + ("changes", "static_checks"), + ("static_checks", "test_shards"), + ("test_ambient_env", "test"), + ): + protected = job_block(workflow, job, next_job) + self.assertIn("runs-on: ubuntu-latest", protected) + self.assertNotIn("runner_plan", protected) + shards = job_block(workflow, "test_shards", "test_ambient_env") + self.assertIn("- shard: 1", shards) + self.assertIn("- shard: 2", shards) + self.assertIn("runs-on: ubuntu-latest", shards) + self.assertNotIn("runner_plan", shards) + self.assertNotIn("matrix.runner", shards) + test_gate = job_block(workflow, "test", "postgres-boot") + self.assertIn( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).test_runner }}", + test_gate, + ) + for job, next_job in (("postgres-boot", "test-windows"), ("test-windows", None)): + protected = job_block(workflow, job, next_job) + self.assertIn("runs-on: ubuntu-latest" if job == "postgres-boot" else "runs-on: windows-latest", protected) + self.assertNotIn("runner_plan", protected) + + def test_frontend_uses_planner_for_test_and_gate(self) -> None: + workflow = self.assert_planner( + "frontend-tests.yml", + "frontend", + ("changes", "frontend", "frontend_gate"), + ) + self.assertIn( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}", + job_block(workflow, "changes", "frontend"), + ) + self.assertIn( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}", + job_block(workflow, "frontend", "frontend-gate"), + ) + self.assertIn( + "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}", + job_block(workflow, "frontend-gate", None), + ) + + def test_protected_lint_workflows_do_not_run_the_planner(self) -> None: + for workflow_name in ( + "architecture-lint.yml", + "lint-action-pinning.yml", + "lint-harness-files.yml", + ): + workflow = (WORKFLOW_ROOT / workflow_name).read_text(encoding="utf-8") + lint = job_block(workflow, "lint", None) + self.assertNotIn("runner_plan:", workflow) + self.assertNotIn("name: Require runner plan", lint) + self.assertIn("runs-on: ubuntu-latest", lint) + + def test_plan_script_is_checked_by_required_action_pinning_workflow(self) -> None: + workflow = (WORKFLOW_ROOT / "lint-action-pinning.yml").read_text(encoding="utf-8") + self.assertTrue(PLAN_SCRIPT.is_file()) + action = (WORKFLOW_ROOT / ".." / "actions" / "plan-external-runners" / "action.yml").resolve() + self.assertTrue(action.is_file()) + action_text = action.read_text(encoding="utf-8") + self.assertIn("using: composite", action_text) + self.assertIn("families:", action_text) + self.assertIn("plan:", action_text) + self.assertIn("runner-plan.py", action_text) + self.assertNotIn("changes_runner:", action_text) + self.assertIn("python3 .github/scripts/runner-plan_test.py", workflow) + self.assertIn("python3 .github/scripts/external-runner-workflow-contract_test.py", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/scripts/runner-plan.py b/.github/scripts/runner-plan.py new file mode 100644 index 0000000000..439653b1c4 --- /dev/null +++ b/.github/scripts/runner-plan.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Build deterministic runner assignments from workflow-owned family data.""" + +import argparse +import hashlib +import json +import re +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class RunnerPlan: + plan: dict[str, object] + warnings: list[str] + + +@dataclass(frozen=True) +class Family: + name: str + tier: str + instances: tuple[dict[str, object], ...] + is_matrix: bool + + +GITHUB_RUNNER = "ubuntu-latest" +VALID_TIERS = frozenset({"light", "standard"}) +FAMILY_NAME = re.compile(r"[a-z][a-z0-9_]*\Z") + + +def _parse_percentage(raw: str | None) -> tuple[int, list[str]]: + if raw is None or raw.strip() == "": + return 0, [] + value = raw.strip() + warning = ( + "KANDEV_CI_EXTERNAL_PERCENT must be an integer from 0 to 100, " + f"got {raw!r}" + ) + if not value.isascii() or not value.isdigit(): + return 0, [warning] + percentage = int(value) + if percentage > 100: + return 0, [warning] + return percentage, [] + + +def _validate_families(raw: object) -> tuple[Family, ...]: + if not isinstance(raw, list) or not raw: + raise ValueError("families must be a non-empty JSON array") + + families: list[Family] = [] + names: set[str] = set() + for entry in raw: + if not isinstance(entry, dict): + raise ValueError("each family must be a JSON object") + name = entry.get("name") + tier = entry.get("tier") + if not isinstance(name, str) or FAMILY_NAME.fullmatch(name) is None: + raise ValueError(f"invalid family name: {name!r}") + if name in names: + raise ValueError(f"duplicate family name: {name}") + if tier not in VALID_TIERS: + raise ValueError(f"family {name!r} has invalid tier: {tier!r}") + + is_matrix = "instances" in entry + instances_value = entry.get("instances", [{}]) + if not isinstance(instances_value, list) or not instances_value: + raise ValueError(f"family {name!r} instances must be a non-empty array") + instances: list[dict[str, object]] = [] + for instance in instances_value: + if not isinstance(instance, dict): + raise ValueError(f"family {name!r} instances must be JSON objects") + if "runner" in instance: + raise ValueError(f"family {name!r} instance cannot define runner") + instances.append(instance) + + names.add(name) + families.append( + Family( + name=name, + tier=tier, + instances=tuple(instances), + is_matrix=is_matrix, + ) + ) + return tuple(families) + + +def _external_for_singleton( + *, workflow: str, run_id: str, family: str, percentage: int +) -> bool: + if percentage <= 0: + return False + if percentage >= 100: + return True + key = f"{workflow}\0{run_id}\0{family}".encode("utf-8") + bucket = int.from_bytes(hashlib.sha256(key).digest()[:8], "big") % 100 + return bucket < percentage + + +def _matrix_assignments( + *, + workflow: str, + run_id: str, + family: str, + instances: tuple[dict[str, object], ...], + percentage: int, +) -> set[int]: + external_count = (len(instances) * percentage) // 100 + ranked: list[tuple[bytes, int]] = [] + for index, instance in enumerate(instances): + matrix_key = json.dumps(instance, sort_keys=True, separators=(",", ":")) + digest = hashlib.sha256( + f"{workflow}\0{run_id}\0{family}\0{matrix_key}".encode("utf-8") + ).digest() + ranked.append((digest, index)) + return {index for _, index in sorted(ranked)[:external_count]} + + +def _plan_family( + *, + workflow: str, + run_id: str, + family: Family, + burst: str, + percentage: int, + external_label: str, +) -> list[dict[str, object]]: + if burst != "true" or not external_label or percentage == 0: + external_indices: set[int] = set() + elif not family.is_matrix: + external_indices = ( + {0} + if _external_for_singleton( + workflow=workflow, + run_id=run_id, + family=family.name, + percentage=percentage, + ) + else set() + ) + else: + external_indices = _matrix_assignments( + workflow=workflow, + run_id=run_id, + family=family.name, + instances=family.instances, + percentage=percentage, + ) + + return [ + { + **instance, + "runner": external_label if index in external_indices else GITHUB_RUNNER, + } + for index, instance in enumerate(family.instances) + ] + + +def build_plan( + *, + workflow: str, + run_id: str, + families: object, + burst: str, + percent: str | None, + light_label: str, + standard_label: str, +) -> RunnerPlan: + percentage, warnings = _parse_percentage(percent) + labels = {"light": light_label, "standard": standard_label} + plan: dict[str, object] = {} + for family in _validate_families(families): + planned = _plan_family( + workflow=workflow, + run_id=run_id, + family=family, + burst=burst, + percentage=percentage, + external_label=labels[family.tier], + ) + if family.is_matrix: + plan[f"{family.name}_matrix"] = {"include": planned} + else: + plan[f"{family.name}_runner"] = planned[0]["runner"] + return RunnerPlan(plan=plan, warnings=warnings) + + +def _write_output(plan: RunnerPlan, output_path: Path) -> None: + value = json.dumps(plan.plan, separators=(",", ":"), sort_keys=True) + with output_path.open("a", encoding="utf-8") as output: + output.write(f"plan={value}\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--workflow", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--families", required=True) + parser.add_argument("--burst", default="") + parser.add_argument("--percent") + parser.add_argument("--light-label", default="") + parser.add_argument("--standard-label", default="") + parser.add_argument("--output", type=Path, default=Path("/dev/stdout")) + args = parser.parse_args() + try: + families = json.loads(args.families) + plan = build_plan( + workflow=args.workflow, + run_id=args.run_id, + families=families, + burst=args.burst, + percent=args.percent, + light_label=args.light_label, + standard_label=args.standard_label, + ) + except (json.JSONDecodeError, ValueError) as error: + parser.error(str(error)) + for warning in plan.warnings: + print(f"::warning::{warning}") + _write_output(plan, args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/runner-plan_test.py b/.github/scripts/runner-plan_test.py new file mode 100644 index 0000000000..8420e3180d --- /dev/null +++ b/.github/scripts/runner-plan_test.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Tests for deterministic external CI runner allocation.""" + +import importlib.util +from pathlib import Path +import unittest + + +SCRIPT = Path(__file__).with_name("runner-plan.py") +SPEC = importlib.util.spec_from_file_location("runner_plan", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +runner_plan = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(runner_plan) + +E2E_FAMILIES = [ + {"name": "changes", "tier": "light"}, + { + "name": "e2e", + "tier": "standard", + "instances": [{"shard": shard} for shard in range(1, 15)], + }, + {"name": "e2e_gate", "tier": "light"}, +] +FRONTEND_FAMILIES = [ + {"name": "frontend", "tier": "standard"}, + {"name": "frontend_gate", "tier": "light"}, +] +BACKEND_MATRIX_FAMILY = [ + { + "name": "backend_test", + "tier": "standard", + "instances": [{"shard": 1, "index": 0}, {"shard": 2, "index": 1}], + } +] + + +class RunnerPlanTest(unittest.TestCase): + def test_missing_percentage_defaults_to_github(self) -> None: + plan = runner_plan.build_plan( + workflow="e2e", + run_id="100", + families=E2E_FAMILIES, + burst="true", + percent=None, + light_label="light-runner", + standard_label="standard-runner", + ) + + self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest") + self.assertFalse(plan.warnings) + + def test_burst_off_ignores_percentage(self) -> None: + plan = runner_plan.build_plan( + workflow="frontend", + run_id="101", + families=FRONTEND_FAMILIES, + burst="false", + percent="100", + light_label="light-runner", + standard_label="standard-runner", + ) + + self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest") + self.assertEqual(plan.plan["frontend_gate_runner"], "ubuntu-latest") + + def test_matrix_percentage_uses_floor_and_is_stable(self) -> None: + first = runner_plan.build_plan( + workflow="e2e", + run_id="102", + families=E2E_FAMILIES, + burst="true", + percent="50", + light_label="light-runner", + standard_label="standard-runner", + ) + second = runner_plan.build_plan( + workflow="e2e", + run_id="102", + families=E2E_FAMILIES, + burst="true", + percent="50", + light_label="light-runner", + standard_label="standard-runner", + ) + + first_matrix = first.plan["e2e_matrix"]["include"] + second_matrix = second.plan["e2e_matrix"]["include"] + self.assertEqual(first_matrix, second_matrix) + self.assertEqual( + sum(item["runner"] == "standard-runner" for item in first_matrix), 7 + ) + + def test_backend_matrix_percentage_uses_floor(self) -> None: + plan = runner_plan.build_plan( + workflow="backend", + run_id="103", + families=BACKEND_MATRIX_FAMILY, + burst="true", + percent="50", + light_label="light-runner", + standard_label="standard-runner", + ) + + matrix = plan.plan["backend_test_matrix"]["include"] + self.assertEqual( + sum(item["runner"] == "standard-runner" for item in matrix), 1 + ) + self.assertEqual( + [(item["shard"], item["index"]) for item in matrix], + [(1, 0), (2, 1)], + ) + + def test_workflow_owned_family_names_need_no_python_change(self) -> None: + plan = runner_plan.build_plan( + workflow="new-workflow", + run_id="109", + families=[{"name": "new_job", "tier": "standard"}], + burst="true", + percent="100", + light_label="light-runner", + standard_label="standard-runner", + ) + + self.assertEqual(plan.plan, {"new_job_runner": "standard-runner"}) + + def test_invalid_family_definition_is_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicate family name"): + runner_plan.build_plan( + workflow="frontend", + run_id="110", + families=[ + {"name": "same", "tier": "light"}, + {"name": "same", "tier": "standard"}, + ], + burst="true", + percent="50", + light_label="light-runner", + standard_label="standard-runner", + ) + + def test_singleton_allocation_is_stable_for_reruns(self) -> None: + first = runner_plan.build_plan( + workflow="frontend", + run_id="104", + families=FRONTEND_FAMILIES, + burst="true", + percent="50", + light_label="light-runner", + standard_label="standard-runner", + ) + second = runner_plan.build_plan( + workflow="frontend", + run_id="104", + families=FRONTEND_FAMILIES, + burst="true", + percent="50", + light_label="light-runner", + standard_label="standard-runner", + ) + self.assertEqual(first.plan, second.plan) + + def test_empty_tier_label_falls_back_to_github(self) -> None: + plan = runner_plan.build_plan( + workflow="frontend", + run_id="105", + families=FRONTEND_FAMILIES, + burst="true", + percent="100", + light_label="", + standard_label="", + ) + + self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest") + + def test_invalid_percentage_fails_closed_with_warning(self) -> None: + plan = runner_plan.build_plan( + workflow="e2e", + run_id="106", + families=E2E_FAMILIES, + burst="true", + percent="101", + light_label="light-runner", + standard_label="standard-runner", + ) + + self.assertTrue(plan.warnings) + self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest") + matrix = plan.plan["e2e_matrix"]["include"] + self.assertTrue(all(item["runner"] == "ubuntu-latest" for item in matrix)) + + def test_unicode_digits_fail_closed_with_warning(self) -> None: + plan = runner_plan.build_plan( + workflow="e2e", + run_id="108", + families=E2E_FAMILIES, + burst="true", + percent="²", + light_label="light-runner", + standard_label="standard-runner", + ) + + self.assertTrue(plan.warnings) + self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest") + matrix = plan.plan["e2e_matrix"]["include"] + self.assertTrue(all(item["runner"] == "ubuntu-latest" for item in matrix)) + + def test_full_percentage_uses_external_for_non_empty_tier(self) -> None: + plan = runner_plan.build_plan( + workflow="frontend", + run_id="107", + families=FRONTEND_FAMILIES, + burst="true", + percent="100", + light_label="light-runner", + standard_label="standard-runner", + ) + + self.assertEqual(plan.plan["frontend_runner"], "standard-runner") + self.assertEqual(plan.plan["frontend_gate_runner"], "light-runner") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/architecture-lint.yml b/.github/workflows/architecture-lint.yml index f5fcf40991..e0c2c3c87e 100644 --- a/.github/workflows/architecture-lint.yml +++ b/.github/workflows/architecture-lint.yml @@ -24,6 +24,8 @@ permissions: jobs: lint: name: Architecture boundaries do not regress + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest steps: diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index c969f677f3..c7a85c9d2c 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -33,11 +33,38 @@ permissions: packages: read jobs: + runner_plan: + name: Plan external runner allocation + runs-on: ubuntu-latest + outputs: + plan: ${{ steps.plan.outputs.plan }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Plan runner allocation + id: plan + uses: ./.github/actions/plan-external-runners + with: + workflow: backend + run-id: ${{ github.run_id }} + families: >- + [{"name":"test","tier":"light"}] + burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }} + percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }} + light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }} + standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }} + # The former trigger-level `paths:` list, applied one level down so a skip # still produces a conclusion. `.github/scripts/changed-paths.py` implements # the same last-match-wins semantics GitHub applies to `paths:`. changes: name: Detect relevant changes + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -106,6 +133,8 @@ jobs: name: Backend Static Checks needs: changes if: needs.changes.outputs.run == 'true' + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest # Pre-baked image bundles Go, golangci-lint, go-licenses, Docker CLI, Node, # pnpm, and the Playwright base — see .github/docker/ci-base/Dockerfile. @@ -226,6 +255,8 @@ jobs: name: Backend Tests (${{ matrix.shard }}/2) needs: changes if: needs.changes.outputs.run == 'true' + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest strategy: fail-fast: false @@ -334,11 +365,13 @@ jobs: if: always() shell: bash working-directory: ${{ github.workspace }} + env: + BACKEND_TEST_SHARD: ${{ matrix.shard }} run: | - report_summary="$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md" + report_summary="$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md" : > "$report_summary" GITHUB_STEP_SUMMARY="$report_summary" \ - INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/test-results-${{ matrix.shard }}.json" \ + INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/test-results-${BACKEND_TEST_SHARD}.json" \ INPUT_MODULEDIRECTORY="$GITHUB_WORKSPACE/apps/backend" \ INPUT_OMIT="successful" \ node ".github/vendor/go-test-action/dist/index.js" @@ -347,12 +380,14 @@ jobs: if: always() shell: bash working-directory: ${{ github.workspace }} + env: + BACKEND_TEST_SHARD: ${{ matrix.shard }} run: | python3 .github/scripts/bounded-step-summary.py \ - --input "$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md" \ + --input "$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md" \ --output "$GITHUB_STEP_SUMMARY" \ - --diagnostics-label "backend-test-results-${{ matrix.shard }}" \ - --diagnostics-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts" + --diagnostics-label "backend-test-results-${BACKEND_TEST_SHARD}" \ + --diagnostics-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#artifacts" - name: Upload test artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -368,6 +403,8 @@ jobs: name: Backend (ambient integration env) needs: changes if: needs.changes.outputs.run == 'true' + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest # test_shards runs in the same clean container and never sets these vars, # so it cannot catch a package that starts reading them again — the class @@ -430,9 +467,10 @@ jobs: # `test-windows` were outside it before this workflow became a gate. test: name: Run Backend Tests - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).test_runner }} needs: - changes + - runner_plan - static_checks - test_shards - test_ambient_env @@ -442,6 +480,7 @@ jobs: steps: - name: Require backend checks to pass env: + RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }} CHANGES_RESULT: ${{ needs.changes.result }} STATIC_CHECKS_RESULT: ${{ needs.static_checks.result }} TEST_SHARDS_RESULT: ${{ needs.test_shards.result }} @@ -451,6 +490,7 @@ jobs: run: | set -euo pipefail + echo "Runner plan: ${RUNNER_PLAN_RESULT}" echo "Change detection: ${CHANGES_RESULT}" echo "Static checks: ${STATIC_CHECKS_RESULT}" echo "Test shards: ${TEST_SHARDS_RESULT}" @@ -458,6 +498,13 @@ jobs: echo "Postgres boot: ${POSTGRES_BOOT_RESULT}" echo "Windows tests: ${TEST_WINDOWS_RESULT}" + # Runner planning must succeed because its outputs select the runner + # fleet. A planner error cannot silently become a hosted fallback. + if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then + echo "Runner planning did not succeed; refusing to report a pass." >&2 + exit 1 + fi + # Change detection itself must succeed: if it errored we do not know # whether the skips below were deliberate. if [[ "${CHANGES_RESULT}" != "success" ]]; then diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index f751f09c46..34bf8f70f6 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -39,12 +39,47 @@ permissions: packages: read jobs: + runner_plan: + name: Plan external runner allocation + runs-on: ubuntu-latest + outputs: + plan: ${{ steps.plan.outputs.plan }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Plan runner allocation + id: plan + uses: ./.github/actions/plan-external-runners + with: + workflow: e2e + run-id: ${{ github.run_id }} + families: >- + [ + {"name":"changes","tier":"light"}, + {"name":"e2e","tier":"standard","instances":[ + {"shard":1},{"shard":2},{"shard":3},{"shard":4}, + {"shard":5},{"shard":6},{"shard":7},{"shard":8}, + {"shard":9},{"shard":10},{"shard":11},{"shard":12}, + {"shard":13},{"shard":14} + ]}, + {"name":"e2e_gate","tier":"light"} + ] + burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }} + percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }} + light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }} + standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }} + # The former trigger-level `paths:` list, applied one level down so a skip # still produces a conclusion. `.github/scripts/changed-paths.py` implements # the same last-match-wins semantics GitHub applies to `paths:`. changes: name: Detect relevant changes - runs-on: ubuntu-latest + needs: runner_plan + runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }} timeout-minutes: 10 outputs: run: ${{ steps.detect.outputs.run }} @@ -113,6 +148,8 @@ jobs: name: Build needs: changes if: needs.changes.outputs.run == 'true' + # Build reads the GitHub API and downloads a timing artifact with the job token. + # Keep this credential-bearing path on the GitHub-hosted boundary. runs-on: ubuntu-latest # Pre-baked image bundles Go, Node 24, pnpm 9.15.9 — see # .github/docker/ci-base/Dockerfile. Uses the `build` tag (with Go) @@ -367,9 +404,9 @@ jobs: e2e: name: E2E Shard ${{ matrix.shard }}/14 - needs: [changes, build] + needs: [changes, build, runner_plan] if: needs.changes.outputs.run == 'true' - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} # Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright browser # binaries (Chromium etc.) — see .github/docker/ci-base/Dockerfile. This is # the biggest speedup since `npx playwright install chromium --with-deps` @@ -398,8 +435,7 @@ jobs: NODE_OPTIONS: --dns-result-order=ipv4first strategy: fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] + matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }} steps: - name: Checkout code @@ -467,9 +503,10 @@ jobs: working-directory: apps/web env: E2E_SHARD: ${{ matrix.shard }} + E2E_MANIFEST_SHARD: ${{ matrix.shard }} KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} - run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${{ matrix.shard }}.json" + run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${E2E_MANIFEST_SHARD}.json" - name: Upload blob report if: always() @@ -1104,6 +1141,8 @@ jobs: # reports to download, and the artifact step would fail on an empty run. # `e2e-gate` below, not this job, is what the merge queue waits on. if: always() && needs.changes.outputs.run == 'true' + # Report aggregation queries GitHub history and downloads artifacts with the + # job token, so it remains on a GitHub-hosted runner during the pilot. runs-on: ubuntu-latest # Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright CLI. # `runtime` tag — no Go needed for merging blob reports. @@ -1473,10 +1512,11 @@ jobs: # count changes. e2e-gate: name: E2E Tests Passed - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }} needs: [ changes, + runner_plan, build, playwright_image, e2e, @@ -1490,6 +1530,7 @@ jobs: steps: - name: Require the E2E jobs to have passed or been skipped env: + RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }} CHANGES_RESULT: ${{ needs.changes.result }} BUILD_RESULT: ${{ needs.build.result }} PLAYWRIGHT_IMAGE_RESULT: ${{ needs.playwright_image.result }} @@ -1502,6 +1543,7 @@ jobs: set -euo pipefail echo "Change detection: ${CHANGES_RESULT}" + echo "Runner plan: ${RUNNER_PLAN_RESULT}" echo "Build: ${BUILD_RESULT}" echo "Playwright image: ${PLAYWRIGHT_IMAGE_RESULT}" echo "E2E shards: ${E2E_RESULT}" @@ -1510,6 +1552,13 @@ jobs: echo "Desktop smoke: ${DESKTOP_E2E_RESULT}" echo "Report merge: ${E2E_REPORT_RESULT}" + # Runner planning must succeed because its outputs select the runner + # fleet. A planner error cannot silently become a hosted fallback. + if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then + echo "Runner planning did not succeed; refusing to report a pass." >&2 + exit 1 + fi + # Change detection itself must succeed: if it errored we do not know # whether the skips below were deliberate. if [[ "${CHANGES_RESULT}" != "success" ]]; then diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml index 9e3fb5d543..1dad80d027 100644 --- a/.github/workflows/frontend-tests.yml +++ b/.github/workflows/frontend-tests.yml @@ -29,12 +29,42 @@ permissions: packages: read jobs: + runner_plan: + name: Plan external runner allocation + runs-on: ubuntu-latest + outputs: + plan: ${{ steps.plan.outputs.plan }} + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Plan runner allocation + id: plan + uses: ./.github/actions/plan-external-runners + with: + workflow: frontend + run-id: ${{ github.run_id }} + families: >- + [ + {"name":"changes","tier":"light"}, + {"name":"frontend","tier":"standard"}, + {"name":"frontend_gate","tier":"light"} + ] + burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }} + percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }} + light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }} + standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }} + # The former trigger-level `paths:` list, applied one level down so a skip # still produces a conclusion. `.github/scripts/changed-paths.py` implements # the same last-match-wins semantics GitHub applies to `paths:`. changes: name: Detect relevant changes - runs-on: ubuntu-latest + needs: runner_plan + runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }} timeout-minutes: 10 outputs: run: ${{ steps.detect.outputs.run }} @@ -103,9 +133,9 @@ jobs: frontend: name: Run Frontend Lint, Tests, and Build - needs: changes + needs: [changes, runner_plan] if: needs.changes.outputs.run == 'true' - runs-on: ubuntu-latest + runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }} # Pre-baked image bundles Node 24, pnpm 9.15.9, Go, and go-licenses — # see .github/docker/ci-base/Dockerfile. container: ghcr.io/kdlbs/kandev-ci:build-latest @@ -121,6 +151,7 @@ jobs: uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 0 + persist-credentials: false - name: Mark workspace safe for git shell: bash @@ -353,22 +384,28 @@ jobs: # conclusion to read instead of a check that stays pending forever. frontend-gate: name: Frontend Tests Passed - runs-on: ubuntu-latest - needs: [changes, frontend] + runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }} + needs: [changes, frontend, runner_plan] if: always() steps: - name: Require the frontend job to have passed or been skipped env: + RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }} CHANGES_RESULT: ${{ needs.changes.result }} FRONTEND_RESULT: ${{ needs.frontend.result }} run: | set -euo pipefail + echo "Runner plan: ${RUNNER_PLAN_RESULT}" # `skipped` is a pass: the change-detection job decided nothing this # workflow covers was touched. `cancelled` is not — a cancelled job # never reached a verdict, and treating that as a pass would let an # untested merge group through. + if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then + echo "Runner planning finished with result: ${RUNNER_PLAN_RESULT}" >&2 + exit 1 + fi if [[ "${CHANGES_RESULT}" != "success" ]]; then echo "Change detection finished with result: ${CHANGES_RESULT}" >&2 exit 1 diff --git a/.github/workflows/lint-action-pinning.yml b/.github/workflows/lint-action-pinning.yml index d219fc4a8f..ba87cbff2b 100644 --- a/.github/workflows/lint-action-pinning.yml +++ b/.github/workflows/lint-action-pinning.yml @@ -34,6 +34,8 @@ permissions: jobs: lint: name: All action refs are SHA-pinned + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest steps: @@ -73,6 +75,12 @@ jobs: - name: Test E2E workflow contract run: python3 .github/scripts/e2e-tests-workflow-contract_test.py + - name: Test external runner workflow contract + run: python3 .github/scripts/external-runner-workflow-contract_test.py + + - name: Test external runner planner + run: python3 .github/scripts/runner-plan_test.py + - name: Test bounded step summary run: python3 .github/scripts/bounded-step-summary_test.py diff --git a/.github/workflows/lint-harness-files.yml b/.github/workflows/lint-harness-files.yml index bee05abe6c..4bea50ceea 100644 --- a/.github/workflows/lint-harness-files.yml +++ b/.github/workflows/lint-harness-files.yml @@ -33,6 +33,8 @@ permissions: jobs: lint: name: Harness files pass lint + # Checkout receives the short-lived job token. Keep this credential-bearing + # path on GitHub-hosted capacity during the external-runner pilot. runs-on: ubuntu-latest steps: diff --git a/docs/ci-merge-queue.md b/docs/ci-merge-queue.md index dfff647134..b9a2e853e0 100644 --- a/docs/ci-merge-queue.md +++ b/docs/ci-merge-queue.md @@ -115,6 +115,79 @@ and it closes the gap where a subject outside their hand-maintained lists changed and nothing checked it. Their contract tests were updated to assert the absence of a filter rather than the presence of specific entries. +### External CI runner capacity + +The normal Linux CI workflows have one activation switch, one percentage +control, and two runner tiers. The switch and percentage are unset by default, +so the workflows use GitHub-hosted runners. + +| Variable | Pilot value | Purpose | +| -------- | ----------- | ------- | +| `KANDEV_CI_EXTERNAL_ENABLED` | unset | Set to `true` to allow external assignments | +| `KANDEV_CI_EXTERNAL_PERCENT` | unset or `0` | Assign 0 to 100 percent of eligible jobs | +| `KANDEV_CI_RUNNER_LIGHT` | `ubicloud-standard-2-ubuntu-2404` | Control and aggregate jobs | +| `KANDEV_CI_RUNNER_STANDARD` | `ubicloud-standard-4-ubuntu-2404` | Eligible browser and frontend test jobs | + +Set `KANDEV_CI_EXTERNAL_ENABLED` to the exact value `true` when the Actions queue +needs extra capacity. Set it to `false`, or remove it, after the queue returns +to its normal level. The two tier variables stay configured between bursts, so +changing the instance type requires only an Actions variable update. + +Set `KANDEV_CI_EXTERNAL_PERCENT` to `50` for a pilot. The E2E matrix then sends +seven of fourteen normal shards to the standard tier. Backend checkout and +report-token jobs remain on GitHub-hosted runners; the aggregate backend gate +and frontend jobs are eligible. Singleton jobs use a stable hash cohort, so +their share approaches the percentage across workflow runs. + +Leave the percentage unset for the default GitHub-only mode. Set it to `100` +to send every eligible job with a non-empty tier label to external capacity. +Malformed or out-of-range values fail closed to GitHub-hosted runners and emit +a planner warning. + +The reusable `.github/actions/plan-external-runners` composite action runs on +the planner job's `ubuntu-latest` checkout. Each workflow declares its eligible +families as JSON, and the action returns one JSON plan with resolved runner +labels: + +```yaml +runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }} +runs-on: ${{ matrix.runner }} +``` + +If a tier variable is empty, that tier uses `ubuntu-latest`. If a non-empty +label is unavailable, GitHub leaves the job queued or fails it. Clear the +invalid label and rerun the workflow. Jobs already queued or running keep their +original runner. New jobs use the current variables. + +The E2E build and report jobs, Docker/Kind shards, Kubernetes compatibility +jobs, Playwright image job, desktop smoke job, backend checkout/static/test +shards, Postgres service job, architecture and harness linters, and Windows job +stay on GitHub-hosted runners. Release, publishing, signing, deployment, and +credential-bearing jobs stay on their existing runners. The workflows do not +add permissions, secrets, or persistent state. + +The same switch and tier labels apply to eligible E2E, backend-gate, and +frontend jobs. Architecture-lint, action-pinning, and harness-lint stay hosted +and do not create planner jobs. The planner does not change job names, test +selection, matrix values, artifacts, dependencies, timeouts, permissions, or +required conclusions. + +Pilot procedure: + +1. Merge the workflow change while `KANDEV_CI_EXTERNAL_ENABLED` and + `KANDEV_CI_EXTERNAL_PERCENT` are unset. +2. Install the [Ubicloud GitHub App](https://www.ubicloud.com/docs/github-actions-integration) for this repository. +3. Set both tier variables to the pilot labels in the table. Leave Premium + Runners disabled. Premium is an account-level setting, not a per-job tier. +4. Set `KANDEV_CI_EXTERNAL_PERCENT=50` for the first pilot. +5. Set `KANDEV_CI_EXTERNAL_ENABLED=true` while the queue is full. +6. Compare at least three runs for queue delay, job duration, runner label, + image setup, and failures. +7. Set the switch to `false` after the queue recovers. Keep the labels and + percentage for the next burst. + +Use the [Ubicloud runner type reference](https://www.ubicloud.com/docs/github-actions-integration/runner-types) when changing a tier. Use GitHub's [runner selection reference](https://docs.github.com/en/actions/how-tos/write-workflows/choose-where-workflows-run/choose-the-runner-for-a-job) for `runs-on` expressions. Do not enable [Ubicloud Premium Runners](https://www.ubicloud.com/docs/github-actions-integration/use-premium-runners) for this pilot. + ### PR context in a merge group A `merge_group` run has no `github.event.pull_request` and no diff --git a/docs/decisions/2026-09-06-opt-in-external-e2e-runners.md b/docs/decisions/2026-09-06-opt-in-external-e2e-runners.md new file mode 100644 index 0000000000..7db829088f --- /dev/null +++ b/docs/decisions/2026-09-06-opt-in-external-e2e-runners.md @@ -0,0 +1,143 @@ +# ADR-2026-09-06-opt-in-external-e2e-runners: Opt in selected Linux CI jobs to external runners + +**Status:** accepted +**Date:** 2026-09-06 +**Area:** infra, workflow, security + +## Context + +The E2E workflow creates fourteen normal Playwright shards, six host-Docker and +Kind shards, two Kubernetes compatibility jobs, and supporting build, report, +and gate jobs. Backend and frontend workflows also contain long Linux test +jobs. Recent timing work reduced shard imbalance, but GitHub-hosted runner wait +can still dominate the merge-queue critical path. + +An external provider can add capacity. However, hard-coded labels make rollback +require another workflow merge. One label for all jobs also prevents low-cost +control jobs from using a smaller instance class. + +Operators need one fast switch during a queue incident, plus a controlled way +to pilot only part of the eligible workload. They also need to keep approved +instance labels configured between incidents. Provider-wide routing moves jobs +outside the reviewed Docker, platform, and credential boundaries. + +## Decision + +Use `KANDEV_CI_EXTERNAL_ENABLED` as the activation switch. A value of `true` +activates configured external tiers for newly dispatched eligible jobs. Any +other value keeps eligible jobs on `ubuntu-latest`. + +Use `KANDEV_CI_EXTERNAL_PERCENT` to control the fraction of eligible jobs that +the external fleet receives while burst mode is active. The default is `0`. +The planner treats `0` as all GitHub-hosted and `100` as all external jobs with +non-empty tier labels. Values from `1` through `99` are allocated +deterministically per job family. Matrix families receive exactly +`floor(N * percentage / 100)` external instances. Singleton families hash the +workflow run into a stable bucket and approach the percentage across runs. A +missing value means `0`; invalid or out-of-range values fail closed to +GitHub-hosted runners with a visible warning. + +Store the complete external labels in two independent repository variables: + +- `KANDEV_CI_RUNNER_LIGHT` for change detection and the required gate. +- `KANDEV_CI_RUNNER_STANDARD` for normal E2E shards and eligible frontend + tests. Credential-bearing build/report and backend checkout paths remain + hosted during the pilot. + +If burst mode is active but a tier label is empty, jobs in that class use +`ubuntu-latest`. A non-empty invalid label remains visible as a queued or failed +job. Do not mix instance types inside the normal shard matrix. + +Apply the same tier contract to eligible read-only Linux jobs in the E2E and +frontend workflows, plus the backend aggregate gate. Keep E2E build/report +jobs, backend checkout/static/test-shard/ambient jobs, architecture and +harness linters, the Playwright image-resolution job, Docker/Kind shards, +Kubernetes compatibility jobs, desktop E2E, Postgres service jobs, and Windows +jobs on `ubuntu-latest` during the initial pilot. Do not apply this variable to +`pull_request_target`, production, deployment, release, publishing, signing, +or other credential-bearing jobs. + +Use Ubicloud x64 Ubuntu 24.04 as the initial provider. Configure standard-2 for +the light tier and standard-4 for the standard tier. Keep Premium Runners +disabled because Ubicloud controls them at the account level. + +Set burst mode to `false` after the queue recovers. This change affects only +newly dispatched jobs. GitHub does not migrate queued or active jobs. + +The external provider is part of the selected jobs' trust boundary. The pilot +adds no secrets or permissions, retains `persist-credentials: false`, and +keeps cache branch protection enabled. Expansion to protected workflows +requires a measured compatibility and security review. + +## Consequences + +- Operators can enable burst capacity or return to GitHub-hosted runners + with one repository-variable change. +- Runner labels remain provider-neutral workflow data. Ubicloud and runner + size remain operator-selected values. +- Operators can resize one tier without changing other job classes. +- Operators can set `KANDEV_CI_EXTERNAL_PERCENT` to `0`, `50`, or `100` to + stage a pilot without editing workflow files. +- Empty tier labels provide a safe GitHub-hosted fallback during partial + configuration. +- An invalid or unavailable configured label can leave new jobs queued until + the operator clears the variable and reruns them. The workflow does not + silently use another fleet. +- External jobs expose their short-lived GitHub token, source, artifacts, and + logs to the approved provider under the existing read-only permissions. +- The pilot can reduce runner wait, but it cannot remove merge-group ordering, + slow tests, cold caches, or artifact-transfer time. +- A planner job adds a small GitHub-hosted dependency before eligible jobs can + start. Exact compute-share percentages across unrelated workflows are not + promised. + +## Alternatives Considered + +### Hard-code Ubicloud labels + +Rejected because enablement, size changes, and rollback each require a +workflow commit and merge-queue pass. + +### Use one runner-label variable as the activation switch + +Rejected because one label gives light and standard workloads the same +instance size. Changing the label also erases the configured paid-runner value +when the operator returns to GitHub-hosted capacity. + +### Use tier labels without a separate activation switch + +Rejected because each burst requires multiple variable changes. A partial +change can unintentionally split eligible jobs across paid and GitHub-hosted +capacity. + +### Enable provider-wide premium routing + +Rejected because it removes job-level selection and routes workloads +whose trust and host-dependency boundaries are outside this pilot. + +### Move every Linux CI job in the first rollout + +Rejected because host-Docker, Kind, Kubernetes compatibility, image-resolution, +desktop, service, and Windows jobs need separate compatibility evidence. Their +smaller initial capacity benefit does not justify widening the pilot boundary. + +### Use a direct percentage expression in `runs-on` + +Rejected because GitHub expressions provide comparisons and boolean operators, +but no random or modulo operator. A read-only planner job can produce a stable +assignment for matrix and singleton job families and can fail closed on invalid +operator input. + +### Distribute an exact percentage of all workflow compute + +Rejected because workflows contain different job-family sizes and runtimes. +The contract allocates matrix instances exactly and singleton jobs by stable +cohort, then measures real queue and execution behavior before any adaptive +policy is considered. + +### Fall back automatically inside a workflow run + +Rejected because GitHub chooses `runs-on` before job execution. Implementing a +second-fleet retry duplicates job graphs and complicates the required gate. +It can also run the same untrusted workload on an unreviewed fleet. Manual +variable rollback keeps the selected trust boundary explicit. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index d48eafb83d..a5f8b1676d 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -245,3 +245,4 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 2026-09-04-generation-fenced-task-environment-ownership | [Fence Task Environment Ownership by Generation](2026-09-04-generation-fenced-task-environment-ownership.md) | accepted | backend | 2026-09-04 | | 2026-09-04-use-repository-token-for-runtime-pin-prs | [Use the built-in Actions token for runtime pin PRs](2026-09-04-use-repository-token-for-runtime-pin-prs.md) | accepted | workflow, security | 2026-09-04 | | 2026-09-05-agent-owned-credential-file-conflicts | [Agent-owned credential file conflicts](2026-09-05-agent-owned-credential-file-conflicts.md) | accepted | backend | 2026-09-05 | +| 2026-09-06-opt-in-external-e2e-runners | [Opt in selected Linux CI jobs to external runners](2026-09-06-opt-in-external-e2e-runners.md) | accepted | infra, workflow, security | 2026-09-06 | diff --git a/docs/plans/external-e2e-runner-capacity/plan.md b/docs/plans/external-e2e-runner-capacity/plan.md new file mode 100644 index 0000000000..1b05112ec7 --- /dev/null +++ b/docs/plans/external-e2e-runner-capacity/plan.md @@ -0,0 +1,175 @@ +--- +created: 2026-09-06 +status: complete +requirements: + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003 +system_design: + - ../../specs/platform/system-design/external-e2e-runner-capacity.md +legacy_specs: [] +--- + +# Implementation Plan: External CI Runner Capacity + +## Overview + +Add two provider-neutral runner tiers, one burst-mode switch, and an optional +deterministic percentage rollout to the normal Linux CI workflows. Protect +runner placement with workflow and planner contract tests. Update the +merge-queue runbook with the Ubicloud configuration, activation, rollback, and +measurement procedure. + +## Scope + +### In scope + +- Route eligible E2E and frontend jobs, plus the backend aggregate gate, + through the light or standard runner tier. +- Keep token-bearing E2E build/report, backend checkout/test, architecture, + action-pinning, and harness jobs on GitHub-hosted runners. +- Use `KANDEV_CI_EXTERNAL_ENABLED` as the single activation switch for both tiers. +- Use `KANDEV_CI_EXTERNAL_PERCENT` to allocate 0 to 100 percent of eligible + job instances when burst mode is active. +- Keep configured tier labels unchanged while burst mode is inactive. +- Keep `playwright_image`, `e2e-containers`, + `e2e-kubernetes-compatibility`, and `desktop-e2e` explicitly on + `ubuntu-latest`. +- Keep backend Postgres/service, Windows, and other protected jobs on + `ubuntu-latest`. +- Add workflow and planner contract coverage for eligible and protected jobs. +- Document the Ubicloud standard-2 and standard-4 pilot, manual rollback, and + trust boundary in the merge-queue runbook. + +### Out of scope + +- Installing the Ubicloud GitHub App or changing live repository Actions + variables from repository code. +- Moving Docker/Kind, compatibility, image-resolution, desktop, Postgres + service, Windows, release, publishing, signing, deployment, or + `pull_request_target` jobs. +- Adding a heavy tier before the first pilot supplies CPU or memory evidence. +- Weighted or adaptive allocation based on historical runtime or queue metrics. +- Guaranteeing an exact percentage of total compute across unrelated workflows. +- Changing shard counts, test selection, worker counts, timeouts, artifact + flow, permissions, required checks, or merge-queue rules. + +## Technical approach + +### Workflow placement + +Eligible singleton jobs read a resolved runner label from the planner's JSON +output: + +```yaml +runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }} +``` + +Matrix jobs read their matrix from that plan and use its `runner` field. +Protected jobs keep an explicit `ubuntu-latest` value and do not depend on the +planner. + +For percentage rollout, add a read-only `runner-plan.py` planner job on +`ubuntu-latest`. It validates the percentage and emits runner assignments for +each eligible job family. Matrix families receive exactly +`floor(N * percentage / 100)` external assignments. Singleton families use a +stable hash bucket. Workflows declare their family data through a reusable +composite action. Downstream jobs consume one JSON plan containing resolved +runner labels and matrices. + +If burst mode is inactive, eligible jobs use GitHub-hosted capacity. If burst +mode is active, each job uses its configured tier. An empty tier uses +`ubuntu-latest`. + +### Contract tests + +Extend the E2E, backend, and frontend workflow contract tests with focused +assertions for every affected job block. Add equivalent assertions that the +architecture-lint, action-pinning, and harness-lint workflows stay hosted and +do not create planner jobs. + +Assert `runs-on: ubuntu-latest` in every protected block. Also assert that +protected jobs do not reference the burst, percentage, or tier variables. Keep +Ubicloud labels out of the workflow and contract tests. + +Add `.github/scripts/runner-plan_test.py` for percentage validation, matrix +floor allocation, singleton stability, rerun stability, empty labels, and +fail-closed malformed input. + +### Operator runbook + +Add an external-runner section to `docs/ci-merge-queue.md`. Explain that the +workflow change is merged while burst mode is inactive. List both tier +variables, their initial Ubicloud values, and the protected job categories. + +Explain that one variable activates or deactivates paid capacity. State that +variable changes affect only new jobs. Define the comparison and rollback +signals. Link the GitHub and Ubicloud source documentation. + +## Tests + +- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1` through `.4`: the workflow + contract test verifies the toggle, two tier variables, and GitHub fallback. +- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5`: existing E2E workflow + contract tests remain unchanged and pass, protecting shard, cache, report, + and gate behavior. +- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6`: the design and runbook + document visible failure plus manual clear-and-rerun recovery. GitHub owns + label scheduling behavior. +- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1` and `.2`: the focused + contract test verifies that protected jobs retain `ubuntu-latest`. Action + pinning and `zizmor` verify that the workflow adds no unpinned action or new + permission path. +- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.3`: post-merge pilot evidence + records selected labels, queue delay, job duration, and reliability for at + least three representative runs. +- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.1` through `.6`: + `runner-plan_test.py` and workflow contract tests verify 0/50/100 behavior, + matrix floor allocation, stable singleton allocation, rerun stability, and + fail-closed malformed input. + +## Work orders + +- [x] [Task 01: Add Opt-in E2E Runner Selection](task-01-add-opt-in-e2e-runner-selection.md) +- [x] [Task 02: Extend Tiered Runners to Linux CI](task-02-extend-tiered-runners-to-linux-ci.md) +- [x] [Task 03: Add Deterministic Percentage Rollout](task-03-deterministic-percentage-rollout.md) + +## Verification results + +- `python3 .github/scripts/e2e-tests-workflow-contract_test.py`: 14 tests passed. +- `python3 .github/scripts/external-runner-workflow-contract_test.py`: 5 tests passed. +- `python3 .github/scripts/runner-plan_test.py`: 8 tests passed. +- `python3 .github/scripts/frontend-tests-workflow-contract_test.py`: 4 tests passed. +- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed. +- `python3 .github/scripts/lint-action-pinning.py`: all 21 workflow files passed. +- `actionlint` v1.7.12: passed for all six touched workflows. +- `zizmor` v1.25.2: no findings for all six touched workflows. +- `python3 scripts/lint-spec-files.py --all`: passed. +- `git diff --check`: passed. + +The full `zizmor .github/workflows` scan can still report findings in existing +workflows outside this change. The six touched workflows report no findings. + +The Ubicloud pilot remains an operator action after merge. It needs at least +three representative runs before a heavier tier or protected job moves. + +Tasks 01, 02, and 03 are complete. The Ubicloud pilot remains an operator +action after merge. + +## Risks + +- A misspelled or unavailable tier label can leave an eligible job queued + until an operator clears the variable and reruns it. +- An active burst with an empty tier label silently uses `ubuntu-latest` for + that class. The runbook must include a pre-activation configuration check. +- An external runner image can differ from GitHub's host image even when the + normal build and shard jobs use repository-owned containers. +- External transparent caching can change cache performance and poisoning risk. + Branch protection must remain enabled for the public repository. +- Moving the report and gate removes a small amount of GitHub-hosted demand but + also makes their prompt completion depend on external capacity. +- Runner capacity can reduce queue delay without improving merge-group + serialization or slow test execution. +- A planner job adds a small GitHub-hosted dependency before eligible jobs can + start. Singleton percentage allocation is a cohort approximation, not an + exact per-run share. diff --git a/docs/plans/external-e2e-runner-capacity/task-01-add-opt-in-e2e-runner-selection.md b/docs/plans/external-e2e-runner-capacity/task-01-add-opt-in-e2e-runner-selection.md new file mode 100644 index 0000000000..30a0c3df6f --- /dev/null +++ b/docs/plans/external-e2e-runner-capacity/task-01-add-opt-in-e2e-runner-selection.md @@ -0,0 +1,116 @@ +--- +id: "01-add-opt-in-e2e-runner-selection" +title: "Add opt-in E2E runner selection" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002 +acceptance_criteria: + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.3 +system_design: + - ../../specs/platform/system-design/external-e2e-runner-capacity.md +--- + +# Task 01: Add Opt-in E2E Runner Selection + +## Summary + +Add two runner tiers and one burst-mode switch to the five eligible E2E jobs. +Protect the placement boundary with a workflow contract test. Document the +Ubicloud configuration, activation, and rollback procedure. + +## In scope + +- Update two light and three standard `runs-on` values in the E2E workflow. +- Assert eligible and protected job placement in the E2E workflow contract + test. +- Add external-runner setup, rollback, security, and measurement guidance to + the merge-queue runbook. + +## Out of scope + +- Changing live GitHub or Ubicloud settings. +- Moving any protected or credential-bearing job. +- Changing workflow behavior beyond runner placement. +- Adding a heavy runner tier. + +## Acceptance + +- When burst mode is inactive, all five eligible jobs use `ubuntu-latest`. +- When burst mode is active, two jobs use the light tier and three use the + standard tier. An empty tier label uses `ubuntu-latest`. +- The four protected E2E jobs remain explicitly on `ubuntu-latest`, and the + workflow adds no secret or permission. +- Contract tests and operator guidance make the initial placement, rollback, + and pilot measurements explicit. + +## Verification + +```bash +python3 .github/scripts/e2e-tests-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +python3 .github/scripts/lint-action-pinning.py +actionlint .github/workflows/e2e-tests.yml +zizmor .github/workflows/e2e-tests.yml +python3 scripts/lint-spec-files.py --all +git diff --check +``` + +## Files likely touched + +- `.github/workflows/e2e-tests.yml` +- `.github/scripts/e2e-tests-workflow-contract_test.py` +- `docs/ci-merge-queue.md` +- `docs/specs/platform/requirements/external-e2e-runner-capacity.md` +- `docs/specs/platform/system-design/external-e2e-runner-capacity.md` +- `docs/plans/external-e2e-runner-capacity/plan.md` +- `docs/plans/external-e2e-runner-capacity/task-01-add-opt-in-e2e-runner-selection.md` + +## Dependencies + +None. + +## Risks + +- String-based workflow contract extraction depends on stable job boundaries. + The test must use the existing `job_block` helper. +- A live provider pilot is the only proof of image and capacity compatibility. + Local checks prove the workflow contract, not external availability. + +## Parallelism + +`sequential` + +## Inputs + +- `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001` and + `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002`. +- The external E2E runner capacity system design. +- ADR 2026-09-06 for the opt-in placement and trust boundary. +- Existing E2E job topology and workflow contract tests. + +## Results + +- Added `KANDEV_CI_EXTERNAL_ENABLED` expressions to the two light jobs and three + standard jobs in `.github/workflows/e2e-tests.yml`. +- Kept the Playwright image, Docker/Kind, Kubernetes compatibility, and desktop + jobs on `ubuntu-latest`. +- Added the eligible and protected job assertions to the E2E workflow contract + test. The full contract suite passes with 14 tests. +- Added the burst activation, tier values, rollback, security boundary, and + pilot measurement procedure to `docs/ci-merge-queue.md`. +- Action pinning tests and lint pass. `actionlint` v1.7.12 passes through + `go run`. Targeted `zizmor` reports no findings. +- The Ubicloud pilot is not run in CI code changes. Operators must collect the + three-run comparison after merge before expanding the pilot. diff --git a/docs/plans/external-e2e-runner-capacity/task-02-extend-tiered-runners-to-linux-ci.md b/docs/plans/external-e2e-runner-capacity/task-02-extend-tiered-runners-to-linux-ci.md new file mode 100644 index 0000000000..f80bc3077d --- /dev/null +++ b/docs/plans/external-e2e-runner-capacity/task-02-extend-tiered-runners-to-linux-ci.md @@ -0,0 +1,119 @@ +--- +id: "02-extend-tiered-runners-to-linux-ci" +title: "Extend tiered runners to Linux CI" +status: done +wave: 1 +depends_on: + - "01-add-opt-in-e2e-runner-selection" +plan: "plan.md" +requirements: + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002 +acceptance_criteria: + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.7 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2 +system_design: + - ../../specs/platform/system-design/external-e2e-runner-capacity.md +--- + +# Task 02: Extend Tiered Runners to Linux CI + +## Summary + +Apply the existing light and standard runner contract to eligible read-only +Linux jobs that can add merge-queue delay in the E2E, backend, and frontend +workflows. Keep checkout-token, report-token, host-service, container, Windows, +release, deployment, architecture-lint, action-pinning, and harness-lint +boundaries on GitHub-hosted runners. + +## In scope + +- Route the backend aggregate gate through the light tier. Keep backend jobs + that check out source or publish reports on GitHub-hosted runners. +- Route frontend control jobs through the light tier and the frontend test job + through the standard tier. +- Keep architecture-lint, action-pinning, and harness-lint checkout jobs on + GitHub-hosted runners. +- Extend workflow contract tests to assert eligible and protected placement. +- Update the merge-queue runbook with the complete eligible-job table. + +## Out of scope + +- Percentage allocation. That is Task 03. +- Moving backend Postgres/service or Windows jobs. +- Moving Docker/Kind, Kubernetes compatibility, image, desktop, release, + publishing, signing, deployment, or credential-bearing jobs. +- Changing test selection, matrices, artifacts, permissions, or required gates. + +## Acceptance + +- With burst mode inactive, every eligible job selects `ubuntu-latest`. +- With burst mode active and a non-empty tier label, eligible jobs select their + light or standard label. Empty labels use `ubuntu-latest`. +- Protected jobs remain explicitly on `ubuntu-latest` and do not reference the + burst or tier variables. +- Existing job names, dependencies, matrices, tests, artifacts, timeouts, + permissions, and required conclusions remain unchanged. + +## Verification + +```bash +python3 .github/scripts/e2e-tests-workflow-contract_test.py +python3 .github/scripts/frontend-tests-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +python3 .github/scripts/lint-action-pinning.py +python3 scripts/lint-harness-files.test.py +python3 scripts/lint-spec-files.py --all +actionlint .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml +zizmor .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml +git diff --check +``` + +## Files likely touched + +- `.github/workflows/backend-tests.yml` +- `.github/workflows/frontend-tests.yml` +- `.github/workflows/architecture-lint.yml` +- `.github/workflows/lint-action-pinning.yml` +- `.github/workflows/lint-harness-files.yml` +- `.github/scripts/e2e-tests-workflow-contract_test.py` +- `.github/scripts/frontend-tests-workflow-contract_test.py` +- Additional workflow contract tests as needed for source-only jobs. +- `docs/ci-merge-queue.md` + +## Dependencies + +Task 01 must be complete so the shared variable names and E2E contract are +already established. + +## Risks + +- Container jobs can use an external VM only when the container workflow does + not require host Docker or service containers. Protected jobs stay on + GitHub-hosted runners until measured evidence exists. +- Short lint jobs can add planner or external capacity overhead if the fleet is + unavailable. Manual burst rollback remains the recovery path. + +## Parallelism + +`sequential` + +## Results + +- Added planner-backed light and standard runner selection to eligible E2E, + backend-gate, and frontend jobs. +- Kept E2E build/report, backend checkout/static/test-shard/ambient, + architecture-lint, action-pinning, harness-lint, Postgres service, Windows, + Docker/Kind, Kubernetes, image, desktop, release, deployment, and + credential-bearing jobs on GitHub-hosted runners. +- Added workflow contract coverage for eligible jobs and protected boundaries. +- The renamed activation variable is `KANDEV_CI_EXTERNAL_ENABLED`. +- Contract, action-pinning, harness-lint, actionlint, specification, and diff + checks passed. diff --git a/docs/plans/external-e2e-runner-capacity/task-03-deterministic-percentage-rollout.md b/docs/plans/external-e2e-runner-capacity/task-03-deterministic-percentage-rollout.md new file mode 100644 index 0000000000..eddf92c130 --- /dev/null +++ b/docs/plans/external-e2e-runner-capacity/task-03-deterministic-percentage-rollout.md @@ -0,0 +1,116 @@ +--- +id: "03-deterministic-percentage-rollout" +title: "Add deterministic percentage rollout" +status: done +wave: 2 +depends_on: + - "02-extend-tiered-runners-to-linux-ci" +plan: "plan.md" +requirements: + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003 +acceptance_criteria: + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.1 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.2 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.3 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.4 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.5 + - AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.6 +system_design: + - ../../specs/platform/system-design/external-e2e-runner-capacity.md +--- + +# Task 03: Add Deterministic Percentage Rollout + +## Summary + +Add a read-only planner that assigns a deterministic percentage of eligible job +instances to external tiers. Use exact floor allocation for matrix families, +stable hash cohorts for singleton jobs, and fail-closed validation for +operator input. + +## In scope + +- Add `.github/scripts/runner-plan.py` and focused unit tests. +- Add planner jobs and outputs to the eligible workflows without changing + their test graph or protected-job boundaries. +- Feed validated runner assignments to matrix and singleton jobs through + `needs` outputs and `fromJSON` where required. +- Document `KANDEV_CI_EXTERNAL_PERCENT`, 0/50/100 behavior, warnings, and + rerun stability in the merge-queue runbook. +- Extend workflow contract tests for planner wiring and protected jobs. + +## Out of scope + +- Automatic activation, adaptive or weighted allocation, or same-run retry. +- Exact compute-share balancing across unrelated workflows. +- Any protected host-Docker, Kind, Kubernetes, service, Windows, release, + deployment, publishing, signing, or credential-bearing job. + +## Acceptance + +- Burst off or percentage `0` assigns all eligible jobs to `ubuntu-latest`. +- Percentage `100` assigns all eligible jobs with non-empty tier labels to the + configured tier; empty labels still use `ubuntu-latest`. +- At 50 percent, a fourteen-instance E2E matrix receives seven external + assignments. Protected backend checkout jobs remain hosted. +- A rerun of the same workflow run receives the same assignments. Singleton + jobs approach the configured percentage across different runs. +- Missing input means `0`. Malformed or out-of-range input emits a visible + warning and assigns all eligible jobs to `ubuntu-latest`. +- Protected jobs never consume planner output and remain on GitHub-hosted + runners. + +## Verification + +```bash +python3 .github/scripts/runner-plan_test.py +python3 .github/scripts/e2e-tests-workflow-contract_test.py +python3 .github/scripts/frontend-tests-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +python3 .github/scripts/lint-action-pinning.py +python3 scripts/lint-spec-files.py --all +actionlint .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml +zizmor .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml +git diff --check +``` + +## Files likely touched + +- `.github/scripts/runner-plan.py` +- `.github/scripts/runner-plan_test.py` +- Eligible workflow files and their contract tests. +- `docs/ci-merge-queue.md` + +## Dependencies + +Task 02 must be complete so all eligible job classes use the same tier +contract before planner outputs are introduced. + +## Risks + +- The planner adds a small GitHub-hosted dependency before eligible jobs start. +- A stable singleton cohort is an approximation across runs, not an exact + per-run percentage. +- Workflow outputs must be constrained to approved labels so operator input + cannot inject an arbitrary runner expression. + +## Parallelism + +`sequential` + +## Results + +- Added `.github/scripts/runner-plan.py` with deterministic matrix and + singleton allocation. +- Added `.github/actions/plan-external-runners/action.yml` so all workflow + planner jobs share one environment and script invocation. Workflows own the + family JSON; Python contains no workflow catalog. +- Added planner tests for unset, 0, 50, 100, empty-label, rerun, and invalid + percentage behavior. +- Added planner jobs with one resolved JSON plan output to eligible workflows. +- Updated the merge-queue runbook with percentage operation and rollback. +- Planner, workflow contract, action-pinning, actionlint, specification, and + diff checks passed. diff --git a/docs/specs/platform/README.md b/docs/specs/platform/README.md index 8791f98a32..61331aa90f 100644 --- a/docs/specs/platform/README.md +++ b/docs/specs/platform/README.md @@ -46,6 +46,7 @@ localization, feature toggles, health, and shared session recovery services. - [Browser console retention](requirements/browser-console-retention.md) - [Diagnostic logging](requirements/diagnostic-logging.md) - [Duration-aware E2E sharding and CI reliability](requirements/e2e-duration-aware-sharding.md) +- [External CI runner capacity](requirements/external-e2e-runner-capacity.md) - [Expected runtime log severity](requirements/expected-runtime-log-severity.md) - [Feature Toggles](requirements/feature-toggles.md) - [Git Credential Lease Reissue](requirements/git-credential-lease-reissue.md) @@ -86,6 +87,7 @@ localization, feature toggles, health, and shared session recovery services. - [Diagnostic logging System Design Part 2](system-design/diagnostic-logging-02.md) - [Expected runtime log severity](system-design/expected-runtime-log-severity.md) - [Duration-aware E2E sharding and CI reliability](system-design/e2e-duration-aware-sharding.md) +- [External CI runner capacity](system-design/external-e2e-runner-capacity.md) - [Health Endpoint — Surface the Running Version](system-design/health-endpoint-version.md) - [Internationalization (i18n)](system-design/i18n.md) - [LSP File Intelligence System Design Part 1](system-design/lsp-file-intelligence-01.md) diff --git a/docs/specs/platform/requirements/external-e2e-runner-capacity.md b/docs/specs/platform/requirements/external-e2e-runner-capacity.md new file mode 100644 index 0000000000..b074c76a9a --- /dev/null +++ b/docs/specs/platform/requirements/external-e2e-runner-capacity.md @@ -0,0 +1,135 @@ +--- +status: active +system: platform +created: 2026-09-06 +owners: + - kandev +--- + +# External CI Runner Capacity Requirements + +## Overview + +The merge queue can wait on GitHub-hosted runner capacity even when test jobs +are balanced. Repository operators need a reversible external runner option +for normal Linux CI workloads. This option must preserve required gates, test +selection, and privileged workflow boundaries. + +The platform system owns this contract because runner capacity is a shared +operational guarantee across CI workflows. The CI automation system continues +to own contributor trust gates for privileged pull-request automation. + +## Terminology + +- **Eligible Linux CI job:** A read-only Linux job in the E2E or frontend + workflows, plus backend aggregate gates that do not receive checkout or + report tokens. Credential-bearing checkout, report, service, and platform + jobs are protected and stay on GitHub-hosted runners. +- **Light tier:** External capacity for control jobs that run for seconds. +- **Standard tier:** External capacity for build, report, browser-test, and + Linux test jobs that need more CPU or memory. +- **Burst mode:** The operator-controlled state that activates configured + external runner tiers for new jobs. +- **Protected CI job:** A host-Docker, Kind, Kubernetes compatibility, image, + desktop, Windows, credential-bearing, release, or deployment job that stays + on a GitHub-hosted runner during the initial rollout. +- **Percentage rollout:** A deterministic allocation of eligible job + instances between GitHub-hosted and configured external capacity. + +## Requirements + +### REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001: Reversible tiered Linux CI runner selection + +**Intent:** Let repository operators configure runner tiers once. They can +then activate or deactivate external capacity with one repository variable. + +**User story:** As a repository operator, I want to use an approved external +runner fleet, so that I can reduce merge-queue wait across Linux CI. + +#### Acceptance criteria + +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1:** When the burst-mode + variable is not `true`, every eligible Linux CI job shall select + `ubuntu-latest`. Configured tier labels shall remain unchanged. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2:** When burst mode is + `true` and the percentage is `100`, each eligible Linux CI job shall use its + configured light or standard runner label. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3:** When burst mode is + `true` and a required tier label is empty, that job class shall select + `ubuntu-latest` for every percentage allocation. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4:** Changing a tier label + shall change the instance type for newly dispatched jobs in that class. It + shall not require a workflow edit or a percentage change. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5:** Changing burst mode, a + tier label, or the percentage shall preserve each workflow's matrix, test + selection, artifacts, dependencies, timeouts, permissions, and required + conclusion. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6:** When a non-empty tier + label cannot accept a job, that job shall remain visibly queued or fail. The + workflow shall not silently select an unapproved fleet. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.7:** The same burst switch and + tier labels shall apply consistently to eligible E2E, backend-gate, and + frontend jobs. Protected lint workflows shall remain GitHub-hosted. + +### REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002: Initial runner trust boundary + +**Intent:** Limit the first external-runner pilot to jobs with reviewed +permissions and runtime dependencies for that provider. + +#### Acceptance criteria + +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1:** During the initial + rollout, protected CI jobs shall select `ubuntu-latest` + regardless of burst mode or configured tier labels. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2:** The external-runner + option shall not add secrets or broaden `GITHUB_TOKEN` permissions. It shall + not persist runner state. It shall not move credential-bearing jobs to the + external fleet. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.3:** Operators shall be able + to identify each job's class, configured tier, and selected fleet. They shall + compare queue delay, execution time, and reliability before expansion. + +### REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003: Deterministic percentage rollout + +**Intent:** Let operators test an external fleet with a controlled fraction of +eligible jobs while the remaining jobs use GitHub-hosted capacity. + +#### Acceptance criteria + +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.1:** When burst mode is not + `true`, the percentage shall have no effect and every eligible job shall use + `ubuntu-latest`. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.2:** When burst mode is + `true` and the percentage is `0`, every eligible job shall use + `ubuntu-latest`. When it is `100`, every eligible job with a non-empty tier + label shall use that tier. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.3:** When the percentage is + between `0` and `100`, the planner shall allocate each job family + deterministically between GitHub-hosted and external capacity. A rerun of + the same workflow run shall keep the same allocation. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.4:** For a matrix family with + `N` instances, the planner shall allocate `floor(N * percentage / 100)` + instances to external capacity. A single-instance family shall use a stable + allocation across workflow runs that approaches the configured percentage. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.5:** A missing percentage + shall be treated as `0`. A malformed or out-of-range percentage shall fail + closed to `ubuntu-latest` and emit a visible planner warning. Neither case + shall select an external label. +- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.6:** Percentage allocation + shall not change job names, matrix values, artifacts, dependencies, test + selection, timeouts, permissions, or required conclusions. + +## Out of scope + +- Automatically activating burst mode from queue metrics. +- Automatically retrying an unavailable external runner label on GitHub-hosted + capacity within the same workflow run. +- Weighted allocation by historical runtime or queue metrics. +- Moving Docker/Kind shards, Kubernetes compatibility, image resolution, + desktop E2E, Postgres service jobs, Windows jobs, or credential-bearing jobs + in the initial rollout. +- Adding a heavy tier before measurements show that a workload needs it. +- Guaranteeing an exact percentage across workflows with different job counts + in one workflow run. +- Enabling an external provider's global or premium-fleet routing setting. +- Selecting ARM runners for x64 E2E binaries and containers. diff --git a/docs/specs/platform/system-design/external-e2e-runner-capacity.md b/docs/specs/platform/system-design/external-e2e-runner-capacity.md new file mode 100644 index 0000000000..2e3a690292 --- /dev/null +++ b/docs/specs/platform/system-design/external-e2e-runner-capacity.md @@ -0,0 +1,237 @@ +--- +status: current +system: platform +requirements: + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002 + - REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003 +created: 2026-09-06 +owners: + - kandev +--- + +# External CI Runner Capacity System Design + +## Purpose and boundaries + +This design adds provider-neutral runner tiers to the normal Linux CI +workflows. Two repository variables store the approved Linux x64 labels. One +burst-mode variable activates those labels when GitHub-hosted capacity is full. +An optional percentage variable lets operators run a deterministic fraction of +eligible jobs on the external fleet during a pilot. + +The tier labels remain configured while burst mode is inactive. An operator +can therefore activate or deactivate paid capacity with one variable change. +The operator can also change an instance type without a workflow edit. + +The design changes only runner placement. The duration-aware planner, shard +manifests, build artifacts, reports, required gates, and workflow permissions +remain authoritative. Host-Docker, Kind, compatibility, image-resolution, +desktop, service, Windows, checkout-token, report-token, release, and +deployment jobs remain outside the initial pilot. + +## Requirement mapping + +| Requirement | Design sections | +| --- | --- | +| `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001` | [Runner-selection contract](#runner-selection-contract), [Job placement](#job-placement), [Failure and recovery](#failure-and-recovery) | +| `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002` | [Security and trust boundary](#security-and-trust-boundary), [Rollout and observability](#rollout-and-observability) | +| `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003` | [Percentage allocation](#percentage-allocation), [Workflow integration](#workflow-integration), [Failure and recovery](#failure-and-recovery) | + +## Runner-selection contract + +The repository stores these Actions variables: + +| Variable | Purpose | Initial Ubicloud value | +| --- | --- | --- | +| `KANDEV_CI_EXTERNAL_ENABLED` | Activates configured external tiers only when its value is `true`. | Unset or `false` | +| `KANDEV_CI_EXTERNAL_PERCENT` | Selects the percentage of eligible job instances assigned to external capacity when burst mode is active. | Unset or `0` | +| `KANDEV_CI_RUNNER_LIGHT` | Selects capacity for change detection and required gates. | `ubicloud-standard-2-ubuntu-2404` | +| `KANDEV_CI_RUNNER_STANDARD` | Selects capacity for eligible browser and frontend test jobs. | `ubicloud-standard-4-ubuntu-2404` | + +The workflow consumes the planner's resolved light-tier value with this +`runs-on` expression: + +```yaml +runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }} +``` + +The planner returns the configured external label or `ubuntu-latest`, never an +expression fragment. Matrix jobs read their matrix from the same JSON plan and +use `runs-on: ${{ matrix.runner }}`. GitHub evaluates the value before it +dispatches the job. + +Percentage rollout cannot be implemented safely with a direct `runs-on` +expression. GitHub expressions provide comparisons and boolean operators but no +random or modulo operator. A planner job therefore computes assignments before +eligible jobs start and exposes one JSON plan through a job output. Downstream +jobs consume that plan with `needs` and `fromJSON`. + +If burst mode is active, the expression selects the complete label for that +tier. An empty tier label uses `ubuntu-latest`. A non-empty invalid label stays +visible as a queued or failed job. + +Variable changes affect jobs that GitHub dispatches after the change. GitHub +does not migrate queued or active jobs. To recover from an external incident, +set `KANDEV_CI_EXTERNAL_ENABLED` to `false` and rerun the workflow. + +The workflow does not parse provider names or instance sizes. It also does not +mix instance types inside one shard cohort. Comparable runner capacity keeps +the duration-aware timing profile useful across normal shards. + +## Percentage allocation + +`.github/scripts/runner-plan.py` is a generic read-only planner that runs on +`ubuntu-latest`. It receives the workflow identifier, run identifier, +workflow-owned JSON family descriptors, burst switch, percentage, and tier +labels. It contains no workflow or job-family catalog. It validates the +percentage and emits one JSON assignment plan plus a visible warning when the +percentage is invalid. + +The planner applies these rules: + +1. Any burst value other than `true`, or a percentage of `0`, assigns every + eligible instance to `ubuntu-latest`. +2. A percentage of `100` assigns every eligible instance with a non-empty tier + label to that tier. Empty tier labels still use `ubuntu-latest`. +3. For a matrix family with `N` instances, it hashes the workflow, job family, + run identifier, and matrix key, sorts the keys, and assigns exactly + `floor(N * percentage / 100)` instances to the external tier. +4. For a singleton family, it hashes the workflow, job-family key, and run + identifier into a stable bucket. A rerun keeps the same allocation, and + different workflow runs approach the configured percentage. +5. A missing percentage is treated as `0`. A non-integer or out-of-range + percentage assigns all instances to `ubuntu-latest` and emits a planner + warning. It never emits an unvalidated external label for invalid input. + +Each job family is allocated independently. At 50 percent, the fourteen E2E +shards receive seven external assignments. Protected backend checkout jobs +remain hosted; the backend aggregate gate and frontend jobs are still +eligible. Singleton jobs use a stable hash cohort, so their share approaches +50 percent across runs. The planner does not promise an exact percentage of +total compute across workflows with different family sizes. + +## Workflow integration + +Each workflow with eligible jobs adds a small planner job on `ubuntu-latest`. +Eligible jobs depend on the planner only for their runner assignment; their +existing test dependencies, matrices, artifacts, timeouts, permissions, and +required conclusions remain unchanged. A workflow with no eligible jobs does +not run the planner. + +The planner steps use the reusable composite action at +`.github/actions/plan-external-runners/action.yml`. The action owns the +validated environment wiring and `runner-plan.py` invocation, while each +workflow owns the family JSON and exposes one `plan` job output. This keeps the +job contract explicit without copying planner logic or teaching Python about +workflow-specific families. + +The JSON plan contains only resolved runner labels and matrix data declared by +the workflow. Protected jobs keep explicit `runs-on: ubuntu-latest` and do not +consume planner output. Contract tests assert the output wiring and the +protected-job boundary. + +## Job placement + +| Job | Initial runner selection | Rationale | +| --- | --- | --- | +| `E2E changes`, `e2e-gate` | Light tier with GitHub fallback | Control jobs are short but can wait many minutes for capacity. | +| `E2E build`, `e2e-report` | `ubuntu-latest` | They query GitHub history or download artifacts with the job token. | +| `e2e` | Standard tier with GitHub fallback | The fourteen normal shards need comparable CPU and memory and do not use explicit repository-token inputs. | +| `Backend changes`, `static_checks`, `test_shards`, `test_ambient_env` | `ubuntu-latest` | Their checkout action receives the short-lived job token. | +| `Backend test` gate | Light tier with GitHub fallback | The aggregate gate is short and has no checkout or service dependency. | +| `Frontend changes`, `frontend-gate` | Light tier with GitHub fallback | Control jobs are short and queue-sensitive. | +| `Frontend frontend` | Standard tier with GitHub fallback | The single frontend test job is a recurring queue bottleneck. | +| `Architecture lint`, action-pinning, harness-lint | `ubuntu-latest` | Their checkout action receives the short-lived job token. | +| `playwright_image` | `ubuntu-latest` | Retains the reviewed host-Docker and GHCR metadata path during the pilot. | +| `e2e-containers` | `ubuntu-latest` | Requires direct host Docker and creates Kind resources. | +| `e2e-kubernetes-compatibility` | `ubuntu-latest` | Requires direct host Docker and pinned Kind clusters. | +| `desktop-e2e` | `ubuntu-latest` | Retains the dedicated desktop container and toolchain boundary during the pilot. | +| Backend `postgres-boot`, Windows, and container/service jobs | `ubuntu-latest` | Preserve host-service, operating-system, and credential boundaries. | + +The runner expression does not alter `needs`, `if`, the matrix, the container +image, the environment, the timeout, or artifacts. The gate keeps +`if: always()` and its current result rules. + +## Workflow contract coverage + +The E2E, backend, and frontend workflow contract tests own the executable +placement contract. Dedicated checks verify that architecture-lint, +action-pinning, and harness-lint do not create planner jobs. They also verify +that every protected CI job uses `runs-on: ubuntu-latest` without the burst, +percentage, or tier variables. + +The contract test also protects the provider-neutral behavior: it asserts the +variable names and GitHub fallback, not the Ubicloud labels. Provider +installation and live variable values remain operator-owned external state. + +## Security and trust boundary + +An external runner executes pull-request and merge-group code. It receives the +short-lived job token, source, artifacts, and logs for its job. Provider +isolation does not make the provider part of GitHub's hosted trust boundary. + +The initial eligible jobs keep the workflow's existing top-level read-only +permissions: `contents: read`, `actions: read`, and `packages: read`. Checkout +continues to use `persist-credentials: false`. Jobs whose checkout, GitHub API, +or artifact action receives the short-lived token remain on `ubuntu-latest`. +The change adds no repository, environment, deployment, publishing, signing, +or production secret. + +The initial Ubicloud installation uses clean ephemeral x64 VMs and just-in-time +runner registration. Keep the default branch protection if the Ubicloud +transparent cache is enabled. Do not replace the pinned `actions/cache` +references during this rollout. + +Jobs triggered by `pull_request_target`, jobs with production or deployment +credentials, checkout-token paths, E2E build/report token paths, releases, +publishing, signing, and Windows builds are not eligible for burst mode. +Future container jobs can use a heavy tier only after a separate host- +dependency, performance, and security review. + +Ubicloud Premium Runners remain disabled. Ubicloud controls Premium Runners at +the account level, so that setting cannot provide job-level tier selection. + +## Failure and recovery + +| Condition | Required behavior | +| --- | --- | +| Burst mode is absent, empty, or not `true` | GitHub dispatches every eligible job to `ubuntu-latest`. | +| Burst mode is `true` and the tier has an approved label | GitHub dispatches new jobs in that class to the configured fleet. | +| Burst mode is `true` and the tier label is empty | GitHub dispatches that job class to `ubuntu-latest`. | +| A non-empty tier label is unavailable or invalid | The job remains visibly queued or fails. No unreviewed fallback runs. | +| External provider has an incident | Set burst mode to `false` and rerun. GitHub does not migrate queued or active jobs. | +| External image differs from the GitHub image | The repository container images own most toolchains. The pilot must verify the host-only change-detection and gate behavior. | +| External cache is cold | Existing dependency and artifact paths continue to work. Cache state can affect duration but not correctness. | + +## Rollout and observability + +1. Merge the workflow while `KANDEV_CI_EXTERNAL_ENABLED` is absent or `false`. +2. Install the Ubicloud GitHub App only for the Kandev repository. +3. Keep transparent-cache branch protection enabled. +4. Disable Premium Runners in the Ubicloud account. +5. Configure `KANDEV_CI_RUNNER_LIGHT` as + `ubicloud-standard-2-ubuntu-2404`. +6. Configure `KANDEV_CI_RUNNER_STANDARD` as + `ubicloud-standard-4-ubuntu-2404`. +7. When the GitHub queue is full, set `KANDEV_CI_EXTERNAL_ENABLED` to `true`. +8. Observe at least three representative pull-request or merge-group runs. + Record runner labels, queue delay, build duration, shard tail, report/gate + delay, cancellations, retries, infrastructure failures, and cache behavior. +9. After the queue recovers, set `KANDEV_CI_EXTERNAL_ENABLED` to `false`. +10. If the pilot shows a regression, disable burst mode and rerun failed jobs. + +Keep the two tier labels configured between bursts. Change the standard-tier +label to `ubicloud-standard-8-ubuntu-2404` only after CPU or memory evidence. + +The existing timing diagnostics separate queue delay from setup and test +execution. The pilot succeeds when runner wait decreases without a higher job +failure, cancellation, retry, cache-verification, or artifact-transfer rate. +GitHub merge-group ordering remains an independent source of serialization and +is not attributed to runner capacity. + +## Related decisions + +- [Opt in selected Linux CI jobs to external runners](../../../decisions/2026-09-06-opt-in-external-e2e-runners.md) +- [Duration-aware E2E sharding uses rolling `main` timings](../../../decisions/2026-08-10-duration-aware-e2e-sharding.md) +- [Cache host-runner E2E browser provisioning](../../../decisions/2026-08-30-e2e-browser-cache.md)