ci: add selectable external CI runners (#3451)

.github/actions/plan-external-runners/action.yml

@@ -0,0 +1,59 @@
+name: Plan external runner allocation
+description: Compute deterministic external-runner assignments for a workflow
+
+inputs:
+  workflow:
+    description: Runner-plan workflow family key
+    required: true
+  run-id:
+    description: Stable workflow run identifier used for allocation
+    required: true
+  families:
+    description: JSON job-family definitions owned by the calling workflow
+    required: true
+  burst:
+    description: Exact true value enables external allocation
+    required: false
+    default: ""
+  percent:
+    description: External allocation percentage from 0 to 100
+    required: false
+    default: ""
+  light-label:
+    description: Configured light-tier runner label
+    required: false
+    default: ""
+  standard-label:
+    description: Configured standard-tier runner label
+    required: false
+    default: ""
+
+outputs:
+  plan:
+    description: JSON object containing runner and matrix outputs
+    value: ${{ steps.plan.outputs.plan }}
+
+runs:
+  using: composite
+  steps:
+    - name: Plan runner allocation
+      id: plan
+      shell: bash
+      env:
+        RUNNER_PLAN_WORKFLOW: ${{ inputs.workflow }}
+        RUNNER_PLAN_RUN_ID: ${{ inputs.run-id }}
+        RUNNER_PLAN_FAMILIES: ${{ inputs.families }}
+        KANDEV_CI_EXTERNAL_ENABLED: ${{ inputs.burst }}
+        KANDEV_CI_EXTERNAL_PERCENT: ${{ inputs.percent }}
+        KANDEV_CI_RUNNER_LIGHT: ${{ inputs.light-label }}
+        KANDEV_CI_RUNNER_STANDARD: ${{ inputs.standard-label }}
+      run: |
+        python3 "$GITHUB_WORKSPACE/.github/scripts/runner-plan.py" \
+          --workflow "$RUNNER_PLAN_WORKFLOW" \
+          --run-id "$RUNNER_PLAN_RUN_ID" \
+          --families "$RUNNER_PLAN_FAMILIES" \
+          --burst "$KANDEV_CI_EXTERNAL_ENABLED" \
+          --percent "$KANDEV_CI_EXTERNAL_PERCENT" \
+          --light-label "$KANDEV_CI_RUNNER_LIGHT" \
+          --standard-label "$KANDEV_CI_RUNNER_STANDARD" \
+          --output "$GITHUB_OUTPUT"

.github/scripts/backend-tests-workflow-contract_test.py

@@ -25,7 +25,7 @@ class BackendTestsWorkflowContractTest(unittest.TestCase):
         generate = step_block(workflow, "Generate test report")
         publish = step_block(workflow, "Publish bounded test report summary")
 
-        temporary_summary = "$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md"
+        temporary_summary = "$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md"
         self.assertIn(
             "uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10",
             checkout,
@@ -37,12 +37,13 @@ class BackendTestsWorkflowContractTest(unittest.TestCase):
         )
         self.assertIn("path: .github/vendor/go-test-action", checkout)
         self.assertIn(f'report_summary="{temporary_summary}"', generate)
+        self.assertIn("BACKEND_TEST_SHARD: ${{ matrix.shard }}", generate)
         self.assertNotIn("${{ runner.temp }}", generate)
         self.assertIn(': > "$report_summary"', generate)
         self.assertIn('GITHUB_STEP_SUMMARY="$report_summary"', generate)
         self.assertIn(
             'INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/'
-            'test-results-${{ matrix.shard }}.json"',
+            'test-results-${BACKEND_TEST_SHARD}.json"',
             generate,
         )
         self.assertIn(
@@ -60,8 +61,8 @@ class BackendTestsWorkflowContractTest(unittest.TestCase):
         self.assertIn(f'--input "{temporary_summary}"', publish)
         self.assertIn('--output "$GITHUB_STEP_SUMMARY"', publish)
         self.assertIn("--diagnostics-label", publish)
-        self.assertIn("backend-test-results-${{ matrix.shard }}", publish)
-        self.assertIn("${{ github.run_id }}#artifacts", publish)
+        self.assertIn("backend-test-results-${BACKEND_TEST_SHARD}", publish)
+        self.assertIn("$GITHUB_RUN_ID#artifacts", publish)
 
     def test_full_json_diagnostics_remain_in_the_named_artifact(self) -> None:
         workflow = BACKEND_WORKFLOW.read_text(encoding="utf-8")

.github/scripts/e2e-tests-workflow-contract_test.py

@@ -193,6 +193,58 @@ cat "${FAKE_DOCKER_MANIFEST}"
         self.assertIn("timeout-minutes: 35", normal_job)
         self.assertNotIn("timeout-minutes: 25", normal_job)
 
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1
+    # @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2
+    def test_external_runner_tiers_are_toggleable_for_eligible_jobs(self) -> None:
+        workflow = E2E_WORKFLOW.read_text(encoding="utf-8")
+
+        light_jobs = {
+            "changes": ("build", "changes_runner"),
+            "e2e-gate": (None, "e2e_gate_runner"),
+        }
+
+        for job, (next_job, output_name) in light_jobs.items():
+            if next_job is None:
+                job_text = workflow.partition("  e2e-gate:\n")[2]
+            else:
+                job_text = job_block(workflow, job, next_job)
+            expected = (
+                "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan)."
+                f"{output_name} }}}}"
+            )
+            self.assertIn(
+                expected,
+                job_text,
+                f"{job} must select its configured tier",
+            )
+        e2e_job = job_block(workflow, "e2e", "playwright_image")
+        self.assertIn("runs-on: ${{ matrix.runner }}", e2e_job)
+        self.assertIn(
+            "matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}",
+            e2e_job,
+        )
+
+        protected_jobs = {
+            "build": "e2e",
+            "e2e-report": "e2e-gate",
+            "playwright_image": "e2e-containers",
+            "e2e-containers": "e2e-kubernetes-compatibility",
+            "e2e-kubernetes-compatibility": "desktop-e2e",
+            "desktop-e2e": "e2e-report",
+        }
+        for job, next_job in protected_jobs.items():
+            protected_text = job_block(workflow, job, next_job)
+            self.assertIn("runs-on: ubuntu-latest", protected_text)
+            self.assertNotIn("runner_plan", protected_text)
+            self.assertNotIn("KANDEV_CI_EXTERNAL", protected_text)
+            self.assertNotIn("KANDEV_CI_RUNNER_", protected_text)
+
     def test_contract_runs_in_the_unfiltered_required_workflow(self) -> None:
         workflow = LINT_WORKFLOW.read_text(encoding="utf-8")
 

.github/scripts/external-runner-workflow-contract_test.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Contract tests for external runner planner wiring and protected jobs."""
+
+from pathlib import Path
+import unittest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW_ROOT = REPO_ROOT / ".github" / "workflows"
+PLAN_SCRIPT = REPO_ROOT / ".github" / "scripts" / "runner-plan.py"
+
+
+def job_block(workflow: str, job: str, next_job: str | None) -> str:
+    marker = f"  {job}:\n"
+    _, separator, remainder = workflow.partition(marker)
+    if not separator:
+        raise AssertionError(f"Workflow has no {job} job")
+    if next_job is None:
+        return remainder
+    return remainder.partition(f"\n  {next_job}:\n")[0]
+
+
+class ExternalRunnerWorkflowContractTest(unittest.TestCase):
+    def assert_planner(
+        self, workflow_name: str, workflow_key: str, families: tuple[str, ...]
+    ) -> str:
+        workflow = (WORKFLOW_ROOT / workflow_name).read_text(encoding="utf-8")
+        planner = job_block(workflow, "runner_plan", "changes")
+        self.assertIn("runs-on: ubuntu-latest", planner)
+        self.assertIn("plan: ${{ steps.plan.outputs.plan }}", planner)
+        self.assertIn("uses: ./.github/actions/plan-external-runners", planner)
+        self.assertIn(f"workflow: {workflow_key}", planner)
+        self.assertIn("families: >-", planner)
+        self.assertIn("run-id: ${{ github.run_id }}", planner)
+        self.assertIn("burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}", planner)
+        self.assertIn("percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}", planner)
+        self.assertIn("light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}", planner)
+        self.assertIn("standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}", planner)
+        for family in families:
+            self.assertIn(f'"name":"{family}"', planner)
+        return workflow
+
+    def test_e2e_uses_planner_for_eligible_jobs_and_keeps_protected_jobs_hosted(self) -> None:
+        workflow = self.assert_planner(
+            "e2e-tests.yml",
+            "e2e",
+            ("changes", "e2e", "e2e_gate"),
+        )
+        self.assertIn(
+            "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}",
+            job_block(workflow, "changes", "build"),
+        )
+        build = job_block(workflow, "build", "e2e")
+        self.assertIn("runs-on: ubuntu-latest", build)
+        self.assertNotIn("runner_plan", build)
+        e2e = job_block(workflow, "e2e", "playwright_image")
+        self.assertIn(
+            "matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}",
+            e2e,
+        )
+        self.assertIn("runs-on: ${{ matrix.runner }}", e2e)
+        report = job_block(workflow, "e2e-report", "e2e-gate")
+        self.assertIn("runs-on: ubuntu-latest", report)
+        self.assertNotIn("runner_plan", report)
+        self.assertIn(
+            "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }}",
+            job_block(workflow, "e2e-gate", None),
+        )
+        for job, next_job in (
+            ("playwright_image", "e2e-containers"),
+            ("e2e-containers", "e2e-kubernetes-compatibility"),
+            ("e2e-kubernetes-compatibility", "desktop-e2e"),
+            ("desktop-e2e", "e2e-report"),
+        ):
+            protected = job_block(workflow, job, next_job)
+            self.assertIn("runs-on: ubuntu-latest", protected)
+            self.assertNotIn("runner_plan", protected)
+            self.assertNotIn("KANDEV_CI_EXTERNAL", protected)
+
+    def test_backend_uses_planner_and_keeps_services_and_windows_hosted(self) -> None:
+        workflow = self.assert_planner(
+            "backend-tests.yml",
+            "backend",
+            ("test",),
+        )
+        for job, next_job in (
+            ("changes", "static_checks"),
+            ("static_checks", "test_shards"),
+            ("test_ambient_env", "test"),
+        ):
+            protected = job_block(workflow, job, next_job)
+            self.assertIn("runs-on: ubuntu-latest", protected)
+            self.assertNotIn("runner_plan", protected)
+        shards = job_block(workflow, "test_shards", "test_ambient_env")
+        self.assertIn("- shard: 1", shards)
+        self.assertIn("- shard: 2", shards)
+        self.assertIn("runs-on: ubuntu-latest", shards)
+        self.assertNotIn("runner_plan", shards)
+        self.assertNotIn("matrix.runner", shards)
+        test_gate = job_block(workflow, "test", "postgres-boot")
+        self.assertIn(
+            "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).test_runner }}",
+            test_gate,
+        )
+        for job, next_job in (("postgres-boot", "test-windows"), ("test-windows", None)):
+            protected = job_block(workflow, job, next_job)
+            self.assertIn("runs-on: ubuntu-latest" if job == "postgres-boot" else "runs-on: windows-latest", protected)
+            self.assertNotIn("runner_plan", protected)
+
+    def test_frontend_uses_planner_for_test_and_gate(self) -> None:
+        workflow = self.assert_planner(
+            "frontend-tests.yml",
+            "frontend",
+            ("changes", "frontend", "frontend_gate"),
+        )
+        self.assertIn(
+            "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}",
+            job_block(workflow, "changes", "frontend"),
+        )
+        self.assertIn(
+            "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}",
+            job_block(workflow, "frontend", "frontend-gate"),
+        )
+        self.assertIn(
+            "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}",
+            job_block(workflow, "frontend-gate", None),
+        )
+
+    def test_protected_lint_workflows_do_not_run_the_planner(self) -> None:
+        for workflow_name in (
+            "architecture-lint.yml",
+            "lint-action-pinning.yml",
+            "lint-harness-files.yml",
+        ):
+            workflow = (WORKFLOW_ROOT / workflow_name).read_text(encoding="utf-8")
+            lint = job_block(workflow, "lint", None)
+            self.assertNotIn("runner_plan:", workflow)
+            self.assertNotIn("name: Require runner plan", lint)
+            self.assertIn("runs-on: ubuntu-latest", lint)
+
+    def test_plan_script_is_checked_by_required_action_pinning_workflow(self) -> None:
+        workflow = (WORKFLOW_ROOT / "lint-action-pinning.yml").read_text(encoding="utf-8")
+        self.assertTrue(PLAN_SCRIPT.is_file())
+        action = (WORKFLOW_ROOT / ".." / "actions" / "plan-external-runners" / "action.yml").resolve()
+        self.assertTrue(action.is_file())
+        action_text = action.read_text(encoding="utf-8")
+        self.assertIn("using: composite", action_text)
+        self.assertIn("families:", action_text)
+        self.assertIn("plan:", action_text)
+        self.assertIn("runner-plan.py", action_text)
+        self.assertNotIn("changes_runner:", action_text)
+        self.assertIn("python3 .github/scripts/runner-plan_test.py", workflow)
+        self.assertIn("python3 .github/scripts/external-runner-workflow-contract_test.py", workflow)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/scripts/runner-plan.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+"""Build deterministic runner assignments from workflow-owned family data."""
+
+import argparse
+import hashlib
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class RunnerPlan:
+    plan: dict[str, object]
+    warnings: list[str]
+
+
+@dataclass(frozen=True)
+class Family:
+    name: str
+    tier: str
+    instances: tuple[dict[str, object], ...]
+    is_matrix: bool
+
+
+GITHUB_RUNNER = "ubuntu-latest"
+VALID_TIERS = frozenset({"light", "standard"})
+FAMILY_NAME = re.compile(r"[a-z][a-z0-9_]*\Z")
+
+
+def _parse_percentage(raw: str | None) -> tuple[int, list[str]]:
+    if raw is None or raw.strip() == "":
+        return 0, []
+    value = raw.strip()
+    warning = (
+        "KANDEV_CI_EXTERNAL_PERCENT must be an integer from 0 to 100, "
+        f"got {raw!r}"
+    )
+    if not value.isascii() or not value.isdigit():
+        return 0, [warning]
+    percentage = int(value)
+    if percentage > 100:
+        return 0, [warning]
+    return percentage, []
+
+
+def _validate_families(raw: object) -> tuple[Family, ...]:
+    if not isinstance(raw, list) or not raw:
+        raise ValueError("families must be a non-empty JSON array")
+
+    families: list[Family] = []
+    names: set[str] = set()
+    for entry in raw:
+        if not isinstance(entry, dict):
+            raise ValueError("each family must be a JSON object")
+        name = entry.get("name")
+        tier = entry.get("tier")
+        if not isinstance(name, str) or FAMILY_NAME.fullmatch(name) is None:
+            raise ValueError(f"invalid family name: {name!r}")
+        if name in names:
+            raise ValueError(f"duplicate family name: {name}")
+        if tier not in VALID_TIERS:
+            raise ValueError(f"family {name!r} has invalid tier: {tier!r}")
+
+        is_matrix = "instances" in entry
+        instances_value = entry.get("instances", [{}])
+        if not isinstance(instances_value, list) or not instances_value:
+            raise ValueError(f"family {name!r} instances must be a non-empty array")
+        instances: list[dict[str, object]] = []
+        for instance in instances_value:
+            if not isinstance(instance, dict):
+                raise ValueError(f"family {name!r} instances must be JSON objects")
+            if "runner" in instance:
+                raise ValueError(f"family {name!r} instance cannot define runner")
+            instances.append(instance)
+
+        names.add(name)
+        families.append(
+            Family(
+                name=name,
+                tier=tier,
+                instances=tuple(instances),
+                is_matrix=is_matrix,
+            )
+        )
+    return tuple(families)
+
+
+def _external_for_singleton(
+    *, workflow: str, run_id: str, family: str, percentage: int
+) -> bool:
+    if percentage <= 0:
+        return False
+    if percentage >= 100:
+        return True
+    key = f"{workflow}\0{run_id}\0{family}".encode("utf-8")
+    bucket = int.from_bytes(hashlib.sha256(key).digest()[:8], "big") % 100
+    return bucket < percentage
+
+
+def _matrix_assignments(
+    *,
+    workflow: str,
+    run_id: str,
+    family: str,
+    instances: tuple[dict[str, object], ...],
+    percentage: int,
+) -> set[int]:
+    external_count = (len(instances) * percentage) // 100
+    ranked: list[tuple[bytes, int]] = []
+    for index, instance in enumerate(instances):
+        matrix_key = json.dumps(instance, sort_keys=True, separators=(",", ":"))
+        digest = hashlib.sha256(
+            f"{workflow}\0{run_id}\0{family}\0{matrix_key}".encode("utf-8")
+        ).digest()
+        ranked.append((digest, index))
+    return {index for _, index in sorted(ranked)[:external_count]}
+
+
+def _plan_family(
+    *,
+    workflow: str,
+    run_id: str,
+    family: Family,
+    burst: str,
+    percentage: int,
+    external_label: str,
+) -> list[dict[str, object]]:
+    if burst != "true" or not external_label or percentage == 0:
+        external_indices: set[int] = set()
+    elif not family.is_matrix:
+        external_indices = (
+            {0}
+            if _external_for_singleton(
+                workflow=workflow,
+                run_id=run_id,
+                family=family.name,
+                percentage=percentage,
+            )
+            else set()
+        )
+    else:
+        external_indices = _matrix_assignments(
+            workflow=workflow,
+            run_id=run_id,
+            family=family.name,
+            instances=family.instances,
+            percentage=percentage,
+        )
+
+    return [
+        {
+            **instance,
+            "runner": external_label if index in external_indices else GITHUB_RUNNER,
+        }
+        for index, instance in enumerate(family.instances)
+    ]
+
+
+def build_plan(
+    *,
+    workflow: str,
+    run_id: str,
+    families: object,
+    burst: str,
+    percent: str | None,
+    light_label: str,
+    standard_label: str,
+) -> RunnerPlan:
+    percentage, warnings = _parse_percentage(percent)
+    labels = {"light": light_label, "standard": standard_label}
+    plan: dict[str, object] = {}
+    for family in _validate_families(families):
+        planned = _plan_family(
+            workflow=workflow,
+            run_id=run_id,
+            family=family,
+            burst=burst,
+            percentage=percentage,
+            external_label=labels[family.tier],
+        )
+        if family.is_matrix:
+            plan[f"{family.name}_matrix"] = {"include": planned}
+        else:
+            plan[f"{family.name}_runner"] = planned[0]["runner"]
+    return RunnerPlan(plan=plan, warnings=warnings)
+
+
+def _write_output(plan: RunnerPlan, output_path: Path) -> None:
+    value = json.dumps(plan.plan, separators=(",", ":"), sort_keys=True)
+    with output_path.open("a", encoding="utf-8") as output:
+        output.write(f"plan={value}\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--workflow", required=True)
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--families", required=True)
+    parser.add_argument("--burst", default="")
+    parser.add_argument("--percent")
+    parser.add_argument("--light-label", default="")
+    parser.add_argument("--standard-label", default="")
+    parser.add_argument("--output", type=Path, default=Path("/dev/stdout"))
+    args = parser.parse_args()
+    try:
+        families = json.loads(args.families)
+        plan = build_plan(
+            workflow=args.workflow,
+            run_id=args.run_id,
+            families=families,
+            burst=args.burst,
+            percent=args.percent,
+            light_label=args.light_label,
+            standard_label=args.standard_label,
+        )
+    except (json.JSONDecodeError, ValueError) as error:
+        parser.error(str(error))
+    for warning in plan.warnings:
+        print(f"::warning::{warning}")
+    _write_output(plan, args.output)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/scripts/runner-plan_test.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+"""Tests for deterministic external CI runner allocation."""
+
+import importlib.util
+from pathlib import Path
+import unittest
+
+
+SCRIPT = Path(__file__).with_name("runner-plan.py")
+SPEC = importlib.util.spec_from_file_location("runner_plan", SCRIPT)
+assert SPEC is not None and SPEC.loader is not None
+runner_plan = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(runner_plan)
+
+E2E_FAMILIES = [
+    {"name": "changes", "tier": "light"},
+    {
+        "name": "e2e",
+        "tier": "standard",
+        "instances": [{"shard": shard} for shard in range(1, 15)],
+    },
+    {"name": "e2e_gate", "tier": "light"},
+]
+FRONTEND_FAMILIES = [
+    {"name": "frontend", "tier": "standard"},
+    {"name": "frontend_gate", "tier": "light"},
+]
+BACKEND_MATRIX_FAMILY = [
+    {
+        "name": "backend_test",
+        "tier": "standard",
+        "instances": [{"shard": 1, "index": 0}, {"shard": 2, "index": 1}],
+    }
+]
+
+
+class RunnerPlanTest(unittest.TestCase):
+    def test_missing_percentage_defaults_to_github(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="e2e",
+            run_id="100",
+            families=E2E_FAMILIES,
+            burst="true",
+            percent=None,
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest")
+        self.assertFalse(plan.warnings)
+
+    def test_burst_off_ignores_percentage(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="frontend",
+            run_id="101",
+            families=FRONTEND_FAMILIES,
+            burst="false",
+            percent="100",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest")
+        self.assertEqual(plan.plan["frontend_gate_runner"], "ubuntu-latest")
+
+    def test_matrix_percentage_uses_floor_and_is_stable(self) -> None:
+        first = runner_plan.build_plan(
+            workflow="e2e",
+            run_id="102",
+            families=E2E_FAMILIES,
+            burst="true",
+            percent="50",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+        second = runner_plan.build_plan(
+            workflow="e2e",
+            run_id="102",
+            families=E2E_FAMILIES,
+            burst="true",
+            percent="50",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        first_matrix = first.plan["e2e_matrix"]["include"]
+        second_matrix = second.plan["e2e_matrix"]["include"]
+        self.assertEqual(first_matrix, second_matrix)
+        self.assertEqual(
+            sum(item["runner"] == "standard-runner" for item in first_matrix), 7
+        )
+
+    def test_backend_matrix_percentage_uses_floor(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="backend",
+            run_id="103",
+            families=BACKEND_MATRIX_FAMILY,
+            burst="true",
+            percent="50",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        matrix = plan.plan["backend_test_matrix"]["include"]
+        self.assertEqual(
+            sum(item["runner"] == "standard-runner" for item in matrix), 1
+        )
+        self.assertEqual(
+            [(item["shard"], item["index"]) for item in matrix],
+            [(1, 0), (2, 1)],
+        )
+
+    def test_workflow_owned_family_names_need_no_python_change(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="new-workflow",
+            run_id="109",
+            families=[{"name": "new_job", "tier": "standard"}],
+            burst="true",
+            percent="100",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        self.assertEqual(plan.plan, {"new_job_runner": "standard-runner"})
+
+    def test_invalid_family_definition_is_rejected(self) -> None:
+        with self.assertRaisesRegex(ValueError, "duplicate family name"):
+            runner_plan.build_plan(
+                workflow="frontend",
+                run_id="110",
+                families=[
+                    {"name": "same", "tier": "light"},
+                    {"name": "same", "tier": "standard"},
+                ],
+                burst="true",
+                percent="50",
+                light_label="light-runner",
+                standard_label="standard-runner",
+            )
+
+    def test_singleton_allocation_is_stable_for_reruns(self) -> None:
+        first = runner_plan.build_plan(
+            workflow="frontend",
+            run_id="104",
+            families=FRONTEND_FAMILIES,
+            burst="true",
+            percent="50",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+        second = runner_plan.build_plan(
+            workflow="frontend",
+            run_id="104",
+            families=FRONTEND_FAMILIES,
+            burst="true",
+            percent="50",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+        self.assertEqual(first.plan, second.plan)
+
+    def test_empty_tier_label_falls_back_to_github(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="frontend",
+            run_id="105",
+            families=FRONTEND_FAMILIES,
+            burst="true",
+            percent="100",
+            light_label="",
+            standard_label="",
+        )
+
+        self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest")
+
+    def test_invalid_percentage_fails_closed_with_warning(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="e2e",
+            run_id="106",
+            families=E2E_FAMILIES,
+            burst="true",
+            percent="101",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        self.assertTrue(plan.warnings)
+        self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest")
+        matrix = plan.plan["e2e_matrix"]["include"]
+        self.assertTrue(all(item["runner"] == "ubuntu-latest" for item in matrix))
+
+    def test_unicode_digits_fail_closed_with_warning(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="e2e",
+            run_id="108",
+            families=E2E_FAMILIES,
+            burst="true",
+            percent="²",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        self.assertTrue(plan.warnings)
+        self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest")
+        matrix = plan.plan["e2e_matrix"]["include"]
+        self.assertTrue(all(item["runner"] == "ubuntu-latest" for item in matrix))
+
+    def test_full_percentage_uses_external_for_non_empty_tier(self) -> None:
+        plan = runner_plan.build_plan(
+            workflow="frontend",
+            run_id="107",
+            families=FRONTEND_FAMILIES,
+            burst="true",
+            percent="100",
+            light_label="light-runner",
+            standard_label="standard-runner",
+        )
+
+        self.assertEqual(plan.plan["frontend_runner"], "standard-runner")
+        self.assertEqual(plan.plan["frontend_gate_runner"], "light-runner")
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/architecture-lint.yml

@@ -24,6 +24,8 @@ permissions:
 jobs:
   lint:
     name: Architecture boundaries do not regress
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
 
     steps:

.github/workflows/backend-tests.yml

@@ -33,11 +33,38 @@ permissions:
   packages: read
 
 jobs:
+  runner_plan:
+    name: Plan external runner allocation
+    runs-on: ubuntu-latest
+    outputs:
+      plan: ${{ steps.plan.outputs.plan }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          persist-credentials: false
+
+      - name: Plan runner allocation
+        id: plan
+        uses: ./.github/actions/plan-external-runners
+        with:
+          workflow: backend
+          run-id: ${{ github.run_id }}
+          families: >-
+            [{"name":"test","tier":"light"}]
+          burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}
+          percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}
+          light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}
+          standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}
+
   # The former trigger-level `paths:` list, applied one level down so a skip
   # still produces a conclusion. `.github/scripts/changed-paths.py` implements
   # the same last-match-wins semantics GitHub applies to `paths:`.
   changes:
     name: Detect relevant changes
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
     timeout-minutes: 10
     outputs:
@@ -106,6 +133,8 @@ jobs:
     name: Backend Static Checks
     needs: changes
     if: needs.changes.outputs.run == 'true'
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
     # Pre-baked image bundles Go, golangci-lint, go-licenses, Docker CLI, Node,
     # pnpm, and the Playwright base — see .github/docker/ci-base/Dockerfile.
@@ -226,6 +255,8 @@ jobs:
     name: Backend Tests (${{ matrix.shard }}/2)
     needs: changes
     if: needs.changes.outputs.run == 'true'
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -334,11 +365,13 @@ jobs:
         if: always()
         shell: bash
         working-directory: ${{ github.workspace }}
+        env:
+          BACKEND_TEST_SHARD: ${{ matrix.shard }}
         run: |
-          report_summary="$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md"
+          report_summary="$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md"
           : > "$report_summary"
           GITHUB_STEP_SUMMARY="$report_summary" \
-          INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/test-results-${{ matrix.shard }}.json" \
+          INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/test-results-${BACKEND_TEST_SHARD}.json" \
           INPUT_MODULEDIRECTORY="$GITHUB_WORKSPACE/apps/backend" \
           INPUT_OMIT="successful" \
             node ".github/vendor/go-test-action/dist/index.js"
@@ -347,12 +380,14 @@ jobs:
         if: always()
         shell: bash
         working-directory: ${{ github.workspace }}
+        env:
+          BACKEND_TEST_SHARD: ${{ matrix.shard }}
         run: |
           python3 .github/scripts/bounded-step-summary.py \
-            --input "$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md" \
+            --input "$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md" \
             --output "$GITHUB_STEP_SUMMARY" \
-            --diagnostics-label "backend-test-results-${{ matrix.shard }}" \
-            --diagnostics-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts"
+            --diagnostics-label "backend-test-results-${BACKEND_TEST_SHARD}" \
+            --diagnostics-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#artifacts"
 
       - name: Upload test artifacts
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -368,6 +403,8 @@ jobs:
     name: Backend (ambient integration env)
     needs: changes
     if: needs.changes.outputs.run == 'true'
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
     # test_shards runs in the same clean container and never sets these vars,
     # so it cannot catch a package that starts reading them again — the class
@@ -430,9 +467,10 @@ jobs:
   # `test-windows` were outside it before this workflow became a gate.
   test:
     name: Run Backend Tests
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).test_runner }}
     needs:
       - changes
+      - runner_plan
       - static_checks
       - test_shards
       - test_ambient_env
@@ -442,6 +480,7 @@ jobs:
     steps:
       - name: Require backend checks to pass
         env:
+          RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }}
           CHANGES_RESULT: ${{ needs.changes.result }}
           STATIC_CHECKS_RESULT: ${{ needs.static_checks.result }}
           TEST_SHARDS_RESULT: ${{ needs.test_shards.result }}
@@ -451,6 +490,7 @@ jobs:
         run: |
           set -euo pipefail
 
+          echo "Runner plan: ${RUNNER_PLAN_RESULT}"
           echo "Change detection: ${CHANGES_RESULT}"
           echo "Static checks: ${STATIC_CHECKS_RESULT}"
           echo "Test shards: ${TEST_SHARDS_RESULT}"
@@ -458,6 +498,13 @@ jobs:
           echo "Postgres boot: ${POSTGRES_BOOT_RESULT}"
           echo "Windows tests: ${TEST_WINDOWS_RESULT}"
 
+          # Runner planning must succeed because its outputs select the runner
+          # fleet. A planner error cannot silently become a hosted fallback.
+          if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then
+            echo "Runner planning did not succeed; refusing to report a pass." >&2
+            exit 1
+          fi
+
           # Change detection itself must succeed: if it errored we do not know
           # whether the skips below were deliberate.
           if [[ "${CHANGES_RESULT}" != "success" ]]; then

.github/workflows/e2e-tests.yml

@@ -39,12 +39,47 @@ permissions:
   packages: read
 
 jobs:
+  runner_plan:
+    name: Plan external runner allocation
+    runs-on: ubuntu-latest
+    outputs:
+      plan: ${{ steps.plan.outputs.plan }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          persist-credentials: false
+
+      - name: Plan runner allocation
+        id: plan
+        uses: ./.github/actions/plan-external-runners
+        with:
+          workflow: e2e
+          run-id: ${{ github.run_id }}
+          families: >-
+            [
+              {"name":"changes","tier":"light"},
+              {"name":"e2e","tier":"standard","instances":[
+                {"shard":1},{"shard":2},{"shard":3},{"shard":4},
+                {"shard":5},{"shard":6},{"shard":7},{"shard":8},
+                {"shard":9},{"shard":10},{"shard":11},{"shard":12},
+                {"shard":13},{"shard":14}
+              ]},
+              {"name":"e2e_gate","tier":"light"}
+            ]
+          burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}
+          percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}
+          light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}
+          standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}
+
   # The former trigger-level `paths:` list, applied one level down so a skip
   # still produces a conclusion. `.github/scripts/changed-paths.py` implements
   # the same last-match-wins semantics GitHub applies to `paths:`.
   changes:
     name: Detect relevant changes
-    runs-on: ubuntu-latest
+    needs: runner_plan
+    runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}
     timeout-minutes: 10
     outputs:
       run: ${{ steps.detect.outputs.run }}
@@ -113,6 +148,8 @@ jobs:
     name: Build
     needs: changes
     if: needs.changes.outputs.run == 'true'
+    # Build reads the GitHub API and downloads a timing artifact with the job token.
+    # Keep this credential-bearing path on the GitHub-hosted boundary.
     runs-on: ubuntu-latest
     # Pre-baked image bundles Go, Node 24, pnpm 9.15.9 — see
     # .github/docker/ci-base/Dockerfile. Uses the `build` tag (with Go)
@@ -367,9 +404,9 @@ jobs:
 
   e2e:
     name: E2E Shard ${{ matrix.shard }}/14
-    needs: [changes, build]
+    needs: [changes, build, runner_plan]
     if: needs.changes.outputs.run == 'true'
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     # Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright browser
     # binaries (Chromium etc.) — see .github/docker/ci-base/Dockerfile. This is
     # the biggest speedup since `npx playwright install chromium --with-deps`
@@ -398,8 +435,7 @@ jobs:
       NODE_OPTIONS: --dns-result-order=ipv4first
     strategy:
       fail-fast: false
-      matrix:
-        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+      matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}
 
     steps:
       - name: Checkout code
@@ -467,9 +503,10 @@ jobs:
         working-directory: apps/web
         env:
           E2E_SHARD: ${{ matrix.shard }}
+          E2E_MANIFEST_SHARD: ${{ matrix.shard }}
           KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json
           KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }}
-        run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${{ matrix.shard }}.json"
+        run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${E2E_MANIFEST_SHARD}.json"
 
       - name: Upload blob report
         if: always()
@@ -1104,6 +1141,8 @@ jobs:
     # reports to download, and the artifact step would fail on an empty run.
     # `e2e-gate` below, not this job, is what the merge queue waits on.
     if: always() && needs.changes.outputs.run == 'true'
+    # Report aggregation queries GitHub history and downloads artifacts with the
+    # job token, so it remains on a GitHub-hosted runner during the pilot.
     runs-on: ubuntu-latest
     # Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright CLI.
     # `runtime` tag — no Go needed for merging blob reports.
@@ -1473,10 +1512,11 @@ jobs:
   # count changes.
   e2e-gate:
     name: E2E Tests Passed
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }}
     needs:
       [
         changes,
+        runner_plan,
         build,
         playwright_image,
         e2e,
@@ -1490,6 +1530,7 @@ jobs:
     steps:
       - name: Require the E2E jobs to have passed or been skipped
         env:
+          RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }}
           CHANGES_RESULT: ${{ needs.changes.result }}
           BUILD_RESULT: ${{ needs.build.result }}
           PLAYWRIGHT_IMAGE_RESULT: ${{ needs.playwright_image.result }}
@@ -1502,6 +1543,7 @@ jobs:
           set -euo pipefail
 
           echo "Change detection: ${CHANGES_RESULT}"
+          echo "Runner plan: ${RUNNER_PLAN_RESULT}"
           echo "Build: ${BUILD_RESULT}"
           echo "Playwright image: ${PLAYWRIGHT_IMAGE_RESULT}"
           echo "E2E shards: ${E2E_RESULT}"
@@ -1510,6 +1552,13 @@ jobs:
           echo "Desktop smoke: ${DESKTOP_E2E_RESULT}"
           echo "Report merge: ${E2E_REPORT_RESULT}"
 
+          # Runner planning must succeed because its outputs select the runner
+          # fleet. A planner error cannot silently become a hosted fallback.
+          if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then
+            echo "Runner planning did not succeed; refusing to report a pass." >&2
+            exit 1
+          fi
+
           # Change detection itself must succeed: if it errored we do not know
           # whether the skips below were deliberate.
           if [[ "${CHANGES_RESULT}" != "success" ]]; then

.github/workflows/frontend-tests.yml

@@ -29,12 +29,42 @@ permissions:
   packages: read
 
 jobs:
+  runner_plan:
+    name: Plan external runner allocation
+    runs-on: ubuntu-latest
+    outputs:
+      plan: ${{ steps.plan.outputs.plan }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+        with:
+          persist-credentials: false
+
+      - name: Plan runner allocation
+        id: plan
+        uses: ./.github/actions/plan-external-runners
+        with:
+          workflow: frontend
+          run-id: ${{ github.run_id }}
+          families: >-
+            [
+              {"name":"changes","tier":"light"},
+              {"name":"frontend","tier":"standard"},
+              {"name":"frontend_gate","tier":"light"}
+            ]
+          burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}
+          percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}
+          light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}
+          standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}
+
   # The former trigger-level `paths:` list, applied one level down so a skip
   # still produces a conclusion. `.github/scripts/changed-paths.py` implements
   # the same last-match-wins semantics GitHub applies to `paths:`.
   changes:
     name: Detect relevant changes
-    runs-on: ubuntu-latest
+    needs: runner_plan
+    runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}
     timeout-minutes: 10
     outputs:
       run: ${{ steps.detect.outputs.run }}
@@ -103,9 +133,9 @@ jobs:
 
   frontend:
     name: Run Frontend Lint, Tests, and Build
-    needs: changes
+    needs: [changes, runner_plan]
     if: needs.changes.outputs.run == 'true'
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}
     # Pre-baked image bundles Node 24, pnpm 9.15.9, Go, and go-licenses —
     # see .github/docker/ci-base/Dockerfile.
     container: ghcr.io/kdlbs/kandev-ci:build-latest
@@ -121,6 +151,7 @@ jobs:
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - name: Mark workspace safe for git
         shell: bash
@@ -353,22 +384,28 @@ jobs:
   # conclusion to read instead of a check that stays pending forever.
   frontend-gate:
     name: Frontend Tests Passed
-    runs-on: ubuntu-latest
-    needs: [changes, frontend]
+    runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}
+    needs: [changes, frontend, runner_plan]
     if: always()
 
     steps:
       - name: Require the frontend job to have passed or been skipped
         env:
+          RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }}
           CHANGES_RESULT: ${{ needs.changes.result }}
           FRONTEND_RESULT: ${{ needs.frontend.result }}
         run: |
           set -euo pipefail
 
+          echo "Runner plan: ${RUNNER_PLAN_RESULT}"
           # `skipped` is a pass: the change-detection job decided nothing this
           # workflow covers was touched. `cancelled` is not — a cancelled job
           # never reached a verdict, and treating that as a pass would let an
           # untested merge group through.
+          if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then
+            echo "Runner planning finished with result: ${RUNNER_PLAN_RESULT}" >&2
+            exit 1
+          fi
           if [[ "${CHANGES_RESULT}" != "success" ]]; then
             echo "Change detection finished with result: ${CHANGES_RESULT}" >&2
             exit 1

.github/workflows/lint-action-pinning.yml

@@ -34,6 +34,8 @@ permissions:
 jobs:
   lint:
     name: All action refs are SHA-pinned
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
 
     steps:
@@ -73,6 +75,12 @@ jobs:
       - name: Test E2E workflow contract
         run: python3 .github/scripts/e2e-tests-workflow-contract_test.py
 
+      - name: Test external runner workflow contract
+        run: python3 .github/scripts/external-runner-workflow-contract_test.py
+
+      - name: Test external runner planner
+        run: python3 .github/scripts/runner-plan_test.py
+
       - name: Test bounded step summary
         run: python3 .github/scripts/bounded-step-summary_test.py
 

.github/workflows/lint-harness-files.yml

@@ -33,6 +33,8 @@ permissions:
 jobs:
   lint:
     name: Harness files pass lint
+    # Checkout receives the short-lived job token. Keep this credential-bearing
+    # path on GitHub-hosted capacity during the external-runner pilot.
     runs-on: ubuntu-latest
 
     steps: