ci: add selectable external CI runners (#3451)
This commit is contained in:
Carlos Florêncio
@@ -0,0 +1,59 @@
|
||||
name: Plan external runner allocation
|
||||
description: Compute deterministic external-runner assignments for a workflow
|
||||
|
||||
inputs:
|
||||
workflow:
|
||||
description: Runner-plan workflow family key
|
||||
required: true
|
||||
run-id:
|
||||
description: Stable workflow run identifier used for allocation
|
||||
required: true
|
||||
families:
|
||||
description: JSON job-family definitions owned by the calling workflow
|
||||
required: true
|
||||
burst:
|
||||
description: Exact true value enables external allocation
|
||||
required: false
|
||||
default: ""
|
||||
percent:
|
||||
description: External allocation percentage from 0 to 100
|
||||
required: false
|
||||
default: ""
|
||||
light-label:
|
||||
description: Configured light-tier runner label
|
||||
required: false
|
||||
default: ""
|
||||
standard-label:
|
||||
description: Configured standard-tier runner label
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
outputs:
|
||||
plan:
|
||||
description: JSON object containing runner and matrix outputs
|
||||
value: ${{ steps.plan.outputs.plan }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Plan runner allocation
|
||||
id: plan
|
||||
shell: bash
|
||||
env:
|
||||
RUNNER_PLAN_WORKFLOW: ${{ inputs.workflow }}
|
||||
RUNNER_PLAN_RUN_ID: ${{ inputs.run-id }}
|
||||
RUNNER_PLAN_FAMILIES: ${{ inputs.families }}
|
||||
KANDEV_CI_EXTERNAL_ENABLED: ${{ inputs.burst }}
|
||||
KANDEV_CI_EXTERNAL_PERCENT: ${{ inputs.percent }}
|
||||
KANDEV_CI_RUNNER_LIGHT: ${{ inputs.light-label }}
|
||||
KANDEV_CI_RUNNER_STANDARD: ${{ inputs.standard-label }}
|
||||
run: |
|
||||
python3 "$GITHUB_WORKSPACE/.github/scripts/runner-plan.py" \
|
||||
--workflow "$RUNNER_PLAN_WORKFLOW" \
|
||||
--run-id "$RUNNER_PLAN_RUN_ID" \
|
||||
--families "$RUNNER_PLAN_FAMILIES" \
|
||||
--burst "$KANDEV_CI_EXTERNAL_ENABLED" \
|
||||
--percent "$KANDEV_CI_EXTERNAL_PERCENT" \
|
||||
--light-label "$KANDEV_CI_RUNNER_LIGHT" \
|
||||
--standard-label "$KANDEV_CI_RUNNER_STANDARD" \
|
||||
--output "$GITHUB_OUTPUT"
|
||||
@@ -25,7 +25,7 @@ class BackendTestsWorkflowContractTest(unittest.TestCase):
|
||||
generate = step_block(workflow, "Generate test report")
|
||||
publish = step_block(workflow, "Publish bounded test report summary")
|
||||
|
||||
temporary_summary = "$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md"
|
||||
temporary_summary = "$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md"
|
||||
self.assertIn(
|
||||
"uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10",
|
||||
checkout,
|
||||
@@ -37,12 +37,13 @@ class BackendTestsWorkflowContractTest(unittest.TestCase):
|
||||
)
|
||||
self.assertIn("path: .github/vendor/go-test-action", checkout)
|
||||
self.assertIn(f'report_summary="{temporary_summary}"', generate)
|
||||
self.assertIn("BACKEND_TEST_SHARD: ${{ matrix.shard }}", generate)
|
||||
self.assertNotIn("${{ runner.temp }}", generate)
|
||||
self.assertIn(': > "$report_summary"', generate)
|
||||
self.assertIn('GITHUB_STEP_SUMMARY="$report_summary"', generate)
|
||||
self.assertIn(
|
||||
'INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/'
|
||||
'test-results-${{ matrix.shard }}.json"',
|
||||
'test-results-${BACKEND_TEST_SHARD}.json"',
|
||||
generate,
|
||||
)
|
||||
self.assertIn(
|
||||
@@ -60,8 +61,8 @@ class BackendTestsWorkflowContractTest(unittest.TestCase):
|
||||
self.assertIn(f'--input "{temporary_summary}"', publish)
|
||||
self.assertIn('--output "$GITHUB_STEP_SUMMARY"', publish)
|
||||
self.assertIn("--diagnostics-label", publish)
|
||||
self.assertIn("backend-test-results-${{ matrix.shard }}", publish)
|
||||
self.assertIn("${{ github.run_id }}#artifacts", publish)
|
||||
self.assertIn("backend-test-results-${BACKEND_TEST_SHARD}", publish)
|
||||
self.assertIn("$GITHUB_RUN_ID#artifacts", publish)
|
||||
|
||||
def test_full_json_diagnostics_remain_in_the_named_artifact(self) -> None:
|
||||
workflow = BACKEND_WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
@@ -193,6 +193,58 @@ cat "${FAKE_DOCKER_MANIFEST}"
|
||||
self.assertIn("timeout-minutes: 35", normal_job)
|
||||
self.assertNotIn("timeout-minutes: 25", normal_job)
|
||||
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1
|
||||
# @covers AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2
|
||||
def test_external_runner_tiers_are_toggleable_for_eligible_jobs(self) -> None:
|
||||
workflow = E2E_WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
light_jobs = {
|
||||
"changes": ("build", "changes_runner"),
|
||||
"e2e-gate": (None, "e2e_gate_runner"),
|
||||
}
|
||||
|
||||
for job, (next_job, output_name) in light_jobs.items():
|
||||
if next_job is None:
|
||||
job_text = workflow.partition(" e2e-gate:\n")[2]
|
||||
else:
|
||||
job_text = job_block(workflow, job, next_job)
|
||||
expected = (
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan)."
|
||||
f"{output_name} }}}}"
|
||||
)
|
||||
self.assertIn(
|
||||
expected,
|
||||
job_text,
|
||||
f"{job} must select its configured tier",
|
||||
)
|
||||
e2e_job = job_block(workflow, "e2e", "playwright_image")
|
||||
self.assertIn("runs-on: ${{ matrix.runner }}", e2e_job)
|
||||
self.assertIn(
|
||||
"matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}",
|
||||
e2e_job,
|
||||
)
|
||||
|
||||
protected_jobs = {
|
||||
"build": "e2e",
|
||||
"e2e-report": "e2e-gate",
|
||||
"playwright_image": "e2e-containers",
|
||||
"e2e-containers": "e2e-kubernetes-compatibility",
|
||||
"e2e-kubernetes-compatibility": "desktop-e2e",
|
||||
"desktop-e2e": "e2e-report",
|
||||
}
|
||||
for job, next_job in protected_jobs.items():
|
||||
protected_text = job_block(workflow, job, next_job)
|
||||
self.assertIn("runs-on: ubuntu-latest", protected_text)
|
||||
self.assertNotIn("runner_plan", protected_text)
|
||||
self.assertNotIn("KANDEV_CI_EXTERNAL", protected_text)
|
||||
self.assertNotIn("KANDEV_CI_RUNNER_", protected_text)
|
||||
|
||||
def test_contract_runs_in_the_unfiltered_required_workflow(self) -> None:
|
||||
workflow = LINT_WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Contract tests for external runner planner wiring and protected jobs."""
|
||||
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW_ROOT = REPO_ROOT / ".github" / "workflows"
|
||||
PLAN_SCRIPT = REPO_ROOT / ".github" / "scripts" / "runner-plan.py"
|
||||
|
||||
|
||||
def job_block(workflow: str, job: str, next_job: str | None) -> str:
|
||||
marker = f" {job}:\n"
|
||||
_, separator, remainder = workflow.partition(marker)
|
||||
if not separator:
|
||||
raise AssertionError(f"Workflow has no {job} job")
|
||||
if next_job is None:
|
||||
return remainder
|
||||
return remainder.partition(f"\n {next_job}:\n")[0]
|
||||
|
||||
|
||||
class ExternalRunnerWorkflowContractTest(unittest.TestCase):
|
||||
def assert_planner(
|
||||
self, workflow_name: str, workflow_key: str, families: tuple[str, ...]
|
||||
) -> str:
|
||||
workflow = (WORKFLOW_ROOT / workflow_name).read_text(encoding="utf-8")
|
||||
planner = job_block(workflow, "runner_plan", "changes")
|
||||
self.assertIn("runs-on: ubuntu-latest", planner)
|
||||
self.assertIn("plan: ${{ steps.plan.outputs.plan }}", planner)
|
||||
self.assertIn("uses: ./.github/actions/plan-external-runners", planner)
|
||||
self.assertIn(f"workflow: {workflow_key}", planner)
|
||||
self.assertIn("families: >-", planner)
|
||||
self.assertIn("run-id: ${{ github.run_id }}", planner)
|
||||
self.assertIn("burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}", planner)
|
||||
self.assertIn("percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}", planner)
|
||||
self.assertIn("light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}", planner)
|
||||
self.assertIn("standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}", planner)
|
||||
for family in families:
|
||||
self.assertIn(f'"name":"{family}"', planner)
|
||||
return workflow
|
||||
|
||||
def test_e2e_uses_planner_for_eligible_jobs_and_keeps_protected_jobs_hosted(self) -> None:
|
||||
workflow = self.assert_planner(
|
||||
"e2e-tests.yml",
|
||||
"e2e",
|
||||
("changes", "e2e", "e2e_gate"),
|
||||
)
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}",
|
||||
job_block(workflow, "changes", "build"),
|
||||
)
|
||||
build = job_block(workflow, "build", "e2e")
|
||||
self.assertIn("runs-on: ubuntu-latest", build)
|
||||
self.assertNotIn("runner_plan", build)
|
||||
e2e = job_block(workflow, "e2e", "playwright_image")
|
||||
self.assertIn(
|
||||
"matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}",
|
||||
e2e,
|
||||
)
|
||||
self.assertIn("runs-on: ${{ matrix.runner }}", e2e)
|
||||
report = job_block(workflow, "e2e-report", "e2e-gate")
|
||||
self.assertIn("runs-on: ubuntu-latest", report)
|
||||
self.assertNotIn("runner_plan", report)
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }}",
|
||||
job_block(workflow, "e2e-gate", None),
|
||||
)
|
||||
for job, next_job in (
|
||||
("playwright_image", "e2e-containers"),
|
||||
("e2e-containers", "e2e-kubernetes-compatibility"),
|
||||
("e2e-kubernetes-compatibility", "desktop-e2e"),
|
||||
("desktop-e2e", "e2e-report"),
|
||||
):
|
||||
protected = job_block(workflow, job, next_job)
|
||||
self.assertIn("runs-on: ubuntu-latest", protected)
|
||||
self.assertNotIn("runner_plan", protected)
|
||||
self.assertNotIn("KANDEV_CI_EXTERNAL", protected)
|
||||
|
||||
def test_backend_uses_planner_and_keeps_services_and_windows_hosted(self) -> None:
|
||||
workflow = self.assert_planner(
|
||||
"backend-tests.yml",
|
||||
"backend",
|
||||
("test",),
|
||||
)
|
||||
for job, next_job in (
|
||||
("changes", "static_checks"),
|
||||
("static_checks", "test_shards"),
|
||||
("test_ambient_env", "test"),
|
||||
):
|
||||
protected = job_block(workflow, job, next_job)
|
||||
self.assertIn("runs-on: ubuntu-latest", protected)
|
||||
self.assertNotIn("runner_plan", protected)
|
||||
shards = job_block(workflow, "test_shards", "test_ambient_env")
|
||||
self.assertIn("- shard: 1", shards)
|
||||
self.assertIn("- shard: 2", shards)
|
||||
self.assertIn("runs-on: ubuntu-latest", shards)
|
||||
self.assertNotIn("runner_plan", shards)
|
||||
self.assertNotIn("matrix.runner", shards)
|
||||
test_gate = job_block(workflow, "test", "postgres-boot")
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).test_runner }}",
|
||||
test_gate,
|
||||
)
|
||||
for job, next_job in (("postgres-boot", "test-windows"), ("test-windows", None)):
|
||||
protected = job_block(workflow, job, next_job)
|
||||
self.assertIn("runs-on: ubuntu-latest" if job == "postgres-boot" else "runs-on: windows-latest", protected)
|
||||
self.assertNotIn("runner_plan", protected)
|
||||
|
||||
def test_frontend_uses_planner_for_test_and_gate(self) -> None:
|
||||
workflow = self.assert_planner(
|
||||
"frontend-tests.yml",
|
||||
"frontend",
|
||||
("changes", "frontend", "frontend_gate"),
|
||||
)
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}",
|
||||
job_block(workflow, "changes", "frontend"),
|
||||
)
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}",
|
||||
job_block(workflow, "frontend", "frontend-gate"),
|
||||
)
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}",
|
||||
job_block(workflow, "frontend-gate", None),
|
||||
)
|
||||
|
||||
def test_protected_lint_workflows_do_not_run_the_planner(self) -> None:
|
||||
for workflow_name in (
|
||||
"architecture-lint.yml",
|
||||
"lint-action-pinning.yml",
|
||||
"lint-harness-files.yml",
|
||||
):
|
||||
workflow = (WORKFLOW_ROOT / workflow_name).read_text(encoding="utf-8")
|
||||
lint = job_block(workflow, "lint", None)
|
||||
self.assertNotIn("runner_plan:", workflow)
|
||||
self.assertNotIn("name: Require runner plan", lint)
|
||||
self.assertIn("runs-on: ubuntu-latest", lint)
|
||||
|
||||
def test_plan_script_is_checked_by_required_action_pinning_workflow(self) -> None:
|
||||
workflow = (WORKFLOW_ROOT / "lint-action-pinning.yml").read_text(encoding="utf-8")
|
||||
self.assertTrue(PLAN_SCRIPT.is_file())
|
||||
action = (WORKFLOW_ROOT / ".." / "actions" / "plan-external-runners" / "action.yml").resolve()
|
||||
self.assertTrue(action.is_file())
|
||||
action_text = action.read_text(encoding="utf-8")
|
||||
self.assertIn("using: composite", action_text)
|
||||
self.assertIn("families:", action_text)
|
||||
self.assertIn("plan:", action_text)
|
||||
self.assertIn("runner-plan.py", action_text)
|
||||
self.assertNotIn("changes_runner:", action_text)
|
||||
self.assertIn("python3 .github/scripts/runner-plan_test.py", workflow)
|
||||
self.assertIn("python3 .github/scripts/external-runner-workflow-contract_test.py", workflow)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build deterministic runner assignments from workflow-owned family data."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunnerPlan:
|
||||
plan: dict[str, object]
|
||||
warnings: list[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Family:
|
||||
name: str
|
||||
tier: str
|
||||
instances: tuple[dict[str, object], ...]
|
||||
is_matrix: bool
|
||||
|
||||
|
||||
GITHUB_RUNNER = "ubuntu-latest"
|
||||
VALID_TIERS = frozenset({"light", "standard"})
|
||||
FAMILY_NAME = re.compile(r"[a-z][a-z0-9_]*\Z")
|
||||
|
||||
|
||||
def _parse_percentage(raw: str | None) -> tuple[int, list[str]]:
|
||||
if raw is None or raw.strip() == "":
|
||||
return 0, []
|
||||
value = raw.strip()
|
||||
warning = (
|
||||
"KANDEV_CI_EXTERNAL_PERCENT must be an integer from 0 to 100, "
|
||||
f"got {raw!r}"
|
||||
)
|
||||
if not value.isascii() or not value.isdigit():
|
||||
return 0, [warning]
|
||||
percentage = int(value)
|
||||
if percentage > 100:
|
||||
return 0, [warning]
|
||||
return percentage, []
|
||||
|
||||
|
||||
def _validate_families(raw: object) -> tuple[Family, ...]:
|
||||
if not isinstance(raw, list) or not raw:
|
||||
raise ValueError("families must be a non-empty JSON array")
|
||||
|
||||
families: list[Family] = []
|
||||
names: set[str] = set()
|
||||
for entry in raw:
|
||||
if not isinstance(entry, dict):
|
||||
raise ValueError("each family must be a JSON object")
|
||||
name = entry.get("name")
|
||||
tier = entry.get("tier")
|
||||
if not isinstance(name, str) or FAMILY_NAME.fullmatch(name) is None:
|
||||
raise ValueError(f"invalid family name: {name!r}")
|
||||
if name in names:
|
||||
raise ValueError(f"duplicate family name: {name}")
|
||||
if tier not in VALID_TIERS:
|
||||
raise ValueError(f"family {name!r} has invalid tier: {tier!r}")
|
||||
|
||||
is_matrix = "instances" in entry
|
||||
instances_value = entry.get("instances", [{}])
|
||||
if not isinstance(instances_value, list) or not instances_value:
|
||||
raise ValueError(f"family {name!r} instances must be a non-empty array")
|
||||
instances: list[dict[str, object]] = []
|
||||
for instance in instances_value:
|
||||
if not isinstance(instance, dict):
|
||||
raise ValueError(f"family {name!r} instances must be JSON objects")
|
||||
if "runner" in instance:
|
||||
raise ValueError(f"family {name!r} instance cannot define runner")
|
||||
instances.append(instance)
|
||||
|
||||
names.add(name)
|
||||
families.append(
|
||||
Family(
|
||||
name=name,
|
||||
tier=tier,
|
||||
instances=tuple(instances),
|
||||
is_matrix=is_matrix,
|
||||
)
|
||||
)
|
||||
return tuple(families)
|
||||
|
||||
|
||||
def _external_for_singleton(
|
||||
*, workflow: str, run_id: str, family: str, percentage: int
|
||||
) -> bool:
|
||||
if percentage <= 0:
|
||||
return False
|
||||
if percentage >= 100:
|
||||
return True
|
||||
key = f"{workflow}\0{run_id}\0{family}".encode("utf-8")
|
||||
bucket = int.from_bytes(hashlib.sha256(key).digest()[:8], "big") % 100
|
||||
return bucket < percentage
|
||||
|
||||
|
||||
def _matrix_assignments(
|
||||
*,
|
||||
workflow: str,
|
||||
run_id: str,
|
||||
family: str,
|
||||
instances: tuple[dict[str, object], ...],
|
||||
percentage: int,
|
||||
) -> set[int]:
|
||||
external_count = (len(instances) * percentage) // 100
|
||||
ranked: list[tuple[bytes, int]] = []
|
||||
for index, instance in enumerate(instances):
|
||||
matrix_key = json.dumps(instance, sort_keys=True, separators=(",", ":"))
|
||||
digest = hashlib.sha256(
|
||||
f"{workflow}\0{run_id}\0{family}\0{matrix_key}".encode("utf-8")
|
||||
).digest()
|
||||
ranked.append((digest, index))
|
||||
return {index for _, index in sorted(ranked)[:external_count]}
|
||||
|
||||
|
||||
def _plan_family(
|
||||
*,
|
||||
workflow: str,
|
||||
run_id: str,
|
||||
family: Family,
|
||||
burst: str,
|
||||
percentage: int,
|
||||
external_label: str,
|
||||
) -> list[dict[str, object]]:
|
||||
if burst != "true" or not external_label or percentage == 0:
|
||||
external_indices: set[int] = set()
|
||||
elif not family.is_matrix:
|
||||
external_indices = (
|
||||
{0}
|
||||
if _external_for_singleton(
|
||||
workflow=workflow,
|
||||
run_id=run_id,
|
||||
family=family.name,
|
||||
percentage=percentage,
|
||||
)
|
||||
else set()
|
||||
)
|
||||
else:
|
||||
external_indices = _matrix_assignments(
|
||||
workflow=workflow,
|
||||
run_id=run_id,
|
||||
family=family.name,
|
||||
instances=family.instances,
|
||||
percentage=percentage,
|
||||
)
|
||||
|
||||
return [
|
||||
{
|
||||
**instance,
|
||||
"runner": external_label if index in external_indices else GITHUB_RUNNER,
|
||||
}
|
||||
for index, instance in enumerate(family.instances)
|
||||
]
|
||||
|
||||
|
||||
def build_plan(
|
||||
*,
|
||||
workflow: str,
|
||||
run_id: str,
|
||||
families: object,
|
||||
burst: str,
|
||||
percent: str | None,
|
||||
light_label: str,
|
||||
standard_label: str,
|
||||
) -> RunnerPlan:
|
||||
percentage, warnings = _parse_percentage(percent)
|
||||
labels = {"light": light_label, "standard": standard_label}
|
||||
plan: dict[str, object] = {}
|
||||
for family in _validate_families(families):
|
||||
planned = _plan_family(
|
||||
workflow=workflow,
|
||||
run_id=run_id,
|
||||
family=family,
|
||||
burst=burst,
|
||||
percentage=percentage,
|
||||
external_label=labels[family.tier],
|
||||
)
|
||||
if family.is_matrix:
|
||||
plan[f"{family.name}_matrix"] = {"include": planned}
|
||||
else:
|
||||
plan[f"{family.name}_runner"] = planned[0]["runner"]
|
||||
return RunnerPlan(plan=plan, warnings=warnings)
|
||||
|
||||
|
||||
def _write_output(plan: RunnerPlan, output_path: Path) -> None:
|
||||
value = json.dumps(plan.plan, separators=(",", ":"), sort_keys=True)
|
||||
with output_path.open("a", encoding="utf-8") as output:
|
||||
output.write(f"plan={value}\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--workflow", required=True)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--families", required=True)
|
||||
parser.add_argument("--burst", default="")
|
||||
parser.add_argument("--percent")
|
||||
parser.add_argument("--light-label", default="")
|
||||
parser.add_argument("--standard-label", default="")
|
||||
parser.add_argument("--output", type=Path, default=Path("/dev/stdout"))
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
families = json.loads(args.families)
|
||||
plan = build_plan(
|
||||
workflow=args.workflow,
|
||||
run_id=args.run_id,
|
||||
families=families,
|
||||
burst=args.burst,
|
||||
percent=args.percent,
|
||||
light_label=args.light_label,
|
||||
standard_label=args.standard_label,
|
||||
)
|
||||
except (json.JSONDecodeError, ValueError) as error:
|
||||
parser.error(str(error))
|
||||
for warning in plan.warnings:
|
||||
print(f"::warning::{warning}")
|
||||
_write_output(plan, args.output)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for deterministic external CI runner allocation."""
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("runner-plan.py")
|
||||
SPEC = importlib.util.spec_from_file_location("runner_plan", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
runner_plan = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(runner_plan)
|
||||
|
||||
E2E_FAMILIES = [
|
||||
{"name": "changes", "tier": "light"},
|
||||
{
|
||||
"name": "e2e",
|
||||
"tier": "standard",
|
||||
"instances": [{"shard": shard} for shard in range(1, 15)],
|
||||
},
|
||||
{"name": "e2e_gate", "tier": "light"},
|
||||
]
|
||||
FRONTEND_FAMILIES = [
|
||||
{"name": "frontend", "tier": "standard"},
|
||||
{"name": "frontend_gate", "tier": "light"},
|
||||
]
|
||||
BACKEND_MATRIX_FAMILY = [
|
||||
{
|
||||
"name": "backend_test",
|
||||
"tier": "standard",
|
||||
"instances": [{"shard": 1, "index": 0}, {"shard": 2, "index": 1}],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
class RunnerPlanTest(unittest.TestCase):
|
||||
def test_missing_percentage_defaults_to_github(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="e2e",
|
||||
run_id="100",
|
||||
families=E2E_FAMILIES,
|
||||
burst="true",
|
||||
percent=None,
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest")
|
||||
self.assertFalse(plan.warnings)
|
||||
|
||||
def test_burst_off_ignores_percentage(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="frontend",
|
||||
run_id="101",
|
||||
families=FRONTEND_FAMILIES,
|
||||
burst="false",
|
||||
percent="100",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest")
|
||||
self.assertEqual(plan.plan["frontend_gate_runner"], "ubuntu-latest")
|
||||
|
||||
def test_matrix_percentage_uses_floor_and_is_stable(self) -> None:
|
||||
first = runner_plan.build_plan(
|
||||
workflow="e2e",
|
||||
run_id="102",
|
||||
families=E2E_FAMILIES,
|
||||
burst="true",
|
||||
percent="50",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
second = runner_plan.build_plan(
|
||||
workflow="e2e",
|
||||
run_id="102",
|
||||
families=E2E_FAMILIES,
|
||||
burst="true",
|
||||
percent="50",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
first_matrix = first.plan["e2e_matrix"]["include"]
|
||||
second_matrix = second.plan["e2e_matrix"]["include"]
|
||||
self.assertEqual(first_matrix, second_matrix)
|
||||
self.assertEqual(
|
||||
sum(item["runner"] == "standard-runner" for item in first_matrix), 7
|
||||
)
|
||||
|
||||
def test_backend_matrix_percentage_uses_floor(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="backend",
|
||||
run_id="103",
|
||||
families=BACKEND_MATRIX_FAMILY,
|
||||
burst="true",
|
||||
percent="50",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
matrix = plan.plan["backend_test_matrix"]["include"]
|
||||
self.assertEqual(
|
||||
sum(item["runner"] == "standard-runner" for item in matrix), 1
|
||||
)
|
||||
self.assertEqual(
|
||||
[(item["shard"], item["index"]) for item in matrix],
|
||||
[(1, 0), (2, 1)],
|
||||
)
|
||||
|
||||
def test_workflow_owned_family_names_need_no_python_change(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="new-workflow",
|
||||
run_id="109",
|
||||
families=[{"name": "new_job", "tier": "standard"}],
|
||||
burst="true",
|
||||
percent="100",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
self.assertEqual(plan.plan, {"new_job_runner": "standard-runner"})
|
||||
|
||||
def test_invalid_family_definition_is_rejected(self) -> None:
|
||||
with self.assertRaisesRegex(ValueError, "duplicate family name"):
|
||||
runner_plan.build_plan(
|
||||
workflow="frontend",
|
||||
run_id="110",
|
||||
families=[
|
||||
{"name": "same", "tier": "light"},
|
||||
{"name": "same", "tier": "standard"},
|
||||
],
|
||||
burst="true",
|
||||
percent="50",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
def test_singleton_allocation_is_stable_for_reruns(self) -> None:
|
||||
first = runner_plan.build_plan(
|
||||
workflow="frontend",
|
||||
run_id="104",
|
||||
families=FRONTEND_FAMILIES,
|
||||
burst="true",
|
||||
percent="50",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
second = runner_plan.build_plan(
|
||||
workflow="frontend",
|
||||
run_id="104",
|
||||
families=FRONTEND_FAMILIES,
|
||||
burst="true",
|
||||
percent="50",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
self.assertEqual(first.plan, second.plan)
|
||||
|
||||
def test_empty_tier_label_falls_back_to_github(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="frontend",
|
||||
run_id="105",
|
||||
families=FRONTEND_FAMILIES,
|
||||
burst="true",
|
||||
percent="100",
|
||||
light_label="",
|
||||
standard_label="",
|
||||
)
|
||||
|
||||
self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest")
|
||||
|
||||
def test_invalid_percentage_fails_closed_with_warning(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="e2e",
|
||||
run_id="106",
|
||||
families=E2E_FAMILIES,
|
||||
burst="true",
|
||||
percent="101",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
self.assertTrue(plan.warnings)
|
||||
self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest")
|
||||
matrix = plan.plan["e2e_matrix"]["include"]
|
||||
self.assertTrue(all(item["runner"] == "ubuntu-latest" for item in matrix))
|
||||
|
||||
def test_unicode_digits_fail_closed_with_warning(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="e2e",
|
||||
run_id="108",
|
||||
families=E2E_FAMILIES,
|
||||
burst="true",
|
||||
percent="²",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
self.assertTrue(plan.warnings)
|
||||
self.assertEqual(plan.plan["changes_runner"], "ubuntu-latest")
|
||||
matrix = plan.plan["e2e_matrix"]["include"]
|
||||
self.assertTrue(all(item["runner"] == "ubuntu-latest" for item in matrix))
|
||||
|
||||
def test_full_percentage_uses_external_for_non_empty_tier(self) -> None:
|
||||
plan = runner_plan.build_plan(
|
||||
workflow="frontend",
|
||||
run_id="107",
|
||||
families=FRONTEND_FAMILIES,
|
||||
burst="true",
|
||||
percent="100",
|
||||
light_label="light-runner",
|
||||
standard_label="standard-runner",
|
||||
)
|
||||
|
||||
self.assertEqual(plan.plan["frontend_runner"], "standard-runner")
|
||||
self.assertEqual(plan.plan["frontend_gate_runner"], "light-runner")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -24,6 +24,8 @@ permissions:
|
||||
jobs:
|
||||
lint:
|
||||
name: Architecture boundaries do not regress
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
@@ -33,11 +33,38 @@ permissions:
|
||||
packages: read
|
||||
|
||||
jobs:
|
||||
runner_plan:
|
||||
name: Plan external runner allocation
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
plan: ${{ steps.plan.outputs.plan }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Plan runner allocation
|
||||
id: plan
|
||||
uses: ./.github/actions/plan-external-runners
|
||||
with:
|
||||
workflow: backend
|
||||
run-id: ${{ github.run_id }}
|
||||
families: >-
|
||||
[{"name":"test","tier":"light"}]
|
||||
burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}
|
||||
percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}
|
||||
light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}
|
||||
standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}
|
||||
|
||||
# The former trigger-level `paths:` list, applied one level down so a skip
|
||||
# still produces a conclusion. `.github/scripts/changed-paths.py` implements
|
||||
# the same last-match-wins semantics GitHub applies to `paths:`.
|
||||
changes:
|
||||
name: Detect relevant changes
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
@@ -106,6 +133,8 @@ jobs:
|
||||
name: Backend Static Checks
|
||||
needs: changes
|
||||
if: needs.changes.outputs.run == 'true'
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
# Pre-baked image bundles Go, golangci-lint, go-licenses, Docker CLI, Node,
|
||||
# pnpm, and the Playwright base — see .github/docker/ci-base/Dockerfile.
|
||||
@@ -226,6 +255,8 @@ jobs:
|
||||
name: Backend Tests (${{ matrix.shard }}/2)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.run == 'true'
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -334,11 +365,13 @@ jobs:
|
||||
if: always()
|
||||
shell: bash
|
||||
working-directory: ${{ github.workspace }}
|
||||
env:
|
||||
BACKEND_TEST_SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
report_summary="$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md"
|
||||
report_summary="$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md"
|
||||
: > "$report_summary"
|
||||
GITHUB_STEP_SUMMARY="$report_summary" \
|
||||
INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/test-results-${{ matrix.shard }}.json" \
|
||||
INPUT_FROMJSONFILE="$GITHUB_WORKSPACE/apps/backend/test-results-${BACKEND_TEST_SHARD}.json" \
|
||||
INPUT_MODULEDIRECTORY="$GITHUB_WORKSPACE/apps/backend" \
|
||||
INPUT_OMIT="successful" \
|
||||
node ".github/vendor/go-test-action/dist/index.js"
|
||||
@@ -347,12 +380,14 @@ jobs:
|
||||
if: always()
|
||||
shell: bash
|
||||
working-directory: ${{ github.workspace }}
|
||||
env:
|
||||
BACKEND_TEST_SHARD: ${{ matrix.shard }}
|
||||
run: |
|
||||
python3 .github/scripts/bounded-step-summary.py \
|
||||
--input "$RUNNER_TEMP/backend-test-summary-${{ matrix.shard }}.md" \
|
||||
--input "$RUNNER_TEMP/backend-test-summary-${BACKEND_TEST_SHARD}.md" \
|
||||
--output "$GITHUB_STEP_SUMMARY" \
|
||||
--diagnostics-label "backend-test-results-${{ matrix.shard }}" \
|
||||
--diagnostics-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}#artifacts"
|
||||
--diagnostics-label "backend-test-results-${BACKEND_TEST_SHARD}" \
|
||||
--diagnostics-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#artifacts"
|
||||
|
||||
- name: Upload test artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
@@ -368,6 +403,8 @@ jobs:
|
||||
name: Backend (ambient integration env)
|
||||
needs: changes
|
||||
if: needs.changes.outputs.run == 'true'
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
# test_shards runs in the same clean container and never sets these vars,
|
||||
# so it cannot catch a package that starts reading them again — the class
|
||||
@@ -430,9 +467,10 @@ jobs:
|
||||
# `test-windows` were outside it before this workflow became a gate.
|
||||
test:
|
||||
name: Run Backend Tests
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).test_runner }}
|
||||
needs:
|
||||
- changes
|
||||
- runner_plan
|
||||
- static_checks
|
||||
- test_shards
|
||||
- test_ambient_env
|
||||
@@ -442,6 +480,7 @@ jobs:
|
||||
steps:
|
||||
- name: Require backend checks to pass
|
||||
env:
|
||||
RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }}
|
||||
CHANGES_RESULT: ${{ needs.changes.result }}
|
||||
STATIC_CHECKS_RESULT: ${{ needs.static_checks.result }}
|
||||
TEST_SHARDS_RESULT: ${{ needs.test_shards.result }}
|
||||
@@ -451,6 +490,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Runner plan: ${RUNNER_PLAN_RESULT}"
|
||||
echo "Change detection: ${CHANGES_RESULT}"
|
||||
echo "Static checks: ${STATIC_CHECKS_RESULT}"
|
||||
echo "Test shards: ${TEST_SHARDS_RESULT}"
|
||||
@@ -458,6 +498,13 @@ jobs:
|
||||
echo "Postgres boot: ${POSTGRES_BOOT_RESULT}"
|
||||
echo "Windows tests: ${TEST_WINDOWS_RESULT}"
|
||||
|
||||
# Runner planning must succeed because its outputs select the runner
|
||||
# fleet. A planner error cannot silently become a hosted fallback.
|
||||
if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then
|
||||
echo "Runner planning did not succeed; refusing to report a pass." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Change detection itself must succeed: if it errored we do not know
|
||||
# whether the skips below were deliberate.
|
||||
if [[ "${CHANGES_RESULT}" != "success" ]]; then
|
||||
|
||||
@@ -39,12 +39,47 @@ permissions:
|
||||
packages: read
|
||||
|
||||
jobs:
|
||||
runner_plan:
|
||||
name: Plan external runner allocation
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
plan: ${{ steps.plan.outputs.plan }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Plan runner allocation
|
||||
id: plan
|
||||
uses: ./.github/actions/plan-external-runners
|
||||
with:
|
||||
workflow: e2e
|
||||
run-id: ${{ github.run_id }}
|
||||
families: >-
|
||||
[
|
||||
{"name":"changes","tier":"light"},
|
||||
{"name":"e2e","tier":"standard","instances":[
|
||||
{"shard":1},{"shard":2},{"shard":3},{"shard":4},
|
||||
{"shard":5},{"shard":6},{"shard":7},{"shard":8},
|
||||
{"shard":9},{"shard":10},{"shard":11},{"shard":12},
|
||||
{"shard":13},{"shard":14}
|
||||
]},
|
||||
{"name":"e2e_gate","tier":"light"}
|
||||
]
|
||||
burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}
|
||||
percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}
|
||||
light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}
|
||||
standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}
|
||||
|
||||
# The former trigger-level `paths:` list, applied one level down so a skip
|
||||
# still produces a conclusion. `.github/scripts/changed-paths.py` implements
|
||||
# the same last-match-wins semantics GitHub applies to `paths:`.
|
||||
changes:
|
||||
name: Detect relevant changes
|
||||
runs-on: ubuntu-latest
|
||||
needs: runner_plan
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
run: ${{ steps.detect.outputs.run }}
|
||||
@@ -113,6 +148,8 @@ jobs:
|
||||
name: Build
|
||||
needs: changes
|
||||
if: needs.changes.outputs.run == 'true'
|
||||
# Build reads the GitHub API and downloads a timing artifact with the job token.
|
||||
# Keep this credential-bearing path on the GitHub-hosted boundary.
|
||||
runs-on: ubuntu-latest
|
||||
# Pre-baked image bundles Go, Node 24, pnpm 9.15.9 — see
|
||||
# .github/docker/ci-base/Dockerfile. Uses the `build` tag (with Go)
|
||||
@@ -367,9 +404,9 @@ jobs:
|
||||
|
||||
e2e:
|
||||
name: E2E Shard ${{ matrix.shard }}/14
|
||||
needs: [changes, build]
|
||||
needs: [changes, build, runner_plan]
|
||||
if: needs.changes.outputs.run == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ matrix.runner }}
|
||||
# Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright browser
|
||||
# binaries (Chromium etc.) — see .github/docker/ci-base/Dockerfile. This is
|
||||
# the biggest speedup since `npx playwright install chromium --with-deps`
|
||||
@@ -398,8 +435,7 @@ jobs:
|
||||
NODE_OPTIONS: --dns-result-order=ipv4first
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
|
||||
matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
@@ -467,9 +503,10 @@ jobs:
|
||||
working-directory: apps/web
|
||||
env:
|
||||
E2E_SHARD: ${{ matrix.shard }}
|
||||
E2E_MANIFEST_SHARD: ${{ matrix.shard }}
|
||||
KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json
|
||||
KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }}
|
||||
run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${{ matrix.shard }}.json"
|
||||
run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${E2E_MANIFEST_SHARD}.json"
|
||||
|
||||
- name: Upload blob report
|
||||
if: always()
|
||||
@@ -1104,6 +1141,8 @@ jobs:
|
||||
# reports to download, and the artifact step would fail on an empty run.
|
||||
# `e2e-gate` below, not this job, is what the merge queue waits on.
|
||||
if: always() && needs.changes.outputs.run == 'true'
|
||||
# Report aggregation queries GitHub history and downloads artifacts with the
|
||||
# job token, so it remains on a GitHub-hosted runner during the pilot.
|
||||
runs-on: ubuntu-latest
|
||||
# Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright CLI.
|
||||
# `runtime` tag — no Go needed for merging blob reports.
|
||||
@@ -1473,10 +1512,11 @@ jobs:
|
||||
# count changes.
|
||||
e2e-gate:
|
||||
name: E2E Tests Passed
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }}
|
||||
needs:
|
||||
[
|
||||
changes,
|
||||
runner_plan,
|
||||
build,
|
||||
playwright_image,
|
||||
e2e,
|
||||
@@ -1490,6 +1530,7 @@ jobs:
|
||||
steps:
|
||||
- name: Require the E2E jobs to have passed or been skipped
|
||||
env:
|
||||
RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }}
|
||||
CHANGES_RESULT: ${{ needs.changes.result }}
|
||||
BUILD_RESULT: ${{ needs.build.result }}
|
||||
PLAYWRIGHT_IMAGE_RESULT: ${{ needs.playwright_image.result }}
|
||||
@@ -1502,6 +1543,7 @@ jobs:
|
||||
set -euo pipefail
|
||||
|
||||
echo "Change detection: ${CHANGES_RESULT}"
|
||||
echo "Runner plan: ${RUNNER_PLAN_RESULT}"
|
||||
echo "Build: ${BUILD_RESULT}"
|
||||
echo "Playwright image: ${PLAYWRIGHT_IMAGE_RESULT}"
|
||||
echo "E2E shards: ${E2E_RESULT}"
|
||||
@@ -1510,6 +1552,13 @@ jobs:
|
||||
echo "Desktop smoke: ${DESKTOP_E2E_RESULT}"
|
||||
echo "Report merge: ${E2E_REPORT_RESULT}"
|
||||
|
||||
# Runner planning must succeed because its outputs select the runner
|
||||
# fleet. A planner error cannot silently become a hosted fallback.
|
||||
if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then
|
||||
echo "Runner planning did not succeed; refusing to report a pass." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Change detection itself must succeed: if it errored we do not know
|
||||
# whether the skips below were deliberate.
|
||||
if [[ "${CHANGES_RESULT}" != "success" ]]; then
|
||||
|
||||
@@ -29,12 +29,42 @@ permissions:
|
||||
packages: read
|
||||
|
||||
jobs:
|
||||
runner_plan:
|
||||
name: Plan external runner allocation
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
plan: ${{ steps.plan.outputs.plan }}
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Plan runner allocation
|
||||
id: plan
|
||||
uses: ./.github/actions/plan-external-runners
|
||||
with:
|
||||
workflow: frontend
|
||||
run-id: ${{ github.run_id }}
|
||||
families: >-
|
||||
[
|
||||
{"name":"changes","tier":"light"},
|
||||
{"name":"frontend","tier":"standard"},
|
||||
{"name":"frontend_gate","tier":"light"}
|
||||
]
|
||||
burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }}
|
||||
percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }}
|
||||
light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }}
|
||||
standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }}
|
||||
|
||||
# The former trigger-level `paths:` list, applied one level down so a skip
|
||||
# still produces a conclusion. `.github/scripts/changed-paths.py` implements
|
||||
# the same last-match-wins semantics GitHub applies to `paths:`.
|
||||
changes:
|
||||
name: Detect relevant changes
|
||||
runs-on: ubuntu-latest
|
||||
needs: runner_plan
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
run: ${{ steps.detect.outputs.run }}
|
||||
@@ -103,9 +133,9 @@ jobs:
|
||||
|
||||
frontend:
|
||||
name: Run Frontend Lint, Tests, and Build
|
||||
needs: changes
|
||||
needs: [changes, runner_plan]
|
||||
if: needs.changes.outputs.run == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}
|
||||
# Pre-baked image bundles Node 24, pnpm 9.15.9, Go, and go-licenses —
|
||||
# see .github/docker/ci-base/Dockerfile.
|
||||
container: ghcr.io/kdlbs/kandev-ci:build-latest
|
||||
@@ -121,6 +151,7 @@ jobs:
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Mark workspace safe for git
|
||||
shell: bash
|
||||
@@ -353,22 +384,28 @@ jobs:
|
||||
# conclusion to read instead of a check that stays pending forever.
|
||||
frontend-gate:
|
||||
name: Frontend Tests Passed
|
||||
runs-on: ubuntu-latest
|
||||
needs: [changes, frontend]
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}
|
||||
needs: [changes, frontend, runner_plan]
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Require the frontend job to have passed or been skipped
|
||||
env:
|
||||
RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }}
|
||||
CHANGES_RESULT: ${{ needs.changes.result }}
|
||||
FRONTEND_RESULT: ${{ needs.frontend.result }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Runner plan: ${RUNNER_PLAN_RESULT}"
|
||||
# `skipped` is a pass: the change-detection job decided nothing this
|
||||
# workflow covers was touched. `cancelled` is not — a cancelled job
|
||||
# never reached a verdict, and treating that as a pass would let an
|
||||
# untested merge group through.
|
||||
if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then
|
||||
echo "Runner planning finished with result: ${RUNNER_PLAN_RESULT}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ "${CHANGES_RESULT}" != "success" ]]; then
|
||||
echo "Change detection finished with result: ${CHANGES_RESULT}" >&2
|
||||
exit 1
|
||||
|
||||
@@ -34,6 +34,8 @@ permissions:
|
||||
jobs:
|
||||
lint:
|
||||
name: All action refs are SHA-pinned
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -73,6 +75,12 @@ jobs:
|
||||
- name: Test E2E workflow contract
|
||||
run: python3 .github/scripts/e2e-tests-workflow-contract_test.py
|
||||
|
||||
- name: Test external runner workflow contract
|
||||
run: python3 .github/scripts/external-runner-workflow-contract_test.py
|
||||
|
||||
- name: Test external runner planner
|
||||
run: python3 .github/scripts/runner-plan_test.py
|
||||
|
||||
- name: Test bounded step summary
|
||||
run: python3 .github/scripts/bounded-step-summary_test.py
|
||||
|
||||
|
||||
@@ -33,6 +33,8 @@ permissions:
|
||||
jobs:
|
||||
lint:
|
||||
name: Harness files pass lint
|
||||
# Checkout receives the short-lived job token. Keep this credential-bearing
|
||||
# path on GitHub-hosted capacity during the external-runner pilot.
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
|
||||
@@ -115,6 +115,79 @@ and it closes the gap where a subject outside their hand-maintained lists
|
||||
changed and nothing checked it. Their contract tests were updated to assert the
|
||||
absence of a filter rather than the presence of specific entries.
|
||||
|
||||
### External CI runner capacity
|
||||
|
||||
The normal Linux CI workflows have one activation switch, one percentage
|
||||
control, and two runner tiers. The switch and percentage are unset by default,
|
||||
so the workflows use GitHub-hosted runners.
|
||||
|
||||
| Variable | Pilot value | Purpose |
|
||||
| -------- | ----------- | ------- |
|
||||
| `KANDEV_CI_EXTERNAL_ENABLED` | unset | Set to `true` to allow external assignments |
|
||||
| `KANDEV_CI_EXTERNAL_PERCENT` | unset or `0` | Assign 0 to 100 percent of eligible jobs |
|
||||
| `KANDEV_CI_RUNNER_LIGHT` | `ubicloud-standard-2-ubuntu-2404` | Control and aggregate jobs |
|
||||
| `KANDEV_CI_RUNNER_STANDARD` | `ubicloud-standard-4-ubuntu-2404` | Eligible browser and frontend test jobs |
|
||||
|
||||
Set `KANDEV_CI_EXTERNAL_ENABLED` to the exact value `true` when the Actions queue
|
||||
needs extra capacity. Set it to `false`, or remove it, after the queue returns
|
||||
to its normal level. The two tier variables stay configured between bursts, so
|
||||
changing the instance type requires only an Actions variable update.
|
||||
|
||||
Set `KANDEV_CI_EXTERNAL_PERCENT` to `50` for a pilot. The E2E matrix then sends
|
||||
seven of fourteen normal shards to the standard tier. Backend checkout and
|
||||
report-token jobs remain on GitHub-hosted runners; the aggregate backend gate
|
||||
and frontend jobs are eligible. Singleton jobs use a stable hash cohort, so
|
||||
their share approaches the percentage across workflow runs.
|
||||
|
||||
Leave the percentage unset for the default GitHub-only mode. Set it to `100`
|
||||
to send every eligible job with a non-empty tier label to external capacity.
|
||||
Malformed or out-of-range values fail closed to GitHub-hosted runners and emit
|
||||
a planner warning.
|
||||
|
||||
The reusable `.github/actions/plan-external-runners` composite action runs on
|
||||
the planner job's `ubuntu-latest` checkout. Each workflow declares its eligible
|
||||
families as JSON, and the action returns one JSON plan with resolved runner
|
||||
labels:
|
||||
|
||||
```yaml
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
```
|
||||
|
||||
If a tier variable is empty, that tier uses `ubuntu-latest`. If a non-empty
|
||||
label is unavailable, GitHub leaves the job queued or fails it. Clear the
|
||||
invalid label and rerun the workflow. Jobs already queued or running keep their
|
||||
original runner. New jobs use the current variables.
|
||||
|
||||
The E2E build and report jobs, Docker/Kind shards, Kubernetes compatibility
|
||||
jobs, Playwright image job, desktop smoke job, backend checkout/static/test
|
||||
shards, Postgres service job, architecture and harness linters, and Windows job
|
||||
stay on GitHub-hosted runners. Release, publishing, signing, deployment, and
|
||||
credential-bearing jobs stay on their existing runners. The workflows do not
|
||||
add permissions, secrets, or persistent state.
|
||||
|
||||
The same switch and tier labels apply to eligible E2E, backend-gate, and
|
||||
frontend jobs. Architecture-lint, action-pinning, and harness-lint stay hosted
|
||||
and do not create planner jobs. The planner does not change job names, test
|
||||
selection, matrix values, artifacts, dependencies, timeouts, permissions, or
|
||||
required conclusions.
|
||||
|
||||
Pilot procedure:
|
||||
|
||||
1. Merge the workflow change while `KANDEV_CI_EXTERNAL_ENABLED` and
|
||||
`KANDEV_CI_EXTERNAL_PERCENT` are unset.
|
||||
2. Install the [Ubicloud GitHub App](https://www.ubicloud.com/docs/github-actions-integration) for this repository.
|
||||
3. Set both tier variables to the pilot labels in the table. Leave Premium
|
||||
Runners disabled. Premium is an account-level setting, not a per-job tier.
|
||||
4. Set `KANDEV_CI_EXTERNAL_PERCENT=50` for the first pilot.
|
||||
5. Set `KANDEV_CI_EXTERNAL_ENABLED=true` while the queue is full.
|
||||
6. Compare at least three runs for queue delay, job duration, runner label,
|
||||
image setup, and failures.
|
||||
7. Set the switch to `false` after the queue recovers. Keep the labels and
|
||||
percentage for the next burst.
|
||||
|
||||
Use the [Ubicloud runner type reference](https://www.ubicloud.com/docs/github-actions-integration/runner-types) when changing a tier. Use GitHub's [runner selection reference](https://docs.github.com/en/actions/how-tos/write-workflows/choose-where-workflows-run/choose-the-runner-for-a-job) for `runs-on` expressions. Do not enable [Ubicloud Premium Runners](https://www.ubicloud.com/docs/github-actions-integration/use-premium-runners) for this pilot.
|
||||
|
||||
### PR context in a merge group
|
||||
|
||||
A `merge_group` run has no `github.event.pull_request` and no
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# ADR-2026-09-06-opt-in-external-e2e-runners: Opt in selected Linux CI jobs to external runners
|
||||
|
||||
**Status:** accepted
|
||||
**Date:** 2026-09-06
|
||||
**Area:** infra, workflow, security
|
||||
|
||||
## Context
|
||||
|
||||
The E2E workflow creates fourteen normal Playwright shards, six host-Docker and
|
||||
Kind shards, two Kubernetes compatibility jobs, and supporting build, report,
|
||||
and gate jobs. Backend and frontend workflows also contain long Linux test
|
||||
jobs. Recent timing work reduced shard imbalance, but GitHub-hosted runner wait
|
||||
can still dominate the merge-queue critical path.
|
||||
|
||||
An external provider can add capacity. However, hard-coded labels make rollback
|
||||
require another workflow merge. One label for all jobs also prevents low-cost
|
||||
control jobs from using a smaller instance class.
|
||||
|
||||
Operators need one fast switch during a queue incident, plus a controlled way
|
||||
to pilot only part of the eligible workload. They also need to keep approved
|
||||
instance labels configured between incidents. Provider-wide routing moves jobs
|
||||
outside the reviewed Docker, platform, and credential boundaries.
|
||||
|
||||
## Decision
|
||||
|
||||
Use `KANDEV_CI_EXTERNAL_ENABLED` as the activation switch. A value of `true`
|
||||
activates configured external tiers for newly dispatched eligible jobs. Any
|
||||
other value keeps eligible jobs on `ubuntu-latest`.
|
||||
|
||||
Use `KANDEV_CI_EXTERNAL_PERCENT` to control the fraction of eligible jobs that
|
||||
the external fleet receives while burst mode is active. The default is `0`.
|
||||
The planner treats `0` as all GitHub-hosted and `100` as all external jobs with
|
||||
non-empty tier labels. Values from `1` through `99` are allocated
|
||||
deterministically per job family. Matrix families receive exactly
|
||||
`floor(N * percentage / 100)` external instances. Singleton families hash the
|
||||
workflow run into a stable bucket and approach the percentage across runs. A
|
||||
missing value means `0`; invalid or out-of-range values fail closed to
|
||||
GitHub-hosted runners with a visible warning.
|
||||
|
||||
Store the complete external labels in two independent repository variables:
|
||||
|
||||
- `KANDEV_CI_RUNNER_LIGHT` for change detection and the required gate.
|
||||
- `KANDEV_CI_RUNNER_STANDARD` for normal E2E shards and eligible frontend
|
||||
tests. Credential-bearing build/report and backend checkout paths remain
|
||||
hosted during the pilot.
|
||||
|
||||
If burst mode is active but a tier label is empty, jobs in that class use
|
||||
`ubuntu-latest`. A non-empty invalid label remains visible as a queued or failed
|
||||
job. Do not mix instance types inside the normal shard matrix.
|
||||
|
||||
Apply the same tier contract to eligible read-only Linux jobs in the E2E and
|
||||
frontend workflows, plus the backend aggregate gate. Keep E2E build/report
|
||||
jobs, backend checkout/static/test-shard/ambient jobs, architecture and
|
||||
harness linters, the Playwright image-resolution job, Docker/Kind shards,
|
||||
Kubernetes compatibility jobs, desktop E2E, Postgres service jobs, and Windows
|
||||
jobs on `ubuntu-latest` during the initial pilot. Do not apply this variable to
|
||||
`pull_request_target`, production, deployment, release, publishing, signing,
|
||||
or other credential-bearing jobs.
|
||||
|
||||
Use Ubicloud x64 Ubuntu 24.04 as the initial provider. Configure standard-2 for
|
||||
the light tier and standard-4 for the standard tier. Keep Premium Runners
|
||||
disabled because Ubicloud controls them at the account level.
|
||||
|
||||
Set burst mode to `false` after the queue recovers. This change affects only
|
||||
newly dispatched jobs. GitHub does not migrate queued or active jobs.
|
||||
|
||||
The external provider is part of the selected jobs' trust boundary. The pilot
|
||||
adds no secrets or permissions, retains `persist-credentials: false`, and
|
||||
keeps cache branch protection enabled. Expansion to protected workflows
|
||||
requires a measured compatibility and security review.
|
||||
|
||||
## Consequences
|
||||
|
||||
- Operators can enable burst capacity or return to GitHub-hosted runners
|
||||
with one repository-variable change.
|
||||
- Runner labels remain provider-neutral workflow data. Ubicloud and runner
|
||||
size remain operator-selected values.
|
||||
- Operators can resize one tier without changing other job classes.
|
||||
- Operators can set `KANDEV_CI_EXTERNAL_PERCENT` to `0`, `50`, or `100` to
|
||||
stage a pilot without editing workflow files.
|
||||
- Empty tier labels provide a safe GitHub-hosted fallback during partial
|
||||
configuration.
|
||||
- An invalid or unavailable configured label can leave new jobs queued until
|
||||
the operator clears the variable and reruns them. The workflow does not
|
||||
silently use another fleet.
|
||||
- External jobs expose their short-lived GitHub token, source, artifacts, and
|
||||
logs to the approved provider under the existing read-only permissions.
|
||||
- The pilot can reduce runner wait, but it cannot remove merge-group ordering,
|
||||
slow tests, cold caches, or artifact-transfer time.
|
||||
- A planner job adds a small GitHub-hosted dependency before eligible jobs can
|
||||
start. Exact compute-share percentages across unrelated workflows are not
|
||||
promised.
|
||||
|
||||
## Alternatives Considered
|
||||
|
||||
### Hard-code Ubicloud labels
|
||||
|
||||
Rejected because enablement, size changes, and rollback each require a
|
||||
workflow commit and merge-queue pass.
|
||||
|
||||
### Use one runner-label variable as the activation switch
|
||||
|
||||
Rejected because one label gives light and standard workloads the same
|
||||
instance size. Changing the label also erases the configured paid-runner value
|
||||
when the operator returns to GitHub-hosted capacity.
|
||||
|
||||
### Use tier labels without a separate activation switch
|
||||
|
||||
Rejected because each burst requires multiple variable changes. A partial
|
||||
change can unintentionally split eligible jobs across paid and GitHub-hosted
|
||||
capacity.
|
||||
|
||||
### Enable provider-wide premium routing
|
||||
|
||||
Rejected because it removes job-level selection and routes workloads
|
||||
whose trust and host-dependency boundaries are outside this pilot.
|
||||
|
||||
### Move every Linux CI job in the first rollout
|
||||
|
||||
Rejected because host-Docker, Kind, Kubernetes compatibility, image-resolution,
|
||||
desktop, service, and Windows jobs need separate compatibility evidence. Their
|
||||
smaller initial capacity benefit does not justify widening the pilot boundary.
|
||||
|
||||
### Use a direct percentage expression in `runs-on`
|
||||
|
||||
Rejected because GitHub expressions provide comparisons and boolean operators,
|
||||
but no random or modulo operator. A read-only planner job can produce a stable
|
||||
assignment for matrix and singleton job families and can fail closed on invalid
|
||||
operator input.
|
||||
|
||||
### Distribute an exact percentage of all workflow compute
|
||||
|
||||
Rejected because workflows contain different job-family sizes and runtimes.
|
||||
The contract allocates matrix instances exactly and singleton jobs by stable
|
||||
cohort, then measures real queue and execution behavior before any adaptive
|
||||
policy is considered.
|
||||
|
||||
### Fall back automatically inside a workflow run
|
||||
|
||||
Rejected because GitHub chooses `runs-on` before job execution. Implementing a
|
||||
second-fleet retry duplicates job graphs and complicates the required gate.
|
||||
It can also run the same untrusted workload on an unreviewed fleet. Manual
|
||||
variable rollback keeps the selected trust boundary explicit.
|
||||
@@ -245,3 +245,4 @@ Read individual ADRs for full context. Create new ones via `/record decision` or
|
||||
| 2026-09-04-generation-fenced-task-environment-ownership | [Fence Task Environment Ownership by Generation](2026-09-04-generation-fenced-task-environment-ownership.md) | accepted | backend | 2026-09-04 |
|
||||
| 2026-09-04-use-repository-token-for-runtime-pin-prs | [Use the built-in Actions token for runtime pin PRs](2026-09-04-use-repository-token-for-runtime-pin-prs.md) | accepted | workflow, security | 2026-09-04 |
|
||||
| 2026-09-05-agent-owned-credential-file-conflicts | [Agent-owned credential file conflicts](2026-09-05-agent-owned-credential-file-conflicts.md) | accepted | backend | 2026-09-05 |
|
||||
| 2026-09-06-opt-in-external-e2e-runners | [Opt in selected Linux CI jobs to external runners](2026-09-06-opt-in-external-e2e-runners.md) | accepted | infra, workflow, security | 2026-09-06 |
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
---
|
||||
created: 2026-09-06
|
||||
status: complete
|
||||
requirements:
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/external-e2e-runner-capacity.md
|
||||
legacy_specs: []
|
||||
---
|
||||
|
||||
# Implementation Plan: External CI Runner Capacity
|
||||
|
||||
## Overview
|
||||
|
||||
Add two provider-neutral runner tiers, one burst-mode switch, and an optional
|
||||
deterministic percentage rollout to the normal Linux CI workflows. Protect
|
||||
runner placement with workflow and planner contract tests. Update the
|
||||
merge-queue runbook with the Ubicloud configuration, activation, rollback, and
|
||||
measurement procedure.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- Route eligible E2E and frontend jobs, plus the backend aggregate gate,
|
||||
through the light or standard runner tier.
|
||||
- Keep token-bearing E2E build/report, backend checkout/test, architecture,
|
||||
action-pinning, and harness jobs on GitHub-hosted runners.
|
||||
- Use `KANDEV_CI_EXTERNAL_ENABLED` as the single activation switch for both tiers.
|
||||
- Use `KANDEV_CI_EXTERNAL_PERCENT` to allocate 0 to 100 percent of eligible
|
||||
job instances when burst mode is active.
|
||||
- Keep configured tier labels unchanged while burst mode is inactive.
|
||||
- Keep `playwright_image`, `e2e-containers`,
|
||||
`e2e-kubernetes-compatibility`, and `desktop-e2e` explicitly on
|
||||
`ubuntu-latest`.
|
||||
- Keep backend Postgres/service, Windows, and other protected jobs on
|
||||
`ubuntu-latest`.
|
||||
- Add workflow and planner contract coverage for eligible and protected jobs.
|
||||
- Document the Ubicloud standard-2 and standard-4 pilot, manual rollback, and
|
||||
trust boundary in the merge-queue runbook.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Installing the Ubicloud GitHub App or changing live repository Actions
|
||||
variables from repository code.
|
||||
- Moving Docker/Kind, compatibility, image-resolution, desktop, Postgres
|
||||
service, Windows, release, publishing, signing, deployment, or
|
||||
`pull_request_target` jobs.
|
||||
- Adding a heavy tier before the first pilot supplies CPU or memory evidence.
|
||||
- Weighted or adaptive allocation based on historical runtime or queue metrics.
|
||||
- Guaranteeing an exact percentage of total compute across unrelated workflows.
|
||||
- Changing shard counts, test selection, worker counts, timeouts, artifact
|
||||
flow, permissions, required checks, or merge-queue rules.
|
||||
|
||||
## Technical approach
|
||||
|
||||
### Workflow placement
|
||||
|
||||
Eligible singleton jobs read a resolved runner label from the planner's JSON
|
||||
output:
|
||||
|
||||
```yaml
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}
|
||||
```
|
||||
|
||||
Matrix jobs read their matrix from that plan and use its `runner` field.
|
||||
Protected jobs keep an explicit `ubuntu-latest` value and do not depend on the
|
||||
planner.
|
||||
|
||||
For percentage rollout, add a read-only `runner-plan.py` planner job on
|
||||
`ubuntu-latest`. It validates the percentage and emits runner assignments for
|
||||
each eligible job family. Matrix families receive exactly
|
||||
`floor(N * percentage / 100)` external assignments. Singleton families use a
|
||||
stable hash bucket. Workflows declare their family data through a reusable
|
||||
composite action. Downstream jobs consume one JSON plan containing resolved
|
||||
runner labels and matrices.
|
||||
|
||||
If burst mode is inactive, eligible jobs use GitHub-hosted capacity. If burst
|
||||
mode is active, each job uses its configured tier. An empty tier uses
|
||||
`ubuntu-latest`.
|
||||
|
||||
### Contract tests
|
||||
|
||||
Extend the E2E, backend, and frontend workflow contract tests with focused
|
||||
assertions for every affected job block. Add equivalent assertions that the
|
||||
architecture-lint, action-pinning, and harness-lint workflows stay hosted and
|
||||
do not create planner jobs.
|
||||
|
||||
Assert `runs-on: ubuntu-latest` in every protected block. Also assert that
|
||||
protected jobs do not reference the burst, percentage, or tier variables. Keep
|
||||
Ubicloud labels out of the workflow and contract tests.
|
||||
|
||||
Add `.github/scripts/runner-plan_test.py` for percentage validation, matrix
|
||||
floor allocation, singleton stability, rerun stability, empty labels, and
|
||||
fail-closed malformed input.
|
||||
|
||||
### Operator runbook
|
||||
|
||||
Add an external-runner section to `docs/ci-merge-queue.md`. Explain that the
|
||||
workflow change is merged while burst mode is inactive. List both tier
|
||||
variables, their initial Ubicloud values, and the protected job categories.
|
||||
|
||||
Explain that one variable activates or deactivates paid capacity. State that
|
||||
variable changes affect only new jobs. Define the comparison and rollback
|
||||
signals. Link the GitHub and Ubicloud source documentation.
|
||||
|
||||
## Tests
|
||||
|
||||
- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1` through `.4`: the workflow
|
||||
contract test verifies the toggle, two tier variables, and GitHub fallback.
|
||||
- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5`: existing E2E workflow
|
||||
contract tests remain unchanged and pass, protecting shard, cache, report,
|
||||
and gate behavior.
|
||||
- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6`: the design and runbook
|
||||
document visible failure plus manual clear-and-rerun recovery. GitHub owns
|
||||
label scheduling behavior.
|
||||
- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1` and `.2`: the focused
|
||||
contract test verifies that protected jobs retain `ubuntu-latest`. Action
|
||||
pinning and `zizmor` verify that the workflow adds no unpinned action or new
|
||||
permission path.
|
||||
- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.3`: post-merge pilot evidence
|
||||
records selected labels, queue delay, job duration, and reliability for at
|
||||
least three representative runs.
|
||||
- `AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.1` through `.6`:
|
||||
`runner-plan_test.py` and workflow contract tests verify 0/50/100 behavior,
|
||||
matrix floor allocation, stable singleton allocation, rerun stability, and
|
||||
fail-closed malformed input.
|
||||
|
||||
## Work orders
|
||||
|
||||
- [x] [Task 01: Add Opt-in E2E Runner Selection](task-01-add-opt-in-e2e-runner-selection.md)
|
||||
- [x] [Task 02: Extend Tiered Runners to Linux CI](task-02-extend-tiered-runners-to-linux-ci.md)
|
||||
- [x] [Task 03: Add Deterministic Percentage Rollout](task-03-deterministic-percentage-rollout.md)
|
||||
|
||||
## Verification results
|
||||
|
||||
- `python3 .github/scripts/e2e-tests-workflow-contract_test.py`: 14 tests passed.
|
||||
- `python3 .github/scripts/external-runner-workflow-contract_test.py`: 5 tests passed.
|
||||
- `python3 .github/scripts/runner-plan_test.py`: 8 tests passed.
|
||||
- `python3 .github/scripts/frontend-tests-workflow-contract_test.py`: 4 tests passed.
|
||||
- `python3 .github/scripts/lint-action-pinning_test.py`: 9 tests passed.
|
||||
- `python3 .github/scripts/lint-action-pinning.py`: all 21 workflow files passed.
|
||||
- `actionlint` v1.7.12: passed for all six touched workflows.
|
||||
- `zizmor` v1.25.2: no findings for all six touched workflows.
|
||||
- `python3 scripts/lint-spec-files.py --all`: passed.
|
||||
- `git diff --check`: passed.
|
||||
|
||||
The full `zizmor .github/workflows` scan can still report findings in existing
|
||||
workflows outside this change. The six touched workflows report no findings.
|
||||
|
||||
The Ubicloud pilot remains an operator action after merge. It needs at least
|
||||
three representative runs before a heavier tier or protected job moves.
|
||||
|
||||
Tasks 01, 02, and 03 are complete. The Ubicloud pilot remains an operator
|
||||
action after merge.
|
||||
|
||||
## Risks
|
||||
|
||||
- A misspelled or unavailable tier label can leave an eligible job queued
|
||||
until an operator clears the variable and reruns it.
|
||||
- An active burst with an empty tier label silently uses `ubuntu-latest` for
|
||||
that class. The runbook must include a pre-activation configuration check.
|
||||
- An external runner image can differ from GitHub's host image even when the
|
||||
normal build and shard jobs use repository-owned containers.
|
||||
- External transparent caching can change cache performance and poisoning risk.
|
||||
Branch protection must remain enabled for the public repository.
|
||||
- Moving the report and gate removes a small amount of GitHub-hosted demand but
|
||||
also makes their prompt completion depend on external capacity.
|
||||
- Runner capacity can reduce queue delay without improving merge-group
|
||||
serialization or slow test execution.
|
||||
- A planner job adds a small GitHub-hosted dependency before eligible jobs can
|
||||
start. Singleton percentage allocation is a cohort approximation, not an
|
||||
exact per-run share.
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
id: "01-add-opt-in-e2e-runner-selection"
|
||||
title: "Add opt-in E2E runner selection"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.3
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/external-e2e-runner-capacity.md
|
||||
---
|
||||
|
||||
# Task 01: Add Opt-in E2E Runner Selection
|
||||
|
||||
## Summary
|
||||
|
||||
Add two runner tiers and one burst-mode switch to the five eligible E2E jobs.
|
||||
Protect the placement boundary with a workflow contract test. Document the
|
||||
Ubicloud configuration, activation, and rollback procedure.
|
||||
|
||||
## In scope
|
||||
|
||||
- Update two light and three standard `runs-on` values in the E2E workflow.
|
||||
- Assert eligible and protected job placement in the E2E workflow contract
|
||||
test.
|
||||
- Add external-runner setup, rollback, security, and measurement guidance to
|
||||
the merge-queue runbook.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Changing live GitHub or Ubicloud settings.
|
||||
- Moving any protected or credential-bearing job.
|
||||
- Changing workflow behavior beyond runner placement.
|
||||
- Adding a heavy runner tier.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- When burst mode is inactive, all five eligible jobs use `ubuntu-latest`.
|
||||
- When burst mode is active, two jobs use the light tier and three use the
|
||||
standard tier. An empty tier label uses `ubuntu-latest`.
|
||||
- The four protected E2E jobs remain explicitly on `ubuntu-latest`, and the
|
||||
workflow adds no secret or permission.
|
||||
- Contract tests and operator guidance make the initial placement, rollback,
|
||||
and pilot measurements explicit.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/e2e-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/lint-action-pinning_test.py
|
||||
python3 .github/scripts/lint-action-pinning.py
|
||||
actionlint .github/workflows/e2e-tests.yml
|
||||
zizmor .github/workflows/e2e-tests.yml
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `.github/workflows/e2e-tests.yml`
|
||||
- `.github/scripts/e2e-tests-workflow-contract_test.py`
|
||||
- `docs/ci-merge-queue.md`
|
||||
- `docs/specs/platform/requirements/external-e2e-runner-capacity.md`
|
||||
- `docs/specs/platform/system-design/external-e2e-runner-capacity.md`
|
||||
- `docs/plans/external-e2e-runner-capacity/plan.md`
|
||||
- `docs/plans/external-e2e-runner-capacity/task-01-add-opt-in-e2e-runner-selection.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
None.
|
||||
|
||||
## Risks
|
||||
|
||||
- String-based workflow contract extraction depends on stable job boundaries.
|
||||
The test must use the existing `job_block` helper.
|
||||
- A live provider pilot is the only proof of image and capacity compatibility.
|
||||
Local checks prove the workflow contract, not external availability.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001` and
|
||||
`REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002`.
|
||||
- The external E2E runner capacity system design.
|
||||
- ADR 2026-09-06 for the opt-in placement and trust boundary.
|
||||
- Existing E2E job topology and workflow contract tests.
|
||||
|
||||
## Results
|
||||
|
||||
- Added `KANDEV_CI_EXTERNAL_ENABLED` expressions to the two light jobs and three
|
||||
standard jobs in `.github/workflows/e2e-tests.yml`.
|
||||
- Kept the Playwright image, Docker/Kind, Kubernetes compatibility, and desktop
|
||||
jobs on `ubuntu-latest`.
|
||||
- Added the eligible and protected job assertions to the E2E workflow contract
|
||||
test. The full contract suite passes with 14 tests.
|
||||
- Added the burst activation, tier values, rollback, security boundary, and
|
||||
pilot measurement procedure to `docs/ci-merge-queue.md`.
|
||||
- Action pinning tests and lint pass. `actionlint` v1.7.12 passes through
|
||||
`go run`. Targeted `zizmor` reports no findings.
|
||||
- The Ubicloud pilot is not run in CI code changes. Operators must collect the
|
||||
three-run comparison after merge before expanding the pilot.
|
||||
@@ -0,0 +1,119 @@
|
||||
---
|
||||
id: "02-extend-tiered-runners-to-linux-ci"
|
||||
title: "Extend tiered runners to Linux CI"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on:
|
||||
- "01-add-opt-in-e2e-runner-selection"
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.7
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/external-e2e-runner-capacity.md
|
||||
---
|
||||
|
||||
# Task 02: Extend Tiered Runners to Linux CI
|
||||
|
||||
## Summary
|
||||
|
||||
Apply the existing light and standard runner contract to eligible read-only
|
||||
Linux jobs that can add merge-queue delay in the E2E, backend, and frontend
|
||||
workflows. Keep checkout-token, report-token, host-service, container, Windows,
|
||||
release, deployment, architecture-lint, action-pinning, and harness-lint
|
||||
boundaries on GitHub-hosted runners.
|
||||
|
||||
## In scope
|
||||
|
||||
- Route the backend aggregate gate through the light tier. Keep backend jobs
|
||||
that check out source or publish reports on GitHub-hosted runners.
|
||||
- Route frontend control jobs through the light tier and the frontend test job
|
||||
through the standard tier.
|
||||
- Keep architecture-lint, action-pinning, and harness-lint checkout jobs on
|
||||
GitHub-hosted runners.
|
||||
- Extend workflow contract tests to assert eligible and protected placement.
|
||||
- Update the merge-queue runbook with the complete eligible-job table.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Percentage allocation. That is Task 03.
|
||||
- Moving backend Postgres/service or Windows jobs.
|
||||
- Moving Docker/Kind, Kubernetes compatibility, image, desktop, release,
|
||||
publishing, signing, deployment, or credential-bearing jobs.
|
||||
- Changing test selection, matrices, artifacts, permissions, or required gates.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- With burst mode inactive, every eligible job selects `ubuntu-latest`.
|
||||
- With burst mode active and a non-empty tier label, eligible jobs select their
|
||||
light or standard label. Empty labels use `ubuntu-latest`.
|
||||
- Protected jobs remain explicitly on `ubuntu-latest` and do not reference the
|
||||
burst or tier variables.
|
||||
- Existing job names, dependencies, matrices, tests, artifacts, timeouts,
|
||||
permissions, and required conclusions remain unchanged.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/e2e-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/frontend-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/lint-action-pinning_test.py
|
||||
python3 .github/scripts/lint-action-pinning.py
|
||||
python3 scripts/lint-harness-files.test.py
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
actionlint .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml
|
||||
zizmor .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `.github/workflows/backend-tests.yml`
|
||||
- `.github/workflows/frontend-tests.yml`
|
||||
- `.github/workflows/architecture-lint.yml`
|
||||
- `.github/workflows/lint-action-pinning.yml`
|
||||
- `.github/workflows/lint-harness-files.yml`
|
||||
- `.github/scripts/e2e-tests-workflow-contract_test.py`
|
||||
- `.github/scripts/frontend-tests-workflow-contract_test.py`
|
||||
- Additional workflow contract tests as needed for source-only jobs.
|
||||
- `docs/ci-merge-queue.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 01 must be complete so the shared variable names and E2E contract are
|
||||
already established.
|
||||
|
||||
## Risks
|
||||
|
||||
- Container jobs can use an external VM only when the container workflow does
|
||||
not require host Docker or service containers. Protected jobs stay on
|
||||
GitHub-hosted runners until measured evidence exists.
|
||||
- Short lint jobs can add planner or external capacity overhead if the fleet is
|
||||
unavailable. Manual burst rollback remains the recovery path.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Results
|
||||
|
||||
- Added planner-backed light and standard runner selection to eligible E2E,
|
||||
backend-gate, and frontend jobs.
|
||||
- Kept E2E build/report, backend checkout/static/test-shard/ambient,
|
||||
architecture-lint, action-pinning, harness-lint, Postgres service, Windows,
|
||||
Docker/Kind, Kubernetes, image, desktop, release, deployment, and
|
||||
credential-bearing jobs on GitHub-hosted runners.
|
||||
- Added workflow contract coverage for eligible jobs and protected boundaries.
|
||||
- The renamed activation variable is `KANDEV_CI_EXTERNAL_ENABLED`.
|
||||
- Contract, action-pinning, harness-lint, actionlint, specification, and diff
|
||||
checks passed.
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
id: "03-deterministic-percentage-rollout"
|
||||
title: "Add deterministic percentage rollout"
|
||||
status: done
|
||||
wave: 2
|
||||
depends_on:
|
||||
- "02-extend-tiered-runners-to-linux-ci"
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.1
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.2
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.3
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.4
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.5
|
||||
- AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.6
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/external-e2e-runner-capacity.md
|
||||
---
|
||||
|
||||
# Task 03: Add Deterministic Percentage Rollout
|
||||
|
||||
## Summary
|
||||
|
||||
Add a read-only planner that assigns a deterministic percentage of eligible job
|
||||
instances to external tiers. Use exact floor allocation for matrix families,
|
||||
stable hash cohorts for singleton jobs, and fail-closed validation for
|
||||
operator input.
|
||||
|
||||
## In scope
|
||||
|
||||
- Add `.github/scripts/runner-plan.py` and focused unit tests.
|
||||
- Add planner jobs and outputs to the eligible workflows without changing
|
||||
their test graph or protected-job boundaries.
|
||||
- Feed validated runner assignments to matrix and singleton jobs through
|
||||
`needs` outputs and `fromJSON` where required.
|
||||
- Document `KANDEV_CI_EXTERNAL_PERCENT`, 0/50/100 behavior, warnings, and
|
||||
rerun stability in the merge-queue runbook.
|
||||
- Extend workflow contract tests for planner wiring and protected jobs.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Automatic activation, adaptive or weighted allocation, or same-run retry.
|
||||
- Exact compute-share balancing across unrelated workflows.
|
||||
- Any protected host-Docker, Kind, Kubernetes, service, Windows, release,
|
||||
deployment, publishing, signing, or credential-bearing job.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Burst off or percentage `0` assigns all eligible jobs to `ubuntu-latest`.
|
||||
- Percentage `100` assigns all eligible jobs with non-empty tier labels to the
|
||||
configured tier; empty labels still use `ubuntu-latest`.
|
||||
- At 50 percent, a fourteen-instance E2E matrix receives seven external
|
||||
assignments. Protected backend checkout jobs remain hosted.
|
||||
- A rerun of the same workflow run receives the same assignments. Singleton
|
||||
jobs approach the configured percentage across different runs.
|
||||
- Missing input means `0`. Malformed or out-of-range input emits a visible
|
||||
warning and assigns all eligible jobs to `ubuntu-latest`.
|
||||
- Protected jobs never consume planner output and remain on GitHub-hosted
|
||||
runners.
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/runner-plan_test.py
|
||||
python3 .github/scripts/e2e-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/frontend-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/lint-action-pinning_test.py
|
||||
python3 .github/scripts/lint-action-pinning.py
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
actionlint .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml
|
||||
zizmor .github/workflows/e2e-tests.yml .github/workflows/backend-tests.yml .github/workflows/frontend-tests.yml .github/workflows/architecture-lint.yml .github/workflows/lint-action-pinning.yml .github/workflows/lint-harness-files.yml
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `.github/scripts/runner-plan.py`
|
||||
- `.github/scripts/runner-plan_test.py`
|
||||
- Eligible workflow files and their contract tests.
|
||||
- `docs/ci-merge-queue.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 02 must be complete so all eligible job classes use the same tier
|
||||
contract before planner outputs are introduced.
|
||||
|
||||
## Risks
|
||||
|
||||
- The planner adds a small GitHub-hosted dependency before eligible jobs start.
|
||||
- A stable singleton cohort is an approximation across runs, not an exact
|
||||
per-run percentage.
|
||||
- Workflow outputs must be constrained to approved labels so operator input
|
||||
cannot inject an arbitrary runner expression.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Results
|
||||
|
||||
- Added `.github/scripts/runner-plan.py` with deterministic matrix and
|
||||
singleton allocation.
|
||||
- Added `.github/actions/plan-external-runners/action.yml` so all workflow
|
||||
planner jobs share one environment and script invocation. Workflows own the
|
||||
family JSON; Python contains no workflow catalog.
|
||||
- Added planner tests for unset, 0, 50, 100, empty-label, rerun, and invalid
|
||||
percentage behavior.
|
||||
- Added planner jobs with one resolved JSON plan output to eligible workflows.
|
||||
- Updated the merge-queue runbook with percentage operation and rollback.
|
||||
- Planner, workflow contract, action-pinning, actionlint, specification, and
|
||||
diff checks passed.
|
||||
@@ -46,6 +46,7 @@ localization, feature toggles, health, and shared session recovery services.
|
||||
- [Browser console retention](requirements/browser-console-retention.md)
|
||||
- [Diagnostic logging](requirements/diagnostic-logging.md)
|
||||
- [Duration-aware E2E sharding and CI reliability](requirements/e2e-duration-aware-sharding.md)
|
||||
- [External CI runner capacity](requirements/external-e2e-runner-capacity.md)
|
||||
- [Expected runtime log severity](requirements/expected-runtime-log-severity.md)
|
||||
- [Feature Toggles](requirements/feature-toggles.md)
|
||||
- [Git Credential Lease Reissue](requirements/git-credential-lease-reissue.md)
|
||||
@@ -86,6 +87,7 @@ localization, feature toggles, health, and shared session recovery services.
|
||||
- [Diagnostic logging System Design Part 2](system-design/diagnostic-logging-02.md)
|
||||
- [Expected runtime log severity](system-design/expected-runtime-log-severity.md)
|
||||
- [Duration-aware E2E sharding and CI reliability](system-design/e2e-duration-aware-sharding.md)
|
||||
- [External CI runner capacity](system-design/external-e2e-runner-capacity.md)
|
||||
- [Health Endpoint — Surface the Running Version](system-design/health-endpoint-version.md)
|
||||
- [Internationalization (i18n)](system-design/i18n.md)
|
||||
- [LSP File Intelligence System Design Part 1](system-design/lsp-file-intelligence-01.md)
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
---
|
||||
status: active
|
||||
system: platform
|
||||
created: 2026-09-06
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# External CI Runner Capacity Requirements
|
||||
|
||||
## Overview
|
||||
|
||||
The merge queue can wait on GitHub-hosted runner capacity even when test jobs
|
||||
are balanced. Repository operators need a reversible external runner option
|
||||
for normal Linux CI workloads. This option must preserve required gates, test
|
||||
selection, and privileged workflow boundaries.
|
||||
|
||||
The platform system owns this contract because runner capacity is a shared
|
||||
operational guarantee across CI workflows. The CI automation system continues
|
||||
to own contributor trust gates for privileged pull-request automation.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Eligible Linux CI job:** A read-only Linux job in the E2E or frontend
|
||||
workflows, plus backend aggregate gates that do not receive checkout or
|
||||
report tokens. Credential-bearing checkout, report, service, and platform
|
||||
jobs are protected and stay on GitHub-hosted runners.
|
||||
- **Light tier:** External capacity for control jobs that run for seconds.
|
||||
- **Standard tier:** External capacity for build, report, browser-test, and
|
||||
Linux test jobs that need more CPU or memory.
|
||||
- **Burst mode:** The operator-controlled state that activates configured
|
||||
external runner tiers for new jobs.
|
||||
- **Protected CI job:** A host-Docker, Kind, Kubernetes compatibility, image,
|
||||
desktop, Windows, credential-bearing, release, or deployment job that stays
|
||||
on a GitHub-hosted runner during the initial rollout.
|
||||
- **Percentage rollout:** A deterministic allocation of eligible job
|
||||
instances between GitHub-hosted and configured external capacity.
|
||||
|
||||
## Requirements
|
||||
|
||||
### REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001: Reversible tiered Linux CI runner selection
|
||||
|
||||
**Intent:** Let repository operators configure runner tiers once. They can
|
||||
then activate or deactivate external capacity with one repository variable.
|
||||
|
||||
**User story:** As a repository operator, I want to use an approved external
|
||||
runner fleet, so that I can reduce merge-queue wait across Linux CI.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.1:** When the burst-mode
|
||||
variable is not `true`, every eligible Linux CI job shall select
|
||||
`ubuntu-latest`. Configured tier labels shall remain unchanged.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.2:** When burst mode is
|
||||
`true` and the percentage is `100`, each eligible Linux CI job shall use its
|
||||
configured light or standard runner label.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.3:** When burst mode is
|
||||
`true` and a required tier label is empty, that job class shall select
|
||||
`ubuntu-latest` for every percentage allocation.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.4:** Changing a tier label
|
||||
shall change the instance type for newly dispatched jobs in that class. It
|
||||
shall not require a workflow edit or a percentage change.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.5:** Changing burst mode, a
|
||||
tier label, or the percentage shall preserve each workflow's matrix, test
|
||||
selection, artifacts, dependencies, timeouts, permissions, and required
|
||||
conclusion.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.6:** When a non-empty tier
|
||||
label cannot accept a job, that job shall remain visibly queued or fail. The
|
||||
workflow shall not silently select an unapproved fleet.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001.7:** The same burst switch and
|
||||
tier labels shall apply consistently to eligible E2E, backend-gate, and
|
||||
frontend jobs. Protected lint workflows shall remain GitHub-hosted.
|
||||
|
||||
### REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002: Initial runner trust boundary
|
||||
|
||||
**Intent:** Limit the first external-runner pilot to jobs with reviewed
|
||||
permissions and runtime dependencies for that provider.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.1:** During the initial
|
||||
rollout, protected CI jobs shall select `ubuntu-latest`
|
||||
regardless of burst mode or configured tier labels.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.2:** The external-runner
|
||||
option shall not add secrets or broaden `GITHUB_TOKEN` permissions. It shall
|
||||
not persist runner state. It shall not move credential-bearing jobs to the
|
||||
external fleet.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002.3:** Operators shall be able
|
||||
to identify each job's class, configured tier, and selected fleet. They shall
|
||||
compare queue delay, execution time, and reliability before expansion.
|
||||
|
||||
### REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003: Deterministic percentage rollout
|
||||
|
||||
**Intent:** Let operators test an external fleet with a controlled fraction of
|
||||
eligible jobs while the remaining jobs use GitHub-hosted capacity.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.1:** When burst mode is not
|
||||
`true`, the percentage shall have no effect and every eligible job shall use
|
||||
`ubuntu-latest`.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.2:** When burst mode is
|
||||
`true` and the percentage is `0`, every eligible job shall use
|
||||
`ubuntu-latest`. When it is `100`, every eligible job with a non-empty tier
|
||||
label shall use that tier.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.3:** When the percentage is
|
||||
between `0` and `100`, the planner shall allocate each job family
|
||||
deterministically between GitHub-hosted and external capacity. A rerun of
|
||||
the same workflow run shall keep the same allocation.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.4:** For a matrix family with
|
||||
`N` instances, the planner shall allocate `floor(N * percentage / 100)`
|
||||
instances to external capacity. A single-instance family shall use a stable
|
||||
allocation across workflow runs that approaches the configured percentage.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.5:** A missing percentage
|
||||
shall be treated as `0`. A malformed or out-of-range percentage shall fail
|
||||
closed to `ubuntu-latest` and emit a visible planner warning. Neither case
|
||||
shall select an external label.
|
||||
- **AC-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003.6:** Percentage allocation
|
||||
shall not change job names, matrix values, artifacts, dependencies, test
|
||||
selection, timeouts, permissions, or required conclusions.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Automatically activating burst mode from queue metrics.
|
||||
- Automatically retrying an unavailable external runner label on GitHub-hosted
|
||||
capacity within the same workflow run.
|
||||
- Weighted allocation by historical runtime or queue metrics.
|
||||
- Moving Docker/Kind shards, Kubernetes compatibility, image resolution,
|
||||
desktop E2E, Postgres service jobs, Windows jobs, or credential-bearing jobs
|
||||
in the initial rollout.
|
||||
- Adding a heavy tier before measurements show that a workload needs it.
|
||||
- Guaranteeing an exact percentage across workflows with different job counts
|
||||
in one workflow run.
|
||||
- Enabling an external provider's global or premium-fleet routing setting.
|
||||
- Selecting ARM runners for x64 E2E binaries and containers.
|
||||
@@ -0,0 +1,237 @@
|
||||
---
|
||||
status: current
|
||||
system: platform
|
||||
requirements:
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002
|
||||
- REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003
|
||||
created: 2026-09-06
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# External CI Runner Capacity System Design
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
This design adds provider-neutral runner tiers to the normal Linux CI
|
||||
workflows. Two repository variables store the approved Linux x64 labels. One
|
||||
burst-mode variable activates those labels when GitHub-hosted capacity is full.
|
||||
An optional percentage variable lets operators run a deterministic fraction of
|
||||
eligible jobs on the external fleet during a pilot.
|
||||
|
||||
The tier labels remain configured while burst mode is inactive. An operator
|
||||
can therefore activate or deactivate paid capacity with one variable change.
|
||||
The operator can also change an instance type without a workflow edit.
|
||||
|
||||
The design changes only runner placement. The duration-aware planner, shard
|
||||
manifests, build artifacts, reports, required gates, and workflow permissions
|
||||
remain authoritative. Host-Docker, Kind, compatibility, image-resolution,
|
||||
desktop, service, Windows, checkout-token, report-token, release, and
|
||||
deployment jobs remain outside the initial pilot.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design sections |
|
||||
| --- | --- |
|
||||
| `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-001` | [Runner-selection contract](#runner-selection-contract), [Job placement](#job-placement), [Failure and recovery](#failure-and-recovery) |
|
||||
| `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-002` | [Security and trust boundary](#security-and-trust-boundary), [Rollout and observability](#rollout-and-observability) |
|
||||
| `REQ-PLATFORM-EXTERNAL-E2E-RUNNER-CAPACITY-003` | [Percentage allocation](#percentage-allocation), [Workflow integration](#workflow-integration), [Failure and recovery](#failure-and-recovery) |
|
||||
|
||||
## Runner-selection contract
|
||||
|
||||
The repository stores these Actions variables:
|
||||
|
||||
| Variable | Purpose | Initial Ubicloud value |
|
||||
| --- | --- | --- |
|
||||
| `KANDEV_CI_EXTERNAL_ENABLED` | Activates configured external tiers only when its value is `true`. | Unset or `false` |
|
||||
| `KANDEV_CI_EXTERNAL_PERCENT` | Selects the percentage of eligible job instances assigned to external capacity when burst mode is active. | Unset or `0` |
|
||||
| `KANDEV_CI_RUNNER_LIGHT` | Selects capacity for change detection and required gates. | `ubicloud-standard-2-ubuntu-2404` |
|
||||
| `KANDEV_CI_RUNNER_STANDARD` | Selects capacity for eligible browser and frontend test jobs. | `ubicloud-standard-4-ubuntu-2404` |
|
||||
|
||||
The workflow consumes the planner's resolved light-tier value with this
|
||||
`runs-on` expression:
|
||||
|
||||
```yaml
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }}
|
||||
```
|
||||
|
||||
The planner returns the configured external label or `ubuntu-latest`, never an
|
||||
expression fragment. Matrix jobs read their matrix from the same JSON plan and
|
||||
use `runs-on: ${{ matrix.runner }}`. GitHub evaluates the value before it
|
||||
dispatches the job.
|
||||
|
||||
Percentage rollout cannot be implemented safely with a direct `runs-on`
|
||||
expression. GitHub expressions provide comparisons and boolean operators but no
|
||||
random or modulo operator. A planner job therefore computes assignments before
|
||||
eligible jobs start and exposes one JSON plan through a job output. Downstream
|
||||
jobs consume that plan with `needs` and `fromJSON`.
|
||||
|
||||
If burst mode is active, the expression selects the complete label for that
|
||||
tier. An empty tier label uses `ubuntu-latest`. A non-empty invalid label stays
|
||||
visible as a queued or failed job.
|
||||
|
||||
Variable changes affect jobs that GitHub dispatches after the change. GitHub
|
||||
does not migrate queued or active jobs. To recover from an external incident,
|
||||
set `KANDEV_CI_EXTERNAL_ENABLED` to `false` and rerun the workflow.
|
||||
|
||||
The workflow does not parse provider names or instance sizes. It also does not
|
||||
mix instance types inside one shard cohort. Comparable runner capacity keeps
|
||||
the duration-aware timing profile useful across normal shards.
|
||||
|
||||
## Percentage allocation
|
||||
|
||||
`.github/scripts/runner-plan.py` is a generic read-only planner that runs on
|
||||
`ubuntu-latest`. It receives the workflow identifier, run identifier,
|
||||
workflow-owned JSON family descriptors, burst switch, percentage, and tier
|
||||
labels. It contains no workflow or job-family catalog. It validates the
|
||||
percentage and emits one JSON assignment plan plus a visible warning when the
|
||||
percentage is invalid.
|
||||
|
||||
The planner applies these rules:
|
||||
|
||||
1. Any burst value other than `true`, or a percentage of `0`, assigns every
|
||||
eligible instance to `ubuntu-latest`.
|
||||
2. A percentage of `100` assigns every eligible instance with a non-empty tier
|
||||
label to that tier. Empty tier labels still use `ubuntu-latest`.
|
||||
3. For a matrix family with `N` instances, it hashes the workflow, job family,
|
||||
run identifier, and matrix key, sorts the keys, and assigns exactly
|
||||
`floor(N * percentage / 100)` instances to the external tier.
|
||||
4. For a singleton family, it hashes the workflow, job-family key, and run
|
||||
identifier into a stable bucket. A rerun keeps the same allocation, and
|
||||
different workflow runs approach the configured percentage.
|
||||
5. A missing percentage is treated as `0`. A non-integer or out-of-range
|
||||
percentage assigns all instances to `ubuntu-latest` and emits a planner
|
||||
warning. It never emits an unvalidated external label for invalid input.
|
||||
|
||||
Each job family is allocated independently. At 50 percent, the fourteen E2E
|
||||
shards receive seven external assignments. Protected backend checkout jobs
|
||||
remain hosted; the backend aggregate gate and frontend jobs are still
|
||||
eligible. Singleton jobs use a stable hash cohort, so their share approaches
|
||||
50 percent across runs. The planner does not promise an exact percentage of
|
||||
total compute across workflows with different family sizes.
|
||||
|
||||
## Workflow integration
|
||||
|
||||
Each workflow with eligible jobs adds a small planner job on `ubuntu-latest`.
|
||||
Eligible jobs depend on the planner only for their runner assignment; their
|
||||
existing test dependencies, matrices, artifacts, timeouts, permissions, and
|
||||
required conclusions remain unchanged. A workflow with no eligible jobs does
|
||||
not run the planner.
|
||||
|
||||
The planner steps use the reusable composite action at
|
||||
`.github/actions/plan-external-runners/action.yml`. The action owns the
|
||||
validated environment wiring and `runner-plan.py` invocation, while each
|
||||
workflow owns the family JSON and exposes one `plan` job output. This keeps the
|
||||
job contract explicit without copying planner logic or teaching Python about
|
||||
workflow-specific families.
|
||||
|
||||
The JSON plan contains only resolved runner labels and matrix data declared by
|
||||
the workflow. Protected jobs keep explicit `runs-on: ubuntu-latest` and do not
|
||||
consume planner output. Contract tests assert the output wiring and the
|
||||
protected-job boundary.
|
||||
|
||||
## Job placement
|
||||
|
||||
| Job | Initial runner selection | Rationale |
|
||||
| --- | --- | --- |
|
||||
| `E2E changes`, `e2e-gate` | Light tier with GitHub fallback | Control jobs are short but can wait many minutes for capacity. |
|
||||
| `E2E build`, `e2e-report` | `ubuntu-latest` | They query GitHub history or download artifacts with the job token. |
|
||||
| `e2e` | Standard tier with GitHub fallback | The fourteen normal shards need comparable CPU and memory and do not use explicit repository-token inputs. |
|
||||
| `Backend changes`, `static_checks`, `test_shards`, `test_ambient_env` | `ubuntu-latest` | Their checkout action receives the short-lived job token. |
|
||||
| `Backend test` gate | Light tier with GitHub fallback | The aggregate gate is short and has no checkout or service dependency. |
|
||||
| `Frontend changes`, `frontend-gate` | Light tier with GitHub fallback | Control jobs are short and queue-sensitive. |
|
||||
| `Frontend frontend` | Standard tier with GitHub fallback | The single frontend test job is a recurring queue bottleneck. |
|
||||
| `Architecture lint`, action-pinning, harness-lint | `ubuntu-latest` | Their checkout action receives the short-lived job token. |
|
||||
| `playwright_image` | `ubuntu-latest` | Retains the reviewed host-Docker and GHCR metadata path during the pilot. |
|
||||
| `e2e-containers` | `ubuntu-latest` | Requires direct host Docker and creates Kind resources. |
|
||||
| `e2e-kubernetes-compatibility` | `ubuntu-latest` | Requires direct host Docker and pinned Kind clusters. |
|
||||
| `desktop-e2e` | `ubuntu-latest` | Retains the dedicated desktop container and toolchain boundary during the pilot. |
|
||||
| Backend `postgres-boot`, Windows, and container/service jobs | `ubuntu-latest` | Preserve host-service, operating-system, and credential boundaries. |
|
||||
|
||||
The runner expression does not alter `needs`, `if`, the matrix, the container
|
||||
image, the environment, the timeout, or artifacts. The gate keeps
|
||||
`if: always()` and its current result rules.
|
||||
|
||||
## Workflow contract coverage
|
||||
|
||||
The E2E, backend, and frontend workflow contract tests own the executable
|
||||
placement contract. Dedicated checks verify that architecture-lint,
|
||||
action-pinning, and harness-lint do not create planner jobs. They also verify
|
||||
that every protected CI job uses `runs-on: ubuntu-latest` without the burst,
|
||||
percentage, or tier variables.
|
||||
|
||||
The contract test also protects the provider-neutral behavior: it asserts the
|
||||
variable names and GitHub fallback, not the Ubicloud labels. Provider
|
||||
installation and live variable values remain operator-owned external state.
|
||||
|
||||
## Security and trust boundary
|
||||
|
||||
An external runner executes pull-request and merge-group code. It receives the
|
||||
short-lived job token, source, artifacts, and logs for its job. Provider
|
||||
isolation does not make the provider part of GitHub's hosted trust boundary.
|
||||
|
||||
The initial eligible jobs keep the workflow's existing top-level read-only
|
||||
permissions: `contents: read`, `actions: read`, and `packages: read`. Checkout
|
||||
continues to use `persist-credentials: false`. Jobs whose checkout, GitHub API,
|
||||
or artifact action receives the short-lived token remain on `ubuntu-latest`.
|
||||
The change adds no repository, environment, deployment, publishing, signing,
|
||||
or production secret.
|
||||
|
||||
The initial Ubicloud installation uses clean ephemeral x64 VMs and just-in-time
|
||||
runner registration. Keep the default branch protection if the Ubicloud
|
||||
transparent cache is enabled. Do not replace the pinned `actions/cache`
|
||||
references during this rollout.
|
||||
|
||||
Jobs triggered by `pull_request_target`, jobs with production or deployment
|
||||
credentials, checkout-token paths, E2E build/report token paths, releases,
|
||||
publishing, signing, and Windows builds are not eligible for burst mode.
|
||||
Future container jobs can use a heavy tier only after a separate host-
|
||||
dependency, performance, and security review.
|
||||
|
||||
Ubicloud Premium Runners remain disabled. Ubicloud controls Premium Runners at
|
||||
the account level, so that setting cannot provide job-level tier selection.
|
||||
|
||||
## Failure and recovery
|
||||
|
||||
| Condition | Required behavior |
|
||||
| --- | --- |
|
||||
| Burst mode is absent, empty, or not `true` | GitHub dispatches every eligible job to `ubuntu-latest`. |
|
||||
| Burst mode is `true` and the tier has an approved label | GitHub dispatches new jobs in that class to the configured fleet. |
|
||||
| Burst mode is `true` and the tier label is empty | GitHub dispatches that job class to `ubuntu-latest`. |
|
||||
| A non-empty tier label is unavailable or invalid | The job remains visibly queued or fails. No unreviewed fallback runs. |
|
||||
| External provider has an incident | Set burst mode to `false` and rerun. GitHub does not migrate queued or active jobs. |
|
||||
| External image differs from the GitHub image | The repository container images own most toolchains. The pilot must verify the host-only change-detection and gate behavior. |
|
||||
| External cache is cold | Existing dependency and artifact paths continue to work. Cache state can affect duration but not correctness. |
|
||||
|
||||
## Rollout and observability
|
||||
|
||||
1. Merge the workflow while `KANDEV_CI_EXTERNAL_ENABLED` is absent or `false`.
|
||||
2. Install the Ubicloud GitHub App only for the Kandev repository.
|
||||
3. Keep transparent-cache branch protection enabled.
|
||||
4. Disable Premium Runners in the Ubicloud account.
|
||||
5. Configure `KANDEV_CI_RUNNER_LIGHT` as
|
||||
`ubicloud-standard-2-ubuntu-2404`.
|
||||
6. Configure `KANDEV_CI_RUNNER_STANDARD` as
|
||||
`ubicloud-standard-4-ubuntu-2404`.
|
||||
7. When the GitHub queue is full, set `KANDEV_CI_EXTERNAL_ENABLED` to `true`.
|
||||
8. Observe at least three representative pull-request or merge-group runs.
|
||||
Record runner labels, queue delay, build duration, shard tail, report/gate
|
||||
delay, cancellations, retries, infrastructure failures, and cache behavior.
|
||||
9. After the queue recovers, set `KANDEV_CI_EXTERNAL_ENABLED` to `false`.
|
||||
10. If the pilot shows a regression, disable burst mode and rerun failed jobs.
|
||||
|
||||
Keep the two tier labels configured between bursts. Change the standard-tier
|
||||
label to `ubicloud-standard-8-ubuntu-2404` only after CPU or memory evidence.
|
||||
|
||||
The existing timing diagnostics separate queue delay from setup and test
|
||||
execution. The pilot succeeds when runner wait decreases without a higher job
|
||||
failure, cancellation, retry, cache-verification, or artifact-transfer rate.
|
||||
GitHub merge-group ordering remains an independent source of serialization and
|
||||
is not attributed to runner capacity.
|
||||
|
||||
## Related decisions
|
||||
|
||||
- [Opt in selected Linux CI jobs to external runners](../../../decisions/2026-09-06-opt-in-external-e2e-runners.md)
|
||||
- [Duration-aware E2E sharding uses rolling `main` timings](../../../decisions/2026-08-10-duration-aware-e2e-sharding.md)
|
||||
- [Cache host-runner E2E browser provisioning](../../../decisions/2026-08-30-e2e-browser-cache.md)
|
||||
Reference in New Issue
Block a user