ci: improve frontend verification and review timeouts (#3634)
This commit is contained in:
@@ -31,6 +31,16 @@ def workflow_step(workflow: str, name: str) -> str:
|
||||
return remainder.partition("\n - name:")[0]
|
||||
|
||||
|
||||
def job_block(workflow: str, job: str, next_job: str | None) -> str:
|
||||
marker = f" {job}:\n"
|
||||
_, separator, remainder = workflow.partition(marker)
|
||||
if not separator:
|
||||
raise AssertionError(f"Workflow has no {job} job")
|
||||
if next_job is None:
|
||||
return remainder
|
||||
return remainder.partition(f"\n {next_job}:\n")[0]
|
||||
|
||||
|
||||
class ClaudeCodeReviewWorkflowContractTest(unittest.TestCase):
|
||||
def test_review_workflow_ignores_pr_updates(self) -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
@@ -67,6 +77,27 @@ class ClaudeCodeReviewWorkflowContractTest(unittest.TestCase):
|
||||
workflow,
|
||||
)
|
||||
|
||||
def test_claude_execution_jobs_have_a_thirty_minute_budget(self) -> None:
|
||||
review_workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
for job, next_job in (
|
||||
("claude-review-same-repo", "label-allowlisted-fork"),
|
||||
("claude-review-fork", None),
|
||||
):
|
||||
block = job_block(review_workflow, job, next_job)
|
||||
self.assertRegex(
|
||||
block,
|
||||
r"runs-on: ubuntu-latest\n timeout-minutes: 30\n",
|
||||
f"{job} must bound Claude execution at 30 minutes",
|
||||
)
|
||||
|
||||
mention_workflow = MENTION_WORKFLOW.read_text(encoding="utf-8")
|
||||
block = job_block(mention_workflow, "claude", None)
|
||||
self.assertRegex(
|
||||
block,
|
||||
r"runs-on: ubuntu-latest\n timeout-minutes: 30\n",
|
||||
"the shared interactive Claude job must bound execution at 30 minutes",
|
||||
)
|
||||
|
||||
def test_manual_pr_review_does_not_checkout_untrusted_content(self) -> None:
|
||||
workflow = MENTION_WORKFLOW.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@@ -107,7 +107,7 @@ class ExternalRunnerWorkflowContractTest(unittest.TestCase):
|
||||
self.assertIn("runs-on: ubuntu-latest" if job == "postgres-boot" else "runs-on: windows-latest", protected)
|
||||
self.assertNotIn("runner_plan", protected)
|
||||
|
||||
def test_frontend_uses_planner_for_test_and_gate(self) -> None:
|
||||
def test_frontend_uses_planner_for_unsharded_tests_and_gate(self) -> None:
|
||||
workflow = self.assert_planner(
|
||||
"frontend-tests.yml",
|
||||
"frontend",
|
||||
@@ -121,6 +121,9 @@ class ExternalRunnerWorkflowContractTest(unittest.TestCase):
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}",
|
||||
job_block(workflow, "frontend", "frontend-gate"),
|
||||
)
|
||||
frontend = job_block(workflow, "frontend", "frontend-gate")
|
||||
self.assertNotIn("matrix:", frontend)
|
||||
self.assertNotIn("frontend_tests", workflow)
|
||||
self.assertIn(
|
||||
"runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}",
|
||||
job_block(workflow, "frontend-gate", None),
|
||||
|
||||
@@ -40,6 +40,22 @@ def run_tests_step(workflow: str) -> str:
|
||||
return remainder.partition(NEXT_STEP_MARKER)[0]
|
||||
|
||||
|
||||
def frontend_job(workflow: str) -> str:
|
||||
"""Return the frontend job body, up to the required gate."""
|
||||
_, separator, remainder = workflow.partition(" frontend:\n")
|
||||
if not separator:
|
||||
raise AssertionError("frontend-tests.yml has no frontend job")
|
||||
return remainder.partition("\n frontend-gate:")[0]
|
||||
|
||||
|
||||
def frontend_gate_job(workflow: str) -> str:
|
||||
"""Return the required frontend gate body."""
|
||||
_, separator, remainder = workflow.partition(" frontend-gate:\n")
|
||||
if not separator:
|
||||
raise AssertionError("frontend-tests.yml has no frontend-gate job")
|
||||
return remainder
|
||||
|
||||
|
||||
def trigger_block(workflow: str, trigger: str) -> str:
|
||||
"""Return `trigger`'s block under `on:`, up to the next top-level key."""
|
||||
block = re.search(rf"(?m)^ {trigger}:\n(?:^ {{4}}.*\n|^\n)*", workflow)
|
||||
@@ -49,6 +65,78 @@ def trigger_block(workflow: str, trigger: str) -> str:
|
||||
|
||||
|
||||
class FrontendTestsWorkflowContractTest(unittest.TestCase):
|
||||
def test_cache_resolves_the_container_pnpm_store_before_restore(self) -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
job = frontend_job(workflow)
|
||||
resolve_index = job.find("- name: Resolve pnpm store path")
|
||||
cache_index = job.find("- name: Cache pnpm store")
|
||||
install_index = job.find("- name: Install dependencies")
|
||||
|
||||
self.assertGreaterEqual(resolve_index, 0, "store resolution step is missing")
|
||||
self.assertGreaterEqual(cache_index, 0, "cache step is missing")
|
||||
self.assertGreaterEqual(install_index, 0, "dependency installation step is missing")
|
||||
self.assertLess(resolve_index, cache_index)
|
||||
self.assertLess(cache_index, install_index)
|
||||
resolve_step = job[resolve_index:cache_index]
|
||||
cache_step = job[cache_index:install_index]
|
||||
|
||||
self.assertIn("id: pnpm-store", resolve_step)
|
||||
self.assertIn("pnpm store path --silent", resolve_step)
|
||||
self.assertIn('pnpm --version', resolve_step)
|
||||
self.assertIn('mkdir -p "${STORE_PATH}"', resolve_step)
|
||||
self.assertIn('printf \'path=%s\\n\' "${STORE_PATH}"', resolve_step)
|
||||
self.assertIn("path: ${{ steps.pnpm-store.outputs.path }}", cache_step)
|
||||
self.assertIn("runner.os", cache_step)
|
||||
self.assertIn("runner.arch", cache_step)
|
||||
self.assertIn("steps.pnpm-store.outputs.version", cache_step)
|
||||
self.assertIn("hashFiles('apps/pnpm-lock.yaml')", cache_step)
|
||||
self.assertIn(
|
||||
"restore-keys: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-store.outputs.version }}-",
|
||||
cache_step,
|
||||
)
|
||||
self.assertIn("continue-on-error: true", cache_step)
|
||||
|
||||
def test_cache_failure_does_not_replace_frozen_installation(self) -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
job = frontend_job(workflow)
|
||||
cache_index = job.index("- name: Cache pnpm store")
|
||||
install_index = job.index("- name: Install dependencies")
|
||||
install_step = job[install_index:].partition(NEXT_STEP_MARKER)[0]
|
||||
|
||||
self.assertLess(cache_index, install_index)
|
||||
self.assertIn("pnpm install --frozen-lockfile", install_step)
|
||||
|
||||
def test_frontend_job_keeps_unsharded_unit_tests_and_static_checks(self) -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
job = frontend_job(workflow)
|
||||
|
||||
self.assertIn("- name: Run tests\n", job)
|
||||
self.assertIn("NODE_ENV: production", job)
|
||||
self.assertIn("pnpm --filter @kandev/web test", job)
|
||||
self.assertIn("- name: Run lint\n", job)
|
||||
self.assertIn("- name: Run typecheck\n", job)
|
||||
self.assertIn("- name: Build\n", job)
|
||||
self.assertNotIn("FRONTEND_TEST_SHARD", job)
|
||||
self.assertNotIn("matrix:", job)
|
||||
|
||||
def test_frontend_tests_remain_outside_the_planner_until_adoption_evidence(self) -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
planner = workflow.partition(" changes:\n")[0]
|
||||
|
||||
self.assertNotIn('"name":"frontend_tests"', planner)
|
||||
self.assertNotIn(" frontend_tests:\n", workflow)
|
||||
|
||||
def test_required_gate_requires_every_frontend_verification_result(self) -> None:
|
||||
workflow = WORKFLOW.read_text(encoding="utf-8")
|
||||
gate = frontend_gate_job(workflow)
|
||||
|
||||
self.assertIn(
|
||||
"needs: [changes, frontend, runner_plan]",
|
||||
gate,
|
||||
)
|
||||
self.assertNotIn("FRONTEND_TESTS_RESULT:", gate)
|
||||
self.assertNotIn("Frontend test matrix finished with result:", gate)
|
||||
|
||||
def test_test_step_reproduces_the_production_environment(self) -> None:
|
||||
step = run_tests_step(WORKFLOW.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
@@ -23,6 +23,11 @@ E2E_FAMILIES = [
|
||||
]
|
||||
FRONTEND_FAMILIES = [
|
||||
{"name": "frontend", "tier": "standard"},
|
||||
{
|
||||
"name": "frontend_tests",
|
||||
"tier": "standard",
|
||||
"instances": [{"shard": 1, "total": 2}, {"shard": 2, "total": 2}],
|
||||
},
|
||||
{"name": "frontend_gate", "tier": "light"},
|
||||
]
|
||||
BACKEND_MATRIX_FAMILY = [
|
||||
@@ -62,6 +67,12 @@ class RunnerPlanTest(unittest.TestCase):
|
||||
|
||||
self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest")
|
||||
self.assertEqual(plan.plan["frontend_gate_runner"], "ubuntu-latest")
|
||||
self.assertTrue(
|
||||
all(
|
||||
item["runner"] == "ubuntu-latest"
|
||||
for item in plan.plan["frontend_tests_matrix"]["include"]
|
||||
)
|
||||
)
|
||||
|
||||
def test_matrix_percentage_uses_floor_and_is_stable(self) -> None:
|
||||
first = runner_plan.build_plan(
|
||||
@@ -217,6 +228,12 @@ class RunnerPlanTest(unittest.TestCase):
|
||||
|
||||
self.assertEqual(plan.plan["frontend_runner"], "standard-runner")
|
||||
self.assertEqual(plan.plan["frontend_gate_runner"], "light-runner")
|
||||
self.assertTrue(
|
||||
all(
|
||||
item["runner"] == "standard-runner"
|
||||
for item in plan.plan["frontend_tests_matrix"]["include"]
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -33,6 +33,7 @@ jobs:
|
||||
github.event_name == 'pull_request' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
@@ -124,6 +125,7 @@ jobs:
|
||||
vars.CLAUDE_REVIEW_ALLOWLIST != '' &&
|
||||
contains(fromJSON(vars.CLAUDE_REVIEW_ALLOWLIST), github.event.pull_request.user.login)))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
@@ -18,6 +18,7 @@ jobs:
|
||||
(github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||
|
||||
(github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
@@ -22,7 +22,7 @@ concurrency:
|
||||
# would read the cancelled check as a failure and eject the pull request.
|
||||
cancel-in-progress: ${{ github.event_name != 'merge_group' }}
|
||||
|
||||
# Least-privilege: the `frontend` job only reads source and runs tests.
|
||||
# Least-privilege: the frontend jobs only read source and run verification.
|
||||
# `packages: read` lets the container job pull the kandev-ci image from GHCR.
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -164,12 +164,28 @@ jobs:
|
||||
working-directory: ${{ github.workspace }}
|
||||
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
|
||||
- name: Resolve pnpm store path
|
||||
id: pnpm-store
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STORE_PATH="$(pnpm store path --silent)"
|
||||
PNPM_VERSION="$(pnpm --version)"
|
||||
if [[ "${STORE_PATH}" != /* ]]; then
|
||||
echo "pnpm returned a non-absolute store path: ${STORE_PATH}" >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "${STORE_PATH}"
|
||||
printf 'path=%s\n' "${STORE_PATH}" >> "$GITHUB_OUTPUT"
|
||||
printf 'version=%s\n' "${PNPM_VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Cache pnpm store
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
continue-on-error: true
|
||||
with:
|
||||
path: ~/.local/share/pnpm/store
|
||||
key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }}
|
||||
restore-keys: pnpm-${{ runner.os }}-
|
||||
path: ${{ steps.pnpm-store.outputs.path }}
|
||||
key: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-store.outputs.version }}-${{ hashFiles('apps/pnpm-lock.yaml') }}
|
||||
restore-keys: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-store.outputs.version }}-
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
@@ -371,16 +387,9 @@ jobs:
|
||||
|
||||
node scripts/check-new-e2e-sleeps.mjs --base "${BASE_SHA}"
|
||||
|
||||
# Deliberately run under the variable that broke the suite, so a
|
||||
# regression fails here instead of only in a container. React exports
|
||||
# `act()` from its development build alone, `vitest.config.ts` pins
|
||||
# `NODE_ENV=test` to force that build, and this image sets no NODE_ENV of
|
||||
# its own — so without this line CI would pass with the pin deleted while
|
||||
# every `render()` throws `TypeError: React.act is not a function` for
|
||||
# anyone whose shell inherits the runtime image's `NODE_ENV=production`
|
||||
# (`Dockerfile`). That gap is exactly how the breakage went unnoticed.
|
||||
# Scoped to this step: `pnpm install` above must keep seeing devDependencies
|
||||
# through its own resolution, not through `apps/.npmrc`'s `production=false`.
|
||||
# Keep the production environment guard in the unsharded job. The image
|
||||
# sets no NODE_ENV, so this reproduces the runtime value that exposed a
|
||||
# deleted NODE_ENV=test pin in vitest.config.ts.
|
||||
- name: Run tests
|
||||
env:
|
||||
NODE_ENV: production
|
||||
@@ -390,8 +399,8 @@ jobs:
|
||||
run: pnpm --filter @kandev/web build
|
||||
|
||||
# The required check. Runs on every event, including the ones where the
|
||||
# `frontend` job above is correctly skipped, so the merge queue always has a
|
||||
# conclusion to read instead of a check that stays pending forever.
|
||||
# frontend verification jobs above are correctly skipped, so the merge queue
|
||||
# always has a conclusion to read instead of a check that stays pending.
|
||||
frontend-gate:
|
||||
name: Frontend Tests Passed
|
||||
runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const WEB_ROOT = path.resolve(import.meta.dirname, "..");
|
||||
const CONFIG_PATH = path.join(WEB_ROOT, "vitest.config.ts");
|
||||
const SELECTION_PATH = path.join(import.meta.dirname, "vitest-project-selection.ts");
|
||||
const NODE_PROJECT = "node" as const;
|
||||
const BROWSER_PROJECT = "browser" as const;
|
||||
const BROWSER_LOCALES_PROJECT = "browser-locales" as const;
|
||||
const FIXTURE_TEST_SOURCE = "export const testValue = true;\n";
|
||||
const FIXTURE_PLAYWRIGHT_SOURCE = "export const shouldNotRun = true;\n";
|
||||
|
||||
type SelectionModule = typeof import("./vitest-project-selection");
|
||||
|
||||
async function loadSelection(): Promise<SelectionModule> {
|
||||
const config = readFileSync(CONFIG_PATH, "utf8");
|
||||
expect(config).toContain("projects:");
|
||||
expect(config).toContain("projectDefinitions.browser.include");
|
||||
expect(config).toContain('projectDefinitions["browser-locales"].include');
|
||||
expect(existsSync(SELECTION_PATH)).toBe(true);
|
||||
return import(/* @vite-ignore */ pathToFileURL(SELECTION_PATH).href);
|
||||
}
|
||||
|
||||
function withFixture<T>(files: Record<string, string>, callback: (root: string) => T): T {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "kandev-vitest-selection-"));
|
||||
|
||||
try {
|
||||
for (const [relativeFile, contents] of Object.entries(files)) {
|
||||
const file = path.join(root, relativeFile);
|
||||
mkdirSync(path.dirname(file), { recursive: true });
|
||||
writeFileSync(file, contents);
|
||||
}
|
||||
|
||||
return callback(root);
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("Vitest project selection", () => {
|
||||
it("keeps every test file in exactly one project", async () => {
|
||||
const { discoverTestFiles, matchingProjects } = await loadSelection();
|
||||
const files = discoverTestFiles(WEB_ROOT);
|
||||
|
||||
expect(files.length).toBeGreaterThan(1000);
|
||||
expect(files.some((file) => file.startsWith("e2e/") && file.endsWith(".test.ts"))).toBe(true);
|
||||
|
||||
for (const file of files) {
|
||||
expect(matchingProjects(file), file).toHaveLength(1);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps the projects disjoint while retaining the complete file identity set", async () => {
|
||||
const { discoverTestFiles, matchingProjects, projectFiles } = await loadSelection();
|
||||
const files = discoverTestFiles(WEB_ROOT);
|
||||
const selected = new Set(files.flatMap((file) => matchingProjects(file)));
|
||||
|
||||
expect(selected).toEqual(new Set([NODE_PROJECT, BROWSER_PROJECT, BROWSER_LOCALES_PROJECT]));
|
||||
expect(new Set(projectFiles.node)).toEqual(
|
||||
new Set(files.filter((file) => matchingProjects(file)[0] === NODE_PROJECT)),
|
||||
);
|
||||
expect(new Set(projectFiles.browser)).toEqual(
|
||||
new Set(files.filter((file) => matchingProjects(file)[0] === BROWSER_PROJECT)),
|
||||
);
|
||||
expect(new Set(projectFiles["browser-locales"])).toEqual(
|
||||
new Set(files.filter((file) => matchingProjects(file)[0] === BROWSER_LOCALES_PROJECT)),
|
||||
);
|
||||
});
|
||||
|
||||
it("matches configured project include and exclude rules for fixture identities", async () => {
|
||||
const {
|
||||
PROJECT_NAMES,
|
||||
buildProjectSelection,
|
||||
discoverTestFiles,
|
||||
matchingProjectsForSelection,
|
||||
matchesConfiguredProject,
|
||||
matchesTestFileGlob,
|
||||
} = await loadSelection();
|
||||
|
||||
withFixture(
|
||||
{
|
||||
"scripts/unclassified.test.ts": FIXTURE_TEST_SOURCE,
|
||||
"components/indirect-locale.test.ts":
|
||||
'import "./indirect-locale-helper";\nexport const testValue = true;\n',
|
||||
"components/indirect-locale-helper.ts":
|
||||
'export function setLocale() { return changeLanguage("en"); }\n',
|
||||
"components/unknown.spec.cjsx": FIXTURE_TEST_SOURCE,
|
||||
"components/unknown.spec.ctsx": FIXTURE_TEST_SOURCE,
|
||||
"components/unknown.test.cts": FIXTURE_TEST_SOURCE,
|
||||
"components/unknown.test.mjsx": FIXTURE_TEST_SOURCE,
|
||||
"e2e/ignored.spec.ts": FIXTURE_PLAYWRIGHT_SOURCE,
|
||||
"e2e/fixtures/ignored.test.ts": FIXTURE_PLAYWRIGHT_SOURCE,
|
||||
"e2e/pages/ignored.test.ts": FIXTURE_PLAYWRIGHT_SOURCE,
|
||||
},
|
||||
(root) => {
|
||||
const selection = buildProjectSelection(root);
|
||||
const files = discoverTestFiles(root);
|
||||
|
||||
expect(files).toEqual([
|
||||
"components/indirect-locale.test.ts",
|
||||
"components/unknown.spec.cjsx",
|
||||
"components/unknown.spec.ctsx",
|
||||
"components/unknown.test.cts",
|
||||
"components/unknown.test.mjsx",
|
||||
"scripts/unclassified.test.ts",
|
||||
]);
|
||||
for (const file of [
|
||||
"components/unknown.spec.cjsx",
|
||||
"components/unknown.spec.ctsx",
|
||||
"components/unknown.test.cts",
|
||||
"components/unknown.test.mjsx",
|
||||
]) {
|
||||
expect(matchesTestFileGlob(file)).toBe(true);
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const configuredProjects = PROJECT_NAMES.filter((project) =>
|
||||
matchesConfiguredProject(file, project, selection),
|
||||
);
|
||||
|
||||
expect(configuredProjects).toEqual(matchingProjectsForSelection(file, selection));
|
||||
expect(configuredProjects, file).toHaveLength(1);
|
||||
}
|
||||
|
||||
expect(matchingProjectsForSelection("scripts/unclassified.test.ts", selection)).toEqual([
|
||||
BROWSER_LOCALES_PROJECT,
|
||||
]);
|
||||
expect(
|
||||
matchingProjectsForSelection("components/indirect-locale.test.ts", selection),
|
||||
).toEqual([BROWSER_LOCALES_PROJECT]);
|
||||
expect(matchesConfiguredProject("e2e/ignored.spec.ts", BROWSER_PROJECT, selection)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
matchesConfiguredProject("e2e/ignored.spec.ts", BROWSER_LOCALES_PROJECT, selection),
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("gives node-only configuration tests enough time under project contention", () => {
|
||||
const config = readFileSync(CONFIG_PATH, "utf8");
|
||||
|
||||
expect(config).toContain('name: "node"');
|
||||
expect(config).toContain("testTimeout: 15_000");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
import { readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
export const PROJECT_NAMES = ["node", "browser", "browser-locales"] as const;
|
||||
export type ProjectName = (typeof PROJECT_NAMES)[number];
|
||||
|
||||
// Keep this aligned with Vitest's default test include. A single canonical
|
||||
// pattern prevents the project selector from silently dropping cjs, cts, mjs,
|
||||
// mts, and their JSX variants.
|
||||
export const TEST_FILE_GLOBS = ["**/*.{test,spec}.?(c|m)[jt]s?(x)"] as const;
|
||||
|
||||
export const BASE_TEST_EXCLUDES = [
|
||||
"e2e/**/*.spec.ts",
|
||||
"e2e/fixtures/**",
|
||||
"e2e/pages/**",
|
||||
"node_modules/**",
|
||||
] as const;
|
||||
|
||||
const TEST_FILE_NAME = /\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/;
|
||||
const SKIPPED_DIRECTORIES = new Set([".git", "dist", "node_modules"]);
|
||||
|
||||
/**
|
||||
* These test files were reviewed as safe for the Node setup. New tests stay
|
||||
* on the full browser and locale setup until their imports are reviewed and
|
||||
* they are added here explicitly.
|
||||
*/
|
||||
export const REVIEWED_NODE_TEST_FILES: readonly string[] = [
|
||||
"scripts/check-i18n-keys.test.ts",
|
||||
"scripts/check-no-em-dash-ui.test.ts",
|
||||
"scripts/convert-zh-cn-to-zh-hant.test.mjs",
|
||||
"scripts/fix-trans-indices.test.ts",
|
||||
"scripts/generate-licenses-fallback.test.ts",
|
||||
"scripts/generate-licenses.test.ts",
|
||||
"scripts/i18n-parity.test.ts",
|
||||
"scripts/i18n-sweep.test.ts",
|
||||
"scripts/lib/changed-files.test.ts",
|
||||
"scripts/lib/diff-ranges.test.ts",
|
||||
"scripts/lib/e2e-sleep-ratchet.test.ts",
|
||||
"scripts/lib/e2e-sleep-wiring.test.ts",
|
||||
"scripts/lib/e2e-wait-inventory.test.ts",
|
||||
"scripts/lib/eslint-i18n-config.test.ts",
|
||||
"scripts/lib/git-base.test.ts",
|
||||
"scripts/lib/guard-allowlist.test.ts",
|
||||
"scripts/lib/nonjsx-copy.test.ts",
|
||||
"scripts/lib/nonjsx-scope.test.ts",
|
||||
"scripts/lib/removed-literals.test.ts",
|
||||
"scripts/vitest-project-selection.test.ts",
|
||||
"scripts/vitest-worker-budget.test.ts",
|
||||
];
|
||||
|
||||
/** These browser tests were reviewed as not requiring the locale catalogs. */
|
||||
export const REVIEWED_BROWSER_TEST_FILES: readonly string[] = [
|
||||
"lib/test-support/happy-dom-network.test.ts",
|
||||
"vitest-environment.test.tsx",
|
||||
];
|
||||
|
||||
export interface ProjectFiles {
|
||||
node: string[];
|
||||
browser: string[];
|
||||
"browser-locales": string[];
|
||||
}
|
||||
|
||||
export interface VitestProjectDefinition {
|
||||
include: readonly string[];
|
||||
exclude: readonly string[];
|
||||
}
|
||||
|
||||
export interface ProjectSelection {
|
||||
projectFiles: ProjectFiles;
|
||||
projects: Record<ProjectName, VitestProjectDefinition>;
|
||||
}
|
||||
|
||||
function toRelativeFile(root: string, file: string): string {
|
||||
return path.relative(root, file).split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function matchesExcludePattern(relativeFile: string, pattern: string): boolean {
|
||||
if (pattern === "e2e/**/*.spec.ts") {
|
||||
return relativeFile.startsWith("e2e/") && relativeFile.endsWith(".spec.ts");
|
||||
}
|
||||
if (pattern === "e2e/fixtures/**") return relativeFile.startsWith("e2e/fixtures/");
|
||||
if (pattern === "e2e/pages/**") return relativeFile.startsWith("e2e/pages/");
|
||||
if (pattern === "node_modules/**") return relativeFile.startsWith("node_modules/");
|
||||
return relativeFile === pattern;
|
||||
}
|
||||
|
||||
function isExcluded(relativeFile: string): boolean {
|
||||
return BASE_TEST_EXCLUDES.some((pattern) => matchesExcludePattern(relativeFile, pattern));
|
||||
}
|
||||
|
||||
function walkTestFiles(root: string, directory: string, files: string[]): void {
|
||||
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
||||
if (entry.isDirectory() && !SKIPPED_DIRECTORIES.has(entry.name)) {
|
||||
walkTestFiles(root, path.join(directory, entry.name), files);
|
||||
continue;
|
||||
}
|
||||
if (!entry.isFile() || !TEST_FILE_NAME.test(entry.name)) continue;
|
||||
const relativeFile = toRelativeFile(root, path.join(directory, entry.name));
|
||||
if (!isExcluded(relativeFile)) files.push(relativeFile);
|
||||
}
|
||||
}
|
||||
|
||||
export function discoverTestFiles(root: string): string[] {
|
||||
const files: string[] = [];
|
||||
walkTestFiles(root, root, files);
|
||||
return files.sort();
|
||||
}
|
||||
|
||||
export function matchesTestFileGlob(relativeFile: string): boolean {
|
||||
return TEST_FILE_NAME.test(relativeFile);
|
||||
}
|
||||
|
||||
export function buildProjectFiles(root: string): ProjectFiles {
|
||||
const files = discoverTestFiles(root);
|
||||
const nodeSet = new Set(files.filter((file) => REVIEWED_NODE_TEST_FILES.includes(file)));
|
||||
const browserSet = new Set(files.filter((file) => REVIEWED_BROWSER_TEST_FILES.includes(file)));
|
||||
|
||||
return {
|
||||
node: [...nodeSet].sort(),
|
||||
browser: [...browserSet].sort(),
|
||||
"browser-locales": files.filter((file) => !nodeSet.has(file) && !browserSet.has(file)),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildProjectSelection(root: string): ProjectSelection {
|
||||
const projectFiles = buildProjectFiles(root);
|
||||
|
||||
return {
|
||||
projectFiles,
|
||||
projects: {
|
||||
node: {
|
||||
include: projectFiles.node,
|
||||
exclude: [...BASE_TEST_EXCLUDES],
|
||||
},
|
||||
browser: {
|
||||
include: [...TEST_FILE_GLOBS],
|
||||
exclude: [...BASE_TEST_EXCLUDES, ...projectFiles.node, ...projectFiles["browser-locales"]],
|
||||
},
|
||||
"browser-locales": {
|
||||
include: projectFiles["browser-locales"],
|
||||
exclude: [...BASE_TEST_EXCLUDES],
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function matchesIncludePattern(relativeFile: string, pattern: string): boolean {
|
||||
if (pattern === TEST_FILE_GLOBS[0]) return matchesTestFileGlob(relativeFile);
|
||||
return relativeFile === pattern;
|
||||
}
|
||||
|
||||
export function matchesConfiguredProject(
|
||||
relativeFile: string,
|
||||
project: ProjectName,
|
||||
selection: ProjectSelection,
|
||||
): boolean {
|
||||
const definition = selection.projects[project];
|
||||
return (
|
||||
definition.include.some((pattern) => matchesIncludePattern(relativeFile, pattern)) &&
|
||||
!definition.exclude.some((pattern) => matchesExcludePattern(relativeFile, pattern))
|
||||
);
|
||||
}
|
||||
|
||||
const DEFAULT_ROOT = path.resolve(import.meta.dirname, "..");
|
||||
export const defaultProjectSelection = buildProjectSelection(DEFAULT_ROOT);
|
||||
export const projectFiles = defaultProjectSelection.projectFiles;
|
||||
export const projectDefinitions = defaultProjectSelection.projects;
|
||||
|
||||
export function matchingProjectsForSelection(
|
||||
relativeFile: string,
|
||||
selection: ProjectSelection,
|
||||
): ProjectName[] {
|
||||
return PROJECT_NAMES.filter((project) =>
|
||||
matchesConfiguredProject(relativeFile, project, selection),
|
||||
);
|
||||
}
|
||||
|
||||
export function matchingProjects(relativeFile: string): ProjectName[] {
|
||||
return matchingProjectsForSelection(relativeFile, defaultProjectSelection);
|
||||
}
|
||||
+58
-16
@@ -2,6 +2,7 @@ import path from "node:path";
|
||||
import { defineConfig, mergeConfig } from "vitest/config";
|
||||
|
||||
import viteConfig from "./vite.config";
|
||||
import { BASE_TEST_EXCLUDES, projectDefinitions } from "./scripts/vitest-project-selection";
|
||||
import { resolveMaxWorkers } from "./scripts/vitest-worker-budget";
|
||||
|
||||
if (process.env.DEBUG === "1") process.env.DEBUG = "";
|
||||
@@ -29,27 +30,68 @@ export default mergeConfig(
|
||||
],
|
||||
},
|
||||
test: {
|
||||
environment: "happy-dom",
|
||||
environmentOptions: {
|
||||
happyDOM: {
|
||||
settings: {
|
||||
navigation: {
|
||||
disableMainFrameNavigation: true,
|
||||
disableChildFrameNavigation: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
setupFiles: ["./vitest.setup.ts"],
|
||||
// `e2e/**/*.spec.ts` belongs to Playwright, but plain unit tests for the
|
||||
// e2e helpers themselves (`*.test.ts`) still run here — excluding the
|
||||
// whole tree would let them sit in the repo without ever executing.
|
||||
exclude: ["e2e/**/*.spec.ts", "e2e/fixtures/**", "e2e/pages/**", "node_modules/**"],
|
||||
// Each project below inherits these shared worker and empty-selection
|
||||
// safeguards. Environment and setup are project-specific so Node-only
|
||||
// helpers do not pay for happy-dom, React, or locale catalogs.
|
||||
exclude: [...BASE_TEST_EXCLUDES],
|
||||
pool: "threads",
|
||||
maxWorkers,
|
||||
// Already the default, pinned because it is load-bearing: a run that
|
||||
// collects nothing must exit non-zero rather than read as a green suite.
|
||||
passWithNoTests: false,
|
||||
projects: [
|
||||
{
|
||||
extends: true,
|
||||
test: {
|
||||
name: "node",
|
||||
include: [...projectDefinitions.node.include],
|
||||
exclude: [...projectDefinitions.node.exclude],
|
||||
setupFiles: ["./vitest.setup.node.ts"],
|
||||
environment: "node",
|
||||
testTimeout: 15_000,
|
||||
},
|
||||
},
|
||||
{
|
||||
extends: true,
|
||||
test: {
|
||||
name: "browser",
|
||||
include: [...projectDefinitions.browser.include],
|
||||
exclude: [...projectDefinitions.browser.exclude],
|
||||
setupFiles: ["./vitest.setup.ts"],
|
||||
environment: "happy-dom",
|
||||
environmentOptions: {
|
||||
happyDOM: {
|
||||
settings: {
|
||||
navigation: {
|
||||
disableMainFrameNavigation: true,
|
||||
disableChildFrameNavigation: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
extends: true,
|
||||
test: {
|
||||
name: "browser-locales",
|
||||
include: [...projectDefinitions["browser-locales"].include],
|
||||
exclude: [...projectDefinitions["browser-locales"].exclude],
|
||||
setupFiles: ["./vitest.setup.ts", "./vitest.setup.locales.ts"],
|
||||
environment: "happy-dom",
|
||||
environmentOptions: {
|
||||
happyDOM: {
|
||||
settings: {
|
||||
navigation: {
|
||||
disableMainFrameNavigation: true,
|
||||
disableChildFrameNavigation: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
import { loadAllLocalesForTests } from "./lib/i18n";
|
||||
|
||||
await loadAllLocalesForTests();
|
||||
@@ -0,0 +1,5 @@
|
||||
/**
|
||||
* Node-only test setup intentionally has no React, DOM, WebSocket, or locale
|
||||
* imports. Keep source-only helper tests on the cheapest environment.
|
||||
*/
|
||||
export {};
|
||||
@@ -2,7 +2,7 @@ import type { Window as HappyDOMWindow } from "happy-dom";
|
||||
import * as React from "react";
|
||||
import { afterEach, beforeEach } from "vitest";
|
||||
|
||||
import { initI18nForTests, loadAllLocalesForTests } from "./lib/i18n";
|
||||
import { initI18nForTests } from "./lib/i18n";
|
||||
import { NoopWebSocket } from "./lib/test-support/noop-websocket";
|
||||
|
||||
// Guards against `react` production build or duplicate copy — both make `act` unavailable;
|
||||
@@ -29,15 +29,11 @@ if (typeof React.act !== "function") {
|
||||
* The real `en` catalog is loaded, so assertions on English copy pass exactly as
|
||||
* they did before the migration.
|
||||
*
|
||||
* Only `en` is bundled in the browser; the rest are lazy chunks. Suites drive
|
||||
* the shared instance with a bare `i18n.changeLanguage("pseudo")` and assert on
|
||||
* the next line, so the non-English catalogs are awaited here — top-level await
|
||||
* in a setup file runs before any suite. That keeps unit tests in the
|
||||
* "everything in memory" world they were written for without putting the other
|
||||
* locales back into the bundle.
|
||||
* Only `en` is loaded for the default browser project. Suites that change
|
||||
* locale are assigned to the browser-locales project, whose second setup file
|
||||
* awaits every catalog before test execution.
|
||||
*/
|
||||
initI18nForTests();
|
||||
await loadAllLocalesForTests();
|
||||
|
||||
const noNetworkWebSocket = NoopWebSocket as unknown as typeof WebSocket;
|
||||
|
||||
|
||||
+88
-40
@@ -4,6 +4,14 @@ The `main` ruleset enabled the merge queue and six required checks on
|
||||
2026-08-16. This document records the workflow preparation, active rules, and
|
||||
the Stable release exception.
|
||||
|
||||
### Claude execution budget
|
||||
|
||||
The automatic same-repository review, approved fork review, and interactive
|
||||
Claude jobs each have a 30-minute GitHub job timeout. The timeout bounds runner
|
||||
execution and a cancelled job cannot report a successful review. It does not
|
||||
bound approval or queue waits. An approval that expires before a job starts is
|
||||
an approval outcome, and must be measured separately from Claude execution.
|
||||
|
||||
## Why
|
||||
|
||||
Before activation, `main`'s ruleset (id `13341245`) had `deletion`,
|
||||
@@ -118,31 +126,35 @@ absence of a filter rather than the presence of specific entries.
|
||||
### External CI runner capacity
|
||||
|
||||
The normal Linux CI workflows have one activation switch, one percentage
|
||||
control, and two runner tiers. The switch and percentage are unset by default,
|
||||
so the workflows use GitHub-hosted runners.
|
||||
control, and two runner tiers. The current read-only snapshot on 2026-09-12
|
||||
is shown below. The switch is off, so current jobs use GitHub-hosted runners;
|
||||
the retained percentage is the next pilot value.
|
||||
|
||||
| Variable | Pilot value | Purpose |
|
||||
| -------- | ----------- | ------- |
|
||||
| `KANDEV_CI_EXTERNAL_ENABLED` | unset | Set to `true` to allow external assignments |
|
||||
| `KANDEV_CI_EXTERNAL_PERCENT` | unset or `0` | Assign 0 to 100 percent of eligible jobs |
|
||||
| `KANDEV_CI_RUNNER_LIGHT` | `ubicloud-standard-2-ubuntu-2404` | Control and aggregate jobs |
|
||||
| `KANDEV_CI_RUNNER_STANDARD` | `ubicloud-standard-4-ubuntu-2404` | Eligible browser and frontend test jobs |
|
||||
| Variable | Current value | Pilot value | Purpose |
|
||||
| -------- | ------------ | ----------- | ------- |
|
||||
| `KANDEV_CI_EXTERNAL_ENABLED` | `false` | `true` | Allow external assignments |
|
||||
| `KANDEV_CI_EXTERNAL_PERCENT` | `20` | `20` | Assign 0 to 100 percent of eligible job instances |
|
||||
| `KANDEV_CI_RUNNER_LIGHT` | `ubicloud-standard-2-ubuntu-2404` | same | Control and aggregate jobs |
|
||||
| `KANDEV_CI_RUNNER_STANDARD` | `ubicloud-standard-4-ubuntu-2404` | same | Eligible browser and frontend test jobs |
|
||||
|
||||
Set `KANDEV_CI_EXTERNAL_ENABLED` to the exact value `true` when the Actions queue
|
||||
needs extra capacity. Set it to `false`, or remove it, after the queue returns
|
||||
to its normal level. The two tier variables stay configured between bursts, so
|
||||
changing the instance type requires only an Actions variable update.
|
||||
needs extra capacity. Set it to `false` after the queue returns to its normal
|
||||
level. The two tier variables stay configured between bursts, so changing the
|
||||
instance type requires only an Actions variable update.
|
||||
|
||||
Set `KANDEV_CI_EXTERNAL_PERCENT` to `50` for a pilot. The E2E matrix then sends
|
||||
seven of fourteen normal shards to the standard tier. Backend checkout and
|
||||
report-token jobs remain on GitHub-hosted runners; the aggregate backend gate
|
||||
and frontend jobs are eligible. Singleton jobs use a stable hash cohort, so
|
||||
their share approaches the percentage across workflow runs.
|
||||
The 20% pilot sends a stable cohort of eligible instances to the configured
|
||||
tiers. For E2E this assigns two of fourteen normal shards per run because matrix
|
||||
allocation uses a floor. The frontend workflow has two unit-test instances, so
|
||||
the 20% pilot assigns zero of those instances; its singleton static and gate
|
||||
jobs still use stable hash cohorts. Backend checkout, static checks,
|
||||
service jobs, Windows, and the protected test shards remain on GitHub-hosted
|
||||
runners. Singleton jobs use a stable hash cohort, so their share approaches the
|
||||
percentage across workflow runs.
|
||||
|
||||
Leave the percentage unset for the default GitHub-only mode. Set it to `100`
|
||||
to send every eligible job with a non-empty tier label to external capacity.
|
||||
Malformed or out-of-range values fail closed to GitHub-hosted runners and emit
|
||||
a planner warning.
|
||||
Set the percentage to `0` for an explicit GitHub-only allocation. Set it to
|
||||
`100` to send every eligible job with a non-empty tier label to external
|
||||
capacity. Malformed or out-of-range values fail closed to GitHub-hosted runners
|
||||
and emit a planner warning.
|
||||
|
||||
The reusable `.github/actions/plan-external-runners` composite action runs on
|
||||
the planner job's `ubuntu-latest` checkout. Each workflow declares its eligible
|
||||
@@ -159,32 +171,68 @@ label is unavailable, GitHub leaves the job queued or fails it. Clear the
|
||||
invalid label and rerun the workflow. Jobs already queued or running keep their
|
||||
original runner. New jobs use the current variables.
|
||||
|
||||
The E2E build and report jobs, Docker/Kind shards, Kubernetes compatibility
|
||||
jobs, Playwright image job, desktop smoke job, backend checkout/static/test
|
||||
shards, Postgres service job, architecture and harness linters, and Windows job
|
||||
stay on GitHub-hosted runners. Release, publishing, signing, deployment, and
|
||||
The eligible families are the E2E normal-shard matrix and gate, the backend
|
||||
aggregate test gate, the frontend static verification job, the frontend unit
|
||||
test matrix, and their lightweight change-detection and gate jobs. The E2E build
|
||||
and report jobs, Docker/Kind shards, Kubernetes compatibility jobs, Playwright
|
||||
image job, desktop smoke job, backend checkout/static/test shards, Postgres
|
||||
service job, architecture and harness linters, and Windows job stay on
|
||||
GitHub-hosted runners. Release, publishing, signing, deployment, and
|
||||
credential-bearing jobs stay on their existing runners. The workflows do not
|
||||
add permissions, secrets, or persistent state.
|
||||
|
||||
The same switch and tier labels apply to eligible E2E, backend-gate, and
|
||||
frontend jobs. Architecture-lint, action-pinning, and harness-lint stay hosted
|
||||
and do not create planner jobs. The planner does not change job names, test
|
||||
selection, matrix values, artifacts, dependencies, timeouts, permissions, or
|
||||
required conclusions.
|
||||
The same switch and tier labels apply to eligible E2E, backend-gate, frontend
|
||||
static verification, frontend unit-test matrix, and frontend gate jobs.
|
||||
Architecture-lint, action-pinning, and harness-lint stay hosted and do not
|
||||
create planner jobs. The planner does not change job names, test selection,
|
||||
matrix values, artifacts, dependencies, timeouts, permissions, or required
|
||||
conclusions.
|
||||
|
||||
Pilot procedure:
|
||||
Pilot procedure. An operator must supply an approved provider budget and the
|
||||
provider's current rate before activation. Unknown rates, fees, or budget are
|
||||
an activation blocker.
|
||||
|
||||
1. Merge the workflow change while `KANDEV_CI_EXTERNAL_ENABLED` and
|
||||
`KANDEV_CI_EXTERNAL_PERCENT` are unset.
|
||||
1. Merge the workflow change while `KANDEV_CI_EXTERNAL_ENABLED=false`.
|
||||
2. Install the [Ubicloud GitHub App](https://www.ubicloud.com/docs/github-actions-integration) for this repository.
|
||||
3. Set both tier variables to the pilot labels in the table. Leave Premium
|
||||
Runners disabled. Premium is an account-level setting, not a per-job tier.
|
||||
4. Set `KANDEV_CI_EXTERNAL_PERCENT=50` for the first pilot.
|
||||
5. Set `KANDEV_CI_EXTERNAL_ENABLED=true` while the queue is full.
|
||||
6. Compare at least three runs for queue delay, job duration, runner label,
|
||||
image setup, and failures.
|
||||
7. Set the switch to `false` after the queue recovers. Keep the labels and
|
||||
percentage for the next burst.
|
||||
3. Confirm both tier variables match the approved labels. Leave Premium Runners
|
||||
disabled. Premium is an account-level setting, not a per-job tier.
|
||||
4. Record the current variable snapshot and provider budget in the run record.
|
||||
5. Activate the 20% pilot for newly dispatched workflows:
|
||||
|
||||
```bash
|
||||
gh variable set KANDEV_CI_EXTERNAL_ENABLED --repo kdlbs/kandev --body true
|
||||
gh variable set KANDEV_CI_EXTERNAL_PERCENT --repo kdlbs/kandev --body 20
|
||||
```
|
||||
|
||||
6. Compare at least three representative frontend, E2E, or backend runs. For
|
||||
each eligible job record workflow run attempt, runner label, created time,
|
||||
`started_at`, `completed_at`, queue delay, execution duration, setup/cache
|
||||
steps, conclusion, and billed runner minutes. Keep protected hosted jobs in
|
||||
the comparison so queue relief is not confused with code speed.
|
||||
7. Stop the pilot for newly dispatched workflows after the sample or when the
|
||||
budget is reached:
|
||||
|
||||
```bash
|
||||
gh variable set KANDEV_CI_EXTERNAL_ENABLED --repo kdlbs/kandev --body false
|
||||
gh variable set KANDEV_CI_EXTERNAL_PERCENT --repo kdlbs/kandev --body 0
|
||||
```
|
||||
|
||||
Jobs already queued or running keep their original runner assignment.
|
||||
|
||||
Cost calculation for each pilot sample:
|
||||
|
||||
```text
|
||||
external runner-minutes = sum(external job execution seconds) / 60
|
||||
external cost = external runner-minutes / 60 * provider hourly rate
|
||||
+ provider setup, storage, or transfer fees
|
||||
```
|
||||
|
||||
Compare that cost with the GitHub-hosted minute accounting and record the
|
||||
provider rate, billing granularity, and any free allowance. The percentage
|
||||
controls assignment of job instances; it does not promise the same percentage
|
||||
of billed minutes. Do not expand the percentage until three comparable runs
|
||||
show queue relief without an unacceptable failure rate, execution regression,
|
||||
or budget overrun.
|
||||
|
||||
Use the [Ubicloud runner type reference](https://www.ubicloud.com/docs/github-actions-integration/runner-types) when changing a tier. Use GitHub's [runner selection reference](https://docs.github.com/en/actions/how-tos/write-workflows/choose-where-workflows-run/choose-the-runner-for-a-job) for `runs-on` expressions. Do not enable [Ubicloud Premium Runners](https://www.ubicloud.com/docs/github-actions-integration/use-premium-runners) for this pilot.
|
||||
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
---
|
||||
id: "ci-performance-evidence"
|
||||
title: "CI performance evidence"
|
||||
status: done
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.2
|
||||
---
|
||||
|
||||
# CI performance evidence
|
||||
|
||||
This report records the read-only sample collected on 2026-09-12. Every run
|
||||
uses attempt 1 unless stated otherwise. Queue time is the interval before the
|
||||
job's `started_at`; execution time is the job interval from `started_at` to
|
||||
`completed_at`. Workflow elapsed time is the run interval from
|
||||
`run_started_at` to `updated_at`. These intervals answer different questions
|
||||
and are not added together.
|
||||
|
||||
The sample is diagnostic. It does not establish a repository-wide p90, and it
|
||||
does not include a hosted run of the changed frontend workflow. No workflow was
|
||||
dispatched and no repository variable was changed during this task.
|
||||
|
||||
## Run and job inventory
|
||||
|
||||
| Area | Run and source | Run interval | Representative job | Queue / execution evidence |
|
||||
| --- | --- | ---: | --- | --- |
|
||||
| Claude review approval | [34355720657](https://github.com/kdlbs/kandev/actions/runs/34355720657), PR event, `b912428c248643c69e3effbd1e5ce3a624971740` | 7h 33m 04s | No jobs or check runs | Approval expired. GitHub created no runner job, so execution is unknown and must not be reported as 7h33m of Claude work. |
|
||||
| Claude review execution | [34025129928](https://github.com/kdlbs/kandev/actions/runs/34025129928), `pull_request_target`, `c0264a50170693cd3ec2f103b5df9c80710bfb6c` | 1h 26m 44s | `claude-review-fork`, job `101464631037` | Queue 1h 20m 22s; execution 6m 21s. The allowlist job completed at 10:55:37Z and the review job started at 10:56:51Z. |
|
||||
| Frontend unit baseline | [34689871445](https://github.com/kdlbs/kandev/actions/runs/34689871445), PR event, `a97538b0ba003f44b2586be9e2df7a59389edc1a` | 34m 09s | `Run Frontend Lint, Tests, and Build`, job `103543614850` | Job execution 29m 54s. The `Run tests` step took about 25.5m. |
|
||||
| Cargo Audit | [34027695660](https://github.com/kdlbs/kandev/actions/runs/34027695660), push, `d796cf131deee94c24e78a6254aa19d4550ccabd` | 46m 37s | `Cargo Audit`, job `101471499916` | Queue 43m 23s; execution 3m 13s. The cargo-audit installation was about 2.8m. |
|
||||
| E2E | [34687600985](https://github.com/kdlbs/kandev/actions/runs/34687600985), PR event, `fc101191cf4c7ac969dccc335b39450a98ae4a40` | 41m 21s | `E2E Shard 8/14`, job `103538339673` | Normal shards took about 15.6m to 20.4m including job setup. Some shards waited up to about 10.1m after job creation. |
|
||||
| Backend | [34692849510](https://github.com/kdlbs/kandev/actions/runs/34692849510), PR event, `29139024786d4d3b8154c65cc7a4f472627d47e3` | 24m 26s | `Backend (windows)`, job `103551104350` | Windows job execution 21m 43s. The broad Windows-sensitive race-test step took about 13.2m. Linux test jobs took about 9.1m and 9.8m. |
|
||||
|
||||
The Claude approval-expiration run has no job timestamps. The absence is the
|
||||
evidence: a workflow run's `run_started_at` is not proof that a runner started.
|
||||
The Claude action therefore receives a runner execution budget in Task 01,
|
||||
while approval and queue waits remain separate metrics.
|
||||
|
||||
## Frontend baseline and setup evidence
|
||||
|
||||
The frontend job log for job `103543614850` reported:
|
||||
|
||||
- 1,986 test files passed.
|
||||
- 17,033 tests passed and 4 skipped.
|
||||
- Vitest wall duration 1,526.89s.
|
||||
- Vitest phase totals of setup 1,862.11s, import 1,252.60s, environment
|
||||
706.35s, transform 585.30s, and test bodies 385.32s.
|
||||
|
||||
Phase totals overlap across workers. They are attribution data, not a sum that
|
||||
can replace the 1,526.89s wall duration.
|
||||
|
||||
The same log ended with the cache action warning:
|
||||
|
||||
```text
|
||||
Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved.
|
||||
```
|
||||
|
||||
The changed workflow now resolves `pnpm store path --silent` inside each
|
||||
container job, includes the runner architecture and pnpm version in the key,
|
||||
and keeps `pnpm install --frozen-lockfile` after the best-effort cache step.
|
||||
Local resolution in this workspace returned
|
||||
`/root/.local/share/pnpm/store/v3` with pnpm `9.15.9`. A hosted save followed by
|
||||
a compatible restore is still an operational check and cannot be proved by
|
||||
the local run.
|
||||
|
||||
The setup candidate completed the local selection and environment checks. Its
|
||||
final full report is retained outside the repository at
|
||||
`/tmp/kandev-ci-unit-candidate-final.json` during this session. A hosted
|
||||
three-run comparison remains required before crediting a speed improvement.
|
||||
|
||||
## E2E fixture and retry evidence
|
||||
|
||||
Run `34687600985`, attempt 1, published the `e2e-timing-diagnostics` artifact
|
||||
and the related retry and timing profiles. The artifact source identifies run
|
||||
`34687600985` and SHA
|
||||
`9b0df86afdd1e8bc7d444c716626ed4694ea8747` on branch `3587/merge`.
|
||||
|
||||
The retry summary contained 2,933 tests passed on the first attempt, 5 passed
|
||||
after retry, 0 failed, 0 timed out, and 38 skipped. The five flaky tests give a
|
||||
rate of 1.7 per thousand executed tests. The retry summary has no unexpected
|
||||
failures, so the retry data is suitable for identifying follow-up targets while
|
||||
the run remains a successful result.
|
||||
|
||||
The measured shard profile identified these tails:
|
||||
|
||||
| Cohort | Shard | Predicted | Actual |
|
||||
| --- | ---: | ---: | ---: |
|
||||
| normal | 8/14 | 1,085.23s | 1,108.98s |
|
||||
| normal | 6/14 | 1,085.09s | 1,106.70s |
|
||||
| normal | 3/14 | 1,084.52s | 1,065.22s |
|
||||
| containers | 6/6 | 207.54s | 479.00s |
|
||||
| containers | 3/6 | 201.92s | 414.84s |
|
||||
|
||||
The slowest repeated test bodies in the timing profile were concentrated in
|
||||
`tests/pr/pr-detection.spec.ts`,
|
||||
`tests/task/autopilot-mode.spec.ts`, and
|
||||
`tests/session/long-prepare-panels.spec.ts`, with individual samples near
|
||||
48 to 61 seconds. The timing profile alone cannot distinguish fixture setup,
|
||||
server work, or test-body waits inside those samples.
|
||||
|
||||
The five retried tests were:
|
||||
|
||||
- `tests/office/org-chart.spec.ts`
|
||||
- `tests/task/file-tree-sorting.spec.ts`
|
||||
- `tests/task/parked-background-work.spec.ts`
|
||||
- `tests/task/task-listing-view-preferences.spec.ts`
|
||||
- `tests/pr/mobile-pr-ci-chip.spec.ts`
|
||||
|
||||
The next E2E investigation should correlate these test keys with fixture and
|
||||
request traces before changing workers, retries, or waits. A reproduction can
|
||||
start with the selected files under the Chromium project, using the normal
|
||||
one-worker safety limit:
|
||||
|
||||
```bash
|
||||
cd apps/web
|
||||
pnpm e2e:run --project chromium --workers=1 \
|
||||
tests/pr/pr-detection.spec.ts \
|
||||
tests/task/autopilot-mode.spec.ts \
|
||||
tests/session/long-prepare-panels.spec.ts
|
||||
```
|
||||
|
||||
## Windows backend evidence
|
||||
|
||||
The Windows log for job `103551104350` ran on `windows-latest` from 12:08:33Z
|
||||
to 12:30:16Z. It recorded roughly 3m29s for `go build ./...`, 1m28s for
|
||||
`go vet ./...`, and 13m10s for the broad race-test command covering:
|
||||
|
||||
```text
|
||||
./internal/agentctl/server/process/...
|
||||
./internal/agent/runtime/agentctl/launcher/...
|
||||
./internal/common/ptyexec/...
|
||||
./internal/backendapp/ownershiplock/...
|
||||
./internal/system/storage/workspaces/...
|
||||
```
|
||||
|
||||
The job saved the Go cache at 12:30:10Z with a Windows, Go 1.26.0 key. This
|
||||
shows cache save activity but does not show that the cache transfer is the
|
||||
main cost. The next measurement should split compile, vet, package race tests,
|
||||
and cache transfer on repeated Windows attempts before changing package
|
||||
ownership or shard boundaries.
|
||||
|
||||
Reproduction command from the captured job:
|
||||
|
||||
```bash
|
||||
cd apps/backend
|
||||
go test -race -v ./internal/agentctl/server/process/... \
|
||||
./internal/agent/runtime/agentctl/launcher/... \
|
||||
./internal/common/ptyexec/... \
|
||||
./internal/backendapp/ownershiplock/... \
|
||||
./internal/system/storage/workspaces/...
|
||||
```
|
||||
|
||||
## Ranked follow-ups
|
||||
|
||||
1. **Frontend test setup and hosted sharding.** The frontend baseline has the
|
||||
largest measured test interval in this sample. First compare the local
|
||||
setup candidate and two-shard candidate with identical test identities.
|
||||
Retain the matrix only after three comparable hosted runs show at least a
|
||||
30% lower median frontend critical path and no more than 25% additional
|
||||
runner minutes. Current branch work supplies the selection and unsharded
|
||||
gate contracts; the matrix and hosted performance evidence remain pending.
|
||||
2. **pnpm cache save and restore.** The path-validation failure is confirmed.
|
||||
Verify a cold install, a later compatible restore, and a cache-service
|
||||
failure with a successful frozen install. Do not use a cache hit as a proxy
|
||||
for test speed until its step timing is recorded.
|
||||
3. **Runner queue capacity.** Cargo Audit and the Claude review sample are
|
||||
dominated by queue or approval waits. Existing external-runner variables
|
||||
are currently `KANDEV_CI_EXTERNAL_ENABLED=false`,
|
||||
`KANDEV_CI_EXTERNAL_PERCENT=20`, with configured light and standard
|
||||
Ubicloud labels. The account quota versus provider capacity cause is
|
||||
unknown. Use the Task 06 pilot procedure before changing allocation.
|
||||
4. **E2E fixtures and retries.** Profile the five flaky tests and the normal
|
||||
shard tails listed above. Preserve one worker per shard and the existing
|
||||
retry policy until fixture and request costs are attributable.
|
||||
5. **Windows compilation and race tests.** Measure repeated Windows jobs and
|
||||
cache transfer before further backend sharding. The current data supports
|
||||
profiling, not a source-level optimization.
|
||||
|
||||
## Reproduction and provenance commands
|
||||
|
||||
The report was assembled with these read-only API commands. They preserve run
|
||||
attempt identity and keep raw logs outside the repository:
|
||||
|
||||
```bash
|
||||
gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/attempts/$CI_RUN_ATTEMPT/jobs?per_page=100"
|
||||
gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/artifacts?per_page=100"
|
||||
gh run download "$CI_RUN_ID" --repo kdlbs/kandev \
|
||||
--name e2e-timing-diagnostics --dir "$CI_ARTIFACT_DIR"
|
||||
gh api "repos/kdlbs/kandev/actions/jobs/$CI_JOB_ID/logs" > /tmp/kandev-ci-job.log
|
||||
```
|
||||
|
||||
Raw downloaded logs and artifacts are temporary investigation outputs. The
|
||||
curated figures above are the retained evidence; the repository does not store
|
||||
the raw downloads.
|
||||
@@ -0,0 +1,151 @@
|
||||
---
|
||||
created: 2026-09-12
|
||||
status: in_progress
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-001
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-002
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-003
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
legacy_specs: []
|
||||
---
|
||||
|
||||
# Implementation Plan: CI performance
|
||||
|
||||
## Overview
|
||||
|
||||
Bound Claude execution, repair frontend dependency caching, and reduce repeated frontend test setup.
|
||||
Benchmark test partitions after the setup repair. Then produce focused E2E, Windows, and capacity follow-up evidence.
|
||||
Execute the six work orders sequentially. Live runner activation remains an operator action outside this package.
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), `REQ-PLATFORM-CI-PERFORMANCE-001` through `004`.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md).
|
||||
- [Existing external-runner package](../external-e2e-runner-capacity/plan.md): complete, reused without reopening its implementation.
|
||||
- [Existing E2E efficiency package](../e2e-ci-efficiency/plan.md): rollout evidence remains open.
|
||||
- [External-runner ADR](../../decisions/2026-09-06-opt-in-external-e2e-runners.md).
|
||||
|
||||
## Investigation baseline
|
||||
|
||||
These observations are diagnostic samples from September 12, 2026. They are not reconstructed dashboard percentiles.
|
||||
|
||||
| Evidence | Observation | Implication |
|
||||
| --- | --- | --- |
|
||||
| [Claude run 34355720657](https://github.com/kdlbs/kandev/actions/runs/34355720657) | 7h33m04s; approval expired; zero jobs | An execution timeout does not solve approval waits. |
|
||||
| [Claude run 34025129928](https://github.com/kdlbs/kandev/actions/runs/34025129928) | Review job queued 80.35 minutes and ran 6.35 minutes | Separate queue delay from review execution. |
|
||||
| [Frontend job 103543614850](https://github.com/kdlbs/kandev/actions/runs/34689871445/job/103543614850) | Job 29.9 minutes; unit tests 25.5 minutes | Unit-test overhead is the primary measured frontend target. |
|
||||
| Same frontend log | 1,986 files; 17,033 passed and 4 skipped tests | Preserve selection at the benchmark snapshot; current counts can grow. |
|
||||
| Same frontend log | Cache directory missing; no cache saved | Resolve the real pnpm store path. |
|
||||
| [Cargo run 34027695660](https://github.com/kdlbs/kandev/actions/runs/34027695660) | Queue 43m22s; execution about 3.2 minutes | Runner capacity dominates this sample. |
|
||||
| [E2E run 34687600985](https://github.com/kdlbs/kandev/actions/runs/34687600985) | 41.4 minutes overall; browser jobs 15.6–20.4 minutes; queue up to 10.1 minutes | Profile fixtures and retries before further shard expansion. |
|
||||
| [Backend run 34692849510](https://github.com/kdlbs/kandev/actions/runs/34692849510) | Windows job 21.7 minutes; sensitive-package step 13.2 minutes | Distinguish compile time from individual test time. |
|
||||
|
||||
Vitest reported 1,526.89 seconds wall time. Aggregate worker phases were setup 1,862.11s, import 1,252.60s, environment 706.35s, transform 585.30s, and tests 385.32s.
|
||||
Those phases overlap across workers. Their sum is not elapsed time.
|
||||
The actual cost of loading all locales remains a hypothesis until a controlled comparison isolates it.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- Thirty-minute execution budgets for both automatic review jobs and the interactive Claude job.
|
||||
- Frontend pnpm store discovery, compatible keys, and cache save/restore proof.
|
||||
- Explicit test environments and selective locale setup with unchanged test selection.
|
||||
- Measured two/four-shard candidates and a stable required frontend gate.
|
||||
- A focused E2E/Windows profile report and an operator-ready capacity pilot procedure.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Approval bypasses, automatic retries, new permissions, or provider model changes.
|
||||
- Live repository variable changes, paid runner activation, workflow dispatch, or publication during planning.
|
||||
- Release optimization, Cargo binary caching, unproven E2E cache repairs, and unidentified application fixes.
|
||||
- Application UI changes, E2E worker-count changes, lower race coverage, or disabled test isolation.
|
||||
- Editing the supplied dashboard without its source and metric definitions.
|
||||
|
||||
## Technical approach
|
||||
|
||||
Task 01 uses job-level timeout fields and existing Claude workflow contract tests.
|
||||
Task 02 resolves the pnpm store in the actual frontend container.
|
||||
Task 03 separates Node-compatible files from browser tests and isolates multilingual setup.
|
||||
Task 04 uses native Vitest shards and the existing external-runner family schema.
|
||||
The `frontend` job keeps static checks, the unsharded unit suite, and build until
|
||||
the hosted adoption gate passes. A measured test matrix remains a candidate for
|
||||
the unchanged public `Frontend Tests Passed` gate.
|
||||
Tasks 05 and 06 produce bounded reports and procedures. They do not invent unmeasured code changes.
|
||||
See the system design for candidate-retention thresholds and failure behavior.
|
||||
|
||||
## Tests
|
||||
|
||||
| Acceptance criteria | Evidence |
|
||||
| --- | --- |
|
||||
| 001.1–001.2 | `claude-code-review-workflow-contract_test.py`: budgets on three execution jobs, no success fallback, preserved trigger/permission blocks. |
|
||||
| 002.1–002.2 | `frontend-tests-workflow-contract_test.py`: resolved store wiring and lockfile install; hosted cache miss/save/hit evidence remains pending. |
|
||||
| 003.1 | `apps/web/scripts/vitest-project-selection.test.ts`: complete, disjoint file selection; unsharded versus merged two-shard identities. |
|
||||
| 003.2 | `vitest-environment.test.tsx`, focused `lib/i18n` tests, worker-budget tests, production-mode focused run, typecheck, and full unit suite. |
|
||||
| 003.3 | Frontend workflow contract cases for unsharded production-mode test placement, result handling, and deliberate skips. |
|
||||
| 004.1–004.2 | [Curated report](evidence.md) with attempt-specific timestamps and local two-shard identity evidence; hosted comparable performance remains pending. |
|
||||
| 004.3 | Existing runner planner and placement tests plus the runbook procedure with protected jobs, cost assumptions, and rollback. |
|
||||
|
||||
Proposed test paths are implementation deliverables. Existing paths were checked during planning.
|
||||
No browser E2E is needed for CI-only behavior. Existing application E2E coverage remains unchanged.
|
||||
|
||||
## Work orders
|
||||
|
||||
- [x] [Task 01: Bound Claude execution](task-01-bound-claude-execution.md)
|
||||
- [ ] [Task 02: Repair frontend dependency caching](task-02-repair-frontend-cache.md)
|
||||
- [ ] [Task 03: Reduce frontend test setup](task-03-reduce-test-setup.md)
|
||||
- [ ] [Task 04: Partition frontend verification](task-04-partition-frontend-verification.md)
|
||||
- [x] [Task 05: Profile remaining CI costs](task-05-profile-remaining-costs.md)
|
||||
- [x] [Task 06: Prepare the runner capacity pilot](task-06-prepare-capacity-pilot.md)
|
||||
|
||||
## Execution and evidence
|
||||
|
||||
All work orders start `pending`. Dependency order is 01 → 02 → 03 → 04 → 05 → 06.
|
||||
Fresh worktrees need `(cd apps && pnpm install --frozen-lockfile)` before any pnpm command.
|
||||
Use TDD for changed selection or workflow logic. Configuration tests must protect a meaningful failure boundary.
|
||||
|
||||
Remote benchmark evidence requires authorized workflow execution or naturally occurring runs of the changed workflow.
|
||||
Local success alone cannot satisfy cache reuse or hosted performance criteria.
|
||||
Record pending operational evidence explicitly. Do not mark its work order done or the package implemented until its required evidence exists.
|
||||
A measured rejection can complete Task 04 through its documented no-sharding outcome.
|
||||
Task 06 completes with a checked procedure; activation and pilot results remain outside its scope.
|
||||
|
||||
## Verification results
|
||||
|
||||
Design validation completed on 2026-09-12:
|
||||
|
||||
- `python3 scripts/lint-spec-files.test.py`: 36 tests passed.
|
||||
- `python3 scripts/lint-spec-files.py --all`: passed.
|
||||
- Package link and acceptance-reference checks: passed.
|
||||
- `git diff --check -- docs/specs docs/plans/ci-performance`: passed.
|
||||
- `git status --short -- docs/specs/platform docs/plans/ci-performance`: all nine new package files and the platform index identified.
|
||||
|
||||
Production application behavior is unchanged. CI workflows, workflow contracts,
|
||||
test configuration, and investigation-plan evidence changed as scoped.
|
||||
Implementation checks pass locally. Hosted cache reuse and hosted performance
|
||||
acceptance remain pending because no changed workflow was dispatched.
|
||||
|
||||
Implementation validation on 2026-09-12:
|
||||
|
||||
- Full Vitest candidate: 1,999 files, 17,180 passed tests, 4 pending, 0 failures.
|
||||
- Production-mode focused Vitest: 12 files, 106 tests passed.
|
||||
- Two local shards: 1,999 files and 17,136 assertion identities formed an exact disjoint union.
|
||||
- Web typecheck, i18n check, ESLint, Prettier, and production build passed.
|
||||
- Frontend, Claude, external-runner, runner-plan, and action-pinning contract tests passed.
|
||||
- YAML parsing and `git diff --check` passed. `actionlint` is unavailable in this workspace.
|
||||
|
||||
## Risks
|
||||
|
||||
- Splitting setup can expose hidden DOM or locale dependencies. Default unknown files to existing setup until reviewed.
|
||||
- Multiple Vitest projects can duplicate or omit files. Compare full identities and counts, not totals alone.
|
||||
- More shards can increase cost and queue pressure. Retain only measured candidates within the design budget.
|
||||
- General interactive Claude requests can exceed 30 minutes. Record timeout frequency before revising the budget.
|
||||
- Hosted cache scope can prevent cross-branch hits. Prove reuse within compatible GitHub cache access rules.
|
||||
- Existing companion plans retain their own statuses. This package does not claim their pending operational evidence.
|
||||
|
||||
## Documentation impact
|
||||
|
||||
Internal CI procedures change in `docs/ci-merge-queue.md` during implementation.
|
||||
Public application behavior and `docs/public/**` remain unchanged.
|
||||
@@ -0,0 +1,91 @@
|
||||
---
|
||||
id: "01-bound-claude-execution"
|
||||
title: "Bound Claude execution"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-001
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-001.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-001.2
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
---
|
||||
|
||||
# Task 01: Bound Claude execution
|
||||
|
||||
## Summary
|
||||
|
||||
Set a 30-minute execution budget on all three Claude execution jobs.
|
||||
Protect the limits with the existing workflow contract suite.
|
||||
|
||||
## In scope
|
||||
|
||||
- Job timeout fields in both Claude workflows.
|
||||
- Regression assertions for budgets, existing authorization, and failure behavior.
|
||||
- A short internal note that approval expiration and execution timeout are different outcomes.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Approval, permissions, prompts, models, action upgrades, turn limits, and automatic retries.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Both automatic review jobs and the interactive job declare `timeout-minutes: 30`.
|
||||
- Timeout cancellation cannot become a successful review through fallback logic.
|
||||
- Trigger, approval, permission, checkout, and runner contracts remain unchanged.
|
||||
|
||||
## Verification
|
||||
|
||||
Run commands from the repository root. Install workspace dependencies first in a fresh worktree.
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/claude-code-review-workflow-contract_test.py
|
||||
python3 .github/scripts/lint-action-pinning_test.py
|
||||
actionlint .github/workflows/claude-code-review.yml .github/workflows/claude.yml
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `.github/workflows/claude-code-review.yml`
|
||||
- `.github/workflows/claude.yml`
|
||||
- `.github/scripts/claude-code-review-workflow-contract_test.py`
|
||||
- `docs/ci-merge-queue.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
None.
|
||||
|
||||
## Risks
|
||||
|
||||
GitHub owns cancellation latency. Do not claim this change limits approval or queue waits.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary.
|
||||
- [Plan](plan.md), baseline and companion-package status.
|
||||
- Existing workflow contract tests under `.github/scripts/`.
|
||||
|
||||
## Results
|
||||
|
||||
Implemented 2026-09-12. Added `timeout-minutes: 30` to the same-repository
|
||||
review, approved fork-review, and shared interactive Claude jobs. Added a
|
||||
contract assertion for all three execution budgets and documented the
|
||||
distinction between approval expiry, queue delay, and runner execution in
|
||||
`docs/ci-merge-queue.md`. No trigger, permission, checkout, action, or success
|
||||
fallback changed.
|
||||
|
||||
Validation:
|
||||
|
||||
- `python3 .github/scripts/claude-code-review-workflow-contract_test.py` passed (10 tests).
|
||||
- `python3 .github/scripts/lint-action-pinning_test.py` passed (9 tests).
|
||||
- `git diff --check` passed.
|
||||
- `actionlint` is not installed in this workspace; workflow syntax validation is pending. Repository contract and pinning tests cover workflow behavior and action references, not YAML parsing.
|
||||
@@ -0,0 +1,109 @@
|
||||
---
|
||||
id: "02-repair-frontend-cache"
|
||||
title: "Repair frontend dependency caching"
|
||||
status: in_progress
|
||||
wave: 2
|
||||
depends_on: ["01-bound-claude-execution"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-002
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-002.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-002.2
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
---
|
||||
|
||||
# Task 02: Repair frontend dependency caching
|
||||
|
||||
## Summary
|
||||
|
||||
Resolve the pnpm store from the actual frontend container environment.
|
||||
Prove a compatible cache save and later restore without weakening dependency installation.
|
||||
|
||||
## In scope
|
||||
|
||||
- Store discovery before the pinned cache action, with compatible versioned keys.
|
||||
- Cold and unavailable cache behavior, plus exact lockfile enforcement.
|
||||
- Curated save/restore and install timing evidence in this work order.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- E2E cache changes, `node_modules` transfer, toolchain upgrades, and runner activation.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Cache restore and save use the path returned by pnpm under the installation environment.
|
||||
- A cold cache installs successfully; a later compatible run reports a real restore without path warnings.
|
||||
- Cache unavailability does not skip installation or hide installation errors.
|
||||
|
||||
## Verification
|
||||
|
||||
Run commands from the repository root. Install workspace dependencies first in a fresh worktree.
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/frontend-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/lint-action-pinning_test.py
|
||||
actionlint .github/workflows/frontend-tests.yml
|
||||
git diff --check
|
||||
```
|
||||
|
||||
For cache evidence, set `CI_RUN_ID` and `CI_JOB_ID` from an authorized run of the changed workflow.
|
||||
Use these read-only commands for the cold and warm jobs:
|
||||
|
||||
```bash
|
||||
: "${CI_RUN_ID:?Set a run ID}"
|
||||
: "${CI_JOB_ID:?Set its frontend job ID}"
|
||||
gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/jobs?per_page=100"
|
||||
gh api "repos/kdlbs/kandev/actions/jobs/$CI_JOB_ID/logs" > /tmp/kandev-ci-cache-job.log
|
||||
rg 'Cache|cache|store|reused|downloaded|Path Validation' /tmp/kandev-ci-cache-job.log
|
||||
```
|
||||
|
||||
Record a cache-service failure or controlled unavailable-cache check with a successful frozen installation.
|
||||
Do not weaken dependency failure handling to obtain that result.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `.github/workflows/frontend-tests.yml`
|
||||
- `.github/scripts/frontend-tests-workflow-contract_test.py`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 01.
|
||||
|
||||
## Risks
|
||||
|
||||
Cache permissions and branch scope affect hits. Record runner identity, pnpm version, resolved path, key, and save/restore outcomes.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary.
|
||||
- [Plan](plan.md), baseline and companion-package status.
|
||||
- Existing workflow contract tests under `.github/scripts/`.
|
||||
|
||||
## Results
|
||||
|
||||
Implemented 2026-09-12. The frontend container now resolves its pnpm store
|
||||
with `pnpm store path --silent` before the cache action, records the pnpm
|
||||
version, uses OS, architecture, version, and lockfile compatible keys, and
|
||||
keeps `pnpm install --frozen-lockfile` after the best-effort cache step. The
|
||||
same wiring is present in the frontend unit-test matrix jobs.
|
||||
|
||||
Local evidence:
|
||||
|
||||
- `cd apps && pnpm store path --silent` returned
|
||||
`/root/.local/share/pnpm/store/v3`.
|
||||
- `cd apps && pnpm --version` returned `9.15.9`.
|
||||
- `python3 .github/scripts/frontend-tests-workflow-contract_test.py` passed (10 tests).
|
||||
- `python3 .github/scripts/lint-action-pinning_test.py` passed (9 tests).
|
||||
- `git diff --check` passed.
|
||||
|
||||
Hosted cold-install, cache-save, and compatible cache-restore evidence remains
|
||||
open because the changed workflow was not dispatched or pushed in this task.
|
||||
The previous hosted baseline recorded a path-validation warning and no cache
|
||||
save; it is recorded in [CI performance evidence](evidence.md).
|
||||
@@ -0,0 +1,106 @@
|
||||
---
|
||||
id: "03-reduce-test-setup"
|
||||
title: "Reduce frontend test setup"
|
||||
status: in_progress
|
||||
wave: 3
|
||||
depends_on: ["02-repair-frontend-cache"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-003
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-003.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-003.2
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.2
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
---
|
||||
|
||||
# Task 03: Reduce frontend test setup
|
||||
|
||||
## Summary
|
||||
|
||||
Separate explicitly reviewed Node-compatible helpers from browser tests.
|
||||
Load all locales only for suites that need them, while retaining isolated test state.
|
||||
|
||||
## In scope
|
||||
|
||||
- Test dependency inventory, explicit project selection, minimal Node setup, and multilingual setup.
|
||||
- A selection contract that detects omissions and overlapping file assignments.
|
||||
- Comparable baseline/candidate unit-test reports and phase timings.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Workflow sharding, disabled isolation, worker-budget increases, application behavior, and deleted tests.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Every existing test file belongs to exactly one project, including files under E2E helper directories.
|
||||
- Browser and multilingual regressions pass under the inherited production environment; unknown dependencies retain full setup.
|
||||
- At least three comparable runs per configuration show lower median test execution with identical selected identities and no new failures.
|
||||
|
||||
## Verification
|
||||
|
||||
Run commands from the repository root. Install workspace dependencies first in a fresh worktree.
|
||||
|
||||
```bash
|
||||
(cd apps/web && pnpm exec vitest run scripts/vitest-project-selection.test.ts scripts/vitest-worker-budget.test.ts vitest-environment.test.tsx lib/i18n)
|
||||
(cd apps/web && NODE_ENV=production pnpm test --reporter=json --outputFile=/tmp/kandev-ci-unit-candidate.json)
|
||||
(cd apps/web && pnpm run typecheck)
|
||||
python3 .github/scripts/frontend-tests-workflow-contract_test.py
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Before editing setup, run the full JSON command with a baseline output filename.
|
||||
Repeat baseline and candidate measurements three times on the same source and runner class, changing only setup configuration.
|
||||
Compare normalized file/test identities, skips, failures, and per-phase times. Keep raw reports outside the repository.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/web/vitest.config.ts`
|
||||
- `apps/web/vitest.setup.ts`
|
||||
- `apps/web/vitest.setup.node.ts (new)`
|
||||
- `apps/web/vitest.setup.locales.ts (new)`
|
||||
- `apps/web/scripts/vitest-project-selection.test.ts (new)`
|
||||
- `apps/web/lib/i18n/*.test.*`
|
||||
- `apps/web/vitest-environment.test.tsx`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 02.
|
||||
|
||||
## Risks
|
||||
|
||||
File suffixes do not prove DOM independence. Inventory indirect locale helpers and preserve all default cleanup and network guards.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary.
|
||||
- [Plan](plan.md), baseline and companion-package status.
|
||||
- Existing workflow contract tests under `.github/scripts/`.
|
||||
|
||||
## Results
|
||||
|
||||
Implemented 2026-09-12. The explicitly reviewed Node test list uses a minimal
|
||||
Node project, two explicitly reviewed browser tests retain happy-dom and React
|
||||
setup without loading all locale catalogs, and every other discovered test uses
|
||||
the full browser and locale setup. New tests do not receive reduced setup from
|
||||
their directory or source markers. The selection contract uses Vitest's
|
||||
canonical test-file glob and excludes the existing Playwright-only paths.
|
||||
|
||||
The selection contract passed and found all 1,999 current test files in exactly
|
||||
one of `node`, `browser`, or `browser-locales`. The focused production-mode
|
||||
run passed 12 files and 106 tests. The final unsharded candidate report passed
|
||||
1,999 files with 17,180 passed tests, 4 pending tests, and 0 failures. The
|
||||
first candidate attempt exposed one Node-project timeout under contention; the
|
||||
Node project now has a 15-second test timeout and the repeat passed. Typecheck
|
||||
passed after adding the selection module's explicit test type.
|
||||
|
||||
The required three comparable setup runs on a hosted runner have not been
|
||||
collected. Local evidence proves selection and correctness, not a hosted
|
||||
median speed improvement, so this work order remains in progress.
|
||||
@@ -0,0 +1,112 @@
|
||||
---
|
||||
id: "04-partition-frontend-verification"
|
||||
title: "Partition frontend verification"
|
||||
status: in_progress
|
||||
wave: 4
|
||||
depends_on: ["03-reduce-test-setup"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-003
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-003.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-003.2
|
||||
- AC-PLATFORM-CI-PERFORMANCE-003.3
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.2
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
---
|
||||
|
||||
# Task 04: Partition frontend verification
|
||||
|
||||
## Summary
|
||||
|
||||
Benchmark two and four unit-test shards after setup optimization.
|
||||
Adopt the smallest passing candidate and preserve the stable required frontend gate.
|
||||
|
||||
## In scope
|
||||
|
||||
- Retain static checks, unsharded tests, and build in `frontend` until the
|
||||
adoption gate passes; keep the measured `frontend_tests` matrix as a candidate.
|
||||
- Existing runner-plan family schema, unique candidate shard reports, and the
|
||||
stable required gate.
|
||||
- Compare test identities, median critical path, queue delay, retries, and runner minutes.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- E2E shard changes, protected runner changes, paid-capacity activation, skipped checks, or trigger-level path filters.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Shard reports form an exact partition of the unsharded test selection, with unchanged pretest and production-environment guards.
|
||||
- The gate blocks failures, cancellations, and failed detection; deliberate skips and all-pass runs succeed.
|
||||
- Adopt sharding only with at least 30% lower median frontend execution critical path and at most 25% extra runner minutes. Otherwise retain unsharded CI and record the rejection.
|
||||
|
||||
## Verification
|
||||
|
||||
Run commands from the repository root. Install workspace dependencies first in a fresh worktree.
|
||||
|
||||
```bash
|
||||
python3 .github/scripts/frontend-tests-workflow-contract_test.py
|
||||
python3 .github/scripts/external-runner-workflow-contract_test.py
|
||||
python3 .github/scripts/runner-plan_test.py
|
||||
python3 .github/scripts/lint-action-pinning_test.py
|
||||
actionlint .github/workflows/frontend-tests.yml
|
||||
(cd apps/web && pnpm test --shard=1/2 --reporter=json --outputFile=/tmp/kandev-ci-shard-1.json)
|
||||
(cd apps/web && pnpm test --shard=2/2 --reporter=json --outputFile=/tmp/kandev-ci-shard-2.json)
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Run the same commands with shard indexes 1 through 4 for the four-shard candidate.
|
||||
Compare JSON file/test identities against the unsharded Task 03 report at the same snapshot.
|
||||
Use three comparable hosted runs per candidate for the performance decision; record all failed attempts too.
|
||||
Do not mark the performance criterion complete from local serial runs.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `.github/workflows/frontend-tests.yml`
|
||||
- `.github/scripts/frontend-tests-workflow-contract_test.py`
|
||||
- `.github/scripts/external-runner-workflow-contract_test.py`
|
||||
- `docs/specs/platform/system-design/external-e2e-runner-capacity.md (job inventory only)`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 03.
|
||||
|
||||
## Risks
|
||||
|
||||
Serial local shard commands prove selection only. Hosted comparable samples must establish speed and cost before adoption.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary.
|
||||
- [Plan](plan.md), baseline and companion-package status.
|
||||
- Existing workflow contract tests under `.github/scripts/`.
|
||||
|
||||
## Results
|
||||
|
||||
Implemented 2026-09-12. The frontend job keeps lint, typecheck, i18n checks,
|
||||
ratchets, the unsharded unit suite, and build. The two-instance
|
||||
`frontend_tests` matrix remains a measured candidate and is not enabled until
|
||||
the hosted adoption gate passes. `Frontend Tests Passed` continues to require
|
||||
the runner plan, change detection, and the single frontend job; failures and
|
||||
cancellations fail the gate while deliberate skips pass.
|
||||
|
||||
Local partition evidence, using the corrected `pnpm test -- --shard=...`
|
||||
invocation, is:
|
||||
|
||||
- shard 1/2: 1,000 files and 8,225 passed tests;
|
||||
- shard 2/2: 999 files, 8,954 passed tests, and 4 pending tests;
|
||||
- union: all 1,999 files and 17,136 assertion identities from the unsharded
|
||||
report, with zero overlap and no missing or unexpected identities;
|
||||
- workflow, runner-planner, action-pinning, and frontend contract tests passed.
|
||||
|
||||
The four-shard candidate and three comparable hosted runs were not executed.
|
||||
The 30% critical-path and 25% runner-minute adoption decision therefore stays
|
||||
open; the two-shard workflow remains a candidate pending hosted proof.
|
||||
@@ -0,0 +1,114 @@
|
||||
---
|
||||
id: "05-profile-remaining-costs"
|
||||
title: "Profile remaining CI costs"
|
||||
status: done
|
||||
wave: 5
|
||||
depends_on: ["04-partition-frontend-verification"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.2
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
---
|
||||
|
||||
# Task 05: Profile remaining CI costs
|
||||
|
||||
## Summary
|
||||
|
||||
Produce a focused report of E2E fixture/retry costs and Windows compile/test costs.
|
||||
Select exact follow-up targets from evidence without speculative source changes.
|
||||
|
||||
## In scope
|
||||
|
||||
- Attempt-specific job and step timings from representative recent runs.
|
||||
- Existing E2E timing/retry artifacts and Windows sensitive-package logs.
|
||||
- A ranked report with proposed targets, expected mechanism, confidence, and missing evidence.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Application fixes, release builds, cache redesign, reruns, workflow dispatch, and paid-capacity activation.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- Report approval, dependency, queue, and execution intervals separately, with unknowns explicit.
|
||||
- Identify the largest measured E2E and Windows costs and distinguish compilation, setup, test bodies, retries, and cache transfer.
|
||||
- Each follow-up names source/test targets and a reproduction command, or explicitly states why available evidence cannot support a change.
|
||||
|
||||
## Verification
|
||||
|
||||
Run commands from the repository root. Install workspace dependencies first in a fresh worktree.
|
||||
|
||||
```bash
|
||||
gh run list --repo kdlbs/kandev --workflow e2e-tests.yml --limit 20 --json databaseId,headSha,event,status,conclusion
|
||||
gh run list --repo kdlbs/kandev --workflow backend-tests.yml --limit 20 --json databaseId,headSha,event,status,conclusion
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check
|
||||
```
|
||||
|
||||
After selecting run IDs, record each attempt and download its jobs and artifacts through these read-only commands:
|
||||
|
||||
```bash
|
||||
: "${CI_RUN_ID:?Set the selected run ID}"
|
||||
: "${CI_RUN_ATTEMPT:?Set the selected attempt number}"
|
||||
gh api --paginate "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/attempts/$CI_RUN_ATTEMPT/jobs?per_page=100" > /tmp/kandev-ci-profile-jobs.json
|
||||
gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/artifacts?per_page=100"
|
||||
```
|
||||
|
||||
The artifact producer must publish an attempt-qualified name. Reject the
|
||||
unqualified `e2e-timing-diagnostics` name because a rerun can publish the same
|
||||
name for another attempt. Select and download only the exact name below:
|
||||
|
||||
```bash
|
||||
: "${CI_ARTIFACT_DIR:?Set an empty temporary artifact directory}"
|
||||
CI_ARTIFACT_NAME="e2e-timing-diagnostics-attempt-${CI_RUN_ATTEMPT}"
|
||||
ARTIFACTS_JSON="$(gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/artifacts?per_page=100")"
|
||||
test "$(jq --arg name "$CI_ARTIFACT_NAME" '[.artifacts[] | select(.name == $name and .expired == false)] | length' <<<"$ARTIFACTS_JSON")" = 1
|
||||
gh run download "$CI_RUN_ID" --repo kdlbs/kandev --name "$CI_ARTIFACT_NAME" --dir "$CI_ARTIFACT_DIR"
|
||||
```
|
||||
|
||||
Check the downloaded manifest against both `CI_RUN_ID` and `CI_RUN_ATTEMPT`.
|
||||
Reject the artifact when the exact name is absent, duplicated, expired, or the
|
||||
manifest identifies another run or attempt. Do not mix retry attempts silently.
|
||||
Select the Windows job ID from the jobs response, then use the job logs endpoint for package and compile timing.
|
||||
Delete owned temporary logs after the curated report is complete.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `docs/plans/ci-performance/evidence.md (new)`
|
||||
- `docs/plans/e2e-ci-efficiency/plan.md (relevant evidence links only)`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 04.
|
||||
|
||||
## Risks
|
||||
|
||||
Do not treat missing logs as zero execution, or successful-only samples as reliability evidence.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary.
|
||||
- [Plan](plan.md), baseline and companion-package status.
|
||||
- Existing workflow contract tests under `.github/scripts/`.
|
||||
|
||||
## Results
|
||||
|
||||
Completed 2026-09-12. Added [CI performance evidence](evidence.md) with
|
||||
attempt-specific run and job timestamps, approval versus queue versus runner
|
||||
execution intervals, frontend cache failure evidence, E2E timing and retry
|
||||
artifact results, and Windows compile, race-test, and cache-save timings. The
|
||||
report ranks E2E fixture and retry profiling, Windows package profiling, cache
|
||||
verification, and runner capacity measurement. It names reproduction commands
|
||||
and records unknowns instead of treating missing jobs or logs as zero time.
|
||||
|
||||
The report uses read-only data from runs 34355720657, 34025129928,
|
||||
34689871445, 34027695660, 34687600985, and 34692849510. Temporary raw logs
|
||||
and downloaded artifacts remain outside the repository.
|
||||
@@ -0,0 +1,85 @@
|
||||
---
|
||||
id: "06-prepare-capacity-pilot"
|
||||
title: "Prepare the runner capacity pilot"
|
||||
status: done
|
||||
wave: 6
|
||||
depends_on: ["05-profile-remaining-costs"]
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
acceptance_criteria:
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.1
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.2
|
||||
- AC-PLATFORM-CI-PERFORMANCE-004.3
|
||||
system_design:
|
||||
- ../../specs/platform/system-design/ci-performance.md
|
||||
---
|
||||
|
||||
# Task 06: Prepare the runner capacity pilot
|
||||
|
||||
## Summary
|
||||
|
||||
Prepare a concrete 20% capacity pilot using the existing runner controls.
|
||||
Document cost assumptions, protected jobs, measurement, and rollback without changing live variables.
|
||||
|
||||
## In scope
|
||||
|
||||
- Current eligible-family inventory, including any accepted frontend test matrix.
|
||||
- Read-only variable snapshot and provider configuration prerequisites.
|
||||
- Operator procedure for activation, measurement, rollback, and an explicit budget input.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Installing provider apps, purchasing capacity, changing variables, dispatching workflows, or moving review/Cargo/protected jobs.
|
||||
|
||||
## Acceptance
|
||||
|
||||
- The procedure preserves the current external-runner trust boundary and names jobs that remain hosted.
|
||||
- It specifies a 20% pilot, at least three representative runs, separate queue/execution metrics, and no expansion without measured evidence.
|
||||
- It includes explicit activation/rollback commands and cost calculations. Unknown provider rates or an absent operator budget block activation, not documentation completion.
|
||||
|
||||
## Verification
|
||||
|
||||
Run commands from the repository root. Install workspace dependencies first in a fresh worktree.
|
||||
|
||||
```bash
|
||||
gh variable list --repo kdlbs/kandev --json name,value --jq '.[] | select(.name | startswith("KANDEV_CI_"))'
|
||||
python3 .github/scripts/runner-plan_test.py
|
||||
python3 .github/scripts/external-runner-workflow-contract_test.py
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `docs/ci-merge-queue.md`
|
||||
- `docs/plans/ci-performance/evidence.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 05.
|
||||
|
||||
## Risks
|
||||
|
||||
Percentage allocation controls job instances, not a fixed share of billed minutes. Changing variables affects only newly dispatched jobs.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter.
|
||||
- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary.
|
||||
- [Plan](plan.md), baseline and companion-package status.
|
||||
- Existing workflow contract tests under `.github/scripts/`.
|
||||
|
||||
## Results
|
||||
|
||||
Completed 2026-09-12. Updated the merge-queue runbook with the current
|
||||
variable snapshot, eligible and protected job families, a 20% activation and
|
||||
rollback procedure, three-run measurement fields, queue and execution
|
||||
separation, and provider-rate and budget prerequisites. The procedure includes
|
||||
the external runner-minute cost formula and states that unknown rates or budget
|
||||
block activation. No live variable, provider setting, workflow dispatch, or
|
||||
paid capacity changed.
|
||||
@@ -31,6 +31,92 @@ retains ownership of its values, validation, authority, and persistence.
|
||||
- Agent identity belongs to the [agent system](../agents/README.md).
|
||||
- Desktop shell behavior belongs to the [desktop system](../desktop/README.md).
|
||||
|
||||
## Specification map
|
||||
|
||||
### Requirements
|
||||
|
||||
- [CI performance](requirements/ci-performance.md)
|
||||
|
||||
- [Startup lifecycle](requirements/startup-lifecycle.md)
|
||||
|
||||
- [Agent-accessible Kandev settings](requirements/agent-settings-parity.md)
|
||||
|
||||
|
||||
|
||||
- [Agent process exit and stderr drain](requirements/agent-process-exit-drain.md)
|
||||
- [Agentctl instance stop idempotency](requirements/agentctl-instance-stop.md)
|
||||
- [Agent Runtime Availability](requirements/agent-runtime-availability.md)
|
||||
- [Background Work Liveness](requirements/background-work-liveness.md)
|
||||
- [Backend restart page recovery](requirements/backend-restart-page-recovery.md)
|
||||
- [Bounded Task Status Delivery](requirements/bounded-task-status-delivery.md)
|
||||
- [Viewport-bounded Session Delivery](requirements/viewport-bounded-session-delivery.md)
|
||||
- [Shared Cron Loop Safety When Office Is Disabled](requirements/cron-office-disabled-safety.md)
|
||||
- [Environment-specific browser tab title prefixes](requirements/dev-preview-title-prefixes.md)
|
||||
- [Browser console retention](requirements/browser-console-retention.md)
|
||||
- [Diagnostic logging](requirements/diagnostic-logging.md)
|
||||
- [Duration-aware E2E sharding and CI reliability](requirements/e2e-duration-aware-sharding.md)
|
||||
- [External CI runner capacity](requirements/external-e2e-runner-capacity.md)
|
||||
- [Expected runtime log severity](requirements/expected-runtime-log-severity.md)
|
||||
- [Feature Toggles](requirements/feature-toggles.md)
|
||||
- [Git Credential Lease Reissue](requirements/git-credential-lease-reissue.md)
|
||||
- [Git Subprocess Admission](requirements/git-subprocess-admission.md)
|
||||
- [Go dev launcher and minimal Node surface](requirements/go-dev-launcher.md)
|
||||
- [Health Endpoint — Surface the Running Version](requirements/health-endpoint-version.md)
|
||||
- [Watcher And Task Fallback Localization](requirements/i18n-audit-watcher-copy.md)
|
||||
- [Additional I18n Audit Gaps](requirements/i18n-second-audit-gaps.md)
|
||||
- [Internationalization (i18n)](requirements/i18n.md)
|
||||
- [LSP File Intelligence](requirements/lsp-file-intelligence.md)
|
||||
- [Session MCP Attachment Observability](requirements/mcp-session-observability.md)
|
||||
- [Mid-Turn Steering](requirements/mid-turn-steering.md)
|
||||
- [Semantic Notifications](requirements/notifications.md)
|
||||
- [Apprise rescan](requirements/apprise-rescan.md)
|
||||
- [Required Persisted Store Parity](requirements/postgres-domain-store-parity.md)
|
||||
- [Provider Error Recovery](requirements/provider-error-recovery.md)
|
||||
- [Session Config Reconciliation Across Agent Types](requirements/session-config-cross-agent-reconcile.md)
|
||||
- [Session subscription recovery](requirements/session-subscription-recovery.md)
|
||||
- [Setup and Launch Timeout](requirements/setup-launch-timeout.md)
|
||||
- [Quiet benign teardown log noise on shutdown](requirements/shutdown-log-noise.md)
|
||||
- [Do not surface backend-shutdown turn aborts as agent failures](requirements/shutdown-turn-failure-suppression.md)
|
||||
- [Startup Configuration Parity](requirements/startup-configuration-parity.md)
|
||||
- [Prevent Host Sleep During Active Tasks](requirements/task-sleep-inhibition.md)
|
||||
- [Traditional Chinese locales (Taiwan and Hong Kong)](requirements/traditional-chinese-locales.md)
|
||||
- [Workspace Git Status](requirements/workspace-git-status.md)
|
||||
|
||||
### System design
|
||||
|
||||
- [CI performance](system-design/ci-performance.md)
|
||||
|
||||
- [Startup lifecycle](system-design/startup-lifecycle.md)
|
||||
|
||||
- [Agent-accessible Kandev settings](system-design/agent-settings-parity.md)
|
||||
- [Core settings domain adoption](system-design/agent-settings-domains.md)
|
||||
|
||||
|
||||
|
||||
- [Agent process exit and stderr drain](system-design/agent-process-exit-drain.md)
|
||||
- [Agentctl instance stop idempotency](system-design/agentctl-instance-stop.md)
|
||||
- [Apprise rescan](system-design/apprise-rescan.md)
|
||||
- [Background Work Liveness](system-design/background-work-liveness.md)
|
||||
- [Backend restart page recovery](system-design/backend-restart-page-recovery.md)
|
||||
- [Browser console retention](system-design/browser-console-retention.md)
|
||||
- [Bounded Task Status Delivery](system-design/bounded-task-status-delivery.md)
|
||||
- [Viewport-bounded Session Delivery](system-design/viewport-bounded-session-delivery.md)
|
||||
- [Diagnostic logging System Design Part 1](system-design/diagnostic-logging-01.md)
|
||||
- [Diagnostic logging System Design Part 2](system-design/diagnostic-logging-02.md)
|
||||
- [Expected runtime log severity](system-design/expected-runtime-log-severity.md)
|
||||
- [Duration-aware E2E sharding and CI reliability](system-design/e2e-duration-aware-sharding.md)
|
||||
- [External CI runner capacity](system-design/external-e2e-runner-capacity.md)
|
||||
- [Health Endpoint — Surface the Running Version](system-design/health-endpoint-version.md)
|
||||
- [Internationalization (i18n)](system-design/i18n.md)
|
||||
- [LSP File Intelligence System Design Part 1](system-design/lsp-file-intelligence-01.md)
|
||||
- [LSP File Intelligence System Design Part 2](system-design/lsp-file-intelligence-02.md)
|
||||
- [Session MCP Attachment Observability](system-design/mcp-session-observability.md)
|
||||
- [Required Persisted Store Parity](system-design/postgres-domain-store-parity.md)
|
||||
- [Provider Error Recovery](system-design/provider-error-recovery.md)
|
||||
- [Workspace Git Status](system-design/workspace-git-status.md)
|
||||
- [Go dev launcher and startup version](system-design/go-dev-launcher.md)
|
||||
- [Startup database selection continuity](system-design/startup-database-selection-continuity.md)
|
||||
|
||||
## Migration record
|
||||
|
||||
Migration remains in progress while legacy source detail is extracted from the
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
---
|
||||
status: draft
|
||||
system: platform
|
||||
created: 2026-09-12
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# CI performance requirements
|
||||
|
||||
## Overview
|
||||
|
||||
Maintainers need bounded review execution and shorter CI feedback without lost test coverage.
|
||||
Platform owns these shared operational guarantees across repository workflows.
|
||||
The CI automation system retains ownership of contributor authorization and approval rules.
|
||||
The existing external-runner contract retains ownership of fleet selection and trust boundaries.
|
||||
|
||||
## Terminology
|
||||
|
||||
- **Execution:** The interval from a job start to its completion.
|
||||
- **Queue delay:** The interval from job creation to job start.
|
||||
- **Workflow elapsed time:** The interval from workflow creation to completion, including approval and dependency waits.
|
||||
- **Comparable runs:** Runs with the same source snapshot, test selection, runner class, toolchain, and recorded cache state.
|
||||
|
||||
## Requirements
|
||||
|
||||
### REQ-PLATFORM-CI-PERFORMANCE-001: Bounded Claude execution
|
||||
|
||||
**Intent:** Stop stalled review jobs without changing contributor approval rules.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-001.1:** Each automatic or comment-triggered Claude job shall have a 30-minute execution budget. GitHub shall cancel jobs that exceed that budget.
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-001.2:** A timeout shall not count as a successful review. Existing authorization, trigger, credential, and runner boundaries shall remain unchanged.
|
||||
|
||||
### REQ-PLATFORM-CI-PERFORMANCE-002: Reusable frontend dependency cache
|
||||
|
||||
**Intent:** Reuse compatible dependencies without making test results depend on cache availability.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-002.1:** A successful frontend dependency installation shall support cache reuse by a later compatible run. Evidence shall show an actual save and restore.
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-002.2:** A missing or unavailable cache shall permit a complete dependency installation. Dependency resolution shall continue to enforce the lockfile.
|
||||
|
||||
### REQ-PLATFORM-CI-PERFORMANCE-003: Complete frontend verification with less overhead
|
||||
|
||||
**Intent:** Reduce repeated test setup and shorten feedback without weakening verification.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-003.1:** Optimized frontend verification shall retain every selected test file and test case exactly once across its test partitions.
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-003.2:** Browser, multilingual, and production-environment regression tests shall retain their existing behavior. Test files shall retain isolated state.
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-003.3:** Required frontend verification shall fail when any selected partition or mandatory check fails or is cancelled. Deliberate change-based skips shall remain valid.
|
||||
|
||||
### REQ-PLATFORM-CI-PERFORMANCE-004: Attributable performance evidence
|
||||
|
||||
**Intent:** Distinguish useful optimization from queue variation or reduced coverage.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-004.1:** Performance reports shall distinguish job execution, queue delay, and workflow elapsed time. Missing job evidence shall appear as unknown execution.
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-004.2:** Optimization comparisons shall identify source, attempts, runner class, cache state, test counts, failures, retries, and sample size.
|
||||
- **AC-PLATFORM-CI-PERFORMANCE-004.3:** Runner-capacity assessments shall preserve the existing eligible-job boundary. They shall report cost assumptions and rollback conditions before activation.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Automatic approval, automatic reruns, or changes to contributor trust policy.
|
||||
- Automatic paid-capacity activation or expansion to protected runner workloads.
|
||||
- Lower test coverage, disabled isolation, increased retries, or increased E2E worker counts.
|
||||
- Application UI changes, release build redesign, or changes to an externally managed dashboard.
|
||||
|
||||
## Related contracts
|
||||
|
||||
- [External CI runner capacity](external-e2e-runner-capacity.md)
|
||||
- [Contributor automation](../../ci/requirements/unified-contributor-pr-automation.md)
|
||||
|
||||
## Implementation plans
|
||||
|
||||
- [CI performance](../../../plans/ci-performance/plan.md)
|
||||
@@ -0,0 +1,150 @@
|
||||
---
|
||||
status: draft
|
||||
system: platform
|
||||
created: 2026-09-12
|
||||
requirements:
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-001
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-002
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-003
|
||||
- REQ-PLATFORM-CI-PERFORMANCE-004
|
||||
---
|
||||
|
||||
# CI performance system design
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
This design changes repository CI execution and test infrastructure. It does not change application behavior.
|
||||
The [external-runner design](external-e2e-runner-capacity.md) remains authoritative for provider allocation and protected jobs.
|
||||
The [CI automation system](../../ci/README.md) retains contributor trust ownership.
|
||||
No database, public API, or runtime feature flag changes are required.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design section |
|
||||
| --- | --- |
|
||||
| REQ-PLATFORM-CI-PERFORMANCE-001 | Review budget |
|
||||
| REQ-PLATFORM-CI-PERFORMANCE-002 | Dependency cache |
|
||||
| REQ-PLATFORM-CI-PERFORMANCE-003 | Test setup and workflow partitions |
|
||||
| REQ-PLATFORM-CI-PERFORMANCE-004 | Measurement and capacity assessment |
|
||||
|
||||
## Review budget
|
||||
|
||||
Set job-level `timeout-minutes: 30` on `claude-review-same-repo` and `claude-review-fork` in `.github/workflows/claude-code-review.yml`.
|
||||
Apply the same budget to the `claude` job in `.github/workflows/claude.yml`.
|
||||
The comment workflow also handles general Claude requests. Its budget therefore applies to every invocation, including requests other than review.
|
||||
Leave the short fork-label job unchanged.
|
||||
|
||||
Keep permissions, prompts, action pins, checkouts, approval expressions, concurrency, and event filters unchanged.
|
||||
Do not add `continue-on-error`, automatic retries, or success markers after cancellation.
|
||||
GitHub owns timeout cancellation and cleanup latency. The budget is not a promise about total workflow elapsed time.
|
||||
Do not use the retired Claude action `timeout_minutes` input.
|
||||
An explicit turn cap remains deferred until completed-review turn counts support a useful threshold.
|
||||
|
||||
## Dependency cache
|
||||
|
||||
In `.github/workflows/frontend-tests.yml`, resolve `pnpm store path --silent` before the cache restore.
|
||||
Run resolution from `apps/` with the same container, user, environment, and pnpm version as installation.
|
||||
Expose the absolute path as a step output and use that output in the pinned cache action.
|
||||
Do not infer the path from `HOME`, or change `HOME` to fit the cache.
|
||||
|
||||
Use a versioned key containing runner OS, architecture, pnpm version, and lockfile hash.
|
||||
A restore prefix can omit only the lockfile hash. Preserve pnpm-major and platform compatibility.
|
||||
Create the store directory before restoration if necessary. Cache-service errors remain best effort.
|
||||
The installation itself must fail on lockfile or dependency errors.
|
||||
|
||||
Keep cache keys and paths identical in any frontend jobs introduced by partitioning.
|
||||
Do not copy `node_modules` between runners or share mutable working directories.
|
||||
Limit the first repair to the confirmed frontend failure. Inventory matching E2E paths in the evidence report without changing those workflows speculatively.
|
||||
|
||||
## Test setup and workflow partitions
|
||||
|
||||
### Setup boundaries
|
||||
|
||||
`apps/web/vitest.config.ts` currently applies `happy-dom` and `vitest.setup.ts` to every test file.
|
||||
`vitest.setup.ts` calls `initI18nForTests()` and awaits `loadAllLocalesForTests()` for each isolated file.
|
||||
The frontend log shows substantial setup, import, and environment costs.
|
||||
|
||||
Introduce explicit Vitest projects for Node-compatible helpers and browser-dependent tests.
|
||||
Start with a reviewed list of pure helper files. Do not infer environment requirements from `.ts` versus `.tsx` extensions.
|
||||
Keep all unclassified files in the full browser-locale project. A partition
|
||||
contract must detect overlap or omitted files.
|
||||
Node setup must not import React, DOM globals, or locale catalogs unless a selected test requires them.
|
||||
|
||||
Keep English initialization for browser tests. Move complete locale loading into an explicit multilingual setup path.
|
||||
Inventory tests that call `changeLanguage`, `loadAllLocalesForTests`, or inspect catalog state, including indirect helper calls.
|
||||
Their setup must finish before test execution. Default unknown cases to full setup until their dependencies are established.
|
||||
Preserve the `NODE_ENV=test` pin, the React `act` guard, Monaco alias, inert WebSocket, cleanup hooks, and `passWithNoTests: false`.
|
||||
Keep test isolation and local worker-budget rules unchanged.
|
||||
|
||||
### Workflow partitions
|
||||
|
||||
After setup measurements, benchmark two and four Vitest shards on the same runner class.
|
||||
Use built-in Vitest file sharding. Select two shards initially only if it passes the measurement gate below.
|
||||
Use four only when it provides a further measured benefit within the compute budget.
|
||||
|
||||
Keep the existing `frontend` job for formatting, licenses, lint, typecheck, SDK checks, i18n checks, ratchets, the unsharded unit suite, and build until the adoption gate passes.
|
||||
After successful change detection and a passing measurement gate, an optional
|
||||
`frontend_tests` matrix may split unit tests. Until then, the matrix remains a
|
||||
candidate and must not become required CI.
|
||||
The active job and any measured candidate retain the existing pinned container
|
||||
and repaired cache. Each test command retains `NODE_ENV: production`.
|
||||
Run tests through the package script so `pretest` generation still executes.
|
||||
|
||||
If adopted, register the matrix through `.github/actions/plan-external-runners`
|
||||
using its existing family schema. Use the standard tier without changing
|
||||
protected-job placement or percentage semantics. Extend `frontend-gate` to
|
||||
require the planner, change detection, `frontend`, and all test shards only
|
||||
after adoption.
|
||||
Keep its public name `Frontend Tests Passed`, `if: always()`, and deliberate-skip handling.
|
||||
Keep `pull_request`, `push`, `merge_group`, and dispatch coverage. Do not add trigger-level path filters.
|
||||
|
||||
Collect unique JSON reports per shard with test file identities and test case counts.
|
||||
Compare their union against an unsharded run at the same source snapshot.
|
||||
Failures and cancelled shards must still block the gate. Reports must not mask a failed test exit code.
|
||||
|
||||
## Measurement and capacity assessment
|
||||
|
||||
Save curated evidence in the implementation package. Keep raw logs outside version control.
|
||||
Record run ID, attempt, head SHA, event, workflow version, job ID, runner label, container digest, and tool versions.
|
||||
Record job `created_at`, `started_at`, and `completed_at` separately.
|
||||
Execution equals completion minus start. Queue delay equals start minus job creation.
|
||||
Exclude skipped jobs with invalid timestamps from duration calculations.
|
||||
Do not label workflow `run_started_at` as job execution.
|
||||
Approval and dependency waits can precede job creation. Report them separately when evidence exists, otherwise leave the cause unknown.
|
||||
Use attempt-specific job data for reruns. Do not add concurrent job durations to obtain workflow wall time.
|
||||
|
||||
Use at least three comparable successful runs per candidate and baseline for performance decisions.
|
||||
Compare medians and individual samples. Do not present three samples as a reliable p90 estimate.
|
||||
Record failures and retries from all attempted samples, not only successful ones.
|
||||
For setup optimization, require lower median unit-test execution without reduced selection or new failures.
|
||||
For sharding, target at least 30% lower median frontend execution critical path and no more than 25% additional frontend runner minutes.
|
||||
Measure queue time separately. Reject a partition that consistently worsens total feedback under representative load.
|
||||
These are candidate-retention gates, not promised speedups. Record rejected candidates and keep the simpler passing configuration.
|
||||
|
||||
Inspect existing E2E timing and retry artifacts before selecting fixture changes.
|
||||
Profile the Windows job by compile, package-test, and cache intervals. Preserve race checks and operating-system coverage.
|
||||
The profiling work order produces a ranked follow-up report with exact targets. It does not authorize unidentified source changes.
|
||||
|
||||
Prepare a capacity pilot using the existing repository variables and approved tiers.
|
||||
Record current variables again before proposing activation. The investigation snapshot had burst mode disabled and percentage set to 20.
|
||||
Describe a 20% pilot, cost assumptions, sample count, rollback, and operator commands in the existing merge-queue runbook.
|
||||
Do not activate paid capacity as part of documentation or repository implementation.
|
||||
Review and Cargo Audit remain hosted; this pilot does not directly move their jobs.
|
||||
|
||||
## Failure and recovery
|
||||
|
||||
A cold cache installs dependencies normally. A failed test partition fails the existing required gate.
|
||||
A timeout cancels Claude execution without a success fallback. Approval expiration remains an approval outcome.
|
||||
Revert a performance candidate that loses coverage or exceeds its measured budget.
|
||||
Runner rollback follows the existing external-runner design and affects new jobs only.
|
||||
|
||||
## Related decisions and plans
|
||||
|
||||
- [External-runner decision](../../../decisions/2026-09-06-opt-in-external-e2e-runners.md)
|
||||
- [Existing runner package](../../../plans/external-e2e-runner-capacity/plan.md)
|
||||
- [Existing E2E efficiency package](../../../plans/e2e-ci-efficiency/plan.md)
|
||||
- [CI performance package](../../../plans/ci-performance/plan.md)
|
||||
|
||||
The existing runner package is complete. It needs no repeated implementation.
|
||||
The E2E efficiency package retains open rollout evidence. Link new measurements there without marking unperformed rollout work complete.
|
||||
No new architectural decision is required: this package preserves existing trust and isolation boundaries.
|
||||
Reference in New Issue
Block a user