From 2c4058c305210abd49cb83b37021a856a2da11fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Sun, 13 Sep 2026 12:06:08 +0100 Subject: [PATCH] ci: improve frontend verification and review timeouts (#3634) --- ...aude-code-review-workflow-contract_test.py | 31 +++ .../external-runner-workflow-contract_test.py | 5 +- .../frontend-tests-workflow-contract_test.py | 88 ++++++++ .github/scripts/runner-plan_test.py | 17 ++ .github/workflows/claude-code-review.yml | 2 + .github/workflows/claude.yml | 1 + .github/workflows/frontend-tests.yml | 41 ++-- .../scripts/vitest-project-selection.test.ts | 150 +++++++++++++ apps/web/scripts/vitest-project-selection.ts | 180 ++++++++++++++++ apps/web/vitest.config.ts | 74 +++++-- apps/web/vitest.setup.locales.ts | 3 + apps/web/vitest.setup.node.ts | 5 + apps/web/vitest.setup.ts | 12 +- docs/ci-merge-queue.md | 128 ++++++++---- docs/plans/ci-performance/evidence.md | 197 ++++++++++++++++++ docs/plans/ci-performance/plan.md | 151 ++++++++++++++ .../task-01-bound-claude-execution.md | 91 ++++++++ .../task-02-repair-frontend-cache.md | 109 ++++++++++ .../task-03-reduce-test-setup.md | 106 ++++++++++ ...task-04-partition-frontend-verification.md | 112 ++++++++++ .../task-05-profile-remaining-costs.md | 114 ++++++++++ .../task-06-prepare-capacity-pilot.md | 85 ++++++++ docs/specs/platform/README.md | 86 ++++++++ .../platform/requirements/ci-performance.md | 79 +++++++ .../platform/system-design/ci-performance.md | 150 +++++++++++++ 25 files changed, 1936 insertions(+), 81 deletions(-) create mode 100644 apps/web/scripts/vitest-project-selection.test.ts create mode 100644 apps/web/scripts/vitest-project-selection.ts create mode 100644 apps/web/vitest.setup.locales.ts create mode 100644 apps/web/vitest.setup.node.ts create mode 100644 docs/plans/ci-performance/evidence.md create mode 100644 docs/plans/ci-performance/plan.md create mode 100644 docs/plans/ci-performance/task-01-bound-claude-execution.md create mode 100644 docs/plans/ci-performance/task-02-repair-frontend-cache.md create mode 100644 docs/plans/ci-performance/task-03-reduce-test-setup.md create mode 100644 docs/plans/ci-performance/task-04-partition-frontend-verification.md create mode 100644 docs/plans/ci-performance/task-05-profile-remaining-costs.md create mode 100644 docs/plans/ci-performance/task-06-prepare-capacity-pilot.md create mode 100644 docs/specs/platform/requirements/ci-performance.md create mode 100644 docs/specs/platform/system-design/ci-performance.md diff --git a/.github/scripts/claude-code-review-workflow-contract_test.py b/.github/scripts/claude-code-review-workflow-contract_test.py index 0552bbc07a..d125c3ca4a 100644 --- a/.github/scripts/claude-code-review-workflow-contract_test.py +++ b/.github/scripts/claude-code-review-workflow-contract_test.py @@ -31,6 +31,16 @@ def workflow_step(workflow: str, name: str) -> str: return remainder.partition("\n - name:")[0] +def job_block(workflow: str, job: str, next_job: str | None) -> str: + marker = f" {job}:\n" + _, separator, remainder = workflow.partition(marker) + if not separator: + raise AssertionError(f"Workflow has no {job} job") + if next_job is None: + return remainder + return remainder.partition(f"\n {next_job}:\n")[0] + + class ClaudeCodeReviewWorkflowContractTest(unittest.TestCase): def test_review_workflow_ignores_pr_updates(self) -> None: workflow = WORKFLOW.read_text(encoding="utf-8") @@ -67,6 +77,27 @@ class ClaudeCodeReviewWorkflowContractTest(unittest.TestCase): workflow, ) + def test_claude_execution_jobs_have_a_thirty_minute_budget(self) -> None: + review_workflow = WORKFLOW.read_text(encoding="utf-8") + for job, next_job in ( + ("claude-review-same-repo", "label-allowlisted-fork"), + ("claude-review-fork", None), + ): + block = job_block(review_workflow, job, next_job) + self.assertRegex( + block, + r"runs-on: ubuntu-latest\n timeout-minutes: 30\n", + f"{job} must bound Claude execution at 30 minutes", + ) + + mention_workflow = MENTION_WORKFLOW.read_text(encoding="utf-8") + block = job_block(mention_workflow, "claude", None) + self.assertRegex( + block, + r"runs-on: ubuntu-latest\n timeout-minutes: 30\n", + "the shared interactive Claude job must bound execution at 30 minutes", + ) + def test_manual_pr_review_does_not_checkout_untrusted_content(self) -> None: workflow = MENTION_WORKFLOW.read_text(encoding="utf-8") diff --git a/.github/scripts/external-runner-workflow-contract_test.py b/.github/scripts/external-runner-workflow-contract_test.py index 14ee5177db..9013547d92 100644 --- a/.github/scripts/external-runner-workflow-contract_test.py +++ b/.github/scripts/external-runner-workflow-contract_test.py @@ -107,7 +107,7 @@ class ExternalRunnerWorkflowContractTest(unittest.TestCase): self.assertIn("runs-on: ubuntu-latest" if job == "postgres-boot" else "runs-on: windows-latest", protected) self.assertNotIn("runner_plan", protected) - def test_frontend_uses_planner_for_test_and_gate(self) -> None: + def test_frontend_uses_planner_for_unsharded_tests_and_gate(self) -> None: workflow = self.assert_planner( "frontend-tests.yml", "frontend", @@ -121,6 +121,9 @@ class ExternalRunnerWorkflowContractTest(unittest.TestCase): "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_runner }}", job_block(workflow, "frontend", "frontend-gate"), ) + frontend = job_block(workflow, "frontend", "frontend-gate") + self.assertNotIn("matrix:", frontend) + self.assertNotIn("frontend_tests", workflow) self.assertIn( "runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }}", job_block(workflow, "frontend-gate", None), diff --git a/.github/scripts/frontend-tests-workflow-contract_test.py b/.github/scripts/frontend-tests-workflow-contract_test.py index a846c266ca..a7aa9d28c2 100644 --- a/.github/scripts/frontend-tests-workflow-contract_test.py +++ b/.github/scripts/frontend-tests-workflow-contract_test.py @@ -40,6 +40,22 @@ def run_tests_step(workflow: str) -> str: return remainder.partition(NEXT_STEP_MARKER)[0] +def frontend_job(workflow: str) -> str: + """Return the frontend job body, up to the required gate.""" + _, separator, remainder = workflow.partition(" frontend:\n") + if not separator: + raise AssertionError("frontend-tests.yml has no frontend job") + return remainder.partition("\n frontend-gate:")[0] + + +def frontend_gate_job(workflow: str) -> str: + """Return the required frontend gate body.""" + _, separator, remainder = workflow.partition(" frontend-gate:\n") + if not separator: + raise AssertionError("frontend-tests.yml has no frontend-gate job") + return remainder + + def trigger_block(workflow: str, trigger: str) -> str: """Return `trigger`'s block under `on:`, up to the next top-level key.""" block = re.search(rf"(?m)^ {trigger}:\n(?:^ {{4}}.*\n|^\n)*", workflow) @@ -49,6 +65,78 @@ def trigger_block(workflow: str, trigger: str) -> str: class FrontendTestsWorkflowContractTest(unittest.TestCase): + def test_cache_resolves_the_container_pnpm_store_before_restore(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + job = frontend_job(workflow) + resolve_index = job.find("- name: Resolve pnpm store path") + cache_index = job.find("- name: Cache pnpm store") + install_index = job.find("- name: Install dependencies") + + self.assertGreaterEqual(resolve_index, 0, "store resolution step is missing") + self.assertGreaterEqual(cache_index, 0, "cache step is missing") + self.assertGreaterEqual(install_index, 0, "dependency installation step is missing") + self.assertLess(resolve_index, cache_index) + self.assertLess(cache_index, install_index) + resolve_step = job[resolve_index:cache_index] + cache_step = job[cache_index:install_index] + + self.assertIn("id: pnpm-store", resolve_step) + self.assertIn("pnpm store path --silent", resolve_step) + self.assertIn('pnpm --version', resolve_step) + self.assertIn('mkdir -p "${STORE_PATH}"', resolve_step) + self.assertIn('printf \'path=%s\\n\' "${STORE_PATH}"', resolve_step) + self.assertIn("path: ${{ steps.pnpm-store.outputs.path }}", cache_step) + self.assertIn("runner.os", cache_step) + self.assertIn("runner.arch", cache_step) + self.assertIn("steps.pnpm-store.outputs.version", cache_step) + self.assertIn("hashFiles('apps/pnpm-lock.yaml')", cache_step) + self.assertIn( + "restore-keys: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-store.outputs.version }}-", + cache_step, + ) + self.assertIn("continue-on-error: true", cache_step) + + def test_cache_failure_does_not_replace_frozen_installation(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + job = frontend_job(workflow) + cache_index = job.index("- name: Cache pnpm store") + install_index = job.index("- name: Install dependencies") + install_step = job[install_index:].partition(NEXT_STEP_MARKER)[0] + + self.assertLess(cache_index, install_index) + self.assertIn("pnpm install --frozen-lockfile", install_step) + + def test_frontend_job_keeps_unsharded_unit_tests_and_static_checks(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + job = frontend_job(workflow) + + self.assertIn("- name: Run tests\n", job) + self.assertIn("NODE_ENV: production", job) + self.assertIn("pnpm --filter @kandev/web test", job) + self.assertIn("- name: Run lint\n", job) + self.assertIn("- name: Run typecheck\n", job) + self.assertIn("- name: Build\n", job) + self.assertNotIn("FRONTEND_TEST_SHARD", job) + self.assertNotIn("matrix:", job) + + def test_frontend_tests_remain_outside_the_planner_until_adoption_evidence(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + planner = workflow.partition(" changes:\n")[0] + + self.assertNotIn('"name":"frontend_tests"', planner) + self.assertNotIn(" frontend_tests:\n", workflow) + + def test_required_gate_requires_every_frontend_verification_result(self) -> None: + workflow = WORKFLOW.read_text(encoding="utf-8") + gate = frontend_gate_job(workflow) + + self.assertIn( + "needs: [changes, frontend, runner_plan]", + gate, + ) + self.assertNotIn("FRONTEND_TESTS_RESULT:", gate) + self.assertNotIn("Frontend test matrix finished with result:", gate) + def test_test_step_reproduces_the_production_environment(self) -> None: step = run_tests_step(WORKFLOW.read_text(encoding="utf-8")) diff --git a/.github/scripts/runner-plan_test.py b/.github/scripts/runner-plan_test.py index 8420e3180d..1bce2a7d96 100644 --- a/.github/scripts/runner-plan_test.py +++ b/.github/scripts/runner-plan_test.py @@ -23,6 +23,11 @@ E2E_FAMILIES = [ ] FRONTEND_FAMILIES = [ {"name": "frontend", "tier": "standard"}, + { + "name": "frontend_tests", + "tier": "standard", + "instances": [{"shard": 1, "total": 2}, {"shard": 2, "total": 2}], + }, {"name": "frontend_gate", "tier": "light"}, ] BACKEND_MATRIX_FAMILY = [ @@ -62,6 +67,12 @@ class RunnerPlanTest(unittest.TestCase): self.assertEqual(plan.plan["frontend_runner"], "ubuntu-latest") self.assertEqual(plan.plan["frontend_gate_runner"], "ubuntu-latest") + self.assertTrue( + all( + item["runner"] == "ubuntu-latest" + for item in plan.plan["frontend_tests_matrix"]["include"] + ) + ) def test_matrix_percentage_uses_floor_and_is_stable(self) -> None: first = runner_plan.build_plan( @@ -217,6 +228,12 @@ class RunnerPlanTest(unittest.TestCase): self.assertEqual(plan.plan["frontend_runner"], "standard-runner") self.assertEqual(plan.plan["frontend_gate_runner"], "light-runner") + self.assertTrue( + all( + item["runner"] == "standard-runner" + for item in plan.plan["frontend_tests_matrix"]["include"] + ) + ) if __name__ == "__main__": diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 77b0ce8274..ecbd5f0b74 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -33,6 +33,7 @@ jobs: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read pull-requests: write @@ -124,6 +125,7 @@ jobs: vars.CLAUDE_REVIEW_ALLOWLIST != '' && contains(fromJSON(vars.CLAUDE_REVIEW_ALLOWLIST), github.event.pull_request.user.login))) runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read pull-requests: write diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index b982a31b6b..3a747ebb18 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -18,6 +18,7 @@ jobs: (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read pull-requests: read diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml index a8bb877ed2..c07937f5b6 100644 --- a/.github/workflows/frontend-tests.yml +++ b/.github/workflows/frontend-tests.yml @@ -22,7 +22,7 @@ concurrency: # would read the cancelled check as a failure and eject the pull request. cancel-in-progress: ${{ github.event_name != 'merge_group' }} -# Least-privilege: the `frontend` job only reads source and runs tests. +# Least-privilege: the frontend jobs only read source and run verification. # `packages: read` lets the container job pull the kandev-ci image from GHCR. permissions: contents: read @@ -164,12 +164,28 @@ jobs: working-directory: ${{ github.workspace }} run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + - name: Resolve pnpm store path + id: pnpm-store + shell: bash + run: | + set -euo pipefail + STORE_PATH="$(pnpm store path --silent)" + PNPM_VERSION="$(pnpm --version)" + if [[ "${STORE_PATH}" != /* ]]; then + echo "pnpm returned a non-absolute store path: ${STORE_PATH}" >&2 + exit 1 + fi + mkdir -p "${STORE_PATH}" + printf 'path=%s\n' "${STORE_PATH}" >> "$GITHUB_OUTPUT" + printf 'version=%s\n' "${PNPM_VERSION}" >> "$GITHUB_OUTPUT" + - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + continue-on-error: true with: - path: ~/.local/share/pnpm/store - key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} - restore-keys: pnpm-${{ runner.os }}- + path: ${{ steps.pnpm-store.outputs.path }} + key: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-store.outputs.version }}-${{ hashFiles('apps/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}-${{ runner.arch }}-${{ steps.pnpm-store.outputs.version }}- - name: Install dependencies run: pnpm install --frozen-lockfile @@ -371,16 +387,9 @@ jobs: node scripts/check-new-e2e-sleeps.mjs --base "${BASE_SHA}" - # Deliberately run under the variable that broke the suite, so a - # regression fails here instead of only in a container. React exports - # `act()` from its development build alone, `vitest.config.ts` pins - # `NODE_ENV=test` to force that build, and this image sets no NODE_ENV of - # its own — so without this line CI would pass with the pin deleted while - # every `render()` throws `TypeError: React.act is not a function` for - # anyone whose shell inherits the runtime image's `NODE_ENV=production` - # (`Dockerfile`). That gap is exactly how the breakage went unnoticed. - # Scoped to this step: `pnpm install` above must keep seeing devDependencies - # through its own resolution, not through `apps/.npmrc`'s `production=false`. + # Keep the production environment guard in the unsharded job. The image + # sets no NODE_ENV, so this reproduces the runtime value that exposed a + # deleted NODE_ENV=test pin in vitest.config.ts. - name: Run tests env: NODE_ENV: production @@ -390,8 +399,8 @@ jobs: run: pnpm --filter @kandev/web build # The required check. Runs on every event, including the ones where the - # `frontend` job above is correctly skipped, so the merge queue always has a - # conclusion to read instead of a check that stays pending forever. + # frontend verification jobs above are correctly skipped, so the merge queue + # always has a conclusion to read instead of a check that stays pending. frontend-gate: name: Frontend Tests Passed runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).frontend_gate_runner }} diff --git a/apps/web/scripts/vitest-project-selection.test.ts b/apps/web/scripts/vitest-project-selection.test.ts new file mode 100644 index 0000000000..78058cefb6 --- /dev/null +++ b/apps/web/scripts/vitest-project-selection.test.ts @@ -0,0 +1,150 @@ +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; + +const WEB_ROOT = path.resolve(import.meta.dirname, ".."); +const CONFIG_PATH = path.join(WEB_ROOT, "vitest.config.ts"); +const SELECTION_PATH = path.join(import.meta.dirname, "vitest-project-selection.ts"); +const NODE_PROJECT = "node" as const; +const BROWSER_PROJECT = "browser" as const; +const BROWSER_LOCALES_PROJECT = "browser-locales" as const; +const FIXTURE_TEST_SOURCE = "export const testValue = true;\n"; +const FIXTURE_PLAYWRIGHT_SOURCE = "export const shouldNotRun = true;\n"; + +type SelectionModule = typeof import("./vitest-project-selection"); + +async function loadSelection(): Promise { + const config = readFileSync(CONFIG_PATH, "utf8"); + expect(config).toContain("projects:"); + expect(config).toContain("projectDefinitions.browser.include"); + expect(config).toContain('projectDefinitions["browser-locales"].include'); + expect(existsSync(SELECTION_PATH)).toBe(true); + return import(/* @vite-ignore */ pathToFileURL(SELECTION_PATH).href); +} + +function withFixture(files: Record, callback: (root: string) => T): T { + const root = mkdtempSync(path.join(tmpdir(), "kandev-vitest-selection-")); + + try { + for (const [relativeFile, contents] of Object.entries(files)) { + const file = path.join(root, relativeFile); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, contents); + } + + return callback(root); + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +describe("Vitest project selection", () => { + it("keeps every test file in exactly one project", async () => { + const { discoverTestFiles, matchingProjects } = await loadSelection(); + const files = discoverTestFiles(WEB_ROOT); + + expect(files.length).toBeGreaterThan(1000); + expect(files.some((file) => file.startsWith("e2e/") && file.endsWith(".test.ts"))).toBe(true); + + for (const file of files) { + expect(matchingProjects(file), file).toHaveLength(1); + } + }); + + it("keeps the projects disjoint while retaining the complete file identity set", async () => { + const { discoverTestFiles, matchingProjects, projectFiles } = await loadSelection(); + const files = discoverTestFiles(WEB_ROOT); + const selected = new Set(files.flatMap((file) => matchingProjects(file))); + + expect(selected).toEqual(new Set([NODE_PROJECT, BROWSER_PROJECT, BROWSER_LOCALES_PROJECT])); + expect(new Set(projectFiles.node)).toEqual( + new Set(files.filter((file) => matchingProjects(file)[0] === NODE_PROJECT)), + ); + expect(new Set(projectFiles.browser)).toEqual( + new Set(files.filter((file) => matchingProjects(file)[0] === BROWSER_PROJECT)), + ); + expect(new Set(projectFiles["browser-locales"])).toEqual( + new Set(files.filter((file) => matchingProjects(file)[0] === BROWSER_LOCALES_PROJECT)), + ); + }); + + it("matches configured project include and exclude rules for fixture identities", async () => { + const { + PROJECT_NAMES, + buildProjectSelection, + discoverTestFiles, + matchingProjectsForSelection, + matchesConfiguredProject, + matchesTestFileGlob, + } = await loadSelection(); + + withFixture( + { + "scripts/unclassified.test.ts": FIXTURE_TEST_SOURCE, + "components/indirect-locale.test.ts": + 'import "./indirect-locale-helper";\nexport const testValue = true;\n', + "components/indirect-locale-helper.ts": + 'export function setLocale() { return changeLanguage("en"); }\n', + "components/unknown.spec.cjsx": FIXTURE_TEST_SOURCE, + "components/unknown.spec.ctsx": FIXTURE_TEST_SOURCE, + "components/unknown.test.cts": FIXTURE_TEST_SOURCE, + "components/unknown.test.mjsx": FIXTURE_TEST_SOURCE, + "e2e/ignored.spec.ts": FIXTURE_PLAYWRIGHT_SOURCE, + "e2e/fixtures/ignored.test.ts": FIXTURE_PLAYWRIGHT_SOURCE, + "e2e/pages/ignored.test.ts": FIXTURE_PLAYWRIGHT_SOURCE, + }, + (root) => { + const selection = buildProjectSelection(root); + const files = discoverTestFiles(root); + + expect(files).toEqual([ + "components/indirect-locale.test.ts", + "components/unknown.spec.cjsx", + "components/unknown.spec.ctsx", + "components/unknown.test.cts", + "components/unknown.test.mjsx", + "scripts/unclassified.test.ts", + ]); + for (const file of [ + "components/unknown.spec.cjsx", + "components/unknown.spec.ctsx", + "components/unknown.test.cts", + "components/unknown.test.mjsx", + ]) { + expect(matchesTestFileGlob(file)).toBe(true); + } + + for (const file of files) { + const configuredProjects = PROJECT_NAMES.filter((project) => + matchesConfiguredProject(file, project, selection), + ); + + expect(configuredProjects).toEqual(matchingProjectsForSelection(file, selection)); + expect(configuredProjects, file).toHaveLength(1); + } + + expect(matchingProjectsForSelection("scripts/unclassified.test.ts", selection)).toEqual([ + BROWSER_LOCALES_PROJECT, + ]); + expect( + matchingProjectsForSelection("components/indirect-locale.test.ts", selection), + ).toEqual([BROWSER_LOCALES_PROJECT]); + expect(matchesConfiguredProject("e2e/ignored.spec.ts", BROWSER_PROJECT, selection)).toBe( + false, + ); + expect( + matchesConfiguredProject("e2e/ignored.spec.ts", BROWSER_LOCALES_PROJECT, selection), + ).toBe(false); + }, + ); + }); + + it("gives node-only configuration tests enough time under project contention", () => { + const config = readFileSync(CONFIG_PATH, "utf8"); + + expect(config).toContain('name: "node"'); + expect(config).toContain("testTimeout: 15_000"); + }); +}); diff --git a/apps/web/scripts/vitest-project-selection.ts b/apps/web/scripts/vitest-project-selection.ts new file mode 100644 index 0000000000..03768e502e --- /dev/null +++ b/apps/web/scripts/vitest-project-selection.ts @@ -0,0 +1,180 @@ +import { readdirSync } from "node:fs"; +import path from "node:path"; + +export const PROJECT_NAMES = ["node", "browser", "browser-locales"] as const; +export type ProjectName = (typeof PROJECT_NAMES)[number]; + +// Keep this aligned with Vitest's default test include. A single canonical +// pattern prevents the project selector from silently dropping cjs, cts, mjs, +// mts, and their JSX variants. +export const TEST_FILE_GLOBS = ["**/*.{test,spec}.?(c|m)[jt]s?(x)"] as const; + +export const BASE_TEST_EXCLUDES = [ + "e2e/**/*.spec.ts", + "e2e/fixtures/**", + "e2e/pages/**", + "node_modules/**", +] as const; + +const TEST_FILE_NAME = /\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/; +const SKIPPED_DIRECTORIES = new Set([".git", "dist", "node_modules"]); + +/** + * These test files were reviewed as safe for the Node setup. New tests stay + * on the full browser and locale setup until their imports are reviewed and + * they are added here explicitly. + */ +export const REVIEWED_NODE_TEST_FILES: readonly string[] = [ + "scripts/check-i18n-keys.test.ts", + "scripts/check-no-em-dash-ui.test.ts", + "scripts/convert-zh-cn-to-zh-hant.test.mjs", + "scripts/fix-trans-indices.test.ts", + "scripts/generate-licenses-fallback.test.ts", + "scripts/generate-licenses.test.ts", + "scripts/i18n-parity.test.ts", + "scripts/i18n-sweep.test.ts", + "scripts/lib/changed-files.test.ts", + "scripts/lib/diff-ranges.test.ts", + "scripts/lib/e2e-sleep-ratchet.test.ts", + "scripts/lib/e2e-sleep-wiring.test.ts", + "scripts/lib/e2e-wait-inventory.test.ts", + "scripts/lib/eslint-i18n-config.test.ts", + "scripts/lib/git-base.test.ts", + "scripts/lib/guard-allowlist.test.ts", + "scripts/lib/nonjsx-copy.test.ts", + "scripts/lib/nonjsx-scope.test.ts", + "scripts/lib/removed-literals.test.ts", + "scripts/vitest-project-selection.test.ts", + "scripts/vitest-worker-budget.test.ts", +]; + +/** These browser tests were reviewed as not requiring the locale catalogs. */ +export const REVIEWED_BROWSER_TEST_FILES: readonly string[] = [ + "lib/test-support/happy-dom-network.test.ts", + "vitest-environment.test.tsx", +]; + +export interface ProjectFiles { + node: string[]; + browser: string[]; + "browser-locales": string[]; +} + +export interface VitestProjectDefinition { + include: readonly string[]; + exclude: readonly string[]; +} + +export interface ProjectSelection { + projectFiles: ProjectFiles; + projects: Record; +} + +function toRelativeFile(root: string, file: string): string { + return path.relative(root, file).split(path.sep).join("/"); +} + +function matchesExcludePattern(relativeFile: string, pattern: string): boolean { + if (pattern === "e2e/**/*.spec.ts") { + return relativeFile.startsWith("e2e/") && relativeFile.endsWith(".spec.ts"); + } + if (pattern === "e2e/fixtures/**") return relativeFile.startsWith("e2e/fixtures/"); + if (pattern === "e2e/pages/**") return relativeFile.startsWith("e2e/pages/"); + if (pattern === "node_modules/**") return relativeFile.startsWith("node_modules/"); + return relativeFile === pattern; +} + +function isExcluded(relativeFile: string): boolean { + return BASE_TEST_EXCLUDES.some((pattern) => matchesExcludePattern(relativeFile, pattern)); +} + +function walkTestFiles(root: string, directory: string, files: string[]): void { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory() && !SKIPPED_DIRECTORIES.has(entry.name)) { + walkTestFiles(root, path.join(directory, entry.name), files); + continue; + } + if (!entry.isFile() || !TEST_FILE_NAME.test(entry.name)) continue; + const relativeFile = toRelativeFile(root, path.join(directory, entry.name)); + if (!isExcluded(relativeFile)) files.push(relativeFile); + } +} + +export function discoverTestFiles(root: string): string[] { + const files: string[] = []; + walkTestFiles(root, root, files); + return files.sort(); +} + +export function matchesTestFileGlob(relativeFile: string): boolean { + return TEST_FILE_NAME.test(relativeFile); +} + +export function buildProjectFiles(root: string): ProjectFiles { + const files = discoverTestFiles(root); + const nodeSet = new Set(files.filter((file) => REVIEWED_NODE_TEST_FILES.includes(file))); + const browserSet = new Set(files.filter((file) => REVIEWED_BROWSER_TEST_FILES.includes(file))); + + return { + node: [...nodeSet].sort(), + browser: [...browserSet].sort(), + "browser-locales": files.filter((file) => !nodeSet.has(file) && !browserSet.has(file)), + }; +} + +export function buildProjectSelection(root: string): ProjectSelection { + const projectFiles = buildProjectFiles(root); + + return { + projectFiles, + projects: { + node: { + include: projectFiles.node, + exclude: [...BASE_TEST_EXCLUDES], + }, + browser: { + include: [...TEST_FILE_GLOBS], + exclude: [...BASE_TEST_EXCLUDES, ...projectFiles.node, ...projectFiles["browser-locales"]], + }, + "browser-locales": { + include: projectFiles["browser-locales"], + exclude: [...BASE_TEST_EXCLUDES], + }, + }, + }; +} + +function matchesIncludePattern(relativeFile: string, pattern: string): boolean { + if (pattern === TEST_FILE_GLOBS[0]) return matchesTestFileGlob(relativeFile); + return relativeFile === pattern; +} + +export function matchesConfiguredProject( + relativeFile: string, + project: ProjectName, + selection: ProjectSelection, +): boolean { + const definition = selection.projects[project]; + return ( + definition.include.some((pattern) => matchesIncludePattern(relativeFile, pattern)) && + !definition.exclude.some((pattern) => matchesExcludePattern(relativeFile, pattern)) + ); +} + +const DEFAULT_ROOT = path.resolve(import.meta.dirname, ".."); +export const defaultProjectSelection = buildProjectSelection(DEFAULT_ROOT); +export const projectFiles = defaultProjectSelection.projectFiles; +export const projectDefinitions = defaultProjectSelection.projects; + +export function matchingProjectsForSelection( + relativeFile: string, + selection: ProjectSelection, +): ProjectName[] { + return PROJECT_NAMES.filter((project) => + matchesConfiguredProject(relativeFile, project, selection), + ); +} + +export function matchingProjects(relativeFile: string): ProjectName[] { + return matchingProjectsForSelection(relativeFile, defaultProjectSelection); +} diff --git a/apps/web/vitest.config.ts b/apps/web/vitest.config.ts index d24be2080e..56c1022858 100644 --- a/apps/web/vitest.config.ts +++ b/apps/web/vitest.config.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { defineConfig, mergeConfig } from "vitest/config"; import viteConfig from "./vite.config"; +import { BASE_TEST_EXCLUDES, projectDefinitions } from "./scripts/vitest-project-selection"; import { resolveMaxWorkers } from "./scripts/vitest-worker-budget"; if (process.env.DEBUG === "1") process.env.DEBUG = ""; @@ -29,27 +30,68 @@ export default mergeConfig( ], }, test: { - environment: "happy-dom", - environmentOptions: { - happyDOM: { - settings: { - navigation: { - disableMainFrameNavigation: true, - disableChildFrameNavigation: true, - }, - }, - }, - }, - setupFiles: ["./vitest.setup.ts"], - // `e2e/**/*.spec.ts` belongs to Playwright, but plain unit tests for the - // e2e helpers themselves (`*.test.ts`) still run here — excluding the - // whole tree would let them sit in the repo without ever executing. - exclude: ["e2e/**/*.spec.ts", "e2e/fixtures/**", "e2e/pages/**", "node_modules/**"], + // Each project below inherits these shared worker and empty-selection + // safeguards. Environment and setup are project-specific so Node-only + // helpers do not pay for happy-dom, React, or locale catalogs. + exclude: [...BASE_TEST_EXCLUDES], pool: "threads", maxWorkers, // Already the default, pinned because it is load-bearing: a run that // collects nothing must exit non-zero rather than read as a green suite. passWithNoTests: false, + projects: [ + { + extends: true, + test: { + name: "node", + include: [...projectDefinitions.node.include], + exclude: [...projectDefinitions.node.exclude], + setupFiles: ["./vitest.setup.node.ts"], + environment: "node", + testTimeout: 15_000, + }, + }, + { + extends: true, + test: { + name: "browser", + include: [...projectDefinitions.browser.include], + exclude: [...projectDefinitions.browser.exclude], + setupFiles: ["./vitest.setup.ts"], + environment: "happy-dom", + environmentOptions: { + happyDOM: { + settings: { + navigation: { + disableMainFrameNavigation: true, + disableChildFrameNavigation: true, + }, + }, + }, + }, + }, + }, + { + extends: true, + test: { + name: "browser-locales", + include: [...projectDefinitions["browser-locales"].include], + exclude: [...projectDefinitions["browser-locales"].exclude], + setupFiles: ["./vitest.setup.ts", "./vitest.setup.locales.ts"], + environment: "happy-dom", + environmentOptions: { + happyDOM: { + settings: { + navigation: { + disableMainFrameNavigation: true, + disableChildFrameNavigation: true, + }, + }, + }, + }, + }, + }, + ], }, }), ); diff --git a/apps/web/vitest.setup.locales.ts b/apps/web/vitest.setup.locales.ts new file mode 100644 index 0000000000..ef20103375 --- /dev/null +++ b/apps/web/vitest.setup.locales.ts @@ -0,0 +1,3 @@ +import { loadAllLocalesForTests } from "./lib/i18n"; + +await loadAllLocalesForTests(); diff --git a/apps/web/vitest.setup.node.ts b/apps/web/vitest.setup.node.ts new file mode 100644 index 0000000000..d34265a8b5 --- /dev/null +++ b/apps/web/vitest.setup.node.ts @@ -0,0 +1,5 @@ +/** + * Node-only test setup intentionally has no React, DOM, WebSocket, or locale + * imports. Keep source-only helper tests on the cheapest environment. + */ +export {}; diff --git a/apps/web/vitest.setup.ts b/apps/web/vitest.setup.ts index 74537ad62d..c2772736db 100644 --- a/apps/web/vitest.setup.ts +++ b/apps/web/vitest.setup.ts @@ -2,7 +2,7 @@ import type { Window as HappyDOMWindow } from "happy-dom"; import * as React from "react"; import { afterEach, beforeEach } from "vitest"; -import { initI18nForTests, loadAllLocalesForTests } from "./lib/i18n"; +import { initI18nForTests } from "./lib/i18n"; import { NoopWebSocket } from "./lib/test-support/noop-websocket"; // Guards against `react` production build or duplicate copy — both make `act` unavailable; @@ -29,15 +29,11 @@ if (typeof React.act !== "function") { * The real `en` catalog is loaded, so assertions on English copy pass exactly as * they did before the migration. * - * Only `en` is bundled in the browser; the rest are lazy chunks. Suites drive - * the shared instance with a bare `i18n.changeLanguage("pseudo")` and assert on - * the next line, so the non-English catalogs are awaited here — top-level await - * in a setup file runs before any suite. That keeps unit tests in the - * "everything in memory" world they were written for without putting the other - * locales back into the bundle. + * Only `en` is loaded for the default browser project. Suites that change + * locale are assigned to the browser-locales project, whose second setup file + * awaits every catalog before test execution. */ initI18nForTests(); -await loadAllLocalesForTests(); const noNetworkWebSocket = NoopWebSocket as unknown as typeof WebSocket; diff --git a/docs/ci-merge-queue.md b/docs/ci-merge-queue.md index f5dec2c4a7..9db6a5c4d4 100644 --- a/docs/ci-merge-queue.md +++ b/docs/ci-merge-queue.md @@ -4,6 +4,14 @@ The `main` ruleset enabled the merge queue and six required checks on 2026-08-16. This document records the workflow preparation, active rules, and the Stable release exception. +### Claude execution budget + +The automatic same-repository review, approved fork review, and interactive +Claude jobs each have a 30-minute GitHub job timeout. The timeout bounds runner +execution and a cancelled job cannot report a successful review. It does not +bound approval or queue waits. An approval that expires before a job starts is +an approval outcome, and must be measured separately from Claude execution. + ## Why Before activation, `main`'s ruleset (id `13341245`) had `deletion`, @@ -118,31 +126,35 @@ absence of a filter rather than the presence of specific entries. ### External CI runner capacity The normal Linux CI workflows have one activation switch, one percentage -control, and two runner tiers. The switch and percentage are unset by default, -so the workflows use GitHub-hosted runners. +control, and two runner tiers. The current read-only snapshot on 2026-09-12 +is shown below. The switch is off, so current jobs use GitHub-hosted runners; +the retained percentage is the next pilot value. -| Variable | Pilot value | Purpose | -| -------- | ----------- | ------- | -| `KANDEV_CI_EXTERNAL_ENABLED` | unset | Set to `true` to allow external assignments | -| `KANDEV_CI_EXTERNAL_PERCENT` | unset or `0` | Assign 0 to 100 percent of eligible jobs | -| `KANDEV_CI_RUNNER_LIGHT` | `ubicloud-standard-2-ubuntu-2404` | Control and aggregate jobs | -| `KANDEV_CI_RUNNER_STANDARD` | `ubicloud-standard-4-ubuntu-2404` | Eligible browser and frontend test jobs | +| Variable | Current value | Pilot value | Purpose | +| -------- | ------------ | ----------- | ------- | +| `KANDEV_CI_EXTERNAL_ENABLED` | `false` | `true` | Allow external assignments | +| `KANDEV_CI_EXTERNAL_PERCENT` | `20` | `20` | Assign 0 to 100 percent of eligible job instances | +| `KANDEV_CI_RUNNER_LIGHT` | `ubicloud-standard-2-ubuntu-2404` | same | Control and aggregate jobs | +| `KANDEV_CI_RUNNER_STANDARD` | `ubicloud-standard-4-ubuntu-2404` | same | Eligible browser and frontend test jobs | Set `KANDEV_CI_EXTERNAL_ENABLED` to the exact value `true` when the Actions queue -needs extra capacity. Set it to `false`, or remove it, after the queue returns -to its normal level. The two tier variables stay configured between bursts, so -changing the instance type requires only an Actions variable update. +needs extra capacity. Set it to `false` after the queue returns to its normal +level. The two tier variables stay configured between bursts, so changing the +instance type requires only an Actions variable update. -Set `KANDEV_CI_EXTERNAL_PERCENT` to `50` for a pilot. The E2E matrix then sends -seven of fourteen normal shards to the standard tier. Backend checkout and -report-token jobs remain on GitHub-hosted runners; the aggregate backend gate -and frontend jobs are eligible. Singleton jobs use a stable hash cohort, so -their share approaches the percentage across workflow runs. +The 20% pilot sends a stable cohort of eligible instances to the configured +tiers. For E2E this assigns two of fourteen normal shards per run because matrix +allocation uses a floor. The frontend workflow has two unit-test instances, so +the 20% pilot assigns zero of those instances; its singleton static and gate +jobs still use stable hash cohorts. Backend checkout, static checks, +service jobs, Windows, and the protected test shards remain on GitHub-hosted +runners. Singleton jobs use a stable hash cohort, so their share approaches the +percentage across workflow runs. -Leave the percentage unset for the default GitHub-only mode. Set it to `100` -to send every eligible job with a non-empty tier label to external capacity. -Malformed or out-of-range values fail closed to GitHub-hosted runners and emit -a planner warning. +Set the percentage to `0` for an explicit GitHub-only allocation. Set it to +`100` to send every eligible job with a non-empty tier label to external +capacity. Malformed or out-of-range values fail closed to GitHub-hosted runners +and emit a planner warning. The reusable `.github/actions/plan-external-runners` composite action runs on the planner job's `ubuntu-latest` checkout. Each workflow declares its eligible @@ -159,32 +171,68 @@ label is unavailable, GitHub leaves the job queued or fails it. Clear the invalid label and rerun the workflow. Jobs already queued or running keep their original runner. New jobs use the current variables. -The E2E build and report jobs, Docker/Kind shards, Kubernetes compatibility -jobs, Playwright image job, desktop smoke job, backend checkout/static/test -shards, Postgres service job, architecture and harness linters, and Windows job -stay on GitHub-hosted runners. Release, publishing, signing, deployment, and +The eligible families are the E2E normal-shard matrix and gate, the backend +aggregate test gate, the frontend static verification job, the frontend unit +test matrix, and their lightweight change-detection and gate jobs. The E2E build +and report jobs, Docker/Kind shards, Kubernetes compatibility jobs, Playwright +image job, desktop smoke job, backend checkout/static/test shards, Postgres +service job, architecture and harness linters, and Windows job stay on +GitHub-hosted runners. Release, publishing, signing, deployment, and credential-bearing jobs stay on their existing runners. The workflows do not add permissions, secrets, or persistent state. -The same switch and tier labels apply to eligible E2E, backend-gate, and -frontend jobs. Architecture-lint, action-pinning, and harness-lint stay hosted -and do not create planner jobs. The planner does not change job names, test -selection, matrix values, artifacts, dependencies, timeouts, permissions, or -required conclusions. +The same switch and tier labels apply to eligible E2E, backend-gate, frontend +static verification, frontend unit-test matrix, and frontend gate jobs. +Architecture-lint, action-pinning, and harness-lint stay hosted and do not +create planner jobs. The planner does not change job names, test selection, +matrix values, artifacts, dependencies, timeouts, permissions, or required +conclusions. -Pilot procedure: +Pilot procedure. An operator must supply an approved provider budget and the +provider's current rate before activation. Unknown rates, fees, or budget are +an activation blocker. -1. Merge the workflow change while `KANDEV_CI_EXTERNAL_ENABLED` and - `KANDEV_CI_EXTERNAL_PERCENT` are unset. +1. Merge the workflow change while `KANDEV_CI_EXTERNAL_ENABLED=false`. 2. Install the [Ubicloud GitHub App](https://www.ubicloud.com/docs/github-actions-integration) for this repository. -3. Set both tier variables to the pilot labels in the table. Leave Premium - Runners disabled. Premium is an account-level setting, not a per-job tier. -4. Set `KANDEV_CI_EXTERNAL_PERCENT=50` for the first pilot. -5. Set `KANDEV_CI_EXTERNAL_ENABLED=true` while the queue is full. -6. Compare at least three runs for queue delay, job duration, runner label, - image setup, and failures. -7. Set the switch to `false` after the queue recovers. Keep the labels and - percentage for the next burst. +3. Confirm both tier variables match the approved labels. Leave Premium Runners + disabled. Premium is an account-level setting, not a per-job tier. +4. Record the current variable snapshot and provider budget in the run record. +5. Activate the 20% pilot for newly dispatched workflows: + + ```bash + gh variable set KANDEV_CI_EXTERNAL_ENABLED --repo kdlbs/kandev --body true + gh variable set KANDEV_CI_EXTERNAL_PERCENT --repo kdlbs/kandev --body 20 + ``` + +6. Compare at least three representative frontend, E2E, or backend runs. For + each eligible job record workflow run attempt, runner label, created time, + `started_at`, `completed_at`, queue delay, execution duration, setup/cache + steps, conclusion, and billed runner minutes. Keep protected hosted jobs in + the comparison so queue relief is not confused with code speed. +7. Stop the pilot for newly dispatched workflows after the sample or when the + budget is reached: + + ```bash + gh variable set KANDEV_CI_EXTERNAL_ENABLED --repo kdlbs/kandev --body false + gh variable set KANDEV_CI_EXTERNAL_PERCENT --repo kdlbs/kandev --body 0 + ``` + + Jobs already queued or running keep their original runner assignment. + +Cost calculation for each pilot sample: + +```text +external runner-minutes = sum(external job execution seconds) / 60 +external cost = external runner-minutes / 60 * provider hourly rate + + provider setup, storage, or transfer fees +``` + +Compare that cost with the GitHub-hosted minute accounting and record the +provider rate, billing granularity, and any free allowance. The percentage +controls assignment of job instances; it does not promise the same percentage +of billed minutes. Do not expand the percentage until three comparable runs +show queue relief without an unacceptable failure rate, execution regression, +or budget overrun. Use the [Ubicloud runner type reference](https://www.ubicloud.com/docs/github-actions-integration/runner-types) when changing a tier. Use GitHub's [runner selection reference](https://docs.github.com/en/actions/how-tos/write-workflows/choose-where-workflows-run/choose-the-runner-for-a-job) for `runs-on` expressions. Do not enable [Ubicloud Premium Runners](https://www.ubicloud.com/docs/github-actions-integration/use-premium-runners) for this pilot. diff --git a/docs/plans/ci-performance/evidence.md b/docs/plans/ci-performance/evidence.md new file mode 100644 index 0000000000..137db835d0 --- /dev/null +++ b/docs/plans/ci-performance/evidence.md @@ -0,0 +1,197 @@ +--- +id: "ci-performance-evidence" +title: "CI performance evidence" +status: done +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-004 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-004.1 + - AC-PLATFORM-CI-PERFORMANCE-004.2 +--- + +# CI performance evidence + +This report records the read-only sample collected on 2026-09-12. Every run +uses attempt 1 unless stated otherwise. Queue time is the interval before the +job's `started_at`; execution time is the job interval from `started_at` to +`completed_at`. Workflow elapsed time is the run interval from +`run_started_at` to `updated_at`. These intervals answer different questions +and are not added together. + +The sample is diagnostic. It does not establish a repository-wide p90, and it +does not include a hosted run of the changed frontend workflow. No workflow was +dispatched and no repository variable was changed during this task. + +## Run and job inventory + +| Area | Run and source | Run interval | Representative job | Queue / execution evidence | +| --- | --- | ---: | --- | --- | +| Claude review approval | [34355720657](https://github.com/kdlbs/kandev/actions/runs/34355720657), PR event, `b912428c248643c69e3effbd1e5ce3a624971740` | 7h 33m 04s | No jobs or check runs | Approval expired. GitHub created no runner job, so execution is unknown and must not be reported as 7h33m of Claude work. | +| Claude review execution | [34025129928](https://github.com/kdlbs/kandev/actions/runs/34025129928), `pull_request_target`, `c0264a50170693cd3ec2f103b5df9c80710bfb6c` | 1h 26m 44s | `claude-review-fork`, job `101464631037` | Queue 1h 20m 22s; execution 6m 21s. The allowlist job completed at 10:55:37Z and the review job started at 10:56:51Z. | +| Frontend unit baseline | [34689871445](https://github.com/kdlbs/kandev/actions/runs/34689871445), PR event, `a97538b0ba003f44b2586be9e2df7a59389edc1a` | 34m 09s | `Run Frontend Lint, Tests, and Build`, job `103543614850` | Job execution 29m 54s. The `Run tests` step took about 25.5m. | +| Cargo Audit | [34027695660](https://github.com/kdlbs/kandev/actions/runs/34027695660), push, `d796cf131deee94c24e78a6254aa19d4550ccabd` | 46m 37s | `Cargo Audit`, job `101471499916` | Queue 43m 23s; execution 3m 13s. The cargo-audit installation was about 2.8m. | +| E2E | [34687600985](https://github.com/kdlbs/kandev/actions/runs/34687600985), PR event, `fc101191cf4c7ac969dccc335b39450a98ae4a40` | 41m 21s | `E2E Shard 8/14`, job `103538339673` | Normal shards took about 15.6m to 20.4m including job setup. Some shards waited up to about 10.1m after job creation. | +| Backend | [34692849510](https://github.com/kdlbs/kandev/actions/runs/34692849510), PR event, `29139024786d4d3b8154c65cc7a4f472627d47e3` | 24m 26s | `Backend (windows)`, job `103551104350` | Windows job execution 21m 43s. The broad Windows-sensitive race-test step took about 13.2m. Linux test jobs took about 9.1m and 9.8m. | + +The Claude approval-expiration run has no job timestamps. The absence is the +evidence: a workflow run's `run_started_at` is not proof that a runner started. +The Claude action therefore receives a runner execution budget in Task 01, +while approval and queue waits remain separate metrics. + +## Frontend baseline and setup evidence + +The frontend job log for job `103543614850` reported: + +- 1,986 test files passed. +- 17,033 tests passed and 4 skipped. +- Vitest wall duration 1,526.89s. +- Vitest phase totals of setup 1,862.11s, import 1,252.60s, environment + 706.35s, transform 585.30s, and test bodies 385.32s. + +Phase totals overlap across workers. They are attribution data, not a sum that +can replace the 1,526.89s wall duration. + +The same log ended with the cache action warning: + +```text +Path Validation Error: Path(s) specified in the action for caching do(es) not exist, hence no cache is being saved. +``` + +The changed workflow now resolves `pnpm store path --silent` inside each +container job, includes the runner architecture and pnpm version in the key, +and keeps `pnpm install --frozen-lockfile` after the best-effort cache step. +Local resolution in this workspace returned +`/root/.local/share/pnpm/store/v3` with pnpm `9.15.9`. A hosted save followed by +a compatible restore is still an operational check and cannot be proved by +the local run. + +The setup candidate completed the local selection and environment checks. Its +final full report is retained outside the repository at +`/tmp/kandev-ci-unit-candidate-final.json` during this session. A hosted +three-run comparison remains required before crediting a speed improvement. + +## E2E fixture and retry evidence + +Run `34687600985`, attempt 1, published the `e2e-timing-diagnostics` artifact +and the related retry and timing profiles. The artifact source identifies run +`34687600985` and SHA +`9b0df86afdd1e8bc7d444c716626ed4694ea8747` on branch `3587/merge`. + +The retry summary contained 2,933 tests passed on the first attempt, 5 passed +after retry, 0 failed, 0 timed out, and 38 skipped. The five flaky tests give a +rate of 1.7 per thousand executed tests. The retry summary has no unexpected +failures, so the retry data is suitable for identifying follow-up targets while +the run remains a successful result. + +The measured shard profile identified these tails: + +| Cohort | Shard | Predicted | Actual | +| --- | ---: | ---: | ---: | +| normal | 8/14 | 1,085.23s | 1,108.98s | +| normal | 6/14 | 1,085.09s | 1,106.70s | +| normal | 3/14 | 1,084.52s | 1,065.22s | +| containers | 6/6 | 207.54s | 479.00s | +| containers | 3/6 | 201.92s | 414.84s | + +The slowest repeated test bodies in the timing profile were concentrated in +`tests/pr/pr-detection.spec.ts`, +`tests/task/autopilot-mode.spec.ts`, and +`tests/session/long-prepare-panels.spec.ts`, with individual samples near +48 to 61 seconds. The timing profile alone cannot distinguish fixture setup, +server work, or test-body waits inside those samples. + +The five retried tests were: + +- `tests/office/org-chart.spec.ts` +- `tests/task/file-tree-sorting.spec.ts` +- `tests/task/parked-background-work.spec.ts` +- `tests/task/task-listing-view-preferences.spec.ts` +- `tests/pr/mobile-pr-ci-chip.spec.ts` + +The next E2E investigation should correlate these test keys with fixture and +request traces before changing workers, retries, or waits. A reproduction can +start with the selected files under the Chromium project, using the normal +one-worker safety limit: + +```bash +cd apps/web +pnpm e2e:run --project chromium --workers=1 \ + tests/pr/pr-detection.spec.ts \ + tests/task/autopilot-mode.spec.ts \ + tests/session/long-prepare-panels.spec.ts +``` + +## Windows backend evidence + +The Windows log for job `103551104350` ran on `windows-latest` from 12:08:33Z +to 12:30:16Z. It recorded roughly 3m29s for `go build ./...`, 1m28s for +`go vet ./...`, and 13m10s for the broad race-test command covering: + +```text +./internal/agentctl/server/process/... +./internal/agent/runtime/agentctl/launcher/... +./internal/common/ptyexec/... +./internal/backendapp/ownershiplock/... +./internal/system/storage/workspaces/... +``` + +The job saved the Go cache at 12:30:10Z with a Windows, Go 1.26.0 key. This +shows cache save activity but does not show that the cache transfer is the +main cost. The next measurement should split compile, vet, package race tests, +and cache transfer on repeated Windows attempts before changing package +ownership or shard boundaries. + +Reproduction command from the captured job: + +```bash +cd apps/backend +go test -race -v ./internal/agentctl/server/process/... \ + ./internal/agent/runtime/agentctl/launcher/... \ + ./internal/common/ptyexec/... \ + ./internal/backendapp/ownershiplock/... \ + ./internal/system/storage/workspaces/... +``` + +## Ranked follow-ups + +1. **Frontend test setup and hosted sharding.** The frontend baseline has the + largest measured test interval in this sample. First compare the local + setup candidate and two-shard candidate with identical test identities. + Retain the matrix only after three comparable hosted runs show at least a + 30% lower median frontend critical path and no more than 25% additional + runner minutes. Current branch work supplies the selection and unsharded + gate contracts; the matrix and hosted performance evidence remain pending. +2. **pnpm cache save and restore.** The path-validation failure is confirmed. + Verify a cold install, a later compatible restore, and a cache-service + failure with a successful frozen install. Do not use a cache hit as a proxy + for test speed until its step timing is recorded. +3. **Runner queue capacity.** Cargo Audit and the Claude review sample are + dominated by queue or approval waits. Existing external-runner variables + are currently `KANDEV_CI_EXTERNAL_ENABLED=false`, + `KANDEV_CI_EXTERNAL_PERCENT=20`, with configured light and standard + Ubicloud labels. The account quota versus provider capacity cause is + unknown. Use the Task 06 pilot procedure before changing allocation. +4. **E2E fixtures and retries.** Profile the five flaky tests and the normal + shard tails listed above. Preserve one worker per shard and the existing + retry policy until fixture and request costs are attributable. +5. **Windows compilation and race tests.** Measure repeated Windows jobs and + cache transfer before further backend sharding. The current data supports + profiling, not a source-level optimization. + +## Reproduction and provenance commands + +The report was assembled with these read-only API commands. They preserve run +attempt identity and keep raw logs outside the repository: + +```bash +gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/attempts/$CI_RUN_ATTEMPT/jobs?per_page=100" +gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/artifacts?per_page=100" +gh run download "$CI_RUN_ID" --repo kdlbs/kandev \ + --name e2e-timing-diagnostics --dir "$CI_ARTIFACT_DIR" +gh api "repos/kdlbs/kandev/actions/jobs/$CI_JOB_ID/logs" > /tmp/kandev-ci-job.log +``` + +Raw downloaded logs and artifacts are temporary investigation outputs. The +curated figures above are the retained evidence; the repository does not store +the raw downloads. diff --git a/docs/plans/ci-performance/plan.md b/docs/plans/ci-performance/plan.md new file mode 100644 index 0000000000..33507cefe6 --- /dev/null +++ b/docs/plans/ci-performance/plan.md @@ -0,0 +1,151 @@ +--- +created: 2026-09-12 +status: in_progress +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-001 + - REQ-PLATFORM-CI-PERFORMANCE-002 + - REQ-PLATFORM-CI-PERFORMANCE-003 + - REQ-PLATFORM-CI-PERFORMANCE-004 +system_design: + - ../../specs/platform/system-design/ci-performance.md +legacy_specs: [] +--- + +# Implementation Plan: CI performance + +## Overview + +Bound Claude execution, repair frontend dependency caching, and reduce repeated frontend test setup. +Benchmark test partitions after the setup repair. Then produce focused E2E, Windows, and capacity follow-up evidence. +Execute the six work orders sequentially. Live runner activation remains an operator action outside this package. + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), `REQ-PLATFORM-CI-PERFORMANCE-001` through `004`. +- [System design](../../specs/platform/system-design/ci-performance.md). +- [Existing external-runner package](../external-e2e-runner-capacity/plan.md): complete, reused without reopening its implementation. +- [Existing E2E efficiency package](../e2e-ci-efficiency/plan.md): rollout evidence remains open. +- [External-runner ADR](../../decisions/2026-09-06-opt-in-external-e2e-runners.md). + +## Investigation baseline + +These observations are diagnostic samples from September 12, 2026. They are not reconstructed dashboard percentiles. + +| Evidence | Observation | Implication | +| --- | --- | --- | +| [Claude run 34355720657](https://github.com/kdlbs/kandev/actions/runs/34355720657) | 7h33m04s; approval expired; zero jobs | An execution timeout does not solve approval waits. | +| [Claude run 34025129928](https://github.com/kdlbs/kandev/actions/runs/34025129928) | Review job queued 80.35 minutes and ran 6.35 minutes | Separate queue delay from review execution. | +| [Frontend job 103543614850](https://github.com/kdlbs/kandev/actions/runs/34689871445/job/103543614850) | Job 29.9 minutes; unit tests 25.5 minutes | Unit-test overhead is the primary measured frontend target. | +| Same frontend log | 1,986 files; 17,033 passed and 4 skipped tests | Preserve selection at the benchmark snapshot; current counts can grow. | +| Same frontend log | Cache directory missing; no cache saved | Resolve the real pnpm store path. | +| [Cargo run 34027695660](https://github.com/kdlbs/kandev/actions/runs/34027695660) | Queue 43m22s; execution about 3.2 minutes | Runner capacity dominates this sample. | +| [E2E run 34687600985](https://github.com/kdlbs/kandev/actions/runs/34687600985) | 41.4 minutes overall; browser jobs 15.6–20.4 minutes; queue up to 10.1 minutes | Profile fixtures and retries before further shard expansion. | +| [Backend run 34692849510](https://github.com/kdlbs/kandev/actions/runs/34692849510) | Windows job 21.7 minutes; sensitive-package step 13.2 minutes | Distinguish compile time from individual test time. | + +Vitest reported 1,526.89 seconds wall time. Aggregate worker phases were setup 1,862.11s, import 1,252.60s, environment 706.35s, transform 585.30s, and tests 385.32s. +Those phases overlap across workers. Their sum is not elapsed time. +The actual cost of loading all locales remains a hypothesis until a controlled comparison isolates it. + +## Scope + +### In scope + +- Thirty-minute execution budgets for both automatic review jobs and the interactive Claude job. +- Frontend pnpm store discovery, compatible keys, and cache save/restore proof. +- Explicit test environments and selective locale setup with unchanged test selection. +- Measured two/four-shard candidates and a stable required frontend gate. +- A focused E2E/Windows profile report and an operator-ready capacity pilot procedure. + +### Out of scope + +- Approval bypasses, automatic retries, new permissions, or provider model changes. +- Live repository variable changes, paid runner activation, workflow dispatch, or publication during planning. +- Release optimization, Cargo binary caching, unproven E2E cache repairs, and unidentified application fixes. +- Application UI changes, E2E worker-count changes, lower race coverage, or disabled test isolation. +- Editing the supplied dashboard without its source and metric definitions. + +## Technical approach + +Task 01 uses job-level timeout fields and existing Claude workflow contract tests. +Task 02 resolves the pnpm store in the actual frontend container. +Task 03 separates Node-compatible files from browser tests and isolates multilingual setup. +Task 04 uses native Vitest shards and the existing external-runner family schema. +The `frontend` job keeps static checks, the unsharded unit suite, and build until +the hosted adoption gate passes. A measured test matrix remains a candidate for +the unchanged public `Frontend Tests Passed` gate. +Tasks 05 and 06 produce bounded reports and procedures. They do not invent unmeasured code changes. +See the system design for candidate-retention thresholds and failure behavior. + +## Tests + +| Acceptance criteria | Evidence | +| --- | --- | +| 001.1–001.2 | `claude-code-review-workflow-contract_test.py`: budgets on three execution jobs, no success fallback, preserved trigger/permission blocks. | +| 002.1–002.2 | `frontend-tests-workflow-contract_test.py`: resolved store wiring and lockfile install; hosted cache miss/save/hit evidence remains pending. | +| 003.1 | `apps/web/scripts/vitest-project-selection.test.ts`: complete, disjoint file selection; unsharded versus merged two-shard identities. | +| 003.2 | `vitest-environment.test.tsx`, focused `lib/i18n` tests, worker-budget tests, production-mode focused run, typecheck, and full unit suite. | +| 003.3 | Frontend workflow contract cases for unsharded production-mode test placement, result handling, and deliberate skips. | +| 004.1–004.2 | [Curated report](evidence.md) with attempt-specific timestamps and local two-shard identity evidence; hosted comparable performance remains pending. | +| 004.3 | Existing runner planner and placement tests plus the runbook procedure with protected jobs, cost assumptions, and rollback. | + +Proposed test paths are implementation deliverables. Existing paths were checked during planning. +No browser E2E is needed for CI-only behavior. Existing application E2E coverage remains unchanged. + +## Work orders + +- [x] [Task 01: Bound Claude execution](task-01-bound-claude-execution.md) +- [ ] [Task 02: Repair frontend dependency caching](task-02-repair-frontend-cache.md) +- [ ] [Task 03: Reduce frontend test setup](task-03-reduce-test-setup.md) +- [ ] [Task 04: Partition frontend verification](task-04-partition-frontend-verification.md) +- [x] [Task 05: Profile remaining CI costs](task-05-profile-remaining-costs.md) +- [x] [Task 06: Prepare the runner capacity pilot](task-06-prepare-capacity-pilot.md) + +## Execution and evidence + +All work orders start `pending`. Dependency order is 01 → 02 → 03 → 04 → 05 → 06. +Fresh worktrees need `(cd apps && pnpm install --frozen-lockfile)` before any pnpm command. +Use TDD for changed selection or workflow logic. Configuration tests must protect a meaningful failure boundary. + +Remote benchmark evidence requires authorized workflow execution or naturally occurring runs of the changed workflow. +Local success alone cannot satisfy cache reuse or hosted performance criteria. +Record pending operational evidence explicitly. Do not mark its work order done or the package implemented until its required evidence exists. +A measured rejection can complete Task 04 through its documented no-sharding outcome. +Task 06 completes with a checked procedure; activation and pilot results remain outside its scope. + +## Verification results + +Design validation completed on 2026-09-12: + +- `python3 scripts/lint-spec-files.test.py`: 36 tests passed. +- `python3 scripts/lint-spec-files.py --all`: passed. +- Package link and acceptance-reference checks: passed. +- `git diff --check -- docs/specs docs/plans/ci-performance`: passed. +- `git status --short -- docs/specs/platform docs/plans/ci-performance`: all nine new package files and the platform index identified. + +Production application behavior is unchanged. CI workflows, workflow contracts, +test configuration, and investigation-plan evidence changed as scoped. +Implementation checks pass locally. Hosted cache reuse and hosted performance +acceptance remain pending because no changed workflow was dispatched. + +Implementation validation on 2026-09-12: + +- Full Vitest candidate: 1,999 files, 17,180 passed tests, 4 pending, 0 failures. +- Production-mode focused Vitest: 12 files, 106 tests passed. +- Two local shards: 1,999 files and 17,136 assertion identities formed an exact disjoint union. +- Web typecheck, i18n check, ESLint, Prettier, and production build passed. +- Frontend, Claude, external-runner, runner-plan, and action-pinning contract tests passed. +- YAML parsing and `git diff --check` passed. `actionlint` is unavailable in this workspace. + +## Risks + +- Splitting setup can expose hidden DOM or locale dependencies. Default unknown files to existing setup until reviewed. +- Multiple Vitest projects can duplicate or omit files. Compare full identities and counts, not totals alone. +- More shards can increase cost and queue pressure. Retain only measured candidates within the design budget. +- General interactive Claude requests can exceed 30 minutes. Record timeout frequency before revising the budget. +- Hosted cache scope can prevent cross-branch hits. Prove reuse within compatible GitHub cache access rules. +- Existing companion plans retain their own statuses. This package does not claim their pending operational evidence. + +## Documentation impact + +Internal CI procedures change in `docs/ci-merge-queue.md` during implementation. +Public application behavior and `docs/public/**` remain unchanged. diff --git a/docs/plans/ci-performance/task-01-bound-claude-execution.md b/docs/plans/ci-performance/task-01-bound-claude-execution.md new file mode 100644 index 0000000000..609bb9b6df --- /dev/null +++ b/docs/plans/ci-performance/task-01-bound-claude-execution.md @@ -0,0 +1,91 @@ +--- +id: "01-bound-claude-execution" +title: "Bound Claude execution" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-001 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-001.1 + - AC-PLATFORM-CI-PERFORMANCE-001.2 +system_design: + - ../../specs/platform/system-design/ci-performance.md +--- + +# Task 01: Bound Claude execution + +## Summary + +Set a 30-minute execution budget on all three Claude execution jobs. +Protect the limits with the existing workflow contract suite. + +## In scope + +- Job timeout fields in both Claude workflows. +- Regression assertions for budgets, existing authorization, and failure behavior. +- A short internal note that approval expiration and execution timeout are different outcomes. + +## Out of scope + +- Approval, permissions, prompts, models, action upgrades, turn limits, and automatic retries. + +## Acceptance + +- Both automatic review jobs and the interactive job declare `timeout-minutes: 30`. +- Timeout cancellation cannot become a successful review through fallback logic. +- Trigger, approval, permission, checkout, and runner contracts remain unchanged. + +## Verification + +Run commands from the repository root. Install workspace dependencies first in a fresh worktree. + +```bash +python3 .github/scripts/claude-code-review-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +actionlint .github/workflows/claude-code-review.yml .github/workflows/claude.yml +git diff --check +``` + +## Files likely touched + +- `.github/workflows/claude-code-review.yml` +- `.github/workflows/claude.yml` +- `.github/scripts/claude-code-review-workflow-contract_test.py` +- `docs/ci-merge-queue.md` + +## Dependencies + +None. + +## Risks + +GitHub owns cancellation latency. Do not claim this change limits approval or queue waits. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter. +- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary. +- [Plan](plan.md), baseline and companion-package status. +- Existing workflow contract tests under `.github/scripts/`. + +## Results + +Implemented 2026-09-12. Added `timeout-minutes: 30` to the same-repository +review, approved fork-review, and shared interactive Claude jobs. Added a +contract assertion for all three execution budgets and documented the +distinction between approval expiry, queue delay, and runner execution in +`docs/ci-merge-queue.md`. No trigger, permission, checkout, action, or success +fallback changed. + +Validation: + +- `python3 .github/scripts/claude-code-review-workflow-contract_test.py` passed (10 tests). +- `python3 .github/scripts/lint-action-pinning_test.py` passed (9 tests). +- `git diff --check` passed. +- `actionlint` is not installed in this workspace; workflow syntax validation is pending. Repository contract and pinning tests cover workflow behavior and action references, not YAML parsing. diff --git a/docs/plans/ci-performance/task-02-repair-frontend-cache.md b/docs/plans/ci-performance/task-02-repair-frontend-cache.md new file mode 100644 index 0000000000..1b1f70d00b --- /dev/null +++ b/docs/plans/ci-performance/task-02-repair-frontend-cache.md @@ -0,0 +1,109 @@ +--- +id: "02-repair-frontend-cache" +title: "Repair frontend dependency caching" +status: in_progress +wave: 2 +depends_on: ["01-bound-claude-execution"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-002 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-002.1 + - AC-PLATFORM-CI-PERFORMANCE-002.2 +system_design: + - ../../specs/platform/system-design/ci-performance.md +--- + +# Task 02: Repair frontend dependency caching + +## Summary + +Resolve the pnpm store from the actual frontend container environment. +Prove a compatible cache save and later restore without weakening dependency installation. + +## In scope + +- Store discovery before the pinned cache action, with compatible versioned keys. +- Cold and unavailable cache behavior, plus exact lockfile enforcement. +- Curated save/restore and install timing evidence in this work order. + +## Out of scope + +- E2E cache changes, `node_modules` transfer, toolchain upgrades, and runner activation. + +## Acceptance + +- Cache restore and save use the path returned by pnpm under the installation environment. +- A cold cache installs successfully; a later compatible run reports a real restore without path warnings. +- Cache unavailability does not skip installation or hide installation errors. + +## Verification + +Run commands from the repository root. Install workspace dependencies first in a fresh worktree. + +```bash +python3 .github/scripts/frontend-tests-workflow-contract_test.py +python3 .github/scripts/lint-action-pinning_test.py +actionlint .github/workflows/frontend-tests.yml +git diff --check +``` + +For cache evidence, set `CI_RUN_ID` and `CI_JOB_ID` from an authorized run of the changed workflow. +Use these read-only commands for the cold and warm jobs: + +```bash +: "${CI_RUN_ID:?Set a run ID}" +: "${CI_JOB_ID:?Set its frontend job ID}" +gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/jobs?per_page=100" +gh api "repos/kdlbs/kandev/actions/jobs/$CI_JOB_ID/logs" > /tmp/kandev-ci-cache-job.log +rg 'Cache|cache|store|reused|downloaded|Path Validation' /tmp/kandev-ci-cache-job.log +``` + +Record a cache-service failure or controlled unavailable-cache check with a successful frozen installation. +Do not weaken dependency failure handling to obtain that result. + +## Files likely touched + +- `.github/workflows/frontend-tests.yml` +- `.github/scripts/frontend-tests-workflow-contract_test.py` + +## Dependencies + +Task 01. + +## Risks + +Cache permissions and branch scope affect hits. Record runner identity, pnpm version, resolved path, key, and save/restore outcomes. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter. +- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary. +- [Plan](plan.md), baseline and companion-package status. +- Existing workflow contract tests under `.github/scripts/`. + +## Results + +Implemented 2026-09-12. The frontend container now resolves its pnpm store +with `pnpm store path --silent` before the cache action, records the pnpm +version, uses OS, architecture, version, and lockfile compatible keys, and +keeps `pnpm install --frozen-lockfile` after the best-effort cache step. The +same wiring is present in the frontend unit-test matrix jobs. + +Local evidence: + +- `cd apps && pnpm store path --silent` returned + `/root/.local/share/pnpm/store/v3`. +- `cd apps && pnpm --version` returned `9.15.9`. +- `python3 .github/scripts/frontend-tests-workflow-contract_test.py` passed (10 tests). +- `python3 .github/scripts/lint-action-pinning_test.py` passed (9 tests). +- `git diff --check` passed. + +Hosted cold-install, cache-save, and compatible cache-restore evidence remains +open because the changed workflow was not dispatched or pushed in this task. +The previous hosted baseline recorded a path-validation warning and no cache +save; it is recorded in [CI performance evidence](evidence.md). diff --git a/docs/plans/ci-performance/task-03-reduce-test-setup.md b/docs/plans/ci-performance/task-03-reduce-test-setup.md new file mode 100644 index 0000000000..2fa411a5f1 --- /dev/null +++ b/docs/plans/ci-performance/task-03-reduce-test-setup.md @@ -0,0 +1,106 @@ +--- +id: "03-reduce-test-setup" +title: "Reduce frontend test setup" +status: in_progress +wave: 3 +depends_on: ["02-repair-frontend-cache"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-003 + - REQ-PLATFORM-CI-PERFORMANCE-004 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-003.1 + - AC-PLATFORM-CI-PERFORMANCE-003.2 + - AC-PLATFORM-CI-PERFORMANCE-004.2 +system_design: + - ../../specs/platform/system-design/ci-performance.md +--- + +# Task 03: Reduce frontend test setup + +## Summary + +Separate explicitly reviewed Node-compatible helpers from browser tests. +Load all locales only for suites that need them, while retaining isolated test state. + +## In scope + +- Test dependency inventory, explicit project selection, minimal Node setup, and multilingual setup. +- A selection contract that detects omissions and overlapping file assignments. +- Comparable baseline/candidate unit-test reports and phase timings. + +## Out of scope + +- Workflow sharding, disabled isolation, worker-budget increases, application behavior, and deleted tests. + +## Acceptance + +- Every existing test file belongs to exactly one project, including files under E2E helper directories. +- Browser and multilingual regressions pass under the inherited production environment; unknown dependencies retain full setup. +- At least three comparable runs per configuration show lower median test execution with identical selected identities and no new failures. + +## Verification + +Run commands from the repository root. Install workspace dependencies first in a fresh worktree. + +```bash +(cd apps/web && pnpm exec vitest run scripts/vitest-project-selection.test.ts scripts/vitest-worker-budget.test.ts vitest-environment.test.tsx lib/i18n) +(cd apps/web && NODE_ENV=production pnpm test --reporter=json --outputFile=/tmp/kandev-ci-unit-candidate.json) +(cd apps/web && pnpm run typecheck) +python3 .github/scripts/frontend-tests-workflow-contract_test.py +git diff --check +``` + +Before editing setup, run the full JSON command with a baseline output filename. +Repeat baseline and candidate measurements three times on the same source and runner class, changing only setup configuration. +Compare normalized file/test identities, skips, failures, and per-phase times. Keep raw reports outside the repository. + +## Files likely touched + +- `apps/web/vitest.config.ts` +- `apps/web/vitest.setup.ts` +- `apps/web/vitest.setup.node.ts (new)` +- `apps/web/vitest.setup.locales.ts (new)` +- `apps/web/scripts/vitest-project-selection.test.ts (new)` +- `apps/web/lib/i18n/*.test.*` +- `apps/web/vitest-environment.test.tsx` + +## Dependencies + +Task 02. + +## Risks + +File suffixes do not prove DOM independence. Inventory indirect locale helpers and preserve all default cleanup and network guards. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter. +- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary. +- [Plan](plan.md), baseline and companion-package status. +- Existing workflow contract tests under `.github/scripts/`. + +## Results + +Implemented 2026-09-12. The explicitly reviewed Node test list uses a minimal +Node project, two explicitly reviewed browser tests retain happy-dom and React +setup without loading all locale catalogs, and every other discovered test uses +the full browser and locale setup. New tests do not receive reduced setup from +their directory or source markers. The selection contract uses Vitest's +canonical test-file glob and excludes the existing Playwright-only paths. + +The selection contract passed and found all 1,999 current test files in exactly +one of `node`, `browser`, or `browser-locales`. The focused production-mode +run passed 12 files and 106 tests. The final unsharded candidate report passed +1,999 files with 17,180 passed tests, 4 pending tests, and 0 failures. The +first candidate attempt exposed one Node-project timeout under contention; the +Node project now has a 15-second test timeout and the repeat passed. Typecheck +passed after adding the selection module's explicit test type. + +The required three comparable setup runs on a hosted runner have not been +collected. Local evidence proves selection and correctness, not a hosted +median speed improvement, so this work order remains in progress. diff --git a/docs/plans/ci-performance/task-04-partition-frontend-verification.md b/docs/plans/ci-performance/task-04-partition-frontend-verification.md new file mode 100644 index 0000000000..c888857524 --- /dev/null +++ b/docs/plans/ci-performance/task-04-partition-frontend-verification.md @@ -0,0 +1,112 @@ +--- +id: "04-partition-frontend-verification" +title: "Partition frontend verification" +status: in_progress +wave: 4 +depends_on: ["03-reduce-test-setup"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-003 + - REQ-PLATFORM-CI-PERFORMANCE-004 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-003.1 + - AC-PLATFORM-CI-PERFORMANCE-003.2 + - AC-PLATFORM-CI-PERFORMANCE-003.3 + - AC-PLATFORM-CI-PERFORMANCE-004.1 + - AC-PLATFORM-CI-PERFORMANCE-004.2 +system_design: + - ../../specs/platform/system-design/ci-performance.md +--- + +# Task 04: Partition frontend verification + +## Summary + +Benchmark two and four unit-test shards after setup optimization. +Adopt the smallest passing candidate and preserve the stable required frontend gate. + +## In scope + +- Retain static checks, unsharded tests, and build in `frontend` until the + adoption gate passes; keep the measured `frontend_tests` matrix as a candidate. +- Existing runner-plan family schema, unique candidate shard reports, and the + stable required gate. +- Compare test identities, median critical path, queue delay, retries, and runner minutes. + +## Out of scope + +- E2E shard changes, protected runner changes, paid-capacity activation, skipped checks, or trigger-level path filters. + +## Acceptance + +- Shard reports form an exact partition of the unsharded test selection, with unchanged pretest and production-environment guards. +- The gate blocks failures, cancellations, and failed detection; deliberate skips and all-pass runs succeed. +- Adopt sharding only with at least 30% lower median frontend execution critical path and at most 25% extra runner minutes. Otherwise retain unsharded CI and record the rejection. + +## Verification + +Run commands from the repository root. Install workspace dependencies first in a fresh worktree. + +```bash +python3 .github/scripts/frontend-tests-workflow-contract_test.py +python3 .github/scripts/external-runner-workflow-contract_test.py +python3 .github/scripts/runner-plan_test.py +python3 .github/scripts/lint-action-pinning_test.py +actionlint .github/workflows/frontend-tests.yml +(cd apps/web && pnpm test --shard=1/2 --reporter=json --outputFile=/tmp/kandev-ci-shard-1.json) +(cd apps/web && pnpm test --shard=2/2 --reporter=json --outputFile=/tmp/kandev-ci-shard-2.json) +git diff --check +``` + +Run the same commands with shard indexes 1 through 4 for the four-shard candidate. +Compare JSON file/test identities against the unsharded Task 03 report at the same snapshot. +Use three comparable hosted runs per candidate for the performance decision; record all failed attempts too. +Do not mark the performance criterion complete from local serial runs. + +## Files likely touched + +- `.github/workflows/frontend-tests.yml` +- `.github/scripts/frontend-tests-workflow-contract_test.py` +- `.github/scripts/external-runner-workflow-contract_test.py` +- `docs/specs/platform/system-design/external-e2e-runner-capacity.md (job inventory only)` + +## Dependencies + +Task 03. + +## Risks + +Serial local shard commands prove selection only. Hosted comparable samples must establish speed and cost before adoption. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter. +- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary. +- [Plan](plan.md), baseline and companion-package status. +- Existing workflow contract tests under `.github/scripts/`. + +## Results + +Implemented 2026-09-12. The frontend job keeps lint, typecheck, i18n checks, +ratchets, the unsharded unit suite, and build. The two-instance +`frontend_tests` matrix remains a measured candidate and is not enabled until +the hosted adoption gate passes. `Frontend Tests Passed` continues to require +the runner plan, change detection, and the single frontend job; failures and +cancellations fail the gate while deliberate skips pass. + +Local partition evidence, using the corrected `pnpm test -- --shard=...` +invocation, is: + +- shard 1/2: 1,000 files and 8,225 passed tests; +- shard 2/2: 999 files, 8,954 passed tests, and 4 pending tests; +- union: all 1,999 files and 17,136 assertion identities from the unsharded + report, with zero overlap and no missing or unexpected identities; +- workflow, runner-planner, action-pinning, and frontend contract tests passed. + +The four-shard candidate and three comparable hosted runs were not executed. +The 30% critical-path and 25% runner-minute adoption decision therefore stays +open; the two-shard workflow remains a candidate pending hosted proof. diff --git a/docs/plans/ci-performance/task-05-profile-remaining-costs.md b/docs/plans/ci-performance/task-05-profile-remaining-costs.md new file mode 100644 index 0000000000..5d7cc9eceb --- /dev/null +++ b/docs/plans/ci-performance/task-05-profile-remaining-costs.md @@ -0,0 +1,114 @@ +--- +id: "05-profile-remaining-costs" +title: "Profile remaining CI costs" +status: done +wave: 5 +depends_on: ["04-partition-frontend-verification"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-004 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-004.1 + - AC-PLATFORM-CI-PERFORMANCE-004.2 +system_design: + - ../../specs/platform/system-design/ci-performance.md +--- + +# Task 05: Profile remaining CI costs + +## Summary + +Produce a focused report of E2E fixture/retry costs and Windows compile/test costs. +Select exact follow-up targets from evidence without speculative source changes. + +## In scope + +- Attempt-specific job and step timings from representative recent runs. +- Existing E2E timing/retry artifacts and Windows sensitive-package logs. +- A ranked report with proposed targets, expected mechanism, confidence, and missing evidence. + +## Out of scope + +- Application fixes, release builds, cache redesign, reruns, workflow dispatch, and paid-capacity activation. + +## Acceptance + +- Report approval, dependency, queue, and execution intervals separately, with unknowns explicit. +- Identify the largest measured E2E and Windows costs and distinguish compilation, setup, test bodies, retries, and cache transfer. +- Each follow-up names source/test targets and a reproduction command, or explicitly states why available evidence cannot support a change. + +## Verification + +Run commands from the repository root. Install workspace dependencies first in a fresh worktree. + +```bash +gh run list --repo kdlbs/kandev --workflow e2e-tests.yml --limit 20 --json databaseId,headSha,event,status,conclusion +gh run list --repo kdlbs/kandev --workflow backend-tests.yml --limit 20 --json databaseId,headSha,event,status,conclusion +python3 scripts/lint-spec-files.py --all +git diff --check +``` + +After selecting run IDs, record each attempt and download its jobs and artifacts through these read-only commands: + +```bash +: "${CI_RUN_ID:?Set the selected run ID}" +: "${CI_RUN_ATTEMPT:?Set the selected attempt number}" +gh api --paginate "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/attempts/$CI_RUN_ATTEMPT/jobs?per_page=100" > /tmp/kandev-ci-profile-jobs.json +gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/artifacts?per_page=100" +``` + +The artifact producer must publish an attempt-qualified name. Reject the +unqualified `e2e-timing-diagnostics` name because a rerun can publish the same +name for another attempt. Select and download only the exact name below: + +```bash +: "${CI_ARTIFACT_DIR:?Set an empty temporary artifact directory}" +CI_ARTIFACT_NAME="e2e-timing-diagnostics-attempt-${CI_RUN_ATTEMPT}" +ARTIFACTS_JSON="$(gh api "repos/kdlbs/kandev/actions/runs/$CI_RUN_ID/artifacts?per_page=100")" +test "$(jq --arg name "$CI_ARTIFACT_NAME" '[.artifacts[] | select(.name == $name and .expired == false)] | length' <<<"$ARTIFACTS_JSON")" = 1 +gh run download "$CI_RUN_ID" --repo kdlbs/kandev --name "$CI_ARTIFACT_NAME" --dir "$CI_ARTIFACT_DIR" +``` + +Check the downloaded manifest against both `CI_RUN_ID` and `CI_RUN_ATTEMPT`. +Reject the artifact when the exact name is absent, duplicated, expired, or the +manifest identifies another run or attempt. Do not mix retry attempts silently. +Select the Windows job ID from the jobs response, then use the job logs endpoint for package and compile timing. +Delete owned temporary logs after the curated report is complete. + +## Files likely touched + +- `docs/plans/ci-performance/evidence.md (new)` +- `docs/plans/e2e-ci-efficiency/plan.md (relevant evidence links only)` + +## Dependencies + +Task 04. + +## Risks + +Do not treat missing logs as zero execution, or successful-only samples as reliability evidence. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter. +- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary. +- [Plan](plan.md), baseline and companion-package status. +- Existing workflow contract tests under `.github/scripts/`. + +## Results + +Completed 2026-09-12. Added [CI performance evidence](evidence.md) with +attempt-specific run and job timestamps, approval versus queue versus runner +execution intervals, frontend cache failure evidence, E2E timing and retry +artifact results, and Windows compile, race-test, and cache-save timings. The +report ranks E2E fixture and retry profiling, Windows package profiling, cache +verification, and runner capacity measurement. It names reproduction commands +and records unknowns instead of treating missing jobs or logs as zero time. + +The report uses read-only data from runs 34355720657, 34025129928, +34689871445, 34027695660, 34687600985, and 34692849510. Temporary raw logs +and downloaded artifacts remain outside the repository. diff --git a/docs/plans/ci-performance/task-06-prepare-capacity-pilot.md b/docs/plans/ci-performance/task-06-prepare-capacity-pilot.md new file mode 100644 index 0000000000..c37631be74 --- /dev/null +++ b/docs/plans/ci-performance/task-06-prepare-capacity-pilot.md @@ -0,0 +1,85 @@ +--- +id: "06-prepare-capacity-pilot" +title: "Prepare the runner capacity pilot" +status: done +wave: 6 +depends_on: ["05-profile-remaining-costs"] +plan: "plan.md" +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-004 +acceptance_criteria: + - AC-PLATFORM-CI-PERFORMANCE-004.1 + - AC-PLATFORM-CI-PERFORMANCE-004.2 + - AC-PLATFORM-CI-PERFORMANCE-004.3 +system_design: + - ../../specs/platform/system-design/ci-performance.md +--- + +# Task 06: Prepare the runner capacity pilot + +## Summary + +Prepare a concrete 20% capacity pilot using the existing runner controls. +Document cost assumptions, protected jobs, measurement, and rollback without changing live variables. + +## In scope + +- Current eligible-family inventory, including any accepted frontend test matrix. +- Read-only variable snapshot and provider configuration prerequisites. +- Operator procedure for activation, measurement, rollback, and an explicit budget input. + +## Out of scope + +- Installing provider apps, purchasing capacity, changing variables, dispatching workflows, or moving review/Cargo/protected jobs. + +## Acceptance + +- The procedure preserves the current external-runner trust boundary and names jobs that remain hosted. +- It specifies a 20% pilot, at least three representative runs, separate queue/execution metrics, and no expansion without measured evidence. +- It includes explicit activation/rollback commands and cost calculations. Unknown provider rates or an absent operator budget block activation, not documentation completion. + +## Verification + +Run commands from the repository root. Install workspace dependencies first in a fresh worktree. + +```bash +gh variable list --repo kdlbs/kandev --json name,value --jq '.[] | select(.name | startswith("KANDEV_CI_"))' +python3 .github/scripts/runner-plan_test.py +python3 .github/scripts/external-runner-workflow-contract_test.py +python3 scripts/lint-spec-files.py --all +git diff --check +``` + +## Files likely touched + +- `docs/ci-merge-queue.md` +- `docs/plans/ci-performance/evidence.md` + +## Dependencies + +Task 05. + +## Risks + +Percentage allocation controls job instances, not a fixed share of billed minutes. Changing variables affects only newly dispatched jobs. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/platform/requirements/ci-performance.md), acceptance IDs in frontmatter. +- [System design](../../specs/platform/system-design/ci-performance.md), corresponding implementation boundary. +- [Plan](plan.md), baseline and companion-package status. +- Existing workflow contract tests under `.github/scripts/`. + +## Results + +Completed 2026-09-12. Updated the merge-queue runbook with the current +variable snapshot, eligible and protected job families, a 20% activation and +rollback procedure, three-run measurement fields, queue and execution +separation, and provider-rate and budget prerequisites. The procedure includes +the external runner-minute cost formula and states that unknown rates or budget +block activation. No live variable, provider setting, workflow dispatch, or +paid capacity changed. diff --git a/docs/specs/platform/README.md b/docs/specs/platform/README.md index 7f56c5f579..90decd1e6b 100644 --- a/docs/specs/platform/README.md +++ b/docs/specs/platform/README.md @@ -31,6 +31,92 @@ retains ownership of its values, validation, authority, and persistence. - Agent identity belongs to the [agent system](../agents/README.md). - Desktop shell behavior belongs to the [desktop system](../desktop/README.md). +## Specification map + +### Requirements + +- [CI performance](requirements/ci-performance.md) + +- [Startup lifecycle](requirements/startup-lifecycle.md) + +- [Agent-accessible Kandev settings](requirements/agent-settings-parity.md) + + + +- [Agent process exit and stderr drain](requirements/agent-process-exit-drain.md) +- [Agentctl instance stop idempotency](requirements/agentctl-instance-stop.md) +- [Agent Runtime Availability](requirements/agent-runtime-availability.md) +- [Background Work Liveness](requirements/background-work-liveness.md) +- [Backend restart page recovery](requirements/backend-restart-page-recovery.md) +- [Bounded Task Status Delivery](requirements/bounded-task-status-delivery.md) +- [Viewport-bounded Session Delivery](requirements/viewport-bounded-session-delivery.md) +- [Shared Cron Loop Safety When Office Is Disabled](requirements/cron-office-disabled-safety.md) +- [Environment-specific browser tab title prefixes](requirements/dev-preview-title-prefixes.md) +- [Browser console retention](requirements/browser-console-retention.md) +- [Diagnostic logging](requirements/diagnostic-logging.md) +- [Duration-aware E2E sharding and CI reliability](requirements/e2e-duration-aware-sharding.md) +- [External CI runner capacity](requirements/external-e2e-runner-capacity.md) +- [Expected runtime log severity](requirements/expected-runtime-log-severity.md) +- [Feature Toggles](requirements/feature-toggles.md) +- [Git Credential Lease Reissue](requirements/git-credential-lease-reissue.md) +- [Git Subprocess Admission](requirements/git-subprocess-admission.md) +- [Go dev launcher and minimal Node surface](requirements/go-dev-launcher.md) +- [Health Endpoint — Surface the Running Version](requirements/health-endpoint-version.md) +- [Watcher And Task Fallback Localization](requirements/i18n-audit-watcher-copy.md) +- [Additional I18n Audit Gaps](requirements/i18n-second-audit-gaps.md) +- [Internationalization (i18n)](requirements/i18n.md) +- [LSP File Intelligence](requirements/lsp-file-intelligence.md) +- [Session MCP Attachment Observability](requirements/mcp-session-observability.md) +- [Mid-Turn Steering](requirements/mid-turn-steering.md) +- [Semantic Notifications](requirements/notifications.md) +- [Apprise rescan](requirements/apprise-rescan.md) +- [Required Persisted Store Parity](requirements/postgres-domain-store-parity.md) +- [Provider Error Recovery](requirements/provider-error-recovery.md) +- [Session Config Reconciliation Across Agent Types](requirements/session-config-cross-agent-reconcile.md) +- [Session subscription recovery](requirements/session-subscription-recovery.md) +- [Setup and Launch Timeout](requirements/setup-launch-timeout.md) +- [Quiet benign teardown log noise on shutdown](requirements/shutdown-log-noise.md) +- [Do not surface backend-shutdown turn aborts as agent failures](requirements/shutdown-turn-failure-suppression.md) +- [Startup Configuration Parity](requirements/startup-configuration-parity.md) +- [Prevent Host Sleep During Active Tasks](requirements/task-sleep-inhibition.md) +- [Traditional Chinese locales (Taiwan and Hong Kong)](requirements/traditional-chinese-locales.md) +- [Workspace Git Status](requirements/workspace-git-status.md) + +### System design + +- [CI performance](system-design/ci-performance.md) + +- [Startup lifecycle](system-design/startup-lifecycle.md) + +- [Agent-accessible Kandev settings](system-design/agent-settings-parity.md) +- [Core settings domain adoption](system-design/agent-settings-domains.md) + + + +- [Agent process exit and stderr drain](system-design/agent-process-exit-drain.md) +- [Agentctl instance stop idempotency](system-design/agentctl-instance-stop.md) +- [Apprise rescan](system-design/apprise-rescan.md) +- [Background Work Liveness](system-design/background-work-liveness.md) +- [Backend restart page recovery](system-design/backend-restart-page-recovery.md) +- [Browser console retention](system-design/browser-console-retention.md) +- [Bounded Task Status Delivery](system-design/bounded-task-status-delivery.md) +- [Viewport-bounded Session Delivery](system-design/viewport-bounded-session-delivery.md) +- [Diagnostic logging System Design Part 1](system-design/diagnostic-logging-01.md) +- [Diagnostic logging System Design Part 2](system-design/diagnostic-logging-02.md) +- [Expected runtime log severity](system-design/expected-runtime-log-severity.md) +- [Duration-aware E2E sharding and CI reliability](system-design/e2e-duration-aware-sharding.md) +- [External CI runner capacity](system-design/external-e2e-runner-capacity.md) +- [Health Endpoint — Surface the Running Version](system-design/health-endpoint-version.md) +- [Internationalization (i18n)](system-design/i18n.md) +- [LSP File Intelligence System Design Part 1](system-design/lsp-file-intelligence-01.md) +- [LSP File Intelligence System Design Part 2](system-design/lsp-file-intelligence-02.md) +- [Session MCP Attachment Observability](system-design/mcp-session-observability.md) +- [Required Persisted Store Parity](system-design/postgres-domain-store-parity.md) +- [Provider Error Recovery](system-design/provider-error-recovery.md) +- [Workspace Git Status](system-design/workspace-git-status.md) +- [Go dev launcher and startup version](system-design/go-dev-launcher.md) +- [Startup database selection continuity](system-design/startup-database-selection-continuity.md) + ## Migration record Migration remains in progress while legacy source detail is extracted from the diff --git a/docs/specs/platform/requirements/ci-performance.md b/docs/specs/platform/requirements/ci-performance.md new file mode 100644 index 0000000000..1d8ec76f80 --- /dev/null +++ b/docs/specs/platform/requirements/ci-performance.md @@ -0,0 +1,79 @@ +--- +status: draft +system: platform +created: 2026-09-12 +owners: + - kandev +--- + +# CI performance requirements + +## Overview + +Maintainers need bounded review execution and shorter CI feedback without lost test coverage. +Platform owns these shared operational guarantees across repository workflows. +The CI automation system retains ownership of contributor authorization and approval rules. +The existing external-runner contract retains ownership of fleet selection and trust boundaries. + +## Terminology + +- **Execution:** The interval from a job start to its completion. +- **Queue delay:** The interval from job creation to job start. +- **Workflow elapsed time:** The interval from workflow creation to completion, including approval and dependency waits. +- **Comparable runs:** Runs with the same source snapshot, test selection, runner class, toolchain, and recorded cache state. + +## Requirements + +### REQ-PLATFORM-CI-PERFORMANCE-001: Bounded Claude execution + +**Intent:** Stop stalled review jobs without changing contributor approval rules. + +#### Acceptance criteria + +- **AC-PLATFORM-CI-PERFORMANCE-001.1:** Each automatic or comment-triggered Claude job shall have a 30-minute execution budget. GitHub shall cancel jobs that exceed that budget. +- **AC-PLATFORM-CI-PERFORMANCE-001.2:** A timeout shall not count as a successful review. Existing authorization, trigger, credential, and runner boundaries shall remain unchanged. + +### REQ-PLATFORM-CI-PERFORMANCE-002: Reusable frontend dependency cache + +**Intent:** Reuse compatible dependencies without making test results depend on cache availability. + +#### Acceptance criteria + +- **AC-PLATFORM-CI-PERFORMANCE-002.1:** A successful frontend dependency installation shall support cache reuse by a later compatible run. Evidence shall show an actual save and restore. +- **AC-PLATFORM-CI-PERFORMANCE-002.2:** A missing or unavailable cache shall permit a complete dependency installation. Dependency resolution shall continue to enforce the lockfile. + +### REQ-PLATFORM-CI-PERFORMANCE-003: Complete frontend verification with less overhead + +**Intent:** Reduce repeated test setup and shorten feedback without weakening verification. + +#### Acceptance criteria + +- **AC-PLATFORM-CI-PERFORMANCE-003.1:** Optimized frontend verification shall retain every selected test file and test case exactly once across its test partitions. +- **AC-PLATFORM-CI-PERFORMANCE-003.2:** Browser, multilingual, and production-environment regression tests shall retain their existing behavior. Test files shall retain isolated state. +- **AC-PLATFORM-CI-PERFORMANCE-003.3:** Required frontend verification shall fail when any selected partition or mandatory check fails or is cancelled. Deliberate change-based skips shall remain valid. + +### REQ-PLATFORM-CI-PERFORMANCE-004: Attributable performance evidence + +**Intent:** Distinguish useful optimization from queue variation or reduced coverage. + +#### Acceptance criteria + +- **AC-PLATFORM-CI-PERFORMANCE-004.1:** Performance reports shall distinguish job execution, queue delay, and workflow elapsed time. Missing job evidence shall appear as unknown execution. +- **AC-PLATFORM-CI-PERFORMANCE-004.2:** Optimization comparisons shall identify source, attempts, runner class, cache state, test counts, failures, retries, and sample size. +- **AC-PLATFORM-CI-PERFORMANCE-004.3:** Runner-capacity assessments shall preserve the existing eligible-job boundary. They shall report cost assumptions and rollback conditions before activation. + +## Out of scope + +- Automatic approval, automatic reruns, or changes to contributor trust policy. +- Automatic paid-capacity activation or expansion to protected runner workloads. +- Lower test coverage, disabled isolation, increased retries, or increased E2E worker counts. +- Application UI changes, release build redesign, or changes to an externally managed dashboard. + +## Related contracts + +- [External CI runner capacity](external-e2e-runner-capacity.md) +- [Contributor automation](../../ci/requirements/unified-contributor-pr-automation.md) + +## Implementation plans + +- [CI performance](../../../plans/ci-performance/plan.md) diff --git a/docs/specs/platform/system-design/ci-performance.md b/docs/specs/platform/system-design/ci-performance.md new file mode 100644 index 0000000000..69e8998f08 --- /dev/null +++ b/docs/specs/platform/system-design/ci-performance.md @@ -0,0 +1,150 @@ +--- +status: draft +system: platform +created: 2026-09-12 +requirements: + - REQ-PLATFORM-CI-PERFORMANCE-001 + - REQ-PLATFORM-CI-PERFORMANCE-002 + - REQ-PLATFORM-CI-PERFORMANCE-003 + - REQ-PLATFORM-CI-PERFORMANCE-004 +--- + +# CI performance system design + +## Purpose and boundaries + +This design changes repository CI execution and test infrastructure. It does not change application behavior. +The [external-runner design](external-e2e-runner-capacity.md) remains authoritative for provider allocation and protected jobs. +The [CI automation system](../../ci/README.md) retains contributor trust ownership. +No database, public API, or runtime feature flag changes are required. + +## Requirement mapping + +| Requirement | Design section | +| --- | --- | +| REQ-PLATFORM-CI-PERFORMANCE-001 | Review budget | +| REQ-PLATFORM-CI-PERFORMANCE-002 | Dependency cache | +| REQ-PLATFORM-CI-PERFORMANCE-003 | Test setup and workflow partitions | +| REQ-PLATFORM-CI-PERFORMANCE-004 | Measurement and capacity assessment | + +## Review budget + +Set job-level `timeout-minutes: 30` on `claude-review-same-repo` and `claude-review-fork` in `.github/workflows/claude-code-review.yml`. +Apply the same budget to the `claude` job in `.github/workflows/claude.yml`. +The comment workflow also handles general Claude requests. Its budget therefore applies to every invocation, including requests other than review. +Leave the short fork-label job unchanged. + +Keep permissions, prompts, action pins, checkouts, approval expressions, concurrency, and event filters unchanged. +Do not add `continue-on-error`, automatic retries, or success markers after cancellation. +GitHub owns timeout cancellation and cleanup latency. The budget is not a promise about total workflow elapsed time. +Do not use the retired Claude action `timeout_minutes` input. +An explicit turn cap remains deferred until completed-review turn counts support a useful threshold. + +## Dependency cache + +In `.github/workflows/frontend-tests.yml`, resolve `pnpm store path --silent` before the cache restore. +Run resolution from `apps/` with the same container, user, environment, and pnpm version as installation. +Expose the absolute path as a step output and use that output in the pinned cache action. +Do not infer the path from `HOME`, or change `HOME` to fit the cache. + +Use a versioned key containing runner OS, architecture, pnpm version, and lockfile hash. +A restore prefix can omit only the lockfile hash. Preserve pnpm-major and platform compatibility. +Create the store directory before restoration if necessary. Cache-service errors remain best effort. +The installation itself must fail on lockfile or dependency errors. + +Keep cache keys and paths identical in any frontend jobs introduced by partitioning. +Do not copy `node_modules` between runners or share mutable working directories. +Limit the first repair to the confirmed frontend failure. Inventory matching E2E paths in the evidence report without changing those workflows speculatively. + +## Test setup and workflow partitions + +### Setup boundaries + +`apps/web/vitest.config.ts` currently applies `happy-dom` and `vitest.setup.ts` to every test file. +`vitest.setup.ts` calls `initI18nForTests()` and awaits `loadAllLocalesForTests()` for each isolated file. +The frontend log shows substantial setup, import, and environment costs. + +Introduce explicit Vitest projects for Node-compatible helpers and browser-dependent tests. +Start with a reviewed list of pure helper files. Do not infer environment requirements from `.ts` versus `.tsx` extensions. +Keep all unclassified files in the full browser-locale project. A partition +contract must detect overlap or omitted files. +Node setup must not import React, DOM globals, or locale catalogs unless a selected test requires them. + +Keep English initialization for browser tests. Move complete locale loading into an explicit multilingual setup path. +Inventory tests that call `changeLanguage`, `loadAllLocalesForTests`, or inspect catalog state, including indirect helper calls. +Their setup must finish before test execution. Default unknown cases to full setup until their dependencies are established. +Preserve the `NODE_ENV=test` pin, the React `act` guard, Monaco alias, inert WebSocket, cleanup hooks, and `passWithNoTests: false`. +Keep test isolation and local worker-budget rules unchanged. + +### Workflow partitions + +After setup measurements, benchmark two and four Vitest shards on the same runner class. +Use built-in Vitest file sharding. Select two shards initially only if it passes the measurement gate below. +Use four only when it provides a further measured benefit within the compute budget. + +Keep the existing `frontend` job for formatting, licenses, lint, typecheck, SDK checks, i18n checks, ratchets, the unsharded unit suite, and build until the adoption gate passes. +After successful change detection and a passing measurement gate, an optional +`frontend_tests` matrix may split unit tests. Until then, the matrix remains a +candidate and must not become required CI. +The active job and any measured candidate retain the existing pinned container +and repaired cache. Each test command retains `NODE_ENV: production`. +Run tests through the package script so `pretest` generation still executes. + +If adopted, register the matrix through `.github/actions/plan-external-runners` +using its existing family schema. Use the standard tier without changing +protected-job placement or percentage semantics. Extend `frontend-gate` to +require the planner, change detection, `frontend`, and all test shards only +after adoption. +Keep its public name `Frontend Tests Passed`, `if: always()`, and deliberate-skip handling. +Keep `pull_request`, `push`, `merge_group`, and dispatch coverage. Do not add trigger-level path filters. + +Collect unique JSON reports per shard with test file identities and test case counts. +Compare their union against an unsharded run at the same source snapshot. +Failures and cancelled shards must still block the gate. Reports must not mask a failed test exit code. + +## Measurement and capacity assessment + +Save curated evidence in the implementation package. Keep raw logs outside version control. +Record run ID, attempt, head SHA, event, workflow version, job ID, runner label, container digest, and tool versions. +Record job `created_at`, `started_at`, and `completed_at` separately. +Execution equals completion minus start. Queue delay equals start minus job creation. +Exclude skipped jobs with invalid timestamps from duration calculations. +Do not label workflow `run_started_at` as job execution. +Approval and dependency waits can precede job creation. Report them separately when evidence exists, otherwise leave the cause unknown. +Use attempt-specific job data for reruns. Do not add concurrent job durations to obtain workflow wall time. + +Use at least three comparable successful runs per candidate and baseline for performance decisions. +Compare medians and individual samples. Do not present three samples as a reliable p90 estimate. +Record failures and retries from all attempted samples, not only successful ones. +For setup optimization, require lower median unit-test execution without reduced selection or new failures. +For sharding, target at least 30% lower median frontend execution critical path and no more than 25% additional frontend runner minutes. +Measure queue time separately. Reject a partition that consistently worsens total feedback under representative load. +These are candidate-retention gates, not promised speedups. Record rejected candidates and keep the simpler passing configuration. + +Inspect existing E2E timing and retry artifacts before selecting fixture changes. +Profile the Windows job by compile, package-test, and cache intervals. Preserve race checks and operating-system coverage. +The profiling work order produces a ranked follow-up report with exact targets. It does not authorize unidentified source changes. + +Prepare a capacity pilot using the existing repository variables and approved tiers. +Record current variables again before proposing activation. The investigation snapshot had burst mode disabled and percentage set to 20. +Describe a 20% pilot, cost assumptions, sample count, rollback, and operator commands in the existing merge-queue runbook. +Do not activate paid capacity as part of documentation or repository implementation. +Review and Cargo Audit remain hosted; this pilot does not directly move their jobs. + +## Failure and recovery + +A cold cache installs dependencies normally. A failed test partition fails the existing required gate. +A timeout cancels Claude execution without a success fallback. Approval expiration remains an approval outcome. +Revert a performance candidate that loses coverage or exceeds its measured budget. +Runner rollback follows the existing external-runner design and affects new jobs only. + +## Related decisions and plans + +- [External-runner decision](../../../decisions/2026-09-06-opt-in-external-e2e-runners.md) +- [Existing runner package](../../../plans/external-e2e-runner-capacity/plan.md) +- [Existing E2E efficiency package](../../../plans/e2e-ci-efficiency/plan.md) +- [CI performance package](../../../plans/ci-performance/plan.md) + +The existing runner package is complete. It needs no repeated implementation. +The E2E efficiency package retains open rollout evidence. Link new measurements there without marking unperformed rollout work complete. +No new architectural decision is required: this package preserves existing trust and isolation boundaries.