name: E2E Tests # No trigger-level `paths:` filter, deliberately. A workflow skipped by one # reports no conclusion at all, and a required check that never reports leaves # a pull request unable to enter a merge queue and a merge group unable to # leave it. `merge_group` cannot carry a `paths:` filter in the first place. # The filter lives in the `changes` job below instead, and `e2e-gate` reports # the single conclusion the queue waits on. See docs/ci-merge-queue.md. on: push: branches: [main] pull_request: branches: [main] merge_group: types: [checks_requested] # Manual trigger for verifying CI on draft PRs and ad-hoc reruns. The # default sharded run uses the same ref the user picks in the dispatch UI. workflow_dispatch: inputs: fail_on_flaky: description: "Fail the run when Playwright needs a retry" required: false type: boolean default: false concurrency: group: ${{ github.workflow }}-${{ github.ref }} # Never cancel a merge-queue run. Its ref is unique per queue entry, so a # cancellation here could only come from a mis-keyed group, and the queue # would read the cancelled check as a failure and eject the pull request. cancel-in-progress: ${{ github.event_name != 'merge_group' }} permissions: contents: read # The build/report jobs read the latest successful main timing artifact. actions: read # Required so the containerized build / e2e shards / e2e-report jobs can # pull the kandev-ci image from GHCR via GITHUB_TOKEN. packages: read jobs: runner_plan: name: Plan external runner allocation runs-on: ubuntu-latest outputs: plan: ${{ steps.plan.outputs.plan }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Plan runner allocation id: plan uses: ./.github/actions/plan-external-runners with: workflow: e2e run-id: ${{ github.run_id }} families: >- [ {"name":"changes","tier":"light"}, {"name":"e2e","tier":"standard","instances":[ {"shard":1},{"shard":2},{"shard":3},{"shard":4}, {"shard":5},{"shard":6},{"shard":7},{"shard":8}, {"shard":9},{"shard":10},{"shard":11},{"shard":12}, {"shard":13},{"shard":14} ]}, {"name":"e2e_gate","tier":"light"} ] burst: ${{ vars.KANDEV_CI_EXTERNAL_ENABLED }} percent: ${{ vars.KANDEV_CI_EXTERNAL_PERCENT }} light-label: ${{ vars.KANDEV_CI_RUNNER_LIGHT }} standard-label: ${{ vars.KANDEV_CI_RUNNER_STANDARD }} # The former trigger-level `paths:` list, applied one level down so a skip # still produces a conclusion. `.github/scripts/changed-paths.py` implements # the same last-match-wins semantics GitHub applies to `paths:`. changes: name: Detect relevant changes needs: runner_plan runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).changes_runner }} timeout-minutes: 10 outputs: run: ${{ steps.detect.outputs.run }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: # The base commit has to be present locally for the diff below. fetch-depth: 0 persist-credentials: false - name: Detect relevant changes id: detect env: PATTERNS: | apps/backend/** apps/desktop/** apps/web/** apps/packages/** apps/pnpm-lock.yaml apps/pnpm-workspace.yaml .github/docker/ci-base/** .github/scripts/resolve-image-digest.sh .github/workflows/ci-base-image.yml .github/workflows/e2e-tests.yml .github/scripts/changed-paths.py !**/*.md EVENT_NAME: ${{ github.event_name }} BASE_REF: ${{ github.base_ref }} MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }} PUSH_BEFORE: ${{ github.event.before }} run: | set -euo pipefail case "${EVENT_NAME}" in # The checkout is the merge ref, whose first parent is the base # branch tip. Diffing against that tip directly collapses to a # two-dot diff over everything main landed since the PR opened, so # take the fork point instead. pull_request) BASE="$(git merge-base HEAD "origin/${BASE_REF}")" ;; # The queue branch is cut from this commit, so it is a true ancestor. merge_group) BASE="${MERGE_GROUP_BASE_SHA}" ;; push) BASE="${PUSH_BEFORE}" ;; # workflow_dispatch is an explicit request to run the suite. *) BASE="" ;; esac # Fail open. A first push to a branch, a force push, or a manual # dispatch leaves no usable base, and running the suite needlessly is # cheaper than merging something no job ever looked at. if [[ -z "${BASE}" || "${BASE}" == "0000000000000000000000000000000000000000" ]] \ || ! git cat-file -e "${BASE}^{commit}" 2>/dev/null; then echo "No usable base commit for '${EVENT_NAME}'; running everything." echo "run=true" >> "$GITHUB_OUTPUT" exit 0 fi # `-z`, not plain --name-only: git quotes non-ASCII paths under its # default core.quotePath, and a quoted path matches no pattern, so a # change to one would skip every job and still report success. git diff --name-only -z "${BASE}" HEAD \ | python3 .github/scripts/changed-paths.py build: name: Build needs: changes if: needs.changes.outputs.run == 'true' # Build reads the GitHub API and downloads a timing artifact with the job token. # Keep this credential-bearing path on the GitHub-hosted boundary. runs-on: ubuntu-latest # Pre-baked image bundles Go, Node 24, pnpm 9.15.9 — see # .github/docker/ci-base/Dockerfile. Uses the `build` tag (with Go) # because this job compiles the backend. container: ghcr.io/kdlbs/kandev-ci:build-latest timeout-minutes: 20 # Container jobs default to `sh`; pin to bash for the few shell steps below. defaults: run: shell: bash steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" # Inside the container the home dir is /root, so the pnpm store lives at # /root/.local/share/pnpm/store. actions/cache resolves `~` against $HOME # so the same path string works for both the host and container variants. - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.local/share/pnpm/store key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} restore-keys: pnpm-${{ runner.os }}- # Same reasoning as the pnpm store above: the image sets GOPATH=/root/go # and leaves GOCACHE at its default, and $HOME is /root, so these are the # same two paths backend-tests.yml caches. Without this the backend build # below starts from a cold module *and* build cache on every run. - name: Cache Go modules uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | ~/go/pkg/mod ~/.cache/go-build key: go-e2e-build-${{ runner.os }}-${{ hashFiles('apps/backend/go.mod', 'apps/backend/go.sum') }} restore-keys: | go-e2e-build-${{ runner.os }}- go-${{ runner.os }}- - name: Install dependencies working-directory: apps run: pnpm install --frozen-lockfile # `build-web-e2e`, not `build-web`: the production web build no longer # bundles the pseudo QA catalog, and tests/i18n/pseudo-coverage.spec.ts # needs it. See apps/web/lib/i18n/bundling.ts. - name: Build backend and web run: make build-backend build-web-e2e # Fixture plugin package consumed by the plugin E2E specs. Built here # (the build job has the Go toolchain) and shipped to the shards as its # own artifact; global-setup fails fast if it is missing. - name: Build E2E fixture plugin package run: make -C apps/backend e2e-plugin-package # Linux/amd64 helper binaries (agentctl + mock-agent) bind-mounted into # containers by the Docker E2E project. Cheap to build (~1s each) so we # always do it; the regular E2E job ignores them. - name: Build Docker E2E helper binaries run: make build-backend-linux-helpers - name: Record immutable E2E build identity run: | set -euo pipefail node <<'NODE' const { createHash } = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); const backendDir = path.resolve("apps/backend"); const artifacts = [ "bin/kandev", "bin/mock-agent", "bin/agentctl-linux-amd64", "bin/mock-agent-linux-amd64", ".build/kandev-plugin-e2e-1.0.0.tar.gz", ]; const checksums = Object.fromEntries( artifacts.map((artifact) => { const contents = fs.readFileSync(path.join(backendDir, artifact)); return [artifact, createHash("sha256").update(contents).digest("hex")]; }), ); fs.writeFileSync( path.join(backendDir, "bin/e2e-build-identity.json"), JSON.stringify( { schema_version: 1, source_revision: process.env.GITHUB_SHA, artifacts: checksums, }, null, 2, ) + "\n", ); NODE - name: Locate latest successful main timing profile artifact id: main-timing-run env: GH_TOKEN: ${{ github.token }} GITHUB_API_URL: ${{ github.api_url }} GITHUB_REPOSITORY: ${{ github.repository }} run: | node <<'NODE' const fs = require("node:fs"); (async () => { const runsUrl = process.env.GITHUB_API_URL + "/repos/" + process.env.GITHUB_REPOSITORY + "/actions/workflows/e2e-tests.yml/runs?branch=main&status=success&per_page=50"; const headers = { accept: "application/vnd.github+json", authorization: "Bearer " + process.env.GH_TOKEN, }; const response = await fetch(runsUrl, { headers, }); if (!response.ok) { console.log( "::warning::Unable to query successful main E2E runs (" + response.status + "); using count fallback.", ); fs.appendFileSync(process.env.GITHUB_OUTPUT, "run_id=\n"); return; } const payload = await response.json(); let run; for (const candidate of payload.workflow_runs ?? []) { if (candidate.head_branch !== "main" || candidate.conclusion !== "success") continue; const artifactsResponse = await fetch( process.env.GITHUB_API_URL + "/repos/" + process.env.GITHUB_REPOSITORY + "/actions/runs/" + candidate.id + "/artifacts?per_page=100", { headers }, ); if (!artifactsResponse.ok) { console.log( "::warning::Unable to inspect artifacts for main E2E run " + candidate.id + " (" + artifactsResponse.status + "); trying an older run.", ); continue; } const artifactsPayload = await artifactsResponse.json(); const hasTimingProfile = (artifactsPayload.artifacts ?? []).some( (artifact) => artifact.name === "e2e-timing-profile" && !artifact.expired, ); if (hasTimingProfile) { run = candidate; break; } console.log( "Skipping main E2E run " + candidate.id + ": no usable timing profile artifact.", ); } fs.appendFileSync(process.env.GITHUB_OUTPUT, "run_id=" + (run?.id ?? "") + "\n"); console.log( run ? "Using successful main E2E run " + run.id + " with a timing profile artifact." : "No successful main E2E run with a timing profile artifact found; using count fallback.", ); })().catch((error) => { console.error(error); fs.appendFileSync(process.env.GITHUB_OUTPUT, "run_id=\n"); }); NODE - name: Download selected main timing profile if: steps.main-timing-run.outputs.run_id != '' continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-timing-profile path: e2e/previous-profile repository: ${{ github.repository }} run-id: ${{ steps.main-timing-run.outputs.run_id }} github-token: ${{ github.token }} - name: Upload selected timing baseline for report job if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-selected-main-timing-profile path: e2e/previous-profile/timing-profile.json if-no-files-found: ignore retention-days: 3 - name: Generate duration-aware E2E shard manifests working-directory: apps/web run: | set -euo pipefail rm -rf "$GITHUB_WORKSPACE/e2e-manifests" if [[ -f "$GITHUB_WORKSPACE/e2e/previous-profile/timing-profile.json" ]]; then echo "Planning from the selected successful main timing profile." pnpm exec tsx e2e/scripts/plan-shards.ts \ --web-root "$GITHUB_WORKSPACE/apps/web" \ --output-dir "$GITHUB_WORKSPACE/e2e-manifests" \ --profile "$GITHUB_WORKSPACE/e2e/previous-profile/timing-profile.json" else echo "Planning with deterministic count fallback; no usable main timing profile was downloaded." pnpm exec tsx e2e/scripts/plan-shards.ts \ --web-root "$GITHUB_WORKSPACE/apps/web" \ --output-dir "$GITHUB_WORKSPACE/e2e-manifests" fi - name: Upload E2E shard manifests uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-shard-manifests path: e2e-manifests/ retention-days: 3 - name: Upload backend build uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-backend-build path: apps/backend/bin/ retention-days: 1 - name: Upload E2E fixture plugin package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-plugin-package path: apps/backend/.build/kandev-plugin-e2e-1.0.0.tar.gz retention-days: 1 # Tar the Vite dist directory so shards can reuse the same static web build. - name: Package web build run: tar -cf apps/web/web-dist.tar -C apps/web dist - name: Upload web build uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-web-build path: apps/web/web-dist.tar retention-days: 1 e2e: name: E2E Shard ${{ matrix.shard }}/14 needs: [changes, build, runner_plan] if: needs.changes.outputs.run == 'true' runs-on: ${{ matrix.runner }} # Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright browser # binaries (Chromium etc.) — see .github/docker/ci-base/Dockerfile. This is # the biggest speedup since `npx playwright install chromium --with-deps` # alone took ~1.5–2 min per shard. # # `--ipc=host` is required for Chromium: the default container `/dev/shm` # is only ~64MiB, which causes tabs to slow to a crawl and renderers to # crash mid-test. # Uses the `runtime` tag — no Go toolchain needed; the shards consume the # pre-built backend artifact from the `build` job. ~600 MB smaller pull # than the build image, saves ~20s per shard at startup. container: image: ghcr.io/kdlbs/kandev-ci:runtime-latest options: --ipc=host # 35 min covers the serial count-fallback tail and setup overhead. The # first queue runs reached the old 25-minute cap with a healthy 167-test # shard before the main timing profile had bootstrapped. timeout-minutes: 35 # Container jobs default to `sh`; pin to bash. defaults: run: shell: bash # Keep localhost resolution deterministic for Node fetches used by the # Playwright fixtures and backend readiness polling. env: NODE_OPTIONS: --dns-result-order=ipv4first strategy: fail-fast: false matrix: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_matrix }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.local/share/pnpm/store key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} restore-keys: pnpm-${{ runner.os }}- - name: Install dependencies working-directory: apps run: pnpm install --frozen-lockfile - name: Download backend build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-backend-build path: apps/backend/bin/ - name: Make backend binaries executable run: chmod +x apps/backend/bin/* - name: Download E2E fixture plugin package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-plugin-package path: apps/backend/.build/ - name: Download web build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-web-build path: apps/web/ - name: Extract web build working-directory: apps/web run: tar -xf web-dist.tar && rm web-dist.tar - name: Enable strict flaky-test diagnostics if: github.event_name == 'workflow_dispatch' && inputs.fail_on_flaky run: echo "E2E_FAIL_ON_FLAKY=1" >> "$GITHUB_ENV" - name: Download E2E shard manifests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-shard-manifests path: e2e-manifests # Pinned to chromium + mobile-chrome so the dedicated `e2e-containers` # job below owns the real-Docker project (which covers both the Docker # executor and the SSH executor's sshd target). Without --project the # container-bound specs would run here too, fail seeding (Docker not # enabled for these shards), and spam every shard. # Cross-job archive extraction does not preserve useful mtimes. Validate # the source revision and every required artifact checksum instead. - name: Run planned E2E tests (shard ${{ matrix.shard }}/14) working-directory: apps/web env: E2E_SHARD: ${{ matrix.shard }} E2E_MANIFEST_SHARD: ${{ matrix.shard }} KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${E2E_MANIFEST_SHARD}.json" - name: Upload blob report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: blob-report-${{ matrix.shard }} path: apps/web/e2e/blob-report/ retention-days: 3 - name: Upload test results if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: test-results-${{ matrix.shard }} path: apps/web/e2e/test-results/ retention-days: 7 # Resolve the runtime tag once before the six container shards start. Fork # pull requests use the public package anonymously, so sharing one bounded # lookup avoids multiplying transient GHCR failures across the matrix. playwright_image: name: Resolve Playwright runtime image needs: changes if: needs.changes.outputs.run == 'true' runs-on: ubuntu-latest timeout-minutes: 5 outputs: digest: ${{ steps.resolve.outputs.digest }} ref: ${{ steps.resolve.outputs.ref }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Log in to GHCR if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Resolve immutable Playwright runtime image id: resolve run: | set -euo pipefail image=ghcr.io/kdlbs/kandev-ci:runtime-latest digest="$(bash .github/scripts/resolve-image-digest.sh "$image")" { echo "digest=$digest" echo "ref=$image@$digest" } >> "$GITHUB_OUTPUT" # Real-container E2E: exercises the Docker, SSH, and Kubernetes executors. # ubuntu-latest ships with Docker preinstalled, so no service container is # required. Container-bound tests are slow (up to ~3 min each), so we shard across # 6 runners — the build job creates a duration-aware manifest for this # cohort, and each runner executes its explicit file selection. # # This job runs the `containers` Playwright project. SSH uses Docker for a # real sshd target; Kubernetes uses a pinned, disposable Kind cluster. # Sharded 6 ways — each shard is its own runner. See apps/web/e2e/README.md. e2e-containers: name: E2E Containers Shard ${{ matrix.shard }}/6 needs: [changes, build, playwright_image] if: needs.changes.outputs.run == 'true' # This job intentionally runs on the host ubuntu runner (NOT the kandev-ci # container image): the containers project needs direct access to the # host Docker daemon, and a docker-in-docker setup would break the tests' # host-style bind mounts (paths under os.tmpdir() inside a runner container # wouldn't exist on the host where the docker daemon resolves them). # We still avoid the slow `playwright install --with-deps` step by copying # browsers out of the kandev-ci image — see the extract step below. runs-on: ubuntu-latest timeout-minutes: 50 strategy: fail-fast: false matrix: shard: [1, 2, 3, 4, 5, 6] steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Verify Docker is available run: docker info - name: Set up pnpm uses: pnpm/action-setup@a8198c4bff370c8506180b035930dea56dbd5288 # v5 with: version: "9.15.9" - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.local/share/pnpm/store key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} restore-keys: pnpm-${{ runner.os }}- - name: Install dependencies working-directory: apps run: pnpm install --frozen-lockfile - name: Start Playwright browser setup timer id: browser_setup_timer run: echo "started_at=$(date +%s)" >> "$GITHUB_OUTPUT" # The kandev-ci image is public, so fork PRs can pull it anonymously. # Their read-only GITHUB_TOKEN can still be rejected by GHCR during an # explicit login because it belongs to the base repository, not the # contributor's fork. Keep authenticated metadata and image requests for # trusted runs. # # A merge_group run has no `github.event.pull_request` at all, so the # condition decides it is a base-repository run. That is the right # answer: the queue branch lives in this repository and its token is a # base-repository token. - name: Log in to GHCR if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} # The runtime image remains the browser source of truth. The cache only # avoids copying its already-installed browsers on warm host runners. # Each run has a unique primary key so a repaired fallback can replace a # cache entry whose verification failed. The restore prefix selects the # newest compatible entry without allowing a rejected exact key to win. - name: Restore Playwright browser cache id: playwright_cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 continue-on-error: true with: path: /tmp/ms-playwright key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }} restore-keys: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}- - name: Configure Playwright browser path run: echo "PLAYWRIGHT_BROWSERS_PATH=/tmp/ms-playwright" >> "$GITHUB_ENV" # A cache hit is usable only after Chromium launches from the restored # directory. `cache-hit` is non-empty for both an exact hit and a # restore-key match. A corrupted or incompatible entry takes the normal # image extraction path below. - name: Verify cached Playwright browser id: playwright_cache_verify if: steps.playwright_cache.outputs.cache-hit != '' continue-on-error: true working-directory: apps/web run: | set -euo pipefail node <<'NODE' const { chromium } = require("@playwright/test"); (async () => { const browser = await chromium.launch({ headless: true }); const page = await browser.newPage(); await page.goto("data:text/html,ok"); await browser.close(); })().catch((error) => { console.error(error); process.exit(1); }); NODE echo "Playwright Chromium cache entry verified." # On a cache miss or failed cache verification, skip the Playwright CDN # download (~7 min and an outage-prone path) by copying the prebaked # browser out of the kandev-ci image. The runtime image is # `FROM mcr.microsoft.com/playwright:...` and ships it at /ms-playwright. # Pulling the resolved digest keeps the fallback tied to the image used # for the cache key; the build image has the same browser layer but is # ~300 MB larger. - name: Extract Playwright browsers from kandev-ci image id: playwright_extract if: steps.playwright_cache.outputs.cache-hit == '' || steps.playwright_cache_verify.outcome != 'success' env: PLAYWRIGHT_RUNTIME_REF: ${{ needs.playwright_image.outputs.ref }} run: | set -euo pipefail docker pull "$PLAYWRIGHT_RUNTIME_REF" mkdir -p /tmp/ms-playwright docker run --rm \ -v /tmp/ms-playwright:/dest \ "$PLAYWRIGHT_RUNTIME_REF" \ cp -a /ms-playwright/. /dest/ # Browser binaries come from the verified cache or the image above. # GitHub's Ubuntu runners usually already have the shared libs Chromium # needs, so smoke-test the copied browser and only hit apt as a bounded # fallback. - name: Verify Playwright browser runtime id: playwright_runtime_verify if: steps.playwright_cache.outputs.cache-hit == '' || steps.playwright_cache_verify.outcome != 'success' working-directory: apps/web run: | set -euo pipefail smoke_test() { node <<'NODE' const { chromium } = require("@playwright/test"); (async () => { const browser = await chromium.launch({ headless: true }); const page = await browser.newPage(); await page.goto("data:text/html,ok"); await browser.close(); })().catch((error) => { console.error(error); process.exit(1); }); NODE } if smoke_test; then echo "Playwright Chromium launches with existing runner packages." exit 0 fi echo "::warning::Playwright Chromium failed to launch; installing missing host packages." timeout 180s npx playwright install-deps chromium smoke_test - name: Save Playwright browser cache id: playwright_cache_save if: success() && (steps.playwright_cache.outputs.cache-hit == '' || steps.playwright_cache_verify.outcome != 'success') uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 continue-on-error: true with: path: /tmp/ms-playwright key: e2e-playwright-${{ runner.os }}-v1.61.1-noble-${{ needs.playwright_image.outputs.digest }}-${{ github.run_id }}-${{ github.run_attempt }} - name: Summarize Playwright browser setup if: always() env: CACHE_HIT: ${{ steps.playwright_cache.outputs.cache-hit }} CACHE_RESTORE_OUTCOME: ${{ steps.playwright_cache.outcome }} CACHE_VERIFY_OUTCOME: ${{ steps.playwright_cache_verify.outcome }} CACHE_EXTRACT_OUTCOME: ${{ steps.playwright_extract.outcome }} CACHE_SAVE_OUTCOME: ${{ steps.playwright_cache_save.outcome }} IMAGE_DIGEST: ${{ needs.playwright_image.outputs.digest }} SETUP_STARTED_AT: ${{ steps.browser_setup_timer.outputs.started_at }} run: | set -euo pipefail started="${SETUP_STARTED_AT:-$(date +%s)}" elapsed="$(( $(date +%s) - started ))" cache_state="miss" setup_mode="image-fallback" if [[ -n "${CACHE_HIT:-}" ]]; then cache_state="hit" fi if [[ -n "${CACHE_HIT:-}" && "${CACHE_VERIFY_OUTCOME:-}" == "success" ]]; then setup_mode="cache-hit" fi { echo "### Playwright browser setup" echo "- image-digest: ${IMAGE_DIGEST:-unresolved}" echo "- cache: ${cache_state}" echo "- mode: ${setup_mode}" echo "- restore: ${CACHE_RESTORE_OUTCOME:-not-run}" echo "- verification: ${CACHE_VERIFY_OUTCOME:-not-run}" echo "- extraction: ${CACHE_EXTRACT_OUTCOME:-not-run}" echo "- save: ${CACHE_SAVE_OUTCOME:-not-run}" echo "- elapsed: ${elapsed}s" } >> "$GITHUB_STEP_SUMMARY" - name: Download backend build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-backend-build path: apps/backend/bin/ - name: Make backend binaries executable run: chmod +x apps/backend/bin/* - name: Download E2E fixture plugin package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-plugin-package path: apps/backend/.build/ - name: Download web build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-web-build path: apps/web/ - name: Extract web build working-directory: apps/web run: tar -xf web-dist.tar && rm web-dist.tar - name: Enable strict flaky-test diagnostics if: github.event_name == 'workflow_dispatch' && inputs.fail_on_flaky run: echo "E2E_FAIL_ON_FLAKY=1" >> "$GITHUB_ENV" - name: Download E2E shard manifests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-shard-manifests path: e2e-manifests - name: Detect Kubernetes specs in this shard id: kubernetes env: MANIFEST_PATH: ${{ github.workspace }}/e2e-manifests/containers/${{ matrix.shard }}.json SHARD_INDEX: ${{ matrix.shard }} run: | set -euo pipefail node <<'NODE' const fs = require("node:fs"); const manifest = JSON.parse(fs.readFileSync(process.env.MANIFEST_PATH, "utf8")); const shard = manifest.shards.find((candidate) => candidate.index === Number(process.env.SHARD_INDEX)); if (!shard) throw new Error(`container shard ${process.env.SHARD_INDEX} is absent`); const hasKubernetes = shard.files.some((file) => file.startsWith("tests/kubernetes/")); fs.appendFileSync(process.env.GITHUB_OUTPUT, `has_kubernetes=${hasKubernetes}\n`); console.log(`Kubernetes specs assigned: ${hasKubernetes}`); NODE # Tool and node versions are intentionally coupled. The Kind fixture # verifies the downloaded checksums again before creating a cluster. - name: Install pinned Kind and kubectl if: steps.kubernetes.outputs.has_kubernetes == 'true' env: KIND_VERSION: v0.32.0 KIND_SHA256_AMD64: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 KUBERNETES_VERSION: v1.36.1 KUBECTL_SHA256_AMD64: 629d3f410e09bf49b64ae7079f7f0bda1191efed311f7d37fdbab0ad5b0ec2b7 KIND_NODE_IMAGE: kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5 run: | set -euo pipefail tool_dir="$RUNNER_TEMP/kandev-kubernetes-tools" mkdir -p "$tool_dir" curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" \ --output "$tool_dir/kind" curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \ --output "$tool_dir/kubectl" echo "${KIND_SHA256_AMD64} $tool_dir/kind" | sha256sum --check --strict echo "${KUBECTL_SHA256_AMD64} $tool_dir/kubectl" | sha256sum --check --strict chmod 0755 "$tool_dir/kind" "$tool_dir/kubectl" "$tool_dir/kind" version "$tool_dir/kubectl" version --client echo "Kind node image: $KIND_NODE_IMAGE" echo "KANDEV_E2E_KIND_BIN=$tool_dir/kind" >> "$GITHUB_ENV" echo "KANDEV_E2E_KUBECTL_BIN=$tool_dir/kubectl" >> "$GITHUB_ENV" echo "KANDEV_E2E_KIND_CLUSTER_NAME=kandev-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.shard }}" >> "$GITHUB_ENV" echo "KANDEV_E2E_KIND_OWNERSHIP_MARKER=$RUNNER_TEMP/kandev-kind-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.shard }}.owned" >> "$GITHUB_ENV" # Validate the same immutable build identity as the regular shards. - name: Run planned container-backed E2E tests shard ${{ matrix.shard }}/6 (Docker + SSH + Kubernetes) working-directory: apps/web env: KANDEV_E2E_CONTAINERS: "1" E2E_SHARD: ${{ matrix.shard }} KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/containers/${{ matrix.shard }}.json" - name: Delete exact Kind cluster if: always() && steps.kubernetes.outputs.has_kubernetes == 'true' run: | set -euo pipefail kind_bin="${KANDEV_E2E_KIND_BIN:-}" cluster_name="${KANDEV_E2E_KIND_CLUSTER_NAME:-}" ownership_marker="${KANDEV_E2E_KIND_OWNERSHIP_MARKER:-}" if [[ -z "$kind_bin" || -z "$cluster_name" || -z "$ownership_marker" ]]; then echo "Kind tools or exact target were never initialized; cleanup is a no-op." exit 0 fi if [[ ! -f "$ownership_marker" ]]; then echo "No reserved ownership marker for $cluster_name; cleanup is a no-op." exit 0 fi if [[ ! -x "$kind_bin" ]]; then echo "Pinned Kind tool is not executable: $kind_bin" >&2 exit 1 fi if [[ ! "$cluster_name" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; then echo "Refusing cleanup for invalid exact Kind cluster name: $cluster_name" >&2 exit 1 fi owned_name="$(tr -d '\r\n' < "$ownership_marker")" if ! [[ "$owned_name" == "$cluster_name" ]]; then echo "Refusing cleanup: marker owns '$owned_name', not '$cluster_name'." >&2 exit 1 fi "$kind_bin" delete cluster --name "$cluster_name" rm -f -- "$ownership_marker" # Named with the same `blob-report-*` prefix the merge job globs for # so the containers results land in the unified HTML report alongside # the sharded chromium/mobile-chrome runs. - name: Upload containers blob report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: blob-report-containers-${{ matrix.shard }} path: apps/web/e2e/blob-report/ retention-days: 3 - name: Upload containers E2E results if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: containers-e2e-test-results-${{ matrix.shard }} path: apps/web/e2e/test-results/ retention-days: 7 # API-only real-cluster smoke on the oldest and current supported Kubernetes # releases. The full 12-case browser lifecycle runs only once, above, on the # current release; this matrix keeps compatibility coverage intentionally small. e2e-kubernetes-compatibility: name: Kubernetes Compatibility (${{ matrix.kubernetes_version }}) needs: [changes, build] if: needs.changes.outputs.run == 'true' runs-on: ubuntu-latest timeout-minutes: 35 strategy: fail-fast: false matrix: include: - id: v1-34-8 kubernetes_version: v1.34.8 kubectl_sha256_amd64: f6249132865c13abe3c9dd5038f5da65849cb86eee1608c001831504e481aa8c node_image: kindest/node:v1.34.8@sha256:02722c2dedddcfc00febf5d27fbeb9b7b2c14294c82109ff4a85d89ac9ba3256 - id: v1-36-1 kubernetes_version: v1.36.1 kubectl_sha256_amd64: 629d3f410e09bf49b64ae7079f7f0bda1191efed311f7d37fdbab0ad5b0ec2b7 node_image: kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5 steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Verify Docker is available run: docker info - name: Set up pnpm uses: pnpm/action-setup@a8198c4bff370c8506180b035930dea56dbd5288 # v5 with: version: "9.15.9" - name: Set up Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: "24" - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.local/share/pnpm/store key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} restore-keys: pnpm-${{ runner.os }}- - name: Install dependencies working-directory: apps run: pnpm install --frozen-lockfile - name: Download backend build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-backend-build path: apps/backend/bin/ - name: Make backend binaries executable run: chmod +x apps/backend/bin/* - name: Download E2E fixture plugin package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-plugin-package path: apps/backend/.build/ - name: Download web build uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-web-build path: apps/web/ - name: Extract web build working-directory: apps/web run: tar -xf web-dist.tar && rm web-dist.tar - name: Install pinned Kind and kubectl env: KIND_VERSION: v0.32.0 KIND_SHA256_AMD64: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 KUBERNETES_VERSION: ${{ matrix.kubernetes_version }} KUBECTL_SHA256_AMD64: ${{ matrix.kubectl_sha256_amd64 }} KIND_NODE_IMAGE: ${{ matrix.node_image }} run: | set -euo pipefail tool_dir="$RUNNER_TEMP/kandev-kubernetes-tools-${{ matrix.id }}" mkdir -p "$tool_dir" curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" \ --output "$tool_dir/kind" curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \ --output "$tool_dir/kubectl" echo "${KIND_SHA256_AMD64} $tool_dir/kind" | sha256sum --check --strict echo "${KUBECTL_SHA256_AMD64} $tool_dir/kubectl" | sha256sum --check --strict chmod 0755 "$tool_dir/kind" "$tool_dir/kubectl" "$tool_dir/kind" version "$tool_dir/kubectl" version --client echo "Kind node image: $KIND_NODE_IMAGE" echo "KANDEV_E2E_KIND_BIN=$tool_dir/kind" >> "$GITHUB_ENV" echo "KANDEV_E2E_KUBECTL_BIN=$tool_dir/kubectl" >> "$GITHUB_ENV" echo "KANDEV_E2E_KUBERNETES_VERSION=$KUBERNETES_VERSION" >> "$GITHUB_ENV" echo "KANDEV_E2E_KIND_CLUSTER_NAME=kandev-e2e-compat-${{ matrix.id }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_ENV" echo "KANDEV_E2E_KIND_OWNERSHIP_MARKER=$RUNNER_TEMP/kandev-kind-compat-${{ matrix.id }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.owned" >> "$GITHUB_ENV" # Validate the build job's immutable source revision and artifact hashes. - name: Run Kubernetes API compatibility smoke working-directory: apps/web env: KANDEV_E2E_CONTAINERS: "1" KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} E2E_FAIL_ON_FLAKY: "1" run: >- pnpm exec playwright test --config e2e/playwright.config.ts --project=kubernetes-compat --reporter=list tests/kubernetes-compat - name: Delete exact Kind compatibility cluster if: always() run: | set -euo pipefail kind_bin="${KANDEV_E2E_KIND_BIN:-}" cluster_name="${KANDEV_E2E_KIND_CLUSTER_NAME:-}" ownership_marker="${KANDEV_E2E_KIND_OWNERSHIP_MARKER:-}" if [[ -z "$kind_bin" || -z "$cluster_name" || -z "$ownership_marker" ]]; then echo "Kind tools or exact target were never initialized; cleanup is a no-op." exit 0 fi if [[ ! -f "$ownership_marker" ]]; then echo "No reserved ownership marker for $cluster_name; cleanup is a no-op." exit 0 fi if [[ ! -x "$kind_bin" ]]; then echo "Pinned Kind tool is not executable: $kind_bin" >&2 exit 1 fi if [[ ! "$cluster_name" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; then echo "Refusing cleanup for invalid exact Kind cluster name: $cluster_name" >&2 exit 1 fi owned_name="$(tr -d '\r\n' < "$ownership_marker")" if ! [[ "$owned_name" == "$cluster_name" ]]; then echo "Refusing cleanup: marker owns '$owned_name', not '$cluster_name'." >&2 exit 1 fi "$kind_bin" delete cluster --name "$cluster_name" rm -f -- "$ownership_marker" - name: Upload Kubernetes compatibility results if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: kubernetes-compatibility-results-${{ matrix.id }} path: apps/web/e2e/test-results/ if-no-files-found: ignore retention-days: 7 desktop-e2e: name: Desktop E2E Smoke needs: changes if: needs.changes.outputs.run == 'true' runs-on: ubuntu-latest # The desktop image contains the Rust toolchain, Tauri dependencies, and # Xvfb. This job does not depend on hosted-runner apt mirrors. container: image: ghcr.io/kdlbs/kandev-ci:desktop-latest options: --ipc=host timeout-minutes: 45 defaults: run: shell: bash steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" # The desktop image uses /root as HOME, so this matches the store path # used by the other container jobs. - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.local/share/pnpm/store key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} restore-keys: pnpm-${{ runner.os }}- - name: Cache Rust build uses: Swatinem/rust-cache@42dc69e1aa15d09112580998cf2ef0119e2e91ae # v2 with: workspaces: apps/desktop/src-tauri -> target key: desktop-e2e - name: Install dependencies working-directory: apps run: pnpm install --frozen-lockfile - name: Run desktop smoke working-directory: apps run: pnpm --filter @kandev/desktop e2e e2e-report: name: Merge E2E Reports needs: [changes, e2e, e2e-containers, desktop-e2e] # `always()` so a failed shard still gets its report merged, but not when # change detection skipped the shards outright: there would be no blob # reports to download, and the artifact step would fail on an empty run. # `e2e-gate` below, not this job, is what the merge queue waits on. if: always() && needs.changes.outputs.run == 'true' # Report aggregation queries GitHub history and downloads artifacts with the # job token, so it remains on a GitHub-hosted runner during the pilot. runs-on: ubuntu-latest # Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright CLI. # `runtime` tag — no Go needed for merging blob reports. container: ghcr.io/kdlbs/kandev-ci:runtime-latest # 8 min (was 5) — the merge now also emits the JSON report used to # cross-check the flake count, and the flake baseline lookup walks recent # main runs looking for a published history artifact. timeout-minutes: 8 # Container jobs default to `sh`; pin to bash. defaults: run: shell: bash steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - name: Mark workspace safe for git run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Cache pnpm store uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.local/share/pnpm/store key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} restore-keys: pnpm-${{ runner.os }}- - name: Install dependencies working-directory: apps run: pnpm install --frozen-lockfile # The published runtime image may predate the report-tool dependency; # keep older images usable while the image rebuild adds unzip permanently. - name: Ensure timing report tools are installed run: | if ! command -v unzip >/dev/null 2>&1; then apt-get update apt-get install -y --no-install-recommends unzip rm -rf /var/lib/apt/lists/* fi - name: Download blob reports uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: path: apps/web/e2e/blob-reports pattern: blob-report-* merge-multiple: true - name: Download E2E shard manifests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 continue-on-error: true with: name: e2e-shard-manifests path: e2e-manifests - name: Download selected main timing profile uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 continue-on-error: true with: name: e2e-selected-main-timing-profile path: e2e/previous-profile - name: Collect timing and retry diagnostics if: always() working-directory: apps/web run: | set -euo pipefail if [[ -f "$GITHUB_WORKSPACE/e2e/previous-profile/timing-profile.json" ]]; then pnpm exec tsx e2e/scripts/e2e-timings.ts \ --input e2e/blob-reports \ --output e2e/timing-profile.json \ --web-root "$GITHUB_WORKSPACE/apps/web" \ --branch "$GITHUB_REF_NAME" \ --sha "$GITHUB_SHA" \ --run-id "$GITHUB_RUN_ID" \ --previous "$GITHUB_WORKSPACE/e2e/previous-profile/timing-profile.json" else pnpm exec tsx e2e/scripts/e2e-timings.ts \ --input e2e/blob-reports \ --output e2e/timing-profile.json \ --web-root "$GITHUB_WORKSPACE/apps/web" \ --branch "$GITHUB_REF_NAME" \ --sha "$GITHUB_SHA" \ --run-id "$GITHUB_RUN_ID" fi pnpm exec tsx e2e/scripts/retry-summary.ts \ --input e2e/blob-reports \ --output e2e/retry-summary.json \ --manifest-dir "$GITHUB_WORKSPACE/e2e-manifests" \ --attachment-output e2e/retry-artifacts \ --artifact-name e2e-timing-diagnostics \ --artifact-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#artifacts" - name: Write E2E timing summary if: always() working-directory: apps/web run: | node <<'NODE' const fs = require("node:fs"); const summaryPath = "e2e/retry-summary.json"; if (!fs.existsSync(summaryPath)) process.exit(0); const summary = JSON.parse(fs.readFileSync(summaryPath, "utf8")); const counts = summary.counts; const diagnosticTests = summary.tests.filter( (test) => test.attempts > 1 || !["passed", "skipped"].includes(test.finalStatus), ); const markdown = (value) => String(value).replaceAll("|", "\\|").replace(/\s+/g, " "); const lines = [ "## E2E timing and retry summary", "", "- Plan mode: " + summary.planning.mode, "- Unknown units: " + summary.planning.unknownUnits, "- Warm units: " + summary.planning.warmUnits, "- Stale profile units: " + summary.planning.staleUnits, "- Passed after retry: " + counts.passedAfterRetry, "- Final failures: " + (counts.failed + counts.timedOut), "", "| Cohort | Shard | Predicted (s) | Actual (s) | Delta (s) | Exit |", "| --- | ---: | ---: | ---: | ---: | ---: |", ...summary.shards.map( (shard) => "| " + shard.cohort + " | " + shard.shard + "/" + shard.shardCount + " | " + shard.predictedSeconds.toFixed(1) + " | " + shard.actualSeconds.toFixed(1) + " | " + shard.deltaSeconds.toFixed(1) + " | " + shard.exitCode + " |", ), ]; if (diagnosticTests.length > 0) { lines.push( "", "### Retry and failure details", "", "| Test | Attempts | Final status | Error | Artifacts |", "| --- | ---: | --- | --- | --- |", ...diagnosticTests.map((test) => { const links = test.attachments .map((attachment) => attachment.artifactUrl ? "[" + markdown(attachment.name + " — " + attachment.path) + "](" + attachment.artifactUrl + ")" : markdown(attachment.name), ) .join(", "); return ( "| `" + markdown(test.key) + "` | " + test.attempts + " | " + test.finalStatus + " | " + test.errorCategory + " | " + (links || "Not captured") + " |" ); }), ); } fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, lines.join("\n") + "\n"); NODE - name: Upload timing profile candidate if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-timing-profile path: apps/web/e2e/timing-profile.json if-no-files-found: ignore retention-days: 30 - name: Upload retry and shard timing summary if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-retry-summary path: apps/web/e2e/retry-summary.json if-no-files-found: ignore retention-days: 14 - name: Upload timing diagnostics bundle if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-timing-diagnostics path: | apps/web/e2e/timing-profile.json apps/web/e2e/retry-summary.json apps/web/e2e/retry-artifacts/ apps/web/e2e/blob-reports/*metadata.json if-no-files-found: ignore retention-days: 14 - name: Merge reports if: hashFiles('apps/web/e2e/blob-reports/**/*.zip') != '' working-directory: apps/web # `-c e2e/playwright.config.ts` is required because the blobs come from # two different absolute checkout paths: the containerized `e2e` shards # check out under `/__w/...` while `e2e-containers` runs on the host # runner and checks out under `/home/runner/work/...`. Playwright # embeds the absolute testDir in each blob and refuses to merge # reports recorded with different test directories — passing the # config makes it normalize against the config's `testDir` instead. # # The `json` reporter is Playwright's own verdict on the run: its # `stats.flaky` counts `TestCase.outcome() === "flaky"`. The flake report # below cross-checks our blob-derived count against it, so a parser bug # cannot quietly move the trended number. `merge-reports` unpacks # `report.jsonl` next to each `report.zip` it reads, so this must stay # after the blob-parsing steps above. env: PLAYWRIGHT_JSON_OUTPUT_FILE: e2e/merged-report.json run: npx playwright merge-reports --config e2e/playwright.config.ts --reporter=html,json ./e2e/blob-reports # The flake trend is carried run to run as an artifact: find the newest # `main` run that published one and use it as this run's baseline. # # This deliberately queries `status=completed`, not the `status=success` # the timing-profile lookup uses, and the asymmetry is the point. A # timing profile is only trustworthy from a run where everything passed, # but flake data is most interesting from runs that did not: restricting # to `success` would drop the history of every run that had a failure, # which is the opposite of what a flake trend is for. Do not "fix" this # to match the other lookup. - name: Locate latest main flake history id: flake-history-run if: always() env: GH_TOKEN: ${{ github.token }} GITHUB_API_URL: ${{ github.api_url }} GITHUB_REPOSITORY: ${{ github.repository }} run: | node <<'NODE' const fs = require("node:fs"); const api = async (url) => { const response = await fetch(url, { headers: { accept: "application/vnd.github+json", authorization: "Bearer " + process.env.GH_TOKEN, }, }); if (!response.ok) throw new Error(url + " -> " + response.status); return response.json(); }; (async () => { const base = process.env.GITHUB_API_URL + "/repos/" + process.env.GITHUB_REPOSITORY; const runs = await api( base + "/actions/workflows/e2e-tests.yml/runs?branch=main&status=completed&per_page=15", ); for (const run of runs.workflow_runs ?? []) { const artifacts = await api(base + "/actions/runs/" + run.id + "/artifacts?per_page=100"); const found = (artifacts.artifacts ?? []).some( (artifact) => artifact.name === "e2e-flake-history" && !artifact.expired, ); if (found) { fs.appendFileSync(process.env.GITHUB_OUTPUT, "run_id=" + run.id + "\n"); console.log("Using flake history from main run " + run.id + "."); return; } } fs.appendFileSync(process.env.GITHUB_OUTPUT, "run_id=\n"); console.log("No main run has published a flake history yet; this run seeds the trend."); })().catch((error) => { console.log("::warning::Unable to locate a flake history baseline: " + error.message); fs.appendFileSync(process.env.GITHUB_OUTPUT, "run_id=\n"); }); NODE - name: Download previous flake history if: always() && steps.flake-history-run.outputs.run_id != '' continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: e2e-flake-history path: e2e/previous-flake-history repository: ${{ github.repository }} run-id: ${{ steps.flake-history-run.outputs.run_id }} github-token: ${{ github.token }} - name: Write E2E flake report if: always() working-directory: apps/web run: | set -euo pipefail if [[ ! -f e2e/retry-summary.json ]]; then echo "No retry summary was produced; skipping the flake report." exit 0 fi history_args=() if [[ -f "$GITHUB_WORKSPACE/e2e/previous-flake-history/flake-history.json" ]]; then history_args=(--history "$GITHUB_WORKSPACE/e2e/previous-flake-history/flake-history.json") fi playwright_args=() if [[ -f e2e/merged-report.json ]]; then playwright_args=(--playwright-json e2e/merged-report.json) fi pnpm exec tsx e2e/scripts/flake-report.ts \ --summary e2e/retry-summary.json \ --output e2e/flake-history.json \ "${history_args[@]}" \ "${playwright_args[@]}" \ --run-id "$GITHUB_RUN_ID" \ --run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" \ --branch "$GITHUB_REF_NAME" \ --sha "$GITHUB_SHA" - name: Upload flake history if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-flake-history path: apps/web/e2e/flake-history.json if-no-files-found: ignore retention-days: 90 - name: Upload merged report uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: playwright-report path: apps/web/playwright-report/ retention-days: 14 - name: Verify required E2E jobs passed if: always() env: E2E_RESULT: ${{ needs.e2e.result }} E2E_CONTAINERS_RESULT: ${{ needs['e2e-containers'].result }} DESKTOP_E2E_RESULT: ${{ needs['desktop-e2e'].result }} run: | set -euo pipefail if [ "$E2E_RESULT" != "success" ]; then echo "e2e finished with result: $E2E_RESULT" >&2 exit 1 fi if [ "$E2E_CONTAINERS_RESULT" != "success" ]; then echo "e2e-containers finished with result: $E2E_CONTAINERS_RESULT" >&2 exit 1 fi if [ "$DESKTOP_E2E_RESULT" != "success" ]; then echo "desktop-e2e finished with result: $DESKTOP_E2E_RESULT" >&2 exit 1 fi # The required check. Runs on every event, including the ones where every job # above is correctly skipped, so the merge queue always has a conclusion to # read instead of a check that stays pending forever. # # A single gate is what makes this workflow usable from a ruleset at all: the # alternative is naming all 14 E2E shards, all 6 container shards, and the # desktop smoke individually, and renumbering the ruleset every time a shard # count changes. e2e-gate: name: E2E Tests Passed runs-on: ${{ fromJSON(needs.runner_plan.outputs.plan).e2e_gate_runner }} needs: [ changes, runner_plan, build, playwright_image, e2e, e2e-containers, e2e-kubernetes-compatibility, desktop-e2e, e2e-report, ] if: always() steps: - name: Require the E2E jobs to have passed or been skipped env: RUNNER_PLAN_RESULT: ${{ needs.runner_plan.result }} CHANGES_RESULT: ${{ needs.changes.result }} BUILD_RESULT: ${{ needs.build.result }} PLAYWRIGHT_IMAGE_RESULT: ${{ needs.playwright_image.result }} E2E_RESULT: ${{ needs.e2e.result }} E2E_CONTAINERS_RESULT: ${{ needs['e2e-containers'].result }} E2E_KUBERNETES_COMPATIBILITY_RESULT: ${{ needs['e2e-kubernetes-compatibility'].result }} DESKTOP_E2E_RESULT: ${{ needs['desktop-e2e'].result }} E2E_REPORT_RESULT: ${{ needs['e2e-report'].result }} run: | set -euo pipefail echo "Change detection: ${CHANGES_RESULT}" echo "Runner plan: ${RUNNER_PLAN_RESULT}" echo "Build: ${BUILD_RESULT}" echo "Playwright image: ${PLAYWRIGHT_IMAGE_RESULT}" echo "E2E shards: ${E2E_RESULT}" echo "E2E container shards: ${E2E_CONTAINERS_RESULT}" echo "Kubernetes compatibility: ${E2E_KUBERNETES_COMPATIBILITY_RESULT}" echo "Desktop smoke: ${DESKTOP_E2E_RESULT}" echo "Report merge: ${E2E_REPORT_RESULT}" # Runner planning must succeed because its outputs select the runner # fleet. A planner error cannot silently become a hosted fallback. if [[ "${RUNNER_PLAN_RESULT}" != "success" ]]; then echo "Runner planning did not succeed; refusing to report a pass." >&2 exit 1 fi # Change detection itself must succeed: if it errored we do not know # whether the skips below were deliberate. if [[ "${CHANGES_RESULT}" != "success" ]]; then echo "Change detection did not succeed; refusing to report a pass." >&2 exit 1 fi # `skipped` is a pass — the change-detection job decided nothing this # workflow covers was touched. `cancelled` is not: a cancelled job # never reached a verdict, and treating that as a pass would let an # untested merge group through. # # This gate is the one check a human opens when a merge is blocked, so # name the job that failed here rather than making them correlate the # list above against twenty shard names. for entry in \ "build:${BUILD_RESULT}" \ "playwright-image:${PLAYWRIGHT_IMAGE_RESULT}" \ "e2e:${E2E_RESULT}" \ "e2e-containers:${E2E_CONTAINERS_RESULT}" \ "e2e-kubernetes-compatibility:${E2E_KUBERNETES_COMPATIBILITY_RESULT}" \ "desktop-e2e:${DESKTOP_E2E_RESULT}" \ "e2e-report:${E2E_REPORT_RESULT}"; do name="${entry%%:*}" result="${entry#*:}" if [[ "${result}" != "success" && "${result}" != "skipped" ]]; then echo "${name} finished with result: ${result}" >&2 exit 1 fi done