ci: speed up e2e checks (#1361)

This commit is contained in:
Carlos Florêncio
2026-06-14 11:32:59 +01:00
committed by GitHub
parent 2dd03b5431
commit 0988087b9f
10 changed files with 112 additions and 132 deletions
+4 -4
View File
@@ -97,7 +97,7 @@ jobs:
retention-days: 1
e2e:
name: E2E Shard ${{ matrix.shard }}/10
name: E2E Shard ${{ matrix.shard }}/14
needs: build
runs-on: ubuntu-latest
# Pre-baked image bundles Node 24, pnpm 9.15.9, and the Playwright browser
@@ -136,7 +136,7 @@ jobs:
strategy:
fail-fast: false
matrix:
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
steps:
- name: Checkout code
@@ -180,9 +180,9 @@ jobs:
# executor and the SSH executor's sshd target). Without --project the
# container-bound specs would run here too, fail seeding (Docker not
# enabled for these shards), and spam every shard.
- name: Run E2E tests (shard ${{ matrix.shard }}/10)
- name: Run E2E tests (shard ${{ matrix.shard }}/14)
working-directory: apps/web
run: npx playwright test --config e2e/playwright.config.ts --project=chromium --project=mobile-chrome --shard=${{ matrix.shard }}/10
run: npx playwright test --config e2e/playwright.config.ts --project=chromium --project=mobile-chrome --shard=${{ matrix.shard }}/14
- name: Upload blob report
if: always()
+8 -3
View File
@@ -67,6 +67,13 @@ export async function waitForSessionEnvironment(
.toBe(options.expectedEnvironmentId);
}
export async function openTaskSession(page: Page, taskId: string): Promise<SessionPage> {
await page.goto(`/t/${taskId}`);
const session = new SessionPage(page);
await session.waitForLoad();
return session;
}
/**
* Seed a task + session and navigate to it, waiting for the first (normal)
* turn to complete. Follow-up prompts can then exercise retry flows from a
@@ -90,9 +97,7 @@ export async function seedIdleSession(
},
);
if (!task.session_id) throw new Error("createTaskWithAgent did not return a session_id");
await testPage.goto(`/t/${task.id}`);
const session = new SessionPage(testPage);
await session.waitForLoad();
const session = await openTaskSession(testPage, task.id);
await session.waitForChatIdle({ timeout: 30_000 });
return session;
}
+65 -38
View File
@@ -52,14 +52,64 @@ async function openChangesTab(testPage: Page) {
/** Click the file row for expansion_test.go to open its diff view. */
async function openExpansionFileDiff(testPage: Page) {
const fileRow = testPage
.locator("button, [role='button'], [class*='file']")
.filter({ hasText: "expansion_test.go" })
.first();
const fileRow = testPage.getByTestId("file-row-expansion_test.go");
await expect(fileRow).toBeVisible({ timeout: 10_000 });
await fileRow.click();
}
async function waitForDiffText(testPage: Page, text: string, timeout = 60_000) {
await expect
.poll(
() =>
testPage.evaluate((expected) => {
for (const container of document.querySelectorAll("diffs-container")) {
if (container.shadowRoot?.textContent?.includes(expected)) return true;
}
return false;
}, text),
{ timeout },
)
.toBe(true);
}
async function hoverUntilGutterSlotAppears(testPage: Page) {
const points = await testPage.evaluate(() => {
const container = document.querySelector("diffs-container");
const shadow = container?.shadowRoot;
if (!shadow) throw new Error("diffs-container shadow root missing");
const line = shadow.querySelector<HTMLElement>("[data-line]");
if (!line) throw new Error("no [data-line] found to hover");
const r = line.getBoundingClientRect();
const y = r.top + r.height / 2;
return [
{ x: r.left + 2, y },
{ x: r.left + 10, y },
{ x: r.left + Math.min(40, r.width / 4), y },
{ x: r.left + r.width / 2, y },
];
});
for (const point of points) {
await testPage.mouse.move(point.x, point.y);
const appeared = await testPage
.waitForFunction(
() =>
Boolean(
document
.querySelector("diffs-container")
?.shadowRoot?.querySelector("[data-gutter-utility-slot]"),
),
null,
{ timeout: 1_500 },
)
.then(() => true)
.catch(() => false);
if (appeared) return;
}
throw new Error("gutter-utility-slot did not appear after hover");
}
test.describe("Diff expansion — Pierre Diffs provider", () => {
test.describe.configure({ retries: 2, timeout: 120_000 });
@@ -73,7 +123,7 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
await openExpansionFileDiff(testPage);
await expect(testPage.locator("diffs-container")).toBeVisible({ timeout: 15_000 });
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({ timeout: 60_000 });
await waitForDiffText(testPage, "HUNK_TOP");
// Pierre's <pre data-diff> uses var(--diffs-bg); our unsafeCSS overrides
// that variable to var(--background) on :host. If the selector ever stops
@@ -112,7 +162,7 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
await openChangesTab(testPage);
await openExpansionFileDiff(testPage);
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({ timeout: 60_000 });
await waitForDiffText(testPage, "HUNK_TOP");
// Pierre 1.1.22 declares [data-gutter-utility-slot] as display:flex with
// top:0/bottom:0 but no align-items, so a fixed-size hover button pins to
@@ -147,7 +197,7 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
await openChangesTab(testPage);
await openExpansionFileDiff(testPage);
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({ timeout: 60_000 });
await waitForDiffText(testPage, "HUNK_TOP");
// Pierre appends the gutter-utility slot wrapper INSIDE the line's
// numberElement on pointer-move (InteractionManager.js: target.numberElement
@@ -157,27 +207,12 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
// calc(1ch - 1lh) on our slotted button — same trick pierre uses on its
// built-in [data-utility-button] — to push it outside the cell into the
// code area. Verify the button's right edge ends up past the cell's right.
const lineCentre = await testPage.evaluate(() => {
const container = document.querySelector("diffs-container");
const shadow = container?.shadowRoot;
if (!shadow) throw new Error("diffs-container shadow root missing");
const line = shadow.querySelector<HTMLElement>("[data-line]");
if (!line) throw new Error("no [data-line] found to hover");
const r = line.getBoundingClientRect();
return { x: r.left + r.width / 2, y: r.top + r.height / 2 };
});
await testPage.mouse.move(lineCentre.x, lineCentre.y);
await hoverUntilGutterSlotAppears(testPage);
const geometry = await testPage.evaluate(async () => {
const geometry = await testPage.evaluate(() => {
const container = document.querySelector("diffs-container")!;
const shadow = container.shadowRoot!;
const deadline = Date.now() + 5_000;
let slotWrapper: HTMLElement | null = null;
while (Date.now() < deadline) {
slotWrapper = shadow.querySelector<HTMLElement>("[data-gutter-utility-slot]");
if (slotWrapper && slotWrapper.parentElement) break;
await new Promise((r) => setTimeout(r, 50));
}
const slotWrapper = shadow.querySelector<HTMLElement>("[data-gutter-utility-slot]");
if (!slotWrapper) throw new Error("gutter-utility-slot did not appear after hover");
const numberCell = slotWrapper.parentElement!;
// The slot wrapper is appended INTO numberCell; our React-rendered button
@@ -216,10 +251,8 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
// On cold CI runners (first test in shard, no V8 code cache), resolveLanguagesAndExecuteTask
// triggers createJavaScriptRegexEngine() which can take 30-40s to JIT-compile.
// diffs-container mounts immediately but content appears only after the engine is ready.
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({ timeout: 60_000 });
await expect(testPage.getByText("HUNK_BOTTOM", { exact: false })).toBeVisible({
timeout: 5_000,
});
await waitForDiffText(testPage, "HUNK_TOP");
await waitForDiffText(testPage, "HUNK_BOTTOM", 5_000);
// Shiki renders each token as a <span style="color: #RRGGBB"> inside the
// diff's shadow DOM. If the worker pool is broken or the @pierre/diffs ↔
@@ -253,9 +286,7 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
await openChangesTab(testPage);
await openExpansionFileDiff(testPage);
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({
timeout: 15_000,
});
await waitForDiffText(testPage, "HUNK_TOP", 15_000);
// Pierre Diffs renders a separator between hunks showing the hidden line count.
// The separator contains img elements (chevron arrows) for expanding.
@@ -272,9 +303,7 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
await openChangesTab(testPage);
await openExpansionFileDiff(testPage);
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({
timeout: 15_000,
});
await waitForDiffText(testPage, "HUNK_TOP", 15_000);
// Wait for expand buttons to appear in the shadow DOM. They load
// asynchronously after full file content is fetched via WebSocket.
@@ -318,9 +347,7 @@ test.describe("Diff expansion — Pierre Diffs provider", () => {
await openExpansionFileDiff(testPage);
// Wait for both hunks and the separator to be present
await expect(testPage.getByText("HUNK_TOP", { exact: false })).toBeVisible({
timeout: 15_000,
});
await waitForDiffText(testPage, "HUNK_TOP", 15_000);
await expect(testPage.getByText(/\d+ unmodified lines/).first()).toBeVisible({
timeout: 20_000,
});
@@ -119,10 +119,10 @@ test.describe("PR checks running timer", () => {
const initialText = await durationEl.textContent();
expect(initialText).toContain("running");
// Wait ~3 seconds and verify the time has changed (incremented)
await testPage.waitForTimeout(3_000);
const updatedText = await durationEl.textContent();
expect(updatedText).toContain("running");
expect(updatedText).not.toBe(initialText);
// Verify the live timer increments without paying a fixed sleep.
await expect
.poll(async () => durationEl.textContent(), { timeout: 3_000 })
.not.toBe(initialText);
await expect(durationEl).toContainText("running");
});
});
@@ -1,5 +1,4 @@
import { test, expect } from "../../fixtures/test-base";
import { KanbanPage } from "../../pages/kanban-page";
import { SessionPage } from "../../pages/session-page";
test.describe("PR watcher missing branch", () => {
@@ -82,21 +81,9 @@ test.describe("PR watcher missing branch", () => {
state: "closed",
});
// --- Navigate to the kanban and click the task ---
const kanban = new KanbanPage(testPage);
await kanban.goto();
const card = kanban.taskCardByTitle("PR #999: Already merged feature");
await expect(card).toBeVisible({ timeout: 15_000 });
// Click → navigate to the session view. The card renders via SSR and its
// router.push click handler only fires once React has hydrated; under shard
// load the first click can land pre-hydration and no-op, so re-click until
// the URL flips. toPass stops on the first success, so no stray re-clicks.
await expect(async () => {
await card.click();
await expect(testPage).toHaveURL(/\/t\//, { timeout: 5_000 });
}).toPass({ timeout: 20_000 });
// --- Navigate to the task session view ---
await testPage.goto(`/t/${task.id}`);
await expect(testPage).toHaveURL(/\/t\//, { timeout: 15_000 });
const session = new SessionPage(testPage);
@@ -21,12 +21,12 @@ test.describe("Long prepare (slow git fetch)", () => {
}) => {
test.setTimeout(120_000);
// Sleep 25s on every `git fetch`/`git pull` run by the backend. The
// worktree fetch timeout is 90s (see worktree/manager.go) so 25s fits
// comfortably under it. 25s is well past the file-tree retry budget
// Sleep 22s on every `git fetch`/`git pull` run by the backend. The
// worktree fetch timeout is 90s (see worktree/manager.go) so 22s fits
// comfortably under it. 22s is still past the file-tree retry budget
// (~18s), which is what we want to exercise.
const delayFile = path.join(backend.tmpDir, "git-delay-ms");
fs.writeFileSync(delayFile, "25000");
fs.writeFileSync(delayFile, "22000");
try {
// Force the worktree executor path; otherwise the task resolves with an
@@ -60,10 +60,11 @@ test.describe("Long prepare (slow git fetch)", () => {
// Wait past the pre-fix retry budget (1+2+5+10 = 18s). The file tree
// must NOT have transitioned to the "manual" (Load Files) state —
// that's the regression.
// that's the regression. On faster CI runners the 22s fetch may already
// have completed by the time this assertion runs, so do not require the
// waiting state to still be visible here.
await testPage.waitForTimeout(19_000);
await expect(fileTreeManual).toHaveCount(0);
await expect(fileTreeWaiting).toBeVisible();
// Fetch eventually returns, worktree creation proceeds, agentctl
// becomes ready. File tree leaves the waiting state automatically.
@@ -1,28 +1,6 @@
import { test, expect } from "../../fixtures/test-base";
import { KanbanPage } from "../../pages/kanban-page";
import { SessionPage } from "../../pages/session-page";
/** Navigate to a kanban card by title and open its session page. */
async function openTaskSession(page: Parameters<typeof KanbanPage>[0], title: string) {
const kanban = new KanbanPage(page);
await kanban.goto();
const card = kanban.taskCardByTitle(title);
await expect(card).toBeVisible({ timeout: 15_000 });
await card.click();
await expect(page).toHaveURL(/\/t\//, { timeout: 15_000 });
const session = new SessionPage(page);
await session.waitForLoad();
// Reload to get fresh SSR state — the session may have started via API before
// the WS connection was established, causing the UI to miss "session is RUNNING"
// events. A reload guarantees hydrated state from the server.
await page.reload();
await session.waitForLoad();
return session;
}
import { openTaskSession } from "../../helpers/session";
/**
* Stop the running session via the tab context menu (session.stop → CANCELLED state),
@@ -63,7 +41,7 @@ test.describe("New session with deleted agent profile", () => {
});
// 2. Create task using the custom profile with a slow response so we can cancel it
await apiClient.createTaskWithAgent(
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Deleted Profile Resume Test",
profile.id,
@@ -76,7 +54,7 @@ test.describe("New session with deleted agent profile", () => {
);
// 3. Navigate to the task and stop the agent to get CANCELLED state
const session = await openTaskSession(testPage, "Deleted Profile Resume Test");
const session = await openTaskSession(testPage, task.id);
await stopRunningSession(session);
// 4. While on the page, force-delete the custom agent profile.
@@ -111,7 +89,7 @@ test.describe("New session with deleted agent profile", () => {
});
// 2. Create task using the custom profile with a slow response so we can cancel it
await apiClient.createTaskWithAgent(
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Deleted Profile Dialog Test",
profile.id,
@@ -124,7 +102,7 @@ test.describe("New session with deleted agent profile", () => {
);
// 3. Navigate to the task and stop the agent to get CANCELLED state
const session = await openTaskSession(testPage, "Deleted Profile Dialog Test");
const session = await openTaskSession(testPage, task.id);
await stopRunningSession(session);
// 4. While on the page, force-delete the custom agent profile.
@@ -1,21 +1,5 @@
import { test, expect } from "../../fixtures/test-base";
import { KanbanPage } from "../../pages/kanban-page";
import { SessionPage } from "../../pages/session-page";
import type { Page } from "@playwright/test";
async function openTaskSession(page: Page, title: string): Promise<SessionPage> {
const kanban = new KanbanPage(page);
await kanban.goto();
const card = kanban.taskCardByTitle(title);
await expect(card).toBeVisible({ timeout: 15_000 });
await card.click();
await expect(page).toHaveURL(/\/t\//, { timeout: 15_000 });
const session = new SessionPage(page);
await session.waitForLoad();
return session;
}
import { openTaskSession } from "../../helpers/session";
test.describe("Session resume boot-message dedup", () => {
// Test restarts the backend multiple times — can be flaky under CI load.
@@ -30,7 +14,7 @@ test.describe("Session resume boot-message dedup", () => {
test.setTimeout(180_000);
// 1. Create the task and wait for the initial agent turn to finish.
await apiClient.createTaskWithAgent(
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Resume Dedup Task",
seedData.agentProfileId,
@@ -42,7 +26,7 @@ test.describe("Session resume boot-message dedup", () => {
},
);
const session = await openTaskSession(testPage, "Resume Dedup Task");
const session = await openTaskSession(testPage, task.id);
await expect(session.chat.getByText("simple mock response", { exact: false })).toBeVisible({
timeout: 30_000,
});
@@ -53,10 +37,10 @@ test.describe("Session resume boot-message dedup", () => {
timeout: 15_000,
});
// 3. Restart the backend three times to produce three "Resumed agent" boot
// messages. The first two must be hidden by the dedup; only the third
// 3. Restart the backend twice to produce two "Resumed agent" boot
// messages. The first must be hidden by the dedup; only the second
// (most recent) should remain visible.
for (let i = 0; i < 3; i++) {
for (let i = 0; i < 2; i++) {
await backend.restart();
await testPage.reload();
await session.waitForLoad();
@@ -68,7 +52,7 @@ test.describe("Session resume boot-message dedup", () => {
});
}
// 4. Key assertion: despite three resumes, only the last "Resumed agent"
// 4. Key assertion: despite two resumes, only the last "Resumed agent"
// row should be rendered.
await expect(session.chat.getByText("Resumed agent Mock", { exact: false })).toHaveCount(1, {
timeout: 15_000,
@@ -14,11 +14,11 @@ test.describe("Task switcher sidebar status", () => {
* status until the user clicked them to trigger a fresh fetch.
*
* Setup:
* Inbox → Working (auto_start, message+15s delay, on_turn_complete → Done) → Done
* Inbox → Working (auto_start, message+4s delay, on_turn_complete → Done) → Done
* 3 tasks in Inbox.
*
* The mock agent sends a message immediately (triggers RUNNING / "Running"),
* then delays 15s (stays in RUNNING), then completes (→ WAITING_FOR_INPUT / "Turn Finished").
* then delays briefly (stays in RUNNING), then completes (→ WAITING_FOR_INPUT / "Turn Finished").
* Each task progresses through: Backlog → Running → Turn Finished.
*/
test("sidebar reflects real-time status updates and preserves state on task switch", async ({
@@ -38,10 +38,11 @@ test.describe("Task switcher sidebar status", () => {
const workingStep = await apiClient.createWorkflowStep(workflow.id, "Working", 1);
const doneStep = await apiClient.createWorkflowStep(workflow.id, "Done", 2);
// message first (triggers RUNNING), delay 15s, then complete
// Message first (triggers RUNNING), then a short delay that is still long
// enough for the sidebar to observe the background task's Running state.
await apiClient.updateWorkflowStep(workingStep.id, {
prompt:
'e2e:message("working...")\ne2e:delay(15000)\ne2e:message("task finished")\n{{task_prompt}}',
'e2e:message("working...")\ne2e:delay(4000)\ne2e:message("task finished")\n{{task_prompt}}',
events: {
on_enter: [{ type: "auto_start_agent" }],
on_turn_complete: [{ type: "move_to_step", config: { step_id: doneStep.id } }],
@@ -118,8 +119,6 @@ test.describe("Task switcher sidebar status", () => {
});
// --- Move Gamma to Working (background task) ---
// Small delay to let the backend settle after Beta's completion
await testPage.waitForTimeout(2_000);
await apiClient.moveTask(taskGamma.id, workflow.id, workingStep.id);
// Gamma should appear in "Running" while running
@@ -40,7 +40,7 @@ test.describe("Manual proceed to next workflow step", () => {
// Work: auto-start agent with a response we can assert on
await apiClient.updateWorkflowStep(workStep.id, {
prompt: 'e2e:delay(2000)\ne2e:message("work step response")\n{{task_prompt}}',
prompt: 'e2e:message("work step response")\n{{task_prompt}}',
events: {
on_enter: [{ type: "auto_start_agent" }],
},
@@ -85,7 +85,6 @@ test.describe("Manual proceed to next workflow step", () => {
await expect(proceedBtn).toBeVisible({ timeout: 10_000 });
// --- Enable plan mode manually (simulates being in a plan-mode workflow step) ---
await testPage.waitForTimeout(1_000);
await session.togglePlanMode();
await expect(session.planPanel).toBeVisible({ timeout: 15_000 });
await expect(session.planModeInput()).toBeVisible({ timeout: 10_000 });
@@ -116,7 +115,7 @@ test.describe("Manual proceed to next workflow step", () => {
* Previously, isMoving stayed true after clicking proceed because the button
* reappeared (for the next step) before isMoving was reset.
*
* Workflow: Spec -> Work -> Review -> QA -> Done (all with auto_start_agent)
* Workflow: Spec -> Work -> Done (all with auto_start_agent)
*/
test("proceed button re-enables across multiple step transitions", async ({
testPage,
@@ -130,7 +129,7 @@ test.describe("Manual proceed to next workflow step", () => {
"Multi Step Proceed Workflow",
);
const stepNames = ["Spec", "Work", "Review", "QA", "Done"];
const stepNames = ["Spec", "Work", "Done"];
const steps: { id: string; name: string }[] = [];
for (let i = 0; i < stepNames.length; i++) {
const step = await apiClient.createWorkflowStep(workflow.id, stepNames[i], i);