kandev/apps/web/scripts/lib/e2e-sleep-wiring.test.ts

/**
 * The ratchet is actually WIRED UP.
 *
 * Every other test here proves the matcher is right and the diff attribution is
 * right. None of them would notice if the CI step, the pre-commit hook or the
 * script's own call to `findNewSleeps` were deleted — the rule would still be
 * correct, its tests would still pass, and the gate would silently never run
 * again. That is the worst failure mode available to a check: it reports a clean
 * tree forever, in the one place nobody thinks to look.
 *
 * So these assert the call sites, not the logic. They are the mutants
 * "delete the CI step", "delete the hook", and "delete the call while keeping
 * the function".
 *
 * Matching is on the script FILENAME rather than on surrounding YAML structure,
 * so reformatting, renaming a step or reordering the workflow does not break
 * them — only actually removing the invocation does.
 */
import fs from "node:fs";
import path from "node:path";

import { ESLint } from "eslint";
import { describe, expect, it } from "vitest";

import {
  e2eSleepSeededGuardDirs,
  SLEEP_EXEMPT_FILES,
  SLEEP_RULE_ID,
} from "../../eslint-rules/no-unsanctioned-sleep.mjs";

const WEB_DIR = path.resolve(import.meta.dirname, "../..");
const REPO_ROOT = path.resolve(WEB_DIR, "../..");

const RATCHET_SCRIPT = "check-new-e2e-sleeps.mjs";

/** ESLint reports a resolved severity numerically; 2 is `"error"`. */
const RULE_ERROR = 2;

/**
 * One ESLint instance for every config-resolution test below.
 *
 * Resolving the first file loads the whole flat config and its plugins, which
 * costs several seconds; later calls hit the instance's cache. Building one
 * instance per test pushed the first past vitest's 5s default and failed it on
 * timeout — a red that says nothing about the guard. Hence a shared instance and
 * an explicit timeout on the tests that use it.
 */
const configFor = (() => {
  let eslint: ESLint | null = null;
  return (file: string) => (eslint ??= new ESLint({ cwd: WEB_DIR })).calculateConfigForFile(file);
})();

const SLEEP_GUARD_TIMEOUT = 60_000;

/** The resolved severity of the sleep rule for `file`, or `undefined` if unset. */
async function sleepSeverity(file: string) {
  const config = await configFor(file);
  return config.rules?.[SLEEP_RULE_ID];
}

const read = (rel: string) => fs.readFileSync(path.join(REPO_ROOT, rel), "utf8");

describe("sleep ratchet wiring", () => {
  /**
   * Both invocations, not just the filename.
   *
   * `toContain(RATCHET_SCRIPT)` alone was too weak: the step has a
   * `pull_request` branch and a `push` branch, so deleting either one left the
   * other's mention of the filename and the assertion still passed. A mutation
   * run caught it. Count the call sites instead.
   */
  it("runs in CI, in the frontend workflow, on both event paths", () => {
    const workflow = read(".github/workflows/frontend-tests.yml");
    expect(workflow).toMatch(/^\s+- name: Run E2E sleep ratchet$/m);
    const calls = workflow.match(new RegExp(`node scripts/${RATCHET_SCRIPT}`, "g")) ?? [];
    expect(calls).toHaveLength(2);
  });

  /**
   * `pull_request` must NOT pass `--base`. The checkout is the merge ref and
   * `pull_request.base.sha` is an ancestor of it, so handing it in makes every
   * commit main landed since the PR opened read as this PR's new code (#2160
   * did exactly this to the i18n ratchet).
   */
  it("does not pass --base on pull_request in CI", () => {
    const workflow = read(".github/workflows/frontend-tests.yml");
    const step = workflow.slice(workflow.indexOf("Run E2E sleep ratchet"));
    // Match the pull_request branch structurally rather than slicing at the
    // first `BASE_SHA=` literal: a future YAML comment mentioning that name
    // would move the slice boundary and quietly shrink what is asserted
    // (review finding). This is strictly tighter than the slice was — it pins
    // the invocation, its bare form, and the early exit, in order.
    //
    // How the event name is read is deliberately NOT pinned. It was
    // `"${{ github.event_name }}"` inline until the merge-queue work moved it
    // to an `EVENT_NAME` env var, which changed nothing about the guarantee
    // here and broke this assertion anyway. What matters is that the
    // pull_request branch runs the ratchet bare and returns: the trailing
    // newline after the script name is what rejects a smuggled `--base`.
    expect(step).toMatch(
      new RegExp(
        `== "pull_request" \\]\\];? then\\n` +
          `\\s+node scripts/${RATCHET_SCRIPT}\\n` +
          `\\s+exit 0\\n`,
      ),
    );
  });

  /**
   * The hook id is matched to end-of-line. `toContain("id: e2e-sleep-ratchet")`
   * was satisfied by `id: e2e-sleep-ratchet-disabled`, so renaming the hook to
   * switch it off passed the test — a mutation run caught it.
   */
  it("runs as a pre-commit hook on e2e changes", () => {
    const config = read(".pre-commit-config.yaml");
    expect(config).toMatch(/^\s+- id: e2e-sleep-ratchet$/m);
    expect(config).toContain(RATCHET_SCRIPT);
    // The hook must fire for the tree it guards.
    expect(config).toContain("^apps/web/e2e/.*\\.tsx?$");
  });

  it("is reachable as a package script", () => {
    const pkg = JSON.parse(read("apps/web/package.json"));
    expect(pkg.scripts["e2e:sleep-ratchet"]).toContain(RATCHET_SCRIPT);
    expect(pkg.scripts["e2e:waits"]).toContain("report-e2e-waits.mjs");
    expect(pkg.scripts["lint:e2e-sleeps"]).toContain("eslint.e2e-sleeps.config.mjs");
  });

  /**
   * The CLI must actually invoke the decision function. A shell that imports it
   * and then exits 0 unconditionally is the "delete the call, keep the function"
   * mutant, and it looks completely healthy from the outside.
   */
  it("the CLI calls findNewSleeps and exits non-zero on violations", () => {
    const cli = read(`apps/web/scripts/${RATCHET_SCRIPT}`);
    expect(cli).toContain("await findNewSleeps(");
    expect(cli).toContain("process.exit(1)");
    // The failure exit must be reachable from the violations branch, not
    // orphaned below an unconditional success exit.
    expect(cli.indexOf("violations.length === 0")).toBeLessThan(cli.indexOf("process.exit(1)"));
  });

  /**
   * The eslint rule is registered in the main config too, on the directories
   * already converted. Losing this registration removes editor feedback and the
   * graduation path, without any test elsewhere noticing.
   */
  it("registers the rule in the main eslint config", () => {
    const config = read("apps/web/eslint.config.mjs");
    expect(config).toContain("e2eSleepGuardFiles");
    expect(config).toContain('"e2e-sleeps/no-unsanctioned-sleep": "error"');
  });

  /**
   * The guard may only grow.
   *
   * This asked ESLint's own config resolution rather than grepping the config
   * text for glob literals. The string form only held while the guard was a
   * list of directory globs: the graduation to a single `e2e/**` entry STRICTLY
   * WIDENED coverage, yet a containment assertion reads that as seven deletions
   * and fails. A test that fails on a tightening is one somebody edits to make
   * the build pass, which is precisely the pressure this is meant to resist.
   *
   * `calculateConfigForFile` answers the question the guard actually exists to
   * answer — "is a sleep in this file an error?" — so it survives any rewrite of
   * the globs and still fails on a real narrowing.
   */
  it(
    "makes a sleep an error in every seeded guard directory",
    async () => {
      for (const dir of e2eSleepSeededGuardDirs) {
        expect(await sleepSeverity(`${dir}/probe.spec.ts`), `${dir} lost its sleep guard`).toEqual([
          RULE_ERROR,
        ]);
      }
    },
    SLEEP_GUARD_TIMEOUT,
  );

  /**
   * The graduation itself: the directories that were NOT on the original seed
   * list are covered now. Without this, narrowing the guard back to the old
   * seven-entry allowlist would pass every other test in this file.
   */
  it(
    "covers the directories the conversion added after the seed",
    async () => {
      for (const dir of [
        "e2e/tests/task",
        "e2e/tests/session",
        "e2e/tests/settings",
        "e2e/helpers",
        "e2e/fixtures",
        "e2e/pages",
      ]) {
        expect(await sleepSeverity(`${dir}/probe.spec.ts`), `${dir} is not guarded`).toEqual([
          RULE_ERROR,
        ]);
      }
    },
    SLEEP_GUARD_TIMEOUT,
  );

  /**
   * The exemption still applies. `causal-waits.ts` implements both banned forms,
   * so a guard that covered it would make the sanctioned helper the one file
   * that can never satisfy the rule it exists to serve.
   */
  it(
    "still exempts the causal-waits helper and its own test",
    async () => {
      for (const file of SLEEP_EXEMPT_FILES) {
        expect(await sleepSeverity(file), `${file} should be exempt`).toBeUndefined();
      }
    },
    SLEEP_GUARD_TIMEOUT,
  );
});