Files
kandev/apps/web/e2e/tests/i18n/pseudo-coverage.spec.ts
T

560 lines
27 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { type Page } from "@playwright/test";
import { test, expect } from "../../fixtures/test-base";
/**
* Pseudo-locale coverage oracle.
*
* Under the pseudo-locale every EXTRACTED message is accented
* (`Language` → `Ĺàńĝũàĝē`). So any user-facing copy that is still plain ASCII
* is, by definition, a string that was never wrapped in a Lingui macro. This
* spec crawls chrome-heavy screens and reports those leftovers.
*
* It runs TWO passes, because copy reaches the user by two routes:
* - visible text nodes, and
* - `COPY_ATTRIBUTES` — the copy a screen-reader user receives and a sighted
* one never sees. That pass is why the oracle is no longer blind to an
* `aria-label` that was never externalized; see docs/i18n.md.
*
* `SCREENS` mirrors `i18nGuardFiles` in eslint.i18n.options.mjs: a screen belongs
* here once the components that render it have been migrated. The eslint guard
* only sees plain literals in JSX, so it and this spec cover different halves of
* the same question — add to both in the PR that migrates a directory.
*
* Gated on `KANDEV_I18N_COVERAGE=1` rather than run in CI: with the migration
* proceeding one directory at a time, a screen's own copy can be clean while
* shared chrome it renders is not yet. It becomes a hard gate when the last
* directory lands and the env guard comes off.
*
* KANDEV_I18N_COVERAGE=1 pnpm e2e -- e2e/tests/i18n/pseudo-coverage.spec.ts
*
* BOTH passes walk `document.body`, deliberately, and NOT the page's `<main>`.
* Scoping to the page under test is the obvious reading of what each test claims,
* and it was tried and rejected on one fact: `@kandev/ui` builds Dialog,
* AlertDialog, DropdownMenu, Tooltip and Select on Radix `Portal`, which mounts
* to `document.body`. Every dialog, menu and tooltip in the app therefore renders
* OUTSIDE `<main>`, so a scoped walk would stop checking the surfaces densest in
* copy — and would do it silently, reporting clean. Trading a visible annoyance
* for an invisible blind spot is the wrong direction for an oracle whose whole
* purpose is the strings nothing else can see.
*
* The cost of body-wide is that a screen can fail for chrome it does not own.
* That cost is now small — migrating the sidebar, settings nav and status bar
* (#2214) took the baseline from 28 findings to 5 — and it buys the one thing
* nothing else has: copy that belongs to no directory, like `@kandev/ui`'s
* hardcoded breadcrumb landmark and sonner's toast-container label, is visible
* only because the walk is body-wide. The fix for chrome noise is migrating
* chrome, not narrowing the oracle.
*/
const COVERAGE_ENABLED = process.env.KANDEV_I18N_COVERAGE === "1";
/**
* Migrated screens whose visible text is overwhelmingly UI chrome, not user data.
*
* `allow` extends `ALLOWED` for one screen only. Use it for text the frontend
* must NOT translate but cannot avoid rendering — records the backend owns, or a
* product name — and say where the value comes from, so the exemption stays
* auditable instead of quietly widening into a place to hide missed strings.
*/
const SCREENS: Array<{ name: string; url: string; allow?: string[] }> = [
{ name: "settings — appearance", url: "/settings/general/appearance" },
{
name: "settings — notifications",
url: "/settings/general/notifications",
// Provider names are rows in the notification_providers table. The backend
// seeds these two (apps/backend/internal/notifications/service/service.go)
// and users name their own Apprise ones, so they are data on the same
// footing as a task title. `Apprise` labels the provider type.
allow: ["Desktop Notifications", "System Notifications", "Apprise"],
},
{ name: "settings — secrets", url: "/settings/general/secrets" },
{ name: "settings — terminal", url: "/settings/general/terminal" },
{
name: "settings — sprites",
url: "/settings/general/sprites",
// Product name of the sandbox provider.
allow: ["Sprites.dev"],
},
{
name: "settings — layouts",
url: "/settings/general/layouts",
// NOTE: everything here must be a PERSISTED name. This list is load-bearing
// in the wrong direction — a broad token also hides genuinely un-migrated
// copy that happens to match it. "Default" already masked the
// default-action button once, which rendered a raw English "Default" until
// `getDefaultActionState` was fixed to resolve it through the catalog.
//
// It fails in the OTHER direction too, and has: these entries duplicate
// strings that live in `lib/layout/layout-profiles.ts`, with nothing keeping
// the two in sync. #2198 changed the Default profile's description and this
// list kept the old wording, so it manufactured a phantom finding for every
// run between `cc6eb4dd5` and this commit. Re-check these against
// `BUILT_IN_LAYOUT_PROFILES` whenever a built-in name or description moves.
//
// Both groups are display strings that are also PERSISTED, so translating
// them in place would write locale-dependent values into a user's saved
// layouts and leave them there after a locale switch:
// - Built-in profile names/descriptions (lib/layout/layout-profiles.ts).
// `upsertBuiltInLayoutOverride` copies `builtIn.name` into the saved
// record the first time a built-in is customized.
// - Dockview panel titles (lib/state/layout-manager/constants.ts), which
// `toSerializedDockview` writes into the stored layout JSON. That path
// is already in `EXCLUDED` in scripts/externalize-strings.mjs.
// Localizing either needs a key/persisted-value split in those modules.
allow: [
"Default",
"Plan Mode",
"Preview Mode",
"VS Code",
"Agent with Files, Changes, and Terminal",
"Agent and Plan side by side",
"Agent and Browser side by side",
"Agent and VS Code side by side",
"Agent",
"Plan",
"Changes",
"Files",
"Browser",
"Terminal",
"PR Details",
"Merge Request",
],
},
{
name: "settings — keyboard shortcuts",
url: "/settings/general/keyboard-shortcuts",
// Modifier/key names label a physical key and are out of scope for
// translation — the same rule the eslint guard's keyboard pattern encodes.
//
// Shortcut names come from CONFIGURABLE_SHORTCUTS in
// lib/keyboard/shortcut-overrides.ts, a registry shared with the
// un-migrated voice-mode settings page; it migrates with that page.
allow: [
"Ctrl",
"Shift",
"Alt",
"Cmd",
"Meta",
"Space",
"Enter",
"Tab",
"Command Panel",
"Command Panel (Alt)",
"File Search",
"Search Task Contents",
"Quick Chat",
"Toggle Bottom Terminal",
"Toggle Sidebar",
"New Task",
"Focus Chat Input",
"Focus CLI Chat Input",
"Toggle Plan Mode",
"Recent Task Switcher",
"Recent Task Switcher (Backward)",
"Voice Input",
"Reverse Chat Search",
"Open Task Pull Request",
],
},
{ name: "settings — task actions", url: "/settings/general/task-actions" },
{ name: "settings — plugins", url: "/settings/plugins" },
// NOT YET: "settings — executors", "settings — account security",
// "settings — account tokens". All three routes' own copy is fully migrated
// and was verified by walking them under pseudo — including every dialog and
// both marketplace tabs, since a static page scan never opens a Radix portal.
// What stops them being entries here is what the fixture renders BESIDE that
// copy, and in each case an `allow` entry would be the wrong fix:
// - `/settings/executors` renders the seeded executor profiles' NAMES, and
// the e2e fixture happens to name them `Local` and `Worktree`. That is
// user data on exactly the footing as the workspace names below, so it
// needs `findUnlocalizedText` to stop treating user data as eligible —
// listing the two values would fix this fixture and leave every developer
// instance broken under different names. The hub cards' own `Docker` and
// `Sprites.dev` labels are brand nouns; they are in the guard's
// `words.exclude` but not in `ALLOWED`, which is a real sync gap in that
// list rather than something this migration should widen it to paper over.
// - Both account routes render `not found` — the router's 404 body, because
// the e2e profile does not register the auth routes, surfaced in the
// sessions/tokens error region. It is a backend diagnostic, English by
// design (docs/i18n.md, "What the backend deliberately does NOT
// translate"), and allowlisting a payload string would hide the next real
// miss in the same region.
// The `plugins` entry above has neither problem and passes with no `allow`.
// Its marketplace entries — names, descriptions, categories, source names and
// index.json URLs — come from the catalog rather than the DOM on load.
//
// NOT YET: "settings — integrations github", "… gitlab", "… jira",
// "… linear", "… sentry". Each page's
// own copy is fully migrated (verified by running this oracle against it —
// every string the integration owns renders accented).
//
// The settings-nav half of this blocker is now GONE: `settings-tree.tsx`,
// `workspaces-group.tsx`, `executors-group.tsx`, `account-group.tsx` and
// `theme-toggle.tsx` are migrated, so "Workspaces", "Integrations",
// "Automations", "Executors", "Voice Mode", "Utility Agents", "External MCP",
// "Plugins" and "Toggle theme" render accented on every screen. Only "System"
// is left, from `sections/settings/system-group.tsx`, which the System-routes
// migration owns.
//
// What still blocks these five entries is the shared integration chrome, not
// the nav. `components/integrations/**` is now migrated, which cleared
// `drafted-integration-enabled-control.tsx` ("Enabled"/"Disabled"),
// `auth-status-banner.tsx` ("Authenticated", "· checked <relative>") and the
// watcher card's loading state. What is left is
// `components/watcher-repository-fields.tsx` ("Repository", "Base Branch",
// "(no repository)"), `STEP_DEFAULT_LABEL` and `stepPlaceholder` — all shared
// with the un-migrated Azure DevOps surface — plus
// `components/integrations/settings-section.tsx` chrome and `@kandev/ui`'s
// built-in dialog "Close" label.
//
// NOT YET: "settings — external mcp", "… prompts", "… voice mode",
// "… utility agents". All four pages' own copy is fully migrated and was
// verified by running this oracle against each of them — every string those
// routes own renders accented, including the 43 in the MCP tool catalog
// (`lib/settings/external-mcp-tools.ts`), which is a `.ts` module no lint rule
// inspects. The collapsed tools preview, the utility agent dialog and the
// inference status note were scanned separately, since this spec only sees
// what a route renders on load.
//
// #2214 migrated the settings nav, which was the original blocker, and these
// four now come back clean of it. What still stops them being entries here is
// copy none of them owns:
// - `aria-label="breadcrumb"` from `@kandev/ui`'s Breadcrumb primitive, and
// Sonner's `Notifications alt+T` toast-region label. Both are the shared-UI
// case docs/i18n.md describes: they need a strings-provider seam in the
// package, not a per-route fix.
// - `Configuration Chat` from `components/config-chat/`, not yet migrated.
// Allowlisting either here would hide misses belonging to whoever owns them,
// which is the failure this list's own comment warns about. The workspace
// names these routes also surfaced are no longer among them: #2220 derives
// them from the boot payload and makes them ineligible, which is the right
// fix — they are user data, and listing them by value would have fixed the
// fixture and left every developer's own instance broken.
//
// Each route also renders data that is correctly English and would need an
// `allow` entry: the built-in prompts' names and bodies and the built-in
// utility agents' names and descriptions (both backend-authored), the agent
// product names and config paths on External MCP, and `Ctrl+Shift+M`.
//
// NOT YET: "settings — workspace workflows"
// (`/settings/workspace/:id/workflows`). The workflow editor's own copy is
// fully migrated and was verified with this oracle against a live instance,
// and the Workspaces branch of the nav it expands is now migrated too. What
// remains is that the route renders the workspace's own name and its
// workflow/step names — user data, on the same footing as the fixture's
// "E2E Workspace". Adding the entry needs `findUnlocalizedText` to stop
// treating user data as eligible, not an allowlist entry per fixture name.
];
/**
* Text that is legitimately un-accented under pseudo: brand/proper nouns, code
* identifiers, and units/symbols. Kept in sync with `words.exclude` in
* apps/web/eslint.i18n.options.mjs.
*/
const ALLOWED = [
"Kandev",
"GitHub",
"GitLab",
"Jira",
"Linear",
"Slack",
"Sentry",
"Azure DevOps",
"ACP",
"MCP",
"SSH",
"URL",
"ID",
"English",
"Pseudo",
"QA",
];
async function activatePseudo(page: Page, url: string) {
await page.goto(url);
await page.evaluate(() => {
document.cookie = "kandev_locale=pseudo; path=/; max-age=31536000; SameSite=Lax";
});
await page.reload();
await expect(page.locator("html")).toHaveAttribute("lang", "pseudo", { timeout: 15_000 });
}
/**
* Attributes that carry display copy rather than a value the app compares.
* Kept in sync with `jsx-attributes.include` in apps/web/eslint.i18n.options.mjs,
* so the guard and this oracle answer the same question about the same set: on an
* allowlisted path lint flags the literal, and here it is checked at render.
*
* `aria-labelledby` / `aria-describedby` / `aria-controls` are deliberately
* absent — they carry element ids, not prose, and are excluded by the guard too.
*
* `title` is checked on EVERY element, not just interactive ones. A `title` is
* exposed as an accessible name or description whatever the element is, and the
* browser renders its native tooltip on hover for any of them — so "the app never
* shows this one" is not true of a rendered `title`. Restricting it to
* interactive elements would need a tag/role/tabindex heuristic, and that
* heuristic would immediately become the place attribute copy hides. The guard
* checks `title` unconditionally too; the two stay in step.
*/
const COPY_ATTRIBUTES = ["aria-label", "aria-description", "title", "placeholder", "alt"];
/**
* Collect copy that looks un-externalized: plain-English visible text nodes, and
* plain-English values of the five copy-bearing attributes.
*
* Attribute findings are returned prefixed with the attribute and the element
* that carries it (`aria-label on button[data-testid=…]: "More information"`),
* because an attribute string is invisible on screen — a bare value gives you
* nothing to grep for and nowhere to look.
*
* Two counters make an empty `leftovers` mean something, because a selector that
* matched no elements reports exactly as clean as a clean screen:
* - `inspectedAttributes` — non-empty attribute values examined. Asserted per
* screen; this is the one that catches a selector matching nothing.
* - `localizedAttributes` — those that rendered FULLY accented, i.e. migrated
* attribute copy the pass demonstrably reached. Not asserted per screen: a
* page of plain inputs and text can legitimately have none, and secrets,
* terminal and sprites do. Pinned once instead, in its own test below.
*/
async function findUnlocalizedCopy(
page: Page,
allowed: string[],
): Promise<{ leftovers: string[]; localizedAttributes: string[]; inspectedAttributes: number }> {
return page.evaluate(
({ allowedList, copyAttributes }) => {
const skipTags = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "CODE", "PRE", "SVG"]);
// A narrower set for attributes. `skipTags` above encodes "this element's
// CONTENT is not prose", which is a different question from whether its
// LABEL is: an `aria-label` on an <svg> icon, or a `title` on a <code>
// chip, is copy the user receives either way. Only the three tags that
// cannot present a label at all are skipped here.
const attrSkipTags = new Set(["SCRIPT", "STYLE", "NOSCRIPT"]);
const wordlike = /[A-Za-z]{4,}/;
const accented = /[À-ɏ]/;
// User data is never copy, and a workspace's name is user data on the same
// footing as a task title. Derived from the boot payload rather than
// listed, because listing it fixes one instance and leaves every other
// one broken: `E2E Workspace` is only the fixture's name, and a developer
// running this against their own instance would have hit the identical
// false positive under a different string.
const workspaceNames = (() => {
const payload = (window as unknown as { __KANDEV_BOOT_PAYLOAD__?: unknown })
.__KANDEV_BOOT_PAYLOAD__;
const state = (payload as { initialState?: { workspaces?: { items?: unknown } } })
?.initialState;
const items = state?.workspaces?.items;
if (!Array.isArray(items)) return [];
return items
.map((item) => (item as { name?: unknown })?.name)
.filter((name): name is string => typeof name === "string" && name.trim().length > 0);
})();
// Longest first: stripping "VS Code" before "Agent and VS Code side by
// side" would leave "side by side" behind and report it as a leftover.
const tokens = [...allowedList, ...workspaceNames].sort((a, b) => b.length - a.length);
/** Still word-like ASCII once allowlisted tokens are removed. */
const hasUnmigratedAscii = (text: string) => {
if (!text || !wordlike.test(text)) return false;
let residue = text;
for (const token of tokens) residue = residue.split(token).join(" ");
return wordlike.test(residue);
};
/** The text pass's rule, unchanged: any accented character clears a node. */
const looksUnlocalized = (text: string) => !accented.test(text) && hasUnmigratedAscii(text);
// The text pass drops zero-size nodes because nothing is on screen to read.
// That test is WRONG for an attribute: a visually hidden control with an
// `aria-label` is precisely the case this pass exists to catch, and every
// `sr-only` node measures as good as zero. What actually disqualifies a
// label is the element not being rendered at all — `display: none`,
// `visibility: hidden`, or a skipped `content-visibility` subtree — because
// then no user receives it, sighted or not. `checkVisibility` asks that
// directly; deliberately WITHOUT `opacityProperty`, since an opacity-0
// element is still in the accessibility tree.
const isRendered = (el: Element) =>
typeof el.checkVisibility !== "function" ||
el.checkVisibility({ contentVisibilityAuto: true, visibilityProperty: true });
const collectText = () => {
const found = new Set<string>();
const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT);
let node: Node | null;
while ((node = walker.nextNode())) {
const text = (node.textContent ?? "").trim();
if (!looksUnlocalized(text)) continue;
const el = node.parentElement;
if (!el || skipTags.has(el.tagName)) continue;
// Same rendered-or-not test the attribute pass uses, rather than the
// zero-size rect this used to apply. The rect check kept `sr-only`
// text ONLY because that utility measures 1px rather than 0 — so
// whether screen-reader-only copy was checked came down to a CSS
// implementation detail, and changing `.sr-only` to `clip` at 0×0
// would have silently dropped it. It IS copy, and a deliberately
// in-scope kind: `theme-toggle.tsx`'s "Toggle theme" reached a user
// only this way. `checkVisibility` keeps it for the right reason —
// the element renders and is in the accessibility tree — while
// correctly dropping `visibility: hidden`, which the rect check kept.
if (!isRendered(el)) continue;
found.add(text.slice(0, 120));
}
return [...found];
};
/** Enough to find the element in the DOM and in the source. */
const describe = (el: Element) => {
const tag = el.tagName.toLowerCase();
const testId = el.getAttribute("data-testid");
if (testId) return `${tag}[data-testid=${testId}]`;
if (el.id) return `${tag}#${el.id}`;
const role = el.getAttribute("role");
return role ? `${tag}[role=${role}]` : tag;
};
const seen = new Map<string, { label: string; count: number }>();
// The positive control. An attribute that HAS been migrated renders as
// `Mōŕē ĩńfōŕmàţĩōń`, which contains no 4-letter ASCII run — so it is
// dropped at the `wordlike` gate and never even reaches the `accented`
// test. Soundness is unaffected (a real miss is pure ASCII and is still
// caught), but it means a working pass and a pass whose selector matched
// NOTHING produce identical empty output. Collecting the accented values
// proves the pass actually reached migrated attribute copy on this
// screen, so the caller can tell those two apart.
const localized: string[] = [];
let inspected = 0;
const inspectAttribute = (el: Element, attr: string) => {
// An empty value is never a missed string, and `alt=""` is load-bearing:
// it marks an image as decorative so screen readers skip it. Reporting
// one would push authors into writing alt text for spacers.
const value = (el.getAttribute(attr) ?? "").trim();
if (!value) return;
inspected += 1;
// NOTE the divergence from the text pass, which clears a node the moment
// it contains one accented character. That rule hides a real shape here:
// an `aria-label` built as "Collapse ${label}" renders
// "Collapse Ĝēńēŕàĺ" — an un-migrated English FRAME around a migrated
// value, which accent-clears-all would call done. Allowlist stripping is
// what separates it from the legitimate case, where the ASCII fragment is
// the allowlisted DATA rather than the frame ("Àćţĩōńś ƒōŕ Agent" on the
// layouts screen strips to nothing).
if (!hasUnmigratedAscii(value)) {
// Fully accented, nothing ASCII left: a migrated attribute, and the
// positive control.
if (accented.test(value)) {
localized.push(`${attr} on ${describe(el)}: "${value.slice(0, 60)}"`);
}
return;
}
// Keyed by attribute+value, not by element: one un-migrated shared
// component renders on twenty rows, and twenty identical findings bury
// the other nineteen strings. The first element carrying it is the
// sample you go and look at.
const key = `${attr} ${value}`;
const hit = seen.get(key);
if (hit) {
hit.count += 1;
return;
}
const label = `${attr} on ${describe(el)}: "${value.slice(0, 120)}"`;
seen.set(key, {
label: accented.test(value) ? `${label} - English frame, migrated value` : label,
count: 1,
});
};
const collectAttributes = () => {
const selector = copyAttributes.map((attr) => `[${attr}]`).join(",");
for (const el of Array.from(document.querySelectorAll(selector))) {
if (attrSkipTags.has(el.tagName.toUpperCase()) || !isRendered(el)) continue;
for (const attr of copyAttributes) inspectAttribute(el, attr);
}
const leftovers = [...seen.values()].map(({ label, count }) =>
count > 1 ? `${label} (×${count})` : label,
);
return { leftovers, localized, inspected };
};
const attributes = collectAttributes();
return {
leftovers: [...collectText(), ...attributes.leftovers],
localizedAttributes: attributes.localized,
inspectedAttributes: attributes.inspected,
};
},
{ allowedList: allowed, copyAttributes: COPY_ATTRIBUTES },
);
}
test.describe("i18n pseudo-locale coverage", () => {
test.skip(
!COVERAGE_ENABLED,
"Set KANDEV_I18N_COVERAGE=1 to run the string-externalization oracle (hard gate at task-40).",
);
for (const screen of SCREENS) {
test(`no un-externalized copy on ${screen.name}`, async ({ testPage }) => {
await activatePseudo(testPage, screen.url);
// Let lazy panels settle before scanning.
await testPage.waitForTimeout(1_000);
const { leftovers, localizedAttributes, inspectedAttributes } = await findUnlocalizedCopy(
testPage,
[...ALLOWED, ...(screen.allow ?? [])],
);
// Control FIRST. A migrated attribute renders accented and so is invisible
// to the leftover check by construction, which means a screen that is
// clean and a selector that matched NOTHING report identically. Assert the
// pass actually read attributes before believing what it did not find.
expect(
inspectedAttributes,
`The attribute pass examined no attribute at all on ${screen.name}, so a ` +
`green leftover check proves nothing about attribute copy here.`,
).toBeGreaterThan(0);
expect(
leftovers,
`Un-externalized strings on ${screen.name}:\n${leftovers.map((s) => ` - ${s}`).join("\n")}` +
`\n\nAttribute pass examined ${inspectedAttributes} attribute(s), of which ` +
`${localizedAttributes.length} rendered fully accented.`,
).toEqual([]);
});
}
/**
* The positive control, pinned once rather than per screen.
*
* `inspectedAttributes` above proves the selector matched; it does not prove
* the pass can tell migrated attribute copy apart from a miss. This does, on
* the one element we know is migrated: `language-settings.tsx` renders
* `aria-label={t("settings:displayLanguage")}` on `#language-select`, so under
* pseudo it must come back accented. If this test fails while the screens
* above pass, the accent detection is broken and every green screen is
* meaningless — which is the failure this whole file exists to not have.
*
* A per-screen version of this assertion was tried and dropped: secrets,
* terminal and sprites legitimately render no fully-accented attribute at all,
* so it fired on three screens that were fine.
*/
test("the attribute pass recognizes migrated attribute copy", async ({ testPage }) => {
await activatePseudo(testPage, "/settings/general/appearance");
await testPage.waitForTimeout(1_000);
const { localizedAttributes } = await findUnlocalizedCopy(testPage, ALLOWED);
expect(
localizedAttributes.join("\n"),
`Expected the migrated aria-label on #language-select to render accented. Got:\n` +
localizedAttributes.join("\n"),
).toContain("language-select");
});
});