kandev/apps/web/e2e/tests/office/office-routing-fallback.spec.ts

import { test, expect } from "../../fixtures/office-fixture";
import { balancedExecutionProfileRouting } from "../../helpers/office-routing";

/**
 * Phase 7 spec #2 — provider fallback under quota_limited.
 *
 * Scenario (per docs/specs/office/requirements/routing.md):
 *   1. Workspace routing enabled with order `claude-acp → codex-acp`,
 *      tier `balanced`.
 *   2. Backend is launched with `KANDEV_PROVIDER_FAILURES=claude-acp:
 *      quota_limited` so the classifier short-circuits to `quota_limited`
 *      for any claude-acp launch.
 *   3. A task is started; the dispatcher records a failed attempt for
 *      claude-acp (outcome=failed_provider_unavailable, error_code=
 *      quota_limited), marks provider health as degraded, then falls
 *      back to codex-acp which launches successfully.
 *   4. Run detail header surfaces the fallback reason; dashboard
 *      provider-health card shows claude-acp as degraded.
 *
 * Per-test injection is handled by restarting the backend with the
 * new env var. The `routingerr.LoadInjectionFromEnv` map is read once
 * via sync.Once at process start, so per-test changes require a
 * backend restart — `backend.restart({ KANDEV_PROVIDER_FAILURES: ... })`
 * is the supported escape hatch from the worker fixture.
 */
test.describe("Office provider routing — fallback", () => {
  test.beforeEach(async ({ apiClient, officeSeed }) => {
    // Routing tests share a worker; the e2eReset wipes routing state
    // (provider_health, route_attempts, runs) so each test starts with
    // a clean catalogue.
    await apiClient.e2eReset(officeSeed.workspaceId, [officeSeed.workflowId]);
  });

  // Restart back to baseline env so KANDEV_MOCK_PROVIDERS /
  // KANDEV_PROVIDER_FAILURES don't leak into sibling specs.
  test.afterAll(async ({ backend }) => {
    await backend.restart();
  });

  test("claude-acp quota fails over to codex-acp", async ({
    backend,
    apiClient,
    officeApi,
    officeSeed,
  }) => {
    test.setTimeout(90_000);
    await backend.restart({
      KANDEV_MOCK_PROVIDERS: "claude-acp,codex-acp,opencode-acp",
      KANDEV_PROVIDER_FAILURES: "claude-acp:quota_limited",
    });

    const routingConfig = await balancedExecutionProfileRouting(
      apiClient,
      officeApi,
      officeSeed.workspaceId,
      ["claude-acp", "codex-acp"],
    );
    await officeApi.updateRouting(officeSeed.workspaceId, routingConfig);

    const task = (await officeApi.createTask(officeSeed.workspaceId, "Fallback test", {
      workflow_id: officeSeed.workflowId,
    })) as { id?: string };
    expect(task.id).toBeTruthy();
    // Office's task→run dispatcher is event-driven off task_assigned;
    // the bare createTask call leaves the task unassigned, so it never
    // hits the routing path. PATCH the assignee to fire the pipeline.
    await officeApi.assignTask(task.id!, officeSeed.agentId);

    // Poll for the run + its attempt list. The first attempt should be
    // claude-acp failed_provider_unavailable; the second codex-acp
    // launched. The scheduler claims runs within a few ticks normally,
    // but a parallel-suite run that just restarted the backend with new
    // KANDEV_MOCK_PROVIDERS needs the full agent-registry rediscovery
    // walk to complete first — 40s rides that out without affecting
    // the happy path (resolves in <5s in isolation).
    let attempts: Array<{
      provider_id: string;
      execution_profile_id: string;
      outcome: string;
      error_code?: string;
    }> = [];
    // `attempts` is captured on each poll so the assertions below read the last
    // observed value rather than needing another request.
    await expect
      .poll(
        async () => {
          const runs = (await officeApi.listRuns(officeSeed.workspaceId)) as {
            runs?: Array<{ id: string; task_id?: string }>;
          };
          const run = (runs.runs ?? []).find((r) => r.task_id === task.id);
          if (!run?.id) return 0;
          attempts = (await officeApi.listRouteAttempts(run.id)).attempts;
          return attempts.length;
        },
        { timeout: 40_000, message: "run never recorded 2 route attempts" },
      )
      .toBeGreaterThanOrEqual(2);
    expect(attempts[0].provider_id).toBe("claude-acp");
    expect(attempts[0].execution_profile_id).toBe(
      routingConfig.provider_profiles["claude-acp"].execution_profile_ids.balanced,
    );
    expect(attempts[0].outcome).toBe("failed_provider_unavailable");
    expect(attempts[0].error_code).toBe("quota_limited");
    expect(attempts[1].provider_id).toBe("codex-acp");
    expect(attempts[1].execution_profile_id).toBe(
      routingConfig.provider_profiles["codex-acp"].execution_profile_ids.balanced,
    );
    expect(attempts[1].execution_profile_id).not.toBe(attempts[0].execution_profile_id);
    expect(attempts[1].outcome).toBe("launched");

    const health = await officeApi.listRoutingHealth(officeSeed.workspaceId);
    const claudeHealth = health.health.find((h) => h.provider_id === "claude-acp");
    expect(claudeHealth?.state).toBe("degraded");
    expect(claudeHealth?.error_code).toBe("quota_limited");
  });
});