From 62d4522e0f327cbb7ad87c978c78174248398971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Sun, 2 Aug 2026 20:14:53 +0100 Subject: [PATCH] feat: add file-backed diagnostic log bundles (#2087) --- .agents/skills/debug/SKILL.md | 43 +- .agents/skills/debug/references/browser.md | 22 +- .agents/skills/debug/references/instance.md | 73 +- .../agent/runtime/agentctl/client_debug.go | 39 + .../runtime/agentctl/client_debug_test.go | 36 + .../agent/runtime/agentctl/client_files.go | 40 + .../lifecycle/diagnostic_materializer.go | 36 + .../agent/runtime/lifecycle/manager_debug.go | 23 + .../runtime/lifecycle/manager_debug_test.go | 60 ++ .../server/adapter/transport/shared/debug.go | 10 + .../agentctl/server/api/acp_debug_export.go | 177 +++++ .../agentctl/server/api/acp_debug_test.go | 62 ++ .../internal/agentctl/server/api/server.go | 4 + .../server/api/workspace_diagnostics.go | 154 ++++ .../server/api/workspace_diagnostics_test.go | 82 ++ .../backendapp/diagnostic_acp_exporter.go | 27 + .../backendapp/diagnostic_session_provider.go | 215 +++++ .../diagnostic_session_provider_test.go | 98 +++ apps/backend/internal/backendapp/helpers.go | 7 +- apps/backend/internal/backendapp/main.go | 18 +- apps/backend/internal/common/config/config.go | 21 +- .../internal/common/config/config_test.go | 9 + .../internal/common/logger/async_sink.go | 220 ++++++ .../internal/common/logger/async_sink_test.go | 113 +++ .../internal/common/logger/backend_logger.go | 302 +++++++ .../internal/common/logger/buffer/buffer.go | 151 ---- .../common/logger/buffer/buffer_test.go | 155 ---- .../internal/common/logger/daily_writer.go | 409 ++++++++++ .../common/logger/daily_writer_test.go | 182 +++++ apps/backend/internal/common/logger/logger.go | 82 +- .../internal/common/logger/logger_test.go | 241 +++++- apps/backend/internal/debug/export.go | 75 -- apps/backend/internal/debug/export_test.go | 73 -- .../backend/internal/gateway/websocket/hub.go | 26 + .../gateway/websocket/hub_broadcast_test.go | 28 + apps/backend/internal/improvekandev/bundle.go | 181 ++--- .../internal/improvekandev/bundle_test.go | 79 +- .../backend/internal/improvekandev/handler.go | 169 ++-- .../internal/improvekandev/handler_test.go | 68 ++ .../integration/improve_kandev_test.go | 5 + apps/backend/internal/launcher/cli/help.go | 4 +- .../internal/launcher/cli/help_test.go | 3 + apps/backend/internal/launcher/env.go | 3 +- apps/backend/internal/launcher/start.go | 17 +- apps/backend/internal/launcher/start_test.go | 54 +- apps/backend/internal/launcher/supervisor.go | 1 + .../internal/launcher/supervisor_test.go | 4 + .../mcp/handlers/diagnostic_bundle.go | 176 +++++ .../mcp/handlers/diagnostic_bundle_test.go | 66 ++ .../backend/internal/mcp/handlers/handlers.go | 8 +- apps/backend/internal/mcp/server/handlers.go | 20 + apps/backend/internal/mcp/server/server.go | 16 + .../internal/mcp/server/server_test.go | 5 +- .../internal/system/frontenderrors/handler.go | 212 +++++ .../system/frontenderrors/handler_test.go | 151 ++++ .../internal/system/frontenderrors/limiter.go | 143 ++++ .../system/frontenderrors/limiter_test.go | 45 ++ .../internal/system/logbundle/archive.go | 748 ++++++++++++++++++ .../system/logbundle/archive_helpers.go | 184 +++++ .../internal/system/logbundle/contracts.go | 147 ++++ .../internal/system/logbundle/disk_unix.go | 13 + .../internal/system/logbundle/disk_windows.go | 17 + .../internal/system/logbundle/handler.go | 256 ++++++ .../internal/system/logbundle/handler_test.go | 163 ++++ apps/backend/internal/system/logbundle/job.go | 129 +++ .../internal/system/logbundle/service.go | 737 +++++++++++++++++ .../internal/system/logbundle/service_test.go | 647 +++++++++++++++ apps/backend/internal/system/logs/files.go | 251 ------ .../internal/system/logs/files_test.go | 289 ------- apps/backend/internal/system/logs/handler.go | 89 --- .../internal/system/logs/handler_test.go | 215 ----- apps/backend/internal/system/logs/tail.go | 197 ----- .../internal/system/logs/tail_buffer_test.go | 66 -- .../backend/internal/system/logs/tail_test.go | 212 ----- apps/backend/internal/system/system.go | 53 +- .../internal/system/system_routes_test.go | 41 + apps/backend/pkg/websocket/actions.go | 1 + apps/cli/src/cli.ts | 4 +- apps/cli/src/dev.ts | 32 +- apps/cli/src/run.ts | 37 +- apps/cli/src/shared.test.ts | 38 + apps/cli/src/shared.ts | 36 +- apps/cli/src/start.ts | 35 +- apps/cli/src/supervisor/backend.ts | 5 + apps/cli/src/supervisor/manifest.test.ts | 6 +- apps/cli/src/supervisor/manifest.ts | 1 + .../[id]/components/agent-channels-tab.tsx | 2 +- .../components/agent-configuration-tab.tsx | 2 +- .../components/agent-instructions-tab.tsx | 2 +- .../[id]/components/agent-memory-tab.tsx | 2 +- .../[id]/components/agent-overview-tab.tsx | 2 +- .../[id]/components/agent-permissions-tab.tsx | 2 +- .../[id]/components/agent-routing-card.tsx | 2 +- .../[id]/components/agent-skills-tab.tsx | 2 +- .../agents/components/create-agent-dialog.tsx | 2 +- .../app/office/components/new-task-dialog.tsx | 2 +- .../app/office/inbox/inbox-page-client.tsx | 2 +- apps/web/app/office/projects/[id]/page.tsx | 2 +- .../[id]/project-executor-section.tsx | 2 +- .../office/projects/[id]/project-header.tsx | 2 +- .../projects/[id]/project-repos-section.tsx | 2 +- .../office/projects/create-project-dialog.tsx | 2 +- .../routines/[id]/routine-detail-view.tsx | 2 +- .../app/office/routines/routines-content.tsx | 2 +- apps/web/app/office/setup/setup-wizard.tsx | 2 +- .../app/office/tasks/use-paginated-tasks.ts | 2 +- .../workspace/activity/activity-feed.tsx | 2 +- .../office/workspace/costs/budgets-tab.tsx | 2 +- .../office/workspace/costs/cost-overview.tsx | 2 +- .../workspace/costs/create-budget-form.tsx | 2 +- .../web/app/office/workspace/routing/page.tsx | 2 +- .../components/danger-zone-section.tsx | 2 +- .../settings/components/git-section.tsx | 2 +- .../settings/components/settings-content.tsx | 2 +- .../workspace/settings/sync/use-sync-state.ts | 2 +- .../office/workspace/skills/skill-list.tsx | 2 +- .../workspace/skills/skills-page-client.tsx | 2 +- apps/web/app/settings/system/logs/page.tsx | 38 +- .../use-app-status-bar-order.ts | 2 +- .../automations/automation-editor.tsx | 2 +- .../trigger-configs/webhook-config.tsx | 2 +- .../azure-devops-task-launcher.tsx | 2 +- .../improve-kandev-dialog-helpers.test.ts | 67 +- .../improve-kandev-dialog-helpers.ts | 60 +- apps/web/components/log-buffer-bridge.tsx | 10 + .../plugins/marketplace-sources-dialog.tsx | 2 +- .../settings/plugins/use-plugin-actions.ts | 2 +- .../plugins/use-plugin-config-form.ts | 2 +- .../settings/ssh-agent-readiness-card.tsx | 2 +- .../settings/system/bundle-customizer.tsx | 533 +++++++++++++ .../settings/system/log-viewer.test.tsx | 133 ++++ .../components/settings/system/log-viewer.tsx | 514 ++++++------ .../task/chat/queued-ghost-list.tsx | 2 +- .../task/chat/queued-ghost-message.tsx | 2 +- .../task/executor-environment-info.tsx | 2 +- .../task/layout-preset-selector.tsx | 2 +- .../components/task/port-forward-dialog.tsx | 2 +- .../sidebar-filter/sidebar-filter-bar.tsx | 2 +- .../task/simple/OfficeSimplePane.tsx | 2 +- .../task/simple/components/labels-picker.tsx | 2 +- apps/web/components/task/simple/task-chat.tsx | 2 +- .../components/task/simple/task-documents.tsx | 2 +- .../components/task/task-plan-revisions.tsx | 2 +- .../web/components/task/use-tunnel-actions.ts | 2 +- apps/web/components/toast-provider.test.tsx | 79 ++ apps/web/components/toast-provider.tsx | 41 +- apps/web/e2e/tests/improve-kandev.spec.ts | 15 +- .../e2e/tests/mobile-improve-kandev.spec.ts | 6 +- apps/web/e2e/tests/system/logs-page.spec.ts | 137 +++- .../tests/system/mobile-logs-bundle.spec.ts | 61 ++ .../plugins/use-auto-update-settings.ts | 2 +- .../domains/session/use-task-environment.ts | 2 +- .../domains/settings/use-automation-runs.ts | 2 +- .../web/hooks/domains/system/use-log-files.ts | 34 - apps/web/hooks/domains/system/use-log-tail.ts | 37 - apps/web/hooks/use-detach-task.ts | 2 +- apps/web/hooks/use-nest-task.ts | 2 +- .../web/hooks/use-optimistic-task-mutation.ts | 2 +- apps/web/lib/api/client.ts | 12 +- .../domains/frontend-error-log-api.test.ts | 72 ++ .../lib/api/domains/frontend-error-log-api.ts | 189 +++++ .../api/domains/improve-kandev-api.test.ts | 34 + .../web/lib/api/domains/improve-kandev-api.ts | 45 +- apps/web/lib/api/domains/system-api.test.ts | 99 ++- apps/web/lib/api/domains/system-api.ts | 73 +- apps/web/lib/logger/buffer.test.ts | 38 + apps/web/lib/logger/buffer.ts | 120 ++- apps/web/lib/logger/capture.test.ts | 18 + apps/web/lib/logger/capture.ts | 66 ++ apps/web/lib/logger/indexeddb-store.test.ts | 37 + apps/web/lib/logger/indexeddb-store.ts | 171 ++++ apps/web/lib/logger/intercept.test.ts | 19 + apps/web/lib/logger/intercept.ts | 221 ++++-- apps/web/lib/logger/runtime.test.ts | 37 + apps/web/lib/logger/runtime.ts | 155 ++++ apps/web/lib/state/slices/index.ts | 1 - apps/web/lib/state/slices/system/index.ts | 1 - .../state/slices/system/system-slice.test.ts | 26 - .../lib/state/slices/system/system-slice.ts | 10 - apps/web/lib/state/slices/system/types.ts | 10 - apps/web/lib/toast/sonner.test.ts | 79 ++ apps/web/lib/toast/sonner.ts | 33 + apps/web/lib/types/backend.ts | 6 +- apps/web/lib/types/system.ts | 65 +- apps/web/lib/ws/handlers/system-events.ts | 10 + apps/web/src/locales/en/settings.json | 51 +- apps/web/src/locales/pseudo/settings.json | 51 +- apps/web/src/settings-routes.tsx | 2 +- ...26-07-30-file-backed-diagnostic-bundles.md | 213 +++++ docs/decisions/INDEX.md | 1 + docs/plans/diagnostic-logging/plan.md | 637 +++++++++++++++ .../task-01-backend-log-sinks.md | 90 +++ .../task-02-frontend-error-endpoint.md | 72 ++ .../task-03-launcher-contracts.md | 79 ++ .../task-04-toast-reporting.md | 86 ++ .../task-05-public-logging-docs.md | 92 +++ .../task-06-debug-workflow-guidance.md | 99 +++ .../task-07-diagnostic-bundle-backend.md | 99 +++ .../task-08-browser-logs-and-bundle-ui.md | 129 +++ .../task-09-remove-legacy-diagnostics.md | 111 +++ ...ask-10-agent-diagnostic-materialization.md | 107 +++ .../task-11-merge-risk-hardening.md | 84 ++ .../task-12-custom-bundle-contracts.md | 91 +++ .../task-13-on-demand-acp-collection.md | 103 +++ .../task-14-bundle-customizer-ui.md | 100 +++ .../task-15-custom-bundle-e2e.md | 83 ++ .../task-16-diagnostic-privacy-docs.md | 88 +++ .../task-17-bundle-customizer-refinement.md | 105 +++ docs/public/cli.md | 4 +- docs/public/configuration.md | 18 +- docs/public/contributing.md | 4 +- docs/public/coverage.json | 7 +- docs/public/docker.md | 2 +- docs/public/k8s.md | 2 +- docs/public/operations.md | 23 +- docs/public/remote-cloud-environment.md | 4 +- docs/specs/INDEX.md | 1 + docs/specs/platform/diagnostic-logging.md | 728 +++++++++++++++++ docs/specs/system-page/spec.md | 38 +- scripts/dev-isolated | 2 +- scripts/kandev-logs | 156 ++-- 221 files changed, 13377 insertions(+), 3103 deletions(-) create mode 100644 apps/backend/internal/agent/runtime/agentctl/client_debug.go create mode 100644 apps/backend/internal/agent/runtime/agentctl/client_debug_test.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/manager_debug.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go create mode 100644 apps/backend/internal/agentctl/server/api/acp_debug_export.go create mode 100644 apps/backend/internal/agentctl/server/api/workspace_diagnostics.go create mode 100644 apps/backend/internal/agentctl/server/api/workspace_diagnostics_test.go create mode 100644 apps/backend/internal/backendapp/diagnostic_acp_exporter.go create mode 100644 apps/backend/internal/backendapp/diagnostic_session_provider.go create mode 100644 apps/backend/internal/backendapp/diagnostic_session_provider_test.go create mode 100644 apps/backend/internal/common/logger/async_sink.go create mode 100644 apps/backend/internal/common/logger/async_sink_test.go create mode 100644 apps/backend/internal/common/logger/backend_logger.go delete mode 100644 apps/backend/internal/common/logger/buffer/buffer.go delete mode 100644 apps/backend/internal/common/logger/buffer/buffer_test.go create mode 100644 apps/backend/internal/common/logger/daily_writer.go create mode 100644 apps/backend/internal/common/logger/daily_writer_test.go delete mode 100644 apps/backend/internal/debug/export.go delete mode 100644 apps/backend/internal/debug/export_test.go create mode 100644 apps/backend/internal/mcp/handlers/diagnostic_bundle.go create mode 100644 apps/backend/internal/mcp/handlers/diagnostic_bundle_test.go create mode 100644 apps/backend/internal/system/frontenderrors/handler.go create mode 100644 apps/backend/internal/system/frontenderrors/handler_test.go create mode 100644 apps/backend/internal/system/frontenderrors/limiter.go create mode 100644 apps/backend/internal/system/frontenderrors/limiter_test.go create mode 100644 apps/backend/internal/system/logbundle/archive.go create mode 100644 apps/backend/internal/system/logbundle/archive_helpers.go create mode 100644 apps/backend/internal/system/logbundle/contracts.go create mode 100644 apps/backend/internal/system/logbundle/disk_unix.go create mode 100644 apps/backend/internal/system/logbundle/disk_windows.go create mode 100644 apps/backend/internal/system/logbundle/handler.go create mode 100644 apps/backend/internal/system/logbundle/handler_test.go create mode 100644 apps/backend/internal/system/logbundle/job.go create mode 100644 apps/backend/internal/system/logbundle/service.go create mode 100644 apps/backend/internal/system/logbundle/service_test.go delete mode 100644 apps/backend/internal/system/logs/files.go delete mode 100644 apps/backend/internal/system/logs/files_test.go delete mode 100644 apps/backend/internal/system/logs/handler.go delete mode 100644 apps/backend/internal/system/logs/handler_test.go delete mode 100644 apps/backend/internal/system/logs/tail.go delete mode 100644 apps/backend/internal/system/logs/tail_buffer_test.go delete mode 100644 apps/backend/internal/system/logs/tail_test.go create mode 100644 apps/backend/internal/system/system_routes_test.go create mode 100644 apps/web/components/settings/system/bundle-customizer.tsx create mode 100644 apps/web/components/settings/system/log-viewer.test.tsx create mode 100644 apps/web/components/toast-provider.test.tsx create mode 100644 apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts delete mode 100644 apps/web/hooks/domains/system/use-log-files.ts delete mode 100644 apps/web/hooks/domains/system/use-log-tail.ts create mode 100644 apps/web/lib/api/domains/frontend-error-log-api.test.ts create mode 100644 apps/web/lib/api/domains/frontend-error-log-api.ts create mode 100644 apps/web/lib/api/domains/improve-kandev-api.test.ts create mode 100644 apps/web/lib/logger/capture.test.ts create mode 100644 apps/web/lib/logger/capture.ts create mode 100644 apps/web/lib/logger/indexeddb-store.test.ts create mode 100644 apps/web/lib/logger/indexeddb-store.ts create mode 100644 apps/web/lib/logger/runtime.test.ts create mode 100644 apps/web/lib/logger/runtime.ts create mode 100644 apps/web/lib/toast/sonner.test.ts create mode 100644 apps/web/lib/toast/sonner.ts create mode 100644 docs/decisions/2026-07-30-file-backed-diagnostic-bundles.md create mode 100644 docs/plans/diagnostic-logging/plan.md create mode 100644 docs/plans/diagnostic-logging/task-01-backend-log-sinks.md create mode 100644 docs/plans/diagnostic-logging/task-02-frontend-error-endpoint.md create mode 100644 docs/plans/diagnostic-logging/task-03-launcher-contracts.md create mode 100644 docs/plans/diagnostic-logging/task-04-toast-reporting.md create mode 100644 docs/plans/diagnostic-logging/task-05-public-logging-docs.md create mode 100644 docs/plans/diagnostic-logging/task-06-debug-workflow-guidance.md create mode 100644 docs/plans/diagnostic-logging/task-07-diagnostic-bundle-backend.md create mode 100644 docs/plans/diagnostic-logging/task-08-browser-logs-and-bundle-ui.md create mode 100644 docs/plans/diagnostic-logging/task-09-remove-legacy-diagnostics.md create mode 100644 docs/plans/diagnostic-logging/task-10-agent-diagnostic-materialization.md create mode 100644 docs/plans/diagnostic-logging/task-11-merge-risk-hardening.md create mode 100644 docs/plans/diagnostic-logging/task-12-custom-bundle-contracts.md create mode 100644 docs/plans/diagnostic-logging/task-13-on-demand-acp-collection.md create mode 100644 docs/plans/diagnostic-logging/task-14-bundle-customizer-ui.md create mode 100644 docs/plans/diagnostic-logging/task-15-custom-bundle-e2e.md create mode 100644 docs/plans/diagnostic-logging/task-16-diagnostic-privacy-docs.md create mode 100644 docs/plans/diagnostic-logging/task-17-bundle-customizer-refinement.md create mode 100644 docs/specs/platform/diagnostic-logging.md diff --git a/.agents/skills/debug/SKILL.md b/.agents/skills/debug/SKILL.md index fe4949126b..8ef3e3c83c 100644 --- a/.agents/skills/debug/SKILL.md +++ b/.agents/skills/debug/SKILL.md @@ -1,7 +1,7 @@ --- name: debug description: Diagnose Kandev bugs, running-instance issues, UI/browser failures, and runtime behavior. Use when the user reports unexpected behavior, asks to investigate, asks to add logs/instrumentation, or when a fix needs root-cause evidence before implementing. Triage first, gather evidence safely, then hand off to /fix for code changes. -allowed-tools: Bash(curl:*) Bash(jq:*) Bash(npx:*) Bash(scripts/kandev-instances:*) Bash(scripts/kandev-logs:*) Bash(scripts/dev-isolated:*) Bash(scripts/kandev-kill:*) Bash(go:*) Bash(rg:*) Bash(grep:*) +allowed-tools: Bash(curl:*) Bash(jq:*) Bash(mktemp:*) Bash(unzip:*) Bash(npx:*) Bash(scripts/kandev-instances:*) Bash(scripts/kandev-logs:*) Bash(scripts/dev-isolated:*) Bash(scripts/kandev-kill:*) Bash(go:*) Bash(rg:*) Bash(grep:*) --- # Debug @@ -19,7 +19,7 @@ through `/fix` when code changes are needed. Create a visible task list: 1. **Triage** - classify the bug and choose the cheapest faithful path -2. **Gather evidence** - targeted test, debug export/logs, browser state, or instrumentation +2. **Gather evidence** - targeted test, source-selectable diagnostic bundle, browser state, or instrumentation 3. **Diagnose** - trace the failure to root cause 4. **Report** - summarize evidence and choose `/fix` when code changes are needed 5. **Clean up** - remove temporary logs, throwaway repro tests, isolated instances, and browser sessions @@ -38,7 +38,7 @@ Pick one path before launching anything: Rules: - Triage before launching anything. - Use logs and targeted tests before browser automation. -- Never mutate the user's live instance. Read-only debug export/logs are allowed; browser interaction must use your isolated instance. +- Never mutate the user's live instance. Creating or downloading an owned diagnostic bundle is read-only; browser interaction must use your isolated instance. - Tear down only what you started. Never `pkill kandev`. ## Evidence Strategy @@ -46,9 +46,40 @@ Rules: Start with the cheapest faithful reproduction: 1. Backend logic: write a throwaway focused Go repro test against the real service path. If it reproduces, convert it via `/fix`. -2. Live instance: use `scripts/kandev-logs --export` or `--level error`; do not relaunch. -3. UI/browser: launch `scripts/dev-isolated --web`, drive `npx playwright-cli`, and correlate console/network state with backend logs. -4. Unknown: trace from the symptom backward through code and add temporary instrumentation only where it will split the search space. +2. Live instance in a task session: call `get_diagnostic_bundle_kandev` with `backend`, `frontend`, or `all`; inspect `manifest.json` before assuming a source is complete. +3. Host-side instance: use `scripts/kandev-logs --source backend|frontend|all`; do not relaunch. Set `KANDEV_API_TOKEN` only when authentication is enabled. +4. UI/browser: launch `scripts/dev-isolated --web`, drive `npx playwright-cli`, and correlate console/network state with a fresh all-source bundle. +5. Unknown: trace from the symptom backward through code and add temporary instrumentation only where it will split the search space. + +### File-first log triage + +Start with the retained backend files before asking for a broad export. Each +Kandev home has `logs/backend-logs.log` plus the two preceding UTC daily files +(`backend-logs-YYYY-MM-DD.log`). The active file appends across same-day +restarts and each daily file is bounded, so search the exact files rather than +loading an entire log into memory: + +```bash +rg --fixed-strings '' '/logs' -g 'backend-logs*.log' +rg --fixed-strings '' '/logs' -g 'backend-logs*.log' +``` + +Prefer a task ID, session ID, or exact route/error string. Add a bounded time +window only after the exact search; do not use a broad `rg` over the whole home +directory because task workspaces and ACP files can contain unrelated private +content. A zero-match task search is inconclusive when the event is an +install-wide startup/API event. + +Request only the needed bundle sources. Standard bundles contain backend and +frontend diagnostic events; a custom bundle can add the allow-listed runtime +index. These sources do not read stored chat transcripts, session messages, or +agent messages. If the +maintainer explicitly needs agent protocol evidence, use the debug-only ACP +source and select the exact authorized sessions; ACP raw/normalized frames may +contain prompts, responses, tool calls, file/MCP data, environment-derived +values, and secrets. Always inspect `manifest.json` and its warnings before +assuming a source is complete, and grep task/session IDs inside the extracted +ZIP before broadening to route text or timestamps. ## Reference Files diff --git a/.agents/skills/debug/references/browser.md b/.agents/skills/debug/references/browser.md index f13c5bb5ab..d5ac861f28 100644 --- a/.agents/skills/debug/references/browser.md +++ b/.agents/skills/debug/references/browser.md @@ -32,16 +32,32 @@ npx playwright-cli snapshot --depth=4 npx playwright-cli console npx playwright-cli console error npx playwright-cli network -npx playwright-cli eval "JSON.stringify(window.__kandevLogBuffer?.snapshot?.() ?? [])" npx playwright-cli goto http://localhost:/some/path ``` -Correlate browser console, frontend log buffer, network activity, and backend logs from: +Correlate browser console and network activity with a fresh standard bundle or +the smallest custom source set that answers the question: ```bash -scripts/kandev-logs --export --level error +scripts/kandev-logs --source all ``` +Kandev retains a bounded three-day console history in that browser. It uploads +the history only after an explicit bundle request; it does not stream console +calls continuously. Inspect the bundle manifest for missing browsers, +truncation, persistence fallback, or dropped entries before treating absence +as evidence. + +The standard bundle contains frontend/backend diagnostic events. A custom +bundle can add the allow-listed runtime index, and neither source reads stored +chat/session/agent messages. ACP protocol frames require the debug-only +**Download with ACP** path, +explicit session selection, and a sensitive-data review; they can contain +prompts, responses, tools, files, MCP data, environment values, and secrets. +When correlating a browser failure with a task, grep the exact task/session ID +in the extracted ZIP and the backend daily files before searching generic +error text. + Close the browser when done unless the user asks to keep it open: ```bash diff --git a/.agents/skills/debug/references/instance.md b/.agents/skills/debug/references/instance.md index f30c8c9e0e..03fc90bb51 100644 --- a/.agents/skills/debug/references/instance.md +++ b/.agents/skills/debug/references/instance.md @@ -10,7 +10,7 @@ scripts/kandev-instances Columns: `PID BACKEND_PORT WEB_PORT AGENTCTL_PORT HOME_DIR REPO_PATH`. -The user's live instance usually has `HOME_DIR=/home/` or backend port `38429`. Never mutate it. Read-only logs/export are allowed. +The user's live instance usually has `HOME_DIR=/home/` or backend port `38429`. Never mutate it. Creating and downloading an owned diagnostic bundle is read-only. ## Launch Isolated Instance @@ -28,24 +28,68 @@ On a clean checkout, pass `--install` or run `make install` once so frontend dep `dev-isolated` prints a `READY` block with ports, log paths, pidfile, and teardown command. Save the backend/web port and pidfile. -## Logs And Export +## Diagnostic bundles -Use `kandev-logs` against the relevant port: +Inside a Kandev task session, request only the evidence needed: -```bash -# Full structured export: -scripts/kandev-logs --export - -# Error-level only: -scripts/kandev-logs --export --level error - -# Tail isolated backend stderr: -scripts/kandev-logs --tail --lines 120 +```text +get_diagnostic_bundle_kandev source=backend +get_diagnostic_bundle_kandev source=frontend +get_diagnostic_bundle_kandev source=all ``` -Filter aggressively. Summarize metadata, error count, recent stack traces, and warning patterns that correlate with the report. +The tool returns an executor-local ZIP path. For a host-side instance, use: -Prefer `scripts/kandev-logs` over generic MCP fetches; it uses plain `curl` and avoids proxy/schema conversion issues. +```bash +scripts/kandev-logs --source backend +scripts/kandev-logs --source frontend +scripts/kandev-logs --source all +``` + +When authentication is enabled, export a personal access token through +`KANDEV_API_TOKEN`; never place it in an argument or print it. No token is +needed when authentication is disabled. + +Use the smallest source set that can answer the question. `backend` covers the +retained daily files; `frontend` asks connected tabs for their bounded local +console history; `runtime` adds only allow-listed session status/executor +metadata; `acp` is an explicit debug-only selection of one to ten authorized +sessions and can include full protocol content. Standard evidence does not +include stored chat/session/agent messages, while ACP evidence deliberately can. + +Extract into a newly created temporary directory, never a repository, home +directory, workspace root, or reused path. Reject ZIP entries that are +absolute or contain `..` before extraction. Inspect `manifest.json` first: +`partial` status, warnings, byte ranges, and loss counters mean a requested +source may be incomplete. + +```bash +diagnostic_tmp="$(mktemp -d)" +if unzip -Z1 "$diagnostic_zip" | grep -Eq '(^/|(^|/)\.\.(/|$))'; then + echo "unsafe diagnostic ZIP paths" >&2 + exit 1 +fi +unzip -q "$diagnostic_zip" -d "$diagnostic_tmp" +jq . "$diagnostic_tmp/manifest.json" +``` + +If a task ID is known, search it exactly before broadening. For a host-side +instance this is often faster than extracting a new ZIP: + +```bash +rg --fixed-strings '' '' +rg --fixed-strings '' '' +rg --fixed-strings '' '/logs' -g 'backend-logs*.log' +rg --fixed-strings '' '/logs' -g 'backend-logs*.log' +``` + +A zero-match task-ID search is inconclusive: install-wide backend events and +browser pages outside recognized task routes may not carry `task_id`. Next +search a precise route/error string or a bounded timestamp window. + +Backend files also live under `/logs/backend-logs.log` with the two +previous UTC daily archives. Prefer a fresh bundle because its manifest records +completeness and includes requested browser evidence. ## Teardown @@ -58,3 +102,4 @@ scripts/kandev-kill --yes ``` `kandev-kill` refuses protected ports without `--force`. Never use `pkill kandev`. +Remove only extraction directories you created. diff --git a/apps/backend/internal/agent/runtime/agentctl/client_debug.go b/apps/backend/internal/agent/runtime/agentctl/client_debug.go new file mode 100644 index 0000000000..9af14dc3c9 --- /dev/null +++ b/apps/backend/internal/agent/runtime/agentctl/client_debug.go @@ -0,0 +1,39 @@ +package client + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" +) + +const maxACPDebugExportBytes = int64(96 * 1024 * 1024) + +// ExportACPDebug requests a bounded raw+normalized ACP ZIP for one exact +// agentctl ACP session. The response body belongs to the caller and must be +// closed, even when the caller chooses not to materialize it. +func (c *Client) ExportACPDebug(ctx context.Context, sessionID string, maxBytes int64) (io.ReadCloser, error) { + if sessionID == "" { + return nil, fmt.Errorf("ACP session ID is required") + } + if maxBytes <= 0 || maxBytes > maxACPDebugExportBytes { + maxBytes = maxACPDebugExportBytes + } + path := "/api/v1/debug/acp/" + url.PathEscape(sessionID) + "/export?max_bytes=" + strconv.FormatInt(maxBytes, 10) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+path, nil) + if err != nil { + return nil, err + } + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + return nil, fmt.Errorf("ACP debug export failed with status %d: %s", resp.StatusCode, string(body)) + } + return resp.Body, nil +} diff --git a/apps/backend/internal/agent/runtime/agentctl/client_debug_test.go b/apps/backend/internal/agent/runtime/agentctl/client_debug_test.go new file mode 100644 index 0000000000..9151c0372d --- /dev/null +++ b/apps/backend/internal/agent/runtime/agentctl/client_debug_test.go @@ -0,0 +1,36 @@ +package client + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" +) + +func TestExportACPDebugRequestsExactSessionAndBound(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/debug/acp/acp-session-1/export" { + t.Errorf("path = %q", r.URL.Path) + } + if r.URL.Query().Get("max_bytes") != "4096" { + t.Errorf("max_bytes = %q", r.URL.Query().Get("max_bytes")) + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("zip-bytes")) + })) + defer server.Close() + + client := &Client{baseURL: server.URL, httpClient: server.Client()} + response, err := client.ExportACPDebug(t.Context(), "acp-session-1", 4096) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(response) + _ = response.Close() + if err != nil { + t.Fatal(err) + } + if string(body) != "zip-bytes" { + t.Fatalf("body = %q", body) + } +} diff --git a/apps/backend/internal/agent/runtime/agentctl/client_files.go b/apps/backend/internal/agent/runtime/agentctl/client_files.go index f86f89dda2..90f27fd049 100644 --- a/apps/backend/internal/agent/runtime/agentctl/client_files.go +++ b/apps/backend/internal/agent/runtime/agentctl/client_files.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/url" @@ -386,6 +387,45 @@ type CopyFilesResponse struct { Error string `json:"error,omitempty"` } +type DiagnosticMaterializeResponse struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` + Error string `json:"error,omitempty"` +} + +// MaterializeDiagnosticBundle streams a bounded archive to the server-owned +// diagnostics directory inside this agentctl workspace. +func (c *Client) MaterializeDiagnosticBundle( + ctx context.Context, + bundleID string, + source io.Reader, +) (*DiagnosticMaterializeResponse, error) { + reqURL := fmt.Sprintf("%s/api/v1/workspace/diagnostics/%s", c.baseURL, url.PathEscape(bundleID)) + request, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL, source) + if err != nil { + return nil, fmt.Errorf("create diagnostic materialization request: %w", err) + } + request.Header.Set("Content-Type", "application/zip") + response, err := c.httpClient.Do(request) + if err != nil { + return nil, fmt.Errorf("materialize diagnostic bundle: %w", err) + } + defer func() { + if err := response.Body.Close(); err != nil { + c.logger.Debug("failed to close diagnostic response body", zap.Error(err)) + } + }() + var result DiagnosticMaterializeResponse + if err := json.NewDecoder(io.LimitReader(response.Body, 64*1024)).Decode(&result); err != nil { + return nil, fmt.Errorf("decode diagnostic materialization response: %w", err) + } + if response.StatusCode != http.StatusOK || result.Error != "" { + return nil, fmt.Errorf("diagnostic materialization failed: status=%d error=%s", + response.StatusCode, result.Error) + } + return &result, nil +} + // CopyFiles ships a batch of pre-planned files (from copyfiles.Plan on the // host) to agentctl's POST /workspace/copy-files endpoint, which writes // them into the workspace under the optional repo subpath. Idempotent — diff --git a/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go b/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go new file mode 100644 index 0000000000..01776804d0 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go @@ -0,0 +1,36 @@ +package lifecycle + +import ( + "context" + "fmt" + "io" +) + +type DiagnosticMaterialization struct { + Path string `json:"path"` + Bytes int64 `json:"bytes"` +} + +func (m *Manager) MaterializeDiagnosticBundle( + ctx context.Context, + taskID, sessionID, bundleID string, + source io.Reader, +) (DiagnosticMaterialization, error) { + execution, exists := m.executionStore.GetBySessionID(sessionID) + if !exists || execution == nil || execution.agentctl == nil { + return DiagnosticMaterialization{}, fmt.Errorf("active task execution not found") + } + if execution.TaskID != taskID || execution.SessionID != sessionID { + return DiagnosticMaterialization{}, fmt.Errorf("task and session do not match the active execution") + } + executionID := execution.ID + result, err := execution.agentctl.MaterializeDiagnosticBundle(ctx, bundleID, source) + if err != nil { + return DiagnosticMaterialization{}, err + } + current, stillCurrent := m.executionStore.GetBySessionID(sessionID) + if !stillCurrent || current.ID != executionID { + return DiagnosticMaterialization{}, fmt.Errorf("task execution changed during diagnostic transfer") + } + return DiagnosticMaterialization{Path: result.Path, Bytes: result.Bytes}, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go b/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go new file mode 100644 index 0000000000..e2c8cc2f9d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go @@ -0,0 +1,23 @@ +package lifecycle + +import ( + "context" + "fmt" + "io" +) + +// ExportACPDebug fetches the selected execution's exact ACP conversation +// export. The caller must perform task/session authorization before invoking +// this method; lifecycle only resolves the in-memory execution seam. +func (m *Manager) ExportACPDebug( + ctx context.Context, taskSessionID string, maxBytes int64, +) (io.ReadCloser, error) { + if taskSessionID == "" { + return nil, fmt.Errorf("task session ID is required") + } + execution, ok := m.GetExecutionBySessionID(taskSessionID) + if !ok || execution == nil || execution.agentctl == nil || execution.ACPSessionID == "" { + return nil, fmt.Errorf("ACP executor is unavailable") + } + return execution.agentctl.ExportACPDebug(ctx, execution.ACPSessionID, maxBytes) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go new file mode 100644 index 0000000000..898cb14468 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go @@ -0,0 +1,60 @@ +package lifecycle + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + + agentctlclient "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/logger" + "go.uber.org/zap" +) + +func TestManagerExportACPDebugUsesExecutionACPConversationID(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/v1/debug/acp/acp-session-1/export" { + t.Errorf("path = %q", r.URL.Path) + } + _, _ = w.Write([]byte("zip")) + })) + defer server.Close() + + parsed, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(parsed.Port()) + if err != nil { + t.Fatal(err) + } + log, err := logger.NewFromZap(zap.NewNop()) + if err != nil { + t.Fatal(err) + } + + mgr := newTestManager(t) + if err := mgr.executionStore.Add(&AgentExecution{ + ID: "execution-1", + SessionID: "task-session-1", + ACPSessionID: "acp-session-1", + agentctl: agentctlclient.NewClient(parsed.Hostname(), port, log), + }); err != nil { + t.Fatal(err) + } + + body, err := mgr.ExportACPDebug(t.Context(), "task-session-1", 4096) + if err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(body) + _ = body.Close() + if err != nil { + t.Fatal(err) + } + if string(data) != "zip" { + t.Fatalf("body = %q", data) + } +} diff --git a/apps/backend/internal/agentctl/server/adapter/transport/shared/debug.go b/apps/backend/internal/agentctl/server/adapter/transport/shared/debug.go index b104d3fab5..a8177b5e65 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/shared/debug.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/shared/debug.go @@ -38,6 +38,16 @@ func ACPDebugEnabled() bool { return debugMode } // env so they resolve to the same directory. func ACPLogDir() string { return resolveACPLogDir() } +// FlushACPLogs makes retained raw/normalized files complete before an +// explicit diagnostic bundle request reads them. It is intentionally not used +// on the frame hot path. +func FlushACPLogs() { + if !debugMode { + return + } + acpLog.flushAll() +} + // ACPRingTail returns up to n most recent normalized events for a session as // raw JSON lines, for the dev-only live-tail endpoint. Returns nil when the // session is unknown. diff --git a/apps/backend/internal/agentctl/server/api/acp_debug_export.go b/apps/backend/internal/agentctl/server/api/acp_debug_export.go new file mode 100644 index 0000000000..014b845981 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/acp_debug_export.go @@ -0,0 +1,177 @@ +package api + +import ( + "archive/zip" + "io" + "net/http" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared" +) + +const ( + defaultACPExportBytes = int64(96 * 1024 * 1024) + maxACPExportBytes = int64(96 * 1024 * 1024) + maxACPExportFiles = 128 +) + +type acpExportFile struct { + name string + path string + kind string + size int64 +} + +func acpDebugExportEnabled() bool { + return os.Getenv("KANDEV_DEBUG_AGENT_MESSAGES") == "true" && + os.Getenv("KANDEV_DEBUG_DEV_MODE") == "true" +} + +func (s *Server) handleACPDebugExport(c *gin.Context) { + sessionID := c.Param("session") + if !validACPExportSession(sessionID) { + c.JSON(http.StatusNotFound, gin.H{"error": "ACP debug export unavailable"}) + return + } + limit := defaultACPExportBytes + if raw := c.Query("max_bytes"); raw != "" { + parsed, err := strconv.ParseInt(raw, 10, 64) + if err != nil || parsed <= 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid ACP export byte limit"}) + return + } + limit = min(parsed, maxACPExportBytes) + } + + shared.FlushACPLogs() + files, err := findACPExportFiles(shared.ACPLogDir(), sessionID) + if err != nil || len(files) == 0 { + c.JSON(http.StatusNotFound, gin.H{"error": "ACP debug export unavailable"}) + return + } + + c.Header("Content-Type", "application/zip") + c.Header("Content-Disposition", `attachment; filename="kandev-acp-debug.zip"`) + writer := zip.NewWriter(c.Writer) + remaining := limit + for _, file := range files { + if remaining <= 0 { + break + } + length := min(file.size, remaining) + offset := file.size - length + source, openErr := os.Open(file.path) + if openErr != nil { + _ = writer.Close() + return + } + header := &zip.FileHeader{Name: "acp/" + file.kind + "/" + file.name, Method: zip.Store} + header.SetMode(0o600) + destination, createErr := writer.CreateHeader(header) + if createErr == nil { + createErr = copyACPSection(destination, source, offset, length) + } + closeErr := source.Close() + if createErr != nil || closeErr != nil { + _ = writer.Close() + return + } + remaining -= length + } + if err := writer.Close(); err != nil { + return + } +} + +func findACPExportFiles(dir, sessionID string) ([]acpExportFile, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + safeSession := sanitizeACPSessionPart(sessionID) + files := make([]acpExportFile, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !isExactACPSessionFilename(name, safeSession) { + continue + } + path := filepath.Join(dir, name) + info, statErr := os.Lstat(path) + if statErr != nil || !info.Mode().IsRegular() { + continue + } + kind := "normalized" + if strings.HasPrefix(name, "raw-") { + kind = "raw" + } + files = append(files, acpExportFile{name: name, path: path, kind: kind, size: info.Size()}) + } + sort.Slice(files, func(i, j int) bool { + if files[i].kind != files[j].kind { + return files[i].kind < files[j].kind + } + return files[i].name < files[j].name + }) + if len(files) > maxACPExportFiles { + files = files[:maxACPExportFiles] + } + return files, nil +} + +func isExactACPSessionFilename(name, safeSession string) bool { + if !strings.HasSuffix(name, ".jsonl") || + (!strings.HasPrefix(name, "raw-") && !strings.HasPrefix(name, "normalized-")) { + return false + } + base := strings.TrimSuffix(name, ".jsonl") + if dot := strings.LastIndexByte(base, '.'); dot > 0 && allASCIIDigits(base[dot+1:]) { + base = base[:dot] + } + return strings.HasSuffix(base, "-"+safeSession) +} + +func validACPExportSession(sessionID string) bool { + return sessionID != "" && len(sessionID) <= 256 && + !strings.ContainsAny(sessionID, "/\\\x00\r\n") +} + +func sanitizeACPSessionPart(value string) string { + if value == "" { + return "unknown" + } + var builder strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': + builder.WriteRune(r) + default: + builder.WriteByte('_') + } + } + return builder.String() +} + +func allASCIIDigits(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func copyACPSection(destination io.Writer, source *os.File, offset, length int64) error { + if _, err := source.Seek(offset, io.SeekStart); err != nil { + return err + } + _, err := io.CopyN(destination, source, length) + return err +} diff --git a/apps/backend/internal/agentctl/server/api/acp_debug_test.go b/apps/backend/internal/agentctl/server/api/acp_debug_test.go index 50a042b8fa..1aea65fdd7 100644 --- a/apps/backend/internal/agentctl/server/api/acp_debug_test.go +++ b/apps/backend/internal/agentctl/server/api/acp_debug_test.go @@ -1,15 +1,22 @@ package api import ( + "archive/zip" + "bytes" "encoding/json" + "io" "net/http" "net/http/httptest" + "os" + "path/filepath" + "strings" "testing" ) const ( envMessages = "KANDEV_DEBUG_AGENT_MESSAGES" envDevMode = "KANDEV_DEBUG_DEV_MODE" + envLogDir = "KANDEV_DEBUG_LOG_DIR" enabled = "true" disabled = "" ) @@ -79,3 +86,58 @@ func TestACPRingTailHandler_UnknownSession(t *testing.T) { t.Errorf("events should serialize as [] not null") } } + +func TestACPDebugExportReturnsOnlyExactSessionFiles(t *testing.T) { + dir := t.TempDir() + t.Setenv(envMessages, enabled) + t.Setenv(envDevMode, enabled) + t.Setenv(envLogDir, dir) + writeACPTestFile(t, filepath.Join(dir, "raw-acp-claude-session-1.jsonl"), "raw-session-1\n") + writeACPTestFile(t, filepath.Join(dir, "normalized-acp-claude-session-1.1.jsonl"), "normalized-session-1\n") + writeACPTestFile(t, filepath.Join(dir, "raw-acp-claude-session-10.jsonl"), "wrong-session\n") + writeACPTestFile(t, filepath.Join(dir, "notes.txt"), "private\n") + + server := newTestServer(t) + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, + "/api/v1/debug/acp/session-1/export?max_bytes=4096", nil) + server.router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("status %d body=%s", rec.Code, rec.Body.String()) + } + reader, err := zip.NewReader(bytes.NewReader(rec.Body.Bytes()), int64(rec.Body.Len())) + if err != nil { + t.Fatalf("zip: %v", err) + } + if len(reader.File) != 2 { + t.Fatalf("entries = %d, want 2", len(reader.File)) + } + for _, entry := range reader.File { + if strings.Contains(entry.Name, "session-10") || entry.Name == "notes.txt" { + t.Fatalf("unexpected export entry %q", entry.Name) + } + body, readErr := io.ReadAll(mustOpenZipEntry(t, entry)) + if readErr != nil { + t.Fatal(readErr) + } + if !strings.Contains(string(body), "session-1") { + t.Fatalf("entry %q body = %q", entry.Name, body) + } + } +} + +func writeACPTestFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.WriteFile(path, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } +} + +func mustOpenZipEntry(t *testing.T, entry *zip.File) io.ReadCloser { + t.Helper() + reader, err := entry.Open() + if err != nil { + t.Fatal(err) + } + return reader +} diff --git a/apps/backend/internal/agentctl/server/api/server.go b/apps/backend/internal/agentctl/server/api/server.go index 341b4b6c6f..4b8382077b 100644 --- a/apps/backend/internal/agentctl/server/api/server.go +++ b/apps/backend/internal/agentctl/server/api/server.go @@ -138,6 +138,7 @@ func (s *Server) setupRoutes() { // Docker, Sprites — to seed the workspace with gitignored config // after the in-container clone). api.POST("/workspace/copy-files", s.handleWorkspaceCopyFiles) + api.POST("/workspace/diagnostics/:id", s.handleWorkspaceDiagnostics) api.POST("/workspace/materialize-repository", s.handleWorkspaceMaterializeRepository) api.POST("/workspace/materialize-repository/remove", s.handleWorkspaceRemoveMaterializedRepository) @@ -218,6 +219,9 @@ func (s *Server) setupRoutes() { if acpDebugTailEnabled() { s.router.GET("/api/v1/debug/acp/:session", s.handleACPRingTail) } + if acpDebugExportEnabled() { + s.router.GET("/api/v1/debug/acp/:session/export", s.handleACPDebugExport) + } } // acpDebugTailEnabled gates the ACP live-tail endpoint on both ACP frame diff --git a/apps/backend/internal/agentctl/server/api/workspace_diagnostics.go b/apps/backend/internal/agentctl/server/api/workspace_diagnostics.go new file mode 100644 index 0000000000..9201230bc6 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/workspace_diagnostics.go @@ -0,0 +1,154 @@ +package api + +import ( + "io" + "net/http" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/gin-gonic/gin" +) + +const maxDiagnosticBundleBytes = int64(256 * 1024 * 1024) + +var diagnosticBundleID = regexp.MustCompile(`^[a-f0-9]{16,64}$`) + +func (s *Server) handleWorkspaceDiagnostics(c *gin.Context) { + id := c.Param("id") + if !diagnosticBundleID.MatchString(id) { + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid diagnostic bundle id"}) + return + } + root := filepath.Join(s.procMgr.WorkDir(), ".kandev", "diagnostics") + if err := os.MkdirAll(root, 0o700); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to prepare diagnostics directory"}) + return + } + _ = os.Chmod(root, 0o700) + cleanupExpiredDiagnostics(root, time.Now().Add(-24*time.Hour)) + file, err := os.CreateTemp(root, ".upload-*.tmp") + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create diagnostic file"}) + return + } + tempPath := file.Name() + defer func() { _ = os.Remove(tempPath) }() + _ = file.Chmod(0o600) + written, copyErr := io.CopyBuffer( + file, + io.LimitReader(c.Request.Body, maxDiagnosticBundleBytes+1), + make([]byte, 1024*1024), + ) + closeErr := file.Close() + if copyErr != nil || closeErr != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write diagnostic file"}) + return + } + if written > maxDiagnosticBundleBytes { + c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "diagnostic bundle is too large"}) + return + } + destination := filepath.Join(root, id+".zip") + if err := os.Rename(tempPath, destination); err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to commit diagnostic file"}) + return + } + _ = os.Chmod(destination, 0o600) + ensureDiagnosticsIgnored(s.procMgr.WorkDir()) + time.AfterFunc(24*time.Hour, func() { + info, err := os.Lstat(destination) + if err == nil && info.Mode().IsRegular() && time.Since(info.ModTime()) >= 24*time.Hour { + _ = os.Remove(destination) + } + }) + c.JSON(http.StatusOK, gin.H{ + "path": filepath.ToSlash(filepath.Join(".kandev", "diagnostics", id+".zip")), + "bytes": written, + }) +} + +func cleanupExpiredDiagnostics(root string, cutoff time.Time) { + entries, err := os.ReadDir(root) + if err != nil { + return + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".zip") { + continue + } + info, err := entry.Info() + if err == nil && info.Mode().IsRegular() && info.ModTime().Before(cutoff) { + _ = os.Remove(filepath.Join(root, entry.Name())) + } + } +} + +func ensureDiagnosticsIgnored(workDir string) { + excludePath, err := diagnosticsExcludePath(workDir) + if err != nil || excludePath == "" { + return + } + if err := os.MkdirAll(filepath.Dir(excludePath), 0o700); err != nil { + return + } + content, err := os.ReadFile(excludePath) + if err != nil && !os.IsNotExist(err) { + return + } + if strings.Contains(string(content), "/.kandev/") { + return + } + file, err := os.OpenFile(excludePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return + } + defer func() { _ = file.Close() }() + _, _ = file.WriteString("\n/.kandev/\n") +} + +func diagnosticsExcludePath(workDir string) (string, error) { + gitPath := filepath.Join(workDir, ".git") + info, err := os.Stat(gitPath) + if err == nil && info.IsDir() { + return filepath.Join(gitPath, "info", "exclude"), nil + } + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + data, err := os.ReadFile(gitPath) + if err != nil { + return "", err + } + const prefix = "gitdir:" + line := strings.TrimSpace(string(data)) + if !strings.HasPrefix(line, prefix) { + return "", nil + } + gitDir := strings.TrimSpace(strings.TrimPrefix(line, prefix)) + if gitDir == "" { + return "", nil + } + if !filepath.IsAbs(gitDir) { + gitDir = filepath.Join(workDir, gitDir) + } + gitDir = filepath.Clean(gitDir) + commonData, commonErr := os.ReadFile(filepath.Join(gitDir, "commondir")) + if commonErr == nil { + commonDir := strings.TrimSpace(string(commonData)) + if commonDir != "" { + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(gitDir, commonDir) + } + gitDir = filepath.Clean(commonDir) + } + } else if !os.IsNotExist(commonErr) { + return "", commonErr + } + return filepath.Join(gitDir, "info", "exclude"), nil +} diff --git a/apps/backend/internal/agentctl/server/api/workspace_diagnostics_test.go b/apps/backend/internal/agentctl/server/api/workspace_diagnostics_test.go new file mode 100644 index 0000000000..d63383d8e2 --- /dev/null +++ b/apps/backend/internal/agentctl/server/api/workspace_diagnostics_test.go @@ -0,0 +1,82 @@ +package api + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestWorkspaceDiagnosticsStreamsToOwnerOnlyServerPath(t *testing.T) { + server, workDir := newCopyFilesTestServer(t) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/workspace/diagnostics/0123456789abcdef", + bytes.NewBufferString("zip-bytes"), + ) + response := httptest.NewRecorder() + server.router.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } + path := filepath.Join(workDir, ".kandev", "diagnostics", "0123456789abcdef.zip") + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 600", info.Mode().Perm()) + } + content, err := os.ReadFile(path) + if err != nil || string(content) != "zip-bytes" { + t.Fatalf("content = %q err=%v", content, err) + } +} + +func TestWorkspaceDiagnosticsRejectsInvalidBundleID(t *testing.T) { + server, _ := newCopyFilesTestServer(t) + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/workspace/diagnostics/..%2Fescape", + bytes.NewBufferString("zip-bytes"), + ) + response := httptest.NewRecorder() + server.router.ServeHTTP(response, request) + if response.Code != http.StatusBadRequest && response.Code != http.StatusNotFound { + t.Fatalf("status = %d body=%s", response.Code, response.Body.String()) + } +} + +func TestEnsureDiagnosticsIgnoredResolvesLinkedWorktreeGitDir(t *testing.T) { + workDir := t.TempDir() + commonDir := filepath.Join(workDir, "common.git") + gitDir := filepath.Join(commonDir, "worktrees", "linked") + if err := os.MkdirAll(filepath.Join(commonDir, "info"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(gitDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(gitDir, "commondir"), []byte("../..\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(workDir, ".git"), + []byte("gitdir: common.git/worktrees/linked\n"), + 0o600, + ); err != nil { + t.Fatal(err) + } + + ensureDiagnosticsIgnored(workDir) + + exclude, err := os.ReadFile(filepath.Join(commonDir, "info", "exclude")) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(exclude, []byte("/.kandev/")) { + t.Fatalf("exclude = %q, want diagnostics pattern", exclude) + } +} diff --git a/apps/backend/internal/backendapp/diagnostic_acp_exporter.go b/apps/backend/internal/backendapp/diagnostic_acp_exporter.go new file mode 100644 index 0000000000..e946c1c4c4 --- /dev/null +++ b/apps/backend/internal/backendapp/diagnostic_acp_exporter.go @@ -0,0 +1,27 @@ +package backendapp + +import ( + "context" + "errors" + "io" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/system/logbundle" +) + +type diagnosticACPExporter struct { + manager *lifecycle.Manager +} + +func newDiagnosticACPExporter(manager *lifecycle.Manager) *diagnosticACPExporter { + return &diagnosticACPExporter{manager: manager} +} + +func (e *diagnosticACPExporter) ExportACP( + ctx context.Context, session logbundle.DiagnosticSession, maxBytes int64, +) (io.ReadCloser, error) { + if e == nil || e.manager == nil { + return nil, errors.New("ACP exporter unavailable") + } + return e.manager.ExportACPDebug(ctx, session.SessionID, maxBytes) +} diff --git a/apps/backend/internal/backendapp/diagnostic_session_provider.go b/apps/backend/internal/backendapp/diagnostic_session_provider.go new file mode 100644 index 0000000000..67a6b3da71 --- /dev/null +++ b/apps/backend/internal/backendapp/diagnostic_session_provider.go @@ -0,0 +1,215 @@ +package backendapp + +import ( + "context" + "sort" + "time" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/system/logbundle" + "github.com/kandev/kandev/internal/task/models" +) + +type diagnosticSessionReader interface { + ListWorkspaces(context.Context) ([]*models.Workspace, error) + ListTasksByWorkspace( + context.Context, string, string, string, string, int, int, string, + bool, bool, bool, bool, + ) ([]*models.Task, int, error) + ListTaskSessions(context.Context, string) ([]*models.TaskSession, error) + GetTaskSession(context.Context, string) (*models.TaskSession, error) + GetExecutorRunningBySessionID(context.Context, string) (*models.ExecutorRunning, error) +} + +type diagnosticSessionProvider struct { + reader diagnosticSessionReader +} + +func newDiagnosticSessionProvider(reader diagnosticSessionReader) *diagnosticSessionProvider { + return &diagnosticSessionProvider{reader: reader} +} + +func (p *diagnosticSessionProvider) ListDiagnosticSessions( + ctx context.Context, + identity authn.Identity, + since time.Time, + selectedIDs []string, +) ([]logbundle.DiagnosticSession, error) { + if p == nil || p.reader == nil { + return []logbundle.DiagnosticSession{}, nil + } + readerCtx := authn.WithIdentity(ctx, identity) + selected := selectedSessionSet(selectedIDs) + rows, err := p.collectCatalog(readerCtx, since, selected) + if err != nil { + return nil, err + } + p.collectExplicit(readerCtx, selected, rows) + return finalizeDiagnosticSessions(rows, selected), nil +} + +const logbundleRuntimePageSize = 500 + +func sessionInWindow(session *models.TaskSession, since time.Time) bool { + return !session.StartedAt.Before(since) || !session.UpdatedAt.Before(since) +} + +func selectedSessionSet(selectedIDs []string) map[string]bool { + selected := make(map[string]bool, len(selectedIDs)) + for _, id := range selectedIDs { + selected[id] = true + } + return selected +} + +func (p *diagnosticSessionProvider) collectCatalog( + ctx context.Context, + since time.Time, + selected map[string]bool, +) (map[string]logbundle.DiagnosticSession, error) { + workspaces, err := p.reader.ListWorkspaces(ctx) + if err != nil { + return nil, err + } + rows := make(map[string]logbundle.DiagnosticSession) + for _, workspace := range workspaces { + if workspace == nil || workspace.ID == "" { + continue + } + if err := p.collectWorkspace(ctx, workspace.ID, since, selected, rows); err != nil { + return nil, err + } + } + return rows, nil +} + +func (p *diagnosticSessionProvider) collectWorkspace( + ctx context.Context, + workspaceID string, + since time.Time, + selected map[string]bool, + rows map[string]logbundle.DiagnosticSession, +) error { + tasks, _, err := p.reader.ListTasksByWorkspace( + ctx, workspaceID, "", "", "", 1, logbundleRuntimePageSize, + "updated_at", true, true, false, true, + ) + if err != nil { + return err + } + for _, task := range tasks { + if task == nil { + continue + } + sessions, err := p.reader.ListTaskSessions(ctx, task.ID) + if err != nil { + return err + } + for _, session := range sessions { + if session == nil || (!selected[session.ID] && !sessionInWindow(session, since)) { + continue + } + rows[session.ID] = p.diagnosticSessionRow(ctx, session, task.Title) + } + } + return nil +} + +// Explicit selections may be older than the normal catalogue window. Read +// those exact rows after the bounded catalogue walk; authorization happens +// inside the task service before any runtime/ACP metadata is consulted. +func (p *diagnosticSessionProvider) collectExplicit( + ctx context.Context, + selected map[string]bool, + rows map[string]logbundle.DiagnosticSession, +) { + for sessionID := range selected { + if _, ok := rows[sessionID]; ok { + continue + } + session, err := p.reader.GetTaskSession(ctx, sessionID) + if err == nil && session != nil { + rows[session.ID] = p.diagnosticSessionRow(ctx, session, "") + } + } +} + +func finalizeDiagnosticSessions( + rows map[string]logbundle.DiagnosticSession, + selected map[string]bool, +) []logbundle.DiagnosticSession { + result := make([]logbundle.DiagnosticSession, 0, len(rows)) + for _, row := range rows { + result = append(result, row) + } + sort.SliceStable(result, func(i, j int) bool { + return result[i].LastActivityAt.After(result[j].LastActivityAt) + }) + if len(result) > logbundleRuntimePageSize { + result = keepSelectedRows(result, selected, logbundleRuntimePageSize) + } + return result +} + +func (p *diagnosticSessionProvider) diagnosticSessionRow( + ctx context.Context, session *models.TaskSession, taskTitle string, +) logbundle.DiagnosticSession { + row := logbundle.DiagnosticSession{ + TaskID: session.TaskID, TaskTitle: taskTitle, SessionID: session.ID, Status: string(session.State), + StartedAt: session.StartedAt, LastActivityAt: session.UpdatedAt, + ACPSessionID: snapshotString(session.Metadata, "acp_session_id"), + Agent: snapshotString(session.AgentProfileSnapshot, "agent_name", "agent_id", "agent"), + Provider: snapshotString(session.AgentProfileSnapshot, "provider", "provider_id"), + Model: snapshotString(session.AgentProfileSnapshot, "model", "model_id"), + ExecutorType: snapshotString(session.ExecutorSnapshot, "type", "executor_type"), + } + if row.ACPSessionID == "" && p.reader != nil { + if running, err := p.reader.GetExecutorRunningBySessionID(ctx, session.ID); err == nil && running != nil { + row.ACPSessionID = running.ResumeToken + } + } + switch { + case row.ACPSessionID != "" && session.AgentExecutionID != "": + row.ACPAvailability = "reachable" + case row.ACPSessionID != "": + row.ACPAvailability = "host_retained" + default: + row.ACPAvailability = "unavailable" + } + return row +} + +func snapshotString(snapshot map[string]interface{}, keys ...string) string { + for _, key := range keys { + if value, ok := snapshot[key].(string); ok && value != "" { + return value + } + } + return "" +} + +func keepSelectedRows( + rows []logbundle.DiagnosticSession, + selected map[string]bool, + limit int, +) []logbundle.DiagnosticSession { + if len(rows) <= limit { + return rows + } + result := append([]logbundle.DiagnosticSession(nil), rows[:limit]...) + seen := make(map[string]bool, len(result)) + for _, row := range result { + seen[row.SessionID] = true + } + for _, row := range rows[limit:] { + if !selected[row.SessionID] || seen[row.SessionID] { + continue + } + result[len(result)-1] = row + seen[row.SessionID] = true + } + sort.SliceStable(result, func(i, j int) bool { + return result[i].LastActivityAt.After(result[j].LastActivityAt) + }) + return result +} diff --git a/apps/backend/internal/backendapp/diagnostic_session_provider_test.go b/apps/backend/internal/backendapp/diagnostic_session_provider_test.go new file mode 100644 index 0000000000..47d790412f --- /dev/null +++ b/apps/backend/internal/backendapp/diagnostic_session_provider_test.go @@ -0,0 +1,98 @@ +package backendapp + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/task/models" +) + +func TestDiagnosticSessionProviderBuildsMessageFreeAllowList(t *testing.T) { + started := time.Date(2026, 8, 2, 12, 0, 0, 0, time.UTC) + reader := &diagnosticSessionReaderStub{ + workspaces: []*models.Workspace{{ID: "ws-1", OwnerID: "user-1"}}, + tasks: map[string][]*models.Task{ + "ws-1": {{ID: "task-1", Title: "Repair diagnostic export", WorkspaceID: "ws-1"}}, + }, + sessions: map[string][]*models.TaskSession{ + "task-1": {{ + ID: "session-1", TaskID: "task-1", State: models.TaskSessionStateRunning, + StartedAt: started, UpdatedAt: started.Add(time.Minute), + AgentProfileSnapshot: map[string]interface{}{ + "agent_name": "claude-acp", "provider": "anthropic", "model": "sonnet", + "title": "must not leak", + }, + ExecutorSnapshot: map[string]interface{}{"type": "local_docker"}, + }}, + }, + } + provider := newDiagnosticSessionProvider(reader) + rows, err := provider.ListDiagnosticSessions( + context.Background(), + authn.Identity{UserID: "user-1", Role: authn.RoleMember}, + started.Add(-time.Hour), nil, + ) + if err != nil { + t.Fatal(err) + } + if len(rows) != 1 { + t.Fatalf("rows = %d, want 1", len(rows)) + } + row := rows[0] + if row.TaskID != "task-1" || row.TaskTitle != "Repair diagnostic export" || + row.SessionID != "session-1" || row.Agent != "claude-acp" || + row.Provider != "anthropic" || row.Model != "sonnet" || row.ExecutorType != "local_docker" { + t.Fatalf("allow-list row = %#v", row) + } + if row.Status != string(models.TaskSessionStateRunning) || !row.LastActivityAt.Equal(started.Add(time.Minute)) { + t.Fatalf("runtime fields = %#v", row) + } + if reader.lastIdentity.UserID != "user-1" { + t.Fatalf("provider did not preserve caller identity: %#v", reader.lastIdentity) + } +} + +type diagnosticSessionReaderStub struct { + workspaces []*models.Workspace + tasks map[string][]*models.Task + sessions map[string][]*models.TaskSession + lastIdentity authn.Identity +} + +func (r *diagnosticSessionReaderStub) ListWorkspaces(ctx context.Context) ([]*models.Workspace, error) { + r.lastIdentity, _ = authn.IdentityFromContext(ctx) + return r.workspaces, nil +} + +func (r *diagnosticSessionReaderStub) ListTasksByWorkspace( + context.Context, string, string, string, string, int, int, string, bool, bool, bool, bool, +) ([]*models.Task, int, error) { + return r.tasks["ws-1"], len(r.tasks["ws-1"]), nil +} + +func (r *diagnosticSessionReaderStub) ListTaskSessions( + _ context.Context, taskID string, +) ([]*models.TaskSession, error) { + return r.sessions[taskID], nil +} + +func (r *diagnosticSessionReaderStub) GetTaskSession( + _ context.Context, sessionID string, +) (*models.TaskSession, error) { + for _, sessions := range r.sessions { + for _, session := range sessions { + if session.ID == sessionID { + return session, nil + } + } + } + return nil, models.ErrTaskSessionNotFound +} + +func (r *diagnosticSessionReaderStub) GetExecutorRunningBySessionID( + context.Context, string, +) (*models.ExecutorRunning, error) { + return nil, models.ErrExecutorRunningNotFound +} diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index 17ea6da13f..061c6717a2 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -1123,6 +1123,9 @@ func registerSecondaryRoutes( if p.repoCloner != nil { ikHandler := improvekandev.NewHandler(p.taskSvc, p.repoCloner, p.version, p.log) + if p.systemSvc != nil { + ikHandler.SetLogBundles(p.systemSvc.LogBundles) + } improvekandev.RegisterRoutes(p.router, ikHandler) improvekandev.CleanupStaleBundles(func(path string, err error) { p.log.Warn("Improve Kandev: failed to clean stale bundle", zap.String("path", path), zap.Error(err)) @@ -1429,6 +1432,9 @@ func registerMCPAndDebugRoutes( mcpHandlers.SetPromptReferenceResolver(p.services.Prompts) mcpHandlers.SetTaskStopper(p.orchestratorSvc) mcpHandlers.SetUserSettingsProvider(p.services.User) + if p.systemSvc != nil && p.systemSvc.LogBundles != nil { + mcpHandlers.SetDiagnosticBundleServices(p.systemSvc.LogBundles, p.lifecycleMgr) + } // Enrich list_tasks responses with associated GitHub PRs (link, title, // number, state) when the github service is available. @@ -1495,7 +1501,6 @@ func registerMCPAndDebugRoutes( p.log.Debug("Registered Debug handlers (HTTP)") if p.devMode { - debughandlers.RegisterExportRoute(p.router, p.version, Commit, p.log) debughandlers.RegisterPprofRoutes(p.router, p.log) debughandlers.RegisterMemoryRoute(p.router, p.log) } diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 1bddce8d28..370ce19520 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -206,14 +206,11 @@ func Run(args []string, build BuildInfo) int { } // 2. Initialize logger - log, err := logger.NewLogger(logger.LoggingConfig{ - Level: cfg.Logging.Level, - Format: cfg.Logging.Format, - OutputPath: cfg.Logging.OutputPath, - MaxSizeMB: cfg.Logging.MaxSizeMB, - MaxBackups: cfg.Logging.MaxBackups, - MaxAgeDays: cfg.Logging.MaxAgeDays, - Compress: cfg.Logging.Compress, + log, err := logger.NewBackendLogger(logger.BackendLoggingConfig{ + HomeDir: cfg.ResolvedHomeDir(), + Level: cfg.Logging.Level, + Format: cfg.Logging.Format, + ConsoleLevel: os.Getenv("KANDEV_CONSOLE_LOG_LEVEL"), }) if err != nil { fmt.Fprintf(os.Stderr, "Failed to initialize logger: %v\n", err) @@ -839,6 +836,11 @@ func startGatewayAndServe( } systemSvc.Storage = storageComposition.handler systemSvc.StorageRuntime = storageComposition.runtime + if systemSvc.LogBundles != nil { + systemSvc.LogBundles.SetNotifier(gateway.Hub) + systemSvc.LogBundles.SetSessionProvider(newDiagnosticSessionProvider(services.Task)) + systemSvc.LogBundles.SetACPExporter(newDiagnosticACPExporter(lifecycleMgr)) + } if systemSvc.Metrics != nil { systemSvc.Metrics.SetBroadcaster(gateway.Hub.BroadcastToSystemMetrics) gateway.Hub.SetSystemMetricsInterestTracker(systemSvc.Metrics) diff --git a/apps/backend/internal/common/config/config.go b/apps/backend/internal/common/config/config.go index 48cb26c7cc..d079b2723b 100644 --- a/apps/backend/internal/common/config/config.go +++ b/apps/backend/internal/common/config/config.go @@ -404,20 +404,8 @@ type FeaturesConfig struct { // LoggingConfig holds logging configuration. type LoggingConfig struct { - Level string `mapstructure:"level"` - Format string `mapstructure:"format"` - OutputPath string `mapstructure:"outputPath"` - - // Rotation options - apply only when OutputPath is a file path - // (ignored for stdout/stderr). Backed by lumberjack. - // - // Note: lumberjack creates the active log file with mode 0600 (owner read/write - // only); the previous os.OpenFile path used 0644. External log shippers or - // sidecars running as a different user will need to run as the same user. - MaxSizeMB int `mapstructure:"maxSizeMb"` // rotate when file exceeds this size; 0 = lumberjack default (100MB) - MaxBackups int `mapstructure:"maxBackups"` // max number of rotated files to retain; 0 = unlimited - MaxAgeDays int `mapstructure:"maxAgeDays"` // max age in days of rotated files; 0 = unlimited - Compress bool `mapstructure:"compress"` // gzip rotated files + Level string `mapstructure:"level"` + Format string `mapstructure:"format"` } // RepositoryDiscoveryConfig holds configuration for local repository scanning. @@ -577,11 +565,6 @@ func setDefaults(v *viper.Viper) { // Logging defaults v.SetDefault("logging.level", "info") v.SetDefault("logging.format", detectDefaultLogFormat()) - v.SetDefault("logging.outputPath", "stdout") - v.SetDefault("logging.maxSizeMb", 100) - v.SetDefault("logging.maxBackups", 5) - v.SetDefault("logging.maxAgeDays", 30) - v.SetDefault("logging.compress", true) // Repository discovery defaults v.SetDefault("repositoryDiscovery.roots", []string{}) diff --git a/apps/backend/internal/common/config/config_test.go b/apps/backend/internal/common/config/config_test.go index fe6a3be363..6f2cd45998 100644 --- a/apps/backend/internal/common/config/config_test.go +++ b/apps/backend/internal/common/config/config_test.go @@ -46,6 +46,15 @@ func minimalValidConfig() *Config { } } +func TestLoggingConfigExposesOnlyLevelAndFormat(t *testing.T) { + cfgType := reflect.TypeOf(LoggingConfig{}) + for _, field := range []string{"OutputPath", "MaxSizeMB", "MaxBackups", "MaxAgeDays", "Compress"} { + if _, exists := cfgType.FieldByName(field); exists { + t.Errorf("LoggingConfig still exposes legacy destination field %s", field) + } + } +} + func TestResolvedHomeDir_Default(t *testing.T) { cfg := &Config{} home, err := os.UserHomeDir() diff --git a/apps/backend/internal/common/logger/async_sink.go b/apps/backend/internal/common/logger/async_sink.go new file mode 100644 index 0000000000..3d5550c0dd --- /dev/null +++ b/apps/backend/internal/common/logger/async_sink.go @@ -0,0 +1,220 @@ +package logger + +import ( + "context" + "io" + "sync" + "sync/atomic" + + "go.uber.org/zap/zapcore" +) + +const ( + lossCapacity = iota + lossReservedCapacity + lossEntryTooLarge + lossWriteError + lossShutdownTimeout + lossReasonCount +) + +var lossReasonNames = [lossReasonCount]string{ + "capacity", "reserved_capacity", "entry_too_large", "write_error", "shutdown_timeout", +} + +type asyncSinkConfig struct { + Name string + MaxEntries int + MaxBytes int + ReservedEntries int + ReservedBytes int + MaxEntryBytes int +} + +type SinkStatistics struct { + Name string `json:"name"` + Accepted map[string]uint64 `json:"accepted"` + Lost map[string]map[string]uint64 `json:"lost"` + QueueEntries int `json:"queue_entries"` + QueueBytes int `json:"queue_bytes"` +} + +type sinkEntry struct { + level zapcore.Level + data []byte +} + +type asyncSink struct { + writer io.Writer + cfg asyncSinkConfig + + mu sync.Mutex + cond *sync.Cond + queue []sinkEntry + inFlight *sinkEntry + pendingEntries int + pendingBytes int + closing bool + timeoutRecorded bool + done chan struct{} + accepted [7]atomic.Uint64 + lost [7][lossReasonCount]atomic.Uint64 +} + +func newAsyncSink(writer io.Writer, cfg asyncSinkConfig) *asyncSink { + sink := &asyncSink{writer: writer, cfg: cfg, done: make(chan struct{})} + sink.cond = sync.NewCond(&sink.mu) + go sink.run() + return sink +} + +func (s *asyncSink) Enqueue(level zapcore.Level, data []byte) bool { + levelIndex := sinkLevelIndex(level) + if len(data) > s.cfg.MaxEntryBytes { + s.lost[levelIndex][lossEntryTooLarge].Add(1) + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if s.closing { + s.lost[levelIndex][lossShutdownTimeout].Add(1) + return false + } + if s.exceedsTotalCapacity(len(data)) { + s.lost[levelIndex][lossCapacity].Add(1) + return false + } + if level < zapcore.WarnLevel && s.exceedsUnreservedCapacity(len(data)) { + s.lost[levelIndex][lossReservedCapacity].Add(1) + return false + } + copied := append([]byte(nil), data...) + s.queue = append(s.queue, sinkEntry{level: level, data: copied}) + s.pendingEntries++ + s.pendingBytes += len(copied) + s.accepted[levelIndex].Add(1) + s.cond.Signal() + return true +} + +func (s *asyncSink) exceedsTotalCapacity(size int) bool { + return s.pendingEntries+1 > s.cfg.MaxEntries || s.pendingBytes+size > s.cfg.MaxBytes +} + +func (s *asyncSink) exceedsUnreservedCapacity(size int) bool { + entryLimit := s.cfg.MaxEntries - s.cfg.ReservedEntries + byteLimit := s.cfg.MaxBytes - s.cfg.ReservedBytes + return s.pendingEntries+1 > entryLimit || s.pendingBytes+size > byteLimit +} + +func (s *asyncSink) run() { + defer close(s.done) + for { + entry, ok := s.next() + if !ok { + return + } + written, err := s.writer.Write(entry.data) + if err != nil || written != len(entry.data) { + s.lost[sinkLevelIndex(entry.level)][lossWriteError].Add(1) + } + s.complete(entry) + } +} + +func (s *asyncSink) next() (sinkEntry, bool) { + s.mu.Lock() + defer s.mu.Unlock() + for len(s.queue) == 0 && !s.closing { + s.cond.Wait() + } + if len(s.queue) == 0 { + return sinkEntry{}, false + } + entry := s.queue[0] + s.queue[0] = sinkEntry{} + s.queue = s.queue[1:] + s.inFlight = &entry + return entry, true +} + +func (s *asyncSink) complete(entry sinkEntry) { + s.mu.Lock() + s.pendingEntries-- + s.pendingBytes -= len(entry.data) + s.inFlight = nil + s.mu.Unlock() +} + +func (s *asyncSink) Stats() SinkStatistics { + stats := SinkStatistics{ + Name: s.cfg.Name, Accepted: make(map[string]uint64), Lost: make(map[string]map[string]uint64), + } + for index, name := range sinkLevelNames { + stats.Accepted[name] = s.accepted[index].Load() + stats.Lost[name] = make(map[string]uint64) + for reason := range lossReasonCount { + stats.Lost[name][lossReasonNames[reason]] = s.lost[index][reason].Load() + } + } + s.mu.Lock() + stats.QueueEntries = s.pendingEntries + stats.QueueBytes = s.pendingBytes + s.mu.Unlock() + return stats +} + +func (s *asyncSink) Close(ctx context.Context) error { + s.mu.Lock() + if !s.closing { + s.closing = true + s.cond.Broadcast() + } + s.mu.Unlock() + select { + case <-s.done: + if syncer, ok := s.writer.(interface{ Sync() error }); ok { + return syncer.Sync() + } + return nil + case <-ctx.Done(): + s.recordShutdownLoss() + return ctx.Err() + } +} + +func (s *asyncSink) recordShutdownLoss() { + s.mu.Lock() + defer s.mu.Unlock() + if s.timeoutRecorded { + return + } + s.timeoutRecorded = true + for _, entry := range s.queue { + s.lost[sinkLevelIndex(entry.level)][lossShutdownTimeout].Add(1) + } + if s.inFlight != nil { + s.lost[sinkLevelIndex(s.inFlight.level)][lossShutdownTimeout].Add(1) + } +} + +var sinkLevelNames = [7]string{"debug", "info", "warn", "error", "dpanic", "panic", "fatal"} + +func sinkLevelIndex(level zapcore.Level) int { + switch { + case level <= zapcore.DebugLevel: + return 0 + case level == zapcore.InfoLevel: + return 1 + case level == zapcore.WarnLevel: + return 2 + case level == zapcore.ErrorLevel: + return 3 + case level == zapcore.DPanicLevel: + return 4 + case level == zapcore.PanicLevel: + return 5 + default: + return 6 + } +} diff --git a/apps/backend/internal/common/logger/async_sink_test.go b/apps/backend/internal/common/logger/async_sink_test.go new file mode 100644 index 0000000000..e115387914 --- /dev/null +++ b/apps/backend/internal/common/logger/async_sink_test.go @@ -0,0 +1,113 @@ +package logger + +import ( + "context" + "sync" + "testing" + "time" + + "go.uber.org/zap/zapcore" +) + +type blockingWriter struct { + once sync.Once + started chan struct{} + release chan struct{} +} + +func (w *blockingWriter) Write(data []byte) (int, error) { + w.once.Do(func() { close(w.started) }) + <-w.release + return len(data), nil +} + +func TestAsyncSinkNeverWaitsForBlockedWriterAndReservesWarnings(t *testing.T) { + writer := &blockingWriter{started: make(chan struct{}), release: make(chan struct{})} + sink := newAsyncSink(writer, asyncSinkConfig{ + Name: "test", MaxEntries: 3, MaxBytes: 30, + ReservedEntries: 1, ReservedBytes: 10, MaxEntryBytes: 20, + }) + t.Cleanup(func() { + close(writer.release) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = sink.Close(ctx) + }) + + if !sink.Enqueue(zapcore.DebugLevel, []byte("first")) { + t.Fatal("first debug entry was rejected") + } + select { + case <-writer.started: + case <-time.After(time.Second): + t.Fatal("writer did not start") + } + if !sink.Enqueue(zapcore.InfoLevel, []byte("second")) { + t.Fatal("second low-priority entry was rejected") + } + + started := time.Now() + if sink.Enqueue(zapcore.InfoLevel, []byte("third")) { + t.Fatal("low-priority entry consumed reserved capacity") + } + if elapsed := time.Since(started); elapsed > 50*time.Millisecond { + t.Fatalf("enqueue waited for blocked writer: %s", elapsed) + } + if !sink.Enqueue(zapcore.WarnLevel, []byte("warning")) { + t.Fatal("warning did not use reserved capacity") + } + if sink.Enqueue(zapcore.ErrorLevel, []byte("overflow")) { + t.Fatal("entry beyond total capacity was accepted") + } + + stats := sink.Stats() + if got := stats.Lost["info"]["reserved_capacity"]; got != 1 { + t.Fatalf("reserved-capacity info loss = %d", got) + } + if got := stats.Lost["error"]["capacity"]; got != 1 { + t.Fatalf("capacity error loss = %d", got) + } +} + +func TestAsyncSinkDropsOversizedEntry(t *testing.T) { + sink := newAsyncSink(discardWriter{}, asyncSinkConfig{ + Name: "test", MaxEntries: 3, MaxBytes: 30, MaxEntryBytes: 4, + }) + if sink.Enqueue(zapcore.ErrorLevel, []byte("12345")) { + t.Fatal("oversized entry was accepted") + } + if got := sink.Stats().Lost["error"]["entry_too_large"]; got != 1 { + t.Fatalf("oversized loss = %d", got) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = sink.Close(ctx) +} + +func TestAsyncSinkCloseHonorsDeadlineAndAccountsForPendingEntry(t *testing.T) { + writer := &blockingWriter{started: make(chan struct{}), release: make(chan struct{})} + sink := newAsyncSink(writer, asyncSinkConfig{ + Name: "test", MaxEntries: 3, MaxBytes: 30, MaxEntryBytes: 20, + }) + if !sink.Enqueue(zapcore.WarnLevel, []byte("warning")) { + t.Fatal("warning was rejected") + } + <-writer.started + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + started := time.Now() + if err := sink.Close(ctx); err == nil { + t.Fatal("Close succeeded despite blocked writer") + } + if elapsed := time.Since(started); elapsed > 100*time.Millisecond { + t.Fatalf("Close exceeded bounded deadline: %s", elapsed) + } + if got := sink.Stats().Lost["warn"]["shutdown_timeout"]; got != 1 { + t.Fatalf("shutdown-timeout loss = %d", got) + } + close(writer.release) +} + +type discardWriter struct{} + +func (discardWriter) Write(data []byte) (int, error) { return len(data), nil } diff --git a/apps/backend/internal/common/logger/backend_logger.go b/apps/backend/internal/common/logger/backend_logger.go new file mode 100644 index 0000000000..6f4fdcf3a3 --- /dev/null +++ b/apps/backend/internal/common/logger/backend_logger.go @@ -0,0 +1,302 @@ +package logger + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + "time" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const ( + textLogFormat = "text" + maxEncodedLogEntry = 256 * 1024 + fileQueueEntries = 8192 + fileQueueBytes = 8 * 1024 * 1024 + stdoutQueueEntries = 2048 + stdoutQueueBytes = 2 * 1024 * 1024 + backendCloseTimeout = 2 * time.Second +) + +type BackendLoggingConfig struct { + HomeDir string + Level string + Format string + ConsoleLevel string + Stdout io.Writer + Stderr io.Writer + Now func() time.Time +} + +type backendLoggerRuntime struct { + fileSink *asyncSink + stdoutSink *asyncSink + fileWriter io.Closer + stderr io.Writer + closeTimeout time.Duration + closeOnce sync.Once + closeErr error +} + +func NewBackendLogger(cfg BackendLoggingConfig) (*Logger, error) { + cfg = backendConfigDefaults(cfg) + fileLevel, err := parseLevel(cfg.Level) + if err != nil { + fileLevel = zapcore.InfoLevel + } + logDir, err := filepath.Abs(filepath.Join(cfg.HomeDir, "logs")) + if err != nil { + return nil, fmt.Errorf("resolve backend log path: %w", err) + } + _, _ = fmt.Fprintf(cfg.Stdout, "Backend logs: %s\n", filepath.Join(logDir, activeBackendLogName)) + + fileWriter := newRetryDailyWriter(logDir, cfg.Now, cfg.Stderr) + fileSink := newAsyncSink(fileWriter, asyncSinkConfig{ + Name: "file", MaxEntries: fileQueueEntries, MaxBytes: fileQueueBytes, + ReservedEntries: 2048, ReservedBytes: 2 * 1024 * 1024, MaxEntryBytes: maxEncodedLogEntry, + }) + stdoutSink := newAsyncSink(cfg.Stdout, asyncSinkConfig{ + Name: "stdout", MaxEntries: stdoutQueueEntries, MaxBytes: stdoutQueueBytes, + ReservedEntries: 512, ReservedBytes: 512 * 1024, MaxEntryBytes: maxEncodedLogEntry, + }) + runtime := &backendLoggerRuntime{ + fileSink: fileSink, stdoutSink: stdoutSink, fileWriter: fileWriter, stderr: cfg.Stderr, + closeTimeout: backendCloseTimeout, + } + encoderConfig := backendEncoderConfig() + fileCore := newAsyncCore(newLogEncoder(cfg.Format, encoderConfig), fileSink, fileLevel) + stdoutLevel := zapcore.WarnLevel + if cfg.ConsoleLevel == "info" { + stdoutLevel = zapcore.InfoLevel + } + stdoutCore := newAsyncCore(newLogEncoder(cfg.Format, encoderConfig), stdoutSink, stdoutLevel) + zapLogger := zap.New(zapcore.NewTee(fileCore, stdoutCore), zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel)) + return &Logger{ + zap: zapLogger, sugar: zapLogger.Sugar(), backend: runtime, + backendLogDir: logDir, + }, nil +} + +func backendConfigDefaults(cfg BackendLoggingConfig) BackendLoggingConfig { + if cfg.HomeDir == "" { + cfg.HomeDir = "." + } + if cfg.Stdout == nil { + cfg.Stdout = os.Stdout + } + if cfg.Stderr == nil { + cfg.Stderr = os.Stderr + } + if cfg.Now == nil { + cfg.Now = time.Now + } + return cfg +} + +func backendEncoderConfig() zapcore.EncoderConfig { + cfg := zap.NewProductionEncoderConfig() + cfg.TimeKey = "timestamp" + cfg.EncodeTime = zapcore.ISO8601TimeEncoder + cfg.EncodeLevel = zapcore.LowercaseLevelEncoder + return cfg +} + +func newLogEncoder(format string, cfg zapcore.EncoderConfig) zapcore.Encoder { + if format == "console" || format == textLogFormat { + cfg.EncodeLevel = zapcore.CapitalColorLevelEncoder + return zapcore.NewConsoleEncoder(cfg) + } + return zapcore.NewJSONEncoder(cfg) +} + +func (r *backendLoggerRuntime) Close() error { + r.closeOnce.Do(func() { + timeout := r.closeTimeout + if timeout <= 0 { + timeout = backendCloseTimeout + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + results := make(chan error, 2) + go func() { results <- r.fileSink.Close(ctx) }() + go func() { results <- r.stdoutSink.Close(ctx) }() + for pending := 2; pending > 0; pending-- { + select { + case err := <-results: + r.closeErr = errors.Join(r.closeErr, err) + case <-ctx.Done(): + r.closeErr = errors.Join(r.closeErr, ctx.Err()) + pending = 1 + } + } + writerResult := make(chan error, 1) + go func() { writerResult <- r.fileWriter.Close() }() + select { + case err := <-writerResult: + r.closeErr = errors.Join(r.closeErr, err) + case <-ctx.Done(): + r.closeErr = errors.Join(r.closeErr, ctx.Err()) + } + }) + return r.closeErr +} + +func (r *backendLoggerRuntime) Stats() []SinkStatistics { + return []SinkStatistics{r.fileSink.Stats(), r.stdoutSink.Stats()} +} + +func (r *backendLoggerRuntime) terminal(level, message string) { + _ = r.Close() + const maxTerminalMessage = 64 * 1024 + if len(message) > maxTerminalMessage { + message = message[:maxTerminalMessage] + } + _, _ = fmt.Fprintf(r.stderr, "%s: %s\n", level, message) +} + +type asyncCore struct { + level zapcore.LevelEnabler + encoder zapcore.Encoder + sink *asyncSink + fields []zapcore.Field +} + +func newAsyncCore(encoder zapcore.Encoder, sink *asyncSink, level zapcore.Level) zapcore.Core { + return &asyncCore{level: zap.LevelEnablerFunc(func(candidate zapcore.Level) bool { + return candidate >= level + }), encoder: encoder, sink: sink} +} + +func (c *asyncCore) Enabled(level zapcore.Level) bool { + return c.level.Enabled(level) +} + +func (c *asyncCore) With(fields []zapcore.Field) zapcore.Core { + clone := *c + clone.fields = append(append([]zapcore.Field(nil), c.fields...), fields...) + return &clone +} + +func (c *asyncCore) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(entry.Level) { + return checked.AddCore(entry, c) + } + return checked +} + +func (c *asyncCore) Write(entry zapcore.Entry, fields []zapcore.Field) error { + encoder := c.encoder.Clone() + for _, field := range c.fields { + field.AddTo(encoder) + } + encoded, err := encoder.EncodeEntry(entry, fields) + if err != nil { + return err + } + c.sink.Enqueue(entry.Level, encoded.Bytes()) + encoded.Free() + return nil +} + +func (c *asyncCore) Sync() error { return nil } + +type retryDailyWriter struct { + mu sync.Mutex + logDir string + now func() time.Time + stderr io.Writer + writer *dailyWriter + nextRetry time.Time + closed bool + stop chan struct{} + done chan struct{} +} + +func newRetryDailyWriter(logDir string, now func() time.Time, stderr io.Writer) *retryDailyWriter { + writer := &retryDailyWriter{ + logDir: logDir, now: now, stderr: stderr, stop: make(chan struct{}), done: make(chan struct{}), + } + writer.activate() + go writer.retryLoop() + return writer +} + +func (w *retryDailyWriter) Write(data []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return 0, os.ErrClosed + } + if w.writer == nil && !w.now().Before(w.nextRetry) { + w.activate() + } + if w.writer == nil { + return 0, fmt.Errorf("backend log file unavailable") + } + written, err := w.writer.Write(data) + if err != nil { + if errors.Is(err, errDailyBackendLogLimit) { + return written, err + } + _ = w.writer.Close() + w.writer = nil + w.nextRetry = w.now().Add(30 * time.Second) + _, _ = fmt.Fprintf(w.stderr, "Backend log file write failed (%v); retrying in 30s\n", err) + } + return written, err +} + +func (w *retryDailyWriter) activate() { + writer, err := newDailyWriter(w.logDir, w.now) + if err != nil { + w.nextRetry = w.now().Add(30 * time.Second) + _, _ = fmt.Fprintf(w.stderr, "Backend log file unavailable (%v); retrying in 30s\n", err) + return + } + if err := writer.MaintenanceError(); err != nil { + _, _ = fmt.Fprintf(w.stderr, "Backend log cleanup deferred: %v\n", err) + } + w.writer = writer + w.nextRetry = time.Time{} +} + +func (w *retryDailyWriter) retryLoop() { + defer close(w.done) + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + w.mu.Lock() + if !w.closed && w.writer == nil && !w.now().Before(w.nextRetry) { + w.activate() + } + w.mu.Unlock() + case <-w.stop: + return + } + } +} + +func (w *retryDailyWriter) Close() error { + w.mu.Lock() + if w.closed { + w.mu.Unlock() + return nil + } + w.closed = true + close(w.stop) + writer := w.writer + w.mu.Unlock() + <-w.done + if writer == nil { + return nil + } + return writer.Close() +} diff --git a/apps/backend/internal/common/logger/buffer/buffer.go b/apps/backend/internal/common/logger/buffer/buffer.go deleted file mode 100644 index 045413258b..0000000000 --- a/apps/backend/internal/common/logger/buffer/buffer.go +++ /dev/null @@ -1,151 +0,0 @@ -// Package buffer provides an in-memory ring buffer of structured log entries. -// Used to capture recent log lines for inclusion in user-submitted reports -// (e.g. Improve Kandev) without unbounded memory growth. -package buffer - -import ( - "sync" - "time" - - "go.uber.org/zap/zapcore" -) - -// DefaultCapacity is the maximum number of entries kept in the default buffer. -const DefaultCapacity = 2000 - -// Entry is a structured log record snapshotted from a zap core. -type Entry struct { - Timestamp time.Time `json:"timestamp"` - Level string `json:"level"` - Logger string `json:"logger,omitempty"` - Caller string `json:"caller,omitempty"` - Message string `json:"message"` - Fields map[string]any `json:"fields,omitempty"` - Stack string `json:"stack,omitempty"` -} - -// RingBuffer is a thread-safe FIFO buffer with a fixed capacity. -type RingBuffer struct { - mu sync.Mutex - entries []Entry - capacity int -} - -// New returns a RingBuffer with the given capacity. A non-positive capacity -// is treated as DefaultCapacity. -func New(capacity int) *RingBuffer { - if capacity <= 0 { - capacity = DefaultCapacity - } - return &RingBuffer{capacity: capacity, entries: make([]Entry, 0, capacity)} -} - -// Push appends e to the buffer, evicting the oldest entry when full. -func (b *RingBuffer) Push(e Entry) { - b.mu.Lock() - defer b.mu.Unlock() - if len(b.entries) >= b.capacity { - copy(b.entries, b.entries[1:]) - b.entries = b.entries[:len(b.entries)-1] - } - b.entries = append(b.entries, e) -} - -// Snapshot returns a copy of the current entries in chronological order. -// Each entry is deep-copied so callers cannot mutate buffered Fields maps. -func (b *RingBuffer) Snapshot() []Entry { - b.mu.Lock() - defer b.mu.Unlock() - out := make([]Entry, len(b.entries)) - for i, e := range b.entries { - out[i] = e - if e.Fields != nil { - fields := make(map[string]any, len(e.Fields)) - for k, v := range e.Fields { - fields[k] = v - } - out[i].Fields = fields - } - } - return out -} - -// Len returns the number of entries currently held. -func (b *RingBuffer) Len() int { - b.mu.Lock() - defer b.mu.Unlock() - return len(b.entries) -} - -// Clear removes all entries. -func (b *RingBuffer) Clear() { - b.mu.Lock() - defer b.mu.Unlock() - b.entries = b.entries[:0] -} - -var ( - defaultOnce sync.Once - defaultBuf *RingBuffer -) - -// Default returns the process-wide ring buffer used by the structured logger. -func Default() *RingBuffer { - defaultOnce.Do(func() { - defaultBuf = New(DefaultCapacity) - }) - return defaultBuf -} - -// Core is a zapcore.Core that pushes every emitted entry into a RingBuffer. -// It performs no encoding or output; it is intended to be teed alongside the -// real output core via zapcore.NewTee. -type Core struct { - zapcore.LevelEnabler - buffer *RingBuffer - fields []zapcore.Field -} - -// NewCore returns a zapcore.Core that writes to buf at the given level. -func NewCore(buf *RingBuffer, level zapcore.LevelEnabler) zapcore.Core { - return &Core{LevelEnabler: level, buffer: buf} -} - -// With returns a child Core with additional fields attached. -func (c *Core) With(fields []zapcore.Field) zapcore.Core { - clone := *c - clone.fields = append(append([]zapcore.Field{}, c.fields...), fields...) - return &clone -} - -// Check adds the core to the checked entry when the level is enabled. -func (c *Core) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { - if c.Enabled(ent.Level) { - return ce.AddCore(ent, c) - } - return ce -} - -// Write captures the entry into the ring buffer. -func (c *Core) Write(ent zapcore.Entry, fields []zapcore.Field) error { - enc := zapcore.NewMapObjectEncoder() - for _, f := range c.fields { - f.AddTo(enc) - } - for _, f := range fields { - f.AddTo(enc) - } - c.buffer.Push(Entry{ - Timestamp: ent.Time, - Level: ent.Level.String(), - Logger: ent.LoggerName, - Caller: ent.Caller.TrimmedPath(), - Message: ent.Message, - Fields: enc.Fields, - Stack: ent.Stack, - }) - return nil -} - -// Sync is a no-op; the ring buffer holds entries in memory only. -func (c *Core) Sync() error { return nil } diff --git a/apps/backend/internal/common/logger/buffer/buffer_test.go b/apps/backend/internal/common/logger/buffer/buffer_test.go deleted file mode 100644 index c9e2756970..0000000000 --- a/apps/backend/internal/common/logger/buffer/buffer_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package buffer - -import ( - "sync" - "testing" - "time" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func TestRingBuffer_PushAndSnapshot(t *testing.T) { - b := New(3) - for i := 0; i < 3; i++ { - b.Push(Entry{Message: string(rune('a' + i))}) - } - snap := b.Snapshot() - if len(snap) != 3 { - t.Fatalf("expected 3 entries, got %d", len(snap)) - } - for i, want := range []string{"a", "b", "c"} { - if snap[i].Message != want { - t.Errorf("entry %d: got %q, want %q", i, snap[i].Message, want) - } - } -} - -func TestRingBuffer_FIFOEvictionAtCapacity(t *testing.T) { - b := New(3) - for i := 0; i < 5; i++ { - b.Push(Entry{Message: string(rune('a' + i))}) - } - snap := b.Snapshot() - if len(snap) != 3 { - t.Fatalf("expected 3 entries after eviction, got %d", len(snap)) - } - for i, want := range []string{"c", "d", "e"} { - if snap[i].Message != want { - t.Errorf("entry %d: got %q, want %q", i, snap[i].Message, want) - } - } -} - -func TestRingBuffer_DefaultCapacity(t *testing.T) { - b := New(0) - if b.capacity != DefaultCapacity { - t.Fatalf("non-positive capacity should default to %d, got %d", DefaultCapacity, b.capacity) - } -} - -func TestRingBuffer_SnapshotIsolation(t *testing.T) { - b := New(5) - b.Push(Entry{Message: "first"}) - snap := b.Snapshot() - snap[0].Message = "mutated" - if got := b.Snapshot()[0].Message; got != "first" { - t.Fatalf("snapshot mutation leaked: got %q", got) - } -} - -func TestRingBuffer_ConcurrentWrites(t *testing.T) { - const writers = 8 - const perWriter = 200 - b := New(writers * perWriter) - var wg sync.WaitGroup - wg.Add(writers) - for w := 0; w < writers; w++ { - go func() { - defer wg.Done() - for i := 0; i < perWriter; i++ { - b.Push(Entry{Timestamp: time.Now(), Message: "x"}) - } - }() - } - wg.Wait() - if got := b.Len(); got != writers*perWriter { - t.Fatalf("expected %d entries, got %d", writers*perWriter, got) - } -} - -func TestRingBuffer_Clear(t *testing.T) { - b := New(3) - b.Push(Entry{Message: "a"}) - b.Push(Entry{Message: "b"}) - b.Clear() - if got := b.Len(); got != 0 { - t.Fatalf("expected 0 entries after clear, got %d", got) - } -} - -func TestDefault_ReturnsSameInstance(t *testing.T) { - first := Default() - second := Default() - if first != second { - t.Fatal("Default() should return the same singleton") - } -} - -func TestCore_CapturesEntriesWithFields(t *testing.T) { - buf := New(10) - core := NewCore(buf, zapcore.DebugLevel) - logger := zap.New(core).With(zap.String("service", "test")) - - logger.Info("hello", zap.Int("count", 42)) - logger.Warn("careful") - - snap := buf.Snapshot() - if len(snap) != 2 { - t.Fatalf("expected 2 entries, got %d", len(snap)) - } - if snap[0].Message != "hello" || snap[0].Level != "info" { - t.Errorf("entry 0: got message=%q level=%q", snap[0].Message, snap[0].Level) - } - if got, _ := snap[0].Fields["service"].(string); got != "test" { - t.Errorf("entry 0 service field: got %v", snap[0].Fields["service"]) - } - if got, _ := snap[0].Fields["count"].(int64); got != 42 { - t.Errorf("entry 0 count field: got %v", snap[0].Fields["count"]) - } - if snap[1].Level != "warn" { - t.Errorf("entry 1 level: got %q", snap[1].Level) - } -} - -func TestRingBuffer_SnapshotDeepCopiesFields(t *testing.T) { - b := New(3) - b.Push(Entry{Message: "m", Fields: map[string]any{"k": "v1"}}) - - snap1 := b.Snapshot() - snap1[0].Fields["k"] = "mutated" - snap1[0].Fields["new"] = "added" - - snap2 := b.Snapshot() - if got := snap2[0].Fields["k"]; got != "v1" { - t.Errorf("buffered field mutated: got %v", got) - } - if _, ok := snap2[0].Fields["new"]; ok { - t.Errorf("buffered field map had extra key added") - } -} - -func TestCore_RespectsLevelEnabler(t *testing.T) { - buf := New(10) - core := NewCore(buf, zapcore.WarnLevel) - logger := zap.New(core) - - logger.Debug("nope") - logger.Info("nope") - logger.Warn("yes") - logger.Error("yes") - - if got := buf.Len(); got != 2 { - t.Fatalf("expected only warn+error to be captured, got %d entries", got) - } -} diff --git a/apps/backend/internal/common/logger/daily_writer.go b/apps/backend/internal/common/logger/daily_writer.go new file mode 100644 index 0000000000..f08076df38 --- /dev/null +++ b/apps/backend/internal/common/logger/daily_writer.go @@ -0,0 +1,409 @@ +package logger + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sync" + "time" +) + +const ( + activeBackendLogName = "backend-logs.log" + backendDayMarkerName = ".backend-logs-day" + rolloverJournalName = ".backend-logs-rollover.json" + dailyBackendLogBytes = 256 * 1024 * 1024 +) + +var datedBackendLogPattern = regexp.MustCompile(`^backend-logs-(\d{4}-\d{2}-\d{2})\.log$`) +var errDailyBackendLogLimit = errors.New("daily backend log byte limit reached") + +type rolloverJournal struct { + SourceDay string `json:"source_day"` + SourceSize int64 `json:"source_size"` + SourceModUnixNano int64 `json:"source_mod_unix_nano"` + Destination string `json:"destination"` + DestinationStart int64 `json:"destination_start"` + CopiedOffset int64 `json:"copied_offset"` +} + +type dailyWriter struct { + mu sync.Mutex + logDir string + now func() time.Time + day string + file *os.File + size int64 + maxBytes int64 + closed bool + maintenanceErr error +} + +func newDailyWriter(logDir string, now func() time.Time) (*dailyWriter, error) { + if now == nil { + now = time.Now + } + if err := os.MkdirAll(logDir, 0o700); err != nil { + return nil, fmt.Errorf("create log directory: %w", err) + } + if err := os.Chmod(logDir, 0o700); err != nil { + return nil, fmt.Errorf("secure log directory: %w", err) + } + writer := &dailyWriter{logDir: logDir, now: now, maxBytes: dailyBackendLogBytes} + if err := writer.prepare(); err != nil { + return nil, err + } + return writer, nil +} + +func (w *dailyWriter) prepare() error { + today := utcDay(w.now()) + if err := w.recoverRollover(); err != nil { + return fmt.Errorf("recover rollover: %w", err) + } + day, err := w.readActiveDay(today) + if err != nil { + return err + } + if day != today { + if err := w.rollover(day); err != nil { + return fmt.Errorf("roll over %s: %w", day, err) + } + } + if err := w.openActive(today); err != nil { + return err + } + w.maintenanceErr = w.cleanup(today) + return nil +} + +func (w *dailyWriter) Write(data []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return 0, os.ErrClosed + } + today := utcDay(w.now()) + if today != w.day { + if err := w.switchDay(today); err != nil { + // Preserve the triggering entry in the active file. The next write + // retries rollover while the marker still names the prior day. + if w.file == nil { + return 0, err + } + return w.file.Write(data) + } + } + if w.size+int64(len(data)) > w.maxBytes { + return 0, errDailyBackendLogLimit + } + written, err := w.file.Write(data) + w.size += int64(written) + return written, err +} + +func (w *dailyWriter) switchDay(today string) error { + previous := w.day + if err := w.file.Close(); err != nil { + return err + } + w.file = nil + if err := w.rollover(previous); err != nil { + return w.reopenPrevious(previous, err) + } + if err := w.openActive(today); err != nil { + return err + } + w.maintenanceErr = w.cleanup(today) + return nil +} + +func (w *dailyWriter) reopenPrevious(day string, rolloverErr error) error { + file, openErr := openOwnerAppend(filepath.Join(w.logDir, activeBackendLogName)) + if openErr != nil { + return errors.Join(rolloverErr, openErr) + } + w.file = file + w.day = day + info, statErr := file.Stat() + if statErr != nil { + _ = file.Close() + w.file = nil + return errors.Join(rolloverErr, statErr) + } + w.size = info.Size() + return rolloverErr +} + +func (w *dailyWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return nil + } + w.closed = true + if w.file == nil { + return nil + } + return w.file.Close() +} + +func (w *dailyWriter) MaintenanceError() error { + w.mu.Lock() + defer w.mu.Unlock() + return w.maintenanceErr +} + +func (w *dailyWriter) readActiveDay(today string) (string, error) { + activePath := filepath.Join(w.logDir, activeBackendLogName) + if _, err := os.Stat(activePath); os.IsNotExist(err) { + return today, nil + } else if err != nil { + return "", fmt.Errorf("stat active log: %w", err) + } + data, err := os.ReadFile(filepath.Join(w.logDir, backendDayMarkerName)) + if err == nil { + if day := string(data); validUTCDay(day) { + return day, nil + } + } + info, err := os.Stat(activePath) + if err != nil { + return "", fmt.Errorf("stat unmarked active log: %w", err) + } + return info.ModTime().UTC().Format(time.DateOnly), nil +} + +func (w *dailyWriter) openActive(day string) error { + file, err := openOwnerAppend(filepath.Join(w.logDir, activeBackendLogName)) + if err != nil { + return fmt.Errorf("open active log: %w", err) + } + if err := writeOwnerFile(filepath.Join(w.logDir, backendDayMarkerName), []byte(day)); err != nil { + _ = file.Close() + return fmt.Errorf("write active day marker: %w", err) + } + w.file = file + w.day = day + info, err := file.Stat() + if err != nil { + _ = file.Close() + w.file = nil + return fmt.Errorf("stat active log: %w", err) + } + w.size = info.Size() + return nil +} + +func (w *dailyWriter) rollover(day string) error { + activePath := filepath.Join(w.logDir, activeBackendLogName) + info, err := os.Stat(activePath) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + journal, err := w.loadOrCreateJournal(day, info) + if err != nil { + return err + } + if err := w.copyRollover(activePath, journal); err != nil { + return err + } + if err := os.Remove(activePath); err != nil && !os.IsNotExist(err) { + return err + } + _ = os.Remove(filepath.Join(w.logDir, backendDayMarkerName)) + return removeIfExists(filepath.Join(w.logDir, rolloverJournalName)) +} + +func (w *dailyWriter) loadOrCreateJournal(day string, info os.FileInfo) (*rolloverJournal, error) { + path := filepath.Join(w.logDir, rolloverJournalName) + if data, err := os.ReadFile(path); err == nil { + var journal rolloverJournal + if err := json.Unmarshal(data, &journal); err != nil { + return nil, fmt.Errorf("decode rollover journal: %w", err) + } + return &journal, nil + } + destination := fmt.Sprintf("backend-logs-%s.log", day) + destinationInfo, err := os.Stat(filepath.Join(w.logDir, destination)) + var destinationStart int64 + if err == nil { + destinationStart = destinationInfo.Size() + } else if !os.IsNotExist(err) { + return nil, err + } + journal := &rolloverJournal{ + SourceDay: day, SourceSize: info.Size(), SourceModUnixNano: info.ModTime().UnixNano(), + Destination: destination, DestinationStart: destinationStart, + } + return journal, writeJSONOwnerFile(path, journal) +} + +func (w *dailyWriter) copyRollover(activePath string, journal *rolloverJournal) error { + source, err := os.Open(activePath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer func() { _ = source.Close() }() + sourceInfo, err := source.Stat() + if err != nil { + return err + } + if sourceInfo.Size() != journal.SourceSize || + sourceInfo.ModTime().UnixNano() != journal.SourceModUnixNano { + journal.SourceSize = sourceInfo.Size() + journal.SourceModUnixNano = sourceInfo.ModTime().UnixNano() + if err := writeJSONOwnerFile(filepath.Join(w.logDir, rolloverJournalName), journal); err != nil { + return err + } + } + destinationPath := filepath.Join(w.logDir, filepath.Base(journal.Destination)) + destination, err := openOwnerAppend(destinationPath) + if err != nil { + return err + } + defer func() { _ = destination.Close() }() + destinationInfo, err := destination.Stat() + if err != nil { + return err + } + copied := destinationInfo.Size() - journal.DestinationStart + if copied < 0 || copied > journal.SourceSize { + return fmt.Errorf("invalid rollover destination size") + } + journal.CopiedOffset = copied + if _, err := source.Seek(copied, io.SeekStart); err != nil { + return err + } + if err := copyJournaled(source, destination, journal, filepath.Join(w.logDir, rolloverJournalName)); err != nil { + return err + } + return destination.Sync() +} + +func copyJournaled(source io.Reader, destination io.Writer, journal *rolloverJournal, path string) error { + buf := make([]byte, 1024*1024) + for { + n, readErr := source.Read(buf) + if n > 0 { + written, writeErr := destination.Write(buf[:n]) + journal.CopiedOffset += int64(written) + if writeErr != nil { + return writeErr + } + if written != n { + return io.ErrShortWrite + } + if err := writeJSONOwnerFile(path, journal); err != nil { + return err + } + } + if errors.Is(readErr, io.EOF) { + return nil + } + if readErr != nil { + return readErr + } + } +} + +func (w *dailyWriter) recoverRollover() error { + path := filepath.Join(w.logDir, rolloverJournalName) + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + var journal rolloverJournal + if err := json.Unmarshal(data, &journal); err != nil { + return err + } + activePath := filepath.Join(w.logDir, activeBackendLogName) + if _, err := os.Stat(activePath); os.IsNotExist(err) { + return removeIfExists(path) + } + if err := w.copyRollover(activePath, &journal); err != nil { + return err + } + if err := os.Remove(activePath); err != nil && !os.IsNotExist(err) { + return err + } + return removeIfExists(path) +} + +func (w *dailyWriter) cleanup(today string) error { + entries, err := os.ReadDir(w.logDir) + if err != nil { + return err + } + cutoff, _ := time.Parse(time.DateOnly, today) + cutoff = cutoff.AddDate(0, 0, -2) + for _, entry := range entries { + match := datedBackendLogPattern.FindStringSubmatch(entry.Name()) + if len(match) != 2 { + continue + } + day, err := time.Parse(time.DateOnly, match[1]) + if err == nil && day.Before(cutoff) { + if err := os.Remove(filepath.Join(w.logDir, entry.Name())); err != nil { + return err + } + } + } + return nil +} + +func utcDay(now time.Time) string { + return now.UTC().Format(time.DateOnly) +} + +func validUTCDay(value string) bool { + parsed, err := time.Parse(time.DateOnly, value) + return err == nil && parsed.Format(time.DateOnly) == value +} + +func openOwnerAppend(path string) (*os.File, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err == nil { + err = os.Chmod(path, 0o600) + } + return file, err +} + +func writeOwnerFile(path string, data []byte) error { + temp := path + ".tmp" + if err := os.WriteFile(temp, data, 0o600); err != nil { + return err + } + if err := os.Chmod(temp, 0o600); err != nil { + return err + } + return os.Rename(temp, path) +} + +func writeJSONOwnerFile(path string, value any) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + return writeOwnerFile(path, data) +} + +func removeIfExists(path string) error { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} diff --git a/apps/backend/internal/common/logger/daily_writer_test.go b/apps/backend/internal/common/logger/daily_writer_test.go new file mode 100644 index 0000000000..71dc81581a --- /dev/null +++ b/apps/backend/internal/common/logger/daily_writer_test.go @@ -0,0 +1,182 @@ +package logger + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestDailyWriterAppendsAndRollsByUTCDay(t *testing.T) { + logDir := t.TempDir() + now := time.Date(2026, 7, 29, 23, 59, 0, 0, time.FixedZone("local", 2*60*60)) + writer, err := newDailyWriter(logDir, func() time.Time { return now }) + if err != nil { + t.Fatalf("newDailyWriter: %v", err) + } + if _, err := writer.Write([]byte("day-one\n")); err != nil { + t.Fatalf("write day one: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close day one: %v", err) + } + + writer, err = newDailyWriter(logDir, func() time.Time { return now }) + if err != nil { + t.Fatalf("restart same day: %v", err) + } + if _, err := writer.Write([]byte("same-day\n")); err != nil { + t.Fatalf("write same day: %v", err) + } + now = time.Date(2026, 7, 30, 0, 1, 0, 0, time.UTC) + if _, err := writer.Write([]byte("day-two\n")); err != nil { + t.Fatalf("write day two: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close day two: %v", err) + } + + archived := readLogFile(t, filepath.Join(logDir, "backend-logs-2026-07-29.log")) + if archived != "day-one\nsame-day\n" { + t.Fatalf("archived log = %q", archived) + } + if active := readLogFile(t, filepath.Join(logDir, "backend-logs.log")); active != "day-two\n" { + t.Fatalf("active log = %q", active) + } +} + +func TestDailyWriterRetentionLeavesUnrelatedFiles(t *testing.T) { + logDir := t.TempDir() + for name := range map[string]bool{ + "backend-logs-2026-07-26.log": true, + "backend-logs-2026-07-27.log": true, + "backend-logs-2026-07-28.log": true, + "backend-logs-2026-07-29.log": true, + "backend-logs-not-a-day.log": true, + "another-service.log": true, + } { + if err := os.WriteFile(filepath.Join(logDir, name), []byte(name), 0o600); err != nil { + t.Fatalf("seed %s: %v", name, err) + } + } + writer, err := newDailyWriter(logDir, func() time.Time { + return time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + }) + if err != nil { + t.Fatalf("newDailyWriter: %v", err) + } + _ = writer.Close() + + for _, name := range []string{"backend-logs-2026-07-28.log", "backend-logs-2026-07-29.log", + "backend-logs-not-a-day.log", "another-service.log"} { + if _, err := os.Stat(filepath.Join(logDir, name)); err != nil { + t.Errorf("expected %s to remain: %v", name, err) + } + } + for _, name := range []string{"backend-logs-2026-07-26.log", "backend-logs-2026-07-27.log"} { + if _, err := os.Stat(filepath.Join(logDir, name)); !os.IsNotExist(err) { + t.Errorf("expected %s to be removed, stat error = %v", name, err) + } + } +} + +func TestDailyWriterResumesJournalWithoutDuplicatingDestination(t *testing.T) { + logDir := t.TempDir() + activePath := filepath.Join(logDir, activeBackendLogName) + source := []byte("first\nsecond\nthird\n") + if err := os.WriteFile(activePath, source, 0o600); err != nil { + t.Fatalf("seed active log: %v", err) + } + info, err := os.Stat(activePath) + if err != nil { + t.Fatalf("stat active log: %v", err) + } + destination := "backend-logs-2026-07-29.log" + prefix := source[:len("first\n")] + if err := os.WriteFile(filepath.Join(logDir, destination), prefix, 0o600); err != nil { + t.Fatalf("seed destination: %v", err) + } + journal := rolloverJournal{ + SourceDay: "2026-07-29", SourceSize: info.Size(), + SourceModUnixNano: info.ModTime().UnixNano(), Destination: destination, + DestinationStart: 0, CopiedOffset: int64(len(prefix)), + } + if err := writeJSONOwnerFile(filepath.Join(logDir, rolloverJournalName), journal); err != nil { + t.Fatalf("seed journal: %v", err) + } + if err := os.WriteFile(filepath.Join(logDir, backendDayMarkerName), []byte("2026-07-29"), 0o600); err != nil { + t.Fatalf("seed marker: %v", err) + } + + now := func() time.Time { return time.Date(2026, 7, 30, 1, 0, 0, 0, time.UTC) } + writer, err := newDailyWriter(logDir, now) + if err != nil { + t.Fatalf("recover writer: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close recovered writer: %v", err) + } + writer, err = newDailyWriter(logDir, now) + if err != nil { + t.Fatalf("restart recovered writer: %v", err) + } + _ = writer.Close() + + if got := readLogFile(t, filepath.Join(logDir, destination)); got != string(source) { + t.Fatalf("recovered destination = %q", got) + } + if _, err := os.Stat(filepath.Join(logDir, rolloverJournalName)); !os.IsNotExist(err) { + t.Fatalf("rollover journal remains after recovery: %v", err) + } +} + +func TestDailyWriterRejectsEntriesBeyondDailyByteLimit(t *testing.T) { + logDir := t.TempDir() + activePath := filepath.Join(logDir, activeBackendLogName) + const dailyLimit = int64(len("full\n")) + if err := os.WriteFile(activePath, []byte("full\n"), 0o600); err != nil { + t.Fatalf("seed active log: %v", err) + } + if err := os.WriteFile(filepath.Join(logDir, backendDayMarkerName), + []byte("2026-07-30"), 0o600); err != nil { + t.Fatalf("seed day marker: %v", err) + } + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + writer, err := newDailyWriter(logDir, func() time.Time { return now }) + if err != nil { + t.Fatalf("newDailyWriter: %v", err) + } + defer func() { _ = writer.Close() }() + if writer.maxBytes != dailyBackendLogBytes { + t.Fatalf("default daily limit = %d, want %d", writer.maxBytes, dailyBackendLogBytes) + } + writer.maxBytes = dailyLimit + + if _, err := writer.Write([]byte("must-not-grow\n")); err == nil { + t.Fatal("write beyond daily byte limit succeeded") + } + info, err := os.Stat(activePath) + if err != nil { + t.Fatalf("stat active log: %v", err) + } + if info.Size() != dailyLimit { + t.Fatalf("active log size = %d, want %d", info.Size(), dailyLimit) + } + now = now.Add(24 * time.Hour) + if _, err := writer.Write([]byte("next\n")); err != nil { + t.Fatalf("next-day write: %v", err) + } + if active := readLogFile(t, activePath); active != "next\n" { + t.Fatalf("next-day active log = %q", active) + } +} + +func readLogFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return strings.ReplaceAll(string(data), "\r\n", "\n") +} diff --git a/apps/backend/internal/common/logger/logger.go b/apps/backend/internal/common/logger/logger.go index ffeea4201d..e7cec3c065 100644 --- a/apps/backend/internal/common/logger/logger.go +++ b/apps/backend/internal/common/logger/logger.go @@ -4,14 +4,11 @@ package logger import ( "context" "os" - "path/filepath" "sync" "go.uber.org/zap" "go.uber.org/zap/zapcore" "gopkg.in/natefinch/lumberjack.v2" - - "github.com/kandev/kandev/internal/common/logger/buffer" ) // Context keys for extracting values from context. @@ -42,10 +39,12 @@ type LoggingConfig struct { // Logger wraps zap.Logger to provide structured logging with helper methods. type Logger struct { - zap *zap.Logger - sugar *zap.SugaredLogger - fields []zap.Field - rotator *lumberjack.Logger // non-nil only when OutputPath is a file path + zap *zap.Logger + sugar *zap.SugaredLogger + fields []zap.Field + rotator *lumberjack.Logger // non-nil only when OutputPath is a file path + backend *backendLoggerRuntime + backendLogDir string } var ( @@ -131,9 +130,7 @@ func NewLogger(cfg LoggingConfig) (*Logger, error) { } outputCore := zapcore.NewCore(encoder, writeSyncer, level) - bufferCore := buffer.NewCore(buffer.Default(), level) - core := zapcore.NewTee(outputCore, bufferCore) - zapLogger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel)) + zapLogger := zap.New(outputCore, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel)) return &Logger{ zap: zapLogger, @@ -177,6 +174,9 @@ func (l *Logger) Sync() error { // rotation cycle can complete cleanly. func (l *Logger) Close() error { _ = l.zap.Sync() + if l.backend != nil { + return l.backend.Close() + } if l.rotator != nil { return l.rotator.Close() } @@ -189,10 +189,12 @@ func (l *Logger) Close() error { func (l *Logger) WithFields(fields ...zap.Field) *Logger { z := l.zap.With(fields...) return &Logger{ - zap: z, - sugar: z.Sugar(), - fields: append(l.fields, fields...), - rotator: l.rotator, + zap: z, + sugar: z.Sugar(), + fields: append(l.fields, fields...), + rotator: l.rotator, + backend: l.backend, + backendLogDir: l.backendLogDir, } } @@ -248,9 +250,32 @@ func (l *Logger) Error(msg string, fields ...zap.Field) { l.zap.Error(msg, fields...) } +// DPanic logs a terminal development-panic entry. Backend loggers perform +// their bounded drain and direct-stderr fallback before panicking. +func (l *Logger) DPanic(msg string, fields ...zap.Field) { + if l.backend != nil { + l.backend.terminal("dpanic", msg) + panic(msg) + } + l.zap.DPanic(msg, fields...) +} + +// Panic logs a terminal entry, drains backend sinks, then panics. +func (l *Logger) Panic(msg string, fields ...zap.Field) { + if l.backend != nil { + l.backend.terminal("panic", msg) + panic(msg) + } + l.zap.Panic(msg, fields...) +} + // Fatal logs a message at fatal level with optional structured fields, // then calls os.Exit(1). func (l *Logger) Fatal(msg string, fields ...zap.Field) { + if l.backend != nil { + l.backend.terminal("fatal", msg) + os.Exit(1) + } l.zap.Fatal(msg, fields...) } @@ -264,28 +289,11 @@ func (l *Logger) Sugar() *zap.SugaredLogger { return l.sugar } -// BufferSnapshot returns a copy of the entries currently held in the -// process-wide log ring buffer used by Improve Kandev reports. -func (l *Logger) BufferSnapshot() []buffer.Entry { - return buffer.Default().Snapshot() -} - -// LogDirectory returns the directory containing the rotating log file managed -// by lumberjack, or empty string when the logger is writing to stdout/stderr -// (no file rotation configured). Powers the System -> Logs page enumeration. -func (l *Logger) LogDirectory() string { - if l == nil || l.rotator == nil || l.rotator.Filename == "" { - return "" +// SinkStatistics returns loss and queue counters for the backend's asynchronous +// file and stdout sinks. Legacy loggers return no sink statistics. +func (l *Logger) SinkStatistics() []SinkStatistics { + if l == nil || l.backend == nil { + return nil } - return filepath.Dir(l.rotator.Filename) -} - -// LogFilename returns the base filename (no directory) of the active -// lumberjack log, e.g. "kandev.log". Rotated files use the same base with a -// timestamp suffix. Empty when no file logging is configured. -func (l *Logger) LogFilename() string { - if l == nil || l.rotator == nil || l.rotator.Filename == "" { - return "" - } - return filepath.Base(l.rotator.Filename) + return l.backend.Stats() } diff --git a/apps/backend/internal/common/logger/logger_test.go b/apps/backend/internal/common/logger/logger_test.go index fa59a6f08e..9ebf61efc4 100644 --- a/apps/backend/internal/common/logger/logger_test.go +++ b/apps/backend/internal/common/logger/logger_test.go @@ -1,14 +1,219 @@ package logger import ( + "bytes" "os" "path/filepath" "strings" + "sync" + "sync/atomic" "testing" + "time" "go.uber.org/zap" ) +type lockedBuffer struct { + mu sync.Mutex + bytes.Buffer +} + +type blockingWriteCloser struct { + mu sync.Mutex + once sync.Once + started chan struct{} + release chan struct{} +} + +func (w *blockingWriteCloser) Write(data []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.once.Do(func() { close(w.started) }) + <-w.release + return len(data), nil +} + +func (w *blockingWriteCloser) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + return nil +} + +func (b *lockedBuffer) Write(data []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.Buffer.Write(data) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.Buffer.String() +} + +func TestNewBackendLoggerUsesIndependentThresholds(t *testing.T) { + homeDir := t.TempDir() + stdout := &lockedBuffer{} + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + log, err := NewBackendLogger(BackendLoggingConfig{ + HomeDir: homeDir, Level: "info", Format: "json", Stdout: stdout, + Stderr: &lockedBuffer{}, Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("NewBackendLogger: %v", err) + } + log.Info("file-only info") + log.Warn("visible warning") + if err := log.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + + fileLog := readLogFile(t, filepath.Join(homeDir, "logs", activeBackendLogName)) + if !strings.Contains(fileLog, "file-only info") || !strings.Contains(fileLog, "visible warning") { + t.Fatalf("file log missing entries: %s", fileLog) + } + stdoutLog := stdout.String() + if strings.Contains(stdoutLog, "file-only info") || !strings.Contains(stdoutLog, "visible warning") { + t.Fatalf("stdout thresholds incorrect: %s", stdoutLog) + } + if !strings.Contains(stdoutLog, filepath.Join(homeDir, "logs", activeBackendLogName)) { + t.Fatalf("startup output does not identify log path: %s", stdoutLog) + } +} + +func TestNewBackendLoggerDebugAndVerboseThresholds(t *testing.T) { + tests := []struct { + name string + level string + consoleLevel string + wantDebugFile bool + wantInfoStdout bool + }{ + {name: "debug", level: "debug", wantDebugFile: true}, + {name: "verbose", level: "info", consoleLevel: "info", wantInfoStdout: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + homeDir := t.TempDir() + stdout := &lockedBuffer{} + log, err := NewBackendLogger(BackendLoggingConfig{ + HomeDir: homeDir, Level: test.level, Format: "json", ConsoleLevel: test.consoleLevel, + Stdout: stdout, Stderr: &lockedBuffer{}, + }) + if err != nil { + t.Fatalf("NewBackendLogger: %v", err) + } + log.Debug("debug entry") + log.Info("info entry") + _ = log.Close() + fileLog := readLogFile(t, filepath.Join(homeDir, "logs", activeBackendLogName)) + if got := strings.Contains(fileLog, "debug entry"); got != test.wantDebugFile { + t.Fatalf("debug file presence = %v, log = %s", got, fileLog) + } + if got := strings.Contains(stdout.String(), "info entry"); got != test.wantInfoStdout { + t.Fatalf("info stdout presence = %v, log = %s", got, stdout.String()) + } + }) + } +} + +func TestNewBackendLoggerSurvivesUnavailableFileSink(t *testing.T) { + homeDir := t.TempDir() + if err := os.WriteFile(filepath.Join(homeDir, "logs"), []byte("not a directory"), 0o600); err != nil { + t.Fatalf("seed conflicting log path: %v", err) + } + stdout := &lockedBuffer{} + stderr := &lockedBuffer{} + log, err := NewBackendLogger(BackendLoggingConfig{ + HomeDir: homeDir, Level: "info", Format: "json", Stdout: stdout, Stderr: stderr, + }) + if err != nil { + t.Fatalf("file failure prevented logger startup: %v", err) + } + log.Warn("stdout remains available") + if err := log.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if !strings.Contains(stdout.String(), "stdout remains available") { + t.Fatalf("stdout fallback missing warning: %s", stdout.String()) + } + if !strings.Contains(stderr.String(), "retrying in 30s") { + t.Fatalf("file activation failure was not reported: %s", stderr.String()) + } + stats := log.SinkStatistics() + if len(stats) != 2 || stats[0].Lost["warn"]["write_error"] != 1 { + t.Fatalf("file sink statistics = %#v", stats) + } +} + +func TestNewBackendLoggerRetriesFileActivation(t *testing.T) { + homeDir := t.TempDir() + logPath := filepath.Join(homeDir, "logs") + if err := os.WriteFile(logPath, []byte("not a directory"), 0o600); err != nil { + t.Fatalf("seed conflicting log path: %v", err) + } + initialNow := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + var nowUnixNano atomic.Int64 + nowUnixNano.Store(initialNow.UnixNano()) + log, err := NewBackendLogger(BackendLoggingConfig{ + HomeDir: homeDir, Level: "info", Format: "json", Stdout: &lockedBuffer{}, + Stderr: &lockedBuffer{}, Now: func() time.Time { return time.Unix(0, nowUnixNano.Load()) }, + }) + if err != nil { + t.Fatalf("NewBackendLogger: %v", err) + } + log.Info("before recovery") + deadline := time.Now().Add(time.Second) + for log.SinkStatistics()[0].Lost["info"]["write_error"] == 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if log.SinkStatistics()[0].Lost["info"]["write_error"] != 1 { + t.Fatal("initial file write failure was not observed") + } + if err := os.Remove(logPath); err != nil { + t.Fatalf("remove conflicting path: %v", err) + } + nowUnixNano.Store(initialNow.Add(31 * time.Second).UnixNano()) + log.Info("after recovery") + if err := log.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if got := readLogFile(t, filepath.Join(logPath, activeBackendLogName)); !strings.Contains(got, "after recovery") { + t.Fatalf("recovered file log = %s", got) + } +} + +func TestBackendLoggerCloseDoesNotWaitOnBlockedFileWriter(t *testing.T) { + writer := &blockingWriteCloser{started: make(chan struct{}), release: make(chan struct{})} + fileSink := newAsyncSink(writer, asyncSinkConfig{ + Name: "file", MaxEntries: 3, MaxBytes: 30, MaxEntryBytes: 20, + }) + stdoutSink := newAsyncSink(discardWriter{}, asyncSinkConfig{ + Name: "stdout", MaxEntries: 3, MaxBytes: 30, MaxEntryBytes: 20, + }) + runtime := &backendLoggerRuntime{ + fileSink: fileSink, stdoutSink: stdoutSink, fileWriter: writer, + stderr: &lockedBuffer{}, closeTimeout: 20 * time.Millisecond, + } + if !fileSink.Enqueue(zap.WarnLevel, []byte("warning")) { + t.Fatal("warning was rejected") + } + <-writer.started + + done := make(chan error, 1) + go func() { done <- runtime.Close() }() + select { + case err := <-done: + if err == nil { + t.Fatal("Close succeeded despite blocked file writer") + } + case <-time.After(100 * time.Millisecond): + close(writer.release) + t.Fatal("Close exceeded bounded deadline") + } + close(writer.release) +} + func TestNewLogger_StdoutHasNoRotator(t *testing.T) { for _, path := range []string{"", "stdout", "stderr"} { t.Run(path, func(t *testing.T) { @@ -98,42 +303,6 @@ func TestWithFields_PropagatesRotator(t *testing.T) { } } -func TestLogDirectory_FileOutputReturnsDir(t *testing.T) { - dir := t.TempDir() - logPath := filepath.Join(dir, "kandev.log") - log, err := NewLogger(LoggingConfig{Level: "info", Format: "json", OutputPath: logPath}) - if err != nil { - t.Fatalf("NewLogger: %v", err) - } - defer func() { _ = log.Close() }() - - if got := log.LogDirectory(); got != dir { - t.Errorf("LogDirectory() = %q, want %q", got, dir) - } -} - -func TestLogDirectory_StdoutReturnsEmpty(t *testing.T) { - for _, path := range []string{"", "stdout", "stderr"} { - t.Run(path, func(t *testing.T) { - log, err := NewLogger(LoggingConfig{Level: "info", Format: "json", OutputPath: path}) - if err != nil { - t.Fatalf("NewLogger: %v", err) - } - defer func() { _ = log.Close() }() - if got := log.LogDirectory(); got != "" { - t.Errorf("LogDirectory() = %q, want empty", got) - } - }) - } -} - -func TestLogDirectory_NilLoggerReturnsEmpty(t *testing.T) { - var l *Logger - if got := l.LogDirectory(); got != "" { - t.Errorf("nil LogDirectory() = %q, want empty", got) - } -} - func TestLoggerClose_IsIdempotent(t *testing.T) { dir := t.TempDir() log, err := NewLogger(LoggingConfig{ diff --git a/apps/backend/internal/debug/export.go b/apps/backend/internal/debug/export.go deleted file mode 100644 index 798ae68e03..0000000000 --- a/apps/backend/internal/debug/export.go +++ /dev/null @@ -1,75 +0,0 @@ -package debug - -import ( - "net/http" - "runtime" - "time" - - "github.com/gin-gonic/gin" - "github.com/kandev/kandev/internal/common/logger" - "github.com/kandev/kandev/internal/common/logger/buffer" -) - -var startTime = time.Now() - -// ExportMetadata contains system runtime information for debugging. -type ExportMetadata struct { - UptimeSeconds float64 `json:"uptime_seconds"` - Version string `json:"version"` - Commit string `json:"commit"` - GoVersion string `json:"go_version"` - OS string `json:"os"` - Arch string `json:"arch"` - Goroutines int `json:"goroutines"` - HeapAllocMB float64 `json:"heap_alloc_mb"` - CapturedAt string `json:"captured_at"` -} - -// ExportResponse is the JSON response for the debug export endpoint. -type ExportResponse struct { - Metadata ExportMetadata `json:"metadata"` - Logs []buffer.Entry `json:"logs"` -} - -// RegisterExportRoute registers GET /api/v1/system/debug/export. -func RegisterExportRoute(router *gin.Engine, version, commit string, _ *logger.Logger) { - api := router.Group("/api/v1/system/debug") - api.GET("/export", handleExport(version, commit)) -} - -func handleExport(version, commit string) gin.HandlerFunc { - return func(c *gin.Context) { - logs := buffer.Default().Snapshot() - - // Optional level filter. - if level := c.Query("level"); level != "" { - filtered := make([]buffer.Entry, 0, len(logs)) - for _, e := range logs { - if e.Level == level { - filtered = append(filtered, e) - } - } - logs = filtered - } - - var mem runtime.MemStats - runtime.ReadMemStats(&mem) - - resp := ExportResponse{ - Metadata: ExportMetadata{ - UptimeSeconds: time.Since(startTime).Seconds(), - Version: version, - Commit: commit, - GoVersion: runtime.Version(), - OS: runtime.GOOS, - Arch: runtime.GOARCH, - Goroutines: runtime.NumGoroutine(), - HeapAllocMB: float64(mem.HeapAlloc) / (1024 * 1024), - CapturedAt: time.Now().UTC().Format(time.RFC3339), - }, - Logs: logs, - } - - c.JSON(http.StatusOK, resp) - } -} diff --git a/apps/backend/internal/debug/export_test.go b/apps/backend/internal/debug/export_test.go deleted file mode 100644 index 7db4d41340..0000000000 --- a/apps/backend/internal/debug/export_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package debug - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "github.com/gin-gonic/gin" - "github.com/kandev/kandev/internal/common/logger/buffer" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestHandleExport(t *testing.T) { - gin.SetMode(gin.TestMode) - - // Push a test entry into the default buffer. - buffer.Default().Push(buffer.Entry{ - Level: "error", - Logger: "test", - Caller: "debug/export_test.go:20", - Message: "something went wrong", - Fields: map[string]any{"task_id": "abc"}, - }) - - router := gin.New() - RegisterExportRoute(router, "1.0.0-test", "deadbeef", nil) - - t.Run("returns metadata and logs", func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/api/v1/system/debug/export", nil) - w := httptest.NewRecorder() - router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - - var resp ExportResponse - err := json.Unmarshal(w.Body.Bytes(), &resp) - require.NoError(t, err) - - assert.Equal(t, "1.0.0-test", resp.Metadata.Version) - assert.Equal(t, "deadbeef", resp.Metadata.Commit) - assert.NotEmpty(t, resp.Metadata.GoVersion) - assert.NotEmpty(t, resp.Metadata.OS) - assert.NotEmpty(t, resp.Metadata.Arch) - assert.Greater(t, resp.Metadata.Goroutines, 0) - assert.Greater(t, resp.Metadata.UptimeSeconds, 0.0) - assert.NotEmpty(t, resp.Metadata.CapturedAt) - - assert.NotEmpty(t, resp.Logs) - }) - - t.Run("filters by level", func(t *testing.T) { - buffer.Default().Push(buffer.Entry{ - Level: "info", - Message: "normal operation", - }) - - req := httptest.NewRequest(http.MethodGet, "/api/v1/system/debug/export?level=error", nil) - w := httptest.NewRecorder() - router.ServeHTTP(w, req) - - assert.Equal(t, http.StatusOK, w.Code) - - var resp ExportResponse - err := json.Unmarshal(w.Body.Bytes(), &resp) - require.NoError(t, err) - - for _, entry := range resp.Logs { - assert.Equal(t, "error", entry.Level) - } - }) -} diff --git a/apps/backend/internal/gateway/websocket/hub.go b/apps/backend/internal/gateway/websocket/hub.go index dc59b2c827..7078fef800 100644 --- a/apps/backend/internal/gateway/websocket/hub.go +++ b/apps/backend/internal/gateway/websocket/hub.go @@ -459,6 +459,32 @@ func (h *Hub) BroadcastToUser(userID string, msg *ws.Message) bool { return h.sendToClients(data, clients, msg.Action) > 0 } +// SendToIdentity sends a notification to every connected client belonging to +// the authenticated identity, without requiring a topic subscription. +func (h *Hub) SendToIdentity(userID string, msg *ws.Message) int { + if userID == "" { + return 0 + } + data, err := json.Marshal(msg) + if err != nil { + h.logger.Error("Failed to marshal identity message", zap.Error(err)) + return 0 + } + h.mu.RLock() + clients := make([]*Client, 0) + for client := range h.clients { + if client.identity.UserID == userID { + clients = append(clients, client) + } + } + h.mu.RUnlock() + h.logger.Debug("SendToIdentity", + zap.String("user_id", userID), + zap.String("action", msg.Action), + zap.Int("recipient_count", len(clients))) + return h.sendToClients(data, clients, msg.Action) +} + // SubscribeToTask subscribes a client to task notifications func (h *Hub) SubscribeToTask(client *Client, taskID string) { h.mu.Lock() diff --git a/apps/backend/internal/gateway/websocket/hub_broadcast_test.go b/apps/backend/internal/gateway/websocket/hub_broadcast_test.go index 35f53e7dc2..96b42f7ffe 100644 --- a/apps/backend/internal/gateway/websocket/hub_broadcast_test.go +++ b/apps/backend/internal/gateway/websocket/hub_broadcast_test.go @@ -3,6 +3,7 @@ package websocket import ( "testing" + "github.com/kandev/kandev/internal/auth/authn" ws "github.com/kandev/kandev/pkg/websocket" ) @@ -169,3 +170,30 @@ func TestBroadcastToUserReportsFalseWhenSubscriberIsClosed(t *testing.T) { t.Fatal("broadcast to a closed subscriber must report no queued recipient") } } + +func TestSendToIdentityTargetsConnectedClientsWithoutSubscription(t *testing.T) { + h := newTestHub(t) + first := newTestClient("first") + first.identity = authn.Identity{UserID: "user-1"} + second := newTestClient("second") + second.identity = authn.Identity{UserID: "user-1"} + other := newTestClient("other") + other.identity = authn.Identity{UserID: "user-2"} + registerTestClient(h, first) + registerTestClient(h, second) + registerTestClient(h, other) + + msg, err := ws.NewNotification("system.logs.capture_requested", map[string]any{"bundle_id": "bundle-1"}) + if err != nil { + t.Fatalf("notification: %v", err) + } + if got := h.SendToIdentity("user-1", msg); got != 2 { + t.Fatalf("queued recipients = %d, want 2", got) + } + if !clientReceived(first) || !clientReceived(second) { + t.Fatal("both matching identity clients must receive the notification") + } + if clientReceived(other) { + t.Fatal("a different identity must not receive the notification") + } +} diff --git a/apps/backend/internal/improvekandev/bundle.go b/apps/backend/internal/improvekandev/bundle.go index f691c7bd15..f6d042fef0 100644 --- a/apps/backend/internal/improvekandev/bundle.go +++ b/apps/backend/internal/improvekandev/bundle.go @@ -1,6 +1,6 @@ // Package improvekandev exposes the HTTP endpoints that bootstrap a hidden -// improve-kandev workflow and capture recent backend/frontend logs into a -// temporary bundle directory referenced by the resulting task. +// improve-kandev workflow and lease an identity-owned diagnostic ZIP into the +// temporary context directory referenced by the resulting task. package improvekandev import ( @@ -9,140 +9,49 @@ import ( "fmt" "os" "path/filepath" - "runtime" - "sort" "strings" "time" - - "github.com/kandev/kandev/internal/common/logger/buffer" ) -// bundlePrefix is the temp-dir prefix used by os.MkdirTemp. -// It is also the marker validateBundleDir uses to refuse arbitrary writes. -const bundlePrefix = "kandev-improve-" +const ( + bundlePrefix = "kandev-improve-" + ownerMarkerName = ".kandev-owner.json" + diagnosticFileName = "diagnostic-bundle.zip" + staleBundleAge = 24 * time.Hour +) -// FrontendLogEntry mirrors apps/web/lib/logger/buffer.ts shape. -type FrontendLogEntry struct { - Timestamp string `json:"timestamp"` - Level string `json:"level"` - Message string `json:"message"` - Args []any `json:"args,omitempty"` - Stack string `json:"stack,omitempty"` - Metadata map[string]any `json:"metadata,omitempty"` +type ownerMarker struct { + Owner string `json:"owner"` } -// metadata is the JSON payload written to /metadata.json. -type metadata struct { - Version string `json:"version"` - OS string `json:"os"` - Arch string `json:"arch"` - GoVersion string `json:"go_version"` - CapturedAt time.Time `json:"captured_at"` - Health map[string]any `json:"health,omitempty"` -} - -// createBundleDir creates a fresh temp directory matching bundlePrefix. -func createBundleDir() (string, error) { - return os.MkdirTemp("", bundlePrefix+"*") -} - -// writeMetadata writes metadata.json into the bundle directory. -// health may be nil if no health snapshot is available. -func writeMetadata(dir, version string, health map[string]any) error { - m := metadata{ - Version: version, - OS: runtime.GOOS, - Arch: runtime.GOARCH, - GoVersion: runtime.Version(), - CapturedAt: time.Now().UTC(), - Health: health, +// createBundleDir creates an owner-only temporary context directory and a +// server-owned marker used to authorize the later diagnostic-bundle lease. +func createBundleDir(owner string) (string, error) { + if owner == "" { + return "", errors.New("owner is required") } - data, err := json.MarshalIndent(m, "", " ") + dir, err := os.MkdirTemp("", bundlePrefix+"*") if err != nil { - return fmt.Errorf("marshal metadata: %w", err) + return "", err } - return os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0o644) + if err := os.Chmod(dir, 0o700); err != nil { + _ = os.RemoveAll(dir) + return "", err + } + data, err := json.Marshal(ownerMarker{Owner: owner}) + if err != nil { + _ = os.RemoveAll(dir) + return "", err + } + if err := os.WriteFile(filepath.Join(dir, ownerMarkerName), data, 0o600); err != nil { + _ = os.RemoveAll(dir) + return "", err + } + return dir, nil } -// writeBackendLog renders the buffer snapshot as plain text and writes it. -func writeBackendLog(dir string, entries []buffer.Entry) error { - var b strings.Builder - for _, e := range entries { - b.WriteString(formatBackendEntry(e)) - b.WriteByte('\n') - } - return os.WriteFile(filepath.Join(dir, "backend.log"), []byte(b.String()), 0o644) -} - -// writeFrontendLog renders frontend entries as plain text and writes them. -func writeFrontendLog(dir string, entries []FrontendLogEntry) error { - var b strings.Builder - for _, e := range entries { - b.WriteString(formatFrontendEntry(e)) - b.WriteByte('\n') - } - return os.WriteFile(filepath.Join(dir, "frontend.log"), []byte(b.String()), 0o644) -} - -// formatBackendEntry renders a single buffer.Entry as a single line of text. -func formatBackendEntry(e buffer.Entry) string { - ts := e.Timestamp.UTC().Format(time.RFC3339Nano) - level := strings.ToUpper(e.Level) - var b strings.Builder - fmt.Fprintf(&b, "%s\t%s\t%s", ts, level, e.Message) - if e.Caller != "" { - fmt.Fprintf(&b, "\t%s", e.Caller) - } - if len(e.Fields) > 0 { - fmt.Fprintf(&b, "\t%s", flattenFields(e.Fields)) - } - if e.Stack != "" { - fmt.Fprintf(&b, "\n%s", e.Stack) - } - return b.String() -} - -// formatFrontendEntry mirrors formatBackendEntry for frontend entries. -func formatFrontendEntry(e FrontendLogEntry) string { - level := strings.ToUpper(e.Level) - var b strings.Builder - fmt.Fprintf(&b, "%s\t%s\t%s", e.Timestamp, level, e.Message) - if len(e.Args) > 0 { - fmt.Fprintf(&b, "\targs=%v", e.Args) - } - if e.Stack != "" { - fmt.Fprintf(&b, "\n%s", e.Stack) - } - return b.String() -} - -// flattenFields renders structured fields as space-separated key=value pairs in -// alphabetical order so log lines are deterministic across runs. -func flattenFields(fields map[string]any) string { - keys := make([]string, 0, len(fields)) - for k := range fields { - keys = append(keys, k) - } - sort.Strings(keys) - var b strings.Builder - for i, k := range keys { - if i > 0 { - b.WriteByte(' ') - } - fmt.Fprintf(&b, "%s=%v", k, fields[k]) - } - return b.String() -} - -// staleBundleAge is the minimum age before a leftover bundle dir is removed -// by CleanupStaleBundles. Recent dirs are preserved in case a task is still -// being created and references them. -const staleBundleAge = 24 * time.Hour - -// CleanupStaleBundles removes kandev-improve-* directories under the OS temp -// dir that are older than staleBundleAge. Errors are logged via onError but -// do not abort the sweep. Safe to call concurrently with active bundles — -// fresh dirs are skipped by the mtime check. +// CleanupStaleBundles removes old server-owned improve-kandev context +// directories. Fresh directories are preserved for in-flight task creation. func CleanupStaleBundles(onError func(path string, err error)) { cleanupStaleBundlesIn(os.TempDir(), staleBundleAge, onError) } @@ -177,14 +86,16 @@ func cleanupStaleBundlesIn(tempDir string, maxAge time.Duration, onError func(pa } } -// validateBundleDir refuses paths that do not match the kandev-improve-* temp -// pattern under the OS temp dir. Returns the cleaned absolute path on success. -func validateBundleDir(dir string) (string, error) { +// validateBundleDir accepts only a server-created temp directory whose +// owner-only marker matches the authenticated caller. +func validateBundleDir(dir, owner string) (string, error) { if dir == "" { return "", errors.New("bundle_dir is required") } - clean := filepath.Clean(dir) - abs, err := filepath.Abs(clean) + if owner == "" { + return "", errors.New("authenticated owner is required") + } + abs, err := filepath.Abs(filepath.Clean(dir)) if err != nil { return "", fmt.Errorf("invalid bundle_dir: %w", err) } @@ -194,7 +105,7 @@ func validateBundleDir(dir string) (string, error) { } resolved, err := filepath.EvalSymlinks(abs) if err != nil { - resolved = abs + return "", errors.New("bundle_dir does not exist") } if !strings.HasPrefix(resolved, tempDir+string(os.PathSeparator)) { return "", errors.New("bundle_dir must be inside the OS temp directory") @@ -206,5 +117,15 @@ func validateBundleDir(dir string) (string, error) { if err != nil || !info.IsDir() { return "", errors.New("bundle_dir does not exist or is not a directory") } + markerPath := filepath.Join(resolved, ownerMarkerName) + markerInfo, err := os.Lstat(markerPath) + if err != nil || !markerInfo.Mode().IsRegular() { + return "", errors.New("bundle_dir owner marker is missing") + } + var marker ownerMarker + data, err := os.ReadFile(markerPath) + if err != nil || json.Unmarshal(data, &marker) != nil || marker.Owner != owner { + return "", errors.New("bundle_dir is not owned by the authenticated user") + } return resolved, nil } diff --git a/apps/backend/internal/improvekandev/bundle_test.go b/apps/backend/internal/improvekandev/bundle_test.go index bbd6cd010f..fee617ec4e 100644 --- a/apps/backend/internal/improvekandev/bundle_test.go +++ b/apps/backend/internal/improvekandev/bundle_test.go @@ -6,12 +6,10 @@ import ( "strings" "testing" "time" - - "github.com/kandev/kandev/internal/common/logger/buffer" ) -func TestCreateBundleDirAndWriteFiles(t *testing.T) { - dir, err := createBundleDir() +func TestCreateBundleDirWritesOwnerMarker(t *testing.T) { + dir, err := createBundleDir("user-1") if err != nil { t.Fatalf("createBundleDir: %v", err) } @@ -21,61 +19,29 @@ func TestCreateBundleDirAndWriteFiles(t *testing.T) { t.Errorf("bundle dir base %q must start with %q", filepath.Base(dir), bundlePrefix) } - if err := writeMetadata(dir, "test", map[string]any{"github_auth": "ok"}); err != nil { - t.Fatalf("writeMetadata: %v", err) - } - data, err := os.ReadFile(filepath.Join(dir, "metadata.json")) + info, err := os.Stat(filepath.Join(dir, ownerMarkerName)) if err != nil { - t.Fatalf("read metadata.json: %v", err) + t.Fatalf("owner marker: %v", err) } - if !strings.Contains(string(data), "\"version\": \"test\"") { - t.Errorf("metadata.json missing version: %s", data) + if info.Mode().Perm() != 0o600 { + t.Fatalf("owner marker mode = %o, want 600", info.Mode().Perm()) } - if !strings.Contains(string(data), "\"github_auth\": \"ok\"") { - t.Errorf("metadata.json missing health: %s", data) + if _, err := validateBundleDir(dir, "user-1"); err != nil { + t.Fatalf("validate owner: %v", err) } - - entries := []buffer.Entry{{ - Timestamp: time.Date(2026, 4, 29, 10, 0, 0, 0, time.UTC), - Level: "info", - Message: "hello world", - Fields: map[string]any{"foo": "bar", "n": 42}, - }} - if err := writeBackendLog(dir, entries); err != nil { - t.Fatalf("writeBackendLog: %v", err) - } - beLog, _ := os.ReadFile(filepath.Join(dir, "backend.log")) - got := string(beLog) - if !strings.Contains(got, "INFO") || !strings.Contains(got, "hello world") { - t.Errorf("backend.log missing expected content: %q", got) - } - if !strings.Contains(got, "foo=bar") || !strings.Contains(got, "n=42") { - t.Errorf("backend.log missing fields: %q", got) - } - - feEntries := []FrontendLogEntry{{ - Timestamp: "2026-04-29T10:00:00.000Z", - Level: "error", - Message: "boom", - Stack: "Error\n at foo", - }} - if err := writeFrontendLog(dir, feEntries); err != nil { - t.Fatalf("writeFrontendLog: %v", err) - } - feLog, _ := os.ReadFile(filepath.Join(dir, "frontend.log")) - if !strings.Contains(string(feLog), "ERROR") || !strings.Contains(string(feLog), "boom") { - t.Errorf("frontend.log missing expected content: %q", feLog) + if _, err := validateBundleDir(dir, "user-2"); err == nil { + t.Fatal("different owner unexpectedly accepted") } } func TestValidateBundleDir_AcceptsValid(t *testing.T) { - dir, err := createBundleDir() + dir, err := createBundleDir("user-1") if err != nil { t.Fatalf("createBundleDir: %v", err) } defer func() { _ = os.RemoveAll(dir) }() - resolved, err := validateBundleDir(dir) + resolved, err := validateBundleDir(dir, "user-1") if err != nil { t.Errorf("validateBundleDir(%q): %v", dir, err) } @@ -96,7 +62,7 @@ func TestValidateBundleDir_RejectsBad(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if _, err := validateBundleDir(tc.dir); err == nil { + if _, err := validateBundleDir(tc.dir, "user-1"); err == nil { t.Errorf("expected error for %q", tc.dir) } }) @@ -140,22 +106,3 @@ func TestCleanupStaleBundles_RemovesOnlyStale(t *testing.T) { t.Errorf("non-bundle dir %q should have been preserved: %v", other, err) } } - -func TestFormatBackendEntry_DeterministicFieldOrder(t *testing.T) { - e := buffer.Entry{ - Timestamp: time.Date(2026, 4, 29, 10, 0, 0, 0, time.UTC), - Level: "warn", - Message: "msg", - Fields: map[string]any{"b": 2, "a": 1, "c": 3}, - } - got := formatBackendEntry(e) - ai := strings.Index(got, "a=1") - bi := strings.Index(got, "b=2") - ci := strings.Index(got, "c=3") - if ai < 0 || bi < 0 || ci < 0 { - t.Fatalf("missing one of a=1/b=2/c=3 in %q", got) - } - if ai >= bi || bi >= ci { - t.Errorf("fields not alphabetically ordered: %q", got) - } -} diff --git a/apps/backend/internal/improvekandev/handler.go b/apps/backend/internal/improvekandev/handler.go index 7b461f33c2..eb7ddc4bb2 100644 --- a/apps/backend/internal/improvekandev/handler.go +++ b/apps/backend/internal/improvekandev/handler.go @@ -2,15 +2,20 @@ package improvekandev import ( "context" + "errors" + "io" "net/http" + "os" "path/filepath" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" "go.uber.org/zap" + "github.com/kandev/kandev/internal/auth/authn" "github.com/kandev/kandev/internal/common/logger" - "github.com/kandev/kandev/internal/common/logger/buffer" + "github.com/kandev/kandev/internal/system/logbundle" taskmodels "github.com/kandev/kandev/internal/task/models" taskservice "github.com/kandev/kandev/internal/task/service" ) @@ -48,13 +53,10 @@ type Cloner interface { // Handler exposes the improve-kandev HTTP endpoints. type Handler struct { - taskSvc *taskservice.Service - cloner Cloner - log *logger.Logger - version string - // snapshot returns the current backend log buffer; defaults to logger's - // process-wide buffer but can be overridden for tests. - snapshot func() []buffer.Entry + taskSvc *taskservice.Service + cloner Cloner + log *logger.Logger + logBundles diagnosticBundleService // gh resolves the authenticated user's login and write access. Defaults // to a gh-CLI shell-out; tests can substitute a fake. gh GitHubInfo @@ -63,27 +65,34 @@ type Handler struct { resolveRemote remoteResolver } -// NewHandler constructs a Handler. version is embedded into bundle metadata. +type diagnosticBundleService interface { + OpenArchive(owner, id string) (*os.File, logbundle.JobView, error) +} + +// NewHandler constructs a Handler. func NewHandler(taskSvc *taskservice.Service, cloner Cloner, version string, log *logger.Logger) *Handler { + _ = version return &Handler{ taskSvc: taskSvc, cloner: cloner, log: log, - version: version, - snapshot: func() []buffer.Entry { return buffer.Default().Snapshot() }, gh: newDefaultGitHubInfo(), resolveRemote: taskservice.ResolveGitRemoteProvider, } } -// RegisterRoutes registers the bootstrap and frontend-log endpoints. +func (h *Handler) SetLogBundles(service diagnosticBundleService) { + h.logBundles = service +} + +// RegisterRoutes registers the bootstrap and diagnostic-bundle lease endpoints. func RegisterRoutes(router *gin.Engine, h *Handler) { if h == nil { return } api := router.Group("/api/v1/system/improve-kandev") api.POST("/bootstrap", h.httpBootstrap) - api.POST("/bundle/frontend-log", h.httpFrontendLog) + api.POST("/bundle/lease", h.httpLeaseBundle) } // BootstrapRequest is the JSON body for POST /bootstrap. @@ -115,19 +124,24 @@ const ( // BootstrapResponse describes the artifacts the dialog needs to submit a task. type BootstrapResponse struct { - RepositoryID string `json:"repository_id"` - WorkflowID string `json:"workflow_id"` - IssueWorkflowID string `json:"issue_workflow_id"` - Branch string `json:"branch"` - BundleDir string `json:"bundle_dir"` - BundleFiles map[string]string `json:"bundle_files"` - GitHubLogin string `json:"github_login"` - HasWriteAccess bool `json:"has_write_access"` - ForkStatus ForkStatus `json:"fork_status"` - ForkMessage string `json:"fork_message,omitempty"` + RepositoryID string `json:"repository_id"` + WorkflowID string `json:"workflow_id"` + IssueWorkflowID string `json:"issue_workflow_id"` + Branch string `json:"branch"` + BundleDir string `json:"bundle_dir"` + BundleFile string `json:"bundle_file"` + GitHubLogin string `json:"github_login"` + HasWriteAccess bool `json:"has_write_access"` + ForkStatus ForkStatus `json:"fork_status"` + ForkMessage string `json:"fork_message,omitempty"` } func (h *Handler) httpBootstrap(c *gin.Context) { + identity, ok := authn.FromGin(c) + if !ok || identity.UserID == "" { + c.JSON(http.StatusUnauthorized, gin.H{"error": "authentication required"}) + return + } var req BootstrapRequest if err := c.ShouldBindJSON(&req); err != nil || req.WorkspaceID == "" { c.JSON(http.StatusBadRequest, gin.H{"error": "workspace_id is required"}) @@ -180,23 +194,12 @@ func (h *Handler) httpBootstrap(c *gin.Context) { return } - dir, err := createBundleDir() + dir, err := createBundleDir(identity.UserID) if err != nil { h.log.Error("improve-kandev: bundle dir creation failed", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to create bundle dir"}) return } - if err := writeMetadata(dir, h.version, nil); err != nil { - h.log.Error("improve-kandev: metadata write failed", zap.Error(err)) - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write metadata"}) - return - } - if err := writeBackendLog(dir, h.snapshot()); err != nil { - h.log.Error("improve-kandev: backend log write failed", zap.Error(err)) - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write backend log"}) - return - } - access := h.resolveGitHubAccess(ctx) c.JSON(http.StatusOK, BootstrapResponse{ @@ -205,15 +208,11 @@ func (h *Handler) httpBootstrap(c *gin.Context) { IssueWorkflowID: issueWorkflow.ID, Branch: defaultBranch, BundleDir: dir, - BundleFiles: map[string]string{ - "metadata": filepath.Join(dir, "metadata.json"), - "backend_log": filepath.Join(dir, "backend.log"), - "frontend_log": filepath.Join(dir, "frontend.log"), - }, - GitHubLogin: access.login, - HasWriteAccess: access.hasWrite, - ForkStatus: access.forkStatus, - ForkMessage: access.forkMessage, + BundleFile: filepath.Join(dir, diagnosticFileName), + GitHubLogin: access.login, + HasWriteAccess: access.hasWrite, + ForkStatus: access.forkStatus, + ForkMessage: access.forkMessage, }) } @@ -439,27 +438,83 @@ func (h *Handler) findAndHealWorkflow( return nil } -// FrontendLogRequest is the JSON body for POST /bundle/frontend-log. -type FrontendLogRequest struct { - BundleDir string `json:"bundle_dir"` - Entries []FrontendLogEntry `json:"entries"` +const maxLeasedBundleBytes = int64(256 * 1024 * 1024) + +type leaseBundleRequest struct { + BundleDir string `json:"bundle_dir"` + BundleID string `json:"bundle_id"` } -func (h *Handler) httpFrontendLog(c *gin.Context) { - var req FrontendLogRequest - if err := c.ShouldBindJSON(&req); err != nil { +func (h *Handler) httpLeaseBundle(c *gin.Context) { + identity, ok := authn.FromGin(c) + if !ok || identity.UserID == "" { + c.JSON(http.StatusUnauthorized, gin.H{"error": "authentication required"}) + return + } + var req leaseBundleRequest + if err := c.ShouldBindJSON(&req); err != nil || req.BundleID == "" { c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload"}) return } - dir, err := validateBundleDir(req.BundleDir) + dir, err := validateBundleDir(req.BundleDir, identity.UserID) if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) return } - if err := writeFrontendLog(dir, req.Entries); err != nil { - h.log.Error("improve-kandev: frontend log write failed", zap.Error(err)) - c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to write frontend log"}) + if h.logBundles == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "diagnostic bundles unavailable"}) return } - c.JSON(http.StatusOK, gin.H{"path": filepath.Join(dir, "frontend.log")}) + archive, view, err := h.logBundles.OpenArchive(identity.UserID, req.BundleID) + if err != nil { + c.JSON(http.StatusConflict, gin.H{"error": "diagnostic bundle is not ready"}) + return + } + defer func() { _ = archive.Close() }() + + path := filepath.Join(dir, diagnosticFileName) + if err := leaseArchive(archive, path); err != nil { + h.log.Error("improve-kandev: diagnostic bundle lease failed", zap.Error(err)) + status := http.StatusInternalServerError + if errors.Is(err, errBundleTooLarge) { + status = http.StatusRequestEntityTooLarge + } + c.JSON(status, gin.H{"error": "failed to lease diagnostic bundle"}) + return + } + time.AfterFunc(staleBundleAge, func() { _ = os.Remove(path) }) + c.JSON(http.StatusOK, gin.H{ + "path": path, "status": view.Status, "sources": view.Sources, + }) +} + +var errBundleTooLarge = errors.New("diagnostic bundle exceeds lease limit") + +func leaseArchive(source io.Reader, path string) error { + temp, err := os.CreateTemp(filepath.Dir(path), ".diagnostic-bundle-*.tmp") + if err != nil { + return err + } + tempPath := temp.Name() + defer func() { + _ = temp.Close() + _ = os.Remove(tempPath) + }() + if err := temp.Chmod(0o600); err != nil { + return err + } + written, err := io.Copy(temp, io.LimitReader(source, maxLeasedBundleBytes+1)) + if err != nil { + return err + } + if written > maxLeasedBundleBytes { + return errBundleTooLarge + } + if err := temp.Sync(); err != nil { + return err + } + if err := temp.Close(); err != nil { + return err + } + return os.Rename(tempPath, path) } diff --git a/apps/backend/internal/improvekandev/handler_test.go b/apps/backend/internal/improvekandev/handler_test.go index 8fe4b18f03..01862783dc 100644 --- a/apps/backend/internal/improvekandev/handler_test.go +++ b/apps/backend/internal/improvekandev/handler_test.go @@ -1,11 +1,20 @@ package improvekandev import ( + "bytes" "context" + "encoding/json" "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" "testing" + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/auth/authn" "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/system/logbundle" taskmodels "github.com/kandev/kandev/internal/task/models" ) @@ -53,6 +62,65 @@ func TestCanonicalWorkspaceID(t *testing.T) { } } +type fakeDiagnosticBundles struct { + path string + owner string + id string +} + +func (f *fakeDiagnosticBundles) OpenArchive(owner, id string) (*os.File, logbundle.JobView, error) { + f.owner, f.id = owner, id + file, err := os.Open(f.path) + return file, logbundle.JobView{ID: id, Status: logbundle.StatusReady, Sources: []string{"backend", "frontend"}}, err +} + +func TestLeaseBundleChecksMarkerOwnerAndCopiesArchive(t *testing.T) { + gin.SetMode(gin.TestMode) + dir, err := createBundleDir("user-1") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + source := filepath.Join(t.TempDir(), "source.zip") + if err := os.WriteFile(source, []byte("zip bytes"), 0o600); err != nil { + t.Fatal(err) + } + bundles := &fakeDiagnosticBundles{path: source} + handler := &Handler{log: logger.Default(), logBundles: bundles} + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "user-1"}) + c.Next() + }) + router.POST("/lease", handler.httpLeaseBundle) + + body, _ := json.Marshal(leaseBundleRequest{BundleDir: dir, BundleID: "bundle-1"}) + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/lease", bytes.NewReader(body))) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", recorder.Code, recorder.Body.String()) + } + if bundles.owner != "user-1" || bundles.id != "bundle-1" { + t.Fatalf("OpenArchive identity = %q/%q", bundles.owner, bundles.id) + } + data, err := os.ReadFile(filepath.Join(dir, diagnosticFileName)) + if err != nil || string(data) != "zip bytes" { + t.Fatalf("leased archive = %q, err=%v", data, err) + } + + otherDir, err := createBundleDir("user-2") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.RemoveAll(otherDir) }) + body, _ = json.Marshal(leaseBundleRequest{BundleDir: otherDir, BundleID: "bundle-1"}) + recorder = httptest.NewRecorder() + router.ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/lease", bytes.NewReader(body))) + if recorder.Code != http.StatusForbidden { + t.Fatalf("other owner status = %d, want 403", recorder.Code) + } +} + func TestResolveGitHubAccess_Writable(t *testing.T) { gh := &fakeGitHubInfo{login: "alice", hasWrite: true} access := newTestHandler(gh).resolveGitHubAccess(context.Background()) diff --git a/apps/backend/internal/integration/improve_kandev_test.go b/apps/backend/internal/integration/improve_kandev_test.go index 9673ff49cc..13e189d1c3 100644 --- a/apps/backend/internal/integration/improve_kandev_test.go +++ b/apps/backend/internal/integration/improve_kandev_test.go @@ -12,6 +12,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" + "github.com/kandev/kandev/internal/auth/authn" "github.com/kandev/kandev/internal/improvekandev" "github.com/kandev/kandev/internal/task/models" ) @@ -37,6 +38,10 @@ func TestImproveKandevBootstrapCreatesBothHiddenWorkflowsIdempotently(t *testing repoPath := t.TempDir() require.NoError(t, exec.Command("git", "init", repoPath).Run()) router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "test-user"}) + c.Next() + }) handler := improvekandev.NewHandler(ts.TaskSvc, improveKandevTestCloner{path: repoPath}, "test", ts.Logger) improvekandev.RegisterRoutes(router, handler) diff --git a/apps/backend/internal/launcher/cli/help.go b/apps/backend/internal/launcher/cli/help.go index 79271cd55e..67d8e4928b 100644 --- a/apps/backend/internal/launcher/cli/help.go +++ b/apps/backend/internal/launcher/cli/help.go @@ -14,8 +14,8 @@ Options: service Manage kandev as an OS service. --version, -V Print CLI version and exit. --port Port for the Go backend. Alias for --backend-port. - --verbose, -v Show info logs from backend. - --debug Show debug logs + agent message dumps. + --verbose, -v Show info logs on stdout (also retained in the backend file). + --debug Retain debug logs in the backend file + agent message dumps. --headless Skip opening the browser. Used by service units. --help, -h Show help. diff --git a/apps/backend/internal/launcher/cli/help_test.go b/apps/backend/internal/launcher/cli/help_test.go index 48cea1b9f7..681ccbfa1f 100644 --- a/apps/backend/internal/launcher/cli/help_test.go +++ b/apps/backend/internal/launcher/cli/help_test.go @@ -16,4 +16,7 @@ func TestHelpDoesNotExposeHiddenBackendMode(t *testing.T) { if strings.Contains(help, "__backend") { t.Fatalf("help exposes hidden backend mode:\n%s", help) } + if !strings.Contains(help, "info logs on stdout") || !strings.Contains(help, "debug logs in the backend file") { + t.Fatalf("help does not explain file/stdout diagnostics:\n%s", help) + } } diff --git a/apps/backend/internal/launcher/env.go b/apps/backend/internal/launcher/env.go index 7853005394..f0f1213bf5 100644 --- a/apps/backend/internal/launcher/env.go +++ b/apps/backend/internal/launcher/env.go @@ -5,7 +5,7 @@ import ( "os" ) -func backendEnv(ports portConfig, logLevel string, debug bool) []string { +func backendEnv(ports portConfig, logLevel, consoleLogLevel string, debug bool) []string { env := os.Environ() env = upsertEnv(env, "KANDEV_SERVER_PORT", fmt.Sprint(ports.BackendPort)) env = upsertEnv(env, "KANDEV_AGENT_STANDALONE_PORT", fmt.Sprint(ports.AgentctlPort)) @@ -13,6 +13,7 @@ func backendEnv(ports portConfig, logLevel string, debug bool) []string { if logLevel != "" { env = upsertEnv(env, "KANDEV_LOG_LEVEL", logLevel) } + env = upsertEnv(env, "KANDEV_CONSOLE_LOG_LEVEL", consoleLogLevel) if debug { env = upsertEnv(env, "KANDEV_DEBUG_AGENT_MESSAGES", "true") env = upsertEnv(env, "KANDEV_DEBUG_PPROF_ENABLED", "true") diff --git a/apps/backend/internal/launcher/start.go b/apps/backend/internal/launcher/start.go index afa68462b2..8995903b5d 100644 --- a/apps/backend/internal/launcher/start.go +++ b/apps/backend/internal/launcher/start.go @@ -71,13 +71,18 @@ func resolveLogLevel(opts Options) string { switch { case opts.Debug: return "debug" - case opts.Verbose: - return "info" default: - return "warn" + return "info" } } +func resolveConsoleLogLevel(opts Options) string { + if opts.Verbose { + return "info" + } + return "warn" +} + func runManagedApp(ctx context.Context, cfg managedAppConfig) int { ignoreBrokenPipeSignal() logStartup(cfg.Header, cfg.Ports, resolveDatabasePath(), cfg.LogLevel) @@ -87,8 +92,10 @@ func runManagedApp(ctx context.Context, cfg managedAppConfig) int { supervisor := newSupervisorFn() attachSignalsFn(supervisor) shutdownDebugf("runManagedApp signal handler attached") - showOutput := cfg.Opts.Verbose || cfg.Opts.Debug - backend, dumpLogs, err := launchBackendFn(cfg.Backend, []string{"__backend"}, cfg.BackendCWD, backendEnv(cfg.Ports, cfg.LogLevel, cfg.Opts.Debug), !showOutput, cfg.Ports, cfg.Mode, supervisor) + env := backendEnv(cfg.Ports, cfg.LogLevel, resolveConsoleLogLevel(cfg.Opts), cfg.Opts.Debug) + backend, dumpLogs, err := launchBackendFn( + cfg.Backend, []string{"__backend"}, cfg.BackendCWD, env, false, cfg.Ports, cfg.Mode, supervisor, + ) if err != nil { fmt.Fprintln(os.Stderr, "[kandev] "+err.Error()) return 1 diff --git a/apps/backend/internal/launcher/start_test.go b/apps/backend/internal/launcher/start_test.go index 443e807e59..024cb284d3 100644 --- a/apps/backend/internal/launcher/start_test.go +++ b/apps/backend/internal/launcher/start_test.go @@ -2,12 +2,59 @@ package launcher import ( "context" + "os" "path/filepath" "reflect" + "strings" "testing" "time" ) +func TestResolveLogThresholdsByMode(t *testing.T) { + t.Setenv("KANDEV_LOG_LEVEL", "") + tests := []struct { + name string + options Options + wantFile string + wantConsole string + }{ + {name: "normal", wantFile: "info", wantConsole: "warn"}, + {name: "debug", options: Options{Debug: true}, wantFile: "debug", wantConsole: "warn"}, + {name: "verbose", options: Options{Verbose: true}, wantFile: "info", wantConsole: "info"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := resolveLogLevel(test.options); got != test.wantFile { + t.Fatalf("file level = %q, want %q", got, test.wantFile) + } + if got := resolveConsoleLogLevel(test.options); got != test.wantConsole { + t.Fatalf("console level = %q, want %q", got, test.wantConsole) + } + }) + } + t.Setenv("KANDEV_LOG_LEVEL", "error") + if got := resolveLogLevel(Options{Debug: true}); got != "error" { + t.Fatalf("explicit file override = %q", got) + } + if os.Getenv("KANDEV_LOG_LEVEL") != "error" { + t.Fatal("test environment changed unexpectedly") + } +} + +func TestBackendEnvCarriesIndependentThresholds(t *testing.T) { + env := backendEnv(portConfig{BackendPort: 1234, AgentctlPort: 5678}, "debug", "warn", true) + joined := strings.Join(env, "\n") + for _, expected := range []string{ + "KANDEV_LOG_LEVEL=debug", + "KANDEV_CONSOLE_LOG_LEVEL=warn", + "KANDEV_DEBUG_AGENT_MESSAGES=true", + } { + if !strings.Contains(joined, expected) { + t.Fatalf("backend environment missing %q", expected) + } + } +} + func TestRunStartUsesSelfExecutableAndBackendCWD(t *testing.T) { oldExecutablePath := executablePath oldLaunchManaged := launchManaged @@ -62,6 +109,7 @@ func TestRunManagedAppAttachesSignalsBeforeBackendLaunch(t *testing.T) { }) var events []string + var launchedQuiet bool newSupervisorFn = func() *processSupervisor { events = append(events, "new-supervisor") return newSupervisor() @@ -71,11 +119,12 @@ func TestRunManagedAppAttachesSignalsBeforeBackendLaunch(t *testing.T) { _ []string, _ string, _ []string, - _ bool, + quiet bool, _ portConfig, _ string, _ *processSupervisor, ) (*restartableBackend, func(), error) { + launchedQuiet = quiet events = append(events, "launch-backend") exitCh := make(chan int, 1) exitCh <- 0 @@ -108,4 +157,7 @@ func TestRunManagedAppAttachesSignalsBeforeBackendLaunch(t *testing.T) { if !reflect.DeepEqual(events, want) { t.Fatalf("events = %v, want %v", events, want) } + if launchedQuiet { + t.Fatal("backend stdout was not forwarded") + } } diff --git a/apps/backend/internal/launcher/supervisor.go b/apps/backend/internal/launcher/supervisor.go index 6507cef801..603454b736 100644 --- a/apps/backend/internal/launcher/supervisor.go +++ b/apps/backend/internal/launcher/supervisor.go @@ -174,6 +174,7 @@ func allowedSupervisorEnv(env []string) map[string]string { "KANDEV_WEB_INTERNAL_URL": true, "KANDEV_AGENT_STANDALONE_PORT": true, "KANDEV_LOG_LEVEL": true, + "KANDEV_CONSOLE_LOG_LEVEL": true, "KANDEV_DEBUG_DEV_MODE": true, "KANDEV_DEBUG_AGENT_MESSAGES": true, "KANDEV_DEBUG_PPROF_ENABLED": true, diff --git a/apps/backend/internal/launcher/supervisor_test.go b/apps/backend/internal/launcher/supervisor_test.go index e849c53940..4d88a37364 100644 --- a/apps/backend/internal/launcher/supervisor_test.go +++ b/apps/backend/internal/launcher/supervisor_test.go @@ -11,6 +11,7 @@ import ( func TestBuildManifestUsesSameBinaryBackendMode(t *testing.T) { env := []string{ "KANDEV_SERVER_PORT=1234", + "KANDEV_CONSOLE_LOG_LEVEL=warn", "UNRELATED=value", } manifest := buildManifest("/opt/kandev/bin/kandev", []string{"__backend"}, "/opt/kandev/bin", env, "/tmp/home", 1234, "run") @@ -27,6 +28,9 @@ func TestBuildManifestUsesSameBinaryBackendMode(t *testing.T) { if manifest.Env["KANDEV_SERVER_PORT"] != "1234" { t.Fatalf("KANDEV_SERVER_PORT = %q", manifest.Env["KANDEV_SERVER_PORT"]) } + if manifest.Env["KANDEV_CONSOLE_LOG_LEVEL"] != "warn" { + t.Fatalf("KANDEV_CONSOLE_LOG_LEVEL = %q", manifest.Env["KANDEV_CONSOLE_LOG_LEVEL"]) + } } func TestPrepareSupervisorEnvWritesExpectedPaths(t *testing.T) { diff --git a/apps/backend/internal/mcp/handlers/diagnostic_bundle.go b/apps/backend/internal/mcp/handlers/diagnostic_bundle.go new file mode 100644 index 0000000000..3b51195962 --- /dev/null +++ b/apps/backend/internal/mcp/handlers/diagnostic_bundle.go @@ -0,0 +1,176 @@ +package handlers + +import ( + "context" + "encoding/json" + "io" + "os" + "time" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/system/logbundle" + userstore "github.com/kandev/kandev/internal/user/store" + ws "github.com/kandev/kandev/pkg/websocket" +) + +type DiagnosticBundleProvider interface { + Create(owner string, sources []string) (logbundle.JobView, bool, error) + Get(owner, id string) (logbundle.JobView, error) + OpenArchive(owner, id string) (*os.File, logbundle.JobView, error) + ManifestSummary(owner, id string) (map[string]any, error) +} + +type DiagnosticBundleMaterializer interface { + MaterializeDiagnosticBundle( + ctx context.Context, + taskID, sessionID, bundleID string, + source io.Reader, + ) (lifecycle.DiagnosticMaterialization, error) +} + +type diagnosticBundleRequest struct { + Source string `json:"source"` + TaskID string `json:"task_id"` + SessionID string `json:"session_id"` +} + +func (h *Handlers) SetDiagnosticBundleServices( + provider DiagnosticBundleProvider, + materializer DiagnosticBundleMaterializer, +) { + h.diagnosticBundles = provider + h.diagnosticMaterializer = materializer +} + +func (h *Handlers) handleGetDiagnosticBundle( + ctx context.Context, + message *ws.Message, +) (*ws.Message, error) { + if h.diagnosticBundles == nil || h.diagnosticMaterializer == nil { + return ws.NewError(message.ID, message.Action, ws.ErrorCodeInternalError, + "Diagnostic bundles are unavailable", nil) + } + request, sources, requestErr := decodeDiagnosticRequest(message.Payload) + if requestErr != nil { + return ws.NewError(message.ID, message.Action, requestErr.code, requestErr.message, nil) + } + owner, authorizationErr := h.authorizeDiagnosticRequest(ctx, request) + if authorizationErr != nil { + return ws.NewError(message.ID, message.Action, + authorizationErr.code, authorizationErr.message, nil) + } + job, _, err := h.diagnosticBundles.Create(owner, sources) + if err != nil { + return ws.NewError(message.ID, message.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + job, err = h.waitForDiagnosticBundle(ctx, owner, job) + if err != nil { + return ws.NewError(message.ID, message.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + archive, _, err := h.diagnosticBundles.OpenArchive(owner, job.ID) + if err != nil { + return ws.NewError(message.ID, message.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + defer func() { _ = archive.Close() }() + manifest, err := h.diagnosticBundles.ManifestSummary(owner, job.ID) + if err != nil { + return ws.NewError(message.ID, message.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + materialized, err := h.diagnosticMaterializer.MaterializeDiagnosticBundle( + ctx, request.TaskID, request.SessionID, job.ID, archive, + ) + if err != nil { + return ws.NewError(message.ID, message.Action, ws.ErrorCodeInternalError, err.Error(), nil) + } + return ws.NewResponse(message.ID, message.Action, map[string]any{ + "path": materialized.Path, "bytes": materialized.Bytes, + "bundle_id": job.ID, "status": job.Status, "sources": job.Sources, + "warnings": job.Warnings, "manifest": manifest, + }) +} + +type diagnosticRequestError struct { + code string + message string +} + +func decodeDiagnosticRequest(payload json.RawMessage) ( + diagnosticBundleRequest, + []string, + *diagnosticRequestError, +) { + var request diagnosticBundleRequest + if err := json.Unmarshal(payload, &request); err != nil { + return request, nil, &diagnosticRequestError{ws.ErrorCodeBadRequest, "Invalid payload"} + } + sources, ok := diagnosticSources(request.Source) + if !ok || request.TaskID == "" || request.SessionID == "" { + return request, nil, &diagnosticRequestError{ + ws.ErrorCodeValidation, "source, task_id, and session_id are required", + } + } + return request, sources, nil +} + +func (h *Handlers) authorizeDiagnosticRequest( + ctx context.Context, + request diagnosticBundleRequest, +) (string, *diagnosticRequestError) { + task, err := h.taskSvc.GetTask(ctx, request.TaskID) + if err != nil || task == nil { + return "", &diagnosticRequestError{ws.ErrorCodeNotFound, "Task not found"} + } + session, err := h.sessionRepo.GetTaskSession(ctx, request.SessionID) + if err != nil || session == nil || session.TaskID != request.TaskID { + return "", &diagnosticRequestError{ws.ErrorCodeValidation, "Session does not belong to task"} + } + if identity, present := authn.IdentityFromContext(ctx); present && identity.UserID != "" { + return identity.UserID, nil + } + return userstore.DefaultUserID, nil +} + +func (h *Handlers) waitForDiagnosticBundle( + ctx context.Context, + owner string, + job logbundle.JobView, +) (logbundle.JobView, error) { + for job.Status == logbundle.StatusCollecting || job.Status == logbundle.StatusBuilding { + timer := time.NewTimer(250 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return logbundle.JobView{}, ctx.Err() + case <-timer.C: + } + var err error + job, err = h.diagnosticBundles.Get(owner, job.ID) + if err != nil { + return logbundle.JobView{}, err + } + } + if job.Status != logbundle.StatusReady && job.Status != logbundle.StatusPartial { + return logbundle.JobView{}, &diagnosticStateError{status: job.Status} + } + return job, nil +} + +type diagnosticStateError struct{ status logbundle.Status } + +func (e *diagnosticStateError) Error() string { + return "diagnostic bundle ended in state " + string(e.status) +} + +func diagnosticSources(source string) ([]string, bool) { + switch source { + case "backend": + return []string{"backend"}, true + case "frontend": + return []string{"frontend"}, true + case "all": + return []string{"backend", "frontend"}, true + default: + return nil, false + } +} diff --git a/apps/backend/internal/mcp/handlers/diagnostic_bundle_test.go b/apps/backend/internal/mcp/handlers/diagnostic_bundle_test.go new file mode 100644 index 0000000000..017d871627 --- /dev/null +++ b/apps/backend/internal/mcp/handlers/diagnostic_bundle_test.go @@ -0,0 +1,66 @@ +package handlers + +import ( + "context" + "io" + "os" + "path/filepath" + "testing" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/system/logbundle" + "github.com/kandev/kandev/internal/task/models" + ws "github.com/kandev/kandev/pkg/websocket" + "github.com/stretchr/testify/require" +) + +type diagnosticMaterializerFake struct { + taskID string + sessionID string + bytes int64 +} + +func (f *diagnosticMaterializerFake) MaterializeDiagnosticBundle( + _ context.Context, + taskID, sessionID, _ string, + source io.Reader, +) (lifecycle.DiagnosticMaterialization, error) { + f.taskID = taskID + f.sessionID = sessionID + count, err := io.Copy(io.Discard, source) + f.bytes = count + return lifecycle.DiagnosticMaterialization{Path: ".kandev/diagnostics/bundle.zip", Bytes: count}, err +} + +func TestGetDiagnosticBundleUsesImmutableTaskSessionAndMaterializesArchive(t *testing.T) { + taskService, repository := newTestTaskService(t) + _, task, session := seedTaskWithSession( + t, taskService, repository, models.TaskSessionStateWaitingForInput, + ) + home := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(home, "logs"), 0o700)) + require.NoError(t, os.WriteFile( + filepath.Join(home, "logs", "backend-logs.log"), []byte("task_id="+task.ID), 0o600, + )) + bundles := logbundle.New(logbundle.Config{HomeDir: home, Log: testLogger(t)}) + ctx, cancel := context.WithCancel(context.Background()) + bundles.Start(ctx) + t.Cleanup(func() { + cancel() + bundles.Stop() + }) + materializer := &diagnosticMaterializerFake{} + handler := &Handlers{ + taskSvc: taskService, sessionRepo: repository, logger: testLogger(t).WithFields(), + } + handler.SetDiagnosticBundleServices(bundles, materializer) + message := makeWSMessage(t, ws.ActionMCPGetDiagnosticBundle, map[string]interface{}{ + "source": "backend", "task_id": task.ID, "session_id": session.ID, + }) + response, err := handler.handleGetDiagnosticBundle(context.Background(), message) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type) + require.Equal(t, task.ID, materializer.taskID) + require.Equal(t, session.ID, materializer.sessionID) + require.Positive(t, materializer.bytes) +} diff --git a/apps/backend/internal/mcp/handlers/handlers.go b/apps/backend/internal/mcp/handlers/handlers.go index e066b1f2d8..d03e8047f0 100644 --- a/apps/backend/internal/mcp/handlers/handlers.go +++ b/apps/backend/internal/mcp/handlers/handlers.go @@ -219,7 +219,9 @@ type Handlers struct { reviewRunner ReviewRunner // Optional task-bound GitHub PR automation controls. - taskPRAutomation TaskPRAutomationService + taskPRAutomation TaskPRAutomationService + diagnosticBundles DiagnosticBundleProvider + diagnosticMaterializer DiagnosticBundleMaterializer } // NewHandlers creates new MCP handlers. @@ -330,6 +332,10 @@ func (h *Handlers) RegisterHandlers(d *ws.Dispatcher) { d.RegisterFunc(ws.ActionTaskWalkthroughDelete, h.handleDeleteWalkthrough) d.RegisterFunc(ws.ActionMCPClarificationTimeout, h.handleClarificationTimeout) count := 26 + if h.diagnosticBundles != nil && h.diagnosticMaterializer != nil { + d.RegisterFunc(ws.ActionMCPGetDiagnosticBundle, h.handleGetDiagnosticBundle) + count++ + } count += h.registerReviewHandlers(d) count += 2 // task PR automation get/update diff --git a/apps/backend/internal/mcp/server/handlers.go b/apps/backend/internal/mcp/server/handlers.go index bde3ac7ace..a149c1fbaa 100644 --- a/apps/backend/internal/mcp/server/handlers.go +++ b/apps/backend/internal/mcp/server/handlers.go @@ -51,6 +51,26 @@ func (s *Server) listWorkspacesHandler() server.ToolHandlerFunc { } } +func (s *Server) getDiagnosticBundleHandler() server.ToolHandlerFunc { + return func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + source, err := req.RequireString("source") + if err != nil || (source != "backend" && source != "frontend" && source != "all") { + return mcp.NewToolResultError("source must be backend, frontend, or all"), nil + } + payload := map[string]string{ + "source": source, "task_id": s.taskID, "session_id": s.sessionID, + } + var result map[string]interface{} + if err := s.backend.RequestPayload( + ctx, ws.ActionMCPGetDiagnosticBundle, payload, &result, + ); err != nil { + return mcp.NewToolResultError(err.Error()), nil + } + data, _ := json.MarshalIndent(result, "", " ") + return mcp.NewToolResultText(string(data)), nil + } +} + func (s *Server) listWorkflowsHandler() server.ToolHandlerFunc { return func(ctx context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { workspaceID, err := req.RequireString("workspace_id") diff --git a/apps/backend/internal/mcp/server/server.go b/apps/backend/internal/mcp/server/server.go index 6c711d0075..309b96c1f5 100644 --- a/apps/backend/internal/mcp/server/server.go +++ b/apps/backend/internal/mcp/server/server.go @@ -522,6 +522,8 @@ func (s *Server) registerTools() { s.registerSetTaskTitleTool() count++ } + s.registerDiagnosticBundleTool() + count++ } s.logger.Info("registered MCP tools", zap.String("mode", s.mode), @@ -530,6 +532,20 @@ func (s *Server) registerTools() { s.rebuildToolArgumentValidators() } +func (s *Server) registerDiagnosticBundleTool() { + s.mcpServer.AddTool( + mcp.NewTool("get_diagnostic_bundle_kandev", + mcp.WithDescription("Collect a bounded diagnostic ZIP for the current task session and materialize it inside this execution workspace. Request backend first for backend/runtime issues, frontend for browser issues, or all only when correlation requires both."), + mcp.WithString("source", + mcp.Required(), + mcp.Enum("backend", "frontend", "all"), + mcp.Description("Diagnostic source to collect: backend, frontend, or all"), + ), + ), + s.wrapHandler("get_diagnostic_bundle_kandev", s.getDiagnosticBundleHandler()), + ) +} + func (s *Server) registerKanbanTools() { // Use NewToolWithRawSchema for parameter-less tools to ensure the schema // includes "properties": {}. The default ToolInputSchema type in mcp-go uses diff --git a/apps/backend/internal/mcp/server/server_test.go b/apps/backend/internal/mcp/server/server_test.go index d819c229da..7263f21dca 100644 --- a/apps/backend/internal/mcp/server/server_test.go +++ b/apps/backend/internal/mcp/server/server_test.go @@ -109,6 +109,7 @@ func TestServerModeTask_RegistersCorrectTools(t *testing.T) { assert.Contains(t, tools, "get_task_conversation_kandev") assert.Contains(t, tools, "get_task_pr_automation_kandev") assert.Contains(t, tools, "update_task_pr_automation_kandev") + assert.Contains(t, tools, "get_diagnostic_bundle_kandev") // Task mode should have plan tools assert.Contains(t, tools, "create_task_plan_kandev") @@ -322,14 +323,14 @@ func TestServerModeTask_ToolCount(t *testing.T) { // 17 kanban (incl. delete + archive task + stop_task + spawn_session + PR automation) + // 1 add_branch_to_task + 1 add_workspace_sources + 1 update_repository_base_branch + // 1 step_complete (ADR 0015) + 1 interaction + 4 plan + 3 walkthrough + - // 1 publish_review_findings + 1 related-tasks = 31. + // 1 publish_review_findings + 1 related-tasks + 1 diagnostic bundle = 32. // Task-document tools (list/get/write) are office-only. assert.Contains(t, tools, "step_complete_kandev", "ADR 0015 explicit-completion signal must be registered in task mode") assert.Contains(t, tools, "show_walkthrough_kandev", "walkthrough tool must be registered in task mode") assert.Contains(t, tools, "publish_review_findings_kandev", "native code-review publishing must be registered in task mode") assert.Contains(t, tools, "spawn_session_kandev", "spawn_session must be registered in task mode") assert.Contains(t, tools, "add_workspace_sources_kandev") - assert.Equal(t, 31, len(tools)) + assert.Equal(t, 32, len(tools)) } func TestServerStepCompleteTool_TaskOnlyAndDiscoverable(t *testing.T) { diff --git a/apps/backend/internal/system/frontenderrors/handler.go b/apps/backend/internal/system/frontenderrors/handler.go new file mode 100644 index 0000000000..9400a8b246 --- /dev/null +++ b/apps/backend/internal/system/frontenderrors/handler.go @@ -0,0 +1,212 @@ +// Package frontenderrors accepts bounded, best-effort reports for error toasts +// already displayed by the browser. +package frontenderrors + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + "go.uber.org/zap" +) + +const ( + maxRequestBodyBytes = 64 * 1024 + titleByteLimit = 8 * 1024 + descriptionByteLimit = 8 * 1024 + urlByteLimit = 8 * 1024 + taskIDByteLimit = 128 + browserFieldByteLimit = 2 * 1024 + stackByteLimit = 16 * 1024 +) + +type Viewport struct { + Width int `json:"width"` + Height int `json:"height"` +} + +type ErrorDetails struct { + Name string `json:"name"` + Message string `json:"message"` + Stack string `json:"stack"` +} + +type Request struct { + ClientTimestamp string `json:"client_timestamp"` + Source string `json:"source"` + TaskID string `json:"task_id"` + Title string `json:"title"` + Description string `json:"description"` + URL string `json:"url"` + UserAgent string `json:"user_agent"` + Language string `json:"language"` + Platform string `json:"platform"` + Viewport *Viewport `json:"viewport"` + Stack string `json:"stack"` + Error *ErrorDetails `json:"error"` +} + +type normalizedRequest struct { + Request + Truncated bool +} + +type Service struct { + log *logger.Logger + limiter *limiter +} + +func New(log *logger.Logger, now func() time.Time) *Service { + return &Service{log: log, limiter: newLimiter(now)} +} + +func Handle(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + identity, ok := authn.FromGin(c) + if !ok || identity.UserID == "" { + c.JSON(http.StatusUnauthorized, gin.H{"error": "authentication required"}) + return + } + if allowed, retry := service.limiter.Allow(identity.UserID); !allowed { + c.Header("Retry-After", strconv.Itoa(max(1, int(retry.Seconds())))) + c.JSON(http.StatusTooManyRequests, gin.H{"error": "rate limit exceeded"}) + return + } + request, requestBytes, status, err := decodeRequest(c) + if err != nil { + c.JSON(status, gin.H{"error": err.Error()}) + return + } + if allowed, retry := service.limiter.AllowBytes(identity.UserID, requestBytes); !allowed { + c.Header("Retry-After", strconv.Itoa(max(1, int(retry.Seconds())))) + c.JSON(http.StatusTooManyRequests, gin.H{"error": "rate limit exceeded"}) + return + } + normalized, err := normalize(request) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } + service.logRequest(identity.UserID, normalized) + c.Status(http.StatusNoContent) + } +} + +func decodeRequest(c *gin.Context) (Request, int, int, error) { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxRequestBodyBytes) + body, err := io.ReadAll(c.Request.Body) + if err != nil { + var maxBytesError *http.MaxBytesError + if errors.As(err, &maxBytesError) { + return Request{}, 0, http.StatusRequestEntityTooLarge, + errors.New("request body too large") + } + return Request{}, 0, http.StatusBadRequest, errors.New("invalid request body") + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var request Request + if err := decoder.Decode(&request); err != nil { + return Request{}, 0, http.StatusBadRequest, errors.New("invalid request body") + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return Request{}, 0, http.StatusBadRequest, + errors.New("request body must contain one JSON object") + } + return request, len(body), http.StatusOK, nil +} + +func normalize(request Request) (normalizedRequest, error) { + if request.Source != "sonner" && request.Source != "toast-provider" { + return normalizedRequest{}, errors.New("unsupported toast source") + } + if strings.TrimSpace(request.Title) == "" && strings.TrimSpace(request.Description) == "" { + return normalizedRequest{}, errors.New("title or description is required") + } + if request.ClientTimestamp != "" { + if _, err := time.Parse(time.RFC3339Nano, request.ClientTimestamp); err != nil { + return normalizedRequest{}, errors.New("client_timestamp must be RFC3339") + } + } + normalized := normalizedRequest{Request: request} + normalized.Title = truncateField(request.Title, titleByteLimit, &normalized.Truncated) + normalized.Description = truncateField(request.Description, descriptionByteLimit, &normalized.Truncated) + normalized.URL = truncateField(redactURL(request.URL), urlByteLimit, &normalized.Truncated) + normalized.TaskID = truncateField(request.TaskID, taskIDByteLimit, &normalized.Truncated) + normalized.UserAgent = truncateField(request.UserAgent, browserFieldByteLimit, &normalized.Truncated) + normalized.Language = truncateField(request.Language, browserFieldByteLimit, &normalized.Truncated) + normalized.Platform = truncateField(request.Platform, browserFieldByteLimit, &normalized.Truncated) + normalized.Stack = truncateField(request.Stack, stackByteLimit, &normalized.Truncated) + if request.Error != nil { + copy := *request.Error + copy.Name = truncateField(copy.Name, browserFieldByteLimit, &normalized.Truncated) + copy.Message = truncateField(copy.Message, titleByteLimit, &normalized.Truncated) + copy.Stack = truncateField(copy.Stack, stackByteLimit, &normalized.Truncated) + normalized.Error = © + } + return normalized, nil +} + +func redactURL(value string) string { + parsed, err := url.Parse(value) + if err != nil { + return "" + } + parsed.User = nil + parsed.RawQuery = "" + parsed.ForceQuery = false + parsed.Fragment = "" + return parsed.String() +} + +func truncateField(value string, limit int, truncated *bool) string { + if len(value) <= limit { + return value + } + end := limit + for end > 0 && !utf8.ValidString(value[:end]) { + end-- + } + *truncated = true + return value[:end] +} + +func (s *Service) logRequest(userID string, request normalizedRequest) { + fields := []zap.Field{ + zap.String("reporting_user_id", userID), + zap.String("client_timestamp", request.ClientTimestamp), + zap.String("source", request.Source), + zap.String("task_id", request.TaskID), + zap.String("title", request.Title), + zap.String("description", request.Description), + zap.String("url", request.URL), + zap.String("user_agent", request.UserAgent), + zap.String("language", request.Language), + zap.String("platform", request.Platform), + zap.String("stack", request.Stack), + zap.Bool("truncated", request.Truncated), + } + if request.Viewport != nil { + fields = append(fields, zap.Int("viewport_width", request.Viewport.Width), + zap.Int("viewport_height", request.Viewport.Height)) + } + if request.Error != nil { + fields = append(fields, + zap.String("error_name", request.Error.Name), + zap.String("error_message", request.Error.Message), + zap.String("error_stack", request.Error.Stack), + ) + } + s.log.Error("frontend error toast", fields...) +} diff --git a/apps/backend/internal/system/frontenderrors/handler_test.go b/apps/backend/internal/system/frontenderrors/handler_test.go new file mode 100644 index 0000000000..4c1c1fd4e9 --- /dev/null +++ b/apps/backend/internal/system/frontenderrors/handler_test.go @@ -0,0 +1,151 @@ +package frontenderrors + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestHandlerLogsBoundedStructuredFrontendError(t *testing.T) { + router, observed := newHandlerTestRouter(t, "user-1") + body := `{ + "client_timestamp":"2026-07-30T12:34:56.789Z", + "source":"sonner", + "task_id":"task-123", + "title":"Failed to save", + "description":"The backend rejected the update", + "url":"http://localhost/settings/system/logs?token=secret#private", + "user_agent":"test-agent", + "language":"en-US", + "platform":"test-platform", + "viewport":{"width":1440,"height":900}, + "stack":"toast stack", + "error":{"name":"TypeError","message":"Failed to fetch","stack":"error stack"} + }` + response := performFrontendErrorRequest(router, body) + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + } + entries := observed.All() + if len(entries) != 1 { + t.Fatalf("logged entries = %d, want 1", len(entries)) + } + entry := entries[0] + if entry.Level != zapcore.ErrorLevel || entry.Message != "frontend error toast" { + t.Fatalf("entry = %s %q", entry.Level, entry.Message) + } + fields := entry.ContextMap() + if fields["task_id"] != "task-123" || fields["source"] != "sonner" || + fields["title"] != "Failed to save" || fields["error_name"] != "TypeError" { + t.Fatalf("structured fields = %#v", fields) + } + if fields["url"] != "http://localhost/settings/system/logs" { + t.Fatalf("logged url = %#v", fields["url"]) + } +} + +func TestHandlerRejectsInvalidAndOversizedRequests(t *testing.T) { + router, observed := newHandlerTestRouter(t, "user-1") + tests := []struct { + name string + body string + want int + }{ + {name: "malformed", body: `{`, want: http.StatusBadRequest}, + {name: "unsupported source", body: `{"source":"other","title":"visible"}`, want: http.StatusBadRequest}, + {name: "empty text", body: `{"source":"sonner","title":" ","description":""}`, want: http.StatusBadRequest}, + {name: "bad timestamp", body: `{"source":"sonner","title":"visible","client_timestamp":"yesterday"}`, want: http.StatusBadRequest}, + {name: "oversized", body: `{"source":"sonner","title":"` + strings.Repeat("x", maxRequestBodyBytes) + `"}`, want: http.StatusRequestEntityTooLarge}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := performFrontendErrorRequest(router, test.body) + if response.Code != test.want { + t.Fatalf("status = %d, want %d, body = %s", response.Code, test.want, response.Body.String()) + } + }) + } + if observed.Len() != 0 { + t.Fatalf("invalid requests produced %d log entries", observed.Len()) + } +} + +func TestHandlerTruncatesUTF8FieldsAndMarksEntry(t *testing.T) { + router, observed := newHandlerTestRouter(t, "user-1") + body, err := json.Marshal(Request{ + Source: "toast-provider", Title: strings.Repeat("🙂", titleByteLimit/2), + TaskID: strings.Repeat("t", taskIDByteLimit+20), + }) + if err != nil { + t.Fatal(err) + } + response := performFrontendErrorRequest(router, string(body)) + if response.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", response.Code, response.Body.String()) + } + fields := observed.All()[0].ContextMap() + title, _ := fields["title"].(string) + taskID, _ := fields["task_id"].(string) + if len(title) > titleByteLimit || !strings.HasSuffix(title, "🙂") { + t.Fatalf("title truncation split UTF-8 or exceeded limit: %d bytes", len(title)) + } + if len(taskID) != taskIDByteLimit || fields["truncated"] != true { + t.Fatalf("task/truncated fields = %d/%#v", len(taskID), fields["truncated"]) + } +} + +func TestHandlerLimitsAcceptedRequestBytes(t *testing.T) { + router, observed := newHandlerTestRouter(t, "user-1") + body := `{"source":"sonner","title":"visible"}` + strings.Repeat(" ", 40*1024) + + if response := performFrontendErrorRequest(router, body); response.Code != http.StatusNoContent { + t.Fatalf("first status = %d, body = %s", response.Code, response.Body.String()) + } + response := performFrontendErrorRequest(router, body) + if response.Code != http.StatusTooManyRequests { + t.Fatalf("second status = %d, want %d, body = %s", + response.Code, http.StatusTooManyRequests, response.Body.String()) + } + if response.Header().Get("Retry-After") == "" { + t.Fatal("missing Retry-After") + } + if observed.Len() != 1 { + t.Fatalf("logged entries = %d, want 1", observed.Len()) + } +} + +func newHandlerTestRouter(t *testing.T, userID string) (*gin.Engine, *observer.ObservedLogs) { + t.Helper() + gin.SetMode(gin.TestMode) + core, observed := observer.New(zapcore.DebugLevel) + log, err := logger.NewFromZap(zap.New(core)) + if err != nil { + t.Fatal(err) + } + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: userID, Role: authn.RoleMember}) + c.Next() + }) + router.POST("/api/v1/system/logs/frontend-errors", Handle(New(log, time.Now))) + return router, observed +} + +func performFrontendErrorRequest(router *gin.Engine, body string) *httptest.ResponseRecorder { + request := httptest.NewRequest(http.MethodPost, "/api/v1/system/logs/frontend-errors", bytes.NewBufferString(body)) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + return response +} diff --git a/apps/backend/internal/system/frontenderrors/limiter.go b/apps/backend/internal/system/frontenderrors/limiter.go new file mode 100644 index 0000000000..54a6752e51 --- /dev/null +++ b/apps/backend/internal/system/frontenderrors/limiter.go @@ -0,0 +1,143 @@ +package frontenderrors + +import ( + "math" + "sync" + "time" +) + +const ( + identityRatePerMinute = 60 + identityBurst = 20 + globalRatePerMinute = 300 + globalBurst = 100 + identityBytesPerMinute = 64 * 1024 + identityBytesBurst = 64 * 1024 + globalBytesPerMinute = 256 * 1024 + globalBytesBurst = 256 * 1024 + identityBucketTTL = 10 * time.Minute + limiterCleanupEvery = time.Minute +) + +type tokenBucket struct { + tokens float64 + last time.Time + lastSeen time.Time +} + +type identityBuckets struct { + requests tokenBucket + bytes tokenBucket + lastSeen time.Time +} + +type limiter struct { + mu sync.Mutex + now func() time.Time + globalRequests tokenBucket + globalBytes tokenBucket + identities map[string]*identityBuckets + lastCleanup time.Time +} + +func newLimiter(now func() time.Time) *limiter { + if now == nil { + now = time.Now + } + current := now() + return &limiter{ + now: now, + globalRequests: newTokenBucket(globalBurst, current), + globalBytes: newTokenBucket(globalBytesBurst, current), + identities: make(map[string]*identityBuckets), + lastCleanup: current, + } +} + +func (l *limiter) Allow(identity string) (bool, time.Duration) { + return l.allow(identity, 1, false) +} + +func (l *limiter) AllowBytes(identity string, size int) (bool, time.Duration) { + return l.allow(identity, max(1, size), true) +} + +func (l *limiter) allow(identity string, cost int, bytes bool) (bool, time.Duration) { + l.mu.Lock() + defer l.mu.Unlock() + now := l.now() + l.cleanup(now) + buckets := l.identities[identity] + if buckets == nil { + buckets = &identityBuckets{ + requests: newTokenBucket(identityBurst, now), + bytes: newTokenBucket(identityBytesBurst, now), + lastSeen: now, + } + l.identities[identity] = buckets + } + buckets.lastSeen = now + if bytes { + return consumeBuckets(&buckets.bytes, &l.globalBytes, cost, + identityBytesPerMinute, identityBytesBurst, + globalBytesPerMinute, globalBytesBurst, now) + } + return consumeBuckets(&buckets.requests, &l.globalRequests, cost, + identityRatePerMinute, identityBurst, globalRatePerMinute, globalBurst, now) +} + +func consumeBuckets(identity, global *tokenBucket, cost, identityRate, identityBurst, + globalRate, globalBurst int, now time.Time) (bool, time.Duration) { + refill(identity, identityRate, identityBurst, now) + refill(global, globalRate, globalBurst, now) + identityRetry := retryAfter(identity, identityRate, cost) + globalRetry := retryAfter(global, globalRate, cost) + if identityRetry > 0 || globalRetry > 0 { + return false, maxDuration(identityRetry, globalRetry) + } + identity.tokens -= float64(cost) + global.tokens -= float64(cost) + return true, 0 +} + +func (l *limiter) cleanup(now time.Time) { + if now.Sub(l.lastCleanup) < limiterCleanupEvery { + return + } + for identity, bucket := range l.identities { + if now.Sub(bucket.lastSeen) > identityBucketTTL { + delete(l.identities, identity) + } + } + l.lastCleanup = now +} + +func newTokenBucket(burst int, now time.Time) tokenBucket { + return tokenBucket{tokens: float64(burst), last: now, lastSeen: now} +} + +func refill(bucket *tokenBucket, ratePerMinute, burst int, now time.Time) { + elapsed := now.Sub(bucket.last).Minutes() + if elapsed > 0 { + bucket.tokens = math.Min(float64(burst), bucket.tokens+elapsed*float64(ratePerMinute)) + bucket.last = now + } +} + +func retryAfter(bucket *tokenBucket, ratePerMinute, cost int) time.Duration { + if bucket.tokens >= float64(cost) { + return 0 + } + seconds := math.Ceil((float64(cost) - bucket.tokens) / (float64(ratePerMinute) / 60)) + if seconds < 1 { + seconds = 1 + } + return time.Duration(seconds) * time.Second +} + +func maxDuration(left, right time.Duration) time.Duration { + if left > right { + return left + } + return right +} diff --git a/apps/backend/internal/system/frontenderrors/limiter_test.go b/apps/backend/internal/system/frontenderrors/limiter_test.go new file mode 100644 index 0000000000..27bb2c15d1 --- /dev/null +++ b/apps/backend/internal/system/frontenderrors/limiter_test.go @@ -0,0 +1,45 @@ +package frontenderrors + +import ( + "fmt" + "testing" + "time" +) + +func TestLimiterEnforcesIdentityAndGlobalBuckets(t *testing.T) { + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + limiter := newLimiter(func() time.Time { return now }) + for index := 0; index < identityBurst; index++ { + if allowed, _ := limiter.Allow("one-user"); !allowed { + t.Fatalf("identity request %d rejected", index) + } + } + if allowed, retry := limiter.Allow("one-user"); allowed || retry < time.Second { + t.Fatalf("identity overflow allowed=%v retry=%s", allowed, retry) + } + + limiter = newLimiter(func() time.Time { return now }) + for index := 0; index < globalBurst; index++ { + if allowed, _ := limiter.Allow(fmt.Sprintf("user-%d", index)); !allowed { + t.Fatalf("global request %d rejected", index) + } + } + if allowed, retry := limiter.Allow("overflow-user"); allowed || retry <= 0 { + t.Fatalf("global overflow allowed=%v retry=%s", allowed, retry) + } +} + +func TestLimiterRefillsAndRemovesStaleIdentities(t *testing.T) { + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + limiter := newLimiter(func() time.Time { return now }) + for range identityBurst { + limiter.Allow("stale-user") + } + now = now.Add(identityBucketTTL + time.Minute) + if allowed, _ := limiter.Allow("new-user"); !allowed { + t.Fatal("request after refill was rejected") + } + if _, exists := limiter.identities["stale-user"]; exists { + t.Fatal("stale identity bucket was not removed") + } +} diff --git a/apps/backend/internal/system/logbundle/archive.go b/apps/backend/internal/system/logbundle/archive.go new file mode 100644 index 0000000000..ad8c46bce8 --- /dev/null +++ b/apps/backend/internal/system/logbundle/archive.go @@ -0,0 +1,748 @@ +package logbundle + +import ( + "archive/zip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "time" + + "go.uber.org/zap" +) + +const ( + maxBackendBytes = int64(160 * 1024 * 1024) + maxACPBackendBytes = int64(96 * 1024 * 1024) + maxACPBytes = int64(96 * 1024 * 1024) + copyChunkBytes = int64(1024 * 1024) + acpKindRaw = "raw" + acpKindNormalized = "normalized" +) + +type backendArchiveFile struct { + Name string `json:"name"` + Size int64 `json:"size"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` +} + +type frontendArchiveFile struct { + Name string `json:"name"` + Bytes int64 `json:"bytes"` + Entries int `json:"entries"` + StorageMode string `json:"storage_mode,omitempty"` + Metadata json.RawMessage `json:"capture_metadata,omitempty"` +} + +type acpArchiveFile struct { + Name string `json:"name"` + Session string `json:"session"` + Kind string `json:"kind"` + Size int64 `json:"size"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` +} + +type archiveManifest struct { + CreatedAt time.Time `json:"created_at"` + Status Status `json:"status"` + RequestedSources []string `json:"requested_sources"` + IncludedSources []string `json:"included_sources"` + Version string `json:"version,omitempty"` + Commit string `json:"commit,omitempty"` + BuildTime string `json:"build_time,omitempty"` + OS string `json:"os"` + Architecture string `json:"architecture"` + BackendFiles []backendArchiveFile `json:"backend_files"` + FrontendFiles []frontendArchiveFile `json:"frontend_files"` + RuntimeEntryCount int `json:"runtime_entry_count,omitempty"` + ACPSessionCount int `json:"acp_session_count,omitempty"` + ACPFiles []acpArchiveFile `json:"acp_files,omitempty"` + Warnings []string `json:"warnings"` + BackendSinkStatistics any `json:"backend_sink_statistics,omitempty"` +} + +type archiveContents struct { + partial bool + warnings []string + included []string + backendFiles []backendArchiveFile + frontendFiles []frontendArchiveFile + runtimeCount int + acpCount int + acpFiles []acpArchiveFile +} + +func (s *Service) build(id string) { + s.mu.Lock() + item := s.jobs[id] + if item == nil || item.Status != StatusBuilding { + s.mu.Unlock() + return + } + if !s.config.Now().UTC().Before(item.BuildDeadline) { + item.Status = StatusExpired + delete(s.active, item.Owner) + _ = os.RemoveAll(item.WorkDir) + s.mu.Unlock() + return + } + snapshot := snapshotJob(item) + s.mu.Unlock() + + archivePath := filepath.Join(snapshot.WorkDir, "diagnostic-bundle.zip") + partial, warnings, err := s.writeArchive(snapshot, archivePath) + + s.mu.Lock() + defer s.mu.Unlock() + item = s.jobs[id] + if item == nil || item.Status != StatusBuilding { + _ = os.Remove(archivePath) + return + } + if err != nil { + item.Status = StatusFailed + addWarning(item, "diagnostic bundle build failed") + item.ExpiresAt = timePointer(s.config.Now().UTC().Add(s.config.ReadyLifetime)) + delete(s.active, item.Owner) + _ = os.Remove(archivePath) + s.config.Log.Error("Failed to build diagnostic bundle", zap.Error(err)) + return + } + for _, warning := range warnings { + addWarning(item, warning) + } + item.Partial = item.Partial || partial + if item.Partial { + item.Status = StatusPartial + } else { + item.Status = StatusReady + } + item.ArchivePath = archivePath + item.ExpiresAt = timePointer(s.config.Now().UTC().Add(s.config.ReadyLifetime)) + delete(s.active, item.Owner) +} + +func snapshotJob(item *job) *job { + copy := *item + copy.Sources = append([]string(nil), item.Sources...) + copy.SessionIDs = append([]string(nil), item.SessionIDs...) + copy.RuntimeSessions = append([]DiagnosticSession(nil), item.RuntimeSessions...) + copy.ACPSessions = append([]DiagnosticSession(nil), item.ACPSessions...) + copy.Warnings = append([]string(nil), item.Warnings...) + copy.Browsers = make(map[string]*browserCapture, len(item.Browsers)) + for id, browser := range item.Browsers { + browserCopy := *browser + browserCopy.CaptureMetadata = append(json.RawMessage(nil), browser.CaptureMetadata...) + copy.Browsers[id] = &browserCopy + } + return © +} + +func (s *Service) writeArchive(item *job, archivePath string) (bool, []string, error) { + file, err := os.OpenFile(archivePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return false, nil, err + } + writer := zip.NewWriter(file) + contents, err := s.populateArchive(writer, item) + if err == nil { + err = addJSONFile(writer, "manifest.json", s.manifest(item, contents)) + } + if err == nil { + err = writer.Close() + } else { + _ = writer.Close() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err == nil { + err = validateArchiveSize(archivePath) + } + if err != nil { + _ = os.Remove(archivePath) + } + return contents.partial, contents.warnings, err +} + +func (s *Service) populateArchive(writer *zip.Writer, item *job) (archiveContents, error) { + result := archiveContents{ + partial: item.Partial, warnings: append([]string(nil), item.Warnings...), + included: make([]string, 0, 2), + } + if slices.Contains(item.Sources, "backend") { + var backendIncluded bool + var err error + backendLimit := maxBackendBytes + if slices.Contains(item.Sources, SourceACP) { + backendLimit = maxACPBackendBytes + } + result.backendFiles, backendIncluded, result.partial, result.warnings, err = + s.addBackendFiles(writer, result.partial, result.warnings, backendLimit) + if err != nil { + return result, err + } + if backendIncluded { + result.included = append(result.included, "backend") + } + } + if slices.Contains(item.Sources, "frontend") { + var err error + result.frontendFiles, err = addFrontendFiles(writer, item) + if err != nil { + return result, err + } + if len(result.frontendFiles) > 0 { + result.included = append(result.included, "frontend") + } else { + result.partial = true + result.warnings = appendUnique(result.warnings, "no frontend log file was included") + } + } + if slices.Contains(item.Sources, SourceRuntime) { + var runtimeTruncated bool + var err error + result.runtimeCount, runtimeTruncated, err = addRuntimeFile( + writer, "runtime/sessions.json", item.RuntimeSessions, + ) + if err != nil { + return result, err + } + if runtimeTruncated { + result.partial = true + result.warnings = appendUnique(result.warnings, "runtime session index was truncated to the archive byte limit") + } + result.included = append(result.included, SourceRuntime) + } + if slices.Contains(item.Sources, SourceACP) { + var err error + result.acpFiles, result.partial, result.warnings, err = s.addACPFiles( + writer, item, result.partial, result.warnings, + ) + if err != nil { + return result, err + } + result.acpCount = len(item.ACPSessions) + if len(result.acpFiles) > 0 { + result.included = append(result.included, SourceACP) + } else { + result.partial = true + result.warnings = appendUnique(result.warnings, "no ACP debug file was included") + } + } + return result, nil +} + +func (s *Service) manifest(item *job, contents archiveContents) archiveManifest { + status := StatusReady + if contents.partial { + status = StatusPartial + } + return archiveManifest{ + CreatedAt: s.config.Now().UTC(), Status: status, + RequestedSources: append([]string(nil), item.Sources...), IncludedSources: contents.included, + Version: s.config.Version, Commit: s.config.Commit, BuildTime: s.config.BuildTime, + OS: runtime.GOOS, Architecture: runtime.GOARCH, + BackendFiles: contents.backendFiles, FrontendFiles: contents.frontendFiles, + RuntimeEntryCount: contents.runtimeCount, ACPSessionCount: contents.acpCount, + ACPFiles: contents.acpFiles, + Warnings: contents.warnings, + BackendSinkStatistics: s.config.Log.SinkStatistics(), + } +} + +func validateArchiveSize(archivePath string) error { + info, err := os.Stat(archivePath) + if err != nil { + return err + } + if info.Size() > maxTemporaryDiskBytes { + return fmt.Errorf("diagnostic bundle exceeds temporary disk limit") + } + return nil +} + +func (s *Service) addBackendFiles( + writer *zip.Writer, partial bool, warnings []string, budget int64, +) ([]backendArchiveFile, bool, bool, []string, error) { + candidates := s.backendCandidates() + remaining := budget + manifestFiles := make([]backendArchiveFile, 0, len(candidates)) + included := false + for _, candidate := range candidates { + info, err := os.Lstat(candidate.Path) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, included, partial, warnings, err + } + if !info.Mode().IsRegular() { + continue + } + if remaining == 0 { + partial = true + warnings = appendUnique(warnings, "backend logs were truncated to the archive byte limit") + continue + } + length := min(info.Size(), remaining) + offset := info.Size() - length + if length < info.Size() { + partial = true + warnings = appendUnique(warnings, "backend logs were truncated to the archive byte limit") + } + source, err := os.Open(candidate.Path) + if err != nil { + return nil, included, partial, warnings, err + } + header := &zip.FileHeader{Name: "backend/" + candidate.Name, Method: zip.Store} + header.SetMode(0o600) + destination, err := writer.CreateHeader(header) + if err == nil { + err = copySection(destination, source, offset, length) + } + closeErr := source.Close() + if err != nil { + return nil, included, partial, warnings, err + } + if closeErr != nil { + return nil, included, partial, warnings, closeErr + } + manifestFiles = append(manifestFiles, backendArchiveFile{ + Name: candidate.Name, Size: info.Size(), Offset: offset, Length: length, + }) + remaining -= length + included = true + } + if !included { + partial = true + warnings = appendUnique(warnings, "no backend log file was included") + } + return manifestFiles, included, partial, warnings, nil +} + +type backendCandidate struct { + Name string + Path string +} + +func (s *Service) backendCandidates() []backendCandidate { + logDir := filepath.Join(s.config.HomeDir, "logs") + now := s.config.Now().UTC() + names := []string{"backend-logs.log"} + for daysAgo := 1; daysAgo <= 2; daysAgo++ { + date := now.AddDate(0, 0, -daysAgo).Format("2006-01-02") + names = append(names, "backend-logs-"+date+".log") + } + out := make([]backendCandidate, 0, len(names)) + for _, name := range names { + out = append(out, backendCandidate{Name: name, Path: filepath.Join(logDir, name)}) + } + return out +} + +func addFrontendFiles(writer *zip.Writer, item *job) ([]frontendArchiveFile, error) { + browsers := make([]*browserCapture, 0, len(item.Browsers)) + for _, browser := range item.Browsers { + browsers = append(browsers, browser) + } + slices.SortFunc(browsers, func(a, b *browserCapture) int { return a.Index - b.Index }) + manifestFiles := make([]frontendArchiveFile, 0, len(browsers)) + for _, browser := range browsers { + info, err := os.Lstat(browser.Path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("frontend capture is not a regular file") + } + source, err := os.Open(browser.Path) + if err != nil { + return nil, err + } + name := "frontend/browser-" + twoDigit(browser.Index) + ".jsonl" + header := &zip.FileHeader{Name: name, Method: zip.Store} + header.SetMode(0o600) + destination, err := writer.CreateHeader(header) + if err == nil { + err = copySection(destination, source, 0, info.Size()) + } + closeErr := source.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + if removeErr := os.Remove(browser.Path); removeErr != nil && !os.IsNotExist(removeErr) { + return nil, removeErr + } + manifestFiles = append(manifestFiles, frontendArchiveFile{ + Name: name, Bytes: info.Size(), Entries: browser.EntryCount, + StorageMode: browser.StorageMode, Metadata: browser.CaptureMetadata, + }) + } + return manifestFiles, nil +} + +func (s *Service) addACPFiles( + writer *zip.Writer, item *job, partial bool, warnings []string, +) ([]acpArchiveFile, bool, []string, error) { + remaining := maxACPBytes + manifestFiles := make([]acpArchiveFile, 0) + collectionCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for index, session := range item.ACPSessions { + if err := collectionCtx.Err(); err != nil { + partial = true + warnings = appendUnique(warnings, "ACP collection deadline reached") + break + } + files, consumed, sessionPartial, sessionWarnings, err := s.addACPFilesForSession( + collectionCtx, writer, item, index+1, session, remaining, + ) + if err != nil { + return nil, partial, warnings, err + } + manifestFiles = append(manifestFiles, files...) + remaining -= consumed + partial = partial || sessionPartial + for _, warning := range sessionWarnings { + warnings = appendUnique(warnings, warning) + } + } + return manifestFiles, partial, warnings, nil +} + +func (s *Service) addACPFilesForSession( + ctx context.Context, + writer *zip.Writer, + item *job, + index int, + session DiagnosticSession, + remaining int64, +) ([]acpArchiveFile, int64, bool, []string, error) { + var exportErr error + if s.config.ACPExporter != nil && remaining > 0 { + exported, consumed, err := s.addACPExport(ctx, writer, item, index, session, remaining) + if err == nil && len(exported) > 0 { + return exported, consumed, false, nil, nil + } + exportErr = err + } + files, err := s.hostACPFiles(session.ACPSessionID) + if err != nil { + return nil, 0, false, nil, err + } + if len(files) == 0 { + warnings := make([]string, 0, 2) + if exportErr != nil { + var invalidErr *invalidACPExportError + if errors.As(exportErr, &invalidErr) { + warnings = append(warnings, "ACP session "+session.SessionID+" executor export was invalid") + } else { + warnings = append(warnings, "ACP session "+session.SessionID+" executor export was unavailable") + } + } + warnings = append(warnings, "ACP session "+session.SessionID+" is unavailable") + return nil, 0, true, warnings, nil + } + return s.addHostACPFiles(writer, index, session, files, remaining) +} + +func (s *Service) addHostACPFiles( + writer *zip.Writer, + index int, + session DiagnosticSession, + files []hostACPFile, + remaining int64, +) ([]acpArchiveFile, int64, bool, []string, error) { + manifestFiles := make([]acpArchiveFile, 0, len(files)) + warnings := make([]string, 0, 1) + partial := false + var consumed int64 + for _, sourceFile := range files { + if remaining <= 0 { + partial = true + warnings = append(warnings, "ACP logs were truncated to the archive byte limit") + break + } + length := min(sourceFile.size, remaining) + offset := sourceFile.size - length + if length < sourceFile.size { + partial = true + warnings = append(warnings, "ACP logs were truncated to the archive byte limit") + } + archiveName := "acp/session-" + twoDigit(index) + "/" + sourceFile.kind + "/" + sourceFile.name + if err := copyACPHostFile(writer, sourceFile, archiveName, offset, length); err != nil { + return nil, 0, partial, warnings, err + } + manifestFiles = append(manifestFiles, acpArchiveFile{ + Name: archiveName, Session: session.SessionID, Kind: sourceFile.kind, + Size: sourceFile.size, Offset: offset, Length: length, + }) + remaining -= length + consumed += length + } + return manifestFiles, consumed, partial, warnings, nil +} + +func copyACPHostFile( + writer *zip.Writer, + sourceFile hostACPFile, + archiveName string, + offset, length int64, +) error { + source, err := os.Open(sourceFile.path) + if err != nil { + return err + } + header := &zip.FileHeader{Name: archiveName, Method: zip.Store} + header.SetMode(0o600) + destination, createErr := writer.CreateHeader(header) + if createErr == nil { + createErr = copySection(destination, source, offset, length) + } + closeErr := source.Close() + if createErr != nil { + return createErr + } + return closeErr +} + +func (s *Service) addACPExport( + ctx context.Context, + writer *zip.Writer, + item *job, + index int, + session DiagnosticSession, + remaining int64, +) ([]acpArchiveFile, int64, error) { + response, err := s.config.ACPExporter.ExportACP(ctx, session, remaining) + if err != nil { + return nil, 0, err + } + if response == nil { + return nil, 0, invalidACPExport("ACP exporter returned an empty response") + } + defer func() { _ = response.Close() }() + tempPath, err := persistACPExport(response, item.WorkDir, remaining) + if err != nil { + return nil, 0, err + } + defer func() { _ = os.Remove(tempPath) }() + entries, consumed, tempFile, err := loadValidatedACPEntries(tempPath, session.ACPSessionID, index, remaining) + if err != nil { + return nil, 0, err + } + defer func() { _ = tempFile.Close() }() + if err := validateACPEntryContents(entries); err != nil { + return nil, 0, err + } + files, err := writeACPEntries(writer, entries, session.SessionID) + if err != nil { + return nil, 0, err + } + return files, consumed, nil +} + +func persistACPExport(response io.Reader, workDir string, remaining int64) (string, error) { + temp, err := os.CreateTemp(workDir, "acp-export-*.zip") + if err != nil { + return "", err + } + tempPath := temp.Name() + count, copyErr := io.Copy(temp, io.LimitReader(response, remaining+1)) + closeErr := temp.Close() + if copyErr != nil || closeErr != nil || count > remaining { + _ = os.Remove(tempPath) + return "", invalidACPExport("ACP export exceeded its byte bound") + } + return tempPath, nil +} + +func loadValidatedACPEntries( + tempPath, sessionID string, + index int, + remaining int64, +) ([]validatedACPEntry, int64, *os.File, error) { + info, err := os.Stat(tempPath) + if err != nil { + return nil, 0, nil, err + } + tempFile, err := os.Open(tempPath) + if err != nil { + return nil, 0, nil, err + } + reader, err := zip.NewReader(tempFile, info.Size()) + if err != nil { + _ = tempFile.Close() + return nil, 0, nil, invalidACPExport("ACP export is not a valid ZIP") + } + if len(reader.File) > maxACPExportEntries { + _ = tempFile.Close() + return nil, 0, nil, invalidACPExport("ACP export contains too many files") + } + entries := make([]validatedACPEntry, 0, len(reader.File)) + seenNames := make(map[string]struct{}, len(reader.File)) + var consumed int64 + for _, entry := range reader.File { + kind, name, ok := validatedACPArchiveName(entry.Name, sessionID) + mode := entry.FileInfo().Mode() + if !ok || !mode.IsRegular() || mode&os.ModeSymlink != 0 || + entry.UncompressedSize64 > uint64(remaining-consumed) { + _ = tempFile.Close() + return nil, 0, nil, invalidACPExport("invalid ACP export entry") + } + archiveName := "acp/session-" + twoDigit(index) + "/" + kind + "/" + name + if _, exists := seenNames[archiveName]; exists { + _ = tempFile.Close() + return nil, 0, nil, invalidACPExport("ACP export contains duplicate files") + } + seenNames[archiveName] = struct{}{} + entries = append(entries, validatedACPEntry{entry: entry, archiveName: archiveName}) + consumed += int64(entry.UncompressedSize64) + } + if len(entries) == 0 { + _ = tempFile.Close() + return nil, 0, nil, invalidACPExport("ACP export contains no recognized files") + } + return entries, consumed, tempFile, nil +} + +func validateACPEntryContents(entries []validatedACPEntry) error { + // Read every entry before touching the final archive. This validates ZIP + // CRCs and prevents a malformed later entry from leaving a valid prefix in + // the archive. Keeping only the bounded response ZIP on disk also preserves + // the job's temporary-storage ceiling. + for _, entry := range entries { + source, err := entry.entry.Open() + if err != nil { + return invalidACPExport("ACP export entry could not be read") + } + count, readErr := io.Copy(io.Discard, source) + closeErr := source.Close() + if readErr != nil || closeErr != nil || count != int64(entry.entry.UncompressedSize64) { + return invalidACPExport("ACP export entry failed validation") + } + } + return nil +} + +func writeACPEntries( + writer *zip.Writer, + entries []validatedACPEntry, + sessionID string, +) ([]acpArchiveFile, error) { + files := make([]acpArchiveFile, 0, len(entries)) + for _, entry := range entries { + source, err := entry.entry.Open() + if err != nil { + return nil, err + } + header := &zip.FileHeader{Name: entry.archiveName, Method: zip.Store} + header.SetMode(0o600) + destination, createErr := writer.CreateHeader(header) + if createErr == nil { + _, createErr = io.CopyN(destination, source, int64(entry.entry.UncompressedSize64)) + } + closeErr := source.Close() + if createErr != nil { + return nil, createErr + } + if closeErr != nil { + return nil, closeErr + } + files = append(files, acpArchiveFile{ + Name: entry.archiveName, Session: sessionID, + Kind: filepath.Base(filepath.Dir(entry.archiveName)), + Size: int64(entry.entry.UncompressedSize64), Length: int64(entry.entry.UncompressedSize64), + }) + } + return files, nil +} + +const maxACPExportEntries = 128 + +type invalidACPExportError struct{ message string } + +func (e *invalidACPExportError) Error() string { return e.message } + +func invalidACPExport(format string, args ...any) error { + return &invalidACPExportError{message: fmt.Sprintf(format, args...)} +} + +type validatedACPEntry struct { + entry *zip.File + archiveName string +} + +func validatedACPArchiveName(name, sessionID string) (string, string, bool) { + clean := filepath.ToSlash(filepath.Clean(name)) + clean = strings.TrimPrefix(clean, "./") + parts := strings.Split(clean, "/") + if len(parts) < 2 || parts[0] == ".." || slices.Contains(parts, "..") { + return "", "", false + } + kind := parts[len(parts)-2] + if kind != acpKindRaw && kind != acpKindNormalized { + return "", "", false + } + base := parts[len(parts)-1] + if !exactACPFilename(base, sanitizeACPPart(sessionID)) { + return "", "", false + } + return kind, base, true +} + +func addJSONFile(writer *zip.Writer, name string, value any) error { + header := &zip.FileHeader{Name: name, Method: zip.Store} + header.SetMode(0o600) + destination, err := writer.CreateHeader(header) + if err != nil { + return err + } + encoder := json.NewEncoder(destination) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +func copySection(destination io.Writer, source *os.File, offset, length int64) error { + if _, err := source.Seek(offset, io.SeekStart); err != nil { + return err + } + buffer := make([]byte, copyChunkBytes) + remaining := length + for remaining > 0 { + size := min(int64(len(buffer)), remaining) + count, err := io.ReadFull(source, buffer[:size]) + if count > 0 { + if _, writeErr := destination.Write(buffer[:count]); writeErr != nil { + return writeErr + } + remaining -= int64(count) + runtime.Gosched() + } + if err != nil && err != io.ErrUnexpectedEOF { + return err + } + } + return nil +} + +func appendUnique(values []string, value string) []string { + if !slices.Contains(values, value) { + return append(values, value) + } + return values +} + +func timePointer(value time.Time) *time.Time { return &value } diff --git a/apps/backend/internal/system/logbundle/archive_helpers.go b/apps/backend/internal/system/logbundle/archive_helpers.go new file mode 100644 index 0000000000..84f640c0ca --- /dev/null +++ b/apps/backend/internal/system/logbundle/archive_helpers.go @@ -0,0 +1,184 @@ +package logbundle + +import ( + "archive/zip" + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" +) + +func addRuntimeFile( + writer *zip.Writer, name string, sessions []DiagnosticSession, +) (int, bool, error) { + var payload bytes.Buffer + payload.WriteByte('[') + included := 0 + truncated := false + for _, session := range sessions { + encoded, err := json.Marshal(session) + if err != nil { + return 0, false, err + } + separator := 0 + if included > 0 { + separator = 1 + } + if int64(payload.Len()+separator+len(encoded)+2) > maxRuntimeBytes { + truncated = true + break + } + if separator != 0 { + payload.WriteByte(',') + } + payload.Write(encoded) + included++ + } + payload.WriteString("]\n") + header := &zip.FileHeader{Name: name, Method: zip.Store} + header.SetMode(0o600) + destination, err := writer.CreateHeader(header) + if err != nil { + return 0, false, err + } + if _, err := destination.Write(payload.Bytes()); err != nil { + return 0, false, err + } + return included, truncated, nil +} + +type hostACPFile struct { + name string + path string + kind string + size int64 +} + +func (s *Service) hostACPFiles(sessionID string) ([]hostACPFile, error) { + if sessionID == "" { + return nil, nil + } + dir := filepath.Join(s.config.HomeDir, "logs", "acp") + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + safeSession := sanitizeACPPart(sessionID) + files := make([]hostACPFile, 0, len(entries)) + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !exactACPFilename(name, safeSession) { + continue + } + path := filepath.Join(dir, name) + info, statErr := os.Lstat(path) + if statErr != nil || !info.Mode().IsRegular() { + continue + } + kind := acpKindNormalized + if strings.HasPrefix(name, "raw-") { + kind = acpKindRaw + } + files = append(files, hostACPFile{name: name, path: path, kind: kind, size: info.Size()}) + } + slices.SortFunc(files, func(a, b hostACPFile) int { + if a.kind != b.kind { + if a.kind == acpKindRaw { + return -1 + } + return 1 + } + return strings.Compare(a.name, b.name) + }) + return files, nil +} + +func exactACPFilename(name, safeSession string) bool { + if !strings.HasSuffix(name, ".jsonl") || + (!strings.HasPrefix(name, "raw-") && !strings.HasPrefix(name, "normalized-")) { + return false + } + base := strings.TrimSuffix(name, ".jsonl") + if dot := strings.LastIndexByte(base, '.'); dot > 0 && asciiDigits(base[dot+1:]) { + base = base[:dot] + } + return strings.HasSuffix(base, "-"+safeSession) +} + +func sanitizeACPPart(value string) string { + if value == "" { + return "unknown" + } + var builder strings.Builder + for _, r := range value { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_', r == '.': + builder.WriteRune(r) + default: + builder.WriteByte('_') + } + } + return builder.String() +} + +func asciiDigits(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// ManifestSummary returns the server-generated manifest without exposing +// archive bytes through JSON. It is used by task-mode MCP after materializing +// the ZIP into the execution workspace. +func (s *Service) ManifestSummary(owner, id string) (map[string]any, error) { + file, _, err := s.OpenArchive(owner, id) + if err != nil { + return nil, err + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + return nil, err + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + return nil, err + } + for _, entry := range reader.File { + if entry.Name != "manifest.json" || entry.UncompressedSize64 > 256*1024 { + continue + } + source, err := entry.Open() + if err != nil { + return nil, err + } + var buffer bytes.Buffer + _, copyErr := io.CopyN(&buffer, source, int64(entry.UncompressedSize64)) + closeErr := source.Close() + if copyErr != nil { + return nil, copyErr + } + if closeErr != nil { + return nil, closeErr + } + var summary map[string]any + if err := json.Unmarshal(buffer.Bytes(), &summary); err != nil { + return nil, err + } + return summary, nil + } + return nil, fmt.Errorf("diagnostic bundle manifest is missing") +} diff --git a/apps/backend/internal/system/logbundle/contracts.go b/apps/backend/internal/system/logbundle/contracts.go new file mode 100644 index 0000000000..8793a68070 --- /dev/null +++ b/apps/backend/internal/system/logbundle/contracts.go @@ -0,0 +1,147 @@ +package logbundle + +import ( + "context" + "io" + "time" + + "github.com/kandev/kandev/internal/auth/authn" +) + +const ( + SourceBackend = "backend" + SourceFrontend = "frontend" + SourceRuntime = "runtime" + SourceACP = "acp" + maxACPSelect = 10 + maxRuntimeRows = 500 + maxSessionID = 256 + maxSessionAge = 72 * time.Hour +) + +// DiagnosticSession is the intentionally small, message-free projection used +// by the runtime index and ACP session picker. TaskTitle is picker-only and +// must remain excluded from archive JSON. Providers must not populate it from +// a generic task/session serialization path. +type DiagnosticSession struct { + TaskID string `json:"task_id"` + TaskTitle string `json:"-"` + SessionID string `json:"session_id"` + ACPSessionID string `json:"-"` + Agent string `json:"agent,omitempty"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + Status string `json:"status,omitempty"` + ExecutorType string `json:"executor_type,omitempty"` + StartedAt time.Time `json:"started_at,omitempty"` + LastActivityAt time.Time `json:"last_activity_at,omitempty"` + ACPAvailability string `json:"acp_availability,omitempty"` +} + +// DiagnosticSessionProvider supplies only sessions visible to the caller. +// selectedIDs lets the provider include explicitly selected sessions that are +// older than the normal three-day catalogue window. +type DiagnosticSessionProvider interface { + ListDiagnosticSessions( + ctx context.Context, + identity authn.Identity, + since time.Time, + selectedIDs []string, + ) ([]DiagnosticSession, error) +} + +// ACPExporter is an optional executor-side source. It returns a bounded ZIP +// containing only the exact selected ACP session's raw/normalized files. +type ACPExporter interface { + ExportACP(ctx context.Context, session DiagnosticSession, maxBytes int64) (io.ReadCloser, error) +} + +type CapabilitiesView struct { + Sources []string `json:"sources"` + ACPDebugEnabled bool `json:"acp_debug_enabled"` + ACPMaxSessions int `json:"acp_max_sessions"` +} + +func normalizeBundleRequest(sources, sessionIDs []string) ([]string, []string, string, error) { + if len(sources) == 0 || len(sources) > 4 { + return nil, nil, "", newError(ErrorInvalid, "sources must select at least one diagnostic source") + } + seen := make(map[string]bool, len(sources)) + for _, source := range sources { + if source != SourceBackend && source != SourceFrontend && + source != SourceRuntime && source != SourceACP { + return nil, nil, "", newError(ErrorInvalid, "unsupported diagnostic source") + } + if seen[source] { + return nil, nil, "", newError(ErrorInvalid, "diagnostic sources must be unique") + } + seen[source] = true + } + normalizedSources := make([]string, 0, len(seen)) + for source := range seen { + normalizedSources = append(normalizedSources, source) + } + slicesSortStrings(normalizedSources) + + normalizedSessions, err := normalizeSessionIDs(sessionIDs) + if err != nil { + return nil, nil, "", err + } + if seen[SourceACP] { + if len(normalizedSessions) == 0 || len(normalizedSessions) > maxACPSelect { + return nil, nil, "", newError(ErrorInvalid, "ACP bundles require one to ten sessions") + } + } else if len(normalizedSessions) > 0 { + return nil, nil, "", newError(ErrorInvalid, "session IDs require the ACP source") + } + key := joinSourceKey(normalizedSources, normalizedSessions) + return normalizedSources, normalizedSessions, key, nil +} + +func normalizeSessionIDs(sessionIDs []string) ([]string, error) { + if len(sessionIDs) == 0 { + return nil, nil + } + seen := make(map[string]bool, len(sessionIDs)) + result := make([]string, 0, len(sessionIDs)) + for _, id := range sessionIDs { + if id == "" || len(id) > maxSessionID { + return nil, newError(ErrorInvalid, "invalid ACP session ID") + } + if seen[id] { + return nil, newError(ErrorInvalid, "ACP session IDs must be unique") + } + seen[id] = true + result = append(result, id) + } + slicesSortStrings(result) + return result, nil +} + +func joinSourceKey(sources, sessionIDs []string) string { + if len(sessionIDs) == 0 { + return joinStrings(sources, ",") + } + return joinStrings(sources, ",") + "\x00" + joinStrings(sessionIDs, ",") +} + +// Small local helpers keep this contracts file independent from the service's +// broader collection implementation while preserving deterministic keys. +func slicesSortStrings(values []string) { + for i := 1; i < len(values); i++ { + for j := i; j > 0 && values[j] < values[j-1]; j-- { + values[j], values[j-1] = values[j-1], values[j] + } + } +} + +func joinStrings(values []string, separator string) string { + if len(values) == 0 { + return "" + } + result := values[0] + for _, value := range values[1:] { + result += separator + value + } + return result +} diff --git a/apps/backend/internal/system/logbundle/disk_unix.go b/apps/backend/internal/system/logbundle/disk_unix.go new file mode 100644 index 0000000000..927bb9487c --- /dev/null +++ b/apps/backend/internal/system/logbundle/disk_unix.go @@ -0,0 +1,13 @@ +//go:build !windows + +package logbundle + +import "syscall" + +func availableDiskBytes(path string) (uint64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, err + } + return stat.Bavail * uint64(stat.Bsize), nil +} diff --git a/apps/backend/internal/system/logbundle/disk_windows.go b/apps/backend/internal/system/logbundle/disk_windows.go new file mode 100644 index 0000000000..701f700462 --- /dev/null +++ b/apps/backend/internal/system/logbundle/disk_windows.go @@ -0,0 +1,17 @@ +//go:build windows + +package logbundle + +import "golang.org/x/sys/windows" + +func availableDiskBytes(path string) (uint64, error) { + directory, err := windows.UTF16PtrFromString(path) + if err != nil { + return 0, err + } + var freeToCaller, totalBytes, totalFree uint64 + if err := windows.GetDiskFreeSpaceEx(directory, &freeToCaller, &totalBytes, &totalFree); err != nil { + return 0, err + } + return freeToCaller, nil +} diff --git a/apps/backend/internal/system/logbundle/handler.go b/apps/backend/internal/system/logbundle/handler.go new file mode 100644 index 0000000000..aa15abf953 --- /dev/null +++ b/apps/backend/internal/system/logbundle/handler.go @@ -0,0 +1,256 @@ +package logbundle + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/auth/authn" +) + +const ( + maxCreateBodyBytes = 8 * 1024 + maxUploadBodyBytes = 1024 * 1024 +) + +type createRequest struct { + Sources []string `json:"sources"` + SessionIDs []string `json:"session_ids"` +} + +type uploadEnvelope struct { + BrowserID string `json:"browser_id"` + CaptureStreamID string `json:"capture_stream_id"` + ChunkIndex int `json:"chunk_index"` + Done bool `json:"done"` + StorageMode string `json:"storage_mode"` + CaptureMetadata json.RawMessage `json:"capture_metadata"` + Entries json.RawMessage `json:"entries"` +} + +type createResponse struct { + JobView + Reused bool `json:"reused"` +} + +type acpSessionResponse struct { + DiagnosticSession + TaskTitle string `json:"task_title,omitempty"` +} + +func RegisterRoutes(group *gin.RouterGroup, service *Service) { + group.POST("/logs/bundles", handleCreate(service)) + group.GET("/logs/capabilities", handleCapabilities(service)) + group.GET("/logs/acp-sessions", handleACPSessions(service)) + group.GET("/logs/bundles/:id", handleGet(service)) + group.POST("/logs/bundles/:id/frontend", handleUpload(service)) + group.GET("/logs/bundles/:id/download", handleDownload(service)) +} + +func handleCreate(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + identity, ok := requestIdentity(c) + if !ok { + return + } + var request createRequest + if err := decodeBoundedJSON(c, maxCreateBodyBytes, &request); err != nil { + writeDecodeError(c, err) + return + } + job, reused, err := service.CreateWithIdentity( + c.Request.Context(), identity, request.Sources, request.SessionIDs, + ) + if err != nil { + writeServiceError(c, err) + return + } + c.JSON(http.StatusAccepted, createResponse{JobView: job, Reused: reused}) + } +} + +func handleCapabilities(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + if _, ok := requestIdentity(c); !ok { + return + } + c.JSON(http.StatusOK, service.Capabilities()) + } +} + +func handleACPSessions(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + identity, ok := requestIdentity(c) + if !ok { + return + } + rows, err := service.ListACPSessions(c.Request.Context(), identity) + if err != nil { + writeServiceError(c, err) + return + } + c.JSON(http.StatusOK, gin.H{"sessions": acpSessionResponses(rows)}) + } +} + +func acpSessionResponses(rows []DiagnosticSession) []acpSessionResponse { + responses := make([]acpSessionResponse, 0, len(rows)) + for _, row := range rows { + responses = append(responses, acpSessionResponse{ + DiagnosticSession: row, + TaskTitle: row.TaskTitle, + }) + } + return responses +} + +func handleGet(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + identity, ok := requestIdentity(c) + if !ok { + return + } + job, err := service.Get(identity.UserID, c.Param("id")) + if err != nil { + writeServiceError(c, err) + return + } + c.JSON(http.StatusOK, job) + } +} + +func handleUpload(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + identity, ok := requestIdentity(c) + if !ok { + return + } + var envelope uploadEnvelope + if err := decodeBoundedJSON(c, maxUploadBodyBytes, &envelope); err != nil { + writeDecodeError(c, err) + return + } + ignored, claimed, err := service.ClaimStream( + identity.UserID, c.Param("id"), envelope.BrowserID, + envelope.CaptureStreamID, envelope.StorageMode, + ) + if err != nil { + writeServiceError(c, err) + return + } + if ignored { + c.Status(http.StatusNoContent) + return + } + var entries []json.RawMessage + if len(envelope.Entries) == 0 { + entries = []json.RawMessage{} + } else if err := json.Unmarshal(envelope.Entries, &entries); err != nil { + if claimed { + service.ReleaseEmptyClaim(identity.UserID, c.Param("id"), + envelope.BrowserID, envelope.CaptureStreamID) + } + c.JSON(http.StatusBadRequest, gin.H{"error": "entries must be a JSON array"}) + return + } + _, err = service.UploadChunk(identity.UserID, c.Param("id"), UploadChunk{ + BrowserID: envelope.BrowserID, CaptureStreamID: envelope.CaptureStreamID, + ChunkIndex: envelope.ChunkIndex, Done: envelope.Done, + StorageMode: envelope.StorageMode, CaptureMetadata: envelope.CaptureMetadata, + Entries: entries, + }) + if err != nil { + if claimed { + service.ReleaseEmptyClaim(identity.UserID, c.Param("id"), + envelope.BrowserID, envelope.CaptureStreamID) + } + writeServiceError(c, err) + return + } + c.Status(http.StatusNoContent) + } +} + +func handleDownload(service *Service) gin.HandlerFunc { + return func(c *gin.Context) { + identity, ok := requestIdentity(c) + if !ok { + return + } + file, _, err := service.OpenArchive(identity.UserID, c.Param("id")) + if err != nil { + writeServiceError(c, err) + return + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to inspect diagnostic bundle"}) + return + } + c.Header("Content-Disposition", `attachment; filename="kandev-diagnostic-logs.zip"`) + c.Header("Content-Type", "application/zip") + http.ServeContent(c.Writer, c.Request, "kandev-diagnostic-logs.zip", info.ModTime(), file) + } +} + +func requestIdentity(c *gin.Context) (authn.Identity, bool) { + identity, ok := authn.FromGin(c) + if !ok || identity.UserID == "" { + c.JSON(http.StatusUnauthorized, gin.H{"error": "authentication required"}) + return authn.Identity{}, false + } + return identity, true +} + +func decodeBoundedJSON(c *gin.Context, limit int64, destination any) error { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, limit) + decoder := json.NewDecoder(c.Request.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { + return err + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("request body must contain one JSON object") + } + return err + } + return nil +} + +func writeDecodeError(c *gin.Context, err error) { + var maxBytesError *http.MaxBytesError + if errors.As(err, &maxBytesError) { + c.JSON(http.StatusRequestEntityTooLarge, gin.H{"error": "request body too large"}) + return + } + c.JSON(http.StatusBadRequest, gin.H{"error": "invalid request body"}) +} + +func writeServiceError(c *gin.Context, err error) { + status := http.StatusInternalServerError + switch { + case IsKind(err, ErrorInvalid): + status = http.StatusBadRequest + case IsKind(err, ErrorNotFound): + status = http.StatusNotFound + case IsKind(err, ErrorGone): + status = http.StatusGone + case IsKind(err, ErrorConflict): + status = http.StatusConflict + case IsKind(err, ErrorIdentityBusy), IsKind(err, ErrorProfileLimit): + status = http.StatusTooManyRequests + c.Header("Retry-After", strconv.Itoa(5)) + case IsKind(err, ErrorSaturated): + status = http.StatusServiceUnavailable + c.Header("Retry-After", strconv.Itoa(5)) + case IsKind(err, ErrorTooLarge): + status = http.StatusRequestEntityTooLarge + } + c.JSON(status, gin.H{"error": err.Error()}) +} diff --git a/apps/backend/internal/system/logbundle/handler_test.go b/apps/backend/internal/system/logbundle/handler_test.go new file mode 100644 index 0000000000..cb58280d80 --- /dev/null +++ b/apps/backend/internal/system/logbundle/handler_test.go @@ -0,0 +1,163 @@ +package logbundle + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/auth/authn" +) + +func TestBundleRoutesEnforceIdentityOwnership(t *testing.T) { + gin.SetMode(gin.TestMode) + service := newTestService(t, Config{HomeDir: t.TempDir(), CaptureWindow: time.Hour}) + router := gin.New() + group := router.Group("/api/v1/system") + group.Use(func(c *gin.Context) { + userID := c.GetHeader("X-Test-User") + if userID != "" { + authn.SetOnGin(c, authn.Identity{UserID: userID, Role: authn.RoleMember}) + } + c.Next() + }) + RegisterRoutes(group, service) + + create := requestBundle(t, router, http.MethodPost, "/api/v1/system/logs/bundles", + "user-1", `{"sources":["frontend"]}`) + if create.Code != http.StatusAccepted { + t.Fatalf("create status = %d body=%s", create.Code, create.Body.String()) + } + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal(create.Body.Bytes(), &created); err != nil { + t.Fatal(err) + } + + foreign := requestBundle(t, router, http.MethodGet, + "/api/v1/system/logs/bundles/"+created.ID, "user-2", "") + if foreign.Code != http.StatusNotFound { + t.Fatalf("foreign status = %d body=%s", foreign.Code, foreign.Body.String()) + } +} + +func TestBundleCapabilityAndACPSessionRoutesUseBackendDebugState(t *testing.T) { + gin.SetMode(gin.TestMode) + provider := &diagnosticSessionProviderStub{sessions: []DiagnosticSession{ + { + TaskID: "task-1", TaskTitle: "Repair diagnostic export", SessionID: "session-1", + Agent: "claude-acp", Status: "running", + }, + }} + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + ACPDebugEnabled: func() bool { return true }, + SessionProvider: provider, + }) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "user-1", Role: authn.RoleMember}) + c.Next() + }) + group := router.Group("/api/v1/system") + RegisterRoutes(group, service) + + capabilities := requestBundle(t, router, http.MethodGet, + "/api/v1/system/logs/capabilities", "", "") + if capabilities.Code != http.StatusOK { + t.Fatalf("capabilities status = %d body=%s", capabilities.Code, capabilities.Body.String()) + } + if !strings.Contains(capabilities.Body.String(), `"acp_debug_enabled":true`) { + t.Fatalf("capabilities = %s", capabilities.Body.String()) + } + + sessions := requestBundle(t, router, http.MethodGet, + "/api/v1/system/logs/acp-sessions", "", "") + if sessions.Code != http.StatusOK || !strings.Contains(sessions.Body.String(), "session-1") { + t.Fatalf("sessions status = %d body=%s", sessions.Code, sessions.Body.String()) + } + if !strings.Contains(sessions.Body.String(), `"task_title":"Repair diagnostic export"`) { + t.Fatalf("ACP picker response omits task title: %s", sessions.Body.String()) + } +} + +func TestDiagnosticSessionTaskTitleStaysOutOfArchiveJSON(t *testing.T) { + field, ok := reflect.TypeOf(DiagnosticSession{}).FieldByName("TaskTitle") + if !ok { + t.Fatal("DiagnosticSession must retain the picker-only TaskTitle field") + } + if actual := field.Tag.Get("json"); actual != "-" { + t.Fatalf("TaskTitle JSON tag = %q, want -", actual) + } +} + +func TestLosingFrontendStreamIsAcknowledgedBeforeEntriesDecode(t *testing.T) { + gin.SetMode(gin.TestMode) + service := newTestService(t, Config{HomeDir: t.TempDir(), CaptureWindow: time.Hour}) + job, _, err := service.Create("user-1", []string{"frontend"}) + if err != nil { + t.Fatal(err) + } + if _, _, err := service.ClaimStream("user-1", job.ID, "browser-a", "winner", "memory"); err != nil { + t.Fatal(err) + } + router := gin.New() + group := router.Group("/api/v1/system") + group.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "user-1", Role: authn.RoleMember}) + c.Next() + }) + RegisterRoutes(group, service) + + body := `{"browser_id":"browser-a","capture_stream_id":"loser","chunk_index":99,` + + `"done":false,"storage_mode":"memory","capture_metadata":null,"entries":{"not":"an array"}}` + response := requestBundle(t, router, http.MethodPost, + "/api/v1/system/logs/bundles/"+job.ID+"/frontend", "", body) + if response.Code != http.StatusNoContent { + t.Fatalf("losing stream status = %d body=%s", response.Code, response.Body.String()) + } +} + +func TestFrontendUploadBodyIsBounded(t *testing.T) { + gin.SetMode(gin.TestMode) + service := newTestService(t, Config{HomeDir: t.TempDir(), CaptureWindow: time.Hour}) + job, _, err := service.Create("user-1", []string{"frontend"}) + if err != nil { + t.Fatal(err) + } + router := gin.New() + group := router.Group("/api/v1/system") + group.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "user-1"}) + c.Next() + }) + RegisterRoutes(group, service) + + body := `{"browser_id":"browser-a","capture_stream_id":"stream-a","chunk_index":0,` + + `"done":false,"entries":["` + strings.Repeat("x", 1024*1024) + `"]}` + response := requestBundle(t, router, http.MethodPost, + "/api/v1/system/logs/bundles/"+job.ID+"/frontend", "", body) + if response.Code != http.StatusRequestEntityTooLarge { + t.Fatalf("oversize status = %d body=%s", response.Code, response.Body.String()) + } +} + +func requestBundle( + t *testing.T, router http.Handler, method, path, userID, body string, +) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(method, path, bytes.NewBufferString(body)) + request.Header.Set("Content-Type", "application/json") + if userID != "" { + request.Header.Set("X-Test-User", userID) + } + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + return response +} diff --git a/apps/backend/internal/system/logbundle/job.go b/apps/backend/internal/system/logbundle/job.go new file mode 100644 index 0000000000..ad62f302c2 --- /dev/null +++ b/apps/backend/internal/system/logbundle/job.go @@ -0,0 +1,129 @@ +package logbundle + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "time" +) + +type Status string + +const ( + StatusCollecting Status = "collecting" + StatusBuilding Status = "building" + StatusReady Status = "ready" + StatusPartial Status = "partial" + StatusFailed Status = "failed" + StatusExpired Status = "expired" +) + +type ErrorKind string + +const ( + ErrorInvalid ErrorKind = "invalid" + ErrorNotFound ErrorKind = "not_found" + ErrorGone ErrorKind = "gone" + ErrorConflict ErrorKind = "conflict" + ErrorIdentityBusy ErrorKind = "identity_busy" + ErrorSaturated ErrorKind = "saturated" + ErrorTooLarge ErrorKind = "too_large" + ErrorProfileLimit ErrorKind = "profile_limit" +) + +type ServiceError struct { + Kind ErrorKind + Message string +} + +func (e *ServiceError) Error() string { return e.Message } + +func IsKind(err error, kind ErrorKind) bool { + var serviceError *ServiceError + return errors.As(err, &serviceError) && serviceError.Kind == kind +} + +func newError(kind ErrorKind, format string, args ...any) error { + return &ServiceError{Kind: kind, Message: fmt.Sprintf(format, args...)} +} + +type JobView struct { + ID string `json:"id"` + Status Status `json:"status"` + Sources []string `json:"sources"` + SessionIDs []string `json:"session_ids,omitempty"` + RuntimeEntryCount int `json:"runtime_entry_count,omitempty"` + ACPSessionCount int `json:"acp_session_count,omitempty"` + BuildDeadline time.Time `json:"build_deadline"` + CaptureDeadline *time.Time `json:"capture_deadline,omitempty"` + ExpiresAt *time.Time `json:"expires_at"` + BrowserProfiles int `json:"browser_profiles"` + FrontendEntryCount int `json:"frontend_entry_count"` + FrontendBytes int64 `json:"frontend_bytes"` + Warnings []string `json:"warnings"` + DownloadURL string `json:"download_url,omitempty"` +} + +type UploadChunk struct { + BrowserID string `json:"browser_id"` + CaptureStreamID string `json:"capture_stream_id"` + ChunkIndex int `json:"chunk_index"` + Done bool `json:"done"` + StorageMode string `json:"storage_mode"` + CaptureMetadata json.RawMessage `json:"capture_metadata"` + Entries []json.RawMessage `json:"entries"` +} + +type browserCapture struct { + ID string + StreamID string + Index int + NextChunk int + EntryCount int + Bytes int64 + Path string + File *os.File + Done bool + StorageMode string + CaptureMetadata json.RawMessage + Truncated bool +} + +type job struct { + ID string + Owner string + Status Status + Sources []string + SessionIDs []string + RuntimeSessions []DiagnosticSession + ACPSessions []DiagnosticSession + SourceKey string + CreatedAt time.Time + BuildDeadline time.Time + CaptureDeadline *time.Time + ExpiresAt *time.Time + WorkDir string + ArchivePath string + Browsers map[string]*browserCapture + FrontendBytes int64 + FrontendEntries int + Warnings []string + Partial bool +} + +func (j *job) view() JobView { + view := JobView{ + ID: j.ID, Status: j.Status, Sources: append([]string(nil), j.Sources...), + SessionIDs: append([]string(nil), j.SessionIDs...), + RuntimeEntryCount: len(j.RuntimeSessions), ACPSessionCount: len(j.ACPSessions), + BuildDeadline: j.BuildDeadline, CaptureDeadline: j.CaptureDeadline, + ExpiresAt: j.ExpiresAt, BrowserProfiles: len(j.Browsers), + FrontendEntryCount: j.FrontendEntries, FrontendBytes: j.FrontendBytes, + Warnings: append([]string(nil), j.Warnings...), + } + if j.Status == StatusReady || j.Status == StatusPartial { + view.DownloadURL = "/api/v1/system/logs/bundles/" + j.ID + "/download" + } + return view +} diff --git a/apps/backend/internal/system/logbundle/service.go b/apps/backend/internal/system/logbundle/service.go new file mode 100644 index 0000000000..d55d74430a --- /dev/null +++ b/apps/backend/internal/system/logbundle/service.go @@ -0,0 +1,737 @@ +// Package logbundle creates bounded, identity-owned diagnostic archives from +// retained backend log files and explicit browser console snapshots. +package logbundle + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "sync" + "time" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + ws "github.com/kandev/kandev/pkg/websocket" + "go.uber.org/zap" +) + +const ( + defaultCaptureWindow = 15 * time.Second + defaultBuildLifetime = 5 * time.Minute + defaultReadyLifetime = 15 * time.Minute + maxActiveJobs = 8 + maxBrowserProfiles = 4 + maxBrowserEntries = 10_000 + maxBrowserBytes = 20 * 1024 * 1024 + maxFrontendBytes = 80 * 1024 * 1024 + maxACPFrontendBytes = 48 * 1024 * 1024 + maxRuntimeBytes = 2 * 1024 * 1024 + maxEntryBytes = 64 * 1024 + maxCaptureMetadata = 8 * 1024 + maxIdentifierBytes = 256 + maxTemporaryDiskBytes = int64(384 * 1024 * 1024) +) + +type IdentityNotifier interface { + SendToIdentity(userID string, message *ws.Message) int +} + +type Config struct { + HomeDir string + Version string + Commit string + BuildTime string + Log *logger.Logger + Now func() time.Time + CaptureWindow time.Duration + BuildLifetime time.Duration + ReadyLifetime time.Duration + AvailableBytes func(path string) (uint64, error) + // ACPDebugEnabled is backend-authoritative. A browser debug flag must never + // be sufficient to authorize ACP export. + ACPDebugEnabled func() bool + SessionProvider DiagnosticSessionProvider + ACPExporter ACPExporter +} + +type Service struct { + config Config + + mu sync.Mutex + jobs map[string]*job + active map[string]string + latest map[string]string + queue chan string + notifier IdentityNotifier + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + start sync.Once + stop sync.Once +} + +func New(config Config) *Service { + if config.HomeDir == "" { + config.HomeDir = "." + } + if config.Log == nil { + config.Log = logger.Default() + } + if config.Now == nil { + config.Now = time.Now + } + if config.CaptureWindow <= 0 { + config.CaptureWindow = defaultCaptureWindow + } + if config.BuildLifetime <= 0 { + config.BuildLifetime = defaultBuildLifetime + } + if config.ReadyLifetime <= 0 { + config.ReadyLifetime = defaultReadyLifetime + } + if config.AvailableBytes == nil { + config.AvailableBytes = availableDiskBytes + } + if config.ACPDebugEnabled == nil { + config.ACPDebugEnabled = func() bool { return os.Getenv("KANDEV_DEBUG_AGENT_MESSAGES") == "true" } + } + return &Service{ + config: config, jobs: make(map[string]*job), active: make(map[string]string), + latest: make(map[string]string), queue: make(chan string, maxActiveJobs), + } +} + +func (s *Service) SetNotifier(notifier IdentityNotifier) { + s.mu.Lock() + s.notifier = notifier + s.mu.Unlock() +} + +func (s *Service) SetSessionProvider(provider DiagnosticSessionProvider) { + s.mu.Lock() + s.config.SessionProvider = provider + s.mu.Unlock() +} + +func (s *Service) SetACPExporter(exporter ACPExporter) { + s.mu.Lock() + s.config.ACPExporter = exporter + s.mu.Unlock() +} + +func (s *Service) Capabilities() CapabilitiesView { + debugEnabled := s.config.ACPDebugEnabled() + sources := []string{SourceBackend, SourceFrontend, SourceRuntime} + if debugEnabled { + sources = append(sources, SourceACP) + } + return CapabilitiesView{ + Sources: sources, ACPDebugEnabled: debugEnabled, ACPMaxSessions: maxACPSelect, + } +} + +func (s *Service) ListACPSessions( + ctx context.Context, identity authn.Identity, +) ([]DiagnosticSession, error) { + if !s.config.ACPDebugEnabled() { + return nil, newError(ErrorNotFound, "ACP debug capture is unavailable") + } + if s.config.SessionProvider == nil { + return []DiagnosticSession{}, nil + } + rows, err := s.config.SessionProvider.ListDiagnosticSessions( + ctx, identity, s.config.Now().UTC().Add(-maxSessionAge), nil, + ) + if err != nil { + return nil, err + } + if len(rows) > maxRuntimeRows { + rows = rows[:maxRuntimeRows] + } + return rows, nil +} + +func (s *Service) Start(parent context.Context) { + s.start.Do(func() { + s.ctx, s.cancel = context.WithCancel(parent) + if err := prepareRootDirectory(s.rootDir()); err != nil { + s.config.Log.Warn("Failed to prepare diagnostic bundle directory", zap.Error(err)) + } + s.wg.Add(2) + go s.buildWorker() + go s.cleanupWorker() + }) +} + +func (s *Service) Stop() { + s.stop.Do(func() { + if s.cancel != nil { + s.cancel() + s.wg.Wait() + } + s.mu.Lock() + for _, item := range s.jobs { + closeBrowserFiles(item) + } + s.mu.Unlock() + }) +} + +func (s *Service) Create(owner string, sources []string) (JobView, bool, error) { + return s.CreateWithIdentity(context.Background(), authn.Identity{ + UserID: owner, Role: authn.RoleAdmin, Synthetic: true, + }, sources, nil) +} + +// CreateWithIdentity creates a source-selectable bundle. The provider is +// consulted before admission for runtime/ACP sources so foreign session IDs +// are rejected without touching files or executor state. +func (s *Service) CreateWithIdentity( + ctx context.Context, + identity authn.Identity, + sources, sessionIDs []string, +) (JobView, bool, error) { + normalized, normalizedSessions, sourceKey, err := normalizeBundleRequest(sources, sessionIDs) + if err != nil { + return JobView{}, false, err + } + if identity.UserID == "" { + return JobView{}, false, newError(ErrorInvalid, "authenticated owner is required") + } + if slices.Contains(normalized, SourceACP) && !s.config.ACPDebugEnabled() { + return JobView{}, false, newError(ErrorInvalid, "ACP debug capture is disabled") + } + runtimeSessions, acpSessions, err := s.materializeDiagnosticSessions( + ctx, identity, normalized, normalizedSessions, + ) + if err != nil { + return JobView{}, false, err + } + now := s.config.Now().UTC() + s.mu.Lock() + defer s.mu.Unlock() + s.expireLocked(now) + + latestKey := identity.UserID + "\x00" + sourceKey + if existing, reused, admissionErr := s.admitLocked(identity.UserID, latestKey); reused || admissionErr != nil { + return existing, reused, admissionErr + } + id, workDir, err := s.createWorkDirectory() + if err != nil { + return JobView{}, false, err + } + item := &job{ + ID: id, Owner: identity.UserID, Sources: normalized, SessionIDs: normalizedSessions, + RuntimeSessions: runtimeSessions, ACPSessions: acpSessions, SourceKey: sourceKey, + CreatedAt: now, BuildDeadline: now.Add(s.config.BuildLifetime), + WorkDir: workDir, Browsers: make(map[string]*browserCapture), + } + if slices.Contains(normalized, "frontend") { + deadline := now.Add(s.config.CaptureWindow) + item.Status = StatusCollecting + item.CaptureDeadline = &deadline + } else { + item.Status = StatusBuilding + } + s.jobs[id] = item + s.active[identity.UserID] = id + s.latest[latestKey] = id + + if item.Status == StatusCollecting { + go s.beginCollection(identity.UserID, id, *item.CaptureDeadline) + } else { + s.enqueueLocked(id) + } + return item.view(), false, nil +} + +func (s *Service) materializeDiagnosticSessions( + ctx context.Context, + identity authn.Identity, + sources, sessionIDs []string, +) ([]DiagnosticSession, []DiagnosticSession, error) { + needsRuntime := slices.Contains(sources, SourceRuntime) + needsACP := slices.Contains(sources, SourceACP) + if !needsRuntime && !needsACP { + return nil, nil, nil + } + if s.config.SessionProvider == nil { + if needsACP { + return nil, nil, newError(ErrorInvalid, "ACP session catalogue is unavailable") + } + return nil, nil, nil + } + records, err := s.config.SessionProvider.ListDiagnosticSessions( + ctx, identity, s.config.Now().UTC().Add(-maxSessionAge), sessionIDs, + ) + if err != nil { + return nil, nil, err + } + var runtimeSessions []DiagnosticSession + if needsRuntime { + runtimeSessions = append(runtimeSessions, records...) + } + if !needsACP { + return runtimeSessions, nil, nil + } + acpSessions, err := selectACPSessions(records, sessionIDs) + if err != nil { + return nil, nil, err + } + return runtimeSessions, acpSessions, nil +} + +func selectACPSessions(records []DiagnosticSession, sessionIDs []string) ([]DiagnosticSession, error) { + byID := make(map[string]DiagnosticSession, len(records)) + for _, record := range records { + byID[record.SessionID] = record + } + selected := make([]DiagnosticSession, 0, len(sessionIDs)) + for _, sessionID := range sessionIDs { + record, ok := byID[sessionID] + if !ok || record.ACPAvailability == "unavailable" { + return nil, newError(ErrorInvalid, "ACP session is unavailable") + } + selected = append(selected, record) + } + return selected, nil +} + +func (s *Service) admitLocked(owner, latestKey string) (JobView, bool, error) { + if id := s.latest[latestKey]; id != "" { + if existing := s.jobs[id]; existing != nil && existing.Status != StatusExpired { + return existing.view(), true, nil + } + } + if id := s.active[owner]; id != "" { + if existing := s.jobs[id]; existing != nil && isActive(existing.Status) { + return JobView{}, false, newError(ErrorIdentityBusy, "another diagnostic bundle is active") + } + delete(s.active, owner) + } + if s.activeCountLocked() >= maxActiveJobs { + return JobView{}, false, newError(ErrorSaturated, "diagnostic bundle capacity is full") + } + return JobView{}, false, nil +} + +func (s *Service) createWorkDirectory() (string, string, error) { + if err := os.MkdirAll(s.rootDir(), 0o700); err != nil { + return "", "", newError(ErrorSaturated, "diagnostic bundle storage unavailable") + } + available, err := s.config.AvailableBytes(s.rootDir()) + if err != nil || available < uint64(maxTemporaryDiskBytes) { + return "", "", newError(ErrorSaturated, "insufficient temporary disk for a diagnostic bundle") + } + id, err := randomID() + if err != nil { + return "", "", err + } + workDir := filepath.Join(s.rootDir(), id) + if err := os.Mkdir(workDir, 0o700); err != nil { + return "", "", err + } + return id, workDir, nil +} + +func (s *Service) Get(owner, id string) (JobView, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.expireLocked(s.config.Now().UTC()) + item := s.ownedJobLocked(owner, id) + if item == nil { + return JobView{}, newError(ErrorNotFound, "diagnostic bundle not found") + } + if item.Status == StatusExpired { + return item.view(), newError(ErrorGone, "diagnostic bundle expired") + } + return item.view(), nil +} + +func (s *Service) OpenArchive(owner, id string) (*os.File, JobView, error) { + s.mu.Lock() + s.expireLocked(s.config.Now().UTC()) + item := s.ownedJobLocked(owner, id) + if item == nil { + s.mu.Unlock() + return nil, JobView{}, newError(ErrorNotFound, "diagnostic bundle not found") + } + view := item.view() + switch item.Status { + case StatusExpired: + s.mu.Unlock() + return nil, view, newError(ErrorGone, "diagnostic bundle expired") + case StatusReady, StatusPartial: + default: + s.mu.Unlock() + return nil, view, newError(ErrorConflict, "diagnostic bundle is not ready") + } + path := item.ArchivePath + s.mu.Unlock() + file, err := os.Open(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, view, newError(ErrorGone, "diagnostic bundle archive is unavailable") + } + return nil, view, err + } + return file, view, nil +} + +func (s *Service) UploadChunk(owner, id string, chunk UploadChunk) (bool, error) { + if err := validateUploadIdentifiers(chunk.BrowserID, chunk.CaptureStreamID, chunk.ChunkIndex); err != nil { + return false, err + } + s.mu.Lock() + defer s.mu.Unlock() + item, err := s.collectingJobLocked(owner, id) + if err != nil { + return false, err + } + browser, ignored, _, err := s.claimBrowserLocked(item, chunk.BrowserID, chunk.CaptureStreamID, chunk.StorageMode) + if err != nil || ignored { + return ignored, err + } + if browser.Done || chunk.ChunkIndex != browser.NextChunk { + return false, newError(ErrorInvalid, "frontend chunks must be sequential") + } + chunkBytes, err := validateChunkPayload(chunk) + if err != nil { + return false, err + } + if captureLimitExceeded(item, browser, len(chunk.Entries), chunkBytes) { + browser.Truncated = true + item.Partial = true + addWarning(item, "frontend capture size limit reached") + return false, newError(ErrorTooLarge, "frontend capture limit reached") + } + if err := writeEntries(browser.File, chunk.Entries); err != nil { + return false, err + } + browser.NextChunk++ + browser.EntryCount += len(chunk.Entries) + browser.Bytes += chunkBytes + item.FrontendEntries += len(chunk.Entries) + item.FrontendBytes += chunkBytes + if chunk.Done { + browser.Done = true + browser.CaptureMetadata = append(json.RawMessage(nil), chunk.CaptureMetadata...) + if err := browser.File.Close(); err != nil { + return false, err + } + browser.File = nil + } + return false, nil +} + +func validateChunkPayload(chunk UploadChunk) (int64, error) { + if !chunk.Done && len(chunk.CaptureMetadata) != 0 && string(chunk.CaptureMetadata) != "null" { + return 0, newError(ErrorInvalid, "capture metadata is allowed only on the final chunk") + } + if len(chunk.CaptureMetadata) > maxCaptureMetadata { + return 0, newError(ErrorTooLarge, "capture metadata is too large") + } + var chunkBytes int64 + for _, entry := range chunk.Entries { + if len(entry) > maxEntryBytes || !json.Valid(entry) { + return 0, newError(ErrorTooLarge, "frontend entry is invalid or too large") + } + chunkBytes += int64(len(entry) + 1) + } + return chunkBytes, nil +} + +func captureLimitExceeded(item *job, browser *browserCapture, entries int, bytes int64) bool { + return browser.EntryCount+entries > maxBrowserEntries || + browser.Bytes+bytes > maxBrowserBytes || + item.FrontendBytes+bytes > frontendByteLimit(item) +} + +func frontendByteLimit(item *job) int64 { + if item != nil && slices.Contains(item.Sources, SourceACP) { + return maxACPFrontendBytes + } + return maxFrontendBytes +} + +func writeEntries(file *os.File, entries []json.RawMessage) error { + for _, entry := range entries { + if _, err := file.Write(entry); err != nil { + return err + } + if _, err := file.Write([]byte{'\n'}); err != nil { + return err + } + } + return nil +} + +// ClaimStream atomically reserves a browser profile for the first responding +// tab. Handlers call it before decoding the entries array so a losing tab is +// acknowledged without per-entry work. +func (s *Service) ClaimStream( + owner, id, browserID, streamID, storageMode string, +) (ignored, newlyClaimed bool, err error) { + if err := validateUploadIdentifiers(browserID, streamID, 0); err != nil { + return false, false, err + } + s.mu.Lock() + defer s.mu.Unlock() + item, err := s.collectingJobLocked(owner, id) + if err != nil { + return false, false, err + } + _, ignored, newlyClaimed, err = s.claimBrowserLocked(item, browserID, streamID, storageMode) + return ignored, newlyClaimed, err +} + +// ReleaseEmptyClaim rolls back a just-created reservation after request +// validation fails. It never removes a stream that has accepted a chunk. +func (s *Service) ReleaseEmptyClaim(owner, id, browserID, streamID string) { + s.mu.Lock() + defer s.mu.Unlock() + item := s.ownedJobLocked(owner, id) + if item == nil { + return + } + browser := item.Browsers[browserID] + if browser == nil || browser.StreamID != streamID || browser.NextChunk != 0 || browser.EntryCount != 0 { + return + } + if browser.File != nil { + _ = browser.File.Close() + } + _ = os.Remove(browser.Path) + delete(item.Browsers, browserID) +} + +func (s *Service) collectingJobLocked(owner, id string) (*job, error) { + s.expireLocked(s.config.Now().UTC()) + item := s.ownedJobLocked(owner, id) + if item == nil { + return nil, newError(ErrorNotFound, "diagnostic bundle not found") + } + if item.Status == StatusExpired { + return nil, newError(ErrorGone, "diagnostic bundle expired") + } + if item.Status != StatusCollecting { + return nil, newError(ErrorConflict, "frontend collection has ended") + } + return item, nil +} + +func (s *Service) claimBrowserLocked( + item *job, browserID, streamID, storageMode string, +) (*browserCapture, bool, bool, error) { + browser := item.Browsers[browserID] + if browser != nil && browser.StreamID != streamID { + return browser, true, false, nil + } + if browser != nil { + return browser, false, false, nil + } + if len(item.Browsers) >= maxBrowserProfiles { + item.Partial = true + addWarning(item, "frontend browser profile limit reached") + return nil, false, false, newError(ErrorProfileLimit, "frontend browser profile limit reached") + } + browser = &browserCapture{ + ID: browserID, StreamID: streamID, + Index: len(item.Browsers) + 1, StorageMode: storageMode, + } + browser.Path = filepath.Join(item.WorkDir, "frontend-"+twoDigit(browser.Index)+".jsonl") + file, err := os.OpenFile(browser.Path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return nil, false, false, err + } + browser.File = file + item.Browsers[browserID] = browser + return browser, false, true, nil +} + +func validateUploadIdentifiers(browserID, streamID string, chunkIndex int) error { + if browserID == "" || len(browserID) > maxIdentifierBytes || + streamID == "" || len(streamID) > maxIdentifierBytes || + chunkIndex < 0 { + return newError(ErrorInvalid, "invalid frontend capture identifiers") + } + return nil +} + +func (s *Service) beginCollection(owner, id string, deadline time.Time) { + s.mu.Lock() + notifier := s.notifier + ctx := s.ctx + s.mu.Unlock() + if notifier != nil { + message, err := ws.NewNotification("system.logs.capture_requested", map[string]any{ + "bundle_id": id, "capture_deadline": deadline, + "max_chunk_bytes": 1024 * 1024, "max_browser_profiles": maxBrowserProfiles, + }) + if err == nil { + notifier.SendToIdentity(owner, message) + } + } + delay := deadline.Sub(s.config.Now().UTC()) + timer := time.NewTimer(max(0, delay)) + defer timer.Stop() + if ctx == nil { + <-timer.C + } else { + select { + case <-timer.C: + case <-ctx.Done(): + return + } + } + s.finishCollection(id) +} + +func (s *Service) finishCollection(id string) { + s.mu.Lock() + defer s.mu.Unlock() + item := s.jobs[id] + if item == nil || item.Status != StatusCollecting { + return + } + for _, browser := range item.Browsers { + if browser.File != nil { + _ = browser.File.Close() + browser.File = nil + } + if !browser.Done { + item.Partial = true + addWarning(item, "frontend capture did not finish before the deadline") + } + } + if len(item.Browsers) == 0 { + item.Partial = true + addWarning(item, "no frontend browser responded") + } + item.Status = StatusBuilding + s.enqueueLocked(id) +} + +func (s *Service) enqueueLocked(id string) { + select { + case s.queue <- id: + default: + item := s.jobs[id] + item.Status = StatusFailed + addWarning(item, "diagnostic bundle build queue is unavailable") + delete(s.active, item.Owner) + } +} + +func (s *Service) buildWorker() { + defer s.wg.Done() + for { + select { + case <-s.ctx.Done(): + return + case id := <-s.queue: + s.build(id) + } + } +} + +func (s *Service) cleanupWorker() { + defer s.wg.Done() + ticker := time.NewTicker(time.Minute) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + s.mu.Lock() + s.expireLocked(s.config.Now().UTC()) + s.mu.Unlock() + } + } +} + +func (s *Service) expireLocked(now time.Time) { + for _, item := range s.jobs { + if isActive(item.Status) && !now.Before(item.BuildDeadline) { + item.Status = StatusExpired + closeBrowserFiles(item) + delete(s.active, item.Owner) + _ = os.RemoveAll(item.WorkDir) + continue + } + if (item.Status == StatusReady || item.Status == StatusPartial || item.Status == StatusFailed) && + item.ExpiresAt != nil && !now.Before(*item.ExpiresAt) { + item.Status = StatusExpired + _ = os.RemoveAll(item.WorkDir) + } + } +} + +func (s *Service) ownedJobLocked(owner, id string) *job { + item := s.jobs[id] + if item == nil || item.Owner != owner { + return nil + } + return item +} + +func (s *Service) activeCountLocked() int { + count := 0 + for _, item := range s.jobs { + if isActive(item.Status) { + count++ + } + } + return count +} + +func (s *Service) rootDir() string { + return filepath.Join(s.config.HomeDir, "tmp", "diagnostic-bundles") +} + +func randomID() (string, error) { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return "", err + } + return hex.EncodeToString(raw), nil +} + +func isActive(status Status) bool { + return status == StatusCollecting || status == StatusBuilding +} + +func addWarning(item *job, warning string) { + if !slices.Contains(item.Warnings, warning) { + item.Warnings = append(item.Warnings, warning) + } +} + +func closeBrowserFiles(item *job) { + for _, browser := range item.Browsers { + if browser.File != nil { + _ = browser.File.Close() + browser.File = nil + } + } +} + +func twoDigit(value int) string { + return string([]byte{'0' + byte(value/10), '0' + byte(value%10)}) +} + +func prepareRootDirectory(path string) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + if err := os.RemoveAll(path); err != nil { + return err + } + return os.Mkdir(path, 0o700) +} diff --git a/apps/backend/internal/system/logbundle/service_test.go b/apps/backend/internal/system/logbundle/service_test.go new file mode 100644 index 0000000000..98e979c06e --- /dev/null +++ b/apps/backend/internal/system/logbundle/service_test.go @@ -0,0 +1,647 @@ +package logbundle + +import ( + "archive/zip" + "bytes" + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestCustomBundleSourcesValidateACPSelectionAndCoalescing(t *testing.T) { + provider := &diagnosticSessionProviderStub{sessions: []DiagnosticSession{ + {TaskID: "task-1", SessionID: "session-1", Agent: "claude-acp", Status: "running"}, + }} + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + ACPDebugEnabled: func() bool { return true }, + SessionProvider: provider, + CaptureWindow: time.Hour, + }) + identity := authn.Identity{UserID: "user-1", Role: authn.RoleMember} + + created, reused, err := service.CreateWithIdentity( + context.Background(), identity, []string{"acp", "runtime"}, []string{"session-1"}, + ) + if err != nil || reused { + t.Fatalf("create custom bundle reused=%v err=%v", reused, err) + } + if !slices.Equal(created.Sources, []string{"acp", "runtime"}) { + t.Fatalf("sources = %v, want [acp runtime]", created.Sources) + } + + equivalent, reused, err := service.CreateWithIdentity( + context.Background(), identity, []string{"runtime", "acp"}, []string{"session-1"}, + ) + if err != nil || !reused || equivalent.ID != created.ID { + t.Fatalf("equivalent custom create = %#v reused=%v err=%v", equivalent, reused, err) + } + + if _, _, err := service.CreateWithIdentity( + context.Background(), identity, []string{"backend"}, []string{"session-1"}, + ); !IsKind(err, ErrorInvalid) { + t.Fatalf("session without ACP error = %v, want invalid", err) + } + if _, _, err := service.CreateWithIdentity( + context.Background(), identity, []string{"acp"}, nil, + ); !IsKind(err, ErrorInvalid) { + t.Fatalf("ACP without session error = %v, want invalid", err) + } +} + +func TestCustomBundleRejectsACPWhenBackendDebugCaptureIsDisabled(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + ACPDebugEnabled: func() bool { return false }, + }) + identity := authn.Identity{UserID: "user-1", Role: authn.RoleMember} + if _, _, err := service.CreateWithIdentity( + context.Background(), identity, []string{"acp"}, []string{"session-1"}, + ); !IsKind(err, ErrorInvalid) { + t.Fatalf("disabled ACP error = %v, want invalid", err) + } +} + +func TestCustomBundleRejectsUnavailableACPSelection(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + ACPDebugEnabled: func() bool { return true }, + SessionProvider: &diagnosticSessionProviderStub{sessions: []DiagnosticSession{{ + TaskID: "task-1", SessionID: "session-1", ACPAvailability: "unavailable", + }}}, + }) + _, _, err := service.CreateWithIdentity( + context.Background(), authn.Identity{UserID: "user-1"}, + []string{SourceACP}, []string{"session-1"}, + ) + if !IsKind(err, ErrorInvalid) { + t.Fatalf("unavailable ACP selection error = %v, want invalid", err) + } +} + +func TestRuntimeSourceWritesAllowListedSessionIndex(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + SessionProvider: &diagnosticSessionProviderStub{sessions: []DiagnosticSession{ + { + TaskID: "task-1", SessionID: "session-1", Agent: "claude-acp", + Provider: "anthropic", Model: "sonnet", Status: "running", + ExecutorType: "local_docker", StartedAt: time.Date(2026, 8, 2, 11, 0, 0, 0, time.UTC), + LastActivityAt: time.Date(2026, 8, 2, 11, 1, 0, 0, time.UTC), + ACPAvailability: "reachable", + }, + }}, + }) + created, _, err := service.CreateWithIdentity( + context.Background(), authn.Identity{UserID: "user-1"}, []string{"runtime"}, nil, + ) + if err != nil { + t.Fatal(err) + } + job := waitForTerminalJob(t, service, "user-1", created.ID) + if job.Status != StatusReady { + t.Fatalf("status = %q, warnings = %v", job.Status, job.Warnings) + } + + file, _, err := service.OpenArchive("user-1", created.ID) + if err != nil { + t.Fatal(err) + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + t.Fatal(err) + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + t.Fatal(err) + } + var runtimeJSON []byte + for _, entry := range reader.File { + if entry.Name != "runtime/sessions.json" { + continue + } + source, openErr := entry.Open() + if openErr != nil { + t.Fatal(openErr) + } + runtimeJSON, err = io.ReadAll(source) + _ = source.Close() + if err != nil { + t.Fatal(err) + } + } + if len(runtimeJSON) == 0 { + t.Fatal("runtime/sessions.json was not included") + } + if string(runtimeJSON) == "" || strings.Contains(string(runtimeJSON), "title") || + strings.Contains(string(runtimeJSON), "message") { + t.Fatalf("runtime index contains disallowed fields: %s", runtimeJSON) + } +} + +func TestRuntimeSourceIsBoundedByItsArchiveBudget(t *testing.T) { + rows := make([]DiagnosticSession, maxRuntimeRows) + for index := range rows { + rows[index] = DiagnosticSession{ + TaskID: "task-1", SessionID: "session-" + twoDigit(index), + Agent: strings.Repeat("agent", 2_000), + } + } + service := newTestService(t, Config{ + HomeDir: t.TempDir(), SessionProvider: &diagnosticSessionProviderStub{sessions: rows}, + }) + created, _, err := service.CreateWithIdentity( + context.Background(), authn.Identity{UserID: "user-1"}, []string{SourceRuntime}, nil, + ) + if err != nil { + t.Fatal(err) + } + job := waitForTerminalJob(t, service, "user-1", created.ID) + if job.Status != StatusPartial { + t.Fatalf("status=%q entries=%d warnings=%v, want truncated partial", job.Status, job.RuntimeEntryCount, job.Warnings) + } + file, _, err := service.OpenArchive("user-1", created.ID) + if err != nil { + t.Fatal(err) + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + t.Fatal(err) + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + t.Fatal(err) + } + for _, entry := range reader.File { + if entry.Name != "runtime/sessions.json" { + continue + } + source, openErr := entry.Open() + if openErr != nil { + t.Fatal(openErr) + } + var decoded []DiagnosticSession + decodeErr := json.NewDecoder(source).Decode(&decoded) + _ = source.Close() + if decodeErr != nil { + t.Fatal(decodeErr) + } + if len(decoded) >= len(rows) { + t.Fatalf("runtime entries=%d, want fewer than %d", len(decoded), len(rows)) + } + } +} + +func TestACPSourceIncludesRawAndNormalizedFilesUnderServerSessionPath(t *testing.T) { + home := t.TempDir() + acpDir := filepath.Join(home, "logs", "acp") + if err := os.MkdirAll(acpDir, 0o700); err != nil { + t.Fatal(err) + } + writeTestFile(t, filepath.Join(acpDir, "raw-acp-claude-acp-session-1.jsonl"), "raw\n") + writeTestFile(t, filepath.Join(acpDir, "normalized-acp-claude-acp-session-1.jsonl"), "normalized\n") + writeTestFile(t, filepath.Join(acpDir, "raw-acp-claude-acp-session-10.jsonl"), "wrong\n") + service := newTestService(t, Config{ + HomeDir: home, ACPDebugEnabled: func() bool { return true }, + SessionProvider: &diagnosticSessionProviderStub{sessions: []DiagnosticSession{{ + TaskID: "task-1", SessionID: "session-1", ACPSessionID: "acp-session-1", + }}}, + }) + created, _, err := service.CreateWithIdentity( + context.Background(), authn.Identity{UserID: "user-1"}, []string{"acp"}, []string{"session-1"}, + ) + if err != nil { + t.Fatal(err) + } + job := waitForTerminalJob(t, service, "user-1", created.ID) + if job.Status != StatusReady { + t.Fatalf("status = %q, warnings = %v", job.Status, job.Warnings) + } + file, _, err := service.OpenArchive("user-1", created.ID) + if err != nil { + t.Fatal(err) + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + t.Fatal(err) + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + t.Fatal(err) + } + var names []string + for _, entry := range reader.File { + names = append(names, entry.Name) + } + slices.Sort(names) + want := []string{ + "acp/session-01/normalized/normalized-acp-claude-acp-session-1.jsonl", + "acp/session-01/raw/raw-acp-claude-acp-session-1.jsonl", + "manifest.json", + } + if !slices.Equal(names, want) { + t.Fatalf("archive entries = %v, want %v", names, want) + } +} + +func TestACPSourceUsesBoundedExecutorExportAndRevalidatesEntries(t *testing.T) { + exporter := &diagnosticACPExporterStub{zipBytes: testACPExportZip(t, map[string]string{ + "raw/raw-acp-claude-acp-session-remote.jsonl": "raw remote\n", + "normalized/normalized-acp-claude-acp-session-remote.jsonl": "normalized remote\n", + })} + service := newTestService(t, Config{ + HomeDir: t.TempDir(), ACPDebugEnabled: func() bool { return true }, ACPExporter: exporter, + SessionProvider: &diagnosticSessionProviderStub{sessions: []DiagnosticSession{{ + TaskID: "task-1", SessionID: "session-1", ACPSessionID: "acp-session-remote", + }}}, + }) + created, _, err := service.CreateWithIdentity( + context.Background(), authn.Identity{UserID: "user-1"}, []string{SourceACP}, []string{"session-1"}, + ) + if err != nil { + t.Fatal(err) + } + job := waitForTerminalJob(t, service, "user-1", created.ID) + if job.Status != StatusReady { + t.Fatalf("status = %q, warnings = %v", job.Status, job.Warnings) + } + if exporter.maxBytes != maxACPBytes { + t.Fatalf("export max bytes = %d, want %d", exporter.maxBytes, maxACPBytes) + } + file, _, err := service.OpenArchive("user-1", created.ID) + if err != nil { + t.Fatal(err) + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + t.Fatal(err) + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + t.Fatal(err) + } + var names []string + for _, entry := range reader.File { + names = append(names, entry.Name) + } + slices.Sort(names) + want := []string{ + "acp/session-01/normalized/normalized-acp-claude-acp-session-remote.jsonl", + "acp/session-01/raw/raw-acp-claude-acp-session-remote.jsonl", + "manifest.json", + } + if !slices.Equal(names, want) { + t.Fatalf("archive entries = %v, want %v", names, want) + } +} + +func TestACPSourceOmitsInvalidExecutorExport(t *testing.T) { + exporter := &diagnosticACPExporterStub{zipBytes: testACPExportZip(t, map[string]string{ + "raw/raw-acp-claude-acp-session-remote.jsonl": "valid before traversal\n", + "raw/../raw-acp-claude-acp-session-remote.jsonl": "traversal\n", + })} + service := newTestService(t, Config{ + HomeDir: t.TempDir(), ACPDebugEnabled: func() bool { return true }, ACPExporter: exporter, + SessionProvider: &diagnosticSessionProviderStub{sessions: []DiagnosticSession{{ + TaskID: "task-1", SessionID: "session-1", ACPSessionID: "acp-session-remote", + }}}, + }) + created, _, err := service.CreateWithIdentity( + context.Background(), authn.Identity{UserID: "user-1"}, []string{SourceACP}, []string{"session-1"}, + ) + if err != nil { + t.Fatal(err) + } + job := waitForTerminalJob(t, service, "user-1", created.ID) + if job.Status != StatusPartial { + t.Fatalf("status = %q, warnings = %v, want partial", job.Status, job.Warnings) + } + file, _, err := service.OpenArchive("user-1", created.ID) + if err != nil { + t.Fatal(err) + } + defer func() { _ = file.Close() }() + info, err := file.Stat() + if err != nil { + t.Fatal(err) + } + reader, err := zip.NewReader(file, info.Size()) + if err != nil { + t.Fatal(err) + } + for _, entry := range reader.File { + if strings.HasPrefix(entry.Name, "acp/") { + t.Fatalf("invalid executor export leaked entry %q", entry.Name) + } + } +} + +type diagnosticACPExporterStub struct { + zipBytes []byte + maxBytes int64 + callCount int +} + +func (e *diagnosticACPExporterStub) ExportACP( + _ context.Context, _ DiagnosticSession, maxBytes int64, +) (io.ReadCloser, error) { + e.maxBytes = maxBytes + e.callCount++ + return io.NopCloser(bytes.NewReader(e.zipBytes)), nil +} + +func testACPExportZip(t *testing.T, files map[string]string) []byte { + t.Helper() + var buffer bytes.Buffer + writer := zip.NewWriter(&buffer) + for name, contents := range files { + entry, err := writer.Create(name) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(entry, contents); err != nil { + t.Fatal(err) + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + return buffer.Bytes() +} + +type diagnosticSessionProviderStub struct { + sessions []DiagnosticSession +} + +func (p *diagnosticSessionProviderStub) ListDiagnosticSessions( + context.Context, authn.Identity, time.Time, []string, +) ([]DiagnosticSession, error) { + return append([]DiagnosticSession(nil), p.sessions...), nil +} + +func TestBackendOnlyBundleContainsOnlyRetainedRegularLogFiles(t *testing.T) { + home := t.TempDir() + logDir := filepath.Join(home, "logs") + if err := os.MkdirAll(logDir, 0o700); err != nil { + t.Fatal(err) + } + now := time.Date(2026, 7, 30, 12, 0, 0, 0, time.UTC) + writeTestFile(t, filepath.Join(logDir, "backend-logs.log"), "current\n") + writeTestFile(t, filepath.Join(logDir, "backend-logs-2026-07-29.log"), "yesterday\n") + writeTestFile(t, filepath.Join(logDir, "backend-logs-2026-07-27.log"), "expired\n") + writeTestFile(t, filepath.Join(logDir, "notes.log"), "private\n") + if err := os.Symlink(filepath.Join(logDir, "notes.log"), filepath.Join(logDir, "backend-logs-2026-07-28.log")); err != nil { + t.Fatal(err) + } + + service := newTestService(t, Config{HomeDir: home, Now: func() time.Time { return now }}) + created, reused, err := service.Create("user-1", []string{"backend"}) + if err != nil { + t.Fatalf("create: %v", err) + } + if reused { + t.Fatal("new job was unexpectedly reused") + } + job := waitForTerminalJob(t, service, "user-1", created.ID) + if job.Status != StatusReady { + t.Fatalf("status = %q, warnings = %v", job.Status, job.Warnings) + } + + file, _, err := service.OpenArchive("user-1", created.ID) + if err != nil { + t.Fatalf("open archive: %v", err) + } + defer func() { _ = file.Close() }() + stat, err := file.Stat() + if err != nil { + t.Fatal(err) + } + reader, err := zip.NewReader(file, stat.Size()) + if err != nil { + t.Fatal(err) + } + var names []string + for _, entry := range reader.File { + names = append(names, entry.Name) + if entry.Method != zip.Store { + t.Fatalf("%s method = %d, want Store", entry.Name, entry.Method) + } + } + slices.Sort(names) + want := []string{ + "backend/backend-logs-2026-07-29.log", + "backend/backend-logs.log", + "manifest.json", + } + if !slices.Equal(names, want) { + t.Fatalf("archive entries = %v, want %v", names, want) + } + if stat.Mode().Perm() != 0o600 { + t.Fatalf("archive mode = %o, want 600", stat.Mode().Perm()) + } +} + +func TestCreateCoalescesEquivalentJobAndRejectsConflictingActiveJob(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + CaptureWindow: time.Hour, + }) + first, reused, err := service.Create("user-1", []string{"frontend", "backend"}) + if err != nil || reused { + t.Fatalf("first create reused=%v err=%v", reused, err) + } + second, reused, err := service.Create("user-1", []string{"backend", "frontend"}) + if err != nil || !reused || second.ID != first.ID { + t.Fatalf("equivalent create = %#v reused=%v err=%v", second, reused, err) + } + if _, _, err := service.Create("user-1", []string{"frontend"}); !IsKind(err, ErrorIdentityBusy) { + t.Fatalf("conflicting create error = %v, want identity busy", err) + } +} + +func TestFrontendUploadBindsFirstStreamAndRequiresSequentialChunks(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + CaptureWindow: time.Hour, + }) + job, _, err := service.Create("user-1", []string{"frontend"}) + if err != nil { + t.Fatal(err) + } + entry := json.RawMessage(`{"level":"error","message":"broken"}`) + ignored, err := service.UploadChunk("user-1", job.ID, UploadChunk{ + BrowserID: "browser-a", CaptureStreamID: "stream-first", ChunkIndex: 0, + Entries: []json.RawMessage{entry}, + }) + if err != nil || ignored { + t.Fatalf("first chunk ignored=%v err=%v", ignored, err) + } + ignored, err = service.UploadChunk("user-1", job.ID, UploadChunk{ + BrowserID: "browser-a", CaptureStreamID: "stream-loser", ChunkIndex: 999, + Entries: []json.RawMessage{entry}, + }) + if err != nil || !ignored { + t.Fatalf("losing stream ignored=%v err=%v", ignored, err) + } + if _, err := service.UploadChunk("user-1", job.ID, UploadChunk{ + BrowserID: "browser-a", CaptureStreamID: "stream-first", ChunkIndex: 2, + }); !IsKind(err, ErrorInvalid) { + t.Fatalf("out-of-order error = %v, want invalid", err) + } +} + +func TestCreateEnforcesGlobalActiveJobLimit(t *testing.T) { + service := newTestService(t, Config{HomeDir: t.TempDir(), CaptureWindow: time.Hour}) + for index := 0; index < maxActiveJobs; index++ { + if _, _, err := service.Create("user-"+twoDigit(index), []string{"frontend"}); err != nil { + t.Fatalf("create active job %d: %v", index, err) + } + } + if _, _, err := service.Create("overflow", []string{"frontend"}); !IsKind(err, ErrorSaturated) { + t.Fatalf("overflow error = %v, want saturated", err) + } +} + +func TestCreateRejectsInsufficientTemporaryDisk(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), + AvailableBytes: func(string) (uint64, error) { + return uint64(maxTemporaryDiskBytes - 1), nil + }, + }) + if _, _, err := service.Create("user-1", []string{"backend"}); !IsKind(err, ErrorSaturated) { + t.Fatalf("insufficient disk error = %v, want saturated", err) + } +} + +func TestActiveAndReadyJobsExpireAtTheirHardDeadlines(t *testing.T) { + service := newTestService(t, Config{ + HomeDir: t.TempDir(), CaptureWindow: time.Hour, + BuildLifetime: 10 * time.Millisecond, ReadyLifetime: 15 * time.Minute, + }) + active, _, err := service.Create("user-1", []string{"frontend"}) + if err != nil { + t.Fatal(err) + } + time.Sleep(20 * time.Millisecond) + if _, err := service.Get("user-1", active.ID); !IsKind(err, ErrorGone) { + t.Fatalf("expired active job error = %v, want gone", err) + } +} + +func TestFrontendCreateNotifiesOnlyThroughIdentityNotifier(t *testing.T) { + service := newTestService(t, Config{HomeDir: t.TempDir(), CaptureWindow: time.Hour}) + notifier := &recordingNotifier{called: make(chan struct{}, 1)} + service.SetNotifier(notifier) + job, _, err := service.Create("user-1", []string{"frontend"}) + if err != nil { + t.Fatal(err) + } + select { + case <-notifier.called: + case <-time.After(time.Second): + t.Fatal("capture notification was not sent") + } + if notifier.owner != "user-1" { + t.Fatalf("notifier owner = %q", notifier.owner) + } + if notifier.message == nil || notifier.message.Action != "system.logs.capture_requested" { + t.Fatalf("notification = %#v", notifier.message) + } + var payload struct { + BundleID string `json:"bundle_id"` + MaxChunkBytes int `json:"max_chunk_bytes"` + MaxBrowserProfiles int `json:"max_browser_profiles"` + } + if err := json.Unmarshal(notifier.message.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload.BundleID != job.ID || payload.MaxChunkBytes != 1024*1024 || + payload.MaxBrowserProfiles != maxBrowserProfiles { + t.Fatalf("capture payload = %#v", payload) + } +} + +type recordingNotifier struct { + owner string + message *ws.Message + called chan struct{} +} + +func (n *recordingNotifier) SendToIdentity(owner string, message *ws.Message) int { + n.owner = owner + n.message = message + select { + case n.called <- struct{}{}: + default: + } + return 1 +} + +func newTestService(t *testing.T, config Config) *Service { + t.Helper() + if config.Log == nil { + var err error + config.Log, err = logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatal(err) + } + } + if config.CaptureWindow == 0 { + config.CaptureWindow = 20 * time.Millisecond + } + service := New(config) + ctx, cancel := context.WithCancel(context.Background()) + service.Start(ctx) + t.Cleanup(func() { + cancel() + service.Stop() + }) + return service +} + +func waitForTerminalJob(t *testing.T, service *Service, owner, id string) JobView { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + job, err := service.Get(owner, id) + if err != nil { + t.Fatal(err) + } + if job.Status == StatusReady || job.Status == StatusPartial || job.Status == StatusFailed { + return job + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("job did not become terminal") + return JobView{} +} + +func writeTestFile(t *testing.T, path, contents string) { + t.Helper() + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + t.Fatal(err) + } + if _, err := io.WriteString(file, contents); err != nil { + _ = file.Close() + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/apps/backend/internal/system/logs/files.go b/apps/backend/internal/system/logs/files.go deleted file mode 100644 index 6c0e0b0a25..0000000000 --- a/apps/backend/internal/system/logs/files.go +++ /dev/null @@ -1,251 +0,0 @@ -// Package logs serves the System -> Logs page: lists log files on disk -// (current + lumberjack-rotated backups), tails the most recent N lines of -// the active log file, and streams individual files for download. -package logs - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "regexp" - "sort" - "strings" - "time" - - "go.uber.org/zap" - - "github.com/kandev/kandev/internal/common/logger" - "github.com/kandev/kandev/internal/common/logger/buffer" -) - -// FileInfo describes one log file on disk. -type FileInfo struct { - Name string `json:"name"` - Size int64 `json:"size"` - Mtime time.Time `json:"mtime"` - Current bool `json:"current"` -} - -// Service enumerates and serves files from the configured log directory. -// The active (un-rotated) file is identified by exact name match against -// currentName (set to the lumberjack Filename's basename, e.g. "kandev.log"). -// When logDir is empty (the process is logging to stdout/stderr only), Tail -// falls back to the in-memory ring buffer so the UI still shows recent logs. -type Service struct { - logDir string - currentName string - log *logger.Logger - memBuffer *buffer.RingBuffer -} - -// NewService constructs a Service. logDir may be empty (logger is writing to -// stdout/stderr); List then returns an empty slice and Tail falls back to the -// shared in-memory ring buffer. currentName is the basename of the active log -// file (e.g. "kandev.log"). -func NewService(logDir, currentName string, log *logger.Logger) *Service { - return &Service{ - logDir: logDir, - currentName: currentName, - log: log, - memBuffer: buffer.Default(), - } -} - -// List returns the log files in the configured directory, sorted by mtime -// descending (newest first). The active file is flagged with Current=true. -// -// Subdirectories are skipped. A missing log directory yields an empty slice -// without error (the logger may be configured for stdout-only output). -func (s *Service) List() ([]FileInfo, error) { - out := []FileInfo{} - if s.logDir == "" { - return out, nil - } - - entries, err := os.ReadDir(s.logDir) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return out, nil - } - s.warn("read log directory failed", zap.String("dir", s.logDir), zap.Error(err)) - return nil, err - } - - for _, entry := range entries { - if entry.IsDir() { - continue - } - fi, statErr := entry.Info() - if statErr != nil { - s.warn("stat log entry failed", zap.String("name", entry.Name()), zap.Error(statErr)) - continue - } - if !fi.Mode().IsRegular() { - continue - } - name := entry.Name() - // Allow-list: only the active log and lumberjack rotations of it. - // A shared logging.outputPath directory may hold unrelated files — - // we must never enumerate them through this endpoint. - if !s.isAllowed(name) { - continue - } - out = append(out, FileInfo{ - Name: name, - Size: fi.Size(), - Mtime: fi.ModTime(), - Current: s.isCurrent(name), - }) - } - - sort.Slice(out, func(i, j int) bool { - return out[i].Mtime.After(out[j].Mtime) - }) - return out, nil -} - -// isCurrent reports whether name is the active (un-rotated) log file. -// Lumberjack rotates "kandev.log" to "kandev-.log[.gz]", so an -// exact basename match is enough to distinguish current from rotated. -func (s *Service) isCurrent(name string) bool { - return s.currentName != "" && name == s.currentName -} - -// lumberjackTimestamp matches the timestamp lumberjack injects between the -// base name and extension when it rotates a log file. Lumberjack formats the -// stamp as time.Format("2006-01-02T15-04-05.000"), so the pattern is exactly -// four digits, a dash, two digits, a dash, two digits, a T, two digits, a -// dash, two digits, a dash, two digits, a dot, three digits — nothing else. -// This is tighter than the previous "non-empty string" check and prevents -// unrelated files like "kandev-cli.log" from matching in shared directories. -var lumberjackTimestamp = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}\.\d{3}$`) - -// isAllowed reports whether name belongs to the configured log set: either -// the active log file itself, or a lumberjack-rotated sibling. Lumberjack -// rotates "." into "-." (optionally gzipped -// to "..gz"). Only names whose embedded timestamp matches lumberjack's -// exact YYYY-MM-DDTHH-MM-SS.sss format are accepted — this prevents unrelated -// files with a matching base prefix from being exposed in shared log dirs. -func (s *Service) isAllowed(name string) bool { - if s.currentName == "" { - return false - } - if name == s.currentName { - return true - } - ext := filepath.Ext(s.currentName) - base := strings.TrimSuffix(s.currentName, ext) - if base == "" || ext == "" { - return false - } - if !strings.HasPrefix(name, base+"-") { - return false - } - rest := strings.TrimPrefix(name, base+"-") - // rest must be "" or ".gz"; strip the - // extension suffix to isolate the timestamp segment. - switch { - case strings.HasSuffix(rest, ext+".gz"): - rest = strings.TrimSuffix(rest, ext+".gz") - case strings.HasSuffix(rest, ext): - rest = strings.TrimSuffix(rest, ext) - default: - return false - } - // Require an exact lumberjack timestamp so files like "kandev-cli.log" - // in a shared directory cannot slip through the allow-list. - return lumberjackTimestamp.MatchString(rest) -} - -// Open returns an *os.File for download, plus its size. name must be a bare -// filename inside the log directory — any path separator, traversal segment, -// or empty value is rejected. Caller closes the returned file. -func (s *Service) Open(name string) (*os.File, int64, error) { - if s.logDir == "" { - return nil, 0, errors.New("logs: log directory not configured") - } - clean, err := s.safeName(name) - if err != nil { - return nil, 0, err - } - // Allow-list check: only the active log file and its lumberjack - // rotations are downloadable. This prevents a shared logging.outputPath - // directory from exposing unrelated files via the download endpoint. - if !s.isAllowed(clean) { - return nil, 0, errors.New("logs: not a log file") - } - // filepath.Base is redundant given safeName already rejects separators, - // but it acts as a syntactic sanitiser for CodeQL's go/path-injection - // taint tracker which does not see safeName as cutting the chain. - full, err := s.containedPath(filepath.Base(clean)) - if err != nil { - return nil, 0, err - } - // Reject symlinks and other non-regular files before opening so an - // adversarial link planted in the log directory cannot escape it. - lstat, err := os.Lstat(full) - if err != nil { - return nil, 0, err - } - if !lstat.Mode().IsRegular() { - return nil, 0, errors.New("logs: not a regular file") - } - f, err := os.Open(full) - if err != nil { - return nil, 0, err - } - fi, err := f.Stat() - if err != nil { - _ = f.Close() - return nil, 0, err - } - return f, fi.Size(), nil -} - -// containedPath joins the (already-safeName-validated) clean filename to the -// configured log directory and verifies that the resolved absolute path is -// still a child of the log directory. This is belt-and-braces alongside -// safeName and gives CodeQL's taint-tracking a syntactic sanitizer it -// recognises ("go/path-injection"). -func (s *Service) containedPath(clean string) (string, error) { - root, err := filepath.Abs(s.logDir) - if err != nil { - return "", fmt.Errorf("logs: resolve log dir: %w", err) - } - joined, err := filepath.Abs(filepath.Join(root, clean)) - if err != nil { - return "", fmt.Errorf("logs: resolve filename: %w", err) - } - rel, err := filepath.Rel(root, joined) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { - return "", errors.New("logs: resolved path escapes the log directory") - } - return joined, nil -} - -// safeName rejects any name that resolves outside the log directory. -func (s *Service) safeName(name string) (string, error) { - if name == "" { - return "", errors.New("logs: empty filename") - } - if strings.ContainsAny(name, `/\`) { - return "", errors.New("logs: path separators not allowed") - } - if name == "." || name == ".." { - return "", errors.New("logs: invalid filename") - } - // filepath.Clean of a single segment without separators is the identity - // for legitimate filenames; anything that changes under Clean is unsafe. - if filepath.Clean(name) != name { - return "", errors.New("logs: invalid filename") - } - return name, nil -} - -func (s *Service) warn(msg string, fields ...zap.Field) { - if s.log == nil { - return - } - s.log.Warn(msg, fields...) -} diff --git a/apps/backend/internal/system/logs/files_test.go b/apps/backend/internal/system/logs/files_test.go deleted file mode 100644 index ea39c61ebd..0000000000 --- a/apps/backend/internal/system/logs/files_test.go +++ /dev/null @@ -1,289 +0,0 @@ -package logs - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/kandev/kandev/internal/common/logger/buffer" -) - -func newTestService(t *testing.T, logDir string) *Service { - t.Helper() - svc := NewService(logDir, "kandev.log", nil) - // Isolate from the process-wide default ring buffer so parallel tests - // cannot pollute Tail() fallbacks when the log file is missing. - svc.memBuffer = buffer.New(100) - return svc -} - -func TestList_EmptyDirReturnsEmptySlice(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - - files, err := svc.List() - if err != nil { - t.Fatalf("List: %v", err) - } - if files == nil { - t.Fatal("List() returned nil, want non-nil empty slice") - } - if len(files) != 0 { - t.Errorf("len(files) = %d, want 0", len(files)) - } -} - -func TestList_MissingDirReturnsEmpty(t *testing.T) { - dir := filepath.Join(t.TempDir(), "does-not-exist") - svc := newTestService(t, dir) - - files, err := svc.List() - if err != nil { - t.Fatalf("List: %v", err) - } - if len(files) != 0 { - t.Errorf("len(files) = %d, want 0", len(files)) - } -} - -func TestList_EmptyLogDirConfigReturnsEmpty(t *testing.T) { - svc := NewService("", "kandev.log", nil) - files, err := svc.List() - if err != nil { - t.Fatalf("List: %v", err) - } - if len(files) != 0 { - t.Errorf("len(files) = %d, want 0 for empty logDir", len(files)) - } -} - -func TestList_SortsNewestFirstAndMarksCurrent(t *testing.T) { - dir := t.TempDir() - older := filepath.Join(dir, "kandev-2026-01-01T00-00-00.000.log") - newer := filepath.Join(dir, "kandev-2026-05-01T00-00-00.000.log.gz") - current := filepath.Join(dir, "kandev.log") - - for _, p := range []string{older, newer, current} { - if err := os.WriteFile(p, []byte("x"), 0o600); err != nil { - t.Fatalf("write %s: %v", p, err) - } - } - - // Force distinct mtimes: older < newer < current. - base := time.Now().Add(-2 * time.Hour) - if err := os.Chtimes(older, base, base); err != nil { - t.Fatalf("chtimes older: %v", err) - } - if err := os.Chtimes(newer, base.Add(time.Hour), base.Add(time.Hour)); err != nil { - t.Fatalf("chtimes newer: %v", err) - } - if err := os.Chtimes(current, base.Add(90*time.Minute), base.Add(90*time.Minute)); err != nil { - t.Fatalf("chtimes current: %v", err) - } - - svc := newTestService(t, dir) - files, err := svc.List() - if err != nil { - t.Fatalf("List: %v", err) - } - if len(files) != 3 { - t.Fatalf("len(files) = %d, want 3", len(files)) - } - - // Newest first ordering. - if files[0].Name != "kandev.log" { - t.Errorf("files[0] = %q, want kandev.log", files[0].Name) - } - if files[1].Name != "kandev-2026-05-01T00-00-00.000.log.gz" { - t.Errorf("files[1] = %q", files[1].Name) - } - if files[2].Name != "kandev-2026-01-01T00-00-00.000.log" { - t.Errorf("files[2] = %q", files[2].Name) - } - - // Exactly one Current. - currentCount := 0 - for _, f := range files { - if f.Current { - currentCount++ - if f.Name != "kandev.log" { - t.Errorf("Current=true on wrong file: %q", f.Name) - } - } - } - if currentCount != 1 { - t.Errorf("currentCount = %d, want 1", currentCount) - } -} - -func TestList_IgnoresNeighboringNonLogFiles(t *testing.T) { - // Simulate logging.outputPath pointing at a shared directory: List must - // only surface the active log + lumberjack rotations, never unrelated - // neighbours like ".env" or "config.yaml". - dir := t.TempDir() - for _, name := range []string{ - "kandev.log", - "kandev-2026-01-01T00-00-00.000.log", - "kandev-2026-05-01T00-00-00.000.log.gz", - ".env", - "config.yaml", - "secrets", - "kandev", // no extension - "other.log", // different base - "kandev-suffixed.txt", // wrong extension - "kandevish-2026-01-01.000.log", // base mismatch (no leading "kandev-") - "kandev-cli.log", // matching prefix but not a timestamp - "kandev-2026-01-01.log", // timestamp missing time part - "kandev-2026-01-01T00-00-00.log", // timestamp missing milliseconds - } { - if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600); err != nil { - t.Fatalf("seed %s: %v", name, err) - } - } - - svc := newTestService(t, dir) - files, err := svc.List() - if err != nil { - t.Fatalf("List: %v", err) - } - - got := map[string]bool{} - for _, f := range files { - got[f.Name] = true - } - want := []string{ - "kandev.log", - "kandev-2026-01-01T00-00-00.000.log", - "kandev-2026-05-01T00-00-00.000.log.gz", - } - for _, w := range want { - if !got[w] { - t.Errorf("List() missing %q", w) - } - } - for _, ban := range []string{ - ".env", "config.yaml", "secrets", "kandev", "other.log", - "kandev-suffixed.txt", "kandevish-2026-01-01.000.log", - "kandev-cli.log", "kandev-2026-01-01.log", "kandev-2026-01-01T00-00-00.log", - } { - if got[ban] { - t.Errorf("List() leaked %q (not a log file)", ban) - } - } - if len(files) != len(want) { - t.Errorf("len(files) = %d, want %d", len(files), len(want)) - } -} - -func TestOpen_RejectsNeighboringNonLogFile(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("SECRET=hunter2"), 0o600); err != nil { - t.Fatalf("seed: %v", err) - } - svc := newTestService(t, dir) - if _, _, err := svc.Open(".env"); err == nil { - t.Fatal("Open(\".env\") returned nil, want allow-list rejection") - } -} - -func TestList_IgnoresSubdirectories(t *testing.T) { - dir := t.TempDir() - if err := os.Mkdir(filepath.Join(dir, "subdir"), 0o755); err != nil { - t.Fatalf("mkdir: %v", err) - } - if err := os.WriteFile(filepath.Join(dir, "kandev.log"), []byte("x"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - - svc := newTestService(t, dir) - files, err := svc.List() - if err != nil { - t.Fatalf("List: %v", err) - } - if len(files) != 1 { - t.Fatalf("len(files) = %d, want 1 (subdir filtered)", len(files)) - } - if files[0].Name != "kandev.log" { - t.Errorf("files[0] = %q", files[0].Name) - } -} - -func TestOpen_ReadsRegularFile(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "kandev.log"), []byte("hello"), 0o644); err != nil { - t.Fatalf("seed: %v", err) - } - svc := newTestService(t, dir) - - f, size, err := svc.Open("kandev.log") - if err != nil { - t.Fatalf("Open: %v", err) - } - t.Cleanup(func() { _ = f.Close() }) - if size != 5 { - t.Errorf("size = %d, want 5", size) - } -} - -func TestOpen_RejectsSymlink(t *testing.T) { - dir := t.TempDir() - target := filepath.Join(dir, "real.log") - if err := os.WriteFile(target, []byte("hello"), 0o644); err != nil { - t.Fatalf("seed: %v", err) - } - link := filepath.Join(dir, "link.log") - if err := os.Symlink(target, link); err != nil { - t.Skipf("symlinks unsupported on this platform: %v", err) - } - svc := newTestService(t, dir) - - if _, _, err := svc.Open("link.log"); err == nil { - t.Fatal("Open(symlink) returned nil, want not-a-regular-file error") - } -} - -func TestOpen_RejectsTraversalAndSeparators(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - - for _, name := range []string{ - "../etc/passwd", - "sub/file.log", - "\\windows\\style.log", - ".", - "..", - "", - } { - t.Run(name, func(t *testing.T) { - if _, _, err := svc.Open(name); err == nil { - t.Fatalf("Open(%q) returned nil, want error", name) - } - }) - } -} - -func TestContainedPath_ReturnedPathIsInsideLogDir(t *testing.T) { - // containedPath's escape-rejection branch is architecturally unreachable - // for single-segment clean names (filepath.Join of a non-empty rootAbs - // and a bare filename without separators always stays inside the root). - // This test exercises the happy path and verifies the postcondition: - // the returned absolute path must be a child of the log directory. - dir := t.TempDir() - svc := newTestService(t, dir) - - // A clean filename that would resolve outside the directory after - // filepath.Join cannot exist by construction (a single segment without - // separators always stays inside the directory). Document this by - // exercising the happy path and verifying the returned path is under dir. - got, err := svc.containedPath("kandev.log") - if err != nil { - t.Fatalf("containedPath: %v", err) - } - rootAbs, _ := filepath.Abs(dir) - rel, err := filepath.Rel(rootAbs, got) - if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { - t.Errorf("containedPath = %q escapes %q", got, rootAbs) - } -} diff --git a/apps/backend/internal/system/logs/handler.go b/apps/backend/internal/system/logs/handler.go deleted file mode 100644 index 760b83cc1f..0000000000 --- a/apps/backend/internal/system/logs/handler.go +++ /dev/null @@ -1,89 +0,0 @@ -package logs - -import ( - "errors" - "fmt" - "io" - "net/http" - "os" - "strconv" - - "github.com/gin-gonic/gin" -) - -// Default and maximum line counts for the tail endpoint. -const ( - defaultTailLines = 1000 - maxTailLines = 10_000 -) - -// HandleList renders GET /api/v1/system/logs: the directory listing. -func HandleList(s *Service) gin.HandlerFunc { - return func(c *gin.Context) { - files, err := s.List() - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"files": files}) - } -} - -// HandleTail renders GET /api/v1/system/logs/tail?n=. n defaults to -// 1000 and is capped at 10000. Invalid values silently fall back to the -// default — the endpoint is read-only and lenient. -func HandleTail(s *Service) gin.HandlerFunc { - return func(c *gin.Context) { - n := resolveTailN(c.Query("n")) - lines, err := s.Tail(n) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - c.JSON(http.StatusOK, gin.H{"lines": lines}) - } -} - -// HandleDownload streams a single log file. The :name path parameter is -// validated to be a bare filename within the configured log directory; -// anything else (path traversal, separators, missing file) returns 4xx. -func HandleDownload(s *Service) gin.HandlerFunc { - return func(c *gin.Context) { - name := c.Param("name") - f, size, err := s.Open(name) - if err != nil { - status := http.StatusBadRequest - if errors.Is(err, os.ErrNotExist) { - status = http.StatusNotFound - } - c.JSON(status, gin.H{"error": err.Error()}) - return - } - defer func() { _ = f.Close() }() - - c.Header("Content-Type", "application/octet-stream") - c.Header("Content-Length", strconv.FormatInt(size, 10)) - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%q", name)) - c.Status(http.StatusOK) - _, _ = io.Copy(c.Writer, f) - } -} - -// resolveTailN parses ?n=… , falling back to the default for empty or -// unparseable values, and clamping to [0, maxTailLines]. -func resolveTailN(raw string) int { - if raw == "" { - return defaultTailLines - } - v, err := strconv.Atoi(raw) - if err != nil { - return defaultTailLines - } - if v < 0 { - return 0 - } - if v > maxTailLines { - return maxTailLines - } - return v -} diff --git a/apps/backend/internal/system/logs/handler_test.go b/apps/backend/internal/system/logs/handler_test.go deleted file mode 100644 index 66aafbf966..0000000000 --- a/apps/backend/internal/system/logs/handler_test.go +++ /dev/null @@ -1,215 +0,0 @@ -package logs - -import ( - "encoding/json" - "fmt" - "net/http/httptest" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/gin-gonic/gin" -) - -func newRouter(svc *Service) *gin.Engine { - gin.SetMode(gin.TestMode) - r := gin.New() - r.GET("/logs", HandleList(svc)) - r.GET("/logs/tail", HandleTail(svc)) - r.GET("/logs/:name/download", HandleDownload(svc)) - return r -} - -func TestHandleList_ReturnsJSON(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "kandev.log"), []byte("hi"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - - if w.Code != 200 { - t.Fatalf("status = %d, want 200", w.Code) - } - var resp struct { - Files []FileInfo `json:"files"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode: %v", err) - } - if len(resp.Files) != 1 { - t.Fatalf("len(files) = %d, want 1", len(resp.Files)) - } - if !resp.Files[0].Current { - t.Error("expected kandev.log to be Current") - } -} - -func TestHandleTail_DefaultN(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - var sb strings.Builder - for i := 1; i <= 50; i++ { - fmt.Fprintf(&sb, "line-%d\n", i) - } - if err := os.WriteFile(path, []byte(sb.String()), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs/tail", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - - if w.Code != 200 { - t.Fatalf("status = %d body=%s", w.Code, w.Body.String()) - } - var resp struct { - Lines []string `json:"lines"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode: %v", err) - } - if len(resp.Lines) != 50 { - t.Errorf("len(lines) = %d, want 50", len(resp.Lines)) - } -} - -func TestHandleTail_ExplicitN(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - var sb strings.Builder - for i := 1; i <= 100; i++ { - fmt.Fprintf(&sb, "line-%d\n", i) - } - if err := os.WriteFile(path, []byte(sb.String()), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs/tail?n=10", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - if w.Code != 200 { - t.Fatalf("status = %d", w.Code) - } - var resp struct { - Lines []string `json:"lines"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode: %v", err) - } - if len(resp.Lines) != 10 { - t.Errorf("len(lines) = %d, want 10", len(resp.Lines)) - } - if resp.Lines[9] != "line-100" { - t.Errorf("last = %q", resp.Lines[9]) - } -} - -func TestHandleTail_CapsAt10000(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - if err := os.WriteFile(path, []byte("a\nb\n"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs/tail?n=999999", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - if w.Code != 200 { - t.Fatalf("status = %d", w.Code) - } - // Just ensure it returns 200 — the cap is internal; the small fixture - // only has 2 lines, so the cap behavior is verified indirectly: a value - // like 999999 must not cause the handler to reject the request. - var resp struct { - Lines []string `json:"lines"` - } - if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { - t.Fatalf("decode: %v", err) - } - if len(resp.Lines) != 2 { - t.Errorf("len(lines) = %d, want 2", len(resp.Lines)) - } -} - -func TestHandleTail_InvalidNFallsBackToDefault(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - if err := os.WriteFile(path, []byte("a\nb\n"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs/tail?n=notanumber", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - if w.Code != 200 { - t.Fatalf("status = %d", w.Code) - } -} - -func TestHandleDownload_StreamsFile(t *testing.T) { - dir := t.TempDir() - if err := os.WriteFile(filepath.Join(dir, "kandev.log"), []byte("payload"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs/kandev.log/download", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - - if w.Code != 200 { - t.Fatalf("status = %d", w.Code) - } - if w.Body.String() != "payload" { - t.Errorf("body = %q, want payload", w.Body.String()) - } - if cd := w.Header().Get("Content-Disposition"); !strings.Contains(cd, "kandev.log") { - t.Errorf("Content-Disposition = %q", cd) - } -} - -func TestHandleDownload_PathTraversalRejected(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - r := newRouter(svc) - - req := httptest.NewRequest("GET", "/logs/..%2Fetc%2Fpasswd/download", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - - if w.Code == 200 { - t.Fatalf("status = %d, want non-200 for traversal", w.Code) - } -} - -func TestHandleDownload_Missing404(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - r := newRouter(svc) - - // Use a name that matches the lumberjack rotation pattern so the - // allow-list lets it through; with the file missing on disk the - // handler must surface os.ErrNotExist as a 404. - req := httptest.NewRequest("GET", "/logs/kandev-2099-01-01T00-00-00.000.log/download", nil) - w := httptest.NewRecorder() - r.ServeHTTP(w, req) - - if w.Code != 404 { - t.Errorf("status = %d, want 404", w.Code) - } -} diff --git a/apps/backend/internal/system/logs/tail.go b/apps/backend/internal/system/logs/tail.go deleted file mode 100644 index 9d84a5fc7d..0000000000 --- a/apps/backend/internal/system/logs/tail.go +++ /dev/null @@ -1,197 +0,0 @@ -package logs - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "os" - "path/filepath" - "slices" - "strings" - - "github.com/kandev/kandev/internal/common/logger/buffer" -) - -// tailChunkSize is the read window stepped backward from the end of file. -const tailChunkSize = 4 * 1024 - -// Tail returns the last n lines of the most recent log activity in order -// (oldest first, newest last). When a log file is configured and non-empty -// it is read backward in fixed-size chunks so memory usage stays bounded. -// When no file is available (stdout-only logger, file missing, or file -// empty) Tail falls back to the in-memory ring buffer so the UI never shows -// "no logs" for a healthy running process. -func (s *Service) Tail(n int) ([]string, error) { - if n <= 0 { - return []string{}, nil - } - lines, err := s.tailFromFile(n) - if err != nil { - return nil, err - } - if len(lines) > 0 { - return lines, nil - } - return s.tailFromBuffer(n), nil -} - -// tailFromFile reads the active log file from disk and returns its tail. -// Returns an empty slice (no error) when there is no file to read. -func (s *Service) tailFromFile(n int) ([]string, error) { - if s.logDir == "" || s.currentName == "" { - return []string{}, nil - } - path := filepath.Join(s.logDir, s.currentName) - f, err := os.Open(path) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return []string{}, nil - } - return nil, err - } - defer func() { _ = f.Close() }() - - fi, err := f.Stat() - if err != nil { - return nil, err - } - size := fi.Size() - if size == 0 { - return []string{}, nil - } - return tailReader(f, size, n) -} - -// tailFromBuffer returns up to n formatted lines from the shared in-memory -// ring buffer. Each entry becomes a single text line in zap's "console" -// format: " [field=value ...]". -func (s *Service) tailFromBuffer(n int) []string { - if s.memBuffer == nil { - return []string{} - } - entries := s.memBuffer.Snapshot() - if len(entries) == 0 { - return []string{} - } - start := 0 - if len(entries) > n { - start = len(entries) - n - } - out := make([]string, 0, len(entries)-start) - for _, e := range entries[start:] { - out = append(out, formatBufferEntry(e)) - } - return out -} - -// formatBufferEntry renders one ring-buffer entry as a single human-readable -// line. Fields are appended as "key=value" pairs, JSON-encoded for non-string -// values so the line stays single-row in the viewer. -func formatBufferEntry(e buffer.Entry) string { - var b strings.Builder - b.WriteString(e.Timestamp.Format("2006-01-02T15:04:05.000Z")) - b.WriteString(" ") - b.WriteString(strings.ToUpper(e.Level)) - if e.Caller != "" { - b.WriteString(" ") - b.WriteString(e.Caller) - } - b.WriteString(" ") - b.WriteString(e.Message) - if len(e.Fields) > 0 { - keys := make([]string, 0, len(e.Fields)) - for k := range e.Fields { - keys = append(keys, k) - } - slices.Sort(keys) - for _, k := range keys { - fmt.Fprintf(&b, " %s=%s", k, formatFieldValue(e.Fields[k])) - } - } - if e.Stack != "" { - b.WriteString(" stack=...") - } - return b.String() -} - -func formatFieldValue(v any) string { - switch typed := v.(type) { - case string: - return typed - case error: - return typed.Error() - default: - if data, err := json.Marshal(typed); err == nil { - return string(data) - } - return fmt.Sprintf("%v", typed) - } -} - -// tailReader reads f backward in tailChunkSize windows until n line breaks -// are accumulated or the start of file is reached. -func tailReader(f io.ReaderAt, size int64, n int) ([]string, error) { - var ( - buf []byte - offset = size - ) - for offset > 0 { - readSize := int64(tailChunkSize) - if offset < readSize { - readSize = offset - } - offset -= readSize - chunk := make([]byte, readSize) - if _, err := f.ReadAt(chunk, offset); err != nil && err != io.EOF { - return nil, err - } - buf = append(chunk, buf...) - if countNewlines(buf) > n { - break - } - } - return lastNLines(buf, n), nil -} - -// countNewlines returns the number of '\n' bytes in b. -func countNewlines(b []byte) int { - count := 0 - for _, c := range b { - if c == '\n' { - count++ - } - } - return count -} - -// lastNLines splits b on '\n' and returns at most the last n lines, preserving -// order. A file that does not end with '\n' still yields its final partial -// line as the last entry. -func lastNLines(b []byte, n int) []string { - if len(b) == 0 { - return []string{} - } - // Strip a single trailing newline so we don't emit a phantom empty line. - trimmed := b - if trimmed[len(trimmed)-1] == '\n' { - trimmed = trimmed[:len(trimmed)-1] - } - // Walk from the end collecting up to n newline-delimited lines. - out := make([]string, 0, n) - end := len(trimmed) - for i := len(trimmed) - 1; i >= 0 && len(out) < n; i-- { - if trimmed[i] == '\n' { - out = append(out, string(trimmed[i+1:end])) - end = i - } - } - if len(out) < n && end > 0 { - out = append(out, string(trimmed[:end])) - } - // out is in reverse order; flip in place. - for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { - out[i], out[j] = out[j], out[i] - } - return out -} diff --git a/apps/backend/internal/system/logs/tail_buffer_test.go b/apps/backend/internal/system/logs/tail_buffer_test.go deleted file mode 100644 index a107618df5..0000000000 --- a/apps/backend/internal/system/logs/tail_buffer_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package logs - -import ( - "errors" - "testing" - "time" - - "github.com/kandev/kandev/internal/common/logger/buffer" -) - -func TestTailFromBuffer_NilBufferReturnsEmpty(t *testing.T) { - svc := &Service{} - if got := svc.tailFromBuffer(5); len(got) != 0 { - t.Fatalf("len = %d, want 0", len(got)) - } -} - -func TestTailFromBuffer_ReturnsLastNLines(t *testing.T) { - rb := buffer.New(10) - for i := 1; i <= 5; i++ { - rb.Push(buffer.Entry{ - Timestamp: time.Date(2026, 1, 1, 0, 0, i, 0, time.UTC), - Level: "info", - Message: "line", - }) - } - svc := &Service{memBuffer: rb} - lines := svc.tailFromBuffer(2) - if len(lines) != 2 { - t.Fatalf("len(lines) = %d, want 2", len(lines)) - } - if lines[0] != "2026-01-01T00:00:04.000Z INFO line" { - t.Errorf("lines[0] = %q", lines[0]) - } - if lines[1] != "2026-01-01T00:00:05.000Z INFO line" { - t.Errorf("lines[1] = %q", lines[1]) - } -} - -func TestFormatBufferEntry_IncludesCallerFieldsAndStack(t *testing.T) { - ts := time.Date(2026, 5, 1, 12, 30, 0, 0, time.UTC) - line := formatBufferEntry(buffer.Entry{ - Timestamp: ts, - Level: "warn", - Caller: "main.go:42", - Message: "boom", - Fields: map[string]any{"count": 3, "name": "alpha"}, - Stack: "trace...", - }) - if line != "2026-05-01T12:30:00.000Z WARN main.go:42 boom count=3 name=alpha stack=..." { - t.Errorf("formatBufferEntry = %q", line) - } -} - -func TestFormatFieldValue_StringErrorAndJSON(t *testing.T) { - if got := formatFieldValue("plain"); got != "plain" { - t.Errorf("string = %q", got) - } - errVal := errors.New("boom") - if got := formatFieldValue(errVal); got != "boom" { - t.Errorf("error = %q", got) - } - if got := formatFieldValue(map[string]int{"a": 1}); got != `{"a":1}` { - t.Errorf("json = %q", got) - } -} diff --git a/apps/backend/internal/system/logs/tail_test.go b/apps/backend/internal/system/logs/tail_test.go deleted file mode 100644 index 078c413e47..0000000000 --- a/apps/backend/internal/system/logs/tail_test.go +++ /dev/null @@ -1,212 +0,0 @@ -package logs - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" -) - -func writeLines(t *testing.T, path string, n int) { - t.Helper() - var sb strings.Builder - for i := 1; i <= n; i++ { - fmt.Fprintf(&sb, "line-%d\n", i) - } - if err := os.WriteFile(path, []byte(sb.String()), 0o600); err != nil { - t.Fatalf("write: %v", err) - } -} - -func TestTail_LargerThanRequested(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - writeLines(t, path, 2000) - - svc := newTestService(t, dir) - lines, err := svc.Tail(1000) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 1000 { - t.Fatalf("len(lines) = %d, want 1000", len(lines)) - } - if lines[0] != "line-1001" { - t.Errorf("first line = %q, want line-1001", lines[0]) - } - if lines[999] != "line-2000" { - t.Errorf("last line = %q, want line-2000", lines[999]) - } - // Order preserved. - for i, want := 1001, 0; want < 1000; i, want = i+1, want+1 { - if lines[want] != fmt.Sprintf("line-%d", i) { - t.Fatalf("lines[%d] = %q, want line-%d", want, lines[want], i) - break - } - } -} - -func TestTail_SmallerThanRequested(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - writeLines(t, path, 500) - - svc := newTestService(t, dir) - lines, err := svc.Tail(1000) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 500 { - t.Fatalf("len(lines) = %d, want 500", len(lines)) - } - if lines[0] != "line-1" { - t.Errorf("first = %q, want line-1", lines[0]) - } - if lines[499] != "line-500" { - t.Errorf("last = %q, want line-500", lines[499]) - } -} - -func TestTail_ZeroReturnsEmpty(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - writeLines(t, path, 100) - - svc := newTestService(t, dir) - lines, err := svc.Tail(0) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 0 { - t.Errorf("len(lines) = %d, want 0", len(lines)) - } -} - -func TestTail_NegativeReturnsEmpty(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - writeLines(t, path, 10) - - svc := newTestService(t, dir) - lines, err := svc.Tail(-5) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 0 { - t.Errorf("len(lines) = %d, want 0 for negative n", len(lines)) - } -} - -func TestTail_MissingFileReturnsEmpty(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - lines, err := svc.Tail(100) - if err != nil { - t.Fatalf("Tail on missing file should not error, got %v", err) - } - if len(lines) != 0 { - t.Errorf("len(lines) = %d, want 0", len(lines)) - } -} - -func TestTail_EmptyFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - if err := os.WriteFile(path, []byte{}, 0o600); err != nil { - t.Fatalf("write empty: %v", err) - } - svc := newTestService(t, dir) - lines, err := svc.Tail(100) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 0 { - t.Errorf("len(lines) = %d, want 0", len(lines)) - } -} - -func TestTail_FileWithoutTrailingNewline(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - // 3 lines but final has no trailing newline. - content := "a\nb\nc" - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - lines, err := svc.Tail(10) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 3 { - t.Fatalf("len(lines) = %d, want 3, got %v", len(lines), lines) - } - if lines[0] != "a" || lines[1] != "b" || lines[2] != "c" { - t.Errorf("lines = %v, want [a b c]", lines) - } -} - -func TestTail_LongLineExceedsChunkSize(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - long := strings.Repeat("x", 10_000) // larger than 4KB chunk - content := "first\n" + long + "\nlast\n" - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - lines, err := svc.Tail(3) - if err != nil { - t.Fatalf("Tail: %v", err) - } - if len(lines) != 3 { - t.Fatalf("len(lines) = %d, want 3", len(lines)) - } - if lines[0] != "first" { - t.Errorf("lines[0] = %q, want first", lines[0]) - } - if lines[1] != long { - t.Errorf("lines[1] truncated: len=%d, want %d", len(lines[1]), len(long)) - } - if lines[2] != "last" { - t.Errorf("lines[2] = %q, want last", lines[2]) - } -} - -func TestOpen_PathTraversalRejected(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - for _, name := range []string{"../etc/passwd", "../../foo.log", "subdir/x.log", "/etc/passwd", ""} { - _, _, err := svc.Open(name) - if err == nil { - t.Errorf("Open(%q) returned no error, want rejection", name) - } - } -} - -func TestOpen_NonexistentFileError(t *testing.T) { - dir := t.TempDir() - svc := newTestService(t, dir) - _, _, err := svc.Open("does-not-exist.log") - if err == nil { - t.Error("Open() returned no error for nonexistent file") - } -} - -func TestOpen_OpensExistingFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "kandev.log") - if err := os.WriteFile(path, []byte("hello"), 0o600); err != nil { - t.Fatalf("write: %v", err) - } - svc := newTestService(t, dir) - f, size, err := svc.Open("kandev.log") - if err != nil { - t.Fatalf("Open: %v", err) - } - defer func() { _ = f.Close() }() - if size != 5 { - t.Errorf("size = %d, want 5", size) - } -} diff --git a/apps/backend/internal/system/system.go b/apps/backend/internal/system/system.go index f84e787534..9524acb246 100644 --- a/apps/backend/internal/system/system.go +++ b/apps/backend/internal/system/system.go @@ -23,9 +23,10 @@ import ( "github.com/kandev/kandev/internal/system/backups" "github.com/kandev/kandev/internal/system/database" "github.com/kandev/kandev/internal/system/disk" + "github.com/kandev/kandev/internal/system/frontenderrors" "github.com/kandev/kandev/internal/system/info" "github.com/kandev/kandev/internal/system/jobs" - "github.com/kandev/kandev/internal/system/logs" + "github.com/kandev/kandev/internal/system/logbundle" "github.com/kandev/kandev/internal/system/metrics" "github.com/kandev/kandev/internal/system/restart" systemsettings "github.com/kandev/kandev/internal/system/settings" @@ -55,16 +56,17 @@ type Wiring struct { // addressable so the cmd/kandev wiring can attach callbacks (Restart) // after construction. type Service struct { - Info *info.Service - Jobs *jobs.Tracker - Disk *disk.Service - Database *database.Service - Backups *backups.Service - Logs *logs.Service - Metrics *metrics.Service - Updates *updates.Service - Restart restart.Manager - Storage *storage.Handler + Info *info.Service + Jobs *jobs.Tracker + Disk *disk.Service + Database *database.Service + Backups *backups.Service + LogBundles *logbundle.Service + FrontendErrors *frontenderrors.Service + Metrics *metrics.Service + Updates *updates.Service + Restart restart.Manager + Storage *storage.Handler // StorageRuntime owns the scheduler, reconciliation, and durable cleanup worker. StorageRuntime *storage.Runtime } @@ -90,8 +92,6 @@ func Provide(cfg *config.Config, log *logger.Logger, pool *db.Pool, eventBus bus backupsSvc := backups.NewService(dataDir, pool, tracker, log) - logDir := log.LogDirectory() - logFile := log.LogFilename() settingsStore, err := systemsettings.NewStore(pool) if err != nil { log.Error("Failed to initialize system settings store", zap.Error(err)) @@ -109,10 +109,14 @@ func Provide(cfg *config.Config, log *logger.Logger, pool *db.Pool, eventBus bus Disk: disk.NewService(homeDir, tracker, log), Database: dbSvc, Backups: backupsSvc, - Logs: logs.NewService(logDir, logFile, log), - Metrics: metricsSvc, - Updates: updates.NewService(pool, build.Version, nil, log, updatesOpts...), - Restart: restart.NewManagerFromEnv(), + LogBundles: logbundle.New(logbundle.Config{ + HomeDir: homeDir, Version: build.Version, Commit: build.Commit, + BuildTime: build.BuildTime, Log: log, + }), + FrontendErrors: frontenderrors.New(log, nil), + Metrics: metricsSvc, + Updates: updates.NewService(pool, build.Version, nil, log, updatesOpts...), + Restart: restart.NewManagerFromEnv(), } } @@ -142,9 +146,12 @@ func (s *Service) RegisterRoutes(router *gin.Engine, log *logger.Logger) { backups.RegisterRoutes(g, s.Backups) - g.GET("/logs", logs.HandleList(s.Logs)) - g.GET("/logs/tail", logs.HandleTail(s.Logs)) - g.GET("/logs/:name/download", logs.HandleDownload(s.Logs)) + if s.FrontendErrors != nil { + g.POST("/logs/frontend-errors", frontenderrors.Handle(s.FrontendErrors)) + } + if s.LogBundles != nil { + logbundle.RegisterRoutes(g, s.LogBundles) + } if s.Metrics != nil { metrics.RegisterRoutes(g, s.Metrics) @@ -164,6 +171,9 @@ func (s *Service) RegisterRoutes(router *gin.Engine, log *logger.Logger) { // StartBackground kicks off the updates poller goroutine. The poller // stops when ctx is cancelled. func (s *Service) StartBackground(ctx context.Context) { + if s.LogBundles != nil { + s.LogBundles.Start(ctx) + } if s.Updates != nil { s.Updates.StartPoller(ctx) } @@ -174,6 +184,9 @@ func (s *Service) StartBackground(ctx context.Context) { // StopBackground joins owned storage background workers. func (s *Service) StopBackground() { + if s.LogBundles != nil { + s.LogBundles.Stop() + } if s.StorageRuntime != nil { s.StorageRuntime.Stop() } diff --git a/apps/backend/internal/system/system_routes_test.go b/apps/backend/internal/system/system_routes_test.go new file mode 100644 index 0000000000..cd9c38398c --- /dev/null +++ b/apps/backend/internal/system/system_routes_test.go @@ -0,0 +1,41 @@ +package system + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/system/frontenderrors" + "go.uber.org/zap" +) + +func TestRegisterRoutesAllowsMemberFrontendErrorReports(t *testing.T) { + gin.SetMode(gin.TestMode) + log, err := logger.NewFromZap(zap.NewNop()) + if err != nil { + t.Fatal(err) + } + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "member-1", Role: authn.RoleMember}) + c.Next() + }) + service := &Service{FrontendErrors: frontenderrors.New(log, nil)} + service.RegisterRoutes(router, log) + + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/system/logs/frontend-errors", + bytes.NewBufferString(`{"source":"sonner","title":"visible error"}`), + ) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + router.ServeHTTP(response, request) + if response.Code != http.StatusNoContent { + t.Fatalf("member report status = %d, body = %s", response.Code, response.Body.String()) + } +} diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go index 6029026bac..da449ad1fa 100644 --- a/apps/backend/pkg/websocket/actions.go +++ b/apps/backend/pkg/websocket/actions.go @@ -381,6 +381,7 @@ const ( ActionMCPPublishReviewFindings = "mcp.publish_review_findings" ActionMCPClarificationTimeout = "mcp.clarification_timeout" ActionMCPSetTaskTitle = "mcp.set_task_title" + ActionMCPGetDiagnosticBundle = "mcp.get_diagnostic_bundle" // Office task handoffs (cross-task context). ActionMCPListRelatedTasks = "mcp.list_related_tasks" diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts index b4c12c3dd9..762d6b1e40 100644 --- a/apps/cli/src/cli.ts +++ b/apps/cli/src/cli.ts @@ -42,8 +42,8 @@ Options: --port Port for the Go backend (the URL kandev opens on in start/run). Alias for --backend-port. Also reads KANDEV_PORT or KANDEV_BACKEND_PORT. - --verbose, -v Show info logs from backend + web. - --debug Show debug logs + agent message dumps. + --verbose, -v Show backend info logs on stdout (also retained in its file). + --debug Retain backend debug logs in its file + agent message dumps. --headless Skip opening the browser. Used by service units. --help, -h Show help. diff --git a/apps/cli/src/dev.ts b/apps/cli/src/dev.ts index bde8403d62..f79134bcbb 100644 --- a/apps/cli/src/dev.ts +++ b/apps/cli/src/dev.ts @@ -6,7 +6,14 @@ import { devKandevHome, HEALTH_TIMEOUT_MS_DEV } from "./constants"; import { resolveHealthTimeoutMs, waitForHealth, waitForUrlReady } from "./health"; import { isInsideKandevTask } from "./kandev-env"; import { createProcessSupervisor } from "./process"; -import { buildBackendEnv, buildWebEnv, logStartupInfo, pickPorts } from "./shared"; +import { + buildBackendEnv, + buildWebEnv, + createOutputRingBuffer, + logStartupInfo, + pickPorts, + resolveBackendLogLevels, +} from "./shared"; import { launchRestartableBackend } from "./supervisor/backend"; import { launchWebApp, openBrowser } from "./web"; @@ -36,10 +43,15 @@ export async function runDev({ repoRoot, backendPort, webPort }: DevOptions): Pr } } - const backendEnv = buildBackendEnv({ ports, extra }); + const levels = resolveBackendLogLevels(); + const backendEnv = buildBackendEnv({ + ports, + logLevel: levels.file, + consoleLogLevel: levels.console, + extra, + }); const webEnv = buildWebEnv({ ports, debug: true }); - const logLevel = - process.env.KANDEV_LOGGING_LEVEL?.trim() || process.env.KANDEV_LOG_LEVEL?.trim() || "info"; + const logLevel = levels.file; const webUrl = `http://localhost:${ports.webPort}`; const appUrl = ports.backendUrl; @@ -58,6 +70,7 @@ export async function runDev({ repoRoot, backendPort, webPort }: DevOptions): Pr path.join("apps", "backend"), "dev", ]); + const output = createOutputRingBuffer(process.stdout); const backend = await launchRestartableBackend({ command: backendCmd, args: backendArgs, @@ -66,13 +79,20 @@ export async function runDev({ repoRoot, backendPort, webPort }: DevOptions): Pr homeDir: backendEnv.KANDEV_HOME_DIR ?? devKandevHome(repoRoot), ports, mode: "dev", - stdio: "inherit", + stdio: ["ignore", "pipe", "inherit"], supervisor, + onProcess: (proc) => output.attach(proc.stdout), }); const healthTimeoutMs = resolveHealthTimeoutMs(HEALTH_TIMEOUT_MS_DEV); console.log("[kandev] starting backend..."); - await waitForHealth(ports.backendUrl, backend.proc, healthTimeoutMs); + await waitForHealth(ports.backendUrl, backend.proc, healthTimeoutMs, () => { + const buffered = output.read(); + if (buffered.trim().length === 0) return; + console.error("[kandev] --- backend stdout (last captured output) ---"); + console.error(buffered.trimEnd()); + console.error("[kandev] --- end backend stdout ---"); + }); console.log(`[kandev] backend ready at ${ports.backendUrl}`); console.log("[kandev] starting web..."); diff --git a/apps/cli/src/run.ts b/apps/cli/src/run.ts index 6acb2e3d5e..d68af4567e 100644 --- a/apps/cli/src/run.ts +++ b/apps/cli/src/run.ts @@ -19,16 +19,21 @@ import { sortVersionsDesc } from "./version"; import { pickAvailablePort } from "./ports"; import { createProcessSupervisor } from "./process"; import { resolveRuntime, validateBundle } from "./runtime"; -import { buildBackendEnv, logStartupInfo } from "./shared"; +import { + buildBackendEnv, + createOutputRingBuffer, + logStartupInfo, + resolveBackendLogLevels, +} from "./shared"; import { launchRestartableBackend } from "./supervisor/backend"; import { openBrowser } from "./web"; export type RunOptions = { runtimeVersion?: string; backendPort?: number; - /** Show info logs from backend + web */ + /** Show backend info logs on stdout */ verbose?: boolean; - /** Show debug logs + agent message dumps */ + /** Retain backend debug logs in its file + agent message dumps */ debug?: boolean; /** Skip browser open. Set by service units. */ headless?: boolean; @@ -43,7 +48,6 @@ type PreparedBundle = { agentctlPort: number; dbPath: string; logLevel: string; - showOutput: boolean; }; /** @@ -181,9 +185,8 @@ async function prepareBundleForLaunch({ const actualBackendPort = backendPort ?? (await pickAvailablePort(DEFAULT_BACKEND_PORT)); const agentctlPort = await pickAvailablePort(DEFAULT_AGENTCTL_PORT); const backendUrl = `http://localhost:${actualBackendPort}`; - const showOutput = verbose || debug; - const logLevel = - process.env.KANDEV_LOG_LEVEL?.trim() || (debug ? "debug" : verbose ? "info" : "warn"); + const levels = resolveBackendLogLevels({ verbose, debug }); + const logLevel = levels.file; const dataDir = resolveDataDir(); fs.mkdirSync(dataDir, { recursive: true }); @@ -198,6 +201,7 @@ async function prepareBundleForLaunch({ backendUrl, }, logLevel, + consoleLogLevel: levels.console, webProxy: false, extra: { KANDEV_DATABASE_PATH: dbPath, @@ -214,7 +218,6 @@ async function prepareBundleForLaunch({ agentctlPort, dbPath, logLevel, - showOutput, }; } @@ -227,14 +230,9 @@ export function attachRingBuffer( stream: NodeJS.ReadableStream | null, maxChars = 64 * 1024, ): () => string { - let buf = ""; - stream?.on("data", (chunk: Buffer | string) => { - buf += typeof chunk === "string" ? chunk : chunk.toString("utf8"); - if (buf.length > maxChars) { - buf = buf.slice(buf.length - maxChars); - } - }); - return () => buf; + const capture = createOutputRingBuffer(null, maxChars); + capture.attach(stream); + return capture.read; } async function launchBundle(prepared: PreparedBundle): Promise<{ @@ -255,6 +253,7 @@ async function launchBundle(prepared: PreparedBundle): Promise<{ const supervisor = createProcessSupervisor(); supervisor.attachSignalHandlers(); + const output = createOutputRingBuffer(process.stdout); const backend = await launchRestartableBackend({ command: prepared.backendBin, @@ -268,17 +267,17 @@ async function launchBundle(prepared: PreparedBundle): Promise<{ backendUrl: prepared.backendUrl, }, mode: "run", - stdio: prepared.showOutput ? ["ignore", "inherit", "inherit"] : ["ignore", "pipe", "inherit"], + stdio: ["ignore", "pipe", "inherit"], supervisor, + onProcess: (proc) => output.attach(proc.stdout), }); const backendProc = backend.proc; - const readBuffered = prepared.showOutput ? () => "" : attachRingBuffer(backendProc.stdout); let dumped = false; const dumpBackendLogs = (): void => { if (dumped) return; dumped = true; - const buffered = readBuffered(); + const buffered = output.read(); if (buffered.trim().length === 0) return; console.error("[kandev] --- backend stdout (last captured output) ---"); console.error(buffered.trimEnd()); diff --git a/apps/cli/src/shared.test.ts b/apps/cli/src/shared.test.ts index 56d1555e29..1db4082059 100644 --- a/apps/cli/src/shared.test.ts +++ b/apps/cli/src/shared.test.ts @@ -1,13 +1,16 @@ import os from "node:os"; +import { PassThrough } from "node:stream"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildBackendEnv, buildWebEnv, + createOutputRingBuffer, listHostNetworkAddresses, logStartupInfo, networkUrlsForPort, + resolveBackendLogLevels, type PortConfig, } from "./shared"; @@ -83,6 +86,41 @@ describe("buildBackendEnv", () => { expect(env.KANDEV_WEB_INTERNAL_URL).toBeUndefined(); }); + + it("uses independent file and console thresholds", () => { + expect(resolveBackendLogLevels({ env: {} })).toEqual({ file: "info", console: "warn" }); + expect(resolveBackendLogLevels({ debug: true, env: {} })).toEqual({ + file: "debug", + console: "warn", + }); + expect(resolveBackendLogLevels({ verbose: true, env: {} })).toEqual({ + file: "info", + console: "info", + }); + expect(resolveBackendLogLevels({ debug: true, env: { KANDEV_LOG_LEVEL: "error" } })).toEqual({ + file: "error", + console: "warn", + }); + }); + + it("tees backend output while retaining a bounded startup tail", () => { + const stream = new PassThrough(); + let forwarded = ""; + const capture = createOutputRingBuffer( + { + write: (chunk) => { + forwarded += chunk.toString(); + return true; + }, + }, + 5, + ); + capture.attach(stream); + stream.write("hello"); + stream.write("world"); + expect(forwarded).toBe("helloworld"); + expect(capture.read()).toBe("world"); + }); }); describe("listHostNetworkAddresses", () => { diff --git a/apps/cli/src/shared.ts b/apps/cli/src/shared.ts index 5e6afcbe5a..596bebc678 100644 --- a/apps/cli/src/shared.ts +++ b/apps/cli/src/shared.ts @@ -54,6 +54,8 @@ export type BackendEnvOptions = { ports: PortConfig; /** Log level: debug, info, warn, error (default: info) */ logLevel?: string; + /** Backend stdout threshold. Normal/debug use warn; verbose uses info. */ + consoleLogLevel?: "warn" | "info"; /** Route browser pages through an external dev web server. Production serves the SPA from Go. */ webProxy?: boolean; /** Additional environment variables to merge */ @@ -67,7 +69,7 @@ export type BackendEnvOptions = { * @returns Environment object for the backend process */ export function buildBackendEnv(options: BackendEnvOptions): NodeJS.ProcessEnv { - const { ports, logLevel, webProxy = true, extra } = options; + const { ports, logLevel, consoleLogLevel = "warn", webProxy = true, extra } = options; if (webProxy && ports.webPort === undefined) { throw new Error("webProxy requires a web port"); } @@ -77,6 +79,7 @@ export function buildBackendEnv(options: BackendEnvOptions): NodeJS.ProcessEnv { ...(webProxy ? { KANDEV_WEB_INTERNAL_URL: `http://localhost:${ports.webPort}` } : {}), KANDEV_AGENT_STANDALONE_PORT: String(ports.agentctlPort), ...(logLevel ? { KANDEV_LOG_LEVEL: logLevel } : {}), + KANDEV_CONSOLE_LOG_LEVEL: consoleLogLevel, ...extra, }; if (!webProxy) { @@ -85,6 +88,37 @@ export function buildBackendEnv(options: BackendEnvOptions): NodeJS.ProcessEnv { return env; } +export function resolveBackendLogLevels({ + verbose = false, + debug = false, + env = process.env, +}: { verbose?: boolean; debug?: boolean; env?: NodeJS.ProcessEnv } = {}): { + file: string; + console: "warn" | "info"; +} { + return { + file: env.KANDEV_LOG_LEVEL?.trim() || (debug ? "debug" : "info"), + console: verbose ? "info" : "warn", + }; +} + +export function createOutputRingBuffer( + sink: Pick | null, + maxChars = 64 * 1024, +): { attach: (stream: NodeJS.ReadableStream | null) => void; read: () => string } { + let buffer = ""; + const attach = (stream: NodeJS.ReadableStream | null): void => { + stream?.on("data", (chunk: Buffer | string) => { + sink?.write(chunk); + buffer += typeof chunk === "string" ? chunk : chunk.toString("utf8"); + if (buffer.length > maxChars) { + buffer = buffer.slice(buffer.length - maxChars); + } + }); + }; + return { attach, read: () => buffer }; +} + export type WebEnvOptions = { ports: PortConfig; /** Set NODE_ENV to production */ diff --git a/apps/cli/src/start.ts b/apps/cli/src/start.ts index 8deb72a6e0..02b3d13688 100644 --- a/apps/cli/src/start.ts +++ b/apps/cli/src/start.ts @@ -22,7 +22,13 @@ import { import { resolveHealthTimeoutMs, waitForHealth } from "./health"; import { getBinaryName } from "./platform"; import { createProcessSupervisor } from "./process"; -import { buildBackendEnv, logStartupInfo, pickBackendPorts } from "./shared"; +import { + buildBackendEnv, + createOutputRingBuffer, + logStartupInfo, + pickBackendPorts, + resolveBackendLogLevels, +} from "./shared"; import { launchRestartableBackend } from "./supervisor/backend"; import { openBrowser } from "./web"; @@ -31,9 +37,9 @@ export type StartOptions = { repoRoot: string; /** Optional preferred backend port (finds available port if not specified) */ backendPort?: number; - /** Show info logs from backend + web */ + /** Show backend info logs on stdout */ verbose?: boolean; - /** Show debug logs + agent message dumps */ + /** Retain backend debug logs in its file + agent message dumps */ debug?: boolean; /** Skip browser open. Set by service units and preview environments. */ headless?: boolean; @@ -65,18 +71,19 @@ export async function runStart({ throw new Error("Backend binary not found. Run `make build` first."); } - // Production mode: use warn log level for clean output unless verbose/debug - const showOutput = verbose || debug; + // File logs retain info by default. Verbose only changes the stdout sink; + // debug lowers the file threshold while stdout remains warn+. fs.mkdirSync(resolveDataDir(), { recursive: true }); // The data dir holds the SQLite DB; keep it owner-only even if it pre-existed // with a looser umask-derived mode. fs.chmodSync(resolveDataDir(), 0o700); const dbPath = resolveDatabasePath(); - const logLevel = - process.env.KANDEV_LOG_LEVEL?.trim() || (debug ? "debug" : verbose ? "info" : "warn"); + const levels = resolveBackendLogLevels({ verbose, debug }); + const logLevel = levels.file; const backendEnv = buildBackendEnv({ ports, logLevel, + consoleLogLevel: levels.console, webProxy: false, extra: { KANDEV_DATABASE_PATH: dbPath, @@ -94,8 +101,7 @@ export async function runStart({ const supervisor = createProcessSupervisor(); supervisor.attachSignalHandlers(); - // Start backend: ignore stdin, show stdout only in verbose/debug mode, always show stderr - // Stderr is always inherited to ensure error messages are visible immediately (no pipe buffering) + const output = createOutputRingBuffer(process.stdout); const backend = await launchRestartableBackend({ command: backendBin, args: [], @@ -104,13 +110,20 @@ export async function runStart({ homeDir: resolveKandevHomeDir(), ports, mode: "start", - stdio: showOutput ? ["ignore", "inherit", "inherit"] : ["ignore", "ignore", "inherit"], + stdio: ["ignore", "pipe", "inherit"], supervisor, + onProcess: (proc) => output.attach(proc.stdout), }); const healthTimeoutMs = resolveHealthTimeoutMs(HEALTH_TIMEOUT_MS_RELEASE); console.log("[kandev] starting backend..."); - await waitForHealth(ports.backendUrl, backend.proc, healthTimeoutMs); + await waitForHealth(ports.backendUrl, backend.proc, healthTimeoutMs, () => { + const buffered = output.read(); + if (buffered.trim().length === 0) return; + console.error("[kandev] --- backend stdout (last captured output) ---"); + console.error(buffered.trimEnd()); + console.error("[kandev] --- end backend stdout ---"); + }); console.log(`[kandev] backend ready at ${ports.backendUrl}`); console.log("[kandev] open: " + ports.backendUrl); diff --git a/apps/cli/src/supervisor/backend.ts b/apps/cli/src/supervisor/backend.ts index ce238d6c20..1bed7e0346 100644 --- a/apps/cli/src/supervisor/backend.ts +++ b/apps/cli/src/supervisor/backend.ts @@ -25,6 +25,7 @@ export async function launchRestartableBackend({ mode, stdio, supervisor, + onProcess, }: { command: string; args: string[]; @@ -35,9 +36,11 @@ export async function launchRestartableBackend({ mode: string; stdio: StdioOptions; supervisor: ReturnType; + onProcess?: (proc: ChildProcess) => void; }): Promise { if (!shouldUseSupervisor(env)) { const proc = spawn(command, args, { cwd, env, stdio }); + onProcess?.(proc); supervisor.children.push(proc as ChildLike); attachBackendExitHandler(proc, supervisor); return { proc, control: null, env }; @@ -63,6 +66,7 @@ export async function launchRestartableBackend({ }); }; const proc = child.start(); + onProcess?.(proc); supervisor.children.push(proc as ChildLike); attachExit(proc); @@ -70,6 +74,7 @@ export async function launchRestartableBackend({ restarting = true; try { const next = await child.restart(); + onProcess?.(next); supervisor.children.push(next as ChildLike); attachExit(next); } finally { diff --git a/apps/cli/src/supervisor/manifest.test.ts b/apps/cli/src/supervisor/manifest.test.ts index 4b49f146f6..90aead64e6 100644 --- a/apps/cli/src/supervisor/manifest.test.ts +++ b/apps/cli/src/supervisor/manifest.test.ts @@ -71,10 +71,14 @@ describe("supervisor launch manifest", () => { expect( allowedEnv({ KANDEV_HOME_DIR: "/tmp/kandev", + KANDEV_CONSOLE_LOG_LEVEL: "warn", AWS_SECRET_ACCESS_KEY: "secret", GITHUB_TOKEN: "secret", }), - ).toEqual({ KANDEV_HOME_DIR: "/tmp/kandev" }); + ).toEqual({ + KANDEV_HOME_DIR: "/tmp/kandev", + KANDEV_CONSOLE_LOG_LEVEL: "warn", + }); }); }); diff --git a/apps/cli/src/supervisor/manifest.ts b/apps/cli/src/supervisor/manifest.ts index 0724bf1ba9..46b87db80a 100644 --- a/apps/cli/src/supervisor/manifest.ts +++ b/apps/cli/src/supervisor/manifest.ts @@ -27,6 +27,7 @@ const ENV_ALLOWLIST = new Set([ "KANDEV_WEB_INTERNAL_URL", "KANDEV_AGENT_STANDALONE_PORT", "KANDEV_LOG_LEVEL", + "KANDEV_CONSOLE_LOG_LEVEL", "KANDEV_DEBUG_DEV_MODE", "KANDEV_DEBUG_AGENT_MESSAGES", "KANDEV_DEBUG_PPROF_ENABLED", diff --git a/apps/web/app/office/agents/[id]/components/agent-channels-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-channels-tab.tsx index b0fa17c7a3..e07ff061f9 100644 --- a/apps/web/app/office/agents/[id]/components/agent-channels-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-channels-tab.tsx @@ -15,7 +15,7 @@ import { Input } from "@kandev/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@kandev/ui/dialog"; import { Tooltip, TooltipContent, TooltipTrigger } from "@kandev/ui/tooltip"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import type { AgentProfile } from "@/lib/state/slices/office/types"; import * as officeApi from "@/lib/api/domains/office-api"; diff --git a/apps/web/app/office/agents/[id]/components/agent-configuration-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-configuration-tab.tsx index 81d1164d99..f75b6bad2e 100644 --- a/apps/web/app/office/agents/[id]/components/agent-configuration-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-configuration-tab.tsx @@ -7,7 +7,7 @@ import { Label } from "@kandev/ui/label"; import { Button } from "@kandev/ui/button"; import { Badge } from "@kandev/ui/badge"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { updateAgentProfile } from "@/lib/api/domains/office-api"; import type { AgentProfile, AgentRole } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/agents/[id]/components/agent-instructions-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-instructions-tab.tsx index 94eca9f627..17b2c3c301 100644 --- a/apps/web/app/office/agents/[id]/components/agent-instructions-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-instructions-tab.tsx @@ -1,7 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import type { AgentProfile } from "@/lib/state/slices/office/types"; import * as officeApi from "@/lib/api/domains/office-api"; import { InstructionFileList } from "./instruction-file-list"; diff --git a/apps/web/app/office/agents/[id]/components/agent-memory-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-memory-tab.tsx index 6e45cbfb62..2cd651635b 100644 --- a/apps/web/app/office/agents/[id]/components/agent-memory-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-memory-tab.tsx @@ -13,7 +13,7 @@ import { Input } from "@kandev/ui/input"; import { Badge } from "@kandev/ui/badge"; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@kandev/ui/dialog"; import { Tooltip, TooltipContent, TooltipTrigger } from "@kandev/ui/tooltip"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import type { AgentProfile } from "@/lib/state/slices/office/types"; import * as officeApi from "@/lib/api/domains/office-api"; diff --git a/apps/web/app/office/agents/[id]/components/agent-overview-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-overview-tab.tsx index c28e755381..b9a777b89b 100644 --- a/apps/web/app/office/agents/[id]/components/agent-overview-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-overview-tab.tsx @@ -7,7 +7,7 @@ import { Label } from "@kandev/ui/label"; import { Button } from "@kandev/ui/button"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; import { IconRefresh } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { updateAgentProfile, getAgentUtilization } from "@/lib/api/domains/office-api"; import type { AgentProfile, AgentRole, ProviderUsage } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/agents/[id]/components/agent-permissions-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-permissions-tab.tsx index 103bb54971..c7dd38447b 100644 --- a/apps/web/app/office/agents/[id]/components/agent-permissions-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-permissions-tab.tsx @@ -7,7 +7,7 @@ import { Label } from "@kandev/ui/label"; import { Input } from "@kandev/ui/input"; import { Badge } from "@kandev/ui/badge"; import { Button } from "@kandev/ui/button"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { updateAgentProfile } from "@/lib/api/domains/office-api"; import type { AgentProfile } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/agents/[id]/components/agent-routing-card.tsx b/apps/web/app/office/agents/[id]/components/agent-routing-card.tsx index 020633d49f..68effb94cd 100644 --- a/apps/web/app/office/agents/[id]/components/agent-routing-card.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-routing-card.tsx @@ -1,7 +1,7 @@ "use client"; import { useEffect, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Card, CardContent, CardHeader, CardTitle } from "@kandev/ui/card"; import { Switch } from "@kandev/ui/switch"; import { Button } from "@kandev/ui/button"; diff --git a/apps/web/app/office/agents/[id]/components/agent-skills-tab.tsx b/apps/web/app/office/agents/[id]/components/agent-skills-tab.tsx index 23c433e00e..13fabee0d4 100644 --- a/apps/web/app/office/agents/[id]/components/agent-skills-tab.tsx +++ b/apps/web/app/office/agents/[id]/components/agent-skills-tab.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, useCallback } from "react"; import Link from "@/components/routing/app-link"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Badge } from "@kandev/ui/badge"; import { Button } from "@kandev/ui/button"; import { Checkbox } from "@kandev/ui/checkbox"; diff --git a/apps/web/app/office/agents/components/create-agent-dialog.tsx b/apps/web/app/office/agents/components/create-agent-dialog.tsx index b6a72feb8c..bd2ef5df49 100644 --- a/apps/web/app/office/agents/components/create-agent-dialog.tsx +++ b/apps/web/app/office/agents/components/create-agent-dialog.tsx @@ -6,7 +6,7 @@ import { Input } from "@kandev/ui/input"; import { Label } from "@kandev/ui/label"; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogFooter } from "@kandev/ui/dialog"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { createAgentProfile } from "@/lib/api/domains/office-api"; import type { AgentRole, AgentProfile } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/components/new-task-dialog.tsx b/apps/web/app/office/components/new-task-dialog.tsx index c6c886ba19..47cc7e9c91 100644 --- a/apps/web/app/office/components/new-task-dialog.tsx +++ b/apps/web/app/office/components/new-task-dialog.tsx @@ -6,7 +6,7 @@ import { Badge } from "@kandev/ui/badge"; import { Textarea } from "@kandev/ui/textarea"; import { Dialog, DialogContent, DialogHeader, DialogFooter, DialogTitle } from "@kandev/ui/dialog"; import { Tooltip, TooltipContent, TooltipTrigger } from "@kandev/ui/tooltip"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { createTask } from "@/lib/api/domains/kanban-api"; import { useIssueDraft, type IssueDraft } from "./new-task-draft"; diff --git a/apps/web/app/office/inbox/inbox-page-client.tsx b/apps/web/app/office/inbox/inbox-page-client.tsx index 8947cc896f..6bcda7c0a0 100644 --- a/apps/web/app/office/inbox/inbox-page-client.tsx +++ b/apps/web/app/office/inbox/inbox-page-client.tsx @@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { IconSearch } from "@tabler/icons-react"; import { Tabs, TabsList, TabsTrigger } from "@kandev/ui/tabs"; import { Input } from "@kandev/ui/input"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { useOfficeRefetch } from "@/hooks/use-office-refetch"; import * as officeApi from "@/lib/api/domains/office-api"; diff --git a/apps/web/app/office/projects/[id]/page.tsx b/apps/web/app/office/projects/[id]/page.tsx index 97fc40e55f..2033b07a99 100644 --- a/apps/web/app/office/projects/[id]/page.tsx +++ b/apps/web/app/office/projects/[id]/page.tsx @@ -6,7 +6,7 @@ import { useRouter } from "@/lib/routing/client-router"; import { IconChevronRight, IconTrash } from "@tabler/icons-react"; import { Button } from "@kandev/ui/button"; import { Separator } from "@kandev/ui/separator"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { getProject, deleteProject } from "@/lib/api/domains/office-api"; import type { Project } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/projects/[id]/project-executor-section.tsx b/apps/web/app/office/projects/[id]/project-executor-section.tsx index e42ecbaaeb..d4d4e8acb9 100644 --- a/apps/web/app/office/projects/[id]/project-executor-section.tsx +++ b/apps/web/app/office/projects/[id]/project-executor-section.tsx @@ -6,7 +6,7 @@ import { Input } from "@kandev/ui/input"; import { Label } from "@kandev/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; import { IconDeviceFloppy } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { updateProject } from "@/lib/api/domains/office-api"; import { useAppStore } from "@/components/state-provider"; import type { Project } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/projects/[id]/project-header.tsx b/apps/web/app/office/projects/[id]/project-header.tsx index abe0c5313e..69ecdf935a 100644 --- a/apps/web/app/office/projects/[id]/project-header.tsx +++ b/apps/web/app/office/projects/[id]/project-header.tsx @@ -6,7 +6,7 @@ import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; import { Textarea } from "@kandev/ui/textarea"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { updateProject } from "@/lib/api/domains/office-api"; import { useAppStore } from "@/components/state-provider"; import type { Project, ProjectStatus } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/projects/[id]/project-repos-section.tsx b/apps/web/app/office/projects/[id]/project-repos-section.tsx index 550e79952f..cc0d14e688 100644 --- a/apps/web/app/office/projects/[id]/project-repos-section.tsx +++ b/apps/web/app/office/projects/[id]/project-repos-section.tsx @@ -1,7 +1,7 @@ "use client"; import { useCallback, useMemo } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { updateProject } from "@/lib/api/domains/office-api"; import { useAppStore } from "@/components/state-provider"; import { useRepositories } from "@/hooks/domains/workspace/use-repositories"; diff --git a/apps/web/app/office/projects/create-project-dialog.tsx b/apps/web/app/office/projects/create-project-dialog.tsx index 1c9ba13628..5a8ef031bf 100644 --- a/apps/web/app/office/projects/create-project-dialog.tsx +++ b/apps/web/app/office/projects/create-project-dialog.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useCallback } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@kandev/ui/dialog"; import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; diff --git a/apps/web/app/office/routines/[id]/routine-detail-view.tsx b/apps/web/app/office/routines/[id]/routine-detail-view.tsx index a0ee4ebcef..052463cefc 100644 --- a/apps/web/app/office/routines/[id]/routine-detail-view.tsx +++ b/apps/web/app/office/routines/[id]/routine-detail-view.tsx @@ -10,7 +10,7 @@ import { Label } from "@kandev/ui/label"; import { Textarea } from "@kandev/ui/textarea"; import { Card, CardContent, CardHeader, CardTitle } from "@kandev/ui/card"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { updateRoutine, diff --git a/apps/web/app/office/routines/routines-content.tsx b/apps/web/app/office/routines/routines-content.tsx index b7c44479ab..ef4ccd9499 100644 --- a/apps/web/app/office/routines/routines-content.tsx +++ b/apps/web/app/office/routines/routines-content.tsx @@ -4,7 +4,7 @@ import { useCallback, useEffect, useState } from "react"; import { Button } from "@kandev/ui/button"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@kandev/ui/tabs"; import { IconPlus } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { listRoutines, diff --git a/apps/web/app/office/setup/setup-wizard.tsx b/apps/web/app/office/setup/setup-wizard.tsx index 6e30756fcc..e123d6a5a9 100644 --- a/apps/web/app/office/setup/setup-wizard.tsx +++ b/apps/web/app/office/setup/setup-wizard.tsx @@ -2,7 +2,7 @@ import { useState, useCallback } from "react"; import { useRouter } from "@/lib/routing/client-router"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { completeOnboarding, importFromFS, diff --git a/apps/web/app/office/tasks/use-paginated-tasks.ts b/apps/web/app/office/tasks/use-paginated-tasks.ts index f2a4d38238..3ba4d78630 100644 --- a/apps/web/app/office/tasks/use-paginated-tasks.ts +++ b/apps/web/app/office/tasks/use-paginated-tasks.ts @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { listTasks, type ListTasksParams } from "@/lib/api/domains/office-extended-api"; import type { TaskFilterState, TaskSortDir, TaskSortField } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/workspace/activity/activity-feed.tsx b/apps/web/app/office/workspace/activity/activity-feed.tsx index 56c042e1d5..a00ba997f4 100644 --- a/apps/web/app/office/workspace/activity/activity-feed.tsx +++ b/apps/web/app/office/workspace/activity/activity-feed.tsx @@ -2,7 +2,7 @@ import { useEffect, useState } from "react"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { listActivity } from "@/lib/api/domains/office-api"; import type { ActivityEntry } from "@/lib/state/slices/office/types"; import { ActivityRow } from "./activity-row"; diff --git a/apps/web/app/office/workspace/costs/budgets-tab.tsx b/apps/web/app/office/workspace/costs/budgets-tab.tsx index 307763d14d..7ca0356c0a 100644 --- a/apps/web/app/office/workspace/costs/budgets-tab.tsx +++ b/apps/web/app/office/workspace/costs/budgets-tab.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from "react"; import { Button } from "@kandev/ui/button"; import { IconPlus } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { listBudgets, deleteBudget } from "@/lib/api/domains/office-api"; import type { BudgetPolicy } from "@/lib/state/slices/office/types"; import { BudgetPolicyCard } from "./budget-policy-card"; diff --git a/apps/web/app/office/workspace/costs/cost-overview.tsx b/apps/web/app/office/workspace/costs/cost-overview.tsx index ab82c78055..c9d0630fdb 100644 --- a/apps/web/app/office/workspace/costs/cost-overview.tsx +++ b/apps/web/app/office/workspace/costs/cost-overview.tsx @@ -9,7 +9,7 @@ import { IconCpu, IconBuilding, } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { getCostsBreakdown } from "@/lib/api/domains/office-api"; import { MetricCard } from "../../components/metric-card"; import type { CostBreakdownItem } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/office/workspace/costs/create-budget-form.tsx b/apps/web/app/office/workspace/costs/create-budget-form.tsx index 1417c6b449..357da89c03 100644 --- a/apps/web/app/office/workspace/costs/create-budget-form.tsx +++ b/apps/web/app/office/workspace/costs/create-budget-form.tsx @@ -5,7 +5,7 @@ import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; import { Card, CardContent } from "@kandev/ui/card"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { createBudget } from "@/lib/api/domains/office-api"; type Props = { diff --git a/apps/web/app/office/workspace/routing/page.tsx b/apps/web/app/office/workspace/routing/page.tsx index 15ec267867..0a79299131 100644 --- a/apps/web/app/office/workspace/routing/page.tsx +++ b/apps/web/app/office/workspace/routing/page.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo, useState } from "react"; import { IconDeviceFloppy } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Button } from "@kandev/ui/button"; import { useAppStore } from "@/components/state-provider"; import { ApiError } from "@/lib/api/client"; diff --git a/apps/web/app/office/workspace/settings/components/danger-zone-section.tsx b/apps/web/app/office/workspace/settings/components/danger-zone-section.tsx index ca4cf38a83..1a27c0d70b 100644 --- a/apps/web/app/office/workspace/settings/components/danger-zone-section.tsx +++ b/apps/web/app/office/workspace/settings/components/danger-zone-section.tsx @@ -3,7 +3,7 @@ import { useState } from "react"; import { useRouter } from "@/lib/routing/client-router"; import { IconTrash } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; import { Label } from "@kandev/ui/label"; diff --git a/apps/web/app/office/workspace/settings/components/git-section.tsx b/apps/web/app/office/workspace/settings/components/git-section.tsx index 0a193ea065..ace432921b 100644 --- a/apps/web/app/office/workspace/settings/components/git-section.tsx +++ b/apps/web/app/office/workspace/settings/components/git-section.tsx @@ -12,7 +12,7 @@ import { import { Input } from "@kandev/ui/input"; import { Button } from "@kandev/ui/button"; import { Badge } from "@kandev/ui/badge"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import * as officeApi from "@/lib/api/domains/office-api"; import type { GitStatusData } from "@/lib/api/domains/office-api"; diff --git a/apps/web/app/office/workspace/settings/components/settings-content.tsx b/apps/web/app/office/workspace/settings/components/settings-content.tsx index 84601e49b4..e892dc278c 100644 --- a/apps/web/app/office/workspace/settings/components/settings-content.tsx +++ b/apps/web/app/office/workspace/settings/components/settings-content.tsx @@ -3,7 +3,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import Image from "@/components/routing/app-image"; import { IconUpload, IconDeviceFloppy } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Input } from "@kandev/ui/input"; import { Switch } from "@kandev/ui/switch"; import { Button } from "@kandev/ui/button"; diff --git a/apps/web/app/office/workspace/settings/sync/use-sync-state.ts b/apps/web/app/office/workspace/settings/sync/use-sync-state.ts index 86dd2d97c6..34cffbe375 100644 --- a/apps/web/app/office/workspace/settings/sync/use-sync-state.ts +++ b/apps/web/app/office/workspace/settings/sync/use-sync-state.ts @@ -1,7 +1,7 @@ "use client"; import { useCallback, useEffect, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import * as officeApi from "@/lib/api/domains/office-api"; import type { SyncDiff } from "@/lib/api/domains/office-api"; diff --git a/apps/web/app/office/workspace/skills/skill-list.tsx b/apps/web/app/office/workspace/skills/skill-list.tsx index a4dd431abd..c2fb6ff84c 100644 --- a/apps/web/app/office/workspace/skills/skill-list.tsx +++ b/apps/web/app/office/workspace/skills/skill-list.tsx @@ -13,7 +13,7 @@ import { IconCode, IconLoader2, } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Badge } from "@kandev/ui/badge"; import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; diff --git a/apps/web/app/office/workspace/skills/skills-page-client.tsx b/apps/web/app/office/workspace/skills/skills-page-client.tsx index 05712a54c1..245741d8ad 100644 --- a/apps/web/app/office/workspace/skills/skills-page-client.tsx +++ b/apps/web/app/office/workspace/skills/skills-page-client.tsx @@ -2,7 +2,7 @@ import { useCallback, useEffect, useState } from "react"; import { IconBoxMultiple } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import * as officeApi from "@/lib/api/domains/office-api"; import type { Skill } from "@/lib/state/slices/office/types"; diff --git a/apps/web/app/settings/system/logs/page.tsx b/apps/web/app/settings/system/logs/page.tsx index d0863298a2..a846769179 100644 --- a/apps/web/app/settings/system/logs/page.tsx +++ b/apps/web/app/settings/system/logs/page.tsx @@ -1,31 +1,17 @@ -import { StateProvider } from "@/components/state-provider"; -import { SystemPageShell } from "@/components/settings/system/system-page-shell"; +"use client"; + +import { useTranslation } from "react-i18next"; import { LogViewer } from "@/components/settings/system/log-viewer"; -import { fetchLogFiles } from "@/lib/api/domains/system-api"; - -export default async function SystemLogsPage() { - let initialState: Record = {}; - try { - const files = await fetchLogFiles({ cache: "no-store" }).catch(() => null); - if (files) { - initialState = { - system: { - logs: { files, tail: [], tailLoaded: false }, - }, - }; - } - } catch { - initialState = {}; - } +import { SystemPageShell } from "@/components/settings/system/system-page-shell"; +export default function SystemLogsPage() { + const { t } = useTranslation(); return ( - - - - - + + + ); } diff --git a/apps/web/components/app-status-bar/use-app-status-bar-order.ts b/apps/web/components/app-status-bar/use-app-status-bar-order.ts index 25296b7726..11337c61dc 100644 --- a/apps/web/components/app-status-bar/use-app-status-bar-order.ts +++ b/apps/web/components/app-status-bar/use-app-status-bar-order.ts @@ -1,7 +1,7 @@ "use client"; import { useCallback, useMemo, useRef, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore, useAppStoreApi } from "@/components/state-provider"; import { createQueuedUserSettingsSync } from "@/lib/user-settings-sync"; import type { AppStatusBarOrderState } from "@/lib/state/slices/settings/types"; diff --git a/apps/web/components/automations/automation-editor.tsx b/apps/web/components/automations/automation-editor.tsx index 887bffb3f3..bf15b7dfd3 100644 --- a/apps/web/components/automations/automation-editor.tsx +++ b/apps/web/components/automations/automation-editor.tsx @@ -3,7 +3,7 @@ import { useState, useEffect, useCallback, useMemo } from "react"; import { useRouter } from "@/lib/routing/client-router"; import { runWithNavigationBlockerBypassed } from "@/lib/routing/navigation-guard"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Separator } from "@kandev/ui/separator"; import { useAppStore } from "@/components/state-provider"; import { getMultiRepoExecutorDisabledReason } from "@/components/task-create-dialog-multi-repo-guard"; diff --git a/apps/web/components/automations/trigger-configs/webhook-config.tsx b/apps/web/components/automations/trigger-configs/webhook-config.tsx index 3cfb876afe..b94cacadf9 100644 --- a/apps/web/components/automations/trigger-configs/webhook-config.tsx +++ b/apps/web/components/automations/trigger-configs/webhook-config.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useMemo, useCallback, useEffect } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Button } from "@kandev/ui/button"; import { Input } from "@kandev/ui/input"; import { Label } from "@kandev/ui/label"; diff --git a/apps/web/components/azure-devops/azure-devops-task-launcher.tsx b/apps/web/components/azure-devops/azure-devops-task-launcher.tsx index c454a71c16..1203aa264c 100644 --- a/apps/web/components/azure-devops/azure-devops-task-launcher.tsx +++ b/apps/web/components/azure-devops/azure-devops-task-launcher.tsx @@ -1,7 +1,7 @@ "use client"; import { useMemo } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { TaskCreateDialog } from "@/components/task-create-dialog"; import { useAppStore } from "@/components/state-provider"; import { cacheAzureDevOpsTaskPullRequest } from "@/hooks/domains/azure-devops/use-azure-devops-task-pull-requests"; diff --git a/apps/web/components/improve-kandev-dialog-helpers.test.ts b/apps/web/components/improve-kandev-dialog-helpers.test.ts index 8279c5e469..d216282e6b 100644 --- a/apps/web/components/improve-kandev-dialog-helpers.test.ts +++ b/apps/web/components/improve-kandev-dialog-helpers.test.ts @@ -3,16 +3,20 @@ import { describe, it, expect, vi, beforeEach } from "vitest"; import { buildImproveKandevDescription } from "./improve-kandev-dialog-helpers"; import type { ImproveKandevBootstrapResponse } from "@/lib/api/domains/improve-kandev-api"; -const uploadFrontendLog = vi.fn(); +const leaseDiagnosticBundle = vi.fn(); +const createDiagnosticBundle = vi.fn(); +const fetchDiagnosticBundle = vi.fn(); vi.mock("@/lib/api/domains/improve-kandev-api", async (orig) => { const actual = await orig(); - return { ...actual, uploadFrontendLog: (...args: unknown[]) => uploadFrontendLog(...args) }; + return { + ...actual, + leaseDiagnosticBundle: (...args: unknown[]) => leaseDiagnosticBundle(...args), + }; }); -vi.mock("@/lib/logger/buffer", () => ({ - snapshotLogs: () => [ - { timestamp: "2026-04-29T10:00:00Z", level: "info", source: "console", message: "hi" }, - ], +vi.mock("@/lib/api/domains/system-api", () => ({ + createDiagnosticBundle: (...args: unknown[]) => createDiagnosticBundle(...args), + fetchDiagnosticBundle: (...args: unknown[]) => fetchDiagnosticBundle(...args), })); const bootstrap: ImproveKandevBootstrapResponse = { @@ -21,11 +25,7 @@ const bootstrap: ImproveKandevBootstrapResponse = { issue_workflow_id: "w2", branch: "main", bundle_dir: "/tmp/kandev-improve-abc", - bundle_files: { - metadata: "/tmp/kandev-improve-abc/metadata.json", - backend_log: "/tmp/kandev-improve-abc/backend.log", - frontend_log: "/tmp/kandev-improve-abc/frontend.log", - }, + bundle_file: "/tmp/kandev-improve-abc/diagnostic-bundle.zip", github_login: "octocat", has_write_access: false, fork_status: "unknown", @@ -33,47 +33,42 @@ const bootstrap: ImproveKandevBootstrapResponse = { describe("buildImproveKandevDescription", () => { beforeEach(() => { - uploadFrontendLog.mockReset(); - uploadFrontendLog.mockResolvedValue({ path: bootstrap.bundle_files.frontend_log }); + leaseDiagnosticBundle.mockReset(); + createDiagnosticBundle.mockReset(); + fetchDiagnosticBundle.mockReset(); + createDiagnosticBundle.mockResolvedValue({ id: "bundle-1", status: "ready" }); + leaseDiagnosticBundle.mockResolvedValue({ + path: bootstrap.bundle_file, + status: "ready", + sources: ["backend", "frontend"], + }); }); it("returns description unchanged when bootstrap is null", async () => { const out = await buildImproveKandevDescription("desc", null, true); expect(out).toBe("desc"); - expect(uploadFrontendLog).not.toHaveBeenCalled(); + expect(createDiagnosticBundle).not.toHaveBeenCalled(); }); it("returns description unchanged when captureLogs is false", async () => { const out = await buildImproveKandevDescription("desc", bootstrap, false); expect(out).toBe("desc"); - expect(uploadFrontendLog).not.toHaveBeenCalled(); + expect(createDiagnosticBundle).not.toHaveBeenCalled(); }); - it("appends bundle file paths and uploads frontend log when captureLogs=true", async () => { + it("creates and leases an all-source diagnostic ZIP when captureLogs=true", async () => { const out = await buildImproveKandevDescription("Original prompt", bootstrap, true); expect(out).toContain("Original prompt"); - expect(out).toContain("Context bundle for the agent:"); - expect(out).toContain(bootstrap.bundle_files.metadata); - expect(out).toContain(bootstrap.bundle_files.backend_log); - expect(out).toContain(bootstrap.bundle_files.frontend_log); - expect(uploadFrontendLog).toHaveBeenCalledWith( - bootstrap.bundle_dir, - expect.arrayContaining([expect.objectContaining({ message: "hi" })]), - ); + expect(out).toContain("frontend + backend logs"); + expect(out).toContain(bootstrap.bundle_file); + expect(createDiagnosticBundle).toHaveBeenCalledWith(["backend", "frontend"]); + expect(leaseDiagnosticBundle).toHaveBeenCalledWith(bootstrap.bundle_dir, "bundle-1"); }); - it("does not abort when frontend log upload fails", async () => { - uploadFrontendLog.mockRejectedValueOnce(new Error("network down")); - await expect(buildImproveKandevDescription("desc", bootstrap, true)).resolves.toContain( - "Context bundle for the agent:", - ); - }); - - it("omits the frontend_log path when upload fails", async () => { - uploadFrontendLog.mockRejectedValueOnce(new Error("network down")); + it("does not abort or reference a missing bundle when collection fails", async () => { + createDiagnosticBundle.mockRejectedValueOnce(new Error("network down")); const out = await buildImproveKandevDescription("desc", bootstrap, true); - expect(out).toContain(bootstrap.bundle_files.metadata); - expect(out).toContain(bootstrap.bundle_files.backend_log); - expect(out).not.toContain(bootstrap.bundle_files.frontend_log); + expect(out).toBe("desc"); + expect(out).not.toContain(bootstrap.bundle_file); }); }); diff --git a/apps/web/components/improve-kandev-dialog-helpers.ts b/apps/web/components/improve-kandev-dialog-helpers.ts index af23d1f218..25e0402672 100644 --- a/apps/web/components/improve-kandev-dialog-helpers.ts +++ b/apps/web/components/improve-kandev-dialog-helpers.ts @@ -1,22 +1,16 @@ "use client"; -import { snapshotLogs } from "@/lib/logger/buffer"; import { - uploadFrontendLog, + leaseDiagnosticBundle, type ImproveKandevBootstrapResponse, } from "@/lib/api/domains/improve-kandev-api"; +import { createDiagnosticBundle, fetchDiagnosticBundle } from "@/lib/api/domains/system-api"; +import type { DiagnosticBundleJob } from "@/lib/types/system"; /** - * Append the bundle file paths to the user-supplied description as a - * machine-readable footer the agent prompt instructs to read. - * - * Behavior: - * - When `bootstrap` is null, returns the description unchanged. - * - When `captureLogs` is false, returns the description unchanged. - * - When `captureLogs` is true, attempts to upload the current in-memory - * frontend log snapshot to the bundle directory and only includes the - * `frontend_log` entry when the upload succeeded — referencing a file - * that was never written would mislead the agent. + * Creates the same owner-scoped frontend+backend ZIP used by System Logs, + * leases it into the Improve Kandev task context, and appends that single path + * to the task description. Diagnostics stay best-effort. */ export async function buildImproveKandevDescription( description: string, @@ -26,24 +20,32 @@ export async function buildImproveKandevDescription( if (!bootstrap) return description; if (!captureLogs) return description; - let frontendLogUploaded = false; try { - await uploadFrontendLog(bootstrap.bundle_dir, snapshotLogs()); - frontendLogUploaded = true; + const created = await createDiagnosticBundle(["backend", "frontend"]); + const completed = await waitForDiagnosticBundle(created); + const lease = await leaseDiagnosticBundle(bootstrap.bundle_dir, completed.id); + return [ + description, + "", + "---", + "Diagnostic bundle for the agent (frontend + backend logs):", + `- ${lease.path}`, + ].join("\n"); } catch { - // Frontend log upload is best-effort — keep submitting the task without it. + return description; } - - const lines = [ - description, - "", - "---", - "Context bundle for the agent:", - `- ${bootstrap.bundle_files.metadata}`, - `- ${bootstrap.bundle_files.backend_log}`, - ]; - if (frontendLogUploaded) { - lines.push(`- ${bootstrap.bundle_files.frontend_log}`); - } - return lines.join("\n"); +} + +async function waitForDiagnosticBundle(initial: DiagnosticBundleJob): Promise { + let job = initial; + const deadline = Date.now() + 5 * 60_000; + while (job.status === "collecting" || job.status === "building") { + if (Date.now() >= deadline) throw new Error("diagnostic bundle timed out"); + await new Promise((resolve) => setTimeout(resolve, 250)); + job = await fetchDiagnosticBundle(job.id); + } + if (job.status !== "ready" && job.status !== "partial") { + throw new Error("diagnostic bundle failed"); + } + return job; } diff --git a/apps/web/components/log-buffer-bridge.tsx b/apps/web/components/log-buffer-bridge.tsx index 0c5d906652..3a4b270216 100644 --- a/apps/web/components/log-buffer-bridge.tsx +++ b/apps/web/components/log-buffer-bridge.tsx @@ -3,14 +3,24 @@ import { useEffect } from "react"; import { installConsoleInterceptor } from "@/lib/logger/intercept"; +import { setLogIdentity } from "@/lib/logger/runtime"; +import { useAppStore } from "@/components/state-provider"; /** * Installs the console + window-error interceptor on the client so recent * frontend logs are captured for Improve Kandev reports. No UI. */ export function LogBufferBridge() { + const auth = useAppStore((state) => state.auth); useEffect(() => { installConsoleInterceptor(); }, []); + useEffect(() => { + if (!auth.authenticated) { + setLogIdentity(null); + return; + } + setLogIdentity(auth.user?.id ?? (auth.mode === "disabled" ? "default-user" : null)); + }, [auth.authenticated, auth.mode, auth.user?.id]); return null; } diff --git a/apps/web/components/settings/plugins/marketplace-sources-dialog.tsx b/apps/web/components/settings/plugins/marketplace-sources-dialog.tsx index 35ad73f151..f1c630306c 100644 --- a/apps/web/components/settings/plugins/marketplace-sources-dialog.tsx +++ b/apps/web/components/settings/plugins/marketplace-sources-dialog.tsx @@ -2,7 +2,7 @@ import { useState } from "react"; import { IconTrash } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { Button } from "@kandev/ui/button"; import { Dialog, diff --git a/apps/web/components/settings/plugins/use-plugin-actions.ts b/apps/web/components/settings/plugins/use-plugin-actions.ts index 6a536107ff..a0f0be13a2 100644 --- a/apps/web/components/settings/plugins/use-plugin-actions.ts +++ b/apps/web/components/settings/plugins/use-plugin-actions.ts @@ -1,7 +1,7 @@ "use client"; import { useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import type { StoreApi } from "zustand"; import { useAppStore, useAppStoreApi } from "@/components/state-provider"; import { useTheme } from "@/components/theme/app-theme"; diff --git a/apps/web/components/settings/plugins/use-plugin-config-form.ts b/apps/web/components/settings/plugins/use-plugin-config-form.ts index 98f5937e39..3a2062d162 100644 --- a/apps/web/components/settings/plugins/use-plugin-config-form.ts +++ b/apps/web/components/settings/plugins/use-plugin-config-form.ts @@ -1,7 +1,7 @@ "use client"; import { useEffect, useMemo, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { getPluginConfig, updatePluginConfig } from "@/lib/api/domains/plugins-api"; import { SECRET_MASK, diff --git a/apps/web/components/settings/ssh-agent-readiness-card.tsx b/apps/web/components/settings/ssh-agent-readiness-card.tsx index fea85acdfe..3d9adc99de 100644 --- a/apps/web/components/settings/ssh-agent-readiness-card.tsx +++ b/apps/web/components/settings/ssh-agent-readiness-card.tsx @@ -8,7 +8,7 @@ import { Label } from "@kandev/ui/label"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@kandev/ui/table"; import { IconCheck, IconCopy, IconLoader2, IconX } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { probeSSHAgents, probeSSHShells } from "@/lib/api/domains/ssh-api"; import type { SSHAgentReadinessRow } from "@/lib/types/http-ssh"; import { copyToClipboard } from "@/lib/utils/copy-to-clipboard"; diff --git a/apps/web/components/settings/system/bundle-customizer.tsx b/apps/web/components/settings/system/bundle-customizer.tsx new file mode 100644 index 0000000000..bf53484052 --- /dev/null +++ b/apps/web/components/settings/system/bundle-customizer.tsx @@ -0,0 +1,533 @@ +"use client"; + +import { useMemo } from "react"; +import { useTranslation } from "react-i18next"; +import { Alert, AlertDescription, AlertTitle } from "@kandev/ui/alert"; +import { Button } from "@kandev/ui/button"; +import { Checkbox } from "@kandev/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@kandev/ui/dialog"; +import { + Drawer, + DrawerContent, + DrawerDescription, + DrawerHeader, + DrawerTitle, +} from "@kandev/ui/drawer"; +import { Spinner } from "@kandev/ui/spinner"; +import { IconExternalLink } from "@tabler/icons-react"; +import { useResponsiveBreakpoint } from "@/hooks/use-responsive-breakpoint"; +import { linkToTask } from "@/lib/links"; +import type { + DiagnosticBundleCapabilities, + DiagnosticBundleSource, + DiagnosticSession, +} from "@/lib/types/system"; + +type BundleCustomizerProps = { + open: boolean; + onOpenChange: (open: boolean) => void; + capabilities: DiagnosticBundleCapabilities | null; + sessions: DiagnosticSession[]; + sessionsLoading: boolean; + selectedSources: DiagnosticBundleSource[]; + selectedSessionIDs: string[]; + submitting: boolean; + onSourcesChange: (sources: DiagnosticBundleSource[]) => void; + onSessionChange: (sessionID: string, checked: boolean) => void; + onSessionSelectionChange: (sessionIDs: string[]) => void; + onSubmit: () => void; +}; + +const SOURCE_ORDER: DiagnosticBundleSource[] = ["backend", "frontend", "runtime", "acp"]; +const SOURCE_COPY: Record = { + backend: { + label: "settings:diagnosticSourceBackend", + description: "settings:diagnosticSourceBackendDescription", + }, + frontend: { + label: "settings:diagnosticSourceFrontend", + description: "settings:diagnosticSourceFrontendDescription", + }, + runtime: { + label: "settings:diagnosticSourceRuntime", + description: "settings:diagnosticSourceRuntimeDescription", + }, + acp: { + label: "settings:diagnosticSourceACP", + description: "settings:diagnosticSourceACPDescription", + }, +}; + +export function BundleCustomizer(props: BundleCustomizerProps) { + const { t } = useTranslation(); + const { isMobile } = useResponsiveBreakpoint(); + const sourceOptions = useMemo(() => { + const available = new Set( + props.capabilities?.sources ?? ["backend", "frontend", "runtime"], + ); + return SOURCE_ORDER.filter( + (source) => + available.has(source) && + (source !== "acp" || props.capabilities?.acp_debug_enabled === true), + ); + }, [props.capabilities?.acp_debug_enabled, props.capabilities?.sources]); + const acpSelected = props.selectedSources.includes("acp"); + const selectableSessions = props.sessions.filter( + (session) => session.acp_availability !== "unavailable", + ); + const validationKey = validationMessageKey( + props.selectedSources, + acpSelected, + props.selectedSessionIDs.length, + selectableSessions.length, + props.capabilities?.acp_max_sessions ?? 10, + ); + const content = ( + + ); + const footer = ( + props.onOpenChange(false)} + onSubmit={props.onSubmit} + /> + ); + const validation = validationKey ? ( +

+ {validationMessage(validationKey, t)} +

+ ) : null; + + if (isMobile) { + return ( + + + + {t("settings:diagnosticCustomizerTitle")} + {t("settings:diagnosticCustomizerDescription")} + +
+ {content} + {validation} +
+ {footer} + + + ); + } + + return ( + + + + {t("settings:diagnosticCustomizerTitle")} + {t("settings:diagnosticCustomizerDescription")} + +
+ {content} + {validation} +
+ {footer} + + + ); +} + +type CustomizerContentProps = { + sourceOptions: DiagnosticBundleSource[]; + selectedSources: DiagnosticBundleSource[]; + selectedSessionIDs: string[]; + sessions: DiagnosticSession[]; + sessionsLoading: boolean; + acpSelected: boolean; + selectableSessions: DiagnosticSession[]; + maxSessions: number; + onSourcesChange: (sources: DiagnosticBundleSource[]) => void; + onSessionChange: (sessionID: string, checked: boolean) => void; + onSessionSelectionChange: (sessionIDs: string[]) => void; +}; + +function CustomizerContent({ + sourceOptions, + selectedSources, + selectedSessionIDs, + sessions, + sessionsLoading, + acpSelected, + selectableSessions, + maxSessions, + onSourcesChange, + onSessionChange, + onSessionSelectionChange, +}: CustomizerContentProps) { + const { t } = useTranslation(); + + return ( +
+ {acpSelected && ( + + {t("settings:diagnosticSourceACP")} + {t("settings:diagnosticACPWarning")} + + )} + + + {acpSelected && ( + + )} +
+ ); +} + +type CustomizerFooterProps = { + submitting: boolean; + invalid: boolean; + onCancel: () => void; + onSubmit: () => void; +}; + +function CustomizerFooter({ submitting, invalid, onCancel, onSubmit }: CustomizerFooterProps) { + const { t } = useTranslation(); + return ( +
+ + +
+ ); +} + +type SourceSectionProps = { + sourceOptions: DiagnosticBundleSource[]; + selectedSources: DiagnosticBundleSource[]; + onSourcesChange: (sources: DiagnosticBundleSource[]) => void; +}; + +function SourceSection({ sourceOptions, selectedSources, onSourcesChange }: SourceSectionProps) { + const { t } = useTranslation(); + const setSource = (source: DiagnosticBundleSource, checked: boolean) => { + const next = checked + ? [...selectedSources, source] + : selectedSources.filter((selected) => selected !== source); + onSourcesChange(SOURCE_ORDER.filter((candidate) => next.includes(candidate))); + }; + return ( +
+

+ {t("settings:diagnosticSourcesTitle")} +

+
+ {sourceOptions.map((source) => ( + setSource(source, checked)} + /> + ))} +
+
+ ); +} + +function SourceRow({ + source, + checked, + onCheckedChange, +}: { + source: DiagnosticBundleSource; + checked: boolean; + onCheckedChange: (checked: boolean) => void; +}) { + const { t } = useTranslation(); + const copy = SOURCE_COPY[source]; + return ( + + ); +} + +type SessionSectionProps = { + sessions: DiagnosticSession[]; + sessionsLoading: boolean; + selectableSessions: DiagnosticSession[]; + selectedSessionIDs: string[]; + maxSessions: number; + onSessionChange: (sessionID: string, checked: boolean) => void; + onSessionSelectionChange: (sessionIDs: string[]) => void; +}; + +function SessionSection({ + sessions, + sessionsLoading, + selectableSessions, + selectedSessionIDs, + maxSessions, + onSessionChange, + onSessionSelectionChange, +}: SessionSectionProps) { + const { t } = useTranslation(); + return ( +
+
+

+ {t("settings:diagnosticSelectSessions")} +

+

+ {t("settings:diagnosticSessionsDescription", { count: maxSessions })} +

+
+ +
+ ); +} + +function SessionResults({ + sessions, + sessionsLoading, + selectableSessions, + selectedSessionIDs, + maxSessions, + onSessionChange, + onSessionSelectionChange, +}: SessionSectionProps) { + const { t } = useTranslation(); + if (sessionsLoading) { + return ( +
+ + {t("settings:diagnosticSessionsLoading")} +
+ ); + } + if (sessions.length === 0) { + return ( +

+ {t("settings:diagnosticACPNoSessions")} +

+ ); + } + const selectAll = () => { + onSessionSelectionChange( + selectableSessions.slice(0, maxSessions).map((session) => session.session_id), + ); + }; + return ( +
+ 0} + hasSelectedSessions={selectedSessionIDs.length > 0} + onSelectAll={selectAll} + onClear={() => onSessionSelectionChange([])} + /> + {sessions.map((session) => ( + = maxSessions} + onSessionChange={onSessionChange} + /> + ))} +

+ {t("settings:diagnosticSelectedSessions", { count: selectedSessionIDs.length })} +

+
+ ); +} + +function SessionSelectionActions({ + hasSelectableSessions, + hasSelectedSessions, + onSelectAll, + onClear, +}: { + hasSelectableSessions: boolean; + hasSelectedSessions: boolean; + onSelectAll: () => void; + onClear: () => void; +}) { + const { t } = useTranslation(); + return ( +
+ + +
+ ); +} + +function SessionRow({ + session, + selectable, + checked, + maxReached, + onSessionChange, +}: { + session: DiagnosticSession; + selectable: boolean; + checked: boolean; + maxReached: boolean; + onSessionChange: (sessionID: string, checked: boolean) => void; +}) { + const { t } = useTranslation(); + const title = session.agent || session.provider || session.session_id; + const taskTitle = session.task_title || session.task_id; + const details = + [session.model, session.status, session.executor_type].filter(Boolean).join(" · ") || + session.session_id; + const checkboxID = `diagnostic-acp-session-${session.session_id}`; + return ( +
+ onSessionChange(session.session_id, value === true)} + className="mt-0.5" + aria-label={session.session_id} + /> +
+ + {taskTitle} + + + +
+ + {availabilityLabel(session.acp_availability, t)} + +
+ ); +} + +function validationMessageKey( + sources: DiagnosticBundleSource[], + acpSelected: boolean, + selectedSessionCount: number, + selectableSessionCount: number, + maxSessions: number, +): string | null { + if (sources.length === 0) return "diagnosticSourceRequired"; + if (acpSelected && selectedSessionCount > maxSessions) return "diagnosticACPSelectionLimit"; + if (acpSelected && (selectableSessionCount === 0 || selectedSessionCount === 0)) { + return "diagnosticACPSelectionRequired"; + } + return null; +} + +function availabilityLabel( + availability: DiagnosticSession["acp_availability"], + t: (key: string) => string, +): string { + if (availability === "reachable") return t("settings:diagnosticSessionReachable"); + if (availability === "host_retained") return t("settings:diagnosticSessionRetained"); + return t("settings:diagnosticSessionUnavailable"); +} + +function validationMessage(key: string, t: (key: string) => string): string { + if (key === "diagnosticSourceRequired") return t("settings:diagnosticSourceRequired"); + if (key === "diagnosticACPSelectionRequired") { + return t("settings:diagnosticACPSelectionRequired"); + } + return t("settings:diagnosticACPSelectionLimit"); +} diff --git a/apps/web/components/settings/system/log-viewer.test.tsx b/apps/web/components/settings/system/log-viewer.test.tsx new file mode 100644 index 0000000000..062a851e0e --- /dev/null +++ b/apps/web/components/settings/system/log-viewer.test.tsx @@ -0,0 +1,133 @@ +import { StrictMode } from "react"; +import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { LogViewer } from "./log-viewer"; + +const createMock = vi.fn(); +const fetchMock = vi.fn(); +const capabilitiesMock = vi.fn(); +const sessionsMock = vi.fn(); +const downloadURLMock = vi.fn((..._args: unknown[]) => "/download/bundle-1"); +const CHECKED_STATE = "checked"; +const DATA_STATE_ATTRIBUTE = "data-state"; + +vi.mock("@/lib/api/domains/system-api", () => ({ + createDiagnosticBundle: (...args: unknown[]) => createMock(...args), + fetchDiagnosticBundle: (...args: unknown[]) => fetchMock(...args), + fetchDiagnosticBundleCapabilities: (...args: unknown[]) => capabilitiesMock(...args), + fetchDiagnosticACPSessions: (...args: unknown[]) => sessionsMock(...args), + buildDiagnosticBundleDownloadUrl: (...args: unknown[]) => downloadURLMock(...args), +})); + +beforeEach(() => { + createMock.mockReset(); + fetchMock.mockReset(); + capabilitiesMock.mockReset(); + sessionsMock.mockReset(); + capabilitiesMock.mockResolvedValue({ + sources: ["backend", "frontend", "runtime"], + acp_debug_enabled: false, + acp_max_sessions: 10, + }); + sessionsMock.mockResolvedValue([]); + downloadURLMock.mockClear(); + vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); +}); + +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); +}); + +describe("LogViewer", () => { + it("uses one customizer with the standard backend and frontend defaults", async () => { + createMock.mockResolvedValue({ + id: "bundle-1", + status: "collecting", + warnings: [], + }); + fetchMock.mockResolvedValue({ + id: "bundle-1", + status: "partial", + warnings: ["One browser did not respond."], + }); + render( + + + , + ); + + expect(screen.getByText("Review before sharing")).toBeTruthy(); + expect(screen.queryByText("Recent log output")).toBeNull(); + expect(screen.queryByTestId("download-diagnostic-bundle")).toBeNull(); + expect(screen.queryByTestId("download-diagnostic-bundle-with-acp")).toBeNull(); + fireEvent.click(screen.getByTestId("customize-diagnostic-bundle")); + expect( + screen.getByRole("checkbox", { name: "Backend logs" }).getAttribute(DATA_STATE_ATTRIBUTE), + ).toBe(CHECKED_STATE); + expect( + screen.getByRole("checkbox", { name: "Frontend logs" }).getAttribute(DATA_STATE_ATTRIBUTE), + ).toBe(CHECKED_STATE); + expect( + screen.getByRole("checkbox", { name: "Runtime index" }).getAttribute(DATA_STATE_ATTRIBUTE), + ).not.toBe(CHECKED_STATE); + fireEvent.click(screen.getByTestId("create-custom-diagnostic-bundle")); + await waitFor(() => expect(createMock).toHaveBeenCalledWith(["backend", "frontend"], [])); + + await waitFor(() => expect(fetchMock).toHaveBeenCalledWith("bundle-1"), { timeout: 1_500 }); + expect(await screen.findByText(/partial ZIP is downloading/)).toBeTruthy(); + expect(downloadURLMock).toHaveBeenCalledWith("bundle-1"); + }); + + it("offers ACP in the one customizer with task links and bounded bulk selection", async () => { + capabilitiesMock.mockResolvedValue({ + sources: ["backend", "frontend", "runtime", "acp"], + acp_debug_enabled: true, + acp_max_sessions: 1, + }); + sessionsMock.mockResolvedValue([ + { + task_id: "task-1", + task_title: "Repair failing diagnostics", + session_id: "session-1", + agent: "claude-acp", + model: "sonnet", + status: "running", + executor_type: "local_docker", + acp_availability: "reachable", + }, + { + task_id: "task-2", + task_title: "Inspect timeout", + session_id: "session-2", + agent: "codex-acp", + acp_availability: "host_retained", + }, + ]); + render(); + await waitFor(() => expect(screen.getByTestId("customize-diagnostic-bundle")).toBeTruthy()); + expect(screen.queryByTestId("download-diagnostic-bundle")).toBeNull(); + expect(screen.queryByTestId("download-diagnostic-bundle-with-acp")).toBeNull(); + fireEvent.click(screen.getByTestId("customize-diagnostic-bundle")); + fireEvent.click(screen.getByRole("checkbox", { name: "ACP debug messages" })); + expect((await screen.findAllByText("ACP debug messages")).length).toBeGreaterThanOrEqual(1); + expect(screen.getByText(/can contain prompts/)).toBeTruthy(); + const create = screen.getByTestId("create-custom-diagnostic-bundle"); + expect(create.hasAttribute("disabled")).toBe(true); + const taskLink = await screen.findByTestId("acp-session-task-link-session-1"); + expect(taskLink.textContent).toContain("Repair failing diagnostics"); + expect(taskLink.getAttribute("href")).toBe("/t/task-1"); + expect(taskLink.getAttribute("target")).toBe("_blank"); + fireEvent.click(screen.getByTestId("select-all-acp-sessions")); + await waitFor(() => expect(create.hasAttribute("disabled")).toBe(false)); + expect(screen.getByText("1 session selected")).toBeTruthy(); + expect( + screen.getByRole("checkbox", { name: "session-1" }).getAttribute(DATA_STATE_ATTRIBUTE), + ).toBe(CHECKED_STATE); + expect( + screen.getByRole("checkbox", { name: "session-2" }).getAttribute(DATA_STATE_ATTRIBUTE), + ).toBe("unchecked"); + fireEvent.click(screen.getByTestId("clear-acp-session-selection")); + await waitFor(() => expect(create.hasAttribute("disabled")).toBe(true)); + }); +}); diff --git a/apps/web/components/settings/system/log-viewer.tsx b/apps/web/components/settings/system/log-viewer.tsx index e7a6306fc9..43c6ba1c9f 100644 --- a/apps/web/components/settings/system/log-viewer.tsx +++ b/apps/web/components/settings/system/log-viewer.tsx @@ -1,269 +1,281 @@ "use client"; -import { Card, CardContent, CardHeader, CardTitle } from "@kandev/ui/card"; +import { useEffect, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { Alert, AlertDescription, AlertTitle } from "@kandev/ui/alert"; import { Button } from "@kandev/ui/button"; +import { Card, CardContent, CardTitle } from "@kandev/ui/card"; import { Spinner } from "@kandev/ui/spinner"; -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@kandev/ui/table"; -import { Badge } from "@kandev/ui/badge"; -import { IconCopy, IconDownload, IconFileText, IconRefresh } from "@tabler/icons-react"; -import { useLogFiles } from "@/hooks/domains/system/use-log-files"; -import { useLogTail } from "@/hooks/domains/system/use-log-tail"; -import { buildLogDownloadUrl } from "@/lib/api/domains/system-api"; -import { copyToClipboard } from "@/lib/utils/copy-to-clipboard"; -import { formatBytes } from "@/lib/utils/format-bytes"; -import { useActionFeedback, type ActionFeedbackState } from "@/hooks/use-action-feedback"; -import { ActionButtonContent } from "./action-button-content"; +import { IconAlertTriangle, IconDownload, IconFileZip } from "@tabler/icons-react"; +import { ApiError } from "@/lib/api/client"; +import { + buildDiagnosticBundleDownloadUrl, + createDiagnosticBundle, + fetchDiagnosticACPSessions, + fetchDiagnosticBundle, + fetchDiagnosticBundleCapabilities, +} from "@/lib/api/domains/system-api"; +import type { + DiagnosticBundleCapabilities, + DiagnosticBundleJob, + DiagnosticBundleSource, + DiagnosticSession, +} from "@/lib/types/system"; +import { BundleCustomizer } from "./bundle-customizer"; -function formatTimestamp(iso: string): string { - if (!iso) return "-"; - const d = new Date(iso); - if (Number.isNaN(d.getTime())) return iso; - return d.toLocaleString(); -} +type ViewState = "idle" | "collecting" | "preparing" | "partial" | "busy" | "error"; -function TailContent({ - tail, - loading, - inMemoryOnly, -}: { - tail: string[]; - loading: boolean; - inMemoryOnly: boolean; -}) { - if (loading && tail.length === 0) { - return ( -
- Loading log... -
- ); - } - if (tail.length === 0) { - return ( -

- No recent log activity captured yet. Logs will appear as Kandev does work. -

- ); - } - return ( -
- {inMemoryOnly && ( -

- Showing the in-memory log buffer (last ~2000 entries). Kandev is currently logging to the - terminal, not to a file - file rotation is disabled. Set logging.outputPath{" "} - in config.yaml to a file path to enable downloadable log files. -

- )} - 
-        {tail.join("\n")}
-
-
- ); -} - -function buildTailFilename(): string { - // YYYY-MM-DDTHH-MM-SS to mirror lumberjack's rotation naming, minus the - // colons (which are invalid on Windows). The user can rename freely. - const stamp = new Date().toISOString().replace(/:/g, "-").replace(/\..+$/, ""); - return `kandev-tail-${stamp}.log`; -} - -function downloadTailAsFile(tail: string[]) { - if (typeof window === "undefined" || tail.length === 0) return; - const blob = new Blob([tail.join("\n") + "\n"], { type: "text/plain" }); - const url = URL.createObjectURL(blob); - const a = document.createElement("a"); - a.href = url; - a.download = buildTailFilename(); - document.body.appendChild(a); - a.click(); - a.remove(); - URL.revokeObjectURL(url); -} - -function TailHeader({ - tail, - current, - refreshState, - copyState, - onRefresh, - onCopy, -}: { - tail: string[]; - current: ReturnType["files"][number] | undefined; - refreshState: ActionFeedbackState; - copyState: ActionFeedbackState; - onRefresh: () => void; - onCopy: () => void; -}) { - const hasTail = tail.length > 0; - return ( -
- - {current ? ( - - ) : ( - - )} - -
- ); -} +const DEFAULT_SOURCES: DiagnosticBundleSource[] = ["backend", "frontend"]; +// eslint-disable-next-line max-lines-per-function -- coordinates bundle polling and shared customizer state. export function LogViewer() { - const { files, isLoading: filesLoading } = useLogFiles(); - const { tail, isLoading: tailLoading, reload: reloadTail } = useLogTail(1000); - const refreshFeedback = useActionFeedback(); - const copyFeedback = useActionFeedback(); + const { t } = useTranslation(); + const [state, setState] = useState("idle"); + const [message, setMessage] = useState(""); + const [capabilities, setCapabilities] = useState(null); + const [sessions, setSessions] = useState([]); + const [sessionsLoading, setSessionsLoading] = useState(false); + const [sessionsLoaded, setSessionsLoaded] = useState(false); + const [customizerOpen, setCustomizerOpen] = useState(false); + const [selectedSources, setSelectedSources] = useState(DEFAULT_SOURCES); + const [selectedSessionIDs, setSelectedSessionIDs] = useState([]); + const [customizerSubmitting, setCustomizerSubmitting] = useState(false); + const mounted = useRef(true); - const onRefresh = () => - void refreshFeedback.run(async () => { - await reloadTail(); - }); + useEffect(() => { + mounted.current = true; + void fetchDiagnosticBundleCapabilities() + .then((next) => mounted.current && setCapabilities(next)) + .catch(() => undefined); + return () => { + mounted.current = false; + }; + }, []); - const onCopy = () => - void copyFeedback.run(async () => { - if (!(await copyToClipboard(tail.join("\n") + "\n"))) { - throw new Error("Clipboard copy failed"); + useEffect(() => { + if (!customizerOpen || !selectedSources.includes("acp") || sessionsLoaded) return; + setSessionsLoading(true); + void fetchDiagnosticACPSessions() + .then((next) => { + if (!mounted.current) return; + setSessions(next); + setSessionsLoaded(true); + }) + .catch(() => { + if (!mounted.current) return; + setSessions([]); + setSessionsLoaded(true); + }) + .finally(() => mounted.current && setSessionsLoading(false)); + }, [customizerOpen, selectedSources, sessionsLoaded]); + + const download = async (sources: DiagnosticBundleSource[], sessionIDs: string[] = []) => { + setMessage(""); + setState("collecting"); + try { + const job = await prepareDiagnosticBundle(sources, sessionIDs, (next) => { + if (mounted.current) setState(next); + }); + if (!mounted.current) return; + if (job.status !== "ready" && job.status !== "partial") { + throw new Error(job.warnings?.[0] ?? t("settings:diagnosticBundlePrepareError")); } + setState(job.status === "partial" ? "partial" : "idle"); + setMessage(bundleMessage(job, t)); + triggerDownload(buildDiagnosticBundleDownloadUrl(job.id)); + } catch (error) { + if (!mounted.current) return; + if (error instanceof ApiError && (error.status === 429 || error.status === 503)) { + const retry = error.retryAfterSeconds ?? 5; + setState("busy"); + setMessage(t("settings:diagnosticBusy", { seconds: retry })); + window.setTimeout(() => mounted.current && setState("idle"), retry * 1_000); + return; + } + setState("error"); + setMessage(error instanceof Error ? error.message : t("settings:diagnosticBundleFailed")); + } + }; + + const openCustomizer = () => { + const available = new Set(capabilities?.sources ?? DEFAULT_SOURCES); + setSelectedSources(DEFAULT_SOURCES.filter((source) => available.has(source))); + setSelectedSessionIDs([]); + setCustomizerSubmitting(false); + setCustomizerOpen(true); + }; + + const submitCustomizer = () => { + setCustomizerSubmitting(true); + setCustomizerOpen(false); + void download(selectedSources, selectedSessionIDs).finally(() => { + if (mounted.current) setCustomizerSubmitting(false); }); + }; - const current = files.find((f) => f.current); - const inMemoryOnly = !filesLoading && files.length === 0; - + const pending = state === "collecting" || state === "preparing"; return ( -
- - - - Recent log output - - - - - - - +
+ + - - - Log files - - - {!files.length && filesLoading && ( -
- Loading files... -
- )} - {files.length > 0 && ( - - - - Name - Kind - Size - Modified - Actions - - - - {files.map((f) => ( - - {f.name} - - - {f.current ? "current" : "rotated"} - - - {formatBytes(f.size)} - {formatTimestamp(f.mtime)} - - - - - ))} - -
- )} - {!filesLoading && files.length === 0 && ( -

- No log files found. Kandev may be logging to stdout (no file rotation configured). -

- )} - - + { + setSelectedSources(sources); + if (!sources.includes("acp")) setSelectedSessionIDs([]); + }} + onSessionChange={(sessionID, checked) => + setSelectedSessionIDs((current) => updateSessionSelection(current, sessionID, checked)) + } + onSessionSelectionChange={setSelectedSessionIDs} + onSubmit={submitCustomizer} + />
); } + +function DiagnosticDisclosure() { + const { t } = useTranslation(); + return ( + + + {t("settings:diagnosticReviewBeforeSharing")} + +

{t("settings:diagnosticReviewDescription")}

+
+

{t("settings:diagnosticNoMessages")}

+

{t("settings:diagnosticIncidentalText")}

+
+ + + ); +} + +type DiagnosticBundleCardProps = { + pending: boolean; + state: ViewState; + message: string; + onCustomize: () => void; +}; + +function DiagnosticBundleCard({ pending, state, message, onCustomize }: DiagnosticBundleCardProps) { + const { t } = useTranslation(); + return ( + + +
+
+ + + {t("settings:diagnosticBundleTitle")} + +

+ {t("settings:diagnosticBundleDescription")} +

+
+ +
+
+

{t("settings:diagnosticBackendEvents")}

+

{t("settings:diagnosticFrontendEvents")}

+
+ {message && } + + + ); +} + +function DiagnosticBundleActions({ + pending, + state, + onCustomize, +}: Omit) { + const { t } = useTranslation(); + return ( +
+ +
+ ); +} + +function DiagnosticBundleMessage({ state, message }: { state: ViewState; message: string }) { + return ( +

+ {message} +

+ ); +} + +function updateSessionSelection(current: string[], sessionID: string, checked: boolean): string[] { + if (!checked) return current.filter((id) => id !== sessionID); + if (current.includes(sessionID)) return current; + return [...current, sessionID]; +} + +async function prepareDiagnosticBundle( + sources: DiagnosticBundleSource[], + sessionIDs: string[], + onStatus: (state: ViewState) => void, +) { + let job = await createDiagnosticBundle(sources, sessionIDs); + while (job.status === "collecting" || job.status === "building") { + onStatus(job.status === "collecting" ? "collecting" : "preparing"); + await delay(500); + job = await fetchDiagnosticBundle(job.id); + } + return job; +} + +function buttonLabel(state: ViewState, t: (key: string) => string): string { + if (state === "collecting") return t("settings:diagnosticCollectingFrontendLogs"); + if (state === "preparing") return t("settings:diagnosticPreparingZip"); + return t("settings:diagnosticCustomize"); +} + +function bundleMessage( + job: DiagnosticBundleJob, + t: (key: string, options?: Record) => string, +): string { + if (job.status !== "partial") return t("settings:diagnosticZipDownloading"); + return job.warnings?.length + ? t("settings:diagnosticPartialZipDownloading", { warnings: job.warnings.join(" ") }) + : t("settings:diagnosticPartialZipUnavailable"); +} + +function triggerDownload(url: string): void { + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = "kandev-diagnostic-logs.zip"; + document.body.appendChild(anchor); + anchor.click(); + anchor.remove(); +} + +function delay(milliseconds: number): Promise { + return new Promise((resolve) => window.setTimeout(resolve, milliseconds)); +} diff --git a/apps/web/components/task/chat/queued-ghost-list.tsx b/apps/web/components/task/chat/queued-ghost-list.tsx index 43d3be88bb..b98e337e9e 100644 --- a/apps/web/components/task/chat/queued-ghost-list.tsx +++ b/apps/web/components/task/chat/queued-ghost-list.tsx @@ -3,7 +3,7 @@ import { useCallback, useEffect, useRef, useState } from "react"; import type { ReactNode } from "react"; import { IconLayoutList, IconPlayerPlay, IconTrash, IconX } from "@tabler/icons-react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useTranslation } from "react-i18next"; import { Button } from "@kandev/ui"; import { Collapsible, CollapsibleContent } from "@kandev/ui/collapsible"; diff --git a/apps/web/components/task/chat/queued-ghost-message.tsx b/apps/web/components/task/chat/queued-ghost-message.tsx index f1648cab0c..6f8d8e6857 100644 --- a/apps/web/components/task/chat/queued-ghost-message.tsx +++ b/apps/web/components/task/chat/queued-ghost-message.tsx @@ -22,7 +22,7 @@ import { IconArrowMerge, } from "@tabler/icons-react"; import ReactMarkdown from "react-markdown"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useTranslation } from "react-i18next"; import { Button } from "@kandev/ui"; import { Textarea } from "@kandev/ui/textarea"; diff --git a/apps/web/components/task/executor-environment-info.tsx b/apps/web/components/task/executor-environment-info.tsx index 657b6fc47f..4fa9f382dd 100644 --- a/apps/web/components/task/executor-environment-info.tsx +++ b/apps/web/components/task/executor-environment-info.tsx @@ -3,7 +3,7 @@ import { useMemo } from "react"; import { IconCopy, IconLoader } from "@tabler/icons-react"; import { Badge } from "@kandev/ui/badge"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { type ContainerLiveStatus, diff --git a/apps/web/components/task/layout-preset-selector.tsx b/apps/web/components/task/layout-preset-selector.tsx index bcb6e08742..d7d8a82f6e 100644 --- a/apps/web/components/task/layout-preset-selector.tsx +++ b/apps/web/components/task/layout-preset-selector.tsx @@ -1,7 +1,7 @@ "use client"; import { useCallback, useRef, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { IconLayout, IconCheck, diff --git a/apps/web/components/task/port-forward-dialog.tsx b/apps/web/components/task/port-forward-dialog.tsx index 5e18d81464..00d94a61fc 100644 --- a/apps/web/components/task/port-forward-dialog.tsx +++ b/apps/web/components/task/port-forward-dialog.tsx @@ -22,7 +22,7 @@ import { listPorts, listTunnels, type ListeningPort } from "@/lib/api/domains/po import { copyToClipboard } from "@/lib/utils/copy-to-clipboard"; import { useTunnelActions } from "./use-tunnel-actions"; import { getBackendConfig } from "@/lib/config"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; function buildPortProxyUrl(sessionId: string, port: number): string { const backendUrl = getBackendConfig().apiBaseUrl; diff --git a/apps/web/components/task/sidebar-filter/sidebar-filter-bar.tsx b/apps/web/components/task/sidebar-filter/sidebar-filter-bar.tsx index d222ddcc22..6f21889242 100644 --- a/apps/web/components/task/sidebar-filter/sidebar-filter-bar.tsx +++ b/apps/web/components/task/sidebar-filter/sidebar-filter-bar.tsx @@ -3,7 +3,7 @@ import { useMemo, useRef } from "react"; import { IconAdjustments, IconPlus } from "@tabler/icons-react"; import { Button } from "@kandev/ui/button"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { useRegisterCommands } from "@/hooks/use-register-commands"; import type { CommandItem } from "@/lib/commands/types"; diff --git a/apps/web/components/task/simple/OfficeSimplePane.tsx b/apps/web/components/task/simple/OfficeSimplePane.tsx index fd2aca97e5..0d7904cd6f 100644 --- a/apps/web/components/task/simple/OfficeSimplePane.tsx +++ b/apps/web/components/task/simple/OfficeSimplePane.tsx @@ -56,7 +56,7 @@ import type { TaskSession, TimelineEvent, } from "@/app/office/tasks/[id]/types"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; const COMMENTABLE_DONE_SESSION_STATES = new Set([ "CREATED", diff --git a/apps/web/components/task/simple/components/labels-picker.tsx b/apps/web/components/task/simple/components/labels-picker.tsx index 0cb1b0306b..8f64ba39d1 100644 --- a/apps/web/components/task/simple/components/labels-picker.tsx +++ b/apps/web/components/task/simple/components/labels-picker.tsx @@ -4,7 +4,7 @@ import { useRef, useState } from "react"; import { IconPlus, IconX } from "@tabler/icons-react"; import { Button } from "@kandev/ui/button"; import { Popover, PopoverContent, PopoverTrigger } from "@kandev/ui/popover"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStore } from "@/components/state-provider"; import { addLabel, removeLabel } from "@/lib/api/domains/office-extended-api"; import { useOptimisticTaskMutation } from "@/hooks/use-optimistic-task-mutation"; diff --git a/apps/web/components/task/simple/task-chat.tsx b/apps/web/components/task/simple/task-chat.tsx index a89ea4f35a..18d4100d3f 100644 --- a/apps/web/components/task/simple/task-chat.tsx +++ b/apps/web/components/task/simple/task-chat.tsx @@ -1,7 +1,7 @@ "use client"; import { useEffect, useMemo, useRef, useState, useCallback } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { IconCode, IconChevronDown, IconSend, IconPaperclip, IconUser } from "@tabler/icons-react"; import { AgentAvatar } from "@/app/office/components/agent-avatar"; import { Button } from "@kandev/ui/button"; diff --git a/apps/web/components/task/simple/task-documents.tsx b/apps/web/components/task/simple/task-documents.tsx index 8f1f2960a4..5011c0e773 100644 --- a/apps/web/components/task/simple/task-documents.tsx +++ b/apps/web/components/task/simple/task-documents.tsx @@ -10,7 +10,7 @@ import { } from "@tabler/icons-react"; import { Button } from "@kandev/ui/button"; import { Textarea } from "@kandev/ui/textarea"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { formatRelativeTime } from "@/lib/utils"; import { listDocuments, diff --git a/apps/web/components/task/task-plan-revisions.tsx b/apps/web/components/task/task-plan-revisions.tsx index 782d7f45c2..4476cdbb26 100644 --- a/apps/web/components/task/task-plan-revisions.tsx +++ b/apps/web/components/task/task-plan-revisions.tsx @@ -13,7 +13,7 @@ import { DialogHeader, DialogTitle, } from "@kandev/ui/dialog"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import type { TaskPlanRevision } from "@/lib/types/http"; import { formatPreciseTime } from "@/lib/utils"; import { AgentLogo } from "@/components/agent-logo"; diff --git a/apps/web/components/task/use-tunnel-actions.ts b/apps/web/components/task/use-tunnel-actions.ts index 894da77f0e..d085c4d479 100644 --- a/apps/web/components/task/use-tunnel-actions.ts +++ b/apps/web/components/task/use-tunnel-actions.ts @@ -2,7 +2,7 @@ import { useState, useCallback } from "react"; import { startTunnel, stopTunnel } from "@/lib/api/domains/port-api"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; export function useTunnelActions( sessionId: string, diff --git a/apps/web/components/toast-provider.test.tsx b/apps/web/components/toast-provider.test.tsx new file mode 100644 index 0000000000..794aa91d53 --- /dev/null +++ b/apps/web/components/toast-provider.test.tsx @@ -0,0 +1,79 @@ +import { act, render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const report = vi.hoisted(() => vi.fn()); + +vi.mock("@/lib/api/domains/frontend-error-log-api", () => ({ + scheduleFrontendErrorReport: report, +})); + +import { ToastProvider, useToast } from "./toast-provider"; + +describe("ToastProvider error reporting", () => { + beforeEach(() => { + report.mockReset(); + }); + + it("reports error creation without changing the rendered toast", () => { + const api = renderProvider(); + let id = ""; + act(() => { + id = api.toast({ + title: "Failed to save", + description: "The backend rejected the update", + variant: "error", + }); + }); + + expect(id).not.toBe(""); + expect(screen.getByText("Failed to save")).toBeTruthy(); + expect(screen.getByText("The backend rejected the update")).toBeTruthy(); + expect(report).toHaveBeenCalledOnce(); + expect(report).toHaveBeenCalledWith({ + source: "toast-provider", + title: "Failed to save", + description: "The backend rejected the update", + }); + }); + + it("reports each transition into error exactly once", () => { + const api = renderProvider(); + let id = ""; + act(() => { + id = api.toast({ title: "Saving", variant: "loading" }); + }); + expect(report).not.toHaveBeenCalled(); + + act(() => api.updateToast(id, { title: "First failure", variant: "error" })); + act(() => api.updateToast(id, { description: "More detail", variant: "error" })); + expect(report).toHaveBeenCalledTimes(1); + expect(report).toHaveBeenLastCalledWith({ + source: "toast-provider", + title: "First failure", + description: undefined, + }); + + act(() => api.updateToast(id, { title: "Recovered", variant: "success" })); + act(() => api.updateToast(id, { title: "Failed again", variant: "error" })); + expect(report).toHaveBeenCalledTimes(2); + expect(report).toHaveBeenLastCalledWith({ + source: "toast-provider", + title: "Failed again", + description: "More detail", + }); + }); +}); + +function renderProvider() { + let current!: ReturnType; + function Capture() { + current = useToast(); + return null; + } + render( + + + , + ); + return current; +} diff --git a/apps/web/components/toast-provider.tsx b/apps/web/components/toast-provider.tsx index cf0b68194f..034cbac52d 100644 --- a/apps/web/components/toast-provider.tsx +++ b/apps/web/components/toast-provider.tsx @@ -3,6 +3,7 @@ import { createContext, useCallback, useContext, useMemo, useRef, useState } from "react"; import { IconCheck, IconX, IconInfoCircle, IconLoader2 } from "@tabler/icons-react"; import { cn, generateUUID } from "@/lib/utils"; +import { scheduleFrontendErrorReport } from "@/lib/api/domains/frontend-error-log-api"; type ToastVariant = "default" | "success" | "error" | "loading"; @@ -53,10 +54,12 @@ const variantStyles: Record< export function ToastProvider({ children }: { children: React.ReactNode }) { const [toasts, setToasts] = useState([]); const timersRef = useRef(new Map>()); + const toastsRef = useRef(new Map()); const removeToast = useCallback((id: string) => { setToasts((prev) => prev.filter((toast) => toast.id !== id)); timersRef.current.delete(id); + toastsRef.current.delete(id); }, []); const scheduleRemoval = useCallback( @@ -78,7 +81,15 @@ export function ToastProvider({ children }: { children: React.ReactNode }) { description: input.description, variant: input.variant ?? "default", }; + toastsRef.current.set(id, nextToast); setToasts((prev) => [...prev, nextToast]); + if (nextToast.variant === "error") { + scheduleFrontendErrorReport({ + source: "toast-provider", + title: nextToast.title, + description: nextToast.description, + }); + } // Loading toasts don't auto-dismiss if (input.variant !== "loading") { scheduleRemoval(id, input.duration ?? 4000); @@ -90,18 +101,24 @@ export function ToastProvider({ children }: { children: React.ReactNode }) { const updateToast = useCallback( (id: string, input: Partial) => { - setToasts((prev) => - prev.map((t) => - t.id === id - ? { - ...t, - ...(input.title !== undefined && { title: input.title }), - ...(input.description !== undefined && { description: input.description }), - ...(input.variant !== undefined && { variant: input.variant }), - } - : t, - ), - ); + const previous = toastsRef.current.get(id); + if (previous) { + const next = { + ...previous, + ...(input.title !== undefined && { title: input.title }), + ...(input.description !== undefined && { description: input.description }), + ...(input.variant !== undefined && { variant: input.variant }), + }; + toastsRef.current.set(id, next); + setToasts((current) => current.map((item) => (item.id === id ? next : item))); + if (next.variant === "error" && previous.variant !== "error") { + scheduleFrontendErrorReport({ + source: "toast-provider", + title: next.title, + description: next.description, + }); + } + } // When transitioning away from loading, schedule auto-dismiss if (input.variant && input.variant !== "loading") { scheduleRemoval(id, input.duration ?? 4000); diff --git a/apps/web/e2e/tests/improve-kandev.spec.ts b/apps/web/e2e/tests/improve-kandev.spec.ts index 35091fa3d4..1245d516bb 100644 --- a/apps/web/e2e/tests/improve-kandev.spec.ts +++ b/apps/web/e2e/tests/improve-kandev.spec.ts @@ -9,7 +9,6 @@ import { test, expect } from "../fixtures/test-base"; */ const BOOTSTRAP_URL = "**/api/v1/system/improve-kandev/bootstrap"; -const FRONTEND_LOG_URL = "**/api/v1/system/improve-kandev/bundle/frontend-log"; const HEALTH_URL = "**/api/v1/system/health"; type ForkStatus = "writable" | "ready" | "blocked_emu" | "unknown"; @@ -61,11 +60,7 @@ async function mockImproveKandevApis( issue_workflow_id: overrides.issueWorkflowId ?? seed.workflowId, branch: "main", bundle_dir: bundleDir, - bundle_files: { - metadata: `${bundleDir}/metadata.json`, - backend_log: `${bundleDir}/backend.log`, - frontend_log: `${bundleDir}/frontend.log`, - }, + bundle_file: `${bundleDir}/diagnostic-bundle.zip`, github_login: overrides.github_login ?? "octocat", has_write_access: hasWrite, fork_status: forkStatus, @@ -73,14 +68,6 @@ async function mockImproveKandevApis( }), }); }); - - await page.route(FRONTEND_LOG_URL, (route) => - route.fulfill({ - status: 200, - contentType: "application/json", - body: JSON.stringify({ path: `${bundleDir}/frontend.log` }), - }), - ); } test.describe("Improve Kandev dialog", () => { diff --git a/apps/web/e2e/tests/mobile-improve-kandev.spec.ts b/apps/web/e2e/tests/mobile-improve-kandev.spec.ts index 3b7e25646a..7a19f1ae4b 100644 --- a/apps/web/e2e/tests/mobile-improve-kandev.spec.ts +++ b/apps/web/e2e/tests/mobile-improve-kandev.spec.ts @@ -25,11 +25,7 @@ test.describe("Improve Kandev on mobile", () => { issue_workflow_id: seedData.workflowId, branch: "main", bundle_dir: "/tmp/kandev-improve-mobile-e2e", - bundle_files: { - metadata: "/tmp/kandev-improve-mobile-e2e/metadata.json", - backend_log: "/tmp/kandev-improve-mobile-e2e/backend.log", - frontend_log: "/tmp/kandev-improve-mobile-e2e/frontend.log", - }, + bundle_file: "/tmp/kandev-improve-mobile-e2e/diagnostic-bundle.zip", github_login: "octocat", has_write_access: false, fork_status: "unknown", diff --git a/apps/web/e2e/tests/system/logs-page.spec.ts b/apps/web/e2e/tests/system/logs-page.spec.ts index 28508fd7b4..c4012e57e6 100644 --- a/apps/web/e2e/tests/system/logs-page.spec.ts +++ b/apps/web/e2e/tests/system/logs-page.spec.ts @@ -1,27 +1,134 @@ +import { readFile } from "node:fs/promises"; +import { inflateRawSync } from "node:zlib"; import { test, expect } from "../../fixtures/test-base"; test.describe("System Logs page", () => { - test("renders the tail viewer card and refresh works without error", async ({ testPage }) => { - test.setTimeout(30_000); - + test("customizes and downloads the default frontend and backend diagnostic ZIP", async ({ + testPage, + prCapture, + }) => { + test.setTimeout(45_000); await testPage.goto("/settings/system/logs"); await expect(testPage.getByTestId("system-page-title")).toHaveText("Logs"); - await expect(testPage.getByTestId("system-log-tail-card")).toBeVisible(); + await expect( + testPage.getByText("Create a diagnostic ZIP with frontend and backend logs."), + ).toBeVisible(); + await expect(testPage.getByText("Review before sharing")).toBeVisible(); + await expect(testPage.getByTestId("system-log-tail-card")).toHaveCount(0); + await prCapture.screenshot("desktop-combined-diagnostic-logs", { + caption: "System Logs clearly discloses the combined frontend and backend ZIP.", + fullPage: true, + }); - const refreshButton = testPage.getByTestId("system-log-tail-refresh"); - await expect(refreshButton).toBeVisible(); + await expect(testPage.getByTestId("download-diagnostic-bundle")).toHaveCount(0); + await testPage.getByTestId("customize-diagnostic-bundle").click(); + const dialog = testPage.getByTestId("diagnostic-bundle-dialog"); + const downloadPromise = testPage.waitForEvent("download"); + await dialog.getByTestId("create-custom-diagnostic-bundle").click(); + const download = await downloadPromise; + expect(download.suggestedFilename()).toBe("kandev-diagnostic-logs.zip"); + const archivePath = await download.path(); + expect(archivePath).not.toBeNull(); + const archive = readStoredZip(await readFile(archivePath!)); + expect(archive.has("manifest.json")).toBe(true); + expect([...archive.keys()].some((name) => name.startsWith("backend/"))).toBe(true); + expect([...archive.keys()].some((name) => name.startsWith("frontend/"))).toBe(true); + const manifest = JSON.parse(archive.get("manifest.json")!.toString("utf8")) as { + requested_sources: string[]; + }; + expect(manifest.requested_sources).toEqual(["backend", "frontend"]); + }); - const tailRequest = testPage.waitForRequest( - (req) => req.url().includes("/api/v1/system/logs/tail") && req.method() === "GET", - { timeout: 10_000 }, + test("opens one wider source customizer without enabling ACP from the browser", async ({ + testPage, + prCapture, + }) => { + await testPage.setViewportSize({ width: 1440, height: 900 }); + await testPage.goto("/settings/system/logs"); + await testPage.getByTestId("customize-diagnostic-bundle").click(); + const dialog = testPage.getByTestId("diagnostic-bundle-dialog"); + await expect(dialog).toBeVisible(); + await expect(dialog.getByText("Runtime index")).toBeVisible(); + await expect(testPage.getByTestId("download-diagnostic-bundle")).toHaveCount(0); + await expect(testPage.getByTestId("download-diagnostic-bundle-with-acp")).toHaveCount(0); + const box = await dialog.boundingBox(); + expect(box?.width ?? 0).toBeGreaterThan(800); + await prCapture.screenshot("desktop-diagnostic-bundle-customizer", { + caption: "The wider diagnostic bundle customizer keeps source choices easy to scan.", + }); + }); + + test("keeps privacy notes stacked and the bundle action on the content axis", async ({ + testPage, + }) => { + await testPage.setViewportSize({ width: 1440, height: 900 }); + await testPage.goto("/settings/system/logs"); + + const noMessages = testPage.getByText( + "Standard bundles do not include chat transcripts, session messages, or agent messages.", ); - await refreshButton.click(); - await tailRequest; + const incidentalText = testPage.getByText( + "Incidental text already written into a log entry is not automatically redacted. Review the ZIP before sharing.", + ); + const description = testPage.getByText( + "Kandev asks your connected browser tabs for their bounded three-day console history, combines it with retained backend log files, and downloads one ZIP.", + ); + const action = testPage.getByTestId("customize-diagnostic-bundle"); + const [noMessagesBox, incidentalTextBox, descriptionBox, actionBox] = await Promise.all([ + noMessages.boundingBox(), + incidentalText.boundingBox(), + description.boundingBox(), + action.boundingBox(), + ]); - // Either the tail content or the empty-tail message is rendered; both are valid. - const content = testPage.getByTestId("system-log-tail-content"); - const empty = testPage.getByTestId("system-log-tail-empty"); - await expect(content.or(empty)).toBeVisible({ timeout: 10_000 }); + expect(noMessagesBox).not.toBeNull(); + expect(incidentalTextBox).not.toBeNull(); + expect(descriptionBox).not.toBeNull(); + expect(actionBox).not.toBeNull(); + expect(incidentalTextBox!.y).toBeGreaterThan(noMessagesBox!.y + noMessagesBox!.height); + expect(Math.abs(actionBox!.x - descriptionBox!.x)).toBeLessThanOrEqual(1); }); }); + +function readStoredZip(buffer: Buffer): Map { + const entries = new Map(); + const endOfCentralDirectory = buffer.lastIndexOf(Buffer.from("PK\x05\x06", "binary")); + if (endOfCentralDirectory < 0) throw new Error("diagnostic ZIP has no central directory"); + const count = buffer.readUInt16LE(endOfCentralDirectory + 10); + let offset = buffer.readUInt32LE(endOfCentralDirectory + 16); + for (let index = 0; index < count; index += 1) { + if (buffer.readUInt32LE(offset) !== 0x02014b50) { + throw new Error("diagnostic ZIP central directory is truncated"); + } + const flags = buffer.readUInt16LE(offset + 8); + const method = buffer.readUInt16LE(offset + 10); + const compressedSize = buffer.readUInt32LE(offset + 20); + const nameLength = buffer.readUInt16LE(offset + 28); + const extraLength = buffer.readUInt16LE(offset + 30); + const commentLength = buffer.readUInt16LE(offset + 32); + const localOffset = buffer.readUInt32LE(offset + 42); + const nameStart = offset + 46; + const name = buffer.subarray(nameStart, nameStart + nameLength).toString("utf8"); + if (flags & 0x1) throw new Error("diagnostic ZIP entry is encrypted"); + if (buffer.readUInt32LE(localOffset) !== 0x04034b50) { + throw new Error("diagnostic ZIP local entry is truncated"); + } + const localNameLength = buffer.readUInt16LE(localOffset + 26); + const localExtraLength = buffer.readUInt16LE(localOffset + 28); + const dataStart = localOffset + 30 + localNameLength + localExtraLength; + const dataEnd = dataStart + compressedSize; + if (dataEnd > buffer.length) throw new Error("diagnostic ZIP entry is truncated"); + const compressed = buffer.subarray(dataStart, dataEnd); + let data: Buffer | null = null; + if (method === 0) { + data = compressed; + } else if (method === 8) { + data = inflateRawSync(compressed); + } + if (!data) throw new Error(`diagnostic ZIP uses unsupported method=${method}`); + entries.set(name, data); + offset += 46 + nameLength + extraLength + commentLength; + } + return entries; +} diff --git a/apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts b/apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts new file mode 100644 index 0000000000..c61f4f7ea6 --- /dev/null +++ b/apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts @@ -0,0 +1,61 @@ +import { test, expect } from "../../fixtures/test-base"; + +test.describe("System Logs mobile", () => { + test("keeps the one customizer action touch-accessible without horizontal overflow", async ({ + testPage, + prCapture, + }) => { + await testPage.setViewportSize({ width: 390, height: 844 }); + await testPage.goto("/settings/system/logs"); + + const action = testPage.getByTestId("customize-diagnostic-bundle"); + await expect(action).toBeVisible(); + expect((await action.boundingBox())?.height ?? 0).toBeGreaterThanOrEqual(44); + await action.tap(); + const drawer = testPage.getByTestId("diagnostic-bundle-drawer"); + await expect(drawer).toBeVisible(); + await expect(testPage.getByTestId("download-diagnostic-bundle")).toHaveCount(0); + const overflow = await testPage.evaluate( + () => document.documentElement.scrollWidth > document.documentElement.clientWidth, + ); + expect(overflow).toBe(false); + await prCapture.screenshot("mobile-combined-diagnostic-logs", { + caption: "The one diagnostic customizer remains touch-accessible on mobile.", + fullPage: true, + }); + }); + + test("opens the customizer in an inset drawer with the standard source defaults", async ({ + testPage, + }) => { + await testPage.setViewportSize({ width: 390, height: 844 }); + await testPage.goto("/settings/system/logs"); + + const customize = testPage.getByTestId("customize-diagnostic-bundle"); + await expect(customize).toBeVisible(); + await customize.tap(); + + const drawer = testPage.getByTestId("diagnostic-bundle-drawer"); + await expect(drawer).toBeVisible(); + await expect(drawer.getByRole("heading", { name: "Evidence sources" })).toBeVisible(); + await expect(drawer.getByRole("checkbox", { name: "Backend logs" })).toHaveAttribute( + "data-state", + "checked", + ); + await expect(drawer.getByRole("checkbox", { name: "Frontend logs" })).toHaveAttribute( + "data-state", + "checked", + ); + await expect(drawer.getByRole("checkbox", { name: "Runtime index" })).toHaveAttribute( + "data-state", + "unchecked", + ); + const create = drawer.getByTestId("create-custom-diagnostic-bundle"); + expect((await create.boundingBox())?.height ?? 0).toBeGreaterThanOrEqual(44); + expect( + await testPage.evaluate( + () => document.documentElement.scrollWidth > document.documentElement.clientWidth, + ), + ).toBe(false); + }); +}); diff --git a/apps/web/hooks/domains/plugins/use-auto-update-settings.ts b/apps/web/hooks/domains/plugins/use-auto-update-settings.ts index 38246fb18c..65750d5ce8 100644 --- a/apps/web/hooks/domains/plugins/use-auto-update-settings.ts +++ b/apps/web/hooks/domains/plugins/use-auto-update-settings.ts @@ -1,7 +1,7 @@ "use client"; import { useEffect, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { getPluginSettings, updatePluginSettings } from "@/lib/api/domains/plugins-api"; export type AutoUpdateSettings = { diff --git a/apps/web/hooks/domains/session/use-task-environment.ts b/apps/web/hooks/domains/session/use-task-environment.ts index c308e0dce1..b3d372ee92 100644 --- a/apps/web/hooks/domains/session/use-task-environment.ts +++ b/apps/web/hooks/domains/session/use-task-environment.ts @@ -1,5 +1,5 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { fetchTaskEnvironmentLive, diff --git a/apps/web/hooks/domains/settings/use-automation-runs.ts b/apps/web/hooks/domains/settings/use-automation-runs.ts index 99222a4a15..bcd72da7e5 100644 --- a/apps/web/hooks/domains/settings/use-automation-runs.ts +++ b/apps/web/hooks/domains/settings/use-automation-runs.ts @@ -1,7 +1,7 @@ "use client"; import { useEffect, useCallback } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { listAutomationRuns, deleteAutomationRun, diff --git a/apps/web/hooks/domains/system/use-log-files.ts b/apps/web/hooks/domains/system/use-log-files.ts deleted file mode 100644 index 7762bc1b85..0000000000 --- a/apps/web/hooks/domains/system/use-log-files.ts +++ /dev/null @@ -1,34 +0,0 @@ -"use client"; - -import { useCallback, useEffect, useState } from "react"; -import { useAppStore } from "@/components/state-provider"; -import { fetchLogFiles } from "@/lib/api/domains/system-api"; - -export function useLogFiles() { - const files = useAppStore((s) => s.system.logs.files); - const setSystemLogs = useAppStore((s) => s.setSystemLogs); - const [isLoading, setIsLoading] = useState(false); - const [error, setError] = useState(null); - const [fetched, setFetched] = useState(false); - - const reload = useCallback(async () => { - setIsLoading(true); - setError(null); - try { - const res = await fetchLogFiles({ cache: "no-store" }); - setSystemLogs(res ?? []); - setFetched(true); - } catch (e) { - setError(e instanceof Error ? e.message : String(e)); - } finally { - setIsLoading(false); - } - }, [setSystemLogs]); - - useEffect(() => { - if (fetched) return; - void reload(); - }, [fetched, reload]); - - return { files, isLoading, error, reload }; -} diff --git a/apps/web/hooks/domains/system/use-log-tail.ts b/apps/web/hooks/domains/system/use-log-tail.ts deleted file mode 100644 index ac652732b8..0000000000 --- a/apps/web/hooks/domains/system/use-log-tail.ts +++ /dev/null @@ -1,37 +0,0 @@ -"use client"; - -import { useCallback, useEffect, useState } from "react"; -import { useAppStore } from "@/components/state-provider"; -import { fetchLogTail } from "@/lib/api/domains/system-api"; - -/** - * Fetches the last `n` lines of the current lumberjack log. The Logs page - * also exposes a Refresh button which re-invokes `reload()`. - */ -export function useLogTail(n = 1000) { - const tail = useAppStore((s) => s.system.logs.tail); - const tailLoaded = useAppStore((s) => s.system.logs.tailLoaded); - const setSystemLogTail = useAppStore((s) => s.setSystemLogTail); - const [isLoading, setIsLoading] = useState(false); - const [error, setError] = useState(null); - - const reload = useCallback(async () => { - setIsLoading(true); - setError(null); - try { - const res = await fetchLogTail(n); - setSystemLogTail(res?.lines ?? []); - } catch (e) { - setError(e instanceof Error ? e.message : String(e)); - } finally { - setIsLoading(false); - } - }, [n, setSystemLogTail]); - - useEffect(() => { - if (tailLoaded) return; - void reload(); - }, [tailLoaded, reload]); - - return { tail, loaded: tailLoaded, isLoading, error, reload }; -} diff --git a/apps/web/hooks/use-detach-task.ts b/apps/web/hooks/use-detach-task.ts index 089a255bb3..339dedfee4 100644 --- a/apps/web/hooks/use-detach-task.ts +++ b/apps/web/hooks/use-detach-task.ts @@ -2,7 +2,7 @@ import { useCallback, useState } from "react"; import type { StoreApi } from "zustand"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { detachTask as requestDetachTask } from "@/lib/api"; import type { Task } from "@/lib/types/http"; import type { AppState } from "@/lib/state/store"; diff --git a/apps/web/hooks/use-nest-task.ts b/apps/web/hooks/use-nest-task.ts index fd27885369..3ab3fe7282 100644 --- a/apps/web/hooks/use-nest-task.ts +++ b/apps/web/hooks/use-nest-task.ts @@ -1,7 +1,7 @@ "use client"; import { useCallback } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { detachTask, updateTask } from "@/lib/api"; import { useAppStoreApi } from "@/components/state-provider"; import type { WorkflowSnapshotData } from "@/lib/state/slices/kanban/types"; diff --git a/apps/web/hooks/use-optimistic-task-mutation.ts b/apps/web/hooks/use-optimistic-task-mutation.ts index c17d546ed4..b181afdba6 100644 --- a/apps/web/hooks/use-optimistic-task-mutation.ts +++ b/apps/web/hooks/use-optimistic-task-mutation.ts @@ -1,7 +1,7 @@ "use client"; import { createContext, useCallback, useContext } from "react"; -import { toast } from "sonner"; +import { toast } from "@/lib/toast/sonner"; import { useAppStoreApi } from "@/components/state-provider"; import type { Task } from "@/app/office/tasks/[id]/types"; import type { OfficeTask } from "@/lib/state/slices/office/types"; diff --git a/apps/web/lib/api/client.ts b/apps/web/lib/api/client.ts index d8f828964c..2ce7998d27 100644 --- a/apps/web/lib/api/client.ts +++ b/apps/web/lib/api/client.ts @@ -17,11 +17,13 @@ export type ApiRequestOptions = { export class ApiError extends Error { readonly status: number; readonly body: unknown; - constructor(message: string, status: number, body: unknown) { + readonly retryAfterSeconds?: number; + constructor(message: string, status: number, body: unknown, retryAfterSeconds?: number) { super(message); this.name = "ApiError"; this.status = status; this.body = body; + this.retryAfterSeconds = retryAfterSeconds; } } @@ -59,7 +61,13 @@ async function throwFromResponse(response: Response): Promise { const errVal = (body as { error?: unknown }).error; if (typeof errVal === "string") message = errVal; } - throw new ApiError(message, response.status, body); + const retryAfter = Number(response.headers.get("Retry-After")); + throw new ApiError( + message, + response.status, + body, + Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : undefined, + ); } export async function fetchJson(pathOrUrl: string, options?: ApiRequestOptions): Promise { diff --git a/apps/web/lib/api/domains/frontend-error-log-api.test.ts b/apps/web/lib/api/domains/frontend-error-log-api.test.ts new file mode 100644 index 0000000000..7e65fe0c31 --- /dev/null +++ b/apps/web/lib/api/domains/frontend-error-log-api.test.ts @@ -0,0 +1,72 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + buildFrontendErrorReport, + deriveTaskID, + scheduleFrontendErrorReport, + sendFrontendErrorReport, +} from "./frontend-error-log-api"; + +describe("frontend error log API", () => { + afterEach(() => { + vi.restoreAllMocks(); + window.history.replaceState({}, "", "/"); + }); + + it.each([ + ["/t/task-one", "task-one"], + ["/tasks/task-two", "task-two"], + ["/office/tasks/task-three", "task-three"], + ["/settings?taskId=query-task", "query-task"], + ["/office/projects/task-four", undefined], + ])("derives task IDs only from recognized routes: %s", (route, expected) => { + expect(deriveTaskID(new URL(route, "http://localhost"))).toBe(expected); + }); + + it("builds bounded browser context without serializing arbitrary React nodes", () => { + window.history.replaceState({}, "", "/t/task-one?token=secret#private"); + const report = buildFrontendErrorReport({ + source: "sonner", + title: "Failed to save", + description: { type: "div", props: { secret: "must-not-be-walked" } }, + error: new TypeError("Failed to fetch"), + }); + + expect(report.source).toBe("sonner"); + expect(report.title).toBe("Failed to save"); + expect(report.description).toBe("[object]"); + expect(report.task_id).toBe("task-one"); + expect(report.error).toMatchObject({ name: "TypeError", message: "Failed to fetch" }); + expect(report.url).toBe(`${window.location.origin}/t/task-one`); + expect(report.url).not.toContain("secret"); + expect(report.viewport).toEqual({ width: window.innerWidth, height: window.innerHeight }); + expect(JSON.stringify(report)).not.toContain("must-not-be-walked"); + }); + + it("uses an authenticated two-second request and silently accepts 204", async () => { + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(new Response(null, { status: 204 })); + await sendFrontendErrorReport({ + source: "sonner", + title: "Failed", + client_timestamp: new Date().toISOString(), + }); + + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining("/api/v1/system/logs/frontend-errors"), + expect.objectContaining({ + method: "POST", + credentials: "include", + signal: expect.any(AbortSignal), + }), + ); + }); + + it("schedules collection off the toast call stack and swallows failures", async () => { + const fetchMock = vi.spyOn(globalThis, "fetch").mockRejectedValue(new Error("offline")); + scheduleFrontendErrorReport({ source: "toast-provider", title: "Failed" }); + expect(fetchMock).not.toHaveBeenCalled(); + await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledOnce()); + }); +}); diff --git a/apps/web/lib/api/domains/frontend-error-log-api.ts b/apps/web/lib/api/domains/frontend-error-log-api.ts new file mode 100644 index 0000000000..3f174f1143 --- /dev/null +++ b/apps/web/lib/api/domains/frontend-error-log-api.ts @@ -0,0 +1,189 @@ +import { fetchJson } from "@/lib/api/client"; + +const REPORT_PATH = "/api/v1/system/logs/frontend-errors"; +const REQUEST_TIMEOUT_MS = 2_000; +const TEXT_LIMIT = 8 * 1024; +const TASK_ID_LIMIT = 128; +const BROWSER_FIELD_LIMIT = 2 * 1024; +const STACK_LIMIT = 16 * 1024; +const MAX_REACT_TEXT_NODES = 32; + +export type FrontendErrorSource = "sonner" | "toast-provider"; + +export type FrontendErrorReportInput = { + source: FrontendErrorSource; + title?: unknown; + description?: unknown; + error?: unknown; +}; + +export type FrontendErrorReport = { + client_timestamp: string; + source: FrontendErrorSource; + task_id?: string; + title?: string; + description?: string; + url?: string; + user_agent?: string; + language?: string; + platform?: string; + viewport?: { width: number; height: number }; + stack?: string; + error?: { name: string; message: string; stack?: string }; +}; + +export function scheduleFrontendErrorReport(input: FrontendErrorReportInput): void { + queueMicrotask(() => { + try { + const report = buildFrontendErrorReport(input); + void sendFrontendErrorReport(report).catch(() => {}); + } catch { + // Diagnostics must never alter or recursively log the original toast. + } + }); +} + +export function buildFrontendErrorReport(input: FrontendErrorReportInput): FrontendErrorReport { + const currentURL = readCurrentURL(); + const report: FrontendErrorReport = { + client_timestamp: new Date().toISOString(), + source: input.source, + title: visibleText(input.title), + description: visibleText(input.description), + stack: bounded(new Error("error toast emitted").stack, STACK_LIMIT), + }; + if (currentURL) { + report.task_id = deriveTaskID(currentURL); + report.url = bounded(`${currentURL.origin}${currentURL.pathname}`, TEXT_LIMIT); + } + addBrowserContext(report); + const error = errorDetails(input.error); + if (error) report.error = error; + return report; +} + +export async function sendFrontendErrorReport(report: FrontendErrorReport): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + await fetchJson(REPORT_PATH, { + init: { + method: "POST", + body: JSON.stringify(report), + signal: controller.signal, + }, + }); + } finally { + clearTimeout(timeout); + } +} + +export function deriveTaskID(url: URL): string | undefined { + const segments = url.pathname.split("/").filter(Boolean); + let candidate: string | null = null; + if ((segments[0] === "t" || segments[0] === "tasks") && segments.length >= 2) { + candidate = segments[1]; + } else if (segments[0] === "office" && segments[1] === "tasks" && segments.length >= 3) { + candidate = segments[2]; + } else { + candidate = url.searchParams.get("taskId"); + } + if (!candidate) return undefined; + try { + return bounded(decodeURIComponent(candidate), TASK_ID_LIMIT); + } catch { + return bounded(candidate, TASK_ID_LIMIT); + } +} + +function addBrowserContext(report: FrontendErrorReport): void { + if (typeof navigator !== "undefined") { + report.user_agent = bounded(navigator.userAgent, BROWSER_FIELD_LIMIT); + report.language = bounded(navigator.language, BROWSER_FIELD_LIMIT); + report.platform = bounded(navigator.platform, BROWSER_FIELD_LIMIT); + } + if (typeof window !== "undefined") { + report.viewport = { width: window.innerWidth, height: window.innerHeight }; + } +} + +function readCurrentURL(): URL | undefined { + if (typeof window === "undefined") return undefined; + try { + return new URL(window.location.href); + } catch { + return undefined; + } +} + +function visibleText(value: unknown): string | undefined { + const state = { nodes: 0 }; + return bounded(extractVisibleText(value, state, 0), TEXT_LIMIT); +} + +function extractVisibleText( + value: unknown, + state: { nodes: number }, + depth: number, +): string | undefined { + if (state.nodes++ >= MAX_REACT_TEXT_NODES || depth > 4 || value == null) return undefined; + switch (typeof value) { + case "string": + return value; + case "number": + case "boolean": + case "bigint": + return String(value); + case "function": + return "[function]"; + case "symbol": + return "[symbol]"; + case "object": + return extractObjectText(value, state, depth); + default: + return undefined; + } +} + +function extractObjectText( + value: object, + state: { nodes: number }, + depth: number, +): string | undefined { + try { + if (Array.isArray(value)) { + const parts: string[] = []; + const count = Math.min(value.length, MAX_REACT_TEXT_NODES); + for (let index = 0; index < count; index++) { + const item = Object.getOwnPropertyDescriptor(value, String(index))?.value; + const text = extractVisibleText(item, state, depth + 1); + if (text) parts.push(text); + } + return parts.join(" ") || "[object]"; + } + const props = Object.getOwnPropertyDescriptor(value, "props")?.value; + if (props && typeof props === "object") { + const children = Object.getOwnPropertyDescriptor(props, "children")?.value; + const text = extractVisibleText(children, state, depth + 1); + if (text) return text; + } + } catch { + return "[object]"; + } + return value instanceof Error ? bounded(value.message, TEXT_LIMIT) : "[object]"; +} + +function errorDetails(value: unknown): FrontendErrorReport["error"] | undefined { + if (!(value instanceof Error)) return undefined; + return { + name: bounded(value.name, BROWSER_FIELD_LIMIT) ?? "Error", + message: bounded(value.message, TEXT_LIMIT) ?? "", + stack: bounded(value.stack, STACK_LIMIT), + }; +} + +function bounded(value: string | undefined, limit: number): string | undefined { + if (!value) return undefined; + if (value.length <= limit) return value; + return value.slice(0, limit); +} diff --git a/apps/web/lib/api/domains/improve-kandev-api.test.ts b/apps/web/lib/api/domains/improve-kandev-api.test.ts new file mode 100644 index 0000000000..cf516bfdd0 --- /dev/null +++ b/apps/web/lib/api/domains/improve-kandev-api.test.ts @@ -0,0 +1,34 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { leaseDiagnosticBundle } from "./improve-kandev-api"; + +describe("leaseDiagnosticBundle", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("posts the server bootstrap directory and owned bundle ID", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + path: "/tmp/kandev-improve-1/diagnostic-bundle.zip", + status: "ready", + sources: ["backend", "frontend"], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ), + ); + vi.stubGlobal("fetch", fetchMock); + + const result = await leaseDiagnosticBundle("/tmp/kandev-improve-1", "bundle-1"); + + const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit]; + expect(url).toContain("/api/v1/system/improve-kandev/bundle/lease"); + expect(init.method).toBe("POST"); + expect(JSON.parse(String(init.body))).toEqual({ + bundle_dir: "/tmp/kandev-improve-1", + bundle_id: "bundle-1", + }); + expect(result.status).toBe("ready"); + }); +}); diff --git a/apps/web/lib/api/domains/improve-kandev-api.ts b/apps/web/lib/api/domains/improve-kandev-api.ts index f49789add2..1ffbdf460e 100644 --- a/apps/web/lib/api/domains/improve-kandev-api.ts +++ b/apps/web/lib/api/domains/improve-kandev-api.ts @@ -1,5 +1,4 @@ import { fetchJson, type ApiRequestOptions } from "../client"; -import type { LogEntry } from "@/lib/logger/buffer"; // Result of the bootstrap fork-capability probe; mirrors backend ForkStatus. // "writable": user has push access to upstream, no fork needed. @@ -16,11 +15,7 @@ export type ImproveKandevBootstrapResponse = { issue_workflow_id: string; branch: string; bundle_dir: string; - bundle_files: { - metadata: string; - backend_log: string; - frontend_log: string; - }; + bundle_file: string; github_login: string; has_write_access: boolean; fork_status: ForkStatus; @@ -41,32 +36,20 @@ export async function bootstrapImproveKandev( }); } -export type FrontendLogPayloadEntry = { - timestamp: string; - level: string; - message: string; - args?: unknown[]; - stack?: string; -}; - -export async function uploadFrontendLog( +export async function leaseDiagnosticBundle( bundleDir: string, - entries: LogEntry[], + bundleId: string, options?: ApiRequestOptions, -): Promise<{ path: string }> { - const payload: FrontendLogPayloadEntry[] = entries.map((e) => ({ - timestamp: e.timestamp, - level: e.level, - message: e.message, - args: e.args, - stack: e.stack, - })); - return fetchJson<{ path: string }>("/api/v1/system/improve-kandev/bundle/frontend-log", { - ...options, - init: { - method: "POST", - body: JSON.stringify({ bundle_dir: bundleDir, entries: payload }), - ...(options?.init ?? {}), +): Promise<{ path: string; status: "ready" | "partial"; sources: string[] }> { + return fetchJson<{ path: string; status: "ready" | "partial"; sources: string[] }>( + "/api/v1/system/improve-kandev/bundle/lease", + { + ...options, + init: { + method: "POST", + body: JSON.stringify({ bundle_dir: bundleDir, bundle_id: bundleId }), + ...(options?.init ?? {}), + }, }, - }); + ); } diff --git a/apps/web/lib/api/domains/system-api.test.ts b/apps/web/lib/api/domains/system-api.test.ts index cff2a69e00..afccf80096 100644 --- a/apps/web/lib/api/domains/system-api.test.ts +++ b/apps/web/lib/api/domains/system-api.test.ts @@ -18,9 +18,12 @@ import { restoreBackup, deleteBackup, buildBackupDownloadUrl, - fetchLogFiles, - fetchLogTail, - buildLogDownloadUrl, + buildDiagnosticBundleDownloadUrl, + createDiagnosticBundle, + fetchDiagnosticBundleCapabilities, + fetchDiagnosticACPSessions, + fetchDiagnosticBundle, + uploadFrontendBundleChunk, fetchUpdates, checkUpdates, applyUpdate, @@ -199,31 +202,75 @@ describe("fetchBackups / createBackup / restoreBackup / deleteBackup", () => { }); describe("logs", () => { - it("fetchLogFiles GETs /logs", async () => { - fetchSpy.mockResolvedValueOnce(jsonResponse([])); - await fetchLogFiles(); - expect(lastCall().url).toBe(`${BASE}/logs`); - expect(method()).toBe("GET"); + it("creates, inspects, uploads, and builds URLs for diagnostic bundles", async () => { + fetchSpy + .mockResolvedValueOnce(jsonResponse({ id: "bundle-1", status: "collecting" })) + .mockResolvedValueOnce(jsonResponse({ id: "bundle-1", status: "ready" })) + .mockResolvedValueOnce(new Response(null, { status: 204 })); + await createDiagnosticBundle(["backend", "frontend"]); + expect(lastCall().url).toBe(`${BASE}/logs/bundles`); + expect(method()).toBe("POST"); + expect(JSON.parse(String(lastCall().init?.body))).toEqual({ + sources: ["backend", "frontend"], + }); + await fetchDiagnosticBundle("bundle-1"); + expect(lastCall().url).toBe(`${BASE}/logs/bundles/bundle-1`); + expect(lastCall().init?.cache).toBe("no-store"); + await uploadFrontendBundleChunk("bundle-1", { + browser_id: "browser", + capture_stream_id: "stream", + chunk_index: 0, + done: true, + storage_mode: "memory", + capture_metadata: null, + entries: [], + }); + expect(lastCall().url).toBe(`${BASE}/logs/bundles/bundle-1/frontend`); + expect(buildDiagnosticBundleDownloadUrl("bundle 1")).toBe( + `${BASE}/logs/bundles/bundle%201/download`, + ); }); - it("fetchLogTail GETs /logs/tail with default n=1000 and no-store cache", async () => { - fetchSpy.mockResolvedValueOnce(jsonResponse({ lines: ["a", "b"] })); - const res = await fetchLogTail(); - const { url, init } = lastCall(); - expect(url).toBe(`${BASE}/logs/tail?n=1000`); - expect((init?.method ?? "GET").toUpperCase()).toBe("GET"); - expect(init?.cache).toBe("no-store"); - expect(res.lines).toEqual(["a", "b"]); - }); - - it("fetchLogTail uses the explicit n parameter when provided", async () => { - fetchSpy.mockResolvedValueOnce(jsonResponse({ lines: [] })); - await fetchLogTail(250); - expect(lastCall().url).toBe(`${BASE}/logs/tail?n=250`); - }); - - it("buildLogDownloadUrl returns the absolute download URL", () => { - expect(buildLogDownloadUrl("kandev.log.1")).toBe(`${BASE}/logs/kandev.log.1/download`); + it("sends selected ACP sessions and reads backend capabilities", async () => { + fetchSpy + .mockResolvedValueOnce( + jsonResponse({ + id: "bundle-acp", + status: "collecting", + sources: ["acp"], + session_ids: ["session-1"], + }), + ) + .mockResolvedValueOnce( + jsonResponse({ + sources: ["backend", "frontend", "runtime", "acp"], + acp_debug_enabled: true, + acp_max_sessions: 10, + }), + ) + .mockResolvedValueOnce( + jsonResponse({ + sessions: [ + { + task_id: "task-1", + task_title: "Repair diagnostic export", + session_id: "session-1", + }, + ], + }), + ); + await createDiagnosticBundle(["acp"], ["session-1"]); + expect(JSON.parse(String(lastCall().init?.body))).toEqual({ + sources: ["acp"], + session_ids: ["session-1"], + }); + const capabilities = await fetchDiagnosticBundleCapabilities(); + expect(lastCall().url).toBe(`${BASE}/logs/capabilities`); + expect(capabilities.acp_debug_enabled).toBe(true); + const sessions = await fetchDiagnosticACPSessions(); + expect(lastCall().url).toBe(`${BASE}/logs/acp-sessions`); + expect(sessions[0]?.session_id).toBe("session-1"); + expect(sessions[0]?.task_title).toBe("Repair diagnostic export"); }); }); diff --git a/apps/web/lib/api/domains/system-api.ts b/apps/web/lib/api/domains/system-api.ts index c0ff12e2ab..3f4056ab2c 100644 --- a/apps/web/lib/api/domains/system-api.ts +++ b/apps/web/lib/api/domains/system-api.ts @@ -5,8 +5,11 @@ import type { DiskUsageResponse, DatabaseStats, SnapshotInfo, - LogFileInfo, - LogTailResponse, + DiagnosticBundleJob, + DiagnosticBundleCapabilities, + DiagnosticBundleSource, + DiagnosticSession, + FrontendLogUploadChunk, UpdatesResponse, JobAcceptResponse, RestartCapability, @@ -126,21 +129,67 @@ export function buildBackupDownloadUrl(name: string, baseUrl?: string): string { // --- Logs --------------------------------------------------------------- -export async function fetchLogFiles(options?: ApiRequestOptions): Promise { - const res = await fetchJson<{ files: LogFileInfo[] }>(`${SYSTEM_BASE}/logs`, options); - return res.files ?? []; -} - -export function fetchLogTail(n = 1000, options?: ApiRequestOptions): Promise { - return fetchJson(`${SYSTEM_BASE}/logs/tail?n=${encodeURIComponent(String(n))}`, { - cache: "no-store", +export function createDiagnosticBundle( + sources: DiagnosticBundleSource[] = ["backend", "frontend"], + sessionIds: string[] = [], + options?: ApiRequestOptions, +): Promise { + return fetchJson(`${SYSTEM_BASE}/logs/bundles`, { ...options, + init: { + ...(options?.init ?? {}), + method: "POST", + body: JSON.stringify({ + sources, + ...(sessionIds.length > 0 ? { session_ids: sessionIds } : {}), + }), + }, }); } -export function buildLogDownloadUrl(name: string, baseUrl?: string): string { +export function fetchDiagnosticBundleCapabilities( + options?: ApiRequestOptions, +): Promise { + return fetchJson(`${SYSTEM_BASE}/logs/capabilities`, { + ...options, + cache: "no-store", + }); +} + +export async function fetchDiagnosticACPSessions( + options?: ApiRequestOptions, +): Promise { + const response = await fetchJson<{ sessions: DiagnosticSession[] }>( + `${SYSTEM_BASE}/logs/acp-sessions`, + { ...options, cache: "no-store" }, + ); + return response.sessions ?? []; +} + +export function fetchDiagnosticBundle( + id: string, + options?: ApiRequestOptions, +): Promise { + return fetchJson(`${SYSTEM_BASE}/logs/bundles/${encodeURIComponent(id)}`, { + ...options, + cache: "no-store", + }); +} + +export function uploadFrontendBundleChunk( + id: string, + chunk: FrontendLogUploadChunk, + options?: ApiRequestOptions, +): Promise { + return fetchJson(`${SYSTEM_BASE}/logs/bundles/${encodeURIComponent(id)}/frontend`, { + ...options, + init: { ...(options?.init ?? {}), method: "POST", body: JSON.stringify(chunk) }, + }); +} + +export function buildDiagnosticBundleDownloadUrl(id: string, baseUrl?: string): string { const root = baseUrl ?? getBackendConfig().apiBaseUrl; - return `${root}${SYSTEM_BASE}/logs/${encodeURIComponent(name)}/download`; + return `${root}${SYSTEM_BASE}/logs/bundles/${encodeURIComponent(id)}/download`; } // --- Jobs --------------------------------------------------------------- diff --git a/apps/web/lib/logger/buffer.test.ts b/apps/web/lib/logger/buffer.test.ts index 625f75511e..3f99de23e3 100644 --- a/apps/web/lib/logger/buffer.test.ts +++ b/apps/web/lib/logger/buffer.test.ts @@ -2,6 +2,8 @@ import { describe, it, expect, beforeEach } from "vitest"; import { DEFAULT_CAPACITY, + MAX_ENTRY_BYTES, + RingBuffer, _resetForTesting, clearLogs, getLogBuffer, @@ -59,4 +61,40 @@ describe("frontend log buffer", () => { clearLogs(); expect(snapshotLogs()).toHaveLength(0); }); + + it("partitions snapshots by authenticated identity", () => { + const buf = getLogBuffer(); + buf.push({ + timestamp: "t1", + level: "info", + source: "console", + message: "alice", + identity_scope: "alice", + }); + buf.push({ + timestamp: "t2", + level: "info", + source: "console", + message: "bob", + identity_scope: "bob", + }); + expect(snapshotLogs("alice").map((entry) => entry.message)).toEqual(["alice"]); + }); + + it("rejects an entry over 64 KiB and sheds low-priority entries first", () => { + const buf = new RingBuffer(2, MAX_ENTRY_BYTES * 2); + expect( + buf.push({ + timestamp: "t", + level: "info", + source: "console", + message: "x".repeat(MAX_ENTRY_BYTES), + }), + ).toBe(false); + buf.push({ timestamp: "1", level: "warn", source: "console", message: "warn" }); + buf.push({ timestamp: "2", level: "debug", source: "console", message: "debug" }); + buf.push({ timestamp: "3", level: "error", source: "console", message: "error" }); + expect(buf.snapshot().map((entry) => entry.message)).toEqual(["warn", "error"]); + expect(buf.statistics()).toEqual({ capacity: 1, entry_too_large: 1 }); + }); }); diff --git a/apps/web/lib/logger/buffer.ts b/apps/web/lib/logger/buffer.ts index 66bc1c2e0b..6c9e86076c 100644 --- a/apps/web/lib/logger/buffer.ts +++ b/apps/web/lib/logger/buffer.ts @@ -1,74 +1,132 @@ /** - * In-memory ring buffer of recent frontend log entries. - * - * Used to capture browser-side `console.*` calls and unhandled errors so they - * can be attached to user-submitted Improve Kandev reports without unbounded - * memory growth. + * Reference-free, bounded staging/fallback buffer for browser diagnostics. + * Console producers never await persistence or network work. */ export type LogLevel = "debug" | "info" | "warn" | "error"; export interface LogEntry { - timestamp: string; // ISO8601 + timestamp: string; level: LogLevel; - source: string; // "console" | "window.onerror" | "unhandledrejection" + source: string; message: string; - args?: unknown[]; // serialized best-effort - stack?: string; // populated when the entry originated from an Error + args?: unknown[]; + stack?: string; + url?: string; + task_id?: string; + identity_scope?: string; } +export type BufferLoss = { + capacity: number; + entry_too_large: number; +}; + export const DEFAULT_CAPACITY = 500; +export const DEFAULT_BYTE_CAPACITY = 2 * 1024 * 1024; +export const MAX_ENTRY_BYTES = 64 * 1024; -class RingBuffer { - private entries: LogEntry[] = []; - private capacity: number; +type StoredEntry = { entry: LogEntry; bytes: number }; - constructor(capacity: number = DEFAULT_CAPACITY) { - this.capacity = Math.max(1, capacity); - } +export class RingBuffer { + private entries: StoredEntry[] = []; + private bytes = 0; + private readonly loss: BufferLoss = { capacity: 0, entry_too_large: 0 }; - push(entry: LogEntry): void { - if (this.entries.length >= this.capacity) { - this.entries.shift(); + constructor( + private readonly capacity: number = DEFAULT_CAPACITY, + private readonly byteCapacity: number = DEFAULT_BYTE_CAPACITY, + ) {} + + push(entry: LogEntry): boolean { + const detached = cloneEntry(entry); + const bytes = encodedBytes(detached); + if (bytes > MAX_ENTRY_BYTES) { + this.loss.entry_too_large += 1; + return false; } - this.entries.push(entry); + while ( + this.entries.length >= Math.max(1, this.capacity) || + this.bytes + bytes > this.byteCapacity + ) { + const eviction = this.evictionIndex(entry.level); + if (eviction < 0) { + this.loss.capacity += 1; + return false; + } + const [removed] = this.entries.splice(eviction, 1); + this.bytes -= removed.bytes; + this.loss.capacity += 1; + } + this.entries.push({ entry: detached, bytes }); + this.bytes += bytes; + return true; } - snapshot(): LogEntry[] { - // Shallow-copy each entry and the args slice so callers cannot mutate - // buffered state (e.g. when the snapshot is later JSON-serialized in place). - return this.entries.map((e) => ({ - ...e, - args: e.args ? [...e.args] : undefined, - })); + snapshot(identityScope?: string): LogEntry[] { + return this.entries + .filter(({ entry }) => identityScope === undefined || entry.identity_scope === identityScope) + .map(({ entry }) => cloneEntry(entry)); } clear(): void { this.entries = []; + this.bytes = 0; } size(): number { return this.entries.length; } + + byteSize(): number { + return this.bytes; + } + + statistics(): BufferLoss { + return { ...this.loss }; + } + + private evictionIndex(incoming: LogLevel): number { + const lowPriority = this.entries.findIndex( + ({ entry }) => entry.level === "debug" || entry.level === "info", + ); + if (lowPriority >= 0) return lowPriority; + return incoming === "warn" || incoming === "error" ? 0 : -1; + } } let defaultBuffer: RingBuffer | null = null; export function getLogBuffer(): RingBuffer { - if (!defaultBuffer) { - defaultBuffer = new RingBuffer(); - } + defaultBuffer ??= new RingBuffer(); return defaultBuffer; } -export function snapshotLogs(): LogEntry[] { - return getLogBuffer().snapshot(); +export function snapshotLogs(identityScope?: string): LogEntry[] { + return getLogBuffer().snapshot(identityScope); } export function clearLogs(): void { getLogBuffer().clear(); } +export function encodedBytes(entry: LogEntry): number { + return new TextEncoder().encode(JSON.stringify(entry)).byteLength; +} + +function cloneEntry(entry: LogEntry): LogEntry { + return { + ...entry, + args: entry.args ? entry.args.map(clonePreview) : undefined, + }; +} + +function clonePreview(value: unknown): unknown { + if (!value || typeof value !== "object") return value; + if (Array.isArray(value)) return value.map(clonePreview); + return { ...(value as Record) }; +} + /** Exposed for tests. */ export function _resetForTesting(): void { defaultBuffer = new RingBuffer(); diff --git a/apps/web/lib/logger/capture.test.ts b/apps/web/lib/logger/capture.test.ts new file mode 100644 index 0000000000..6ab8f72dcc --- /dev/null +++ b/apps/web/lib/logger/capture.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { chunkEntries } from "./capture"; +import type { LogEntry } from "./buffer"; + +describe("frontend log capture chunks", () => { + it("keeps sequential chunks below the requested target", () => { + const entries: LogEntry[] = Array.from({ length: 5 }, (_, index) => ({ + timestamp: String(index), + level: "info", + source: "console", + message: "x".repeat(40), + })); + const chunks = chunkEntries(entries, 160); + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.flat()).toEqual(entries); + expect(chunks.every((chunk) => chunk.length > 0)).toBe(true); + }); +}); diff --git a/apps/web/lib/logger/capture.ts b/apps/web/lib/logger/capture.ts new file mode 100644 index 0000000000..1ef38a7f26 --- /dev/null +++ b/apps/web/lib/logger/capture.ts @@ -0,0 +1,66 @@ +import { uploadFrontendBundleChunk } from "@/lib/api/domains/system-api"; +import type { LogEntry } from "./buffer"; +import { browserInstallationID, browserLogMetadata, snapshotBrowserLogs } from "./runtime"; + +const TARGET_CHUNK_BYTES = 800 * 1024; + +export type CaptureRequest = { + bundle_id: string; + capture_deadline: string; + max_chunk_bytes: number; + max_browser_profiles: number; +}; + +export async function handleBrowserLogCapture( + request: CaptureRequest, + identityScope: string | null, +): Promise { + if (!identityScope || !request.bundle_id || Date.now() >= Date.parse(request.capture_deadline)) { + return; + } + const streamID = randomID(); + const entries = await snapshotBrowserLogs(identityScope); + const chunks = chunkEntries(entries, Math.min(TARGET_CHUNK_BYTES, request.max_chunk_bytes)); + const storageMode = browserLogMetadata().storage_mode === "memory" ? "memory" : "indexeddb"; + for (let index = 0; index < chunks.length; index++) { + const done = index === chunks.length - 1; + await uploadFrontendBundleChunk(request.bundle_id, { + browser_id: browserInstallationID(), + capture_stream_id: streamID, + chunk_index: index, + done, + storage_mode: storageMode, + capture_metadata: done ? boundedMetadata(browserLogMetadata()) : null, + entries: chunks[index], + }); + } +} + +export function chunkEntries(entries: LogEntry[], maxBytes: number): LogEntry[][] { + const chunks: LogEntry[][] = [[]]; + let currentBytes = 0; + for (const entry of entries) { + const bytes = new TextEncoder().encode(JSON.stringify(entry)).byteLength + 1; + if (chunks.at(-1)!.length > 0 && currentBytes + bytes > maxBytes) { + chunks.push([]); + currentBytes = 0; + } + chunks.at(-1)!.push(entry); + currentBytes += bytes; + } + return chunks; +} + +function boundedMetadata(metadata: Record): Record { + const serialized = JSON.stringify(metadata); + if (new TextEncoder().encode(serialized).byteLength <= 8 * 1024) return metadata; + return { storage_mode: metadata.storage_mode, metadata_truncated: true }; +} + +function randomID(): string { + try { + return crypto.randomUUID(); + } catch { + return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + } +} diff --git a/apps/web/lib/logger/indexeddb-store.test.ts b/apps/web/lib/logger/indexeddb-store.test.ts new file mode 100644 index 0000000000..28e9e63c1c --- /dev/null +++ b/apps/web/lib/logger/indexeddb-store.test.ts @@ -0,0 +1,37 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it } from "vitest"; +import { retentionPlan } from "./indexeddb-store"; + +describe("IndexedDB log retention", () => { + it("prunes age first and then oldest records until count and byte caps hold", () => { + const now = Date.UTC(2026, 6, 30); + const records = [ + { + id: 1, + identity_scope: "a", + timestamp_ms: now - 4 * 86_400_000, + bytes: 1, + entry: {} as never, + }, + ...Array.from({ length: 10_001 }, (_, index) => ({ + id: index + 2, + identity_scope: "a", + timestamp_ms: now - 1_000 + index, + bytes: 10, + entry: {} as never, + })), + ]; + const result = retentionPlan(records, now); + expect(result.removeIDs).toEqual([1, 2]); + expect(result.retainedCount).toBe(10_000); + }); + + it("walks the timestamp index without materializing the full store", () => { + const source = readFileSync("lib/logger/indexeddb-store.ts", "utf8"); + + expect(source).toContain('index("timestamp_ms")'); + expect(source).toContain("index.openCursor()"); + expect(source).not.toContain("store.getAll()"); + }); +}); diff --git a/apps/web/lib/logger/indexeddb-store.ts b/apps/web/lib/logger/indexeddb-store.ts new file mode 100644 index 0000000000..6cb5e6d410 --- /dev/null +++ b/apps/web/lib/logger/indexeddb-store.ts @@ -0,0 +1,171 @@ +import type { LogEntry } from "./buffer"; + +const DATABASE_NAME = "kandev-diagnostic-logs-v1"; +const STORE_NAME = "entries"; +const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000; +const MAX_ENTRIES = 10_000; +const MAX_BYTES = 20 * 1024 * 1024; + +type PersistedEntry = { + id?: number; + identity_scope: string; + timestamp_ms: number; + bytes: number; + entry: LogEntry; +}; + +export class IndexedDBLogStore { + private database: Promise | null = null; + + async append(entries: LogEntry[]): Promise { + if (entries.length === 0) return; + const database = await this.open(); + const transaction = database.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + for (const entry of entries) { + const identity = entry.identity_scope; + if (!identity) continue; + const serialized = JSON.stringify(entry); + store.add({ + identity_scope: identity, + timestamp_ms: Date.parse(entry.timestamp) || Date.now(), + bytes: new TextEncoder().encode(serialized).byteLength, + entry, + } satisfies PersistedEntry); + } + await transactionDone(transaction); + await this.prune(); + } + + async snapshot(identityScope: string): Promise { + const database = await this.open(); + const transaction = database.transaction(STORE_NAME, "readonly"); + const index = transaction.objectStore(STORE_NAME).index("identity_scope"); + const records = await requestResult( + index.getAll(IDBKeyRange.only(identityScope)), + ); + await transactionDone(transaction); + const cutoff = Date.now() - THREE_DAYS_MS; + return records + .filter((record) => record.timestamp_ms >= cutoff) + .sort((left, right) => left.timestamp_ms - right.timestamp_ms) + .map((record) => record.entry); + } + + async clear(): Promise { + const database = await this.open(); + const transaction = database.transaction(STORE_NAME, "readwrite"); + transaction.objectStore(STORE_NAME).clear(); + await transactionDone(transaction); + } + + private open(): Promise { + if (this.database) return this.database; + if (typeof indexedDB === "undefined") return Promise.reject(new Error("IndexedDB unavailable")); + this.database = new Promise((resolve, reject) => { + const request = indexedDB.open(DATABASE_NAME, 1); + request.onupgradeneeded = () => { + const store = request.result.createObjectStore(STORE_NAME, { + keyPath: "id", + autoIncrement: true, + }); + store.createIndex("identity_scope", "identity_scope", { unique: false }); + store.createIndex("timestamp_ms", "timestamp_ms", { unique: false }); + }; + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error ?? new Error("IndexedDB open failed")); + }); + return this.database; + } + + private async prune(): Promise { + const database = await this.open(); + const transaction = database.transaction(STORE_NAME, "readwrite"); + const store = transaction.objectStore(STORE_NAME); + const index = store.index("timestamp_ms"); + const totals = await scanAndDeleteExpired(index, Date.now() - THREE_DAYS_MS); + await deleteOldestUntilWithinBounds(index, totals); + await transactionDone(transaction); + } +} + +type RetentionTotals = { count: number; bytes: number }; + +function scanAndDeleteExpired(index: IDBIndex, cutoff: number): Promise { + return new Promise((resolve, reject) => { + const totals = { count: 0, bytes: 0 }; + const request = index.openCursor(); + request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed")); + request.onsuccess = () => { + const cursor = request.result; + if (!cursor) { + resolve(totals); + return; + } + const record = cursor.value as PersistedEntry; + if (record.timestamp_ms < cutoff) { + cursor.delete(); + } else { + totals.count += 1; + totals.bytes += record.bytes; + } + cursor.continue(); + }; + }); +} + +function deleteOldestUntilWithinBounds(index: IDBIndex, totals: RetentionTotals): Promise { + if (totals.count <= MAX_ENTRIES && totals.bytes <= MAX_BYTES) return Promise.resolve(); + return new Promise((resolve, reject) => { + const request = index.openCursor(); + request.onerror = () => reject(request.error ?? new Error("IndexedDB cursor failed")); + request.onsuccess = () => { + const cursor = request.result; + if (!cursor || (totals.count <= MAX_ENTRIES && totals.bytes <= MAX_BYTES)) { + resolve(); + return; + } + const record = cursor.value as PersistedEntry; + cursor.delete(); + totals.count -= 1; + totals.bytes -= record.bytes; + cursor.continue(); + }; + }); +} + +export function retentionPlan( + records: PersistedEntry[], + now: number, +): { removeIDs: number[]; retainedCount: number; retainedBytes: number } { + const cutoff = now - THREE_DAYS_MS; + const ordered = [...records].sort((left, right) => left.timestamp_ms - right.timestamp_ms); + let count = ordered.length; + let bytes = ordered.reduce((total, record) => total + record.bytes, 0); + const removeIDs: number[] = []; + for (const record of ordered) { + if (record.id === undefined) continue; + if (record.timestamp_ms >= cutoff && count <= MAX_ENTRIES && bytes <= MAX_BYTES) break; + removeIDs.push(record.id); + count -= 1; + bytes -= record.bytes; + } + return { removeIDs, retainedCount: count, retainedBytes: bytes }; +} + +function requestResult(request: IDBRequest): Promise { + return new Promise((resolve, reject) => { + request.onsuccess = () => resolve(request.result); + request.onerror = () => reject(request.error ?? new Error("IndexedDB request failed")); + }); +} + +function transactionDone(transaction: IDBTransaction): Promise { + return new Promise((resolve, reject) => { + transaction.oncomplete = () => resolve(); + transaction.onerror = () => + reject(transaction.error ?? new Error("IndexedDB transaction failed")); + transaction.onabort = () => + reject(transaction.error ?? new Error("IndexedDB transaction aborted")); + }); +} diff --git a/apps/web/lib/logger/intercept.test.ts b/apps/web/lib/logger/intercept.test.ts index b42e557663..0ce40f6ef4 100644 --- a/apps/web/lib/logger/intercept.test.ts +++ b/apps/web/lib/logger/intercept.test.ts @@ -101,4 +101,23 @@ describe("console interceptor", () => { true, ); }); + + it("keeps reference-free bounded previews without traversing caller objects", () => { + installConsoleInterceptor(); + const hostile = new Proxy( + {}, + { + get() { + throw new Error("must not read properties"); + }, + getPrototypeOf() { + throw new Error("must not inspect prototypes"); + }, + }, + ); + console.info("preview", hostile, ..."x".repeat(30).split("")); + const entry = snapshotLogs().at(-1)!; + expect(entry.args?.[0]).toEqual({ type: "object" }); + expect(entry.args).toHaveLength(19); + }); }); diff --git a/apps/web/lib/logger/intercept.ts b/apps/web/lib/logger/intercept.ts index 0b345bf45f..f51c5d251d 100644 --- a/apps/web/lib/logger/intercept.ts +++ b/apps/web/lib/logger/intercept.ts @@ -1,116 +1,181 @@ -/** - * Console + window-error interceptor that mirrors entries into the in-memory - * log ring buffer (see `./buffer.ts`). Idempotent — safe to call multiple - * times. Designed to be installed once from the top-level client boot. - */ - -import { getLogBuffer, type LogEntry, type LogLevel } from "./buffer"; - -let installed = false; +import { deriveTaskID } from "@/lib/api/domains/frontend-error-log-api"; +import { encodedBytes, MAX_ENTRY_BYTES, type LogEntry, type LogLevel } from "./buffer"; +import { stageLogEntry } from "./runtime"; const LEVELS: LogLevel[] = ["debug", "info", "warn", "error"]; +const MAX_ARGUMENTS = 20; +const STRING_LIMIT = 4 * 1024; +const ERROR_MESSAGE_LIMIT = 8 * 1024; +const STACK_LIMIT = 16 * 1024; -function safeStringify(arg: unknown): unknown { - if ( - typeof arg === "string" || - typeof arg === "number" || - typeof arg === "boolean" || - arg === null - ) { - return arg; - } - if (arg instanceof Error) { - return { name: arg.name, message: arg.message, stack: arg.stack }; +let installed = false; +let removeWindowListeners: (() => void) | null = null; +let restoreConsole: (() => void) | null = null; + +function preview(value: unknown): unknown { + switch (typeof value) { + case "string": + return bounded(value, STRING_LIMIT); + case "number": + case "boolean": + return value; + case "undefined": + return { type: "undefined" }; + case "bigint": + return bounded(value.toString(), STRING_LIMIT); + case "symbol": + return { type: "symbol" }; + case "function": + return { type: "function" }; + case "object": + if (value === null) return null; + return errorPreview(value) ?? { type: "object" }; + default: + return { type: "unknown" }; } +} + +function errorPreview(value: object): { name: string; message: string; stack?: string } | null { try { - JSON.stringify(arg); - return arg; + if (!(value instanceof Error)) return null; + return { + name: bounded(value.name || "Error", STRING_LIMIT), + message: bounded(value.message, ERROR_MESSAGE_LIMIT), + stack: value.stack ? bounded(value.stack, STACK_LIMIT) : undefined, + }; } catch { - return String(arg); + return null; } } function extractMessage(first: unknown): string { - if (typeof first === "string") return first; - if (first instanceof Error) return first.message; - return String(first ?? ""); + if (typeof first === "string") return bounded(first, ERROR_MESSAGE_LIMIT); + if (typeof first === "number" || typeof first === "boolean" || typeof first === "bigint") { + return bounded(String(first), ERROR_MESSAGE_LIMIT); + } + if (first && typeof first === "object") { + const error = errorPreview(first); + if (error) return error.message; + } + return first == null ? "" : `[${typeof first}]`; } -function buildEntry(level: LogLevel, source: string, args: unknown[]): LogEntry { - const message = extractMessage(args[0]); - // Preserve the originating Error's stack on the entry so the bundle keeps it - // even though we serialize args by value below. - const stack = args[0] instanceof Error ? args[0].stack : undefined; - return { +export function buildLogEntry(level: LogLevel, source: string, args: unknown[]): LogEntry { + const firstError = + args[0] && typeof args[0] === "object" ? errorPreview(args[0] as object) : null; + const entry: LogEntry = { timestamp: new Date().toISOString(), level, source, - message, - args: args.length > 1 ? args.slice(1).map(safeStringify) : undefined, - stack, + message: extractMessage(args[0]), + args: args.length > 1 ? args.slice(1, MAX_ARGUMENTS).map(preview) : undefined, + stack: firstError?.stack, }; + addLocation(entry); + while (encodedBytes(entry) > MAX_ENTRY_BYTES && entry.args?.length) entry.args.pop(); + if (encodedBytes(entry) > MAX_ENTRY_BYTES) entry.stack = undefined; + if (encodedBytes(entry) > MAX_ENTRY_BYTES) entry.message = bounded(entry.message, STRING_LIMIT); + return entry; } export function installConsoleInterceptor(): void { - if (installed) return; - if (typeof window === "undefined") return; + if (installed || typeof window === "undefined") return; installed = true; - - const buffer = getLogBuffer(); - + const originals = new Map void>(); for (const level of LEVELS) { const original = console[level]?.bind(console); if (!original) continue; + originals.set(level, original); console[level] = (...args: unknown[]) => { try { - buffer.push(buildEntry(level, "console", args)); + stageLogEntry(buildLogEntry(level, "console", args)); } catch { - // never let interceptor break the host console call + // Diagnostics cannot alter console behavior. } original(...args); }; } + restoreConsole = () => { + for (const [level, original] of originals) console[level] = original; + }; - window.addEventListener("error", (event) => { - buffer.push({ - timestamp: new Date().toISOString(), - level: "error", - source: "window.onerror", - message: event.message ?? "Uncaught error", - args: [ - { - filename: event.filename, - lineno: event.lineno, - colno: event.colno, - error: - event.error instanceof Error - ? { name: event.error.name, stack: event.error.stack } - : undefined, - }, - ], - stack: event.error instanceof Error ? event.error.stack : undefined, - }); - }); + const onError = (event: ErrorEvent) => { + try { + const error = + event.error && typeof event.error === "object" ? errorPreview(event.error) : null; + stageLogEntry( + buildWindowEntry("window.onerror", event.message || "Uncaught error", error?.stack, { + filename: bounded(event.filename || "", STRING_LIMIT), + line: event.lineno, + column: event.colno, + error: error ?? undefined, + }), + ); + } catch { + // Best effort only. + } + }; + const onRejection = (event: PromiseRejectionEvent) => { + try { + const error = + event.reason && typeof event.reason === "object" ? errorPreview(event.reason) : null; + stageLogEntry( + buildWindowEntry( + "unhandledrejection", + error?.message ?? extractMessage(event.reason), + error?.stack, + error ?? preview(event.reason), + ), + ); + } catch { + // Best effort only. + } + }; + window.addEventListener("error", onError); + window.addEventListener("unhandledrejection", onRejection); + removeWindowListeners = () => { + window.removeEventListener("error", onError); + window.removeEventListener("unhandledrejection", onRejection); + }; +} - window.addEventListener("unhandledrejection", (event) => { - const reason = event.reason; - const message = - reason instanceof Error ? reason.message : String(reason ?? "Unhandled rejection"); - buffer.push({ - timestamp: new Date().toISOString(), - level: "error", - source: "unhandledrejection", - message, - args: - reason instanceof Error - ? [{ name: reason.name, stack: reason.stack }] - : [safeStringify(reason)], - stack: reason instanceof Error ? reason.stack : undefined, - }); - }); +function buildWindowEntry( + source: string, + message: string, + stack: string | undefined, + arg: unknown, +) { + const entry: LogEntry = { + timestamp: new Date().toISOString(), + level: "error", + source, + message: bounded(message, ERROR_MESSAGE_LIMIT), + args: [arg], + stack: stack ? bounded(stack, STACK_LIMIT) : undefined, + }; + addLocation(entry); + return entry; +} + +function addLocation(entry: LogEntry): void { + try { + const url = new URL(window.location.href); + entry.url = bounded(url.href, ERROR_MESSAGE_LIMIT); + entry.task_id = deriveTaskID(url); + } catch { + // Location is optional. + } +} + +function bounded(value: string, limit: number): string { + return value.length <= limit ? value : value.slice(0, limit); } /** Exposed for tests. */ export function _resetInstalledForTesting(): void { + removeWindowListeners?.(); + restoreConsole?.(); + removeWindowListeners = null; + restoreConsole = null; installed = false; } diff --git a/apps/web/lib/logger/runtime.test.ts b/apps/web/lib/logger/runtime.test.ts new file mode 100644 index 0000000000..9993683df6 --- /dev/null +++ b/apps/web/lib/logger/runtime.test.ts @@ -0,0 +1,37 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { _resetForTesting, MAX_ENTRY_BYTES } from "./buffer"; +import { _resetRuntimeForTesting, snapshotBrowserLogs, stageLogEntry } from "./runtime"; + +const storeMocks = vi.hoisted(() => ({ + append: vi.fn(), + snapshot: vi.fn(), +})); + +vi.mock("./indexeddb-store", () => ({ + IndexedDBLogStore: class { + append = storeMocks.append; + snapshot = storeMocks.snapshot; + }, +})); + +beforeEach(() => { + _resetForTesting(); + _resetRuntimeForTesting(); + storeMocks.append.mockReset(); + storeMocks.snapshot.mockReset().mockResolvedValue([]); +}); + +describe("browser logger runtime", () => { + it("does not stage entries rejected by the bounded memory buffer", async () => { + stageLogEntry({ + timestamp: new Date().toISOString(), + level: "error", + source: "window.error", + message: "x".repeat(MAX_ENTRY_BYTES + 1), + }); + + await snapshotBrowserLogs("default-user"); + + expect(storeMocks.append).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/lib/logger/runtime.ts b/apps/web/lib/logger/runtime.ts new file mode 100644 index 0000000000..8bff401809 --- /dev/null +++ b/apps/web/lib/logger/runtime.ts @@ -0,0 +1,155 @@ +import { encodedBytes, getLogBuffer, snapshotLogs, type LogEntry, type LogLevel } from "./buffer"; +import { IndexedDBLogStore } from "./indexeddb-store"; + +const DRAIN_ENTRY_LIMIT = 50; +const DRAIN_BYTE_LIMIT = 256 * 1024; +const STAGING_ENTRY_LIMIT = 500; +const STAGING_BYTE_LIMIT = 2 * 1024 * 1024; + +type Staged = { entry: LogEntry; bytes: number }; + +const store = new IndexedDBLogStore(); +let identityScope: string | null = "default-user"; +let staging: Staged[] = []; +let stagingBytes = 0; +let scheduled = false; +let storageMode: "indexeddb" | "memory" = "indexeddb"; +let persistenceFailures = 0; +let stagingDropped = 0; + +export function setLogIdentity(scope: string | null): void { + identityScope = scope; +} + +export function stageLogEntry(entry: Omit): void { + const scoped = { ...entry, identity_scope: identityScope ?? undefined }; + if (!getLogBuffer().push(scoped)) return; + if (!identityScope || storageMode === "memory") return; + const bytes = encodedBytes(scoped); + if (!makeStagingRoom(entry.level, bytes)) { + stagingDropped += 1; + return; + } + staging.push({ entry: scoped, bytes }); + stagingBytes += bytes; + scheduleDrain(); +} + +export async function snapshotBrowserLogs(scope: string): Promise { + await flushStaging(); + if (storageMode === "indexeddb") { + try { + return await store.snapshot(scope); + } catch { + degradePersistence(); + } + } + return snapshotLogs(scope); +} + +export function browserLogMetadata(): Record { + return { + storage_mode: storageMode, + persistence_failures: persistenceFailures, + staging_dropped: stagingDropped, + memory_loss: getLogBuffer().statistics(), + }; +} + +export function browserInstallationID(): string { + const key = "kandev-diagnostic-browser-id"; + try { + const existing = localStorage.getItem(key); + if (existing) return existing; + const created = randomID(); + localStorage.setItem(key, created); + return created; + } catch { + return randomID(); + } +} + +function makeStagingRoom(level: LogLevel, bytes: number): boolean { + while (staging.length >= STAGING_ENTRY_LIMIT || stagingBytes + bytes > STAGING_BYTE_LIMIT) { + const low = staging.findIndex( + (candidate) => candidate.entry.level === "debug" || candidate.entry.level === "info", + ); + let index = low; + if (index < 0) index = level === "warn" || level === "error" ? 0 : -1; + if (index < 0) return false; + const [removed] = staging.splice(index, 1); + stagingBytes -= removed.bytes; + stagingDropped += 1; + } + return true; +} + +function scheduleDrain(): void { + if (scheduled) return; + scheduled = true; + const drain = () => { + scheduled = false; + void drainBatch(); + }; + if (typeof requestIdleCallback === "function") { + requestIdleCallback(drain, { timeout: 1_000 }); + } else { + setTimeout(drain, 250); + } +} + +async function drainBatch(): Promise { + if (storageMode === "memory" || staging.length === 0) return; + const batch: Staged[] = []; + let bytes = 0; + while (staging.length > 0 && batch.length < DRAIN_ENTRY_LIMIT) { + const next = staging[0]; + if (batch.length > 0 && bytes + next.bytes > DRAIN_BYTE_LIMIT) break; + staging.shift(); + stagingBytes -= next.bytes; + batch.push(next); + bytes += next.bytes; + } + try { + await store.append(batch.map(({ entry }) => entry)); + } catch { + for (const item of batch.reverse()) { + staging.unshift(item); + stagingBytes += item.bytes; + } + degradePersistence(); + return; + } + if (staging.length > 0) scheduleDrain(); +} + +async function flushStaging(): Promise { + while (storageMode === "indexeddb" && staging.length > 0) { + await drainBatch(); + } +} + +function degradePersistence(): void { + persistenceFailures += 1; + storageMode = "memory"; + staging = []; + stagingBytes = 0; +} + +function randomID(): string { + try { + return crypto.randomUUID(); + } catch { + return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + } +} + +export function _resetRuntimeForTesting(): void { + identityScope = "default-user"; + staging = []; + stagingBytes = 0; + scheduled = false; + storageMode = "indexeddb"; + persistenceFailures = 0; + stagingDropped = 0; +} diff --git a/apps/web/lib/state/slices/index.ts b/apps/web/lib/state/slices/index.ts index 2ac87b91d9..cd3dd8ff5a 100644 --- a/apps/web/lib/state/slices/index.ts +++ b/apps/web/lib/state/slices/index.ts @@ -83,7 +83,6 @@ export type { SystemSliceState, SystemSliceActions, SystemBackupsState, - SystemLogsState, SystemJobsMap, } from "./system/types"; export type { diff --git a/apps/web/lib/state/slices/system/index.ts b/apps/web/lib/state/slices/system/index.ts index eb89f95a5c..ac6397d072 100644 --- a/apps/web/lib/state/slices/system/index.ts +++ b/apps/web/lib/state/slices/system/index.ts @@ -4,6 +4,5 @@ export type { SystemSliceState, SystemSliceActions, SystemBackupsState, - SystemLogsState, SystemJobsMap, } from "./types"; diff --git a/apps/web/lib/state/slices/system/system-slice.test.ts b/apps/web/lib/state/slices/system/system-slice.test.ts index 6872742e88..f143a1fe2a 100644 --- a/apps/web/lib/state/slices/system/system-slice.test.ts +++ b/apps/web/lib/state/slices/system/system-slice.test.ts @@ -8,7 +8,6 @@ import type { DiskUsageResponse, DatabaseStats, SnapshotInfo, - LogFileInfo, UpdatesResponse, SystemJob, StorageOverviewResponse, @@ -68,13 +67,6 @@ const SNAPSHOT: SnapshotInfo = { kind: "manual", }; -const LOG_FILE: LogFileInfo = { - name: "kandev.log", - size: 2048, - mtime: TS, - current: true, -}; - const UPDATES: UpdatesResponse = { current: "1.2.3", latest: "1.2.4", @@ -157,7 +149,6 @@ describe("system slice", () => { expect(s.system.diskUsage).toBeNull(); expect(s.system.database).toBeNull(); expect(s.system.backups).toEqual({ items: [], loaded: false }); - expect(s.system.logs).toEqual({ files: [], tail: [], tailLoaded: false }); expect(s.system.updates).toBeNull(); expect(s.system.jobs).toEqual({}); }); @@ -194,23 +185,6 @@ describe("system slice", () => { expect(store.getState().system.backups).toEqual({ items: [], loaded: true }); }); - it("setSystemLogs replaces only the files (tail stays untouched)", () => { - const store = makeStore(); - store.getState().setSystemLogTail(["line 1", "line 2"]); - store.getState().setSystemLogs([LOG_FILE]); - expect(store.getState().system.logs.files).toEqual([LOG_FILE]); - expect(store.getState().system.logs.tail).toEqual(["line 1", "line 2"]); - expect(store.getState().system.logs.tailLoaded).toBe(true); - }); - - it("setSystemLogTail flips tailLoaded to true", () => { - const store = makeStore(); - expect(store.getState().system.logs.tailLoaded).toBe(false); - store.getState().setSystemLogTail(["hello"]); - expect(store.getState().system.logs.tail).toEqual(["hello"]); - expect(store.getState().system.logs.tailLoaded).toBe(true); - }); - it("setSystemUpdates stores the response", () => { const store = makeStore(); store.getState().setSystemUpdates(UPDATES); diff --git a/apps/web/lib/state/slices/system/system-slice.ts b/apps/web/lib/state/slices/system/system-slice.ts index af799e82a2..260c581313 100644 --- a/apps/web/lib/state/slices/system/system-slice.ts +++ b/apps/web/lib/state/slices/system/system-slice.ts @@ -7,7 +7,6 @@ export const defaultSystemState: SystemSliceState = { diskUsage: null, database: null, backups: { items: [], loaded: false }, - logs: { files: [], tail: [], tailLoaded: false }, updates: null, jobs: {}, metrics: null, @@ -42,15 +41,6 @@ export const createSystemSlice: StateCreator< set((draft) => { draft.system.backups = { items, loaded: true }; }), - setSystemLogs: (files) => - set((draft) => { - draft.system.logs.files = files; - }), - setSystemLogTail: (lines) => - set((draft) => { - draft.system.logs.tail = lines; - draft.system.logs.tailLoaded = true; - }), setSystemUpdates: (updates) => set((draft) => { draft.system.updates = updates; diff --git a/apps/web/lib/state/slices/system/types.ts b/apps/web/lib/state/slices/system/types.ts index f85489d3cf..1632cd890e 100644 --- a/apps/web/lib/state/slices/system/types.ts +++ b/apps/web/lib/state/slices/system/types.ts @@ -3,7 +3,6 @@ import type { DiskUsageResponse, DatabaseStats, SnapshotInfo, - LogFileInfo, UpdatesResponse, SystemJob, SystemMetricsSnapshot, @@ -18,12 +17,6 @@ export type SystemBackupsState = { loaded: boolean; }; -export type SystemLogsState = { - files: LogFileInfo[]; - tail: string[]; - tailLoaded: boolean; -}; - export type SystemJobsMap = Record; export type SystemSliceState = { @@ -32,7 +25,6 @@ export type SystemSliceState = { diskUsage: DiskUsageResponse | null; database: DatabaseStats | null; backups: SystemBackupsState; - logs: SystemLogsState; updates: UpdatesResponse | null; jobs: SystemJobsMap; metrics: SystemMetricsSnapshot | null; @@ -50,8 +42,6 @@ export type SystemSliceActions = { setSystemDiskUsage: (usage: DiskUsageResponse) => void; setSystemDatabase: (stats: DatabaseStats) => void; setSystemBackups: (items: SnapshotInfo[]) => void; - setSystemLogs: (files: LogFileInfo[]) => void; - setSystemLogTail: (lines: string[]) => void; setSystemUpdates: (updates: UpdatesResponse) => void; upsertSystemJob: (job: SystemJob) => void; clearSystemJob: (jobId: string) => void; diff --git a/apps/web/lib/toast/sonner.test.ts b/apps/web/lib/toast/sonner.test.ts new file mode 100644 index 0000000000..b4b0a2077d --- /dev/null +++ b/apps/web/lib/toast/sonner.test.ts @@ -0,0 +1,79 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => { + const order: string[] = []; + return { + order, + upstreamError: vi.fn(() => { + order.push("delegate"); + return "toast-id"; + }), + report: vi.fn(() => order.push("report")), + }; +}); + +vi.mock("sonner", () => ({ + toast: Object.assign(vi.fn(), { + error: mocks.upstreamError, + success: vi.fn(), + warning: vi.fn(), + info: vi.fn(), + }), +})); + +vi.mock("@/lib/api/domains/frontend-error-log-api", () => ({ + scheduleFrontendErrorReport: mocks.report, +})); + +import { toast } from "./sonner"; + +describe("reporting Sonner seam", () => { + beforeEach(() => { + mocks.order.length = 0; + vi.clearAllMocks(); + }); + + it("delegates error rendering before scheduling exactly one report", () => { + const result = toast.error("Failed to save", { + description: "The backend rejected the update", + }); + + expect(result).toBe("toast-id"); + expect(mocks.order).toEqual(["delegate", "report"]); + expect(mocks.report).toHaveBeenCalledOnce(); + expect(mocks.report).toHaveBeenCalledWith({ + source: "sonner", + title: "Failed to save", + description: "The backend rejected the update", + error: undefined, + }); + }); + + it("prevents application call sites from bypassing the reporting seam", () => { + const webRoot = path.resolve(process.cwd()); + const wrapper = path.join(webRoot, "lib", "toast", "sonner.ts"); + const bypasses = sourceFiles(webRoot).filter((file) => { + if (file === wrapper) return false; + if (/\.test\.(ts|tsx)$/.test(file)) return false; + return /import\s+\{\s*toast\s*\}\s+from\s+["']sonner["']/.test(fs.readFileSync(file, "utf8")); + }); + expect(bypasses).toEqual([]); + }); +}); + +function sourceFiles(root: string): string[] { + const files: string[] = []; + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + if (entry.name === "node_modules" || entry.name === ".next") continue; + const fullPath = path.join(root, entry.name); + if (entry.isDirectory()) { + files.push(...sourceFiles(fullPath)); + } else if (/\.(ts|tsx)$/.test(entry.name)) { + files.push(fullPath); + } + } + return files; +} diff --git a/apps/web/lib/toast/sonner.ts b/apps/web/lib/toast/sonner.ts new file mode 100644 index 0000000000..52f90c6113 --- /dev/null +++ b/apps/web/lib/toast/sonner.ts @@ -0,0 +1,33 @@ +import { toast as upstreamToast } from "sonner"; + +import { scheduleFrontendErrorReport } from "@/lib/api/domains/frontend-error-log-api"; + +const reportingError = ( + ...args: Parameters +): ReturnType => { + const result = upstreamToast.error(...args); + const options = args[1]; + scheduleFrontendErrorReport({ + source: "sonner", + title: args[0], + description: ownDataValue(options, "description"), + error: args[0] instanceof Error ? args[0] : ownDataValue(options, "error"), + }); + return result; +}; + +export const toast = new Proxy(upstreamToast, { + get(target, property, receiver) { + if (property === "error") return reportingError; + return Reflect.get(target, property, receiver); + }, +}) as typeof upstreamToast; + +function ownDataValue(value: unknown, property: string): unknown { + if (!value || typeof value !== "object") return undefined; + try { + return Object.getOwnPropertyDescriptor(value, property)?.value; + } catch { + return undefined; + } +} diff --git a/apps/web/lib/types/backend.ts b/apps/web/lib/types/backend.ts index a372547419..46fc7f06ea 100644 --- a/apps/web/lib/types/backend.ts +++ b/apps/web/lib/types/backend.ts @@ -1,4 +1,5 @@ import type { TaskPlanEventPayload, TaskPlanRevisionEventPayload } from "./task-plan-events"; +import type { CaptureRequest } from "@/lib/logger/capture"; export type BackendMessageType = keyof BackendMessageMap; @@ -403,7 +404,6 @@ export type UserSettingsUpdatedPayload = Omit< workspace_id: string; repository_ids: string[]; }; - export type ShellOutputPayload = { task_id: string; session_id: string; @@ -524,6 +524,10 @@ export type BackendMessageMap = OfficeBackendMessageMap & "session.git.event": BackendMessage<"session.git.event", GitEventPayload>; "system.job.update": BackendMessage<"system.job.update", import("./system").SystemJob>; "system.metrics.updated": BackendMessage<"system.metrics.updated", SystemMetricsSnapshot>; + "system.logs.capture_requested": BackendMessage< + "system.logs.capture_requested", + CaptureRequest + >; "system.update_available": BackendMessage<"system.update_available", UpdateAvailablePayload>; "workspace.created": BackendMessage<"workspace.created", WorkspacePayload>; "workspace.updated": BackendMessage<"workspace.updated", WorkspacePayload>; diff --git a/apps/web/lib/types/system.ts b/apps/web/lib/types/system.ts index 881dc5b2ef..31ccf5154e 100644 --- a/apps/web/lib/types/system.ts +++ b/apps/web/lib/types/system.ts @@ -54,16 +54,65 @@ export interface SnapshotInfo { kind: SnapshotKind; } -export interface LogFileInfo { - name: string; - size: number; - /** ISO timestamp. */ - mtime: string; - current: boolean; +export type DiagnosticBundleStatus = + | "collecting" + | "building" + | "ready" + | "partial" + | "failed" + | "expired"; + +export type DiagnosticBundleSource = "backend" | "frontend" | "runtime" | "acp"; + +export interface DiagnosticBundleCapabilities { + sources: DiagnosticBundleSource[]; + acp_debug_enabled: boolean; + acp_max_sessions: number; } -export interface LogTailResponse { - lines: string[]; +export type DiagnosticSessionAvailability = "host_retained" | "reachable" | "unavailable"; + +export interface DiagnosticSession { + task_id: string; + /** Returned only by the authorized ACP picker; never included in archive metadata. */ + task_title?: string; + session_id: string; + agent?: string; + provider?: string; + model?: string; + status?: string; + executor_type?: string; + started_at?: string; + last_activity_at?: string; + acp_availability?: DiagnosticSessionAvailability; +} + +export interface DiagnosticBundleJob { + id: string; + status: DiagnosticBundleStatus; + sources: DiagnosticBundleSource[]; + session_ids?: string[]; + reused?: boolean; + build_deadline: string; + capture_deadline?: string; + expires_at: string | null; + browser_profiles: number; + frontend_entry_count: number; + frontend_bytes: number; + warnings: string[]; + runtime_entry_count?: number; + acp_session_count?: number; + download_url?: string; +} + +export interface FrontendLogUploadChunk { + browser_id: string; + capture_stream_id: string; + chunk_index: number; + done: boolean; + storage_mode: "indexeddb" | "memory"; + capture_metadata: Record | null; + entries: unknown[]; } export interface UpdatesResponse { diff --git a/apps/web/lib/ws/handlers/system-events.ts b/apps/web/lib/ws/handlers/system-events.ts index a59bbf98e1..831ef43a4c 100644 --- a/apps/web/lib/ws/handlers/system-events.ts +++ b/apps/web/lib/ws/handlers/system-events.ts @@ -1,6 +1,7 @@ import type { StoreApi } from "zustand"; import type { AppState } from "@/lib/state/store"; import type { WsHandlers } from "@/lib/ws/handlers/types"; +import { handleBrowserLogCapture } from "@/lib/logger/capture"; export function registerSystemEventsHandlers(store: StoreApi): WsHandlers { return { @@ -13,5 +14,14 @@ export function registerSystemEventsHandlers(store: StoreApi): WsHandl "system.metrics.updated": (message) => { store.getState().setSystemMetricsSnapshot(message.payload); }, + "system.logs.capture_requested": (message) => { + const auth = store.getState().auth; + const identityScope = auth.authenticated + ? (auth.user?.id ?? (auth.mode === "disabled" ? "default-user" : null)) + : null; + void handleBrowserLogCapture(message.payload, identityScope).catch(() => { + // Capture errors must not recurse through the intercepted console. + }); + }, }; } diff --git a/apps/web/src/locales/en/settings.json b/apps/web/src/locales/en/settings.json index 1323aeb49c..23040b136e 100644 --- a/apps/web/src/locales/en/settings.json +++ b/apps/web/src/locales/en/settings.json @@ -381,5 +381,54 @@ "visualEditorUnavailable": "Visual editor unavailable", "voiceMode": "Voice Mode", "vsCodeRemoteSsh": "VS Code Remote SSH", - "whenEnabledInstallYourProjectS": "When enabled, install your project's dependencies (e.g. <2>{{command}} via repository setup scripts) to avoid missing type errors." + "whenEnabledInstallYourProjectS": "When enabled, install your project's dependencies (e.g. <2>{{command}} via repository setup scripts) to avoid missing type errors.", + "logsPageTitle": "Logs", + "logsPageDescription": "Download a bounded diagnostic ZIP containing frontend and backend logs.", + "diagnosticReviewBeforeSharing": "Review before sharing", + "diagnosticReviewDescription": "This ZIP combines up to three days of backend logs with locally retained frontend console events from connected browsers. It may contain URLs, console arguments, stacks, paths, runtime metadata, and user-visible errors.", + "diagnosticBundleTitle": "Frontend + backend diagnostic logs", + "diagnosticBundleDescription": "Kandev asks your connected browser tabs for their bounded three-day console history, combines it with retained backend log files, and downloads one ZIP.", + "diagnosticBackendEvents": "Backend logs include startup, lifecycle, API, executor, and error events written by Kandev.", + "diagnosticFrontendEvents": "Frontend logs include browser console errors, warnings, and diagnostic toast reports from this browser profile.", + "diagnosticNoMessages": "Standard bundles do not include chat transcripts, session messages, or agent messages.", + "diagnosticIncidentalText": "Incidental text already written into a log entry is not automatically redacted. Review the ZIP before sharing.", + "diagnosticCustomize": "Customize bundle", + "diagnosticCustomizerTitle": "Customize diagnostic bundle", + "diagnosticCustomizerDescription": "Choose the evidence to include. Backend and frontend logs are selected by default, and nothing is saved as a preference.", + "diagnosticSourcesTitle": "Evidence sources", + "diagnosticSourceBackend": "Backend logs", + "diagnosticSourceBackendDescription": "Up to three days of retained Kandev backend log files.", + "diagnosticSourceFrontend": "Frontend logs", + "diagnosticSourceFrontendDescription": "Bounded console events from connected browser tabs.", + "diagnosticSourceRuntime": "Runtime index", + "diagnosticSourceRuntimeDescription": "Allow-listed session status and executor metadata; no messages or prompts.", + "diagnosticSourceACP": "ACP debug messages", + "diagnosticSourceACPDescription": "Raw and normalized agent protocol frames for selected sessions.", + "diagnosticSelectSessions": "Select ACP sessions", + "diagnosticSessionsDescription_one": "Choose up to {{count}} eligible session. Unavailable sessions cannot be collected.", + "diagnosticSessionsDescription_other": "Choose up to {{count}} eligible sessions. Unavailable sessions cannot be collected.", + "diagnosticSessionsLoading": "Loading eligible sessions…", + "diagnosticSessionUnavailable": "Unavailable", + "diagnosticSessionRetained": "Host-retained", + "diagnosticSessionReachable": "Reachable", + "diagnosticSelectedSessions_one": "{{count}} session selected", + "diagnosticSelectedSessions_other": "{{count}} sessions selected", + "diagnosticSelectAllACPSessions": "Select all ACP sessions", + "diagnosticClearACPSelection": "Clear ACP selection", + "diagnosticOpenTask": "Open task {{title}} in a new tab", + "diagnosticSourceRequired": "Select at least one evidence source.", + "diagnosticACPSelectionRequired": "Select at least one eligible ACP session.", + "diagnosticACPSelectionLimit": "Select no more than ten ACP sessions.", + "diagnosticACPWarning": "ACP debug messages can contain prompts, responses, tool calls, file contents, MCP data, environment-derived values, and secrets. Review this ZIP carefully before sharing.", + "diagnosticACPNoSessions": "ACP download is available in debug mode, but this account has no eligible sessions to include.", + "diagnosticBundleCancel": "Cancel", + "diagnosticBundleCreate": "Create bundle", + "diagnosticBundlePrepareError": "The diagnostic bundle could not be prepared.", + "diagnosticBundleFailed": "The diagnostic bundle failed.", + "diagnosticBusy": "Diagnostics are busy. Try again in {{seconds}} seconds.", + "diagnosticCollectingFrontendLogs": "Collecting frontend logs…", + "diagnosticPreparingZip": "Preparing ZIP…", + "diagnosticZipDownloading": "Your diagnostic ZIP is downloading.", + "diagnosticPartialZipDownloading": "A partial ZIP is downloading: {{warnings}}", + "diagnosticPartialZipUnavailable": "A partial diagnostic ZIP is downloading; some frontend logs were unavailable." } diff --git a/apps/web/src/locales/pseudo/settings.json b/apps/web/src/locales/pseudo/settings.json index 016d1fe07a..28cfd1f59f 100644 --- a/apps/web/src/locales/pseudo/settings.json +++ b/apps/web/src/locales/pseudo/settings.json @@ -381,5 +381,54 @@ "visualEditorUnavailable": "Vĩśũàĺ ēďĩţōŕ ũńàvàĩĺàƀĺē", "voiceMode": "Vōĩćē Ḿōďē", "vsCodeRemoteSsh": "VŚ Ćōďē Ŕēḿōţē ŚŚĤ", - "whenEnabledInstallYourProjectS": "Ŵĥēń ēńàƀĺēď, ĩńśţàĺĺ ŷōũŕ ƥŕōĵēćţ'ś ďēƥēńďēńćĩēś (ē.ĝ. <2>{{command}} vĩà ŕēƥōśĩţōŕŷ śēţũƥ śćŕĩƥţś) ţō àvōĩď ḿĩśśĩńĝ ţŷƥē ēŕŕōŕś." + "whenEnabledInstallYourProjectS": "Ŵĥēń ēńàƀĺēď, ĩńśţàĺĺ ŷōũŕ ƥŕōĵēćţ'ś ďēƥēńďēńćĩēś (ē.ĝ. <2>{{command}} vĩà ŕēƥōśĩţōŕŷ śēţũƥ śćŕĩƥţś) ţō àvōĩď ḿĩśśĩńĝ ţŷƥē ēŕŕōŕś.", + "logsPageTitle": "Ĺōĝś", + "logsPageDescription": "Ďōŵńĺōàď à ƀōũńďēď ďĩàĝńōśţĩć ŹĨƤ ćōńţàĩńĩńĝ ƒŕōńţēńď àńď ƀàćķēńď ĺōĝś.", + "diagnosticReviewBeforeSharing": "Ŕēvĩēŵ ƀēƒōŕē śĥàŕĩńĝ", + "diagnosticReviewDescription": "Ţĥĩś ŹĨƤ ćōḿƀĩńēś ũƥ ţō ţĥŕēē ďàŷś ōƒ ƀàćķēńď ĺōĝś ŵĩţĥ ĺōćàĺĺŷ ŕēţàĩńēď ƒŕōńţēńď ćōńśōĺē ēvēńţś ƒŕōḿ ćōńńēćţēď ƀŕōŵśēŕś. Ĩţ ḿàŷ ćōńţàĩń ŨŔĹś, ćōńśōĺē àŕĝũḿēńţś, śţàćķś, ƥàţĥś, ŕũńţĩḿē ḿēţàďàţà, àńď ũśēŕ-vĩśĩƀĺē ēŕŕōŕś.", + "diagnosticBundleTitle": "Ƒŕōńţēńď + ƀàćķēńď ďĩàĝńōśţĩć ĺōĝś", + "diagnosticBundleDescription": "Ķàńďēv àśķś ŷōũŕ ćōńńēćţēď ƀŕōŵśēŕ ţàƀś ƒōŕ ţĥēĩŕ ƀōũńďēď ţĥŕēē-ďàŷ ćōńśōĺē ĥĩśţōŕŷ, ćōḿƀĩńēś ĩţ ŵĩţĥ ŕēţàĩńēď ƀàćķēńď ĺōĝ ƒĩĺēś, àńď ďōŵńĺōàďś ōńē ŹĨƤ.", + "diagnosticBackendEvents": "Ɓàćķēńď ĺōĝś ĩńćĺũďē śţàŕţũƥ, ĺĩƒēćŷćĺē, ÀƤĨ, ēẋēćũţōŕ, àńď ēŕŕōŕ ēvēńţś ŵŕĩţţēń ƀŷ Ķàńďēv.", + "diagnosticFrontendEvents": "Ƒŕōńţēńď ĺōĝś ĩńćĺũďē ƀŕōŵśēŕ ćōńśōĺē ēŕŕōŕś, ŵàŕńĩńĝś, àńď ďĩàĝńōśţĩć ţōàśţ ŕēƥōŕţś.", + "diagnosticNoMessages": "Śţàńďàŕď ƀũńďĺēś ďō ńōţ ĩńćĺũďē ćĥàţ ţŕàńśćŕĩƥţś, śēśśĩōń ḿēśśàĝēś, ōŕ àĝēńţ ḿēśśàĝēś.", + "diagnosticIncidentalText": "Ĩńćĩďēńţàĺ ţēẋţ àĺŕēàďŷ ŵŕĩţţēń ĩńţō à ĺōĝ ēńţŕŷ ĩś ńōţ àũţōḿàţĩćàĺĺŷ ŕēďàćţēď.", + "diagnosticCustomize": "Ćũśţōḿĩźē ƀũńďĺē", + "diagnosticCustomizerTitle": "Ćũśţōḿĩźē ďĩàĝńōśţĩć ƀũńďĺē", + "diagnosticCustomizerDescription": "Ćĥōōśē ţĥē ēvĩďēńćē ţō ĩńćĺũďē. Ɓàćķēńď àńď ƒŕōńţēńď ĺōĝś àŕē śēĺēćţēď ƀŷ ďēƒàũĺţ, àńď ńōţĥĩńĝ ĩś śàvēď àś à ƥŕēƒēŕēńćē.", + "diagnosticSourcesTitle": "Ēvĩďēńćē śōũŕćēś", + "diagnosticSourceBackend": "Ɓàćķēńď ĺōĝś", + "diagnosticSourceBackendDescription": "Ŭƥ ţō ţĥŕēē ďàŷś ōƒ ŕēţàĩńēď Ķàńďēv ƀàćķēńď ĺōĝ ƒĩĺēś.", + "diagnosticSourceFrontend": "Ƒŕōńţēńď ĺōĝś", + "diagnosticSourceFrontendDescription": "Ɓōũńďēď ćōńśōĺē ēvēńţś ƒŕōḿ ćōńńēćţēď ƀŕōŵśēŕ ţàƀś.", + "diagnosticSourceRuntime": "Ŕũńţĩḿē ĩńďēẋ", + "diagnosticSourceRuntimeDescription": "Àĺĺōŵ-ĺĩśţēď śēśśĩōń śţàţũś àńď ēẋēćũţōŕ ḿēţàďàţà; ńō ḿēśśàĝēś ōŕ ƥŕōḿƥţś.", + "diagnosticSourceACP": "ÀĆƤ ďēƀũĝ ḿēśśàĝēś", + "diagnosticSourceACPDescription": "Ŕàŵ àńď ńōŕḿàĺĩźēď àĝēńţ ƥŕōţōćōĺ ƒŕàḿēś ƒōŕ śēĺēćţēď śēśśĩōńś.", + "diagnosticSelectSessions": "Śēĺēćţ ÀĆƤ śēśśĩōńś", + "diagnosticSessionsDescription_one": "Ćĥōōśē ũƥ ţō {{count}} ēĺĩĝĩƀĺē śēśśĩōń. Ŭńàvàĩĺàƀĺē śēśśĩōńś ćàńńōţ ƀē ćōĺĺēćţēď.", + "diagnosticSessionsDescription_other": "Ćĥōōśē ũƥ ţō {{count}} ēĺĩĝĩƀĺē śēśśĩōńś. Ŭńàvàĩĺàƀĺē śēśśĩōńś ćàńńōţ ƀē ćōĺĺēćţēď.", + "diagnosticSessionsLoading": "Ĺōàďĩńĝ ēĺĩĝĩƀĺē śēśśĩōńś…", + "diagnosticSessionUnavailable": "Ŭńàvàĩĺàƀĺē", + "diagnosticSessionRetained": "Ĥōśţ-ŕēţàĩńēď", + "diagnosticSessionReachable": "Ŕēàćĥàƀĺē", + "diagnosticSelectedSessions_one": "{{count}} śēśśĩōń śēĺēćţēď", + "diagnosticSelectedSessions_other": "{{count}} śēśśĩōńś śēĺēćţēď", + "diagnosticSelectAllACPSessions": "Śēĺēćţ àĺĺ ÀĆƤ śēśśĩōńś", + "diagnosticClearACPSelection": "Ćĺēàŕ ÀĆƤ śēĺēćţĩōń", + "diagnosticOpenTask": "Ōƥēń ţàśķ {{title}} ĩń à ńēŵ ţàƀ", + "diagnosticSourceRequired": "Śēĺēćţ àţ ĺēàśţ ōńē ēvĩďēńćē śōũŕćē.", + "diagnosticACPSelectionRequired": "Śēĺēćţ àţ ĺēàśţ ōńē ēĺĩĝĩƀĺē ÀĆƤ śēśśĩōń.", + "diagnosticACPSelectionLimit": "Śēĺēćţ ńō ḿōŕē ţĥàń ţēń ÀĆƤ śēśśĩōńś.", + "diagnosticACPWarning": "ÀĆƤ ďēƀũĝ ḿēśśàĝēś ćàń ćōńţàĩń ƥŕōḿƥţś, ŕēśƥōńśēś, ţōōĺ ćàĺĺś, ƒĩĺē ćōńţēńţ, ḾĆƤ ďàţà, ēńvĩŕōńḿēńţ vàĺũēś, àńď śēćŕēţś.", + "diagnosticACPNoSessions": "ÀĆƤ ďōŵńĺōàď ĩś àvàĩĺàƀĺē ĩń ďēƀũĝ ḿōďē, ƀũţ ţĥĩś àććōũńţ ĥàś ńō ēĺĩĝĩƀĺē śēśśĩōńś.", + "diagnosticBundleCancel": "Ćàńćēĺ", + "diagnosticBundleCreate": "Ćŕēàţē ƀũńďĺē", + "diagnosticBundlePrepareError": "Ţĥē ďĩàĝńōśţĩć ƀũńďĺē ćōũĺď ńōţ ƀē ƥŕēƥàŕēď.", + "diagnosticBundleFailed": "Ţĥē ďĩàĝńōśţĩć ƀũńďĺē ƒàĩĺēď.", + "diagnosticBusy": "Ďĩàĝńōśţĩćś àŕē ƀũśŷ. Ţŕŷ àĝàĩń ĩń {{seconds}} śēćōńďś.", + "diagnosticCollectingFrontendLogs": "Ćōĺĺēćţĩńĝ ƒŕōńţēńď ĺōĝś…", + "diagnosticPreparingZip": "Ƥŕēƥàŕĩńĝ ŹĨƤ…", + "diagnosticZipDownloading": "Ŷōũŕ ďĩàĝńōśţĩć ŹĨƤ ĩś ďōŵńĺōàďĩńĝ.", + "diagnosticPartialZipDownloading": "À ƥàŕţĩàĺ ŹĨƤ ĩś ďōŵńĺōàďĩńĝ: {{warnings}}", + "diagnosticPartialZipUnavailable": "À ƥàŕţĩàĺ ďĩàĝńōśţĩć ŹĨƤ ĩś ďōŵńĺōàďĩńĝ; śōḿē ƒŕōńţēńď ĺōĝś ŵēŕē ũńàvàĩĺàƀĺē." } diff --git a/apps/web/src/settings-routes.tsx b/apps/web/src/settings-routes.tsx index 6540d0d5c6..972d8b978b 100644 --- a/apps/web/src/settings-routes.tsx +++ b/apps/web/src/settings-routes.tsx @@ -227,7 +227,7 @@ const SETTINGS_ROUTES: Record = { "/settings/system/logs": () => ( diff --git a/docs/decisions/2026-07-30-file-backed-diagnostic-bundles.md b/docs/decisions/2026-07-30-file-backed-diagnostic-bundles.md new file mode 100644 index 0000000000..9dbf6d8340 --- /dev/null +++ b/docs/decisions/2026-07-30-file-backed-diagnostic-bundles.md @@ -0,0 +1,213 @@ +# ADR-2026-07-30-file-backed-diagnostic-bundles: File-backed diagnostic bundles + +**Status:** accepted +**Date:** 2026-07-30 +**Area:** backend, frontend, infra, protocol, workflow + +## Context + +Frontend error toasts and console failures can be the only user-visible +evidence of a problem, while backend diagnostics are split between terminal +output, optional files, an in-memory ring buffer, a System Logs viewer, and a +dev-only debug export. Kandev already captures a bounded browser console buffer +and uploads it on demand for Improve Kandev, but that path is not a clear +user-downloadable artifact or a reusable agent workflow. The design has +meaningful privacy, retention, transport, and access alternatives. + +## Decision + +The backend owns an always-on current-day file at +`/logs/backend-logs.log`, rolls it at UTC midnight to +`backend-logs-YYYY-MM-DD.log`, appends across same-day restarts, and retains +the current plus two preceding UTC days. Each daily file has a fixed 256 MiB +write bound so sustained or hostile diagnostic traffic cannot consume +unbounded disk before midnight. Normal/debug/verbose file and stdout thresholds +remain defined by the diagnostic-logging spec. Destination and lumberjack +rotation settings are removed. + +The backend structured ring buffer, Logs-page tail and file table, individual +file downloads, and dev-only log export are removed. Files become the backend +source of truth. Runtime metadata moves into a diagnostic bundle manifest. + +Diagnostic collection is best effort and deliberately stays off application +critical paths. Backend file and stdout sinks drain independent bounded, +priority-aware queues so a blocked terminal cannot stall the authoritative +file. Startup log-path failures degrade to stderr and background retry rather +than preventing Kandev from starting. Browser interception stages only +reference-free bounded values and persists compact batches asynchronously +during idle time. Toast reporting is scheduled after the toast is handed to the +UI library and is never awaited. Automatic toast reports remove URL query and +fragment data and are guarded by both count and byte token buckets. Bundle +capture and archive construction have fixed per-job byte/profile limits plus a +bounded, coalescing background worker so one export cannot create unbounded +CPU, disk, memory, or connection work. + +The browser intercepts every console level plus uncaught errors and rejections, +but retains entries locally in bounded IndexedDB for three UTC days. It never +continuously uploads console history. When an authenticated bundle job requests +frontend evidence, the backend sends a WebSocket capture notification only to +connections for the requesting identity. Responding tabs upload bounded chunks; +the first per-tab capture stream to claim a browser-profile installation ID +owns that upload so interleaved tabs cannot mix snapshots. Local records are +partitioned by authenticated Kandev identity so an account switch inside one +browser profile cannot disclose the previous user's history. + +Users download one clearly disclosed combined ZIP from System Logs. It contains +separate `backend/` and `frontend/` directories plus `manifest.json`. Frontend +capture may time out without blocking backend evidence; the bundle then becomes +partial and records the omission. Any signed-in user retains the existing +install-wide Logs-page access boundary, while bundle job ownership and frontend +capture remain identity-scoped. + +The standard bundle never reads persisted chat/session message tables or raw +agent protocol captures. Its UI names the frontend and backend event classes +and explains that this source boundary cannot remove incidental content already +emitted into a selected log. `manifest.json` remains mandatory. A separate +optional `runtime/` source provides a bounded authorized task/session index +(IDs, agent/provider/model, status, timestamps, and executor type) without +titles, prompts, responses, tool payloads, file content, identities, or stored +message bodies. + +Raw ACP is a distinct debug-only, explicit-consent source. It is available only +when backend-authoritative ACP message capture is enabled, always requires the +user to select one or more sessions, and carries a stronger disclosure that raw +and normalized frames can contain prompts, responses, tool inputs/results, +workspace file content, MCP payloads, environment-derived values, and secrets. +Non-admin users may select only sessions on tasks they own; admins may select +any eligible session. Auth-disabled local installations use their synthetic +admin identity. + +Kandev does not continuously centralize ACP frames. For selected sessions the +bundle worker reads retained host files or fetches a bounded export on demand +from reachable executor-side `agentctl` instances, revalidates every entry and +archive path, and records unavailable/stopped sessions as partial. The request +selects at most ten sessions, uses at most two concurrent executor fetches, and +has a 30-second ACP collection deadline. Standard jobs keep their existing +160 MiB backend/80 MiB frontend budgets. ACP-inclusive jobs reserve 96 MiB +backend, 48 MiB frontend, 96 MiB ACP, and 2 MiB runtime while retaining the +256 MiB uncompressed archive and 384 MiB temporary-disk bounds. + +Agents use the same source-selectable bundle API. They request `backend`, +`frontend`, or `all`, extract into a fresh temporary directory, and grep the +artifact locally—normally by exact task ID first. Kandev does not add a +server-side free-text log query. Task-session agents use an owner-scoped MCP +tool that derives identity from the current execution and materializes the ZIP +into that execution instead of exposing a reusable credential. Improve Kandev +creates an authenticated browser-owned bundle first, then leases a verified +copy into its task context. Public bundle jobs expire 15 minutes after becoming +downloadable; Improve Kandev keeps its copied artifact for the existing +24-hour cleanup window. + +Raw ACP and the runtime index are intentionally not added to the task-mode MCP +or Improve Kandev source aliases. ACP selection remains a human-reviewed +download flow because its content and permission boundary differ materially +from the standard support bundle. + +The observable contract lives in +[`docs/specs/platform/diagnostic-logging.md`](../specs/platform/diagnostic-logging.md). + +## Consequences + +Users and agents receive the same inspectable artifact, backend logs survive +same-day restarts, and large log searches happen with standard local tools +instead of an unbounded HTTP response. Backend memory no longer duplicates the +file sink. Frontend history stays browser-owned unless a user or agent +explicitly requests it. + +Frontend-only evidence requires a connected browser during the collection +window. IndexedDB adds a bounded local persistence surface, WebSocket +request/HTTP chunk response adds a short-lived capture protocol, and bundles +need expiry and cleanup ownership. A combined ZIP may be partial, so consumers +must inspect the manifest before assuming both sources are present. + +Some lower-priority logs can be deliberately omitted during sustained resource +pressure. The manifest exposes queue, persistence, and archive loss so support +does not mistake a best-effort bundle for complete evidence. Diagnostic bundle +requests can be temporarily busy or coalesced instead of competing with normal +product activity. + +The fixed daily file bound limits worst-case disk growth but can omit later +same-day evidence after sustained pressure. Count and byte limits reduce how +quickly automatic browser reports consume that budget, while UTC rollover +restores capacity without operator action. + +Fixed byte/profile caps mean a very large retained backend file or a fifth +browser profile may be represented only partially. Newest backend bytes are +preferred and the manifest records exact included ranges. ZIP payloads are +stored without CPU-heavy compression, trading a larger download for predictable +application performance. + +Any signed-in user can still download install-wide backend diagnostics. This +preserves current behavior but carries broader sensitive content once browser +logs are included, so the page disclosure and identity-scoped frontend capture +are mandatory. + +The System Logs page becomes a focused action rather than a live diagnostic +viewer. Mobile and desktop share the workflow, with a touch-sized primary +action and no wide table. Desktop customization uses a dialog; phone uses an +inset, safe-area-aware bottom drawer with one internal scroll owner. Debug mode +shows one **Customize bundle** action. It starts with backend and frontend +evidence selected, while the runtime index and ACP evidence remain explicit +opt-ins. Selecting ACP opens the required session selection in that same +customizer. + +Raw ACP is intentionally more sensitive than the standard bundle. The UI and +API cannot truthfully promise that an ACP-inclusive archive excludes agent or +session messages. Maintainers gain exact wire evidence, but users must make a +session-scoped choice and review a stronger warning before collection. + +On-demand executor reads can omit evidence from stopped, removed, unreachable, +or non-debug executors. This avoids continuous network/disk overhead and limits +backend custody of raw frames, at the cost of some historical completeness. + +## Alternatives Considered + +- **Continuously stream every frontend log to the backend.** Rejected because it + uploads sensitive browser history without an explicit diagnostic request and + adds persistent network/storage overhead. +- **Continuously centralize all debug ACP frames.** Rejected because raw frames + can contain secrets and full work content, would add hot-path network/storage + overhead, and would retain sensitive executor data without an explicit user + request. +- **Put ACP in every debug-mode bundle automatically.** Rejected because the + standard no-transcript disclosure would become misleading and a single click + could export unrelated sessions. ACP requires explicit session selection. +- **Offer only normalized ACP events.** Rejected because maintainers sometimes + need raw wire evidence to diagnose adapter normalization, while normalized + events can still contain sensitive message/tool content. The UI therefore + includes both behind the same strong warning. +- **Read only host-resident ACP files.** Rejected because common Docker and + remote executors retain debug files inside their agentctl environment. + Bounded on-demand export covers reachable selected sessions without making + raw-frame streaming continuous. +- **Export task/session database records as a debug source.** Rejected in favor + of a small allow-listed runtime index that excludes titles, message bodies, + tool payloads, file content, identities, and arbitrary configuration. +- **Upload only the current 500-entry memory buffer.** Rejected because it + cannot match the selected three-day diagnostic window across reloads. +- **Have the initiating frontend attach logs directly to the ZIP request.** + Rejected in favor of a backend-requested capture protocol that also supports + agent-triggered frontend bundles and multiple connected browser profiles. +- **Keep the backend ring buffer for the page and debug endpoint.** Rejected + because the always-on file sink is now authoritative and duplicate memory + retention creates inconsistent evidence windows. +- **Add a filtered JSON log-query endpoint for agents.** Rejected because agents + can select a smaller source bundle, extract it, and use mature local search + tools without maintaining a second parser/query contract. +- **Expose only one all-sources bundle.** Rejected because agents often need + backend evidence only and should not trigger frontend capture unnecessarily. +- **Let agents call the HTTP API with a reusable user token.** Rejected because + in-session MCP already resolves the task owner and can materialize an + identity-scoped artifact without exposing a broad credential to the agent. +- **Make bundles admin-only.** Rejected to preserve the existing authenticated + System Logs boundary selected for this feature. +- **Keep the tail viewer beside bundle download.** Rejected because it retains + duplicate endpoints, state, and UI without adding evidence absent from the + files. +- **Use local-time retention and rollover.** Rejected because UTC is + deterministic across machines and daylight-saving transitions. +- **Rely on three-day retention without a daily byte bound.** Rejected because + retention limits age but a single current-day file can still exhaust disk. +- **Make the daily byte bound configurable.** Rejected to keep diagnostic + resource behavior predictable and avoid restoring the removed rotation + configuration surface. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index a989f700f6..84b67066cb 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -91,6 +91,7 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 2026-07-29-interactive-accent-surface-semantics | [Separate Brand Accent from Interactive Surface Fills](2026-07-29-interactive-accent-surface-semantics.md) | accepted | frontend | 2026-07-29 | | 2026-07-30-embedded-editor-executor-capabilities | [Derive Embedded Editor Availability from the Active Executor](2026-07-30-embedded-editor-executor-capabilities.md) | accepted | backend, frontend, protocol | 2026-07-30 | | 2026-07-30-session-owned-mcp-observability | [Keep MCP Attachment Evidence Session Owned](2026-07-30-session-owned-mcp-observability.md) | accepted | backend, frontend, protocol, security | 2026-07-30 | +| 2026-07-30-file-backed-diagnostic-bundles | [File-backed diagnostic bundles](2026-07-30-file-backed-diagnostic-bundles.md) | accepted | backend, frontend, infra, protocol, workflow | 2026-07-30 | | 2026-07-30-runtime-task-state-before-running-event | [Publish Task State Before Running Session State](2026-07-30-runtime-task-state-before-running-event.md) | accepted | backend, frontend, protocol, workflow | 2026-07-30 | | 2026-07-31-isolate-manual-pr-review-content | [Isolate Manual PR Review Content](2026-07-31-isolate-manual-pr-review-content.md) | accepted | infra, workflow, security | 2026-07-31 | | 2026-07-31-agent-generated-task-titles | [Bind Agent Title Generation to Pending Tasks](2026-07-31-agent-generated-task-titles.md) | accepted | backend, frontend, protocol | 2026-07-31 | diff --git a/docs/plans/diagnostic-logging/plan.md b/docs/plans/diagnostic-logging/plan.md new file mode 100644 index 0000000000..6a64b997a3 --- /dev/null +++ b/docs/plans/diagnostic-logging/plan.md @@ -0,0 +1,637 @@ +--- +spec: docs/specs/platform/diagnostic-logging.md +created: 2026-07-30 +status: implemented +--- + +# Implementation Plan: Diagnostic logging + +## Overview + +Establish the fixed daily backend files first, because every later endpoint and +bundle depends on their path, retention, and sink semantics. Add bounded error- +toast ingestion and align every launcher. In parallel, implement frontend toast +reporting and the backend diagnostic-bundle job/protocol. Then replace the +browser memory-only console capture with bounded local three-day retention and +the System Logs download workflow. Finally remove legacy ring-buffer/viewer +paths, migrate Improve Kandev, update docs, and teach agents to download, +extract, and grep source-selectable bundles. The follow-up extension preserves +that standard flow while adding an optional sanitized runtime index and an +explicit, session-selected ACP source fetched on demand only in debug mode. + +The implementation follows +[ADR-2026-07-30-file-backed-diagnostic-bundles](../../decisions/2026-07-30-file-backed-diagnostic-bundles.md). + +--- + +## Backend daily files and sinks + +- In `apps/backend/internal/common/logger/logger.go`, preserve `NewLogger` for + agentctl and focused callers, and add a backend-specific constructor whose + file and stdout cores have independent level enablers. +- Add a backend daily writer for + `/logs/backend-logs.log`: create the directory owner-only, + append on same-UTC-day restarts, roll at UTC midnight or first startup after + a boundary, and create new active files as `0600`. +- Name completed files `backend-logs-YYYY-MM-DD.log`. Keep the current UTC day + plus the preceding two UTC days, deleting only strictly recognized older + daily filenames and preserving unrelated neighbors. +- Persist the active file's UTC-day identity rather than inferring it only from + mtime, and inject a clock for deterministic rollover tests. Serialize + write/rollover/close so concurrent entries cannot be lost or split across + the wrong day. +- Make rollover crash-recoverable with an owner-only transaction journal + containing source day/identity/size, destination, and copied offset. Resume + an interrupted merge exactly once and never replace an existing dated file. +- Give file and stdout independent priority-aware asynchronous queues. Enforce + the spec's entry/byte/reserved capacities and 256 KiB encoded-entry maximum; + expose per-sink loss counters without recursively logging each drop. Emit a + compact recovery summary only after that sink can accept it. +- Expose a read-only sink-statistics snapshot for manifest construction + (accepted/dropped by sink, level, and reason); this is not another log buffer. +- Drain both queues for at most two seconds on graceful close. Give terminal + DPanic/Panic/Fatal entries the bounded flush plus capped direct-stderr path + defined by the spec. +- In `apps/backend/internal/backendapp/main.go`, resolve the fixed path from + `cfg.ResolvedHomeDir()`, construct the dual-sink logger, and print the + absolute path directly to stdout before normal structured startup logs. +- In `apps/backend/internal/common/config/config.go`, remove the destination + and lumberjack rotation fields/defaults. Keep `logging.level` and + `logging.format`; `logging.level` becomes the file threshold. +- Do not expand the existing process-wide ring-buffer contract. Its consumers + are migrated and the buffer is removed after the bundle workflow lands. +- File open/create and retention-cleanup failures must not prevent product + startup. Start with stderr/stdout sinks, report the intended path, and retry + file activation every 30 seconds. + +## Error-toast ingestion + +- Add `apps/backend/internal/system/frontenderrors/handler.go` with + `POST /api/v1/system/logs/frontend-errors`. +- Define a typed request and normalization helper in the logs package. Limit + the body to 64 KiB, validate source and visible text, UTF-8-safely bound each + field including the optional 128-byte task ID, and record truncation. +- Log a fixed `frontend error toast` message at error level with client values + only as structured fields, then return `204`. +- Add bounded in-memory token buckets: per identity 60/minute with burst 20, + process-wide 300/minute with burst 100. Return `429` plus `Retry-After` and + keep limiter identity state TTL-bounded. +- Register the route on the authenticated non-admin System group. + +## Launchers and CLI contracts + +- In the native Go launcher, separate file and stdout thresholds, forward + backend stdout in every mode, and retain captured startup-failure output. +- Add internal `KANDEV_CONSOLE_LOG_LEVEL`: `warn` normally and in debug, `info` + only for `--verbose`. `KANDEV_LOG_LEVEL` remains the file threshold (`info` + normally, `debug` for `--debug`, unless explicitly overridden). +- Apply the same contract to the TypeScript development/release supervisors. + When release stdout is piped for failure capture, tee it to parent stdout. +- Update native and TypeScript CLI help so `--debug` promises detail in the + backend file while `--verbose` remains the stdout-info opt-in. + +--- + +## Frontend toast reporting + +### Best-effort report client + +- Add `apps/web/lib/api/domains/frontend-error-log-api.ts` with the request + type, environment capture, safe text/error extraction, route task-ID + derivation, and a fire-and-forget call through the authenticated API client. +- Capture full URL, browser identity fields, language, platform, viewport, + client timestamp, emission stack, and underlying `Error` details. +- Schedule the request only after delegating to the original toast implementation + and never await it. Bound local context work, use a two-second deadline, + and catch every rejection. Do not log it to `console.error`, retry it, or + emit another toast. + +### Toast seams + +- Add `apps/web/lib/toast/sonner.ts`, re-export the Sonner API used by Kandev, + and wrap `toast.error` so it delegates first and schedules reporting + immediately afterward in the same turn. +- Mechanically replace application `toast` imports from `sonner` with the + Kandev wrapper. Keep the shared toaster renderer on upstream Sonner. +- Add a source-scan assertion so application call sites cannot bypass the seam. +- In `apps/web/components/toast-provider.tsx`, report error creation and + transitions into the error variant without changing IDs, timers, ordering, + ARIA behavior, placement, styling, or dismissal. + +--- + +## Diagnostic bundle backend + +### Job and archive service + +- Add a dedicated `apps/backend/internal/system/logbundle` package rather than + growing the existing file-list handler. It owns job state, temporary working + directories, archive construction, expiry cleanup, and a clock/filesystem + seam for tests. +- Bundle jobs are identity-owned, use unguessable IDs, create owner-only files, + have a five-minute collecting/building deadline, expire 15 minutes after + becoming ready/partial, and have states `collecting`, `building`, `ready`, + `partial`, `failed`, and `expired`. +- Move collection, copy, JSONL writing, and ZIP construction to a bounded + background worker. Allow one active job per identity, eight collecting/queued + jobs process-wide, and one build process-wide. Coalesce the same source set; + return `429` for a different identity-local active request and `503` for + global saturation, both with `Retry-After: 5`. +- Backend capture copies only recognized active/dated log files inside the + rolling three-day window into `backend/`, preserving bytes and names. Apply a + 160 MiB backend budget newest-first; include only the newest byte range when + a file exceeds the remaining budget. +- Accept at most four browser profiles, 20 MiB each and 80 MiB total frontend + data. Cap uncompressed archive payload at 256 MiB and temporary job data at + 384 MiB, checking available space before admission. Use ZIP `Store`, stream + 1 MiB chunks, and yield between chunks rather than spending CPU on deflate. +- Runtime metadata formerly exposed by debug export goes into + `manifest.json`. Manifest construction is deterministic and records sources, + included filenames, browser/connection counts, storage modes, truncation, + timeouts, warnings, and expiry without listing another user's identity. +- ZIP entry names are server-owned. Reject symlinks and non-regular backend + files, and enforce containment before opening any source or working path. + +### Frontend request/response protocol + +- Add `POST /api/v1/system/logs/bundles`, + `GET /api/v1/system/logs/bundles/:id`, + `POST /api/v1/system/logs/bundles/:id/frontend`, and + `GET /api/v1/system/logs/bundles/:id/download`. +- A frontend source sends `system.logs.capture_requested` through the WebSocket + hub only to connected clients matching the HTTP caller's identity. Extend + the hub with an identity-targeted send that returns the number of queued + connections; do not use a global broadcast. +- Collect for at most 15 seconds. Track targeted/responded connection counts, + accept sequential ≤1 MiB chunks, cap each browser at 10,000 entries/20 MiB, + and cap each job at four browsers/80 MiB. +- Require an opaque per-tab `capture_stream_id`. The first accepted chunk + atomically claims its browser ID; acknowledge later streams for that browser + without decoding/appending entries. Validate sequence only within the winning + stream so tab interleaving cannot mix snapshots. +- Validate every frontend entry again on the backend. Write one server-named + JSONL file per distinct browser profile. +- Backend-only jobs skip WebSocket capture and can build immediately. +- Register the bundle worker/expiry loop with explicit start/stop ownership and + leak tests per backend goroutine conventions. + +--- + +## Browser console retention and capture + +### Local store + +- Replace the memory-only implementation behind `apps/web/lib/logger/buffer.ts` + with a storage facade: IndexedDB is authoritative, while the 500-entry + memory ring remains the fallback. +- Preserve the existing interceptor seams. Normalize console arguments into + bounded JSON-safe data off the intercepted console call path and add URL, + recognized route task ID, browser installation ID, serialized size, and + storage mode. +- The interceptor creates a reference-free preview: inspect at most 20 + arguments, keep capped primitives and `Error` fields, and represent other + values by safe type descriptors without retaining objects, walking + prototypes, or invoking getters. +- Enforce a 500-entry/2 MiB staging queue. Drain at most 50 entries/256 KiB per + transaction after 250 ms or in idle time with a one-second timeout fallback; + never await IndexedDB opening, serialization, pruning, or quota work from a + console call. Shed debug/info before warn/error and retain loss counters. +- Retain three UTC days and enforce 10,000 entries/20 MiB per browser profile, + with 64 KiB per entry. Prune expired records first, then oldest-first. +- Store one opaque random browser installation ID per browser profile. All tabs + share it; never expose it as a ZIP filename or authorization credential. +- Partition entries by the current authenticated Kandev identity and snapshot + only that partition. Apply entry/byte caps across all partitions so account + churn cannot create unbounded browser storage. +- Make initialization, capture, snapshot, pruning, quota failure, and memory + fallback testable without relying on a real browser database. + +### WebSocket responder and uploads + +- Add a narrow top-level hook/bridge that handles + `system.logs.capture_requested`, snapshots local history, chunks it under the + advertised limit, uploads sequentially, and always sends a final `done` + chunk when possible. +- Every tab creates a random `capture_stream_id` for its response. Multiple tabs + may answer; the backend atomically accepts one stream per browser ID, so the + client does not need cross-tab leader election. +- Put bounded loss/persistence/truncation counters in the final `done` chunk's + `capture_metadata`; upload no more than 8 KiB and let the backend validate it. +- Capture/upload failures stay silent in the console to avoid recursion. The + backend's partial manifest is the diagnostic surface. + +--- + +## System Logs user workflow + +- Replace `LogViewer` with one focused diagnostic-bundle card. Remove tail, + copy, refresh, file table, badges, and individual file actions. +- Explain in visible plain language that the combined ZIP contains install-wide + backend logs plus locally retained browser console events, full URLs, + console arguments, stacks, and runtime metadata; disclose the three-day, + four-browser, and fixed byte limits and tell users to review before sharing. +- The primary **Download diagnostic bundle** action requests both sources, + polls/subscribes to job progress, downloads ready or partial ZIPs, and shows + why a partial bundle lacks frontend data. +- Reuse `SystemPageShell`, `Card`, `Button`, and existing action-feedback/status + patterns. The route remains `/settings/system/logs`. + +### Mobile design contract + +- Desktop and phone use the same one-dimensional card and shared state/action. + There is no table, sidebar, overlay, or alternate navigation to adapt. +- The existing System settings route is the mobile entry point. The disclosure + precedes the primary action; status/warnings follow it. +- Use the normal page as the single scroll owner. The action has at least a + 44px active height, remains full-width when helpful on phones, and requires + no hover or wide-viewport interaction. +- Add a `mobile-chrome` E2E scenario proving disclosure, touch activation, + progress/partial behavior, download availability, and no horizontal overflow. + +--- + +## Custom bundles, runtime index, and ACP debug evidence + +### Backend source and permission contracts + +- Extend `apps/backend/internal/system/logbundle` source normalization and job + identity to support `runtime` and `acp` plus a normalized ACP session-ID set. + Preserve `backend`/`frontend` behavior and make coalescing key off both source + and selected-session sets. +- Add authenticated capabilities and ACP-candidate endpoints under + `/api/v1/system/logs`. Cap candidate rows at 500, return only allow-listed + session/runtime fields, and derive candidate visibility through existing task + authorization: owners see their sessions, admins may see all. +- Add a runtime-index provider that emits `runtime/sessions.json` from at most + 500 authorized sessions in the three-day diagnostic window plus explicitly + selected ACP sessions. Use a dedicated DTO so task titles, descriptions, + identities, messages, prompts, tool data, files, arbitrary metadata, and + configuration cannot enter by accident. +- Gate ACP capability, candidate listing, and job creation on the backend's + effective `KANDEV_DEBUG_AGENT_MESSAGES` state. A client-side debug flag is + presentation only and never authorizes collection. +- Keep `manifest.json` mandatory. Add requested/included runtime and ACP source + fields, selected/available/omitted session counts, executor availability, + byte ranges, deadlines, and warnings without recording user identities or + filesystem paths. + +### On-demand ACP export + +- Add an agentctl internal export route beside the existing debug ACP tail. It + is registered only while ACP debug logging is enabled, accepts an exact + session and clamped byte budget, and produces a bounded ZIP of recognized + raw/normalized retained JSONL files for only that session. +- Add a runtime/lifecycle client seam that locates the selected execution and + streams the export from reachable local, Docker, Sprites, or remote agentctl + instances. Host-resident standalone logs use the same recognized-filename + reader. Do not centralize frames before a request. +- Authorize all selected sessions before I/O. Fetch at most two executors + concurrently, collect at most ten sessions for 30 seconds total, stream in + fixed chunks, and cancel/close every partial response on deadline or job + cancellation. +- Revalidate returned ZIP structure, regular-file status, per-entry/session + ownership, JSONL filenames, and bytes before copying into server-numbered + `acp/session-NN/` paths. Treat stopped/unreachable/expired/invalid sessions as + partial rather than falling back to another session or broader directory. +- Preserve the existing 256 MiB archive and 384 MiB temporary-disk caps. + Standard bundles retain 160 MiB backend and 80 MiB frontend budgets; + ACP-inclusive jobs use 96 MiB backend, 48 MiB frontend, 96 MiB ACP, and + 2 MiB runtime budgets with no redistribution from omitted sources. + +### Logs page customization + +- Replace the single action copy in + `apps/web/components/settings/system/log-viewer.tsx` with a shared bundle + request controller and one **Customize bundle** entry point. +- Fetch backend capabilities instead of inferring ACP availability from + `window.__KANDEV_DEBUG`. The customizer defaults to backend+frontend, + offers runtime as an opt-in, and exposes ACP as an opt-in only when the + backend allows it. Selecting ACP loads candidates on demand and requires one + to ten eligible sessions before submit. In debug mode it remains available + with an explanatory empty state when the caller has no eligible session. +- Add exact translated copy describing frontend and backend event classes, + stating that standard bundles do not read stored chat/session/agent messages, + and warning that incidental text already emitted to logs may remain. +- When ACP is selected, replace the standard reassurance with the explicit raw + protocol warning covering prompts, responses, tools, files, MCP data, + environment-derived values, and secrets. Show each session's authorized task + title as a new-tab task link, per-session availability, and explicit + select-all/clear controls that honor the server's maximum; explain why + unavailable sessions cannot be collected. Keep task titles out of the + runtime index and all archive metadata. +- Keep all selection, validation, polling, partial/error status, and download + logic shared. Do not add source preferences to persistent settings or a + database; each customizer opening starts from the same defaults. + +### Mobile design contract + +- Desktop uses the existing design-system `Dialog`; phone uses the shipped + inset `Drawer` pattern from `mobile-menu-sheet`/`mobile-picker-sheet`. The + Settings > Logs route and diagnostic card remain the entry point. +- The disclosure/header and bottom actions stay fixed; the source/session list + is the only `min-h-0 overflow-y-auto` scroll owner. Clear the bottom safe area + and keep document horizontal overflow at zero. +- Stack card actions on phones, keep each primary/secondary trigger and drawer + row at least 44px high, preserve visible labels, and avoid hover-only details. +- Add desktop and `mobile-chrome` E2E flows proving standard download, + customization, ACP session validation, high-sensitivity disclosure, partial + unavailable-session behavior, safe-area/overflow geometry, and equivalent + final bundle contents. + +--- + +## Agent diagnostic materialization + +- Register task-mode `get_diagnostic_bundle_kandev` with a required + `sources` enum (`backend`, `frontend`, or `all`) and no identity, task, + session, path, or token arguments. +- Forward the current MCP server's task/session IDs. The existing in-session MCP + identity scoper resolves the task owner before the backend handler creates a + bundle; external credentialed MCP callers retain their authenticated identity. +- Wait for ready/partial within the job's hard lifetime, then materialize the + ZIP through the existing lifecycle/agentctl file-transfer seam to + `.kandev/diagnostics/.zip` inside the current execution workspace, + mode `0600`. Never trust a caller-selected destination. +- Return the executor-local path plus manifest source/completeness/loss summary. + A backend-only call never sends a frontend WebSocket capture request. +- Clean materialized files on session/task workspace cleanup. The repository's + ignored `.kandev/` directory keeps the artifact out of git status. +- Keep `scripts/kandev-logs` as a host-side fallback: no credential in + auth-disabled mode; under auth it reads a PAT only from + `KANDEV_API_TOKEN`, sends it as Bearer auth, and never prints it. + +--- + +## Legacy-path removal and Improve Kandev migration + +- Remove the backend structured ring-buffer core/package and + `Logger.BufferSnapshot`. +- Remove System Logs list/tail/individual-download handlers, service state, + frontend API methods/hooks/store fields/components, and their tests. +- Remove dev-only `/api/v1/system/debug/export` and update + `scripts/kandev-logs` to create/download source-selectable bundle jobs. +- Migrate Improve Kandev's “Include logs” path to the public authenticated + all-sources job. After bootstrap returns, the initiating browser creates and + waits for ready/partial, then calls + `POST /api/v1/system/improve-kandev/bundle/lease` with its job ID and bootstrap + directory. The backend verifies both belong to the same identity, copies + `diagnostic-bundle.zip`, and returns the path appended to the task + description. Bootstrap itself never waits for the 15-second collection. +- Bind every bootstrap directory to its authenticated creator with a + server-written owner marker outside the archive. Prefix/path validation is + containment defense, not authorization. +- Update Improve Kandev's disclosure from “small in-memory ring buffer” to exact + bundle contents and partial/truncation behavior. +- Reuse archive construction and validation while keeping the longer explicit + Improve Kandev lease separate from 15-minute public bundle jobs. + +--- + +## Tests + +- **Daily writer and dual sinks:** fake-clock Go tests cover normal/debug/ + verbose thresholds, same/cross-day restarts, midnight rollover, concurrent + writes, `0600`, strict cleanup matching, rollover retry, blocked I/O, + independent-sink isolation, exact capacities, priority shedding, loss + accounting, bounded close/fatal behavior, startup fallback/retry, journaled + crash points, collision recovery, and producer non-blocking behavior. +- **Configuration:** Go tests prove removed destination/rotation keys are + ignored and level/format remain. +- **Toast endpoint:** Gin tests cover auth, valid sources, optional task ID, + malformed/oversized bodies, missing text, UTF-8 truncation, and fixed + server-owned message/level, per-identity/global token buckets, TTL cleanup, + `429`, and `Retry-After`. +- **Launchers:** native and TypeScript tests cover every file/stdout level + matrix and warning stdout forwarding. +- **Toast seams:** Vitest covers environment/route capture, silent failure, + delegate-before-report ordering, legacy transitions, non-awaited scheduling, + two-second deadline, and direct-import prevention. +- **Bundle service:** Go tests cover identity ownership, source selection, + backend allow-list/containment, manifest determinism, ZIP layout, chunk + ordering/bounds, browser dedupe, partial/timeout states, expiry cleanup, + frontend identity isolation, interleaved capture streams, exact coalescing/ + `429`/`503` capacity, per-source/job/temp byte caps, newest-range truncation, + disk-space refusal, five-minute cancellation, and start/stop leak safety. +- **Runtime index and capabilities:** Go handler/service tests cover debug + capability gating, owner/admin session visibility, foreign-ID non-disclosure, + source/session validation, coalescing keys, allow-listed runtime DTO fields, + 500-row/three-day bounds, and failure-to-partial behavior. +- **ACP exporter and collector:** Agentctl tests cover debug-disabled `404`, + exact-session file matching, raw+normalized rotation selection, clamped + newest-byte ranges, malicious paths, and bounded streaming. Runtime/logbundle + tests cover host and executor sources, owner/admin authorization before I/O, + ten-session/two-fetch/30-second limits, cancellation, partial unavailable + sessions, per-source budgets, archive renaming, manifest mapping, and no + continuous-copy goroutine. +- **Browser store:** Vitest with an IndexedDB test implementation covers + three-day/entry/byte pruning, 64 KiB entry limits, cyclic/unserializable + values, reference-free object descriptors, argument/field caps, shared + browser ID, identity partitioning/account switches, quota failure, memory + fallback, snapshots, exact staging/batch limits, overflow priority, and the + invariant that slow persistence cannot delay the intercepted console call. +- **Capture bridge:** Vitest covers WS request handling, per-tab stream IDs, + sequential chunks, done marker, interleaved duplicate-tab safety, and silent + upload failures. +- **System Logs UI:** component tests cover disclosure, job states, partial + warning, download, and accessible action. +- **Custom Logs UI:** Vitest covers standard source defaults, capability-driven + ACP visibility, custom source validation, required/owned session selection, + runtime-index selection, standard versus ACP disclosure, translated copy, + shared polling/download state, unavailable sessions, and desktop dialog versus + phone drawer presentation. +- **Legacy removal:** source scans reject the old tail/list/debug-export APIs + and backend ring-buffer imports. +- **Improve Kandev:** backend/frontend tests cover reuse of the all-sources ZIP + job, owner-verified lease, non-blocking bootstrap, partial archive, and + task-description path. +- **Agent diagnostics:** Go/MCP integration tests cover task-owner identity, + source selection, no caller-selected path, local and remote materialization, + cleanup, backend-only no-capture behavior, and manifest summary. + +## E2E tests + +- Update `apps/web/e2e/tests/system/logs-page.spec.ts` using TDD: navigate through + the System Logs UI, assert the full disclosure, request a bundle, answer the + backend capture request through the real frontend bridge, and verify the + downloaded ZIP exposes separate backend/frontend entries and a manifest. +- Add `apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts` for the + `mobile-chrome` project: use `.tap()`, verify the ≥44px action, partial-state + feedback when capture is unavailable, downloadable backend evidence, and no + document horizontal overflow. +- Extend the desktop spec with debug-disabled standard/custom behavior and a + debug-enabled seeded session catalogue. Verify that **Download with ACP…** + cannot submit without a session, the strong warning is visible, selected raw + and normalized evidence plus the sanitized runtime index appear in the ZIP, + and unavailable sessions yield an explicit partial result. +- Extend the mobile spec to open Customize and ACP selection through the inset + drawer, complete the same source/session workflow, assert ≥44px controls, + internal scrolling, viewport containment, safe-area clearance, focus return, + and zero document horizontal overflow. +- Use `pnpm e2e:run` so production backend/web artifacts are rebuilt before the + focused specs run. + +--- + +## Public documentation + +- Update `docs/public/configuration.md` to remove destination/lumberjack + settings and document fixed daily files and the three-day UTC policy. +- Update `docs/public/contributing.md` for startup path output and the + diagnostics-bundle workflow. +- Update `docs/public/docker.md`, `docs/public/operations.md`, and + `docs/public/k8s.md` for resolved file paths, container stdout thresholds, + source-selectable bundle downloads, browser availability, and sensitive data. +- Document that the System Logs page downloads backend plus frontend evidence, + that frontend history stays local until requested, and that partial bundles + must be checked via `manifest.json`. +- Update CLI/operations/configuration/remote-cloud docs to distinguish the + standard message-table-free source boundary from raw ACP's explicit + high-sensitivity content, owner/admin selection rules, host versus executor + availability, runtime index fields, fixed ACP limits, and partial behavior. +- Update `.agents/skills/debug` guidance to request standard sources first and + use ACP only after a human-selected debug bundle is available. Continue exact + task-ID/session-ID grep and never describe standard no-transcript language as + applying to an ACP-inclusive archive. + +--- + +## Debug harness + +- Update `.agents/skills/debug/SKILL.md` and its instance/browser references to + use `get_diagnostic_bundle_kandev` in task sessions and request `backend` + first for backend/live-instance issues, `frontend` for browser evidence, or + `all` only when correlation requires both. +- Teach agents to create a fresh temporary directory, download the ZIP, extract + it safely, inspect `manifest.json`, exact-grep task ID first, then broaden by + session ID, route/error text, and time window. +- Treat a zero task-ID match as evidence, not proof: global/startup lines and + frontend pages outside task routes may omit it. +- Update `scripts/kandev-logs` to drive bundle jobs rather than debug export or + the removed stderr-tail convention. Under auth it reads only the + `KANDEV_API_TOKEN` environment variable. Never extract over an existing + directory. + +--- + +## Verification Results + +Tasks 01–11 record the implemented standard bundle results in their individual +task files. Tasks 12–17 now record the implemented custom source/runtime-index, +on-demand ACP, consolidated customizer UI, E2E, and privacy-guidance work. No +database migration or browser schema migration is required by this extension. + +--- + +## Implementation waves and parallel candidates + +The default is sequential execution in the primary conversation. + +Wave 1: + +- [x] [Task 01: Backend log sinks](task-01-backend-log-sinks.md) + +Wave 2: + +- [x] [Task 02: Frontend-error endpoint](task-02-frontend-error-endpoint.md) +- [x] [Task 03: Launcher contracts](task-03-launcher-contracts.md) + +Task 02 and Task 03 are parallel-safe after Task 01. Parallel execution still +requires explicit user authorization. + +Wave 3: + +- [x] [Task 04: Toast reporting](task-04-toast-reporting.md) +- [x] [Task 07: Diagnostic bundle backend](task-07-diagnostic-bundle-backend.md) + +Task 04 owns frontend toast files; Task 07 owns backend bundle/gateway files. +They are parallel-safe after their dependencies with explicit authorization. + +Wave 4: + +- [x] [Task 08: Browser logs and bundle UI](task-08-browser-logs-and-bundle-ui.md) + +Wave 5: + +- [x] [Task 10: Agent diagnostic materialization](task-10-agent-diagnostic-materialization.md) + +Wave 6: + +- [x] [Task 09: Remove legacy diagnostics](task-09-remove-legacy-diagnostics.md) + +Wave 7: + +- [x] [Task 05: Public logging documentation](task-05-public-logging-docs.md) +- [x] [Task 06: Debug workflow guidance](task-06-debug-workflow-guidance.md) + +Task 05 and Task 06 are parallel-safe after Tasks 09 and 10 because public docs +and agent-harness files are disjoint. Parallel execution still requires +explicit user authorization. + +Wave 8: + +- [x] [Task 11: Merge-risk hardening](task-11-merge-risk-hardening.md) + +Task 11 is a sequential PR-remediation task covering the current-main lint +integration, bounded automatic toast ingestion, the fixed daily file bound, +URL privacy hardening, and allocation-free IndexedDB pruning. + +Wave 9: + +- [x] [Task 12: Custom bundle contracts and runtime index](task-12-custom-bundle-contracts.md) + +Wave 10: + +- [x] [Task 13: On-demand ACP collection](task-13-on-demand-acp-collection.md) + +Task 13 depends on Task 12's source/session and manifest contracts. Both tasks +are sequential because they share logbundle job/archive state and permission +boundaries. + +Wave 11: + +- [x] [Task 14: Bundle customizer UI](task-14-bundle-customizer-ui.md) + +Wave 12: + +- [x] [Task 15: Custom bundle E2E](task-15-custom-bundle-e2e.md) +- [x] [Task 16: Diagnostic privacy documentation](task-16-diagnostic-privacy-docs.md) + +Task 15 follows the completed backend/frontend behavior. Task 16 is +parallel-safe with Task 15 because it owns public docs and debug guidance while +Task 15 owns Playwright fixtures/specs; parallel execution still requires +explicit user authorization. + +Wave 13: + +- [x] [Task 17: Bundle customizer refinement](task-17-bundle-customizer-refinement.md) + +## Risks + +- Frontend console data is highly sensitive. It remains browser-local until an + explicit request, but combined bundle disclosure and identity routing must be + unambiguous. +- IndexedDB quota/private-mode failures must degrade to bounded memory without + breaking console behavior or bundle creation. +- WebSocket capture and HTTP chunk upload span two transports; ownership, + expiry, disconnect, duplicate tabs, and late chunks need deterministic tests. +- ZIP creation must not follow symlinks, accept client paths, zip-slip on + extraction guidance, or grow beyond frontend/backend bounds. +- Raw ACP can contain prompts, responses, tools, files, MCP payloads, and + secrets. Capability gating, explicit session selection, owner/admin + authorization before I/O, high-sensitivity copy, and server-owned archive + paths are release blockers. +- Executor-side ACP evidence may disappear with stopped or removed containers. + On-demand collection must report this as partial without centralizing frames + continuously or blocking session/agent work. +- The runtime index must be built from a field allow-list, not generic model or + metadata serialization, so future message-bearing fields cannot enter by + accident. +- Desktop Dialog and phone Drawer must share state/validation while retaining + one phone scroll owner, safe-area clearance, ≥44px targets, and no horizontal + overflow. +- Any authenticated user retains install-wide backend log access by explicit + product choice. +- Removing the ring buffer affects Logs, debug export, Improve Kandev, logger + construction, scripts, tests, and user copy; migration must be atomic. +- Task-ID filtering is best effort. Logs before task resolution or outside task + routes require session, route, error-text, or time-window correlation. diff --git a/docs/plans/diagnostic-logging/task-01-backend-log-sinks.md b/docs/plans/diagnostic-logging/task-01-backend-log-sinks.md new file mode 100644 index 0000000000..2dda760e31 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-01-backend-log-sinks.md @@ -0,0 +1,90 @@ +--- +id: "01-backend-log-sinks" +title: "Backend log sinks" +status: completed +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 01: Backend log sinks + +## Acceptance + +- Backend owns an owner-only `/logs/backend-logs.log`, appends + across same-UTC-day restarts, and rolls it at UTC midnight or the first + startup after a day boundary. +- Completed files use `backend-logs-YYYY-MM-DD.log`; cleanup preserves only the + current UTC day and two preceding days while leaving unrelated neighbors. +- Normal, debug, and verbose file/stdout thresholds match the spec. The backend + constructor does not add a new dependency on the legacy in-memory buffer. +- Log producers never wait for file or stdout I/O. Independent file/stdout + queues enforce the exact entry/byte/reserved capacities and 256 KiB entry cap + from the spec, shed debug/info first, and expose per-sink loss counters. +- A read-only atomic sink-statistics snapshot is available to the later bundle + manifest without retaining log entries. +- Close drains for at most two seconds; terminal DPanic/Panic/Fatal behavior + uses the bounded direct-stderr fallback from the spec. +- File-open and cleanup failures do not prevent startup. The backend reports + fallback and retries file activation every 30 seconds. +- Destination and lumberjack rotation settings no longer influence backend + logging; the daily writer exposes the strict active/dated file set consumed + by the later bundle service. + +## Verification + +```bash +cd apps/backend +go test ./internal/common/logger ./internal/common/config ./internal/backendapp +``` + +## Files likely touched + +- `apps/backend/internal/common/logger/logger.go` +- `apps/backend/internal/common/logger/async_sink.go` +- `apps/backend/internal/common/logger/daily_writer.go` +- `apps/backend/internal/common/logger/logger_test.go` +- `apps/backend/internal/common/logger/daily_writer_test.go` +- `apps/backend/internal/common/config/config.go` +- `apps/backend/internal/common/config/config_test.go` +- `apps/backend/internal/backendapp/main.go` + +## Dependencies + +None. + +## Parallelism + +Sequential. This establishes the shared logging contract consumed by every +later task. + +## Inputs + +- Spec: What, Persistence guarantees, and the first five Scenarios. +- Plan: Backend / Fixed diagnostic file and dual sinks. +- ADR: fixed path, three-day UTC history, removed destination/rotation config. +- Existing patterns: `logger.NewLogger` and file-sink lifecycle. + +## Risks + +- Cleanup must never use a broad glob or delete unrelated log-directory files. +- The daily writer needs an explicit persisted day marker or equally robust + mechanism; an externally changed file mtime cannot silently choose its date. +- A rollover transaction journal must resume every tested crash point exactly + once and must never overwrite an existing dated destination. +- Midnight rollover must serialize with writes and retry without losing the + entry that observed the day boundary. +- Slow stdout/files must not cause request, agent, or workflow goroutines to + wait on logging. The implementation must make its finite queue and recovery + accounting testable with a blocked writer. +- A blocked stdout queue must not delay or shed entries accepted by a writable + file queue, and vice versa. +- `Logger.WithFields` and `Close` must retain every backend sink. +- File creation and cleanup must behave consistently on Windows and Unix. + +## Output contract + +Report the threshold matrix, UTC rollover/retention behavior, cleanup matching +rule, files changed, exact test command/result, blockers or risks, and update +this task plus `plan.md` status in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-02-frontend-error-endpoint.md b/docs/plans/diagnostic-logging/task-02-frontend-error-endpoint.md new file mode 100644 index 0000000000..386a860369 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-02-frontend-error-endpoint.md @@ -0,0 +1,72 @@ +--- +id: "02-frontend-error-endpoint" +title: "Frontend-error endpoint" +status: completed +wave: 2 +depends_on: ["01-backend-log-sinks"] +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 02: Frontend-error endpoint + +## Acceptance + +- Authenticated install users can submit the exact bounded + `POST /api/v1/system/logs/frontend-errors` contract and receive `204`. +- A valid request creates one fixed `frontend error toast` error entry with + client data, including an optional bounded `task_id`, only in structured + fields. +- Malformed, oversized, source-invalid, text-empty, and truncation cases follow + the spec without weakening the existing log-download allow-list. +- Per-identity 60/minute burst-20 and process-wide 300/minute burst-100 token + buckets return `429` plus `Retry-After`; stale identity buckets are removed. +- The endpoint lives in `internal/system/frontenderrors`, not the legacy + `internal/system/logs` package removed by Task 09. + +## Verification + +```bash +cd apps/backend +go test ./internal/system/frontenderrors ./internal/system +``` + +## Files likely touched + +- `apps/backend/internal/system/frontenderrors/handler.go` +- `apps/backend/internal/system/frontenderrors/handler_test.go` +- `apps/backend/internal/system/frontenderrors/limiter.go` +- `apps/backend/internal/system/frontenderrors/limiter_test.go` +- `apps/backend/internal/system/system.go` +- `apps/backend/internal/system/system_routes_test.go` + +## Dependencies + +- Task 01 provides the dual-sink logger and fixed file used by the endpoint. + +## Parallelism + +Parallel-safe with Task 03 after Task 01. The files are disjoint from the +launcher implementations and no shared schema or generated contract changes. + +## Inputs + +- Spec: API surface, Permissions, Failure modes, and report Scenarios. +- Plan: Backend / Frontend-error ingestion. +- Existing patterns: System route composition and `system/logs` handler tests. + +## Risks + +- Body limiting must occur before JSON decoding. +- Truncation must preserve valid UTF-8 and must not turn client data into a log + message or level. +- `task_id` is an untrusted correlation field, not authorization or proof that + the reporting user owns a task. +- Limiter identity keys come only from authenticated context; task IDs and + browser fields never select a bucket. + +## Output contract + +Report the final request shape and limits, files changed, exact test +command/result, blockers or risks, and update this task plus `plan.md` status +in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-03-launcher-contracts.md b/docs/plans/diagnostic-logging/task-03-launcher-contracts.md new file mode 100644 index 0000000000..a2781b0176 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-03-launcher-contracts.md @@ -0,0 +1,79 @@ +--- +id: "03-launcher-contracts" +title: "Launcher contracts" +status: completed +wave: 2 +depends_on: ["01-backend-log-sinks"] +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 03: Launcher contracts + +## Acceptance + +- Native and TypeScript normal/debug launch modes pass file `info`/`debug` + thresholds while keeping stdout at `warn`; verbose passes `info` to both. +- Backend stdout is forwarded in every launch mode without losing the + startup-failure capture used by health checks. +- Native and TypeScript help text describe `--debug` as file diagnostics and + `--verbose` as stdout-info output. + +## Verification + +```bash +cd apps/backend +go test ./internal/launcher +``` + +```bash +cd apps +pnpm --filter @kandev/cli test -- src/shared.test.ts src/start.test.ts src/run.test.ts src/dev.test.ts +``` + +## Files likely touched + +- `apps/backend/internal/launcher/start.go` +- `apps/backend/internal/launcher/start_test.go` +- `apps/backend/internal/launcher/env.go` +- `apps/backend/internal/launcher/process.go` +- `apps/backend/internal/launcher/process_test.go` +- `apps/backend/internal/launcher/supervisor.go` +- `apps/backend/internal/launcher/cli/help.go` +- `apps/cli/src/shared.ts` +- `apps/cli/src/shared.test.ts` +- `apps/cli/src/dev.ts` +- `apps/cli/src/start.ts` +- `apps/cli/src/start.test.ts` +- `apps/cli/src/run.ts` +- `apps/cli/src/run.test.ts` +- `apps/cli/src/cli.ts` + +## Dependencies + +- Task 01 defines the environment variables and backend sink behavior the + launchers drive. + +## Parallelism + +Parallel-safe with Task 02 after Task 01. It owns launcher/CLI files only. + +## Inputs + +- Spec: What and normal/debug/verbose Scenarios. +- Plan: Launchers and CLI contracts. +- Existing patterns: Go `startProcess` captured output and TypeScript release + `attachRingBuffer`. + +## Risks + +- TypeScript release mode must tee piped stdout without consuming or duplicating + the health-failure buffer. +- Restart manifests must preserve the internal console-threshold environment. +- `KANDEV_LOG_LEVEL` remains an explicit file-level override. + +## Output contract + +Report the resolved matrix for every launcher mode, files changed, exact test +commands/results, blockers or risks, and update this task plus `plan.md` status +in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-04-toast-reporting.md b/docs/plans/diagnostic-logging/task-04-toast-reporting.md new file mode 100644 index 0000000000..2919b38671 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-04-toast-reporting.md @@ -0,0 +1,86 @@ +--- +id: "04-toast-reporting" +title: "Toast reporting" +status: completed +wave: 3 +depends_on: ["02-frontend-error-endpoint"] +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 04: Toast reporting + +## Acceptance + +- Sonner error toasts and legacy error creation/error transitions each send + exactly one best-effort report with visible text and available rich browser + context. +- Recognized task routes add their task ID to the report; non-task routes omit + it without affecting the toast or report. +- Reporting success or failure does not change toast rendering, IDs, timing, + ordering, dismissal, ARIA behavior, or responsive behavior. +- Toast rendering is handed to the original library before reporting is + scheduled. The report is never awaited, uses a two-second deadline, and bounded + context collection cannot create a noticeable synchronous serialization path. +- Application Sonner call sites use the reporting wrapper, and a focused + source-scan test prevents new direct `sonner` toast imports. + +## Verification + +```bash +cd apps +pnpm --filter @kandev/web test -- lib/api/domains/frontend-error-log-api.test.ts lib/toast/sonner.test.ts components/toast-provider.test.tsx +``` + +```bash +cd apps/web +pnpm run typecheck +``` + +## Files likely touched + +- `apps/web/lib/api/domains/frontend-error-log-api.ts` +- `apps/web/lib/api/domains/frontend-error-log-api.test.ts` +- `apps/web/lib/toast/sonner.ts` +- `apps/web/lib/toast/sonner.test.ts` +- `apps/web/components/toast-provider.tsx` +- `apps/web/components/toast-provider.test.tsx` +- Application files returned by + `rg -l 'import \\{ toast \\} from "sonner"' apps/web --glob '*.{ts,tsx}'` + (import-line-only migration to `@/lib/toast/sonner`) + +## Dependencies + +- Task 02 defines and registers the report endpoint. + +## Parallelism + +Sequential after Task 02 because the frontend client must implement the +approved backend contract. + +## Inputs + +- Spec: API surface, Failure modes, and toast-report Scenarios. +- Plan: Frontend sections. +- Frontend guidance: shared behavior only; no mobile composition change. +- Existing patterns: `fetchJson`, Sonner call sites, and `ToastProvider`. + +## Risks + +- Fire-and-forget handling must not produce unhandled promise rejections or + recurse through console interception. +- Tests assert delegate-before-report call order for both toast systems. +- "Immediate" reporting must not become synchronous toast latency; slow or + offline networking must leave the UI path unaffected. +- Rich context and generated stacks are sensitive and must remain bounded by + the backend contract. +- Route parsing must accept only the documented task route shapes and must not + mistake unrelated path segments for task IDs. +- Non-text React nodes must not crash serialization or suppress the original + toast. + +## Output contract + +Report source coverage, preserved UI behavior, files changed, exact test +commands/results, blockers or risks, and update this task plus `plan.md` status +in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-05-public-logging-docs.md b/docs/plans/diagnostic-logging/task-05-public-logging-docs.md new file mode 100644 index 0000000000..f796a2e123 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-05-public-logging-docs.md @@ -0,0 +1,92 @@ +--- +id: "05-public-logging-docs" +title: "Public logging documentation" +status: completed +wave: 7 +depends_on: + - "01-backend-log-sinks" + - "02-frontend-error-endpoint" + - "03-launcher-contracts" + - "04-toast-reporting" + - "07-diagnostic-bundle-backend" + - "08-browser-logs-and-bundle-ui" + - "09-remove-legacy-diagnostics" + - "10-agent-diagnostic-materialization" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 05: Public logging documentation + +## Acceptance + +- Public configuration docs remove the destination and lumberjack rotation + settings and accurately explain the fixed active file, UTC daily rollover, + same-day restart append, rolling three-day retention, and file/stdout + threshold matrix. +- Development and Docker docs identify the actual resolved log path and warn + that bundles contain install-wide backend evidence plus sensitive browser + URLs, console arguments, and stacks. +- Public docs explain source-selectable agent bundles, browser-requested + capture, three-day local frontend retention, partial manifests, and the lack + of continuous frontend upload. +- Public docs explain best-effort queue loss, bundle byte/profile truncation, + 256 MiB archive bounds, 15-minute ready lease, and `429`/`503` busy behavior. +- Public-doc validation passes with no stale examples of custom backend output + destinations, individual System Logs downloads, tails, or debug exports. + +## Verification + +```bash +rg -n "logging\\.(outputPath|maxSizeMb|maxBackups|maxAgeDays|compress)|KANDEV_LOGGING_(OUTPUTPATH|MAXSIZEMB|MAXBACKUPS|MAXAGEDAYS|COMPRESS)|backend-logs\\.log|debug/export|logs/tail|diagnostic bundle" docs/public +``` + +```bash +node --test scripts/validate-public-docs.test.mjs +``` + +```bash +node scripts/validate-public-docs.mjs +``` + +## Files likely touched + +- `docs/public/configuration.md` +- `docs/public/contributing.md` +- `docs/public/docker.md` +- `docs/public/operations.md` +- `docs/public/k8s.md` + +## Dependencies + +Tasks 01–04 and 07–09 must settle the exact shipped behavior before public +wording is finalized. + +## Parallelism + +Parallel-safe with Task 06 after Tasks 09 and 10. It owns public documentation; +Task 06 owns agent-harness files and the log helper script. + +## Inputs + +- Spec: complete observable contract. +- ADR: consequences and rejected compatibility alternatives. +- Plan: Public documentation. +- Docs-maintainer skill requirements. + +## Risks + +- `logging.level` must be described as the file threshold, not a universal + sink level. +- “Three days” must consistently mean the current UTC calendar day plus the + two preceding UTC calendar days. +- Docker `/data` and development `.kandev-dev` examples must use their resolved + homes rather than hard-coded `~/.kandev`. +- Docs must distinguish the backend's three on-disk days from frontend history, + which stays local to a browser until a bundle requests it. + +## Output contract + +Report public docs changed, exact validation commands/results, stale references +found or removed, blockers or risks, and update this task plus `plan.md` status +in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-06-debug-workflow-guidance.md b/docs/plans/diagnostic-logging/task-06-debug-workflow-guidance.md new file mode 100644 index 0000000000..d45bea1c62 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-06-debug-workflow-guidance.md @@ -0,0 +1,99 @@ +--- +id: "06-debug-workflow-guidance" +title: "Debug workflow guidance" +status: completed +wave: 7 +depends_on: + - "09-remove-legacy-diagnostics" + - "10-agent-diagnostic-materialization" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 06: Debug workflow guidance + +## Acceptance + +- Task-session debug uses `get_diagnostic_bundle_kandev` and requests backend, + frontend, or all based on the bug class instead of calling the removed debug + export or tail. +- The workflow creates a fresh temporary directory, downloads the selected ZIP, + extracts it safely without overwriting an existing path, and inspects + `manifest.json` before trusting source completeness. +- When a task ID is known, the workflow searches it exactly with + `rg --fixed-strings` before broadening to session ID, route/error text, or a + bounded time window. +- A zero-match task-ID search is described as inconclusive because global logs + and frontend pages outside recognized task routes may omit `task_id`. +- Live-instance safety remains unchanged: bundle creation/download is read-only, + agents do not relaunch the user's instance, and they remove only temporary + extraction directories they created. +- Host-side `scripts/kandev-logs` works without credentials only when auth is + disabled; authenticated use reads a PAT from `KANDEV_API_TOKEN` and never + accepts or prints it as an argument. + +## Verification + +```bash +git diff --check -- .agents/skills/debug scripts/kandev-logs +``` + +```bash +python3 scripts/lint-harness-files.test.py +``` + +```bash +scripts/kandev-logs --help +``` + +```bash +rg -n "bundle|manifest|backend|frontend|task ID|task_id|fixed-strings" \ + .agents/skills/debug/SKILL.md \ + .agents/skills/debug/references/instance.md \ + .agents/skills/debug/references/browser.md \ + scripts/kandev-logs +``` + +## Files likely touched + +- `.agents/skills/debug/SKILL.md` +- `.agents/skills/debug/references/instance.md` +- `.agents/skills/debug/references/browser.md` +- `scripts/kandev-logs` +- focused shell tests for `scripts/kandev-logs` + +## Dependencies + +- Task 09 removes legacy diagnostics. +- Task 10 provides owner-scoped MCP materialization and the authenticated + host-helper contract. + +## Parallelism + +Parallel-safe with Task 05 after Tasks 09 and 10. It owns agent-harness files +and the log helper; Task 05 owns public documentation. + +## Inputs + +- Spec: Agent diagnostics, bundle API, partial-bundle behavior, and scenarios. +- Plan: Debug harness. +- Harness-improvement skill: update the narrowest existing skill and preserve + progressive disclosure. +- Existing patterns: debug triage gate, live-instance safety, + `kandev-instances`, `kandev-logs`, and browser/backend correlation. + +## Risks + +- Agents must inspect `manifest.json`; an all-sources or frontend bundle can be + partial when no browser responds. +- Extraction must reject traversal and must never target a workspace, + repository root, home directory, or reused broad path. +- Task IDs are correlation hints, not authorization or complete trace IDs. +- The helper must work for default and isolated instances and must not hard-code + `~/.kandev`. + +## Output contract + +Report source-selection guidance, extraction safety, exact evidence order, +files changed, validation commands/results, remaining correlation gaps, and +update this task plus `plan.md` status in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-07-diagnostic-bundle-backend.md b/docs/plans/diagnostic-logging/task-07-diagnostic-bundle-backend.md new file mode 100644 index 0000000000..e3baf43893 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-07-diagnostic-bundle-backend.md @@ -0,0 +1,99 @@ +--- +id: "07-diagnostic-bundle-backend" +title: "Diagnostic bundle backend" +status: completed +wave: 3 +depends_on: ["01-backend-log-sinks"] +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 07: Diagnostic bundle backend + +## Acceptance + +- Authenticated users can create identity-owned bundle jobs for any non-empty + subset of `backend` and `frontend`, inspect status, upload bounded frontend + chunks, and download ready/partial ZIPs. +- Backend-only jobs copy the strict retained daily file set and build without + waiting for a browser. +- Frontend jobs send `system.logs.capture_requested` only to WebSocket clients + matching the caller identity, collect for at most 15 seconds, deduplicate by + browser profile, atomically bind the first per-tab capture stream for each + browser, and validate sequential chunk/body/browser bounds. +- ZIPs contain only server-owned paths under `backend/`, `frontend/`, and + `manifest.json`; source symlinks, path traversal, and non-regular files are + rejected. +- Jobs, working files, and ZIPs are owner-only. Collection/build has a + five-minute hard lifetime; ready/partial ZIPs expire after 15 minutes. +- Enforce one active job per identity, eight collecting/queued process-wide, + and one builder. Equivalent source sets coalesce; conflicts return `429` and + global saturation `503`, both with `Retry-After: 5`. +- Enforce four browsers, 20 MiB/profile, 80 MiB frontend, 160 MiB backend, + 256 MiB uncompressed payload, and 384 MiB temporary-disk limits. Backend + truncation selects newest byte ranges and makes the manifest/job partial. +- ZIP payloads use `Store` and 1 MiB yielded chunks rather than deflate. + +## Verification + +```bash +cd apps/backend +go test ./internal/system/logbundle ./internal/system ./internal/gateway/websocket +``` + +```bash +cd apps/backend +go test -race ./internal/system/logbundle ./internal/gateway/websocket +``` + +## Files likely touched + +- `apps/backend/internal/system/logbundle/service.go` +- `apps/backend/internal/system/logbundle/job.go` +- `apps/backend/internal/system/logbundle/archive.go` +- `apps/backend/internal/system/logbundle/handler.go` +- `apps/backend/internal/system/logbundle/*_test.go` +- `apps/backend/internal/system/system.go` +- `apps/backend/internal/gateway/websocket/hub.go` +- `apps/backend/internal/gateway/websocket/hub_broadcast_test.go` +- `apps/backend/internal/backendapp/helpers.go` + +## Dependencies + +- Task 01 provides the strict active/dated backend file set. + +## Parallelism + +Parallel-safe with Task 04 after Task 01. It owns backend bundle/gateway files; +Task 04 owns frontend toast files. + +## Inputs + +- Spec: System Logs page and bundle contents, Diagnostic bundle jobs, + Permissions, Failure modes, and bundle Scenarios. +- Plan: Diagnostic bundle backend. +- ADR: files are authoritative, capture is explicit, agents select sources. +- Existing patterns: system route composition, WebSocket identity routing, + owner-scoped jobs, and bounded archive/file services. + +## Risks + +- The bundle owner comes only from authenticated context; browser IDs are + dedupe hints and cannot authorize access. +- Capture crosses WebSocket notification and HTTP chunk upload. Late, + duplicate, disconnected, expired, and out-of-order responses must converge + deterministically. +- A browser ID is claimed atomically by one `capture_stream_id`; losing streams + are acknowledged before entry decoding so duplicate tabs cannot amplify CPU. +- ZIP entry names and source paths must never use client-supplied path data. +- Background cleanup must stop and join without goroutine leaks. +- Bundle work must not run expensive filesystem or archive work in request + handlers or permit an unbounded number of concurrent exports. +- Admission checks temporary free space before accepting a job; cancellation + removes partial files without waiting for the 15-minute ready lease. + +## Output contract + +Report the final job state machine, identity routing, limits, archive layout, +files changed, exact test commands/results, blockers/risks, and update this task +plus `plan.md` status in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-08-browser-logs-and-bundle-ui.md b/docs/plans/diagnostic-logging/task-08-browser-logs-and-bundle-ui.md new file mode 100644 index 0000000000..392c83e3db --- /dev/null +++ b/docs/plans/diagnostic-logging/task-08-browser-logs-and-bundle-ui.md @@ -0,0 +1,129 @@ +--- +id: "08-browser-logs-and-bundle-ui" +title: "Browser logs and bundle UI" +status: completed +wave: 4 +depends_on: + - "04-toast-reporting" + - "07-diagnostic-bundle-backend" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 08: Browser logs and bundle UI + +## Acceptance + +- Console debug/info/warn/error calls, uncaught errors, and unhandled + rejections retain normal browser behavior while entering a bounded three-day + IndexedDB history with URL and recognized-route task ID. +- Retention enforces 64 KiB per entry and 10,000-entry/20 MiB per browser + profile limits. IndexedDB failure degrades to the 500-entry memory buffer. +- Entries are partitioned by authenticated Kandev identity; account switches + in one browser profile cannot include the prior account's frontend history. +- Every tab handles identity-scoped `system.logs.capture_requested` and uploads + a sequential bounded snapshot with a random per-tab `capture_stream_id`, + without logging its own failures or causing console recursion. +- The final `done` chunk includes at most 8 KiB of loss, persistence, storage, + and truncation metadata for the manifest. +- Interception creates reference-free previews only: at most 20 arguments, + bounded primitives/Error fields, and non-traversing type descriptors for + other values. It never retains arbitrary caller objects. +- The staging queue is capped at 500 entries/2 MiB. IndexedDB work drains at + most 50 entries/256 KiB after 250 ms or idle time with a one-second timeout; + full staging sheds debug/info before warn/error and records loss. +- System Logs renders one clear combined-download workflow with sensitive-data + disclosure and collecting/preparing/partial/busy/error feedback, including + `Retry-After` handling. Tail, copy, refresh, file table, and individual + download UI are absent. +- Desktop and phone provide the same capability. The phone action is + touch-accessible, uses the page as its single scroll owner, and has no + horizontal overflow. + +## Verification + +```bash +cd apps +pnpm --filter @kandev/web test -- \ + lib/logger/buffer.test.ts \ + lib/logger/intercept.test.ts \ + lib/logger/indexeddb-store.test.ts \ + lib/api/domains/system-api.test.ts \ + components/log-buffer-bridge.test.tsx \ + components/settings/system/log-viewer.test.tsx +``` + +```bash +cd apps/web +pnpm run typecheck +``` + +```bash +cd apps/web +pnpm e2e:run e2e/tests/system/logs-page.spec.ts \ + e2e/tests/system/mobile-logs-bundle.spec.ts +``` + +## Files likely touched + +- `apps/web/lib/logger/buffer.ts` +- `apps/web/lib/logger/buffer.test.ts` +- `apps/web/lib/logger/intercept.ts` +- `apps/web/lib/logger/intercept.test.ts` +- `apps/web/lib/logger/indexeddb-store.ts` +- `apps/web/lib/logger/indexeddb-store.test.ts` +- `apps/web/components/log-buffer-bridge.tsx` +- `apps/web/components/log-buffer-bridge.test.tsx` +- `apps/web/lib/ws/handlers/system-events.ts` +- `apps/web/lib/ws/handlers/system-events.test.ts` +- `apps/web/lib/api/domains/system-api.ts` +- `apps/web/lib/api/domains/system-api.test.ts` +- `apps/web/components/settings/system/log-viewer.tsx` +- `apps/web/components/settings/system/log-viewer.test.tsx` +- `apps/web/app/settings/system/logs/page.tsx` +- `apps/web/e2e/tests/system/logs-page.spec.ts` +- `apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts` + +## Dependencies + +- Task 04 provides shared route/task-ID extraction and preserves toast behavior. +- Task 07 defines the WS capture notification, chunk upload, job status, and + download contracts. + +## Parallelism + +Sequential after Tasks 04 and 07 because it implements both approved contracts. + +## Inputs + +- Spec: Browser console history, System Logs page, bundle APIs, failure modes, + persistence, desktop/mobile scenarios. +- Plan: Browser console retention and capture; System Logs user workflow. +- Frontend guidance: API calls through domain clients/hooks, no direct component + fetches, shadcn imports, and explicit sensitive-data copy. +- Mobile guidance: one focal action, ≥44px touch target, shared business logic, + one scroll owner, and `mobile-chrome` outcome coverage. +- Existing patterns: console interceptor/buffer, log page action feedback, + SystemPageShell, WS handler registry, and Improve Kandev frontend upload. + +## Risks + +- Serialization must handle cycles, DOM/host objects, getters, large values, + quota failures, and unavailable IndexedDB without breaking console calls. +- Object preview must not invoke getters or retain references; hostile proxies + must fail closed to a generic descriptor within the fixed argument cap. +- Slow or blocked IndexedDB must be testably unable to delay console methods, + rendering, or the WebSocket event loop. +- Capture failures must not call intercepted console methods. +- Multiple tabs share one browser ID and store; duplicate responses must remain + safe even when they race. +- Identity partitioning must fail closed while auth state is unavailable or + changing; it cannot fall back to returning every local partition. +- Download E2E must verify user-visible behavior and ZIP contents against a + freshly rebuilt production bundle. + +## Output contract + +Report local storage bounds/fallback, WS upload behavior, desktop/mobile UI +contract, files changed, exact unit/typecheck/E2E results, blockers/risks, and +update this task plus `plan.md` status in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-09-remove-legacy-diagnostics.md b/docs/plans/diagnostic-logging/task-09-remove-legacy-diagnostics.md new file mode 100644 index 0000000000..320e06f2f8 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-09-remove-legacy-diagnostics.md @@ -0,0 +1,111 @@ +--- +id: "09-remove-legacy-diagnostics" +title: "Remove legacy diagnostics" +status: completed +wave: 6 +depends_on: + - "07-diagnostic-bundle-backend" + - "08-browser-logs-and-bundle-ui" + - "10-agent-diagnostic-materialization" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 09: Remove legacy diagnostics + +## Acceptance + +- Backend logging no longer allocates or writes the process-wide structured + ring buffer, and no product path imports it. +- System Logs list/tail/individual-download APIs and their frontend + hooks/state/components are removed after the bundle UI has replaced them. +- Dev-only `/api/v1/system/debug/export` is removed; runtime metadata comes + from bundle manifests. +- Improve Kandev creates an authenticated all-sources browser job. Its + owner-verified lease endpoint copies the ready/partial ZIP for 24 hours and + gives the agent that path instead of separate snapshots; bootstrap never + waits for frontend collection. +- Bootstrap writes a server-owned identity marker; lease authorization checks + that marker and bundle ownership rather than trusting the submitted temp path. +- `scripts/kandev-logs` no longer calls debug export or tails the isolated + stderr convention; the focused agent-guidance task can finalize its UX on + the source-selectable bundle API. + +## Verification + +```bash +cd apps/backend +go test ./internal/common/logger ./internal/debug ./internal/improvekandev ./internal/system/... +``` + +```bash +cd apps +pnpm --filter @kandev/web test -- \ + components/improve-kandev-dialog-helpers.test.ts \ + lib/api/domains/improve-kandev-api.test.ts \ + lib/state/slices/system/system-slice.test.ts +``` + +```bash +rg -n "buffer\\.Default|BufferSnapshot|debug/export|logs/tail|buildLogDownloadUrl|useLogTail" \ + apps/backend apps/web scripts +``` + +## Files likely touched + +- `apps/backend/internal/common/logger/logger.go` +- `apps/backend/internal/common/logger/logger_test.go` +- `apps/backend/internal/common/logger/buffer/` +- `apps/backend/internal/debug/export.go` +- `apps/backend/internal/debug/export_test.go` +- `apps/backend/internal/system/logs/` +- `apps/backend/internal/system/system.go` +- `apps/backend/internal/backendapp/helpers.go` +- `apps/backend/internal/improvekandev/bundle.go` +- `apps/backend/internal/improvekandev/handler.go` +- `apps/backend/internal/improvekandev/*_test.go` +- `apps/web/hooks/domains/system/use-log-files.ts` +- `apps/web/hooks/domains/system/use-log-tail.ts` +- `apps/web/lib/state/slices/system/` +- `apps/web/components/improve-kandev-dialog-helpers.ts` +- `apps/web/components/improve-kandev-dialog-create.tsx` +- `apps/web/lib/api/domains/improve-kandev-api.ts` +- `scripts/kandev-logs` + +## Dependencies + +- Task 07 supplies the shared bundle service and manifest metadata. +- Task 08 replaces the Logs page and supplies browser capture. +- Task 10 establishes the replacement authenticated agent workflow before the + debug export disappears. + +## Parallelism + +Sequential. It removes compatibility paths only after all consumers have +migrated. + +## Inputs + +- Spec: Agent diagnostics, legacy route removal, Improve Kandev reuse, and + out-of-scope viewer/query behavior. +- Plan: Legacy-path removal and Improve Kandev migration. +- Existing patterns: logger buffer core, debug export, System Logs + service/hooks/store, and Improve Kandev temp bundle. + +## Risks + +- Removal spans Go and TypeScript consumers; source scans must distinguish + unrelated ring buffers used by agents, terminals, or plugins. +- The new `internal/system/frontenderrors` package is explicitly retained while + the legacy `internal/system/logs` list/tail/download package is removed. +- Improve Kandev agents need a readable ZIP path and updated instructions to + extract it before diagnosis. +- The 24-hour Improve Kandev task-context lease must not weaken the public + bundle API's 15-minute expiry or leak files beyond its existing cleanup + window. + +## Output contract + +Report removed APIs/buffer consumers, Improve Kandev migration and expiry +behavior, files changed, exact tests/source scans, blockers/risks, and update +this task plus `plan.md` status in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-10-agent-diagnostic-materialization.md b/docs/plans/diagnostic-logging/task-10-agent-diagnostic-materialization.md new file mode 100644 index 0000000000..12d4667468 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-10-agent-diagnostic-materialization.md @@ -0,0 +1,107 @@ +--- +id: "10-agent-diagnostic-materialization" +title: "Agent diagnostic materialization" +status: completed +wave: 5 +depends_on: + - "07-diagnostic-bundle-backend" + - "08-browser-logs-and-bundle-ui" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 10: Agent diagnostic materialization + +## Acceptance + +- Task-mode MCP registers `get_diagnostic_bundle_kandev` with one required + source enum (`backend`, `frontend`, or `all`) and no task, session, user, + path, URL, or credential arguments. +- The MCP server supplies its immutable task/session IDs. Existing MCP identity + scoping resolves the task owner before the handler creates the bundle, and + task/session-pair authorization happens before bundle or runtime access. +- The handler waits for ready/partial within the five-minute job lifetime and + streams the ZIP through agentctl to a server-selected owner-only + `.kandev/diagnostics/.zip` path in that execution workspace. +- Transfer is streaming and capped at 256 MiB; it does not marshal the ZIP into + JSON/base64 or buffer it wholly in backend, MCP, WebSocket, or agentctl + memory. Remote and local executors use the same bounded contract. +- The MCP result contains the executor-local path plus manifest completeness, + source, byte-range, and loss summary. Backend-only never requests frontend + capture. +- Materialized artifacts are removed on session/task workspace cleanup or + after 24 hours, whichever comes first, and never appear in git status. +- `scripts/kandev-logs` supports host-side jobs. With auth enabled it reads a + PAT only from `KANDEV_API_TOKEN`; it never accepts or prints raw credentials. + +## Verification + +```bash +cd apps/backend +go test ./internal/mcp/server ./internal/mcp/handlers \ + ./internal/agent/runtime/lifecycle ./internal/agent/runtime/agentctl \ + ./internal/agentctl/server/api +``` + +```bash +cd apps/backend +go test -race ./internal/mcp/handlers ./internal/agent/runtime/lifecycle +``` + +```bash +scripts/kandev-logs --help +``` + +## Files likely touched + +- `apps/backend/internal/mcp/server/server.go` +- `apps/backend/internal/mcp/server/server_test.go` +- `apps/backend/internal/mcp/handlers/handlers.go` +- `apps/backend/internal/mcp/handlers/diagnostic_bundle.go` +- `apps/backend/internal/mcp/handlers/diagnostic_bundle_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_execution.go` +- `apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go` +- `apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer_test.go` +- `apps/backend/internal/agent/runtime/agentctl/client_diagnostics.go` +- `apps/backend/internal/agentctl/server/api/workspace_diagnostics.go` +- `apps/backend/internal/agentctl/server/api/workspace_diagnostics_test.go` +- `apps/backend/internal/system/logbundle/service.go` +- `scripts/kandev-logs` +- focused shell tests for `scripts/kandev-logs` + +## Dependencies + +- Task 07 supplies identity-owned source-selectable jobs and archives. +- Task 08 supplies frontend capture for frontend/all requests. + +## Parallelism + +Sequential after Tasks 07 and 08. This crosses MCP, runtime, agentctl, and the +bundle service and must settle before legacy debug export is removed. + +## Inputs + +- Spec: Agent diagnostics, Permissions, performance limits, and agent scenarios. +- Plan: Agent diagnostic materialization. +- Existing patterns: task-mode MCP registration, in-session identity scoping, + task/session-pair authorization, lifecycle execution lookup, and agentctl + workspace file services. + +## Risks + +- Never accept an agent-selected output path or infer identity from tool input. +- The task/session pair must be authorized before resolving its execution; a + caller cannot combine its own session with another task. +- Streaming must propagate cancellation and close both source/destination + handles without buffering a 256 MiB archive. +- A delayed materialization from an old execution cannot write into a successor + execution; bind transfer to immutable execution identity and validate it + again before the destination is committed. +- Cleanup removes only the server-owned diagnostics directory and never a + workspace root, repository, or caller-selected path. + +## Output contract + +Report tool schema, identity and task/session ownership, local/remote streaming +path, archive/cleanup bounds, files changed, exact tests/results, blockers or +risks, and update this task plus `plan.md` status in the same conversation. diff --git a/docs/plans/diagnostic-logging/task-11-merge-risk-hardening.md b/docs/plans/diagnostic-logging/task-11-merge-risk-hardening.md new file mode 100644 index 0000000000..c15800553b --- /dev/null +++ b/docs/plans/diagnostic-logging/task-11-merge-risk-hardening.md @@ -0,0 +1,84 @@ +--- +id: "11-merge-risk-hardening" +title: "Merge-risk hardening" +status: completed +wave: 8 +depends_on: + - "01-backend-log-sinks" + - "02-frontend-error-endpoint" + - "04-toast-reporting" + - "08-browser-logs-and-bundle-ui" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 11: Merge-risk hardening + +## Confirmed root causes + +- The PR head passed lint before `main` added three effective lines to + `apps/web/lib/types/backend.ts`; the merge result now exceeds the enforced + 600-line limit. +- Request-count limiting and age-only file retention do not bound bytes written + by authenticated automatic toast reports during one UTC day. +- Automatic toast reports retain URL query and fragment data even though task + correlation can be derived before redaction. +- IndexedDB pruning materializes and sorts every retained record after each + persistence batch, creating avoidable allocation and main-thread work. + +## Acceptance + +- The current `main` merge result passes web lint and typecheck without + suppressing the line limit. +- The automatic-toast endpoint retains its request-count buckets and also + enforces exact bounded-body byte buckets per identity and process-wide. +- A daily backend file never grows beyond 256 MiB; entries that would cross the + bound are dropped without disabling the writer, and the next UTC day accepts + writes normally. +- Automatic toast reports preserve task-ID derivation but send only URL origin + and pathname. +- IndexedDB retention uses the existing timestamp index and bounded cursor work + instead of `getAll()` plus an in-memory sort. No IndexedDB version/schema + migration is introduced. +- No rendered UI, navigation, touch, scrolling, or responsive behavior changes; + existing desktop/mobile bundle E2E coverage remains applicable. + +## TDD and verification + +1. Add failing Go coverage for byte-weighted endpoint rejection and the daily + file bound. +2. Add failing Vitest coverage for URL redaction and cursor-based retention. +3. Implement the minimum fixes, then run: + +```bash +cd apps/backend +go test ./internal/common/logger ./internal/system/frontenderrors +``` + +```bash +cd apps +pnpm --filter @kandev/web test -- \ + lib/api/domains/frontend-error-log-api.test.ts \ + lib/logger/indexeddb-store.test.ts +pnpm --filter @kandev/web typecheck +pnpm --filter @kandev/web lint +``` + +## Risks and out of scope + +- This task does not change the accepted authenticated install-wide backend + bundle access boundary or restore removed legacy log APIs/configuration. +- The browser persistence change must use the existing IndexedDB schema. +- Diagnostic pressure remains best effort; loss must not block application + work. + +## Verification results + +- `go test -race ./internal/common/logger ./internal/system/frontenderrors`: + 32 tests passed. +- Focused Vitest: 10 tests passed across the frontend-error client and + IndexedDB store. +- Web typecheck and lint: passed. +- CI-style golangci-lint against `origin/main`: passed. +- Public docs: 58 validator tests and all 41 published pages passed. +- `git diff --check`: passed. diff --git a/docs/plans/diagnostic-logging/task-12-custom-bundle-contracts.md b/docs/plans/diagnostic-logging/task-12-custom-bundle-contracts.md new file mode 100644 index 0000000000..eb3822bbcb --- /dev/null +++ b/docs/plans/diagnostic-logging/task-12-custom-bundle-contracts.md @@ -0,0 +1,91 @@ +--- +id: "12-custom-bundle-contracts" +title: "Custom bundle contracts and runtime index" +status: done +wave: 9 +depends_on: + - "07-diagnostic-bundle-backend" + - "11-merge-risk-hardening" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 12: Custom bundle contracts and runtime index + +## Acceptance + +- Bundle jobs accept unique `backend`, `frontend`, `runtime`, and `acp` + sources; ACP requires one to ten unique session IDs, and coalescing includes + the normalized session set. +- Capabilities and ACP-candidate endpoints use backend-authoritative debug + state and existing task authorization: owners see their sessions, admins may + see all, and foreign IDs are rejected without existence disclosure. +- `runtime/sessions.json` contains only the approved bounded session/runtime + DTO fields, newest-first, with no titles, identities, arbitrary metadata, + messages, prompts, tool payloads, files, or configuration. + +## Verification + +```bash +cd apps/backend +go test ./internal/system/logbundle ./internal/system ./internal/backendapp +go test -race ./internal/system/logbundle ./internal/system +``` + +## Files likely touched + +- `apps/backend/internal/system/logbundle/job.go` +- `apps/backend/internal/system/logbundle/service.go` +- `apps/backend/internal/system/logbundle/archive.go` +- `apps/backend/internal/system/logbundle/handler.go` +- `apps/backend/internal/system/logbundle/runtime_index.go` +- `apps/backend/internal/system/logbundle/*_test.go` +- `apps/backend/internal/system/system.go` +- `apps/backend/internal/backendapp/helpers.go` +- `apps/backend/internal/backendapp/helpers_test.go` + +## Dependencies + +- Task 07 provides the current owner-scoped job/archive service. +- Task 11 fixes the current resource ceilings this extension must preserve. + +## Parallelism + +Sequential. Task 13 depends on these source, authorization, job-key, and +manifest contracts and touches the same logbundle files. + +## Inputs + +- Spec: System Logs page and bundle contents, Diagnostic bundle jobs, + Permissions, Failure modes, and runtime-index scenarios. +- Plan: Backend source and permission contracts. +- ADR: raw ACP is separate explicit consent; runtime metadata is field + allow-listed rather than a database export. + +## Risks + +- Generic model/metadata serialization could silently add message-bearing or + identity fields later; the DTO must be explicit and pinned by negative tests. +- Authorization must complete before filesystem/executor access and must not + reveal whether a foreign session exists. +- No backend database migration is required or permitted for this extension. + +## Output contract + +Report request/response shapes, authorization behavior, runtime DTO fields, +source/job-key changes, exact tests/results, blockers/risks, and synchronize +this task plus `plan.md` status. + +## Results + +Implemented source/session contracts and runtime-index plumbing. + +- Added backend-authoritative `backend`, `frontend`, `runtime`, and `acp` + source normalization with deterministic selected-session coalescing keys. +- Added capability and ACP-candidate endpoints with debug gating and bounded + allow-listed session records. +- Added the task-service adapter with owner/admin authorization before session + metadata access and no generic task/session serialization. +- Added `runtime/sessions.json` archive output and manifest counts. +- Verification: `go test ./internal/system/logbundle ./internal/system ./internal/backendapp -count=1` (pass; 251 tests). +- Verification: `go test ./internal/backendapp -run TestDiagnosticSessionProvider -count=1` (pass). diff --git a/docs/plans/diagnostic-logging/task-13-on-demand-acp-collection.md b/docs/plans/diagnostic-logging/task-13-on-demand-acp-collection.md new file mode 100644 index 0000000000..42f35df6ae --- /dev/null +++ b/docs/plans/diagnostic-logging/task-13-on-demand-acp-collection.md @@ -0,0 +1,103 @@ +--- +id: "13-on-demand-acp-collection" +title: "On-demand ACP collection" +status: done +wave: 10 +depends_on: + - "12-custom-bundle-contracts" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 13: On-demand ACP collection + +## Acceptance + +- Debug-enabled agentctl exports only recognized retained raw and normalized + files for the exact requested session under a server-clamped byte limit; + debug-disabled, unknown, expired, and malicious requests fail closed. +- The backend collects explicitly selected host or reachable executor ACP + evidence only after owner/admin authorization, with at most ten sessions, + two concurrent executor reads, and a 30-second total deadline. +- ACP-inclusive archives use server-numbered paths, enforce the 96/48/96/2 MiB + source budgets and existing global/temp caps, and become partial with exact + manifest warnings for unavailable, invalid, or truncated sessions. + +## Verification + +```bash +cd apps/backend +go test ./internal/agentctl/server/api \ + ./internal/agent/runtime/agentctl \ + ./internal/agent/runtime/lifecycle \ + ./internal/system/logbundle +go test -race ./internal/agentctl/server/api \ + ./internal/agent/runtime/agentctl \ + ./internal/system/logbundle +``` + +## Files likely touched + +- `apps/backend/internal/agentctl/server/api/server.go` +- `apps/backend/internal/agentctl/server/api/acp_debug_export.go` +- `apps/backend/internal/agentctl/server/api/acp_debug_export_test.go` +- `apps/backend/internal/agentctl/server/adapter/transport/shared/acplog.go` +- `apps/backend/internal/agent/runtime/agentctl/client.go` +- `apps/backend/internal/agent/runtime/agentctl/client_files.go` +- `apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go` +- `apps/backend/internal/system/logbundle/acp_collector.go` +- `apps/backend/internal/system/logbundle/archive.go` +- `apps/backend/internal/system/logbundle/service.go` +- `apps/backend/internal/system/logbundle/*_test.go` +- `apps/backend/internal/backendapp/helpers.go` + +## Dependencies + +- Task 12 defines accepted sources, selected-session job identity, capability + gating, authorization, and manifest DTOs. + +## Parallelism + +Sequential. It crosses agentctl, lifecycle/runtime, and the shared logbundle +archive/job state established by Task 12. + +## Inputs + +- Spec: performance/resource contract, ACP internal export API, Permissions, + Failure modes, Persistence guarantees, and ACP scenarios. +- Plan: On-demand ACP export. +- Agentctl guidance: recognized debug filenames, writer rotation/retention, + and debug-only privacy contract. + +## Risks + +- Raw frames may contain secrets and full work content; never broaden a + selected session to a directory scan or reuse client-provided ZIP paths. +- Docker/remote executors may disappear while collecting. Cancellation must + close bodies/temp files without stalling the agent or product request path. +- Do not add continuous ACP forwarding, a permanent backend ACP store, or an + unbounded archive/decompression path. + +## Output contract + +Report exporter/collector contracts, executor coverage, concurrency/deadline +bounds, archive validation, partial semantics, exact tests/results, +blockers/risks, and synchronize this task plus `plan.md` status. + +## Results + +Implemented on-demand ACP export and collection across agentctl, lifecycle, +backendapp, and the diagnostic archive service. + +- Added a debug/dev-gated agentctl export route that returns only exact-session + raw and normalized files, with server-owned ZIP paths, regular-file checks, + byte/file limits, and rotated-file matching. +- Added authenticated runtime-client and lifecycle seams plus the backendapp + exporter/provider adapters. Session authorization happens before executor or + host-file access; explicit selections are limited to ten and can include + host-retained sessions while unavailable sessions are rejected. +- Added bounded, 30-second ACP collection with 96 MiB ACP and 96 MiB backend + budgets. Remote ZIPs are fully validated before any archive entry is written; + invalid, unavailable, and truncated evidence produces partial warnings. +- Verification: `go test ./internal/agentctl/server/api ./internal/agent/runtime/agentctl ./internal/agent/runtime/lifecycle ./internal/system/logbundle ./internal/backendapp -count=1` (pass; 1,468 tests). +- Verification: `go test -race ./internal/agentctl/server/api ./internal/agent/runtime/agentctl ./internal/agent/runtime/lifecycle ./internal/system/logbundle -count=1` (pass; 1,242 tests). diff --git a/docs/plans/diagnostic-logging/task-14-bundle-customizer-ui.md b/docs/plans/diagnostic-logging/task-14-bundle-customizer-ui.md new file mode 100644 index 0000000000..0586f4c89a --- /dev/null +++ b/docs/plans/diagnostic-logging/task-14-bundle-customizer-ui.md @@ -0,0 +1,100 @@ +--- +id: "14-bundle-customizer-ui" +title: "Bundle customizer UI" +status: done +wave: 11 +depends_on: + - "12-custom-bundle-contracts" + - "13-on-demand-acp-collection" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 14: Bundle customizer UI + +## Acceptance + +- Logs shows translated standard-source boundaries and frontend/backend event + classes, plus **Download standard bundle** and **Customize bundle**; ACP + controls derive only from backend capabilities. +- Debug-capable users also see **Download with ACP…**. ACP submission requires + one to ten eligible selected sessions and replaces the standard reassurance + with the exact raw-protocol content/secrets warning. With no eligible session, + the action remains visible and opens a non-submittable explanatory empty state. +- Desktop uses a Dialog and phone an inset Drawer with shared source/session, + validation, polling, partial/error, and download logic; the phone surface has + one scroll owner, safe-area actions, ≥44px targets, and no horizontal overflow. + +## Verification + +```bash +cd apps +pnpm --filter @kandev/web test -- \ + lib/api/domains/system-api.test.ts \ + components/settings/system/log-viewer.test.tsx +``` + +```bash +cd apps/web +pnpm run i18n:ratchet +pnpm run typecheck +``` + +## Files likely touched + +- `apps/web/lib/types/system.ts` +- `apps/web/lib/api/domains/system-api.ts` +- `apps/web/lib/api/domains/system-api.test.ts` +- `apps/web/components/settings/system/log-viewer.tsx` +- `apps/web/components/settings/system/log-viewer.test.tsx` +- `apps/web/components/settings/system/bundle-customizer.tsx` +- `apps/web/hooks/use-responsive-breakpoint.ts` +- `apps/web/src/locales/en/settings.json` +- `apps/web/src/locales/pseudo/settings.json` + +## Dependencies + +- Task 12 provides capabilities, candidates, runtime source, and request DTOs. +- Task 13 provides actual ACP availability/partial behavior. + +## Parallelism + +Sequential after Tasks 12 and 13. The component and API types must land +together so debug capability cannot be inferred client-side. + +## Inputs + +- Spec: System Logs page and bundle contents plus desktop/mobile scenarios. +- Plan: Logs page customization and mobile design contract. +- Mobile reference: `mobile-menu-sheet` and `mobile-picker-sheet` Drawer + composition; existing System Logs card is the route/card baseline. + +## Risks + +- Standard no-transcript copy must explicitly exclude ACP-inclusive bundles and + must not promise that incidental text already emitted into logs is redacted. +- Dialog/Drawer wrappers must not duplicate request state or let source/session + validation drift across viewports. +- This task adds no persisted preference and no browser/database migration. + +## Output contract + +Report desktop/mobile interaction, translated disclosure, source/session +defaults and validation, files changed, exact tests/results, rendered mobile +evidence, blockers/risks, and synchronize this task plus `plan.md` status. + +## Results + +Implemented the translated standard/custom/ACP bundle workflow. + +- Added backend-driven capability and session API clients/types, with standard + frontend/backend defaults, optional runtime index, and explicit one-to-ten + ACP session selection. +- Added a shared desktop Dialog / phone inset Drawer customizer with a single + mobile scroll owner, safe-area footer, 44px targets, no horizontal overflow, + empty/validation states, and the ACP content/secrets disclosure. +- Updated Logs copy to explain backend/frontend event classes, incidental text, + and that standard bundles exclude stored chat, session, and agent messages. +- Verification: focused system API/Logs UI tests pass (28 tests); `pnpm run + i18n:ratchet`, `pnpm run i18n:check`, `pnpm run typecheck`, and `pnpm run + lint` all pass. diff --git a/docs/plans/diagnostic-logging/task-15-custom-bundle-e2e.md b/docs/plans/diagnostic-logging/task-15-custom-bundle-e2e.md new file mode 100644 index 0000000000..fc5e6626d2 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-15-custom-bundle-e2e.md @@ -0,0 +1,83 @@ +--- +id: "15-custom-bundle-e2e" +title: "Custom bundle E2E" +status: done +wave: 12 +depends_on: + - "14-bundle-customizer-ui" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 15: Custom bundle E2E + +## Acceptance + +- Desktop E2E proves standard/custom/debug-capability states, required ACP + session selection, owner-visible candidates, strong disclosure, runtime + index fields, raw+normalized ZIP entries, and partial unavailable sessions. +- Mobile E2E completes the same source/session outcome through the inset drawer + and proves internal scrolling, viewport containment, focus/dismiss behavior, + safe-area clearance, ≥44px targets, and no document horizontal overflow. +- Tests rebuild production backend/web artifacts and inspect the downloaded ZIP + rather than asserting only button visibility. + +## Verification + +```bash +cd apps/web +pnpm e2e:run e2e/tests/system/logs-page.spec.ts \ + e2e/tests/system/mobile-logs-bundle.spec.ts +``` + +## Files likely touched + +- `apps/web/e2e/tests/system/logs-page.spec.ts` +- `apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts` +- `apps/web/e2e/fixtures/backend.ts` +- `apps/web/e2e/helpers/api-client.ts` +- `apps/backend/internal/debug/fixture.go` + +## Dependencies + +- Task 14 completes the end-to-end backend/frontend behavior and selectors. + +## Parallelism + +Parallel-safe with Task 16 after Task 14. This task owns E2E fixtures/specs; +Task 16 owns public docs and agent skill guidance. + +## Inputs + +- Spec scenarios for standard disclosure, debug gating, authorization, + selected/unavailable ACP sessions, runtime index, and phone Drawer behavior. +- Plan E2E section and mobile design contract. +- Existing Logs desktop/mobile specs and production-build E2E fixtures. + +## Risks + +- Fixture ACP content must be synthetic and must not read a developer's real + debug directory, database, credentials, sessions, or executor data. +- ZIP assertions must verify absence of message-bearing sources in standard and + runtime-only bundles as well as presence in the explicit ACP fixture. + +## Output contract + +Report seeded states, downloaded artifact assertions, desktop/mobile commands +and results, screenshots/traces on failure, cleanup evidence, blockers/risks, +and synchronize this task plus `plan.md` status. + +## Results + +Added desktop and mobile coverage for the customizer and standard bundle +workflow. The desktop spec parses the downloaded stored ZIP, verifies +`manifest.json`, backend/frontend entries, and the requested source set, then +checks the source customizer, runtime-index option, and debug-only ACP button +boundary. The mobile spec verifies the inset Drawer, internal source surface, +44px actions, and document-width containment. + +- Verification: `pnpm e2e:run e2e/tests/system/logs-page.spec.ts` (pass; 2 + Chromium tests with production backend/web builds). +- Verification: `pnpm e2e:run --no-build --project mobile-chrome + e2e/tests/system/mobile-logs-bundle.spec.ts` (pass; 2 mobile tests after the + production build). diff --git a/docs/plans/diagnostic-logging/task-16-diagnostic-privacy-docs.md b/docs/plans/diagnostic-logging/task-16-diagnostic-privacy-docs.md new file mode 100644 index 0000000000..39c8543363 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-16-diagnostic-privacy-docs.md @@ -0,0 +1,88 @@ +--- +id: "16-diagnostic-privacy-docs" +title: "Diagnostic privacy documentation" +status: done +wave: 12 +depends_on: + - "14-bundle-customizer-ui" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 16: Diagnostic privacy documentation + +## Acceptance + +- Public docs distinguish standard frontend/backend event evidence from stored + message tables and truthfully warn that incidental emitted log text remains. +- Debug docs explain raw+normalized ACP contents, owner/admin session selection, + debug capability gating, on-demand host/executor collection, fixed limits, + partial unavailable sessions, runtime-index fields, and no continuous ACP + centralization. +- Debug skills inspect standard sources first, treat ACP as explicit + human-provided sensitive evidence, inspect `manifest.json`, and exact-grep + task/session IDs without claiming ACP is message-free. + +## Verification + +```bash +rg -n "ACP|agent messages|session messages|runtime index|diagnostic bundle" \ + docs/public .agents/skills/debug +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +make lint-harness +``` + +## Files likely touched + +- `docs/public/cli.md` +- `docs/public/configuration.md` +- `docs/public/docker.md` +- `docs/public/k8s.md` +- `docs/public/operations.md` +- `docs/public/remote-cloud-environment.md` +- `.agents/skills/debug/SKILL.md` +- `.agents/skills/debug/references/browser.md` +- `.agents/skills/debug/references/instance.md` + +## Dependencies + +- Task 14 settles shipped labels, capability behavior, limits, and partial UI + wording before public and agent guidance is finalized. + +## Parallelism + +Parallel-safe with Task 15 after Task 14. It owns docs/skill files only. + +## Inputs + +- Spec complete privacy, source, permission, failure, persistence, and scenario + contracts. +- ADR consent/on-demand decisions and rejected alternatives. +- Docs-maintainer and debug-skill conventions. + +## Risks + +- “Does not read stored messages” must not be shortened into “contains no + messages”; frontend/backend logs can carry incidental emitted content, and + ACP deliberately carries message/tool/file payloads. +- Existing remote-cloud wording currently says to exclude ACP from support + bundles; update it to distinguish standard bundles from explicit selected + ACP debug bundles rather than deleting the privacy warning. + +## Output contract + +Report docs/skill changes, stale claims removed, exact validators/results, +blockers/risks, and synchronize this task plus `plan.md` status. + +## Results + +Updated public Logs guidance, the remote-cloud ACP warning, and debug skill +references with source boundaries, event classes, ACP sensitivity, fixed +budgets, partial behavior, manifest-first inspection, and exact task/session +grep commands. Standard evidence is explicitly distinguished from stored +messages and from opt-in ACP protocol content. + +- Verification: `node --test scripts/validate-public-docs.test.mjs` (58 passed). +- Verification: `node scripts/validate-public-docs.mjs` (41 docs validated). +- Verification: `make lint-harness` (112 harness files passed). diff --git a/docs/plans/diagnostic-logging/task-17-bundle-customizer-refinement.md b/docs/plans/diagnostic-logging/task-17-bundle-customizer-refinement.md new file mode 100644 index 0000000000..df0d8f7535 --- /dev/null +++ b/docs/plans/diagnostic-logging/task-17-bundle-customizer-refinement.md @@ -0,0 +1,105 @@ +--- +id: "17-bundle-customizer-refinement" +title: "Bundle customizer refinement" +status: done +wave: 13 +depends_on: + - "12-custom-bundle-contracts" + - "14-bundle-customizer-ui" + - "15-custom-bundle-e2e" +plan: "plan.md" +spec: "../../specs/platform/diagnostic-logging.md" +--- + +# Task 17: Bundle customizer refinement + +## Acceptance + +- System Logs exposes one **Customize bundle** entry point. Its default is the + backend/frontend bundle; runtime and debug-only ACP evidence are optional + sources in the same customizer. +- The wider desktop Dialog and phone Drawer share selection state. Selecting + ACP lazy-loads candidates, shows the high-sensitivity warning, and provides + select-all/clear controls that never exceed the backend session limit. +- Each authorized ACP candidate shows its task title as a safe new-tab link. + The candidate API carries that title intentionally, while the runtime index, + manifest, and archive continue to exclude it. + +## Verification + +```bash +cd apps/backend +go test ./internal/system/logbundle ./internal/backendapp +``` + +```bash +cd apps +pnpm --filter @kandev/web test -- --run components/settings/system/log-viewer.test.tsx +``` + +```bash +cd apps/web +pnpm e2e:run e2e/tests/system/logs-page.spec.ts +pnpm e2e:run --no-build --project mobile-chrome e2e/tests/system/mobile-logs-bundle.spec.ts +``` + +## Files likely touched + +- `apps/backend/internal/system/logbundle/contracts.go` +- `apps/backend/internal/system/logbundle/handler.go` +- `apps/backend/internal/system/logbundle/handler_test.go` +- `apps/backend/internal/backendapp/diagnostic_session_provider.go` +- `apps/web/lib/types/system.ts` +- `apps/web/components/settings/system/log-viewer.tsx` +- `apps/web/components/settings/system/bundle-customizer.tsx` +- `apps/web/components/settings/system/log-viewer.test.tsx` +- `apps/web/e2e/tests/system/logs-page.spec.ts` +- `apps/web/e2e/tests/system/mobile-logs-bundle.spec.ts` +- `apps/web/src/locales/en/settings.json` +- `apps/web/src/locales/pseudo/settings.json` +- `docs/specs/platform/diagnostic-logging.md` +- `docs/specs/system-page/spec.md` +- `docs/public/operations.md` +- `docs/decisions/2026-07-30-file-backed-diagnostic-bundles.md` + +## Mobile design contract + +- The existing System Logs route remains the mobile entry point. Its one + Customize action opens the existing inset Drawer rather than a compressed + desktop dialog. +- The source/session body remains the only internal scroll owner; the + safe-area-aware footer keeps Cancel/Create reachable. Select-all and clear + controls are visible touch targets rather than hover-only affordances. +- The same source/session state drives desktop and mobile. Mobile coverage + proves the Drawer, ACP source choice, selection controls, and no horizontal + overflow. + +## Risks + +- Task titles are user-visible task metadata, but must be returned only after + existing task authorization and must never leak into bundle files. +- The picker can have more eligible sessions than its backend maximum. Bulk + selection must stay deterministic and bounded. +- The wider desktop Dialog must remain viewport-contained; the phone Drawer + must retain its current safe-area and scroll ownership behavior. + +## Results + +- Consolidated the Logs page to one **Customize bundle** action with + backend/frontend selected by default and runtime/ACP left optional. +- Added lazy ACP candidate loading, bounded select-all/clear controls, and an + authorized task-title link that opens each candidate's Kandev task in a new + tab. Task titles are API-picker metadata only and remain excluded from the + runtime index, manifest, and archive. +- Widened the desktop customizer without imposing empty fixed-height space; the + phone flow remains an inset Drawer with one scroll owner and touch-sized + controls. +- Refined the Logs-page hierarchy with matched card padding, vertically ordered + privacy/source notes, and a desktop action aligned under the bundle + description. The phone page retains the same single-column flow and Drawer + entry point. +- Updated the public operations guide and diagnostic-bundle decision to match + the consolidated flow. +- Verified with backend package tests (252), focused web tests (28), production + web build, desktop E2E (3), mobile E2E (2), web typecheck, lint, and i18n + checks. diff --git a/docs/public/cli.md b/docs/public/cli.md index fd378e4bd9..3e5be029f8 100644 --- a/docs/public/cli.md +++ b/docs/public/cli.md @@ -97,7 +97,7 @@ kandev service [service options] | `--port <1-65535>` | Request an exact backend port. `--backend-port` is an alias; `--port=` forms also work. | | `--headless`, `--no-browser` | Do not open a browser. | | `--verbose`, `-v` | Show backend info output. | -| `--debug` | Show debug output and enable diagnostic endpoints and ACP frame logs. See the security warning below. | +| `--debug` | Record debug output in the backend file and enable diagnostic endpoints and ACP frame logs; stdout remains concise. See the security warning below. | | `--version`, `-V` | Print the native runtime version. | | `--help`, `-h`, `help` | Print help. | @@ -193,7 +193,7 @@ Flags take precedence over the equivalent port variables. `KANDEV_BACKEND_PORT` The launcher also sets the selected server and `agentctl` ports for the backend. Treat its supervisor socket and manifest under `/supervisor/` as private implementation state, not a control API. -`--debug` sets `KANDEV_DEBUG_AGENT_MESSAGES=true` and `KANDEV_DEBUG_PPROF_ENABLED=true`. ACP logs can contain full prompts, file content, and tool calls, while diagnostic endpoints expose process details. Use debug mode only on a trusted machine and remove retained debug logs afterward. [Configuration](./configuration.md) lists their location and retention controls. +`--debug` sets `KANDEV_DEBUG_AGENT_MESSAGES=true` and `KANDEV_DEBUG_PPROF_ENABLED=true`. Debug events go to `/logs/backend-logs.log`; warn and above still appear on stdout. ACP logs can contain full prompts, file content, and tool calls, while diagnostic endpoints expose process details. Use debug mode only on a trusted machine and remove retained debug logs afterward. [Configuration](./configuration.md) lists locations and retention. ## Data and cleanup diff --git a/docs/public/configuration.md b/docs/public/configuration.md index 3dcce2f977..063c6a9ec7 100644 --- a/docs/public/configuration.md +++ b/docs/public/configuration.md @@ -38,7 +38,6 @@ Viper maps a nested YAML key by replacing `.` with `_`, adding `KANDEV_`, and up ```text database.dbName -> KANDEV_DATABASE_DBNAME -logging.maxSizeMb -> KANDEV_LOGGING_MAXSIZEMB repoClone.basePath -> KANDEV_REPOCLONE_BASEPATH ``` @@ -138,15 +137,12 @@ The voice fallback sends audio to the configured OpenAI transcription service an | YAML key | Environment variable | Default | Current behavior | |---|---|---|---| -| `logging.level` | `KANDEV_LOG_LEVEL` | `info` | `debug`, `info`, `warn`, or `error`. The CLI normally supplies `warn`, `info`, or `debug`. | +| `logging.level` | `KANDEV_LOG_LEVEL` | `info` | File threshold: `debug`, `info`, `warn`, or `error`. `--debug` selects `debug`; normal and `--verbose` launches select `info`. | | `logging.format` | `KANDEV_LOGGING_FORMAT` | `text`, or `json` in production/Kubernetes | `text` or `json`; `auto` is not accepted. | -| `logging.outputPath` | `KANDEV_LOGGING_OUTPUTPATH` | `stdout` | `stdout`, `stderr`, or a file path. | -| `logging.maxSizeMb` | `KANDEV_LOGGING_MAXSIZEMB` | `100` | File-output rotation threshold. Zero uses lumberjack's 100 MiB default. | -| `logging.maxBackups` | `KANDEV_LOGGING_MAXBACKUPS` | `5` | Rotated-file count; zero means unlimited. | -| `logging.maxAgeDays` | `KANDEV_LOGGING_MAXAGEDAYS` | `30` | Rotated-file age; zero means unlimited. | -| `logging.compress` | `KANDEV_LOGGING_COMPRESS` | `true` | Gzip rotated files. | -The format default becomes JSON when `KUBERNETES_SERVICE_HOST` is non-empty or `KANDEV_ENV` is exactly `production`/`prod`; otherwise it is text. Rotation settings apply only to file output. Active log files are created owner-only (`0600`) on Unix, so a log shipper running as another user cannot read them without an explicit permission design. +Every backend launch writes to `/logs/backend-logs.log` and prints that resolved path at startup. The active file appends across same-day restarts and accepts at most 256 MiB; later entries are dropped until the next UTC day rather than allowing diagnostics to fill the disk. At the next UTC day it rolls to `backend-logs-YYYY-MM-DD.log`; Kandev retains the current UTC day and the two preceding days. Files are owner-only (`0600`) on Unix. + +Normal launches write info and above to the file and warn and above to stdout. `--debug` writes debug and above to the file while stdout remains warn and above. `--verbose` writes info and above to both. The format default becomes JSON when `KUBERNETES_SERVICE_HOST` is non-empty or `KANDEV_ENV` is exactly `production`/`prod`; otherwise it is text. Debug output may contain repository paths, subprocess output, prompts, file content, and tool-call data. Treat it as sensitive. @@ -195,7 +191,6 @@ server: logging: format: "json" - outputPath: "/var/log/kandev/backend.log" repositoryDiscovery: roots: @@ -257,11 +252,6 @@ auth: logging: level: "info" format: "text" - outputPath: "stdout" - maxSizeMb: 100 - maxBackups: 5 - maxAgeDays: 30 - compress: true repositoryDiscovery: roots: [] # automatic scan roots; explicit paths need not be included diff --git a/docs/public/contributing.md b/docs/public/contributing.md index 16a1073ef8..526fc9b542 100644 --- a/docs/public/contributing.md +++ b/docs/public/contributing.md @@ -32,7 +32,7 @@ make bootstrap make dev ``` -This is the normal development path. The TypeScript supervisor starts the Go backend and Vite, selects available ports, points Go at Vite, and isolates application state under the checkout's `.kandev-dev/`. Use the printed URLs. +This is the normal development path. The TypeScript supervisor starts the Go backend and Vite, selects available ports, points Go at Vite, and isolates application state under the checkout's `.kandev-dev/`. Use the printed URLs. Backend logs append to `.kandev-dev/logs/backend-logs.log`; startup prints the resolved path. `make dev-web` starts only Vite on its fixed development port; it has no live API by itself. `make dev-backend` starts only the backend with the normal production-profile home unless you override it. For an intentionally isolated backend-only run: @@ -40,7 +40,7 @@ This is the normal development path. The TypeScript supervisor starts the Go bac KANDEV_HOME_DIR="$PWD/.kandev-dev" KANDEV_DEBUG_DEV_MODE=true make dev-backend ``` -Use `make build` for a production build. Use `make start` for a production-shaped local start; it installs dependencies, builds and synchronizes the embedded web application, then launches Kandev. +Use `make build` for a production build. Use `make start` for a production-shaped local start; it installs dependencies, builds and synchronizes the embedded web application, then launches Kandev and writes `/logs/backend-logs.log`. ## Find the owner diff --git a/docs/public/coverage.json b/docs/public/coverage.json index 0abf5444a2..207437fb27 100644 --- a/docs/public/coverage.json +++ b/docs/public/coverage.json @@ -271,12 +271,16 @@ "sources": [ "apps/backend/internal/system/settings/store.go", "apps/backend/internal/system/storage/handler.go", + "apps/backend/internal/system/logbundle/service.go", + "apps/backend/internal/mcp/handlers/diagnostic_bundle.go", "apps/web/components/settings/system/feature-toggles-settings.tsx", "apps/web/components/settings/system/storage/storage-maintenance-settings.tsx" ], "tests": [ "apps/backend/internal/system/settings/store_test.go", "apps/backend/internal/system/storage/handler_test.go", + "apps/backend/internal/system/logbundle/service_test.go", + "apps/backend/internal/mcp/handlers/diagnostic_bundle_test.go", "apps/web/components/settings/system/feature-toggles-settings.test.tsx", "apps/web/components/settings/system/storage/storage-maintenance-settings.test.tsx" ], @@ -290,7 +294,8 @@ "/settings/system/status", "/settings/system/storage", "/settings/system/updates" - ] + ], + "mcpTools": ["get_diagnostic_bundle_kandev"] }, { "id": "deployment-and-platforms", diff --git a/docs/public/docker.md b/docs/public/docker.md index 6fd6d9ccd2..2752a2c947 100644 --- a/docs/public/docker.md +++ b/docs/public/docker.md @@ -280,7 +280,7 @@ healthcheck: start_period: 15s ``` -Container logs go to stdout/stderr by default. If configuration sends logs to a file, place that path on persistent storage and account for Kandev's file rotation settings. +The backend writes its daily files to `/data/logs/backend-logs.log` on the persistent volume and prints the path at startup. Each daily file accepts at most 256 MiB, and Kandev retains the current UTC day plus the two preceding days. `docker logs` shows the bounded stdout stream; use **Settings > System > Logs** for a frontend+backend diagnostic ZIP. ## Upgrade and remove diff --git a/docs/public/k8s.md b/docs/public/k8s.md index b52bce3122..ed89541a14 100644 --- a/docs/public/k8s.md +++ b/docs/public/k8s.md @@ -140,7 +140,7 @@ See [Configuration](configuration.md) for exact YAML and `KANDEV_` names. Import | `KANDEV_LOG_LEVEL` | `info` | Backend log threshold | | `KANDEV_DATABASE_DRIVER` | `sqlite` by default | Set `postgres` for an external database | -Kubernetes detection makes the default log format JSON. Logs remain on stdout unless `logging.outputPath` is changed. +Kubernetes detection makes the default log format JSON. Kandev writes daily files under `/data/logs/`, caps each day at 256 MiB, and emits warn-and-above to stdout by default; ensure the home path is persistent if the three-day file history must survive pod replacement. ### PostgreSQL diff --git a/docs/public/operations.md b/docs/public/operations.md index 483927afb7..559d2cf3fe 100644 --- a/docs/public/operations.md +++ b/docs/public/operations.md @@ -252,18 +252,31 @@ It does not erase the entire Kandev home: backups, `master.key`, logs, service m ## Logs and diagnostics -Open **Settings > System > Logs** for recent structured backend events. Kandev maintains an in-memory ring of exactly 2,000 events that passed the configured log level; the page requests the newest 1,000. This buffer disappears on restart. +Open **Settings > System > Logs** and select **Customize bundle** to create and +download a diagnostic ZIP. Backend and frontend diagnostic evidence are +selected by default; you can add the allow-listed runtime index, and in debug +mode raw and normalized agent-protocol (ACP) evidence. Adding ACP opens an +explicit session picker, with each session's Kandev task title linked for +identification. The archive has source directories and `manifest.json`; inspect +the manifest for requested sources, warnings, captured ranges, truncation, and +loss before treating it as complete. -Default interactive launcher output is warning level. `kandev --verbose` selects info level and shows backend output; `kandev --debug` selects debug level and also enables ACP message dumps. An explicit `KANDEV_LOG_LEVEL` overrides the flag-selected level. Those dumps can contain full prompts, file contents, and tool calls. Use debug mode only on a trusted machine, collect the minimum needed, then disable and remove the files. +Standard bundles do not read stored chat transcripts, session messages, or agent messages. Backend evidence covers startup, lifecycle, API, executor, and error events; frontend evidence covers bounded browser console errors/warnings and diagnostic toast reports. Incidental text already emitted into a backend/frontend log entry is not automatically redacted, so review the ZIP before sharing. The runtime index contains only allow-listed session status and executor metadata. -Logging configuration defaults are `outputPath: stdout`, 100 MB rotation size, five rotated files, 30-day rotated-file age, and gzip compression. Rotation applies only when `logging.outputPath` is a file, and active files are created owner-readable. The Logs page can list/download the exact active filename and timestamped rotations when that filename has an extension; for an extensionless output path it does not enumerate rotated siblings. With stdout logging, use the in-memory tail plus the process manager: +ACP evidence is a separate, debug-only opt-in. It is limited to one to ten sessions the caller is authorized to view and may contain prompts, responses, tool calls, file contents, MCP data, environment-derived values, and secrets. Collection is on demand from retained host files or reachable executors; unavailable, expired, invalid, or truncated sessions make the archive partial instead of broadening the request or continuously copying ACP frames. + +Every launch writes info and above to `/logs/backend-logs.log`; normal and debug stdout show warn and above. `kandev --verbose` also shows info on stdout. `kandev --debug` records debug events in the file and enables ACP message dumps. The backend appends on same-day restarts, caps each daily file at 256 MiB, rolls at UTC midnight, and retains the current day plus the two previous UTC days. + +The browser keeps a bounded three-day console history locally and sends it only when an authenticated bundle request asks for frontend evidence. Console calls are not continuously uploaded. Error toasts are reported immediately, but automatic reports remove URL query strings and fragments and use count and byte rate limits. A bundle can be partial if no browser responds, IndexedDB is unavailable, queues shed entries, or byte/profile limits truncate data. Ready bundles expire after 15 minutes; one active job per user and bounded global capacity can return `429` or `503`. Standard bundles cap backend/frontend/runtime sources at 160/80/2 MiB; ACP-inclusive bundles use 96/48/2 MiB plus 96 MiB ACP, with no budget transfer between sources. + +Process-manager logs remain useful for live output: - Linux service: `kandev service logs -f` reads the systemd journal. - macOS service: the same command tails `/logs/service.out` and `service.err`. - Docker: `docker logs -f kandev`. - Kubernetes: `kubectl logs -f deployment/kandev`. -When reporting an incident, record timestamp/timezone, Kandev version and commit from **System > About**, task ID, session ID, executor type, repository/branch, and relevant provider request IDs. **System > Licenses** is a generated inventory of shipped npm and Go dependencies, not a runtime health check. +When reporting an incident, record timestamp/timezone, Kandev version and commit from **System > About**, task ID, session ID, executor type, repository/branch, and relevant provider request IDs. Bundles can contain install-wide backend events, full browser URLs, console arguments, and stacks; review them as sensitive data before sharing. Agents can request backend, frontend, or all sources and should search a known task ID exactly before broadening. **System > Licenses** is a generated inventory of shipped npm and Go dependencies, not a runtime health check. ## Disk use and environment cleanup @@ -337,7 +350,7 @@ drawer mirrors it as the saved left sequence followed by the saved right sequenc | Backups page reports a 15-second create timeout | Reload the backup list and inspect the `backup-create` job/log | Large `VACUUM INTO` jobs can still finish; avoid double-clicking and ensure free disk | | Backup/maintenance fails on PostgreSQL | Active driver on Database page | Use `pg_dump`, provider snapshots, and PostgreSQL maintenance; System backup/vacuum/reset is SQLite-only | | Restored data looks stale | Whether the backend was restarted immediately | Quit/restart; do not keep using the old open database connections | -| Logs page has no downloadable files | `logging.outputPath` and service/container logs | `stdout` is the default; use in-memory tail or configure a file sink | +| Diagnostic bundle is partial | `manifest.json` warnings, source status, and loss counters | Keep a Kandev browser open for frontend capture; use backend-only when browser evidence is unnecessary | | Update check returns HTTP 429 | Time since last **Check now** | Wait at least 30 seconds; background checks retry every six hours | | **Apply update** is absent | Install mode/method and `/service/install.json` | Expected for system, unmanaged, local-checkout, or invalid-metadata installs. A managed npm, npx, or Homebrew user service should offer Apply; reinstall it with the same flags to refresh its identity and metadata, or use the manual package-manager flow. | | Metrics show unavailable | OS support, disk path, executor connectivity | Select supported metrics and verify permissions/network; the collector reports errors per sample | diff --git a/docs/public/remote-cloud-environment.md b/docs/public/remote-cloud-environment.md index 5e05f683a0..8cc8b5ece3 100644 --- a/docs/public/remote-cloud-environment.md +++ b/docs/public/remote-cloud-environment.md @@ -152,7 +152,7 @@ The Azure tooling is optional and only needed for Azure Repos operations. See [G ## Development state and persistence -Normal `make dev` isolates state under `/.kandev-dev/`, including its SQLite database, logs, task workspaces, and other Kandev home data. It does not use `~/.kandev` by default. The selected development profile uses embedded SQLite and the in-memory event bus, so PostgreSQL and NATS are not prerequisites. +Normal `make dev` isolates state under `/.kandev-dev/`, including its SQLite database, task workspaces, and `.kandev-dev/logs/backend-logs.log`. Startup prints the resolved log path. It does not use `~/.kandev` by default. The selected development profile uses embedded SQLite and the in-memory event bus, so PostgreSQL and NATS are not prerequisites. When the checkout itself is a Kandev task workspace, the launcher deliberately clears a parent-provided `KANDEV_DATABASE_PATH` and uses the local `.kandev-dev` state. In a normal shell, an explicit `KANDEV_DATABASE_PATH` is honored. If that path is outside a `.kandev-dev` directory, the launcher treats it as production data, copies an existing database to `~/.kandev/data/backups/dev-prod-db-.db`, retains the newest five such snapshots, and aborts startup if the copy fails. @@ -168,7 +168,7 @@ make clean-db That target removes the entire `.kandev-dev` tree, not only the SQLite file. Any tasks, repositories, sessions, and debug logs below it are lost. Ephemeral-VM deletion likewise loses uncommitted changes and provider-local secrets unless the provider volume persists; push or export wanted work before destroying the workspace. -Development mode writes raw ACP frames under `.kandev-dev/logs/acp`. They can include full prompts, file content, and tool payloads. The built-in writer rotates and prunes them, but you should still exclude them from support bundles and delete the development home when no longer needed. +Development mode writes raw ACP frames under `.kandev-dev/logs/acp`. They can include full prompts, file content, and tool payloads. The built-in writer rotates and prunes them, but standard support bundles exclude ACP; include ACP only through the explicit debug bundle session picker after reviewing the selected sessions. Delete the development home when no longer needed. ## Build and test commands diff --git a/docs/specs/INDEX.md b/docs/specs/INDEX.md index c27ae20dde..7aff772669 100644 --- a/docs/specs/INDEX.md +++ b/docs/specs/INDEX.md @@ -44,6 +44,7 @@ Product-wide capabilities that are not tied to a single feature area. | [semantic-notifications](platform/notifications.md) | shipped | | [workspace-git-status](platform/workspace-git-status.md) | shipped | | [bounded-task-status-delivery](platform/bounded-task-status-delivery.md) | approved | +| [diagnostic-logging](platform/diagnostic-logging.md) | approved | ## tasks/ — task & workflow model diff --git a/docs/specs/platform/diagnostic-logging.md b/docs/specs/platform/diagnostic-logging.md new file mode 100644 index 0000000000..637a8a9b1b --- /dev/null +++ b/docs/specs/platform/diagnostic-logging.md @@ -0,0 +1,728 @@ +--- +status: building +created: 2026-07-30 +owner: tbd +--- + +# Diagnostic logging + +Decision: [ADR-2026-07-30-file-backed-diagnostic-bundles](../../decisions/2026-07-30-file-backed-diagnostic-bundles.md) + +Implementation plan: [diagnostic-logging](../../plans/diagnostic-logging/plan.md) + +## Why + +Users can see frontend failures that leave no backend evidence, and support +cannot rely on one known artifact containing recent backend and browser +diagnostics. The product needs a clearly disclosed bundle that users can +download and agents can inspect without continuously uploading browser console +history or returning an unbounded log export. + +## What + +### Backend files and terminal output + +- Every backend process targets + `/logs/backend-logs.log`, where `` is the resolved + `KANDEV_HOME_DIR` or its `~/.kandev` default, and writes there whenever the + path is available. +- Backend startup prints the absolute log-file path to stdout before the + backend becomes ready. +- A normal run writes `info` and higher entries to the file and `warn` and + higher entries to stdout. +- A debug run writes `debug` and higher entries to the file while stdout + remains `warn` and higher. +- An explicitly verbose run writes `info` and higher entries to both the file + and stdout. +- `logging.level` / `KANDEV_LOG_LEVEL` remains the supported override for the + file threshold, and `logging.format` remains supported. + `logging.outputPath`, `logging.maxSizeMb`, `logging.maxBackups`, + `logging.maxAgeDays`, and `logging.compress` are removed; the diagnostic + path and daily retention policy are not configurable. +- `backend-logs.log` represents the current UTC calendar day. At UTC midnight, + the backend atomically rolls it to `backend-logs-YYYY-MM-DD.log` and creates + a new owner-readable, owner-writable active file. +- A restart during the same UTC day appends to `backend-logs.log`. On the first + startup after a day boundary, the previous active file is rolled to the date + it represents before new entries are written. +- Cleanup retains a rolling three-day UTC window: the current active day and + at most the two preceding dated daily files. Older recognized daily files + are removed; unrelated files in the log directory are preserved. +- Each daily backend file accepts at most 256 MiB. Once the active file reaches + that bound, entries that would exceed it are dropped without blocking + application work and are reflected in file-sink loss statistics. UTC + rollover resumes normal writes into the next day's file. +- Rollover is crash-recoverable. A small owner-only rollover journal records + the source day, source identity/size, destination, and copied offset before + an existing dated destination is extended. Recovery resumes that transaction + exactly once and never replaces an existing dated file. +- The structured backend in-memory ring buffer is removed. The System Logs + page, Improve Kandev, and agent diagnostics use the files and bundles + described below. + +### Error-toast reporting + +- Every error toast displayed through either supported frontend toast system + is reported immediately to the backend and written as an `error` entry named + `frontend error toast`. +- Toast reporting includes the visible toast text plus rich browser context: + the browser URL origin and pathname with query and fragment removed, browser + identity fields, viewport dimensions, a client timestamp, the toast + implementation source, the current task ID when it can be derived from a + recognized task route or query parameter, a toast-emission call stack when + available, and underlying error metadata when the caller still provides it. +- Toast reporting is best effort. A reporting failure does not suppress, + replace, duplicate, or otherwise alter the original toast and does not + create a retry toast. +- Frontend report bodies and individual fields are bounded before they reach + the logger. The backend never treats a client-supplied value as a log level + or log message template. +- The endpoint has an in-memory token bucket per authenticated identity + allowing 60 reports per minute with a burst of 20, plus a process-wide + bucket allowing 300 reports per minute with a burst of 100. Exhausted + buckets return `429 Too Many Requests` with `Retry-After`; the frontend + silently discards that response. Independent byte buckets admit at most + 64 KiB per minute per identity with a 64 KiB burst and 256 KiB per minute + process-wide with a 256 KiB burst, measured from the bounded request body. + +### Browser console history + +- The frontend intercepts `console.debug`, `console.info`, `console.warn`, and + `console.error`, plus `window.error` and `unhandledrejection`, without + changing their normal browser behavior. +- Entries remain local to the browser until a diagnostic bundle explicitly + requests them. Kandev does not continuously stream console logs to the + backend. +- Browser history is stored in IndexedDB for a rolling three-day UTC window. + Each browser profile has an opaque random installation ID shared by its tabs. +- Entries are partitioned by the current authenticated Kandev identity. A + capture returns only that identity's partition; signing into another account + in the same browser profile cannot expose the previous account's entries. + Auth-disabled mode uses the existing synthetic single-user scope. +- The browser store keeps at most 10,000 entries and 20 MiB after + serialization across all identity partitions. Each entry is capped at + 64 KiB. Age is pruned first, then the oldest entries are evicted until both + bounds hold. +- Entries contain a client timestamp, level, source, message, bounded + JSON-safe arguments, available stack, full URL, and a task ID derived from a + recognized task route when available. +- Console-call staging never retains an arbitrary caller object. It preserves + primitive values and bounded `Error` fields. Other objects, functions, DOM + values, and proxies become reference-free type descriptors without walking + prototypes or invoking getters. At most 20 arguments are inspected; strings + are capped at 4 KiB, error messages at 8 KiB, and stacks at 16 KiB before the + overall 64 KiB entry cap. +- If IndexedDB is unavailable or fails, the frontend falls back to the existing + 500-entry in-memory buffer. A resulting bundle identifies the degraded + capture in its manifest. + +### Performance and resource contract + +- Diagnostic capture is best effort and is never on an application-critical + path. Emitting a backend log, displaying a toast, or invoking a browser + console method must not wait for filesystem I/O, IndexedDB I/O, a network + request, ZIP creation, or another browser tab. +- File and stdout use independent bounded asynchronous queues so a blocked + terminal cannot stall the authoritative file. The file queue accepts at most + 8,192 entries or 8 MiB and reserves 2,048 entries or 2 MiB for `warn+`. The + stdout queue accepts at most 2,048 entries or 2 MiB and reserves 512 entries + or 512 KiB for `warn+`. An encoded entry over 256 KiB is dropped. Producers + never wait for sink I/O; `debug`/`info` is shed before reserved capacity is + used, and even `warn+` is dropped rather than blocking when its queue is full. + Each sink records atomic loss counters by level and reason. +- Graceful shutdown drains both queues for at most two seconds, then records + any remaining entries as lost. `DPanic`, `Panic`, and `Fatal` are terminal + exceptions: the logger attempts the same bounded drain and writes one capped + final entry directly to stderr before the process terminates. +- Toast reporting begins after the original toast has been handed to its UI + library and is never awaited. It performs bounded context collection, uses a + two-second request deadline, and has no retry queue. "Immediately" means the + report is scheduled in the same toast-emission turn, not that rendering waits + for an HTTP round trip. +- Browser interception does only constant-time staging work on the console + call path, where constant-time means bounded by the fixed argument and field + limits above. Its reference-free staging queue holds at most 500 entries or + 2 MiB. It persists at most 50 entries or 256 KiB per transaction, scheduled + after 250 ms or during browser idle time with a one-second timeout fallback. + IndexedDB opening, serialization, pruning, and quota handling never run + synchronously in the intercepted console call. A full staging queue drops + lower-priority `debug`/`info` entries first and records loss metadata; it + never delays the original console call. +- Bundle collection, JSONL writing, cleanup, and download preparation run + outside the route handler. An identity owns at most one active job, at most + eight jobs may collect or wait process-wide, and one archive build runs + process-wide. An equivalent source request reuses the existing job. A + different request from an identity with an active job returns `429`; global + saturation returns `503`; both include `Retry-After: 5`. +- A job accepts at most four browser profiles, 20 MiB per profile and 80 MiB + total frontend data. Without ACP, backend payload remains capped at 160 MiB + and frontend payload at 80 MiB. When ACP is selected, the per-source budgets + are 96 MiB backend, 48 MiB frontend, 96 MiB ACP, and 2 MiB runtime index. + Omitted sources do not transfer their budget to another source. Backend and + ACP files are selected newest-first; when a selected file exceeds its + remaining source budget, only its newest bytes are included and the manifest + records the source byte range. Total uncompressed archive payload remains + capped at 256 MiB and a job may use at most 384 MiB of temporary disk. + Creation fails safely when that temporary-space budget is unavailable. +- ACP collection is request-driven, permits at most ten selected sessions, + fetches from at most two reachable executor-side `agentctl` instances at a + time, and has a 30-second total collection deadline. It never continuously + copies protocol frames to the backend. ACP collection, like frontend capture, + runs outside product-critical paths and may produce a partial bundle. +- Log payloads use ZIP `Store` rather than CPU-heavy compression. Copy and ZIP + writing use 1 MiB chunks and yield between chunks. Collection/build jobs have + a five-minute hard lifetime; ready/partial archives expire 15 minutes after + becoming downloadable. +- The bundle manifest records capture, persistence, queue, and archive + truncation/loss counters. Diagnostic failures and pressure must be visible in + the artifact without creating recursive log traffic. + +### System Logs page and bundle contents + +- `/settings/system/logs` no longer renders a log tail, file table, individual + file downloads, copy action, or refresh action. +- The page states that a standard bundle does not read stored chat history, + session transcripts, agent responses, prompts, tool messages, database rows, + or workspace files. This is a source boundary, not a redaction guarantee: + incidental content already emitted into a selected log may still appear. +- The frontend disclosure names `console.debug`, `console.info`, + `console.warn`, `console.error`, uncaught JavaScript errors, unhandled promise + rejections, bounded browser/runtime context, and user-visible error toasts + that are reported into backend logs. +- The backend disclosure names structured application/runtime, HTTP/service, + executor, integration, startup/shutdown, warning/error, identifier, path, + performance, and diagnostic-loss events. Backend inclusion copies emitted + file lines; it does not query product tables or construct transcripts. +- The page exposes one always-visible **Customize bundle** action. It opens + source selection with backend and frontend selected by default; the optional + runtime index and ACP evidence start unselected. Collection, preparing, + ready/partial, busy-with-retry, and failure states remain on the page without + navigating away. +- When the backend reports that raw ACP capture is enabled, ACP is an optional + source in that same customizer. Selecting it requires the user to choose one + or more eligible sessions before collection starts. ACP is never silently + included by a one-click action. The source remains available when no eligible + session exists; the picker then shows an explanatory empty state and keeps + submission disabled. +- The ACP selection surface explicitly warns that raw and normalized protocol + frames can contain full prompts, agent responses, tool arguments/results, + workspace file content, MCP payloads, environment-derived values, and + secrets. The standard no-transcript disclosure never appears to cover an + ACP-inclusive bundle. +- ACP candidate rows show their authorized Kandev task title as a link that + opens that task in a new tab. The title is available only to identify a + picker row; it is not written to the runtime index, manifest, or archive. + The picker provides explicit select-all and clear-selection controls. Select + all includes the first eligible sessions within the backend's maximum rather + than exceeding the collection limit. +- `manifest.json` is always included and is not a selectable source. The + optional runtime index contains only task/session IDs, agent/provider/model, + status, timestamps, executor type, and ACP availability. It excludes task + titles, descriptions, prompts, responses, tool payloads, file content, and + stored message bodies. +- Desktop renders source/session customization in a modal dialog. Phone uses + an inset bottom drawer patterned after the shipped mobile picker/menu + surfaces, with one internally scrolling body, a fixed disclosure/header, + safe-area-aware actions, no horizontal overflow, and at least 44px touch + targets. Both viewports share selection, validation, job, and download logic. +- The combined ZIP uses this layout: + +```text +manifest.json +backend/backend-logs.log +backend/backend-logs-YYYY-MM-DD.log +frontend/browser-01.jsonl +frontend/browser-02.jsonl +runtime/sessions.json +acp/session-01/raw-acp.jsonl +acp/session-01/normalized-acp.jsonl +``` + +- `backend/` contains the recognized active and dated files still inside the + three-day retention window. File bytes are copied without reformatting. +- `frontend/` contains one JSON Lines file per distinct responding browser + profile. Multiple tabs from one browser profile are deduplicated. Client + values never determine archive paths or filenames. +- `runtime/sessions.json` is present only when the runtime source is requested. + It contains at most 500 authorized sessions updated within the backend + three-day diagnostic window, newest first. When ACP sessions are selected, + their rows are included even if they fall outside that window. +- `acp/` is present only for explicitly selected sessions while ACP debug + capture is enabled. It contains raw and normalized retained JSONL frames + collected on demand from host-resident files or reachable executor-side + `agentctl` instances. Archive directories are server-numbered; client-supplied + session IDs never become ZIP paths. `manifest.json` maps each directory to + its authorized task/session and records unavailable or truncated files. +- `manifest.json` identifies requested and included sources, capture time, + Kandev version/commit, OS/architecture, uptime and bounded runtime metrics, + backend filenames, frontend browser/connection counts, storage mode, omitted + or truncated data, selected ACP session availability and byte ranges, runtime + index counts, capture timeouts, and archive expiry. +- Bundle archives and working files are owner-only and live in a Kandev-owned + temporary directory. User/API bundle jobs expire 15 minutes after creation. +- If frontend or ACP capture is unavailable or incomplete, a multi-source + bundle still becomes downloadable with backend files. Its state is `partial`, + and the manifest plus UI state explains each unavailable source/session. +- Any backend/ACP byte-range truncation, frontend profile omission, runtime + index truncation, queue loss, or archive-cap truncation also makes the bundle + `partial`; the manifest records exact source byte ranges and aggregate loss + counters. + +### Agent diagnostics + +- Agents use the same bundle API and request only `backend`, `frontend`, or + `all`. A backend-only request never waits for a browser. +- The task-mode MCP tool does not expose raw ACP or runtime-index selection. + ACP capture is an explicit human download flow because its stronger content + and permission contract must be reviewed before collection. +- Task-session agents normally use the task-mode + `get_diagnostic_bundle_kandev` MCP tool. It accepts only a source selection; + it derives the task, session, and authenticated owner from the MCP dispatch + context, creates the identity-owned job, waits for ready/partial, and + materializes the ZIP as an owner-only file under the execution workspace's + ignored `.kandev/diagnostics/` directory. It returns that local path and a + manifest summary; it never accepts a user ID, task ID, output path, or token. +- `scripts/kandev-logs` remains the host-side fallback. Authentication-disabled + instances need only a port. Authenticated use reads a PAT exclusively from + `KANDEV_API_TOKEN`; it never accepts or prints a token on the command line. +- Agent guidance downloads the selected ZIP, extracts it into a fresh temporary + directory, searches an exact task ID first, and broadens to session ID, + route/error text, or a bounded time window when needed. +- The dev-only `/api/v1/system/debug/export` log response is removed. Runtime + metadata moves to `manifest.json`; no replacement unbounded JSON log query is + introduced. +- Improve Kandev uses the same archive builder instead of writing snapshots + from the removed backend ring buffer. When **Include logs** is selected, the + initiating browser creates an authenticated `all` job and waits for + ready/partial before task submission. It then calls an Improve Kandev lease + endpoint with that caller-owned bundle ID and bootstrap directory; the + backend verifies common ownership, copies the ZIP to + `diagnostic-bundle.zip`, and returns the path appended to the task + description. The bootstrap route never waits for frontend collection. + The task-context copy retains the existing 24-hour cleanup window so a newly + launched agent can read it. + +## API surface + +### `POST /api/v1/system/logs/frontend-errors` + +The authenticated browser submits: + +```json +{ + "client_timestamp": "2026-07-30T12:34:56.789Z", + "source": "sonner", + "task_id": "d7e9e92e-eeb3-41e3-86d9-b6eca5760ac1", + "title": "Failed to save settings", + "description": "The backend rejected the update", + "url": "http://localhost:38429/settings/system/logs?tab=current#tail", + "user_agent": "Mozilla/5.0 ...", + "language": "en-US", + "platform": "Linux x86_64", + "viewport": { + "width": 1440, + "height": 900 + }, + "stack": "Error: error toast emitted\n at ...", + "error": { + "name": "TypeError", + "message": "Failed to fetch", + "stack": "TypeError: Failed to fetch\n at ..." + } +} +``` + +- `source` is `sonner` or `toast-provider`. +- `title` and `description` are optional individually, but at least one must + contain visible text. +- `client_timestamp` is RFC 3339 when supplied. The backend log timestamp + remains authoritative. +- `task_id` is optional and capped at 128 bytes. The frontend derives it only + from recognized `/t/:taskId`, `/tasks/:taskId`, `/office/tasks/:taskId`, or + `taskId` query-parameter routes; other pages omit it. +- `url`, `user_agent`, `language`, `platform`, `viewport`, `stack`, and + `error` are best-effort client observations and may be absent. +- The request body is capped at 64 KiB. Before logging, the backend + UTF-8-safely truncates title, description, error message, and URL to 8 KiB + each; task ID to 128 bytes; user agent and platform/language fields to 2 KiB + each; and stacks to 16 KiB each. It adds `truncated: true` when any value was + shortened. +- A valid report returns `204 No Content`. +- Invalid JSON, an unsupported source, or a report without visible text + returns `400 Bad Request`. A body over 64 KiB returns + `413 Request Entity Too Large`. +- An exhausted identity or process-wide report bucket returns + `429 Too Many Requests` with `Retry-After`. + +The endpoint logs one structured `error` entry. Client fields remain fields; +they cannot replace the fixed `frontend error toast` message. + +### Diagnostic bundle jobs + +`POST /api/v1/system/logs/bundles` accepts: + +```json +{ + "sources": ["backend", "frontend", "runtime", "acp"], + "session_ids": ["session-uuid-1", "session-uuid-2"] +} +``` + +- `sources` is a non-empty unique subset of `backend`, `frontend`, `runtime`, + and `acp`. `session_ids` is rejected unless `acp` is selected. Selecting + `acp` requires one to ten unique session IDs. +- The backend resolves each ACP session to an authorized task and current or + retained debug source. A non-admin may select only sessions on tasks they + own; an admin may select any eligible session. Unknown, foreign, or duplicate + IDs fail the request without revealing whether a foreign session exists. +- `acp` is rejected unless backend-authoritative ACP debug capture is enabled. + A frontend runtime debug flag alone cannot enable or authorize ACP export. +- A valid request returns `202 Accepted` with `id`, `status`, `reused`, + `build_deadline`, and nullable `expires_at`. `expires_at` is populated when + the job becomes ready/partial. An equivalent non-expired job for the same + identity, source set, and normalized ACP session set returns that job with + `reused: true`. +- Backend-only jobs can become ready immediately. Jobs containing `frontend` + collect browser evidence for at most 15 seconds. Jobs containing `acp` + collect selected executor evidence for at most 30 seconds; frontend and ACP + collection may overlap under the global job bounds. +- An invalid or empty source set returns `400 Bad Request`. +- A different request while that identity owns an active job returns + `429 Too Many Requests`; process-wide job saturation returns + `503 Service Unavailable`. Both include `Retry-After: 5`. + +`GET /api/v1/system/logs/capabilities` returns backend-authoritative bundle +capabilities for the current identity: + +```json +{ + "sources": ["backend", "frontend", "runtime"], + "acp_debug_enabled": false, + "acp_max_sessions": 10 +} +``` + +- `acp` appears in `sources` whenever ACP debug capture is enabled. Candidate + count may be zero; the UI still shows the debug action and its empty state. +- This endpoint reveals capability only. It does not return paths, frames, + another user's session IDs, or the value of environment variables. + +`GET /api/v1/system/logs/acp-sessions` returns at most 500 eligible sessions, +newest first, and is available only while ACP debug capture is enabled: + +```json +{ + "sessions": [ + { + "task_id": "task-uuid", + "task_title": "Investigate browser disconnect", + "session_id": "session-uuid", + "agent": "claude-acp", + "provider": "anthropic", + "model": "sonnet", + "status": "running", + "executor_type": "local_docker", + "last_activity_at": "2026-08-02T12:00:00Z", + "acp_availability": "reachable" + } + ] +} +``` + +- `acp_availability` is `host_retained`, `reachable`, or `unavailable`. + Unavailable sessions remain selectable only when a retained host file might + still be discovered during collection; otherwise the control disables them + with an explanation. +- Non-admin responses contain only caller-owned sessions. Admin responses may + contain all eligible sessions. The task title is returned only for the + authorized ACP-picker row and is not copied into an archive. Descriptions, + messages, prompts, tool payloads, file content, user identity, and log paths + are never returned. + +Reachable executor-side `agentctl` instances expose an internal +`GET /api/v1/debug/acp/:session/export?max_bytes=N` route only while +`KANDEV_DEBUG_AGENT_MESSAGES=true`: + +- The response is a bounded ZIP containing only recognized retained raw and + normalized ACP JSONL files for that exact session. `N` is clamped to the + backend's remaining ACP budget. +- The route never accepts a filesystem path or task/user identity and never + returns unrelated session files. Unknown sessions return `404`; disabled + debug capture returns `404`; unreadable or expired evidence returns `410`. +- The lifecycle/runtime client is the only bundle consumer. Browser clients + never call executor-side `agentctl` directly, and the backend revalidates ZIP + entry names, byte bounds, and selected-session ownership before inclusion. + +When frontend collection starts, the backend sends authenticated WebSocket +notification `system.logs.capture_requested` to every connected frontend for +the requesting identity: + +```json +{ + "bundle_id": "01J...", + "capture_deadline": "2026-07-30T12:35:15Z", + "max_chunk_bytes": 1048576, + "max_browser_profiles": 4 +} +``` + +Each frontend snapshots its local three-day store and uploads sequential chunks +to `POST /api/v1/system/logs/bundles/:id/frontend`: + +```json +{ + "browser_id": "opaque-browser-id", + "capture_stream_id": "opaque-per-tab-response-id", + "chunk_index": 0, + "done": false, + "storage_mode": "indexeddb", + "capture_metadata": null, + "entries": [] +} +``` + +- Each request body is capped at 1 MiB; each entry remains capped at 64 KiB. +- The first accepted chunk atomically binds a browser ID to its + `capture_stream_id` for that job. Chunks must be sequential within that + stream. Chunks from other tabs using the same browser ID are acknowledged + without parsing or appending their entries, so concurrent tabs cannot produce + a mixed snapshot. +- A caller may upload only to its own unexpired bundle job. +- The backend accepts at most 10,000 entries and 20 MiB per browser profile. +- The backend accepts at most four profiles and 80 MiB of frontend data for the + whole job. Later profiles receive `429` and are recorded as omitted. +- The final `done` chunk includes at most 8 KiB of server-validated + `capture_metadata`: dropped counts by level/reason, persistence failures, + entry truncations, and storage mode. Earlier chunks omit it. +- Accepted chunks return `204 No Content`; invalid ordering or bounds return + `400` or `413`; unknown/expired jobs return `404` or `410`. + +`GET /api/v1/system/logs/bundles/:id` returns the caller-owned job state: +`collecting`, `building`, `ready`, `partial`, `failed`, or `expired`, plus +source/capture counts, selected ACP session counts and aggregate availability, +warnings, expiry, and a download URL when available. + +`GET /api/v1/system/logs/bundles/:id/download` returns +`application/zip` for `ready` and `partial` jobs. It returns `409 Conflict` +while work is pending, `404` for unknown jobs, and `410 Gone` after expiry. + +`POST /api/v1/system/improve-kandev/bundle/lease` accepts a caller-owned +ready/partial `bundle_id` plus the caller's current Improve Kandev +`bundle_dir`. It verifies both artifacts belong to the authenticated identity, +copies the ZIP as `diagnostic-bundle.zip`, and returns that owner-only path. +Pending, foreign, expired, or invalid artifacts are rejected without copying. +Bootstrap directories carry a server-written owner marker outside the ZIP; a +path prefix or caller-supplied directory alone never proves ownership. + +The existing log list, tail, individual-file download, and dev debug-export +routes are removed. + +## Permissions + +All diagnostic routes use the existing authenticated install-user boundary. Any +signed-in user may create and download a bundle, matching the current System +Logs access rule. A user can inspect, upload to, or download only bundle jobs +created under the same identity. With authentication disabled, the synthetic +single-user identity keeps local behavior unchanged. + +Raw ACP selection adds a stricter task-ownership boundary. A non-admin may list +and include ACP evidence only for sessions whose task owner matches the +authenticated identity. An admin may list and include any eligible session. +Authentication-disabled local installs use the synthetic admin identity. The +backend authorizes every selected session before contacting host or executor +storage; possessing or guessing a session ID does not grant access. + +Frontend capture notifications and uploads never cross identities. Bundle +ownership is derived from the authenticated request, not a client-supplied user +or browser ID. + +Task-mode MCP bundle requests derive identity from the immutable current +execution task through the existing MCP identity scoper. The tool does not +accept a caller-supplied identity, task/session pair, destination, or reusable +credential. + +Backend and bundle files use owner-only permissions (`0600`) where Unix +permission bits are supported. Diagnostic archives may contain install-wide +backend activity, full URLs, console arguments, stacks, paths, browser +metadata, and user-visible backend error text. The page must present that +disclosure before the download action. + +ACP-inclusive archives carry a distinct high-sensitivity disclosure and may +contain prompts, responses, tool arguments/results, file content, MCP payloads, +environment-derived values, and secrets. Standard bundles never read raw ACP +files or persisted message/transcript tables. The runtime index is authorized +with the same task/session rules and excludes message bodies and user identity. + +## Failure modes + +- If the log directory cannot be created or the active file cannot be opened, + backend startup prints the intended path and a warning to stderr, continues + with stdout/stderr logging, and retries the file sink every 30 seconds. + Retention-cleanup failures are warnings and do not prevent startup. +- If midnight rollover fails after startup, the backend emits an error to + stderr and its existing sinks, continues writing to the active file, and + retries rollover on the next write without dropping the triggering entry. +- If browser persistence fails, capture continues in the bounded memory + fallback and the bundle manifest marks the degraded storage mode. +- If no eligible frontend is connected, a frontend-only bundle becomes + `partial` with no frontend file; an `all` bundle still includes backend files. +- If some frontend connections time out, disconnect, reject the request, or + exceed bounds, successful distinct-browser captures remain and the manifest + lists aggregate omissions without exposing another user's identity. +- If ACP debug capture is disabled, an ACP request fails validation before a + job is created. If a selected host file or executor becomes unavailable after + authorization, other selected sources continue and the bundle becomes + `partial` with a per-session availability warning in the manifest. +- If executor ACP export times out, exceeds its clamp, returns invalid ZIP + paths, or cannot be revalidated, that session is omitted. The backend does + not retry continuously, does not copy other sessions, and does not fail a + valid backend/frontend/runtime source. +- If the runtime index query fails, other selected sources continue and the + bundle becomes `partial`; it never falls back to a broader unscoped query. +- If ZIP construction fails, the job becomes `failed`, temporary partial files + are removed, and the UI surfaces a normal error toast. +- If a backend or browser diagnostic queue is saturated, lower-priority entries + may be omitted and the manifest records the loss; the product path that + produced the log continues without waiting. +- If the bundle worker is busy, the request receives a busy response or reuses + the caller's equivalent active job. It does not create unbounded concurrent + archive or capture work. +- If the browser report endpoint receives a network, authentication, + validation, or server failure, the original error toast remains unchanged + and the reporting promise is discarded without retry. +- Unsupported or non-text toast content is omitted from the corresponding text + field; reporting proceeds only when visible text can be extracted. +- Daily rollover uses UTC calendar boundaries and never uses client timestamps + to select a backend file. + +## Persistence guarantees + +- `backend-logs.log` preserves entries accepted by the file queue during the + current UTC day until the fixed 256 MiB daily bound. Pressure drops, startup + fallback gaps, bounded-shutdown loss, and archive truncation are explicit in + counters and bundle manifests. +- Dated archives preserve the two preceding UTC calendar days. Files outside + that rolling window are deleted at startup and after successful rollover. +- Browser console history remains in the browser profile for three UTC days, + subject to its entry/byte caps. It is not server-persistent before capture. +- Raw ACP files retain their existing debug-only executor/host retention: at + most 48 hours by default, bounded by per-file rotation and total file count. + Bundle creation reads selected files on demand and does not create permanent + backend ACP storage. Executor teardown may make evidence unavailable before + the retention age. +- The runtime index is generated per bundle from authorized current database + state, capped to the diagnostic window/row limit, and is not persisted as a + separate product record. +- A frontend bundle can contain only histories returned by browser profiles + connected during its 15-second collection window. +- Bundle jobs and ZIPs survive frontend navigation. Collecting/building jobs + are cancelled after five minutes; ready/partial archives expire 15 minutes + after becoming downloadable. They are not durable product data. +- Improve Kandev's task-context ZIP is a separate internal lease on the shared + archive builder and remains available for up to 24 hours. +- No toast report is stored in SQLite or queued in browser storage for retry. +- Concurrent backend processes sharing one Kandev home are unsupported. + +## Scenarios + +- **GIVEN** no explicit logging override, **WHEN** Kandev starts normally, + **THEN** stdout prints the absolute backend log path, the file contains + `info+`, and stdout contains only `warn+` backend entries. +- **GIVEN** Kandev starts with debug logging, **WHEN** it emits debug through + error entries, **THEN** all are in the file while only warn/error are on + stdout. +- **GIVEN** a same-UTC-day restart, **WHEN** the backend opens its log, + **THEN** it appends without losing earlier entries. +- **GIVEN** the configured log directory is temporarily unwritable, **WHEN** + Kandev starts, **THEN** product startup succeeds with a stderr warning and + the backend retries file activation every 30 seconds. +- **GIVEN** more than three UTC days of backend files, **WHEN** cleanup runs, + **THEN** only the current and two preceding days remain. +- **GIVEN** an error toast on a recognized task route, **WHEN** its report is + accepted, **THEN** one backend error entry includes its visible text, + browser context, and `task_id`. +- **GIVEN** console activity over three days, **WHEN** browser retention runs, + **THEN** expired and oldest-over-cap entries are removed without uploading + retained entries. +- **GIVEN** IndexedDB is slow or unavailable, **WHEN** an application emits a + console entry, **THEN** the original console behavior returns without waiting + for persistence and bounded staging/fallback preserves best-effort evidence. +- **GIVEN** a backend file sink is slow or blocked, **WHEN** application code + emits logs, **THEN** logging does not block that code; lower-priority entries + are shed first and the eventual bundle reports any loss. +- **GIVEN** stdout is blocked while the file sink is writable, **WHEN** backend + logs are emitted, **THEN** the stdout queue may shed entries without stalling + or dropping entries accepted by the independent file queue. +- **GIVEN** two users sign into Kandev from the same browser profile, **WHEN** + the second user requests a bundle, **THEN** its frontend files contain only + entries captured under the second user's identity partition. +- **GIVEN** a signed-in user opens System Logs, **WHEN** the page renders, + **THEN** it names the frontend/backend event classes, states that the standard + bundle does not read stored session or agent messages, warns about incidental + emitted log content, and exposes one Customize bundle action without a tail + or file table. +- **GIVEN** ACP debug capture is disabled, **WHEN** a user opens System Logs, + **THEN** no ACP download action or ACP custom source is offered and a crafted + ACP bundle request is rejected. +- **GIVEN** ACP debug capture is enabled, **WHEN** a user selects ACP debug + messages in **Customize bundle**, **THEN** the session picker opens with the + high-sensitivity disclosure and collection cannot start until at least one + authorized session is selected. +- **GIVEN** the ACP picker lists an authorized session, **WHEN** a user reviews + it, **THEN** its task title links to the task in a new tab and select-all + includes no more than the allowed number of eligible sessions. +- **GIVEN** a non-admin can observe another user's session ID, **WHEN** they + list ACP candidates or submit that ID directly, **THEN** the session is not + disclosed or included and the response does not reveal its existence. +- **GIVEN** an admin selects two reachable sessions and one stopped unavailable + executor session, **WHEN** the debug bundle finishes, **THEN** raw and + normalized files for the reachable sessions are included, the unavailable + session is omitted, and the bundle is downloadable as `partial` with an + explicit manifest warning. +- **GIVEN** the runtime index is selected without ACP, **WHEN** a bundle is + built, **THEN** it contains only authorized bounded session/runtime metadata + and no task titles, prompts, responses, tool payloads, files, or message + bodies. +- **GIVEN** two tabs from one browser profile answer a bundle request, **WHEN** + their chunks interleave, **THEN** the first capture stream owns that browser + ID and the ZIP contains one internally consistent frontend JSONL snapshot. +- **GIVEN** one browser answers and another times out, **WHEN** collection + expires, **THEN** the combined ZIP is downloadable as `partial` with backend + files, the successful frontend file, and a manifest warning. +- **GIVEN** retained backend files exceed 160 MiB, **WHEN** a bundle is built, + **THEN** it includes newest byte ranges within the budget, becomes `partial`, + and records each source offset and omitted byte count in the manifest. +- **GIVEN** an agent needs only task-scoped backend evidence, **WHEN** it + calls `get_diagnostic_bundle_kandev` for `backend`, **THEN** the current task + owner scopes the job, no frontend capture is triggered, and the tool returns + an executor-local ZIP path the agent can extract and exact-grep. +- **GIVEN** an agent requests frontend evidence while no browser is connected, + **WHEN** collection finishes, **THEN** it receives a partial ZIP whose + manifest states that no frontend capture was available. +- **GIVEN** a phone user downloads diagnostics, **WHEN** collection completes, + **THEN** the same disclosure, progress, partial-state warning, and archive + download remain available without horizontal scrolling or desktop-only + controls. +- **GIVEN** a phone user customizes a bundle, **WHEN** they select sources and + ACP sessions, **THEN** an inset bottom drawer keeps disclosure and actions + reachable with one internal scroll owner, safe-area clearance, ≥44px targets, + and the same validation/result as desktop. + +## Out of scope + +- Continuous browser-log upload, hosted telemetry, or crash-reporting service. +- Continuous central upload or permanent backend retention of raw ACP frames. +- Including raw ACP in the standard bundle, Improve Kandev bundle, or + task-mode `get_diagnostic_bundle_kandev` response. +- Exporting database contents, task titles/descriptions, chat transcripts, + stored agent messages, workspace files, secrets/configuration, or environment + dumps as standalone bundle sources. +- Preserving backend or frontend diagnostic history beyond three UTC days. +- A server-side free-text/regex log search or unbounded JSON log export. +- Viewing, copying, refreshing, or individually downloading log files from the + System Logs page. +- Retrying failed error-toast reports. +- Changing toast copy, placement, duration, or styling. +- Adding a setting for log path, retention, browser capture, or bundle bounds. +- Supporting configurable size-based, per-process, or local-time backend + rotation. diff --git a/docs/specs/system-page/spec.md b/docs/specs/system-page/spec.md index 07f7d30286..22bd2d396d 100644 --- a/docs/specs/system-page/spec.md +++ b/docs/specs/system-page/spec.md @@ -10,7 +10,7 @@ owner: tbd Today, settings-style operational concerns (health, disk usage, current version, OSS attribution, log access, database maintenance) have no home in kandev's UI. The existing settings sidebar is product-configuration only (executors, agents, integrations); the only system-shaped entry — Changelog — sits awkwardly inside it. When something goes wrong, users have no path inside the app to see "where is my database, how big is it, what version am I on, is there an update, where are the logs." This forces them into the filesystem and into GitHub, and it makes recoverable problems (bloated SQLite, corrupt state) look unrecoverable. -Radarr and Sonarr solve this with a dedicated **System** area: a group of read-only diagnostic pages plus a small number of gated maintenance actions. This spec brings the same shape to kandev, scoped to the kandev-specific surface (database diagnostics, SQLite backups/maintenance, worktrees, GitHub releases, and lumberjack log files). +Radarr and Sonarr solve this with a dedicated **System** area: a group of read-only diagnostic pages plus a small number of gated maintenance actions. This spec brings the same shape to kandev, scoped to the kandev-specific surface (database diagnostics, SQLite backups/maintenance, worktrees, GitHub releases, and daily backend log files). ## What (v1) @@ -35,9 +35,21 @@ A new **System** group is added to the existing settings sidebar (`apps/web/comp - Per-row actions: **Download**, **Restore** (gated like Factory Reset), **Delete** (confirm-only). - **Create snapshot** button at the top of the list; uses the existing `VACUUM INTO` path. 5. **Logs** — `/settings/system/logs` - - Static log viewer: last 1000 lines of the current lumberjack log file, with a **Refresh** button. No live tail in v1. - - List of rotated log files with name/size/mtime and a per-row **Download** button. - - **Download current** button at the top. + - One **Customize bundle** action that opens source selection with backend + and frontend evidence selected by default; the runtime index and + debug-only ACP evidence remain explicit opt-ins. Bundle collection keeps + collecting/preparing/partial/busy/error feedback on the page. + - Visible disclosure that the ZIP contains up to three days of install-wide + backend logs and up to four connected browser profiles subject to fixed + byte limits, including URLs, console arguments, stacks, and runtime + metadata. + - The page does not render a tail, file table, copy/refresh controls, or individual-file downloads. + - If a source is unavailable, truncated by the fixed diagnostic budgets, or + reports queue loss, the page still downloads a partial ZIP and names the + omission without blocking normal application work. + - The active files, browser capture, ZIP lifecycle, sink thresholds, and + frontend-error ingestion contracts are defined by + [Diagnostic logging](../platform/diagnostic-logging.md). 6. **Updates** — `/settings/system/updates` - Shows running version, latest available version, and a "new version available" badge if newer. - **Check now** button — forces a re-poll (rate-limited 30s per process). @@ -81,9 +93,12 @@ POST /api/v1/system/backups - create snapshot; 202 + jobId GET /api/v1/system/backups/:name/download - stream snapshot file POST /api/v1/system/backups/:name/restore - restore; body { confirm: "RESTORE" } DELETE /api/v1/system/backups/:name - delete snapshot -GET /api/v1/system/logs - { files: [{ name, size, mtime }] } -GET /api/v1/system/logs/tail?n=1000 - last N lines of current log -GET /api/v1/system/logs/:name/download - stream log file +POST /api/v1/system/logs/bundles - create source-selectable diagnostic ZIP job +GET /api/v1/system/logs/bundles/:id - bundle collection/build status +POST /api/v1/system/logs/bundles/:id/frontend - bounded frontend capture chunk +GET /api/v1/system/logs/bundles/:id/download - stream ready/partial ZIP +POST /api/v1/system/logs/frontend-errors - bounded/rate-limited error-toast report +POST /api/v1/system/improve-kandev/bundle/lease - owner-verified 24h task-context copy GET /api/v1/system/updates - { current, latest, latestCheckedAt, releaseUrl, install, applySupported } POST /api/v1/system/updates/check - force GitHub re-poll; rate-limited 30s POST /api/v1/system/updates/apply - queue service-only self-update; body { confirm: "UPDATE" } @@ -148,7 +163,7 @@ The page reads the JSON statically; no backend endpoint is needed. - **GIVEN** the user clicks **Factory Reset**, types `RESET`, and confirms, **WHEN** the backend executes, **THEN** a fresh snapshot is created first, all tables are dropped and migrations re-run, the backend restarts, and the frontend redirects to the empty onboarding state once it reconnects. - **GIVEN** the backend cannot reach GitHub, **WHEN** the poller fires, **THEN** the failure is logged but the previous `latest_version` and `latest_version_checked_at` remain in `kandev_meta`; the Updates page surfaces the stale value with a "Last checked