diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index df9e226227..ac658d7d30 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -529,9 +529,9 @@ jobs: needs: changes if: needs.changes.outputs.run == 'true' runs-on: windows-latest - # The race-sensitive Windows package suite can exceed 15 minutes on a - # busy hosted runner; keep the cap finite while allowing normal completion. - timeout-minutes: 20 + # The race-sensitive Windows package suite can approach 20 minutes on a + # busy hosted runner; keep the cap finite while allowing runner variance. + timeout-minutes: 30 # Least-privilege: this job only reads source and runs tests, no PR/issue # writes or release operations. Flagged by CodeQL workflow audit + matching # CodeRabbit nitpick. diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 67b2f9690d..4b92d57b5b 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -117,7 +117,7 @@ jobs: # .github/docker/ci-base/Dockerfile. Uses the `build` tag (with Go) # because this job compiles the backend. container: ghcr.io/kdlbs/kandev-ci:build-latest - timeout-minutes: 10 + timeout-minutes: 20 # Container jobs default to `sh`; pin to bash for the few shell steps below. defaults: run: @@ -179,6 +179,42 @@ jobs: - name: Build Docker E2E helper binaries run: make build-backend-linux-helpers + - name: Record immutable E2E build identity + run: | + set -euo pipefail + node <<'NODE' + const { createHash } = require("node:crypto"); + const fs = require("node:fs"); + const path = require("node:path"); + + const backendDir = path.resolve("apps/backend"); + const artifacts = [ + "bin/kandev", + "bin/mock-agent", + "bin/agentctl-linux-amd64", + "bin/mock-agent-linux-amd64", + ".build/kandev-plugin-e2e-1.0.0.tar.gz", + ]; + const checksums = Object.fromEntries( + artifacts.map((artifact) => { + const contents = fs.readFileSync(path.join(backendDir, artifact)); + return [artifact, createHash("sha256").update(contents).digest("hex")]; + }), + ); + fs.writeFileSync( + path.join(backendDir, "bin/e2e-build-identity.json"), + JSON.stringify( + { + schema_version: 1, + source_revision: process.env.GITHUB_SHA, + artifacts: checksums, + }, + null, + 2, + ) + "\n", + ); + NODE + - name: Locate latest successful main timing profile artifact id: main-timing-run env: @@ -424,19 +460,14 @@ jobs: # executor and the SSH executor's sshd target). Without --project the # container-bound specs would run here too, fail seeding (Docker not # enabled for these shards), and spam every shard. - # global-setup.ts refuses to run a backend binary older than any - # apps/backend source. That guard exists for local runs, where nothing - # else notices `pnpm run build:e2e` rebuilding only the frontend. Here - # the binary is staged out of band: the `build` job compiled it from - # this same commit, and the shard's own checkout stamps every source - # file with a *later* mtime than the artifact it downloads. Freshness is - # already guaranteed by `needs: build`; the mtime comparison is not - # meaningful across jobs. + # Cross-job archive extraction does not preserve useful mtimes. Validate + # the source revision and every required artifact checksum instead. - name: Run planned E2E tests (shard ${{ matrix.shard }}/14) working-directory: apps/web env: E2E_SHARD: ${{ matrix.shard }} - KANDEV_E2E_SKIP_FRESHNESS: "1" + KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json + KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/normal/${{ matrix.shard }}.json" - name: Upload blob report @@ -455,15 +486,14 @@ jobs: path: apps/web/e2e/test-results/ retention-days: 7 - # Real-Docker E2E: exercises the local_docker executor end-to-end. + # Real-container E2E: exercises the Docker, SSH, and Kubernetes executors. # ubuntu-latest ships with Docker preinstalled, so no service container is - # required. Container-bound tests are slow (~2 min each), so we shard across + # required. Container-bound tests are slow (up to ~3 min each), so we shard across # 6 runners — the build job creates a duration-aware manifest for this # cohort, and each runner executes its explicit file selection. # - # This job runs the `containers` Playwright project, which covers both the - # Docker executor and the SSH executor (since the SSH executor's e2e tests - # also rely on Docker to host a real sshd container as the remote target). + # This job runs the `containers` Playwright project. SSH uses Docker for a + # real sshd target; Kubernetes uses a pinned, disposable Kind cluster. # Sharded 6 ways — each shard is its own runner. See apps/web/e2e/README.md. e2e-containers: name: E2E Containers Shard ${{ matrix.shard }}/6 @@ -477,7 +507,7 @@ jobs: # We still avoid the slow `playwright install --with-deps` step by copying # browsers out of the kandev-ci image — see the extract step below. runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 50 strategy: fail-fast: false matrix: @@ -743,16 +773,96 @@ jobs: name: e2e-shard-manifests path: e2e-manifests - # Same out-of-band staging as the chromium shards above — see the comment - # on that step for why the backend-binary freshness guard is skipped. - - name: Run planned container-backed E2E tests shard ${{ matrix.shard }}/6 (Docker executor + SSH executor) + - name: Detect Kubernetes specs in this shard + id: kubernetes + env: + MANIFEST_PATH: ${{ github.workspace }}/e2e-manifests/containers/${{ matrix.shard }}.json + SHARD_INDEX: ${{ matrix.shard }} + run: | + set -euo pipefail + node <<'NODE' + const fs = require("node:fs"); + + const manifest = JSON.parse(fs.readFileSync(process.env.MANIFEST_PATH, "utf8")); + const shard = manifest.shards.find((candidate) => candidate.index === Number(process.env.SHARD_INDEX)); + if (!shard) throw new Error(`container shard ${process.env.SHARD_INDEX} is absent`); + const hasKubernetes = shard.files.some((file) => file.startsWith("tests/kubernetes/")); + fs.appendFileSync(process.env.GITHUB_OUTPUT, `has_kubernetes=${hasKubernetes}\n`); + console.log(`Kubernetes specs assigned: ${hasKubernetes}`); + NODE + + # Tool and node versions are intentionally coupled. The Kind fixture + # verifies the downloaded checksums again before creating a cluster. + - name: Install pinned Kind and kubectl + if: steps.kubernetes.outputs.has_kubernetes == 'true' + env: + KIND_VERSION: v0.32.0 + KIND_SHA256_AMD64: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 + KUBERNETES_VERSION: v1.36.1 + KUBECTL_SHA256_AMD64: 629d3f410e09bf49b64ae7079f7f0bda1191efed311f7d37fdbab0ad5b0ec2b7 + KIND_NODE_IMAGE: kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5 + run: | + set -euo pipefail + tool_dir="$RUNNER_TEMP/kandev-kubernetes-tools" + mkdir -p "$tool_dir" + curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" \ + --output "$tool_dir/kind" + curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ + "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \ + --output "$tool_dir/kubectl" + echo "${KIND_SHA256_AMD64} $tool_dir/kind" | sha256sum --check --strict + echo "${KUBECTL_SHA256_AMD64} $tool_dir/kubectl" | sha256sum --check --strict + chmod 0755 "$tool_dir/kind" "$tool_dir/kubectl" + "$tool_dir/kind" version + "$tool_dir/kubectl" version --client + echo "Kind node image: $KIND_NODE_IMAGE" + echo "KANDEV_E2E_KIND_BIN=$tool_dir/kind" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KUBECTL_BIN=$tool_dir/kubectl" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KIND_CLUSTER_NAME=kandev-e2e-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.shard }}" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KIND_OWNERSHIP_MARKER=$RUNNER_TEMP/kandev-kind-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${{ matrix.shard }}.owned" >> "$GITHUB_ENV" + + # Validate the same immutable build identity as the regular shards. + - name: Run planned container-backed E2E tests shard ${{ matrix.shard }}/6 (Docker + SSH + Kubernetes) working-directory: apps/web env: KANDEV_E2E_CONTAINERS: "1" E2E_SHARD: ${{ matrix.shard }} - KANDEV_E2E_SKIP_FRESHNESS: "1" + KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json + KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} run: bash e2e/scripts/run-planned-shard.sh "$GITHUB_WORKSPACE/e2e-manifests/containers/${{ matrix.shard }}.json" + - name: Delete exact Kind cluster + if: always() && steps.kubernetes.outputs.has_kubernetes == 'true' + run: | + set -euo pipefail + kind_bin="${KANDEV_E2E_KIND_BIN:-}" + cluster_name="${KANDEV_E2E_KIND_CLUSTER_NAME:-}" + ownership_marker="${KANDEV_E2E_KIND_OWNERSHIP_MARKER:-}" + if [[ -z "$kind_bin" || -z "$cluster_name" || -z "$ownership_marker" ]]; then + echo "Kind tools or exact target were never initialized; cleanup is a no-op." + exit 0 + fi + if [[ ! -f "$ownership_marker" ]]; then + echo "No reserved ownership marker for $cluster_name; cleanup is a no-op." + exit 0 + fi + if [[ ! -x "$kind_bin" ]]; then + echo "Pinned Kind tool is not executable: $kind_bin" >&2 + exit 1 + fi + if [[ ! "$cluster_name" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; then + echo "Refusing cleanup for invalid exact Kind cluster name: $cluster_name" >&2 + exit 1 + fi + owned_name="$(tr -d '\r\n' < "$ownership_marker")" + if ! [[ "$owned_name" == "$cluster_name" ]]; then + echo "Refusing cleanup: marker owns '$owned_name', not '$cluster_name'." >&2 + exit 1 + fi + "$kind_bin" delete cluster --name "$cluster_name" + rm -f -- "$ownership_marker" + # Named with the same `blob-report-*` prefix the merge job globs for # so the containers results land in the unified HTML report alongside # the sharded chromium/mobile-chrome runs. @@ -772,6 +882,164 @@ jobs: path: apps/web/e2e/test-results/ retention-days: 7 + # API-only real-cluster smoke on the oldest and current supported Kubernetes + # releases. The full 12-case browser lifecycle runs only once, above, on the + # current release; this matrix keeps compatibility coverage intentionally small. + e2e-kubernetes-compatibility: + name: Kubernetes Compatibility (${{ matrix.kubernetes_version }}) + needs: [changes, build] + if: needs.changes.outputs.run == 'true' + runs-on: ubuntu-latest + timeout-minutes: 35 + strategy: + fail-fast: false + matrix: + include: + - id: v1-34-8 + kubernetes_version: v1.34.8 + kubectl_sha256_amd64: f6249132865c13abe3c9dd5038f5da65849cb86eee1608c001831504e481aa8c + node_image: kindest/node:v1.34.8@sha256:02722c2dedddcfc00febf5d27fbeb9b7b2c14294c82109ff4a85d89ac9ba3256 + - id: v1-36-1 + kubernetes_version: v1.36.1 + kubectl_sha256_amd64: 629d3f410e09bf49b64ae7079f7f0bda1191efed311f7d37fdbab0ad5b0ec2b7 + node_image: kindest/node:v1.36.1@sha256:3489c7674813ba5d8b1a9977baea8a6e553784dab7b84759d1014dbd78f7ebd5 + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Verify Docker is available + run: docker info + + - name: Set up pnpm + uses: pnpm/action-setup@a8198c4bff370c8506180b035930dea56dbd5288 # v5 + with: + version: "9.15.9" + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "24" + + - name: Cache pnpm store + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.local/share/pnpm/store + key: pnpm-${{ runner.os }}-${{ hashFiles('apps/pnpm-lock.yaml') }} + restore-keys: pnpm-${{ runner.os }}- + + - name: Install dependencies + working-directory: apps + run: pnpm install --frozen-lockfile + + - name: Download backend build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: e2e-backend-build + path: apps/backend/bin/ + + - name: Make backend binaries executable + run: chmod +x apps/backend/bin/* + + - name: Download E2E fixture plugin package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: e2e-plugin-package + path: apps/backend/.build/ + + - name: Download web build + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: e2e-web-build + path: apps/web/ + + - name: Extract web build + working-directory: apps/web + run: tar -xf web-dist.tar && rm web-dist.tar + + - name: Install pinned Kind and kubectl + env: + KIND_VERSION: v0.32.0 + KIND_SHA256_AMD64: 50030de23cf40a18505f20426f6a8506bedf13c6e509244bd1fa9463721b0f54 + KUBERNETES_VERSION: ${{ matrix.kubernetes_version }} + KUBECTL_SHA256_AMD64: ${{ matrix.kubectl_sha256_amd64 }} + KIND_NODE_IMAGE: ${{ matrix.node_image }} + run: | + set -euo pipefail + tool_dir="$RUNNER_TEMP/kandev-kubernetes-tools-${{ matrix.id }}" + mkdir -p "$tool_dir" + curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ + "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" \ + --output "$tool_dir/kind" + curl --fail --location --silent --show-error --retry 3 --retry-delay 2 --retry-all-errors \ + "https://dl.k8s.io/release/${KUBERNETES_VERSION}/bin/linux/amd64/kubectl" \ + --output "$tool_dir/kubectl" + echo "${KIND_SHA256_AMD64} $tool_dir/kind" | sha256sum --check --strict + echo "${KUBECTL_SHA256_AMD64} $tool_dir/kubectl" | sha256sum --check --strict + chmod 0755 "$tool_dir/kind" "$tool_dir/kubectl" + "$tool_dir/kind" version + "$tool_dir/kubectl" version --client + echo "Kind node image: $KIND_NODE_IMAGE" + echo "KANDEV_E2E_KIND_BIN=$tool_dir/kind" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KUBECTL_BIN=$tool_dir/kubectl" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KUBERNETES_VERSION=$KUBERNETES_VERSION" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KIND_CLUSTER_NAME=kandev-e2e-compat-${{ matrix.id }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_ENV" + echo "KANDEV_E2E_KIND_OWNERSHIP_MARKER=$RUNNER_TEMP/kandev-kind-compat-${{ matrix.id }}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.owned" >> "$GITHUB_ENV" + + # Validate the build job's immutable source revision and artifact hashes. + - name: Run Kubernetes API compatibility smoke + working-directory: apps/web + env: + KANDEV_E2E_CONTAINERS: "1" + KANDEV_E2E_BUILD_IDENTITY: ${{ github.workspace }}/apps/backend/bin/e2e-build-identity.json + KANDEV_E2E_EXPECTED_SOURCE_REVISION: ${{ github.sha }} + E2E_FAIL_ON_FLAKY: "1" + run: >- + pnpm exec playwright test --config e2e/playwright.config.ts + --project=kubernetes-compat --reporter=list tests/kubernetes-compat + + - name: Delete exact Kind compatibility cluster + if: always() + run: | + set -euo pipefail + kind_bin="${KANDEV_E2E_KIND_BIN:-}" + cluster_name="${KANDEV_E2E_KIND_CLUSTER_NAME:-}" + ownership_marker="${KANDEV_E2E_KIND_OWNERSHIP_MARKER:-}" + if [[ -z "$kind_bin" || -z "$cluster_name" || -z "$ownership_marker" ]]; then + echo "Kind tools or exact target were never initialized; cleanup is a no-op." + exit 0 + fi + if [[ ! -f "$ownership_marker" ]]; then + echo "No reserved ownership marker for $cluster_name; cleanup is a no-op." + exit 0 + fi + if [[ ! -x "$kind_bin" ]]; then + echo "Pinned Kind tool is not executable: $kind_bin" >&2 + exit 1 + fi + if [[ ! "$cluster_name" =~ ^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$ ]]; then + echo "Refusing cleanup for invalid exact Kind cluster name: $cluster_name" >&2 + exit 1 + fi + owned_name="$(tr -d '\r\n' < "$ownership_marker")" + if ! [[ "$owned_name" == "$cluster_name" ]]; then + echo "Refusing cleanup: marker owns '$owned_name', not '$cluster_name'." >&2 + exit 1 + fi + "$kind_bin" delete cluster --name "$cluster_name" + rm -f -- "$ownership_marker" + + - name: Upload Kubernetes compatibility results + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: kubernetes-compatibility-results-${{ matrix.id }} + path: apps/web/e2e/test-results/ + if-no-files-found: ignore + retention-days: 7 + desktop-e2e: name: Desktop E2E Smoke needs: changes @@ -1197,7 +1465,16 @@ jobs: e2e-gate: name: E2E Tests Passed runs-on: ubuntu-latest - needs: [changes, build, e2e, e2e-containers, desktop-e2e, e2e-report] + needs: + [ + changes, + build, + e2e, + e2e-containers, + e2e-kubernetes-compatibility, + desktop-e2e, + e2e-report, + ] if: always() steps: @@ -1207,6 +1484,7 @@ jobs: BUILD_RESULT: ${{ needs.build.result }} E2E_RESULT: ${{ needs.e2e.result }} E2E_CONTAINERS_RESULT: ${{ needs['e2e-containers'].result }} + E2E_KUBERNETES_COMPATIBILITY_RESULT: ${{ needs['e2e-kubernetes-compatibility'].result }} DESKTOP_E2E_RESULT: ${{ needs['desktop-e2e'].result }} E2E_REPORT_RESULT: ${{ needs['e2e-report'].result }} run: | @@ -1216,6 +1494,7 @@ jobs: echo "Build: ${BUILD_RESULT}" echo "E2E shards: ${E2E_RESULT}" echo "E2E container shards: ${E2E_CONTAINERS_RESULT}" + echo "Kubernetes compatibility: ${E2E_KUBERNETES_COMPATIBILITY_RESULT}" echo "Desktop smoke: ${DESKTOP_E2E_RESULT}" echo "Report merge: ${E2E_REPORT_RESULT}" @@ -1238,6 +1517,7 @@ jobs: "build:${BUILD_RESULT}" \ "e2e:${E2E_RESULT}" \ "e2e-containers:${E2E_CONTAINERS_RESULT}" \ + "e2e-kubernetes-compatibility:${E2E_KUBERNETES_COMPATIBILITY_RESULT}" \ "desktop-e2e:${DESKTOP_E2E_RESULT}" \ "e2e-report:${E2E_REPORT_RESULT}"; do name="${entry%%:*}" diff --git a/AGENTS.md b/AGENTS.md index 6c25d51f3b..5ebc465a1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ apps/ - **Web-only scripts** live in `apps/web/package.json`; run them from `apps/web` (for example `pnpm run i18n:check` or `pnpm run i18n:ratchet`) or use `pnpm --filter @kandev/web ...`, not from `apps/`. - **Desktop**: Tauri shell (`cd apps && pnpm --filter @kandev/desktop build|e2e`; Rust tests from `apps/desktop/src-tauri`) - **UI**: Shadcn components via `@kandev/ui` -- **E2E**: Playwright (`cd apps/web && pnpm e2e:raw`). The `containers` project (gated on `KANDEV_E2E_CONTAINERS=1`, formerly `docker`) covers both the Docker executor and the SSH executor — anything that needs a real Docker daemon on the host lives there. See `apps/web/e2e/README.md`. +- **E2E**: Playwright (`cd apps/web && pnpm e2e:raw`). The `containers` project (gated on `KANDEV_E2E_CONTAINERS=1`, formerly `docker`) covers Docker, SSH, and Kind-backed Kubernetes executor scenarios — anything that needs a real Docker daemon on the host lives there. See `apps/web/e2e/README.md`. - **GitHub repo**: `https://github.com/kdlbs/kandev` - **Container image**: `ghcr.io/kdlbs/kandev` (GitHub Container Registry) diff --git a/apps/backend/AGENTS.md b/apps/backend/AGENTS.md index 2330c8e157..84b5760079 100644 --- a/apps/backend/AGENTS.md +++ b/apps/backend/AGENTS.md @@ -23,6 +23,7 @@ apps/backend/ │ │ ├── dto/ # Agent data transfer objects │ │ ├── executor/ # Executor types, checks, and service │ │ ├── handlers/ # Agent event handlers +│ │ ├── kubernetes/ # Kubernetes config, Pod/PVC composition, admission, and client streaming │ │ ├── registry/ # Agent type registry and defaults │ │ ├── settings/ # Agent settings │ │ ├── mcpconfig/ # MCP server configuration @@ -69,6 +70,7 @@ apps/backend/ │ │ └── secretadapter/ # Upsert-style adapter over secrets.SecretStore │ ├── i18n/ # Localization for backend-rendered browser/share artifacts │ ├── jira/ # Jira/Atlassian Cloud integration (config, REST client, poller) +│ ├── kubernetes/ # Kubernetes diagnostics and exact recorded-session status API │ ├── linear/ # Linear integration (config, GraphQL client, poller) │ ├── lsp/ # LSP server │ ├── mcp/ # MCP protocol support @@ -101,15 +103,9 @@ apps/backend/ - Handles event-driven state transitions via workflow engine - Located in `internal/orchestrator/` -**Cancellation progress projection:** `orchestrator.Service.CancellationPending(sessionID)` is a -runtime-only, session-scoped view of accepted cancellation work. Serialization that carries the -boolean with ordering identity uses the atomic `CancellationPendingSnapshot(sessionID)` provider, -whose process-local revision increments on first-begin and last-end transitions. The task DTO -package exposes both the compatibility boolean provider and snapshot seam; boot state, task-session -HTTP/WS lists and detail responses, and the session-scoped WebSocket notification must project -explicit `true`/`false` values plus the revision. Keep count, revision, and publication queue updates -in one critical section, drain event-bus sends outside it, and never persist this transient marker or -turn it into a coarse session lifecycle state. +**Cancellation progress projection:** `orchestrator.Service.CancellationPending(sessionID)` is a runtime-only, session-scoped view of accepted cancellation work. Serialization that carries the boolean with ordering identity uses the atomic `CancellationPendingSnapshot(sessionID)` provider, whose process-local revision increments on first-begin and last-end transitions. +The task DTO package exposes both the compatibility boolean provider and snapshot seam; boot state, task-session HTTP/WS lists and detail responses, and the session-scoped WebSocket notification must project explicit `true`/`false` values plus the revision. +Keep count, revision, and publication queue updates in one critical section, drain event-bus sends outside it, and never persist this transient marker or turn it into a coarse session lifecycle state. **Watcher Dispatch Coordinator** (`internal/orchestrator/watcher_dispatch.go`) is the single pipeline that turns a freshly-observed external issue (Linear, Jira, future) into a Kandev task. Bus subscribers for each integration forward the event to `WatcherDispatchCoordinator.Dispatch` with a per-integration `WatcherSource` implementation (`source_linear.go`, `source_jira.go`). Source methods carry the integration-specific bits (reserve dedup, build task request, attach task ID, release, auto-start params); the coordinator owns the cross-cutting pipeline (create task, decide auto-start, error/release handling). Add a new watcher = implement `WatcherSource` + register a one-line bus subscriber. Do NOT add another `createXIssueTask` mirror. @@ -140,7 +136,7 @@ replace state verification, installation association, or HMAC verification. **Lifecycle Manager** (`internal/agent/runtime/lifecycle/`) manages agent instances under the runtime: - `Manager` (`manager.go`, `manager_*.go`) - central coordinator for agent lifecycle -- `ExecutorBackend` interface (`executor_backend.go`) - abstracts execution environment (Docker, Standalone, Sprites, Remote Docker) +- `ExecutorBackend` interface (`executor_backend.go`) - abstracts execution environment (Docker, Standalone, Sprites, SSH, Kubernetes, Remote Docker) - `ExecutionStore` (`execution_store.go`) - thread-safe in-memory execution tracking - `session.go` - ACP session initialization and resume - `streams.go` - WebSocket stream connections to agentctl @@ -164,7 +160,11 @@ Standalone agentctl is launched in its own process group so terminal Ctrl+C is h - `local_pc` - Standalone process on host - `local_docker` - Docker container on host - `sprites` - Sprites cloud environment -- `remote_docker`, `remote_vps`, `k8s` - Planned +- `ssh` - Remote SSH host +- `k8s` - Namespaced Kubernetes Pod with optional PVC workspace +- `remote_docker`, `remote_vps` - Planned + +**Kubernetes lifecycle:** `executors_running` is the authoritative resource inventory. Persist the exact Pod/PVC names, UIDs, full `kandev.ai/*` identity, workload snapshot, and internal runtime-secret references before reporting a launch as durable. Ordinary stop and backend shutdown preserve resources; terminal cleanup deletes the Pod and only a Kandev-created PVC after exact identity checks and confirmed absence. Reconnect uses the current executor connection config but the recorded workload/resource snapshot, and any ambiguity fails closed. Keep agentctl reachable only through a process-local loopback port-forward; never add a Service or place resolved credentials in a Pod spec. **Remote SSH executor platforms:** Treat supported remote OS/arch values as an end-to-end contract. Platform probe/normalization, lifecycle support checks, agentctl helper resolution, platform default shell, SSH readiness endpoints, frontend response types, and tests must stay aligned. Preserve raw unsupported platform details in user-facing errors, but use normalized values for supported-platform matching. Keep shell defaults platform-aware: Darwin defaults to `zsh`, Linux defaults to `bash`, unless an explicit shell is saved. diff --git a/apps/backend/go.mod b/apps/backend/go.mod index c1b0a0b328..cfe9a98ace 100644 --- a/apps/backend/go.mod +++ b/apps/backend/go.mod @@ -3,7 +3,7 @@ module github.com/kandev/kandev go 1.26.0 require ( - github.com/Masterminds/semver/v3 v3.2.1 + github.com/Masterminds/semver/v3 v3.4.0 github.com/UserExistsError/conpty v0.1.4 github.com/bmatcuk/doublestar/v4 v4.10.0 github.com/coder/acp-go-sdk v0.13.5 @@ -11,7 +11,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/godbus/dbus/v5 v5.1.0 github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.3 + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-plugin v1.8.0 github.com/jackc/pgx/v5 v5.9.2 @@ -45,6 +45,10 @@ require ( google.golang.org/protobuf v1.36.11 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.35.5 + k8s.io/apimachinery v0.35.5 + k8s.io/client-go v0.35.5 + sigs.k8s.io/yaml v1.6.0 ) require ( @@ -62,18 +66,24 @@ require ( github.com/dlclark/regexp2/v2 v2.5.1 // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/fatih/color v1.13.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/yamux v0.1.2 // indirect @@ -81,6 +91,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/klauspost/cpuid/v2 v2.2.4 // indirect @@ -92,8 +103,11 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/nats-io/nkeys v0.4.6 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/oklog/run v1.1.0 // indirect @@ -104,11 +118,12 @@ require ( github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.7.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/pflag v1.0.9 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 // indirect @@ -116,12 +131,24 @@ require ( go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/multierr v1.10.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.3.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/term v0.43.0 // indirect golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.3 // indirect ) replace github.com/coder/acp-go-sdk => github.com/kdlbs/acp-go-sdk v0.13.6-0.20260821092416-13e3f4dc12c2 diff --git a/apps/backend/go.sum b/apps/backend/go.sum index abcb22a6b4..027bc75282 100644 --- a/apps/backend/go.sum +++ b/apps/backend/go.sum @@ -40,12 +40,14 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/UserExistsError/conpty v0.1.4 h1:+3FhJhiqhyEJa+K5qaK3/w6w+sN3Nh9O9VbJyBS02to= github.com/UserExistsError/conpty v0.1.4/go.mod h1:PDglKIkX3O/2xVk0MV9a6bCWxRmPVfxqZoTG/5sSd9I= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= @@ -76,6 +78,7 @@ github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -91,6 +94,8 @@ github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -105,6 +110,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -119,6 +126,14 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -162,6 +177,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -194,8 +211,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= @@ -224,6 +241,7 @@ github.com/jhump/protoreflect v1.17.0 h1:qOEr613fac2lOuTgWN4tPAtLL7fUSbuJL5X5Xum github.com/jhump/protoreflect v1.17.0/go.mod h1:h9+vUUL38jiBzck8ck+6G/aeMX8Z4QUY/NiJPwPNi+8= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -242,6 +260,7 @@ github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -276,11 +295,18 @@ github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/nats.go v1.31.0 h1:/WFBHEc/dOKBF6qf1TZhrdEfTmOZ5JzdJ+Y3m6Y/p7E= github.com/nats-io/nats.go v1.31.0/go.mod h1:di3Bm5MLsoB4Bx61CBTsxuarI36WbhAwOm8QrW39+i8= github.com/nats-io/nkeys v0.4.6 h1:IzVe95ru2CT6ta874rt9saQRkWfe2nFj1NtvYSLqMzY= @@ -315,13 +341,15 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.16.0 h1:rGGH0XDZhdUOryiDWjmIvUSWpbNqisK8Wk0Vyefw8hc= github.com/spf13/viper v1.16.0/go.mod h1:yg78JgCJcbrQOvV9YLXgkLaZqUidkY9K+Dd1FofRzQg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -349,6 +377,8 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -389,6 +419,10 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -477,6 +511,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -550,6 +586,8 @@ golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -704,6 +742,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= @@ -721,9 +763,29 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.35.5 h1:BrFeUDGY/LBtlA1R5RoxhlYRHs76RnQBc6xbm/y7hsQ= +k8s.io/api v0.35.5/go.mod h1:xWkFhMnoPZdTAQh95Rlw3zZpUUNVlFHcuESUYd06BWM= +k8s.io/apimachinery v0.35.5 h1:lbjjjUfVeVqFbiOpyhqZHc8DhiYkWOxSNij7lHx2U8Y= +k8s.io/apimachinery v0.35.5/go.mod h1:NNi1taPOpep0jOj+oRha3mBJPqvi0hGdaV8TCqGQ+cc= +k8s.io/client-go v0.35.5 h1:wUrgqVSmFRw75bgSHY7X0G/hZM/QYpV0Hg7SYYOYpFk= +k8s.io/client-go v0.35.5/go.mod h1:Z0mDcAJsX1Y7RQfuQlJipiRtqf8Mhk2VDu1/JvRqdGo= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3 h1:u08YRbVUi59ri4YD6cg0UqNM4Dimn0sIl+wldcx5PYw= +sigs.k8s.io/structured-merge-diff/v6 v6.3.3/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/apps/backend/internal/agent/executor/executor.go b/apps/backend/internal/agent/executor/executor.go index 44b566c35c..dd31f8e2cc 100644 --- a/apps/backend/internal/agent/executor/executor.go +++ b/apps/backend/internal/agent/executor/executor.go @@ -19,6 +19,7 @@ const ( NameRemoteDocker = agentruntime.RuntimeRemoteDocker NameSprites = agentruntime.RuntimeSprites NameSSH = agentruntime.RuntimeSSH + NameKubernetes = agentruntime.RuntimeKubernetes ) // ExecutorTypeToBackend maps an ExecutorType to its corresponding executor Name. @@ -36,6 +37,8 @@ func ExecutorTypeToBackend(execType models.ExecutorType) Name { return NameSprites case models.ExecutorTypeSSH: return NameSSH + case models.ExecutorTypeKubernetes: + return NameKubernetes case models.ExecutorTypeMockRemote: return NameStandalone default: diff --git a/apps/backend/internal/agent/executor/executor_test.go b/apps/backend/internal/agent/executor/executor_test.go new file mode 100644 index 0000000000..158bb14007 --- /dev/null +++ b/apps/backend/internal/agent/executor/executor_test.go @@ -0,0 +1,17 @@ +package executor + +import ( + "testing" + + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +func TestExecutorTypeToBackendMapsKubernetes(t *testing.T) { + t.Parallel() + + got := ExecutorTypeToBackend(models.ExecutorTypeKubernetes) + if got != agentruntime.RuntimeKubernetes { + t.Fatalf("ExecutorTypeToBackend(k8s) = %q, want k8s", got) + } +} diff --git a/apps/backend/internal/agent/handlers/git_handlers.go b/apps/backend/internal/agent/handlers/git_handlers.go index 7ddf0a685f..512ea8e359 100644 --- a/apps/backend/internal/agent/handlers/git_handlers.go +++ b/apps/backend/internal/agent/handlers/git_handlers.go @@ -305,7 +305,8 @@ func (h *GitHandlers) wsPull(ctx context.Context, msg *ws.Message) (*ws.Message, return nil, fmt.Errorf("session_id is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -330,7 +331,8 @@ func (h *GitHandlers) wsPush(ctx context.Context, msg *ws.Message) (*ws.Message, return nil, fmt.Errorf("session_id is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -368,7 +370,8 @@ func (h *GitHandlers) wsContribution(ctx context.Context, msg *ws.Message, opera return nil, fmt.Errorf("expected_remote_head is required") } - agentClient, err := h.getAgentCtlClient(ctx, req.SessionID) + agentClient, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -394,7 +397,8 @@ func (h *GitHandlers) wsRebase(ctx context.Context, msg *ws.Message) (*ws.Messag return nil, fmt.Errorf("base_branch is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -422,7 +426,8 @@ func (h *GitHandlers) wsMerge(ctx context.Context, msg *ws.Message) (*ws.Message return nil, fmt.Errorf("base_branch is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -450,7 +455,8 @@ func (h *GitHandlers) wsAbort(ctx context.Context, msg *ws.Message) (*ws.Message return nil, fmt.Errorf("operation must be 'merge' or 'rebase'") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -477,7 +483,8 @@ func (h *GitHandlers) wsCommit(ctx context.Context, msg *ws.Message) (*ws.Messag return nil, fmt.Errorf("message is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -505,7 +512,8 @@ func (h *GitHandlers) wsRenameBranch(ctx context.Context, msg *ws.Message) (*ws. return nil, fmt.Errorf("new_name is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -542,7 +550,8 @@ func (h *GitHandlers) wsReset(ctx context.Context, msg *ws.Message) (*ws.Message return nil, fmt.Errorf("invalid reset mode: %s (must be soft, mixed, or hard)", req.Mode) } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -566,7 +575,8 @@ func (h *GitHandlers) wsStage(ctx context.Context, msg *ws.Message) (*ws.Message return nil, fmt.Errorf("session_id is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -590,7 +600,8 @@ func (h *GitHandlers) wsUnstage(ctx context.Context, msg *ws.Message) (*ws.Messa return nil, fmt.Errorf("session_id is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -618,7 +629,8 @@ func (h *GitHandlers) wsDiscard(ctx context.Context, msg *ws.Message) (*ws.Messa return nil, fmt.Errorf("paths are required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -645,7 +657,8 @@ func (h *GitHandlers) wsCreatePR(ctx context.Context, msg *ws.Message) (*ws.Mess return nil, fmt.Errorf("title is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -696,7 +709,8 @@ func (h *GitHandlers) wsRevertCommit(ctx context.Context, msg *ws.Message) (*ws. return nil, fmt.Errorf("commit_sha is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { return nil, err } @@ -724,7 +738,8 @@ func (h *GitHandlers) wsCommitDiff(ctx context.Context, msg *ws.Message) (*ws.Me return nil, fmt.Errorf("commit_sha is required") } - client, err := h.getAgentCtlClient(ctx, req.SessionID) + client, releaseClient, err := h.getAgentCtlClient(ctx, req.SessionID) + defer releaseClient() if err != nil { if isSessionNotReadyError(err) { return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ @@ -757,18 +772,19 @@ func isSessionNotReadyError(err error) bool { // getAgentCtlClient gets the agentctl client for a session. // Uses GetOrEnsureExecution so git operations survive backend restarts — // they're workspace-oriented and don't require a running agent process. -func (h *GitHandlers) getAgentCtlClient(ctx context.Context, sessionID string) (*client.Client, error) { +func (h *GitHandlers) getAgentCtlClient(ctx context.Context, sessionID string) (*client.Client, func(), error) { + noRelease := func() {} execution, err := h.lifecycleMgr.GetOrEnsureExecution(ctx, sessionID) if err != nil { - return nil, fmt.Errorf("no agent running for session %s: %w", sessionID, err) + return nil, noRelease, fmt.Errorf("no agent running for session %s: %w", sessionID, err) } - c := execution.GetAgentCtlClient() + c, releaseClient := execution.AcquireAgentCtlClient() if c == nil { - return nil, fmt.Errorf("agent client not available for session %s", sessionID) + return nil, releaseClient, fmt.Errorf("agent client not available for session %s", sessionID) } - return c, nil + return c, releaseClient, nil } // GitCommitsRequest for session.git.commits action @@ -845,7 +861,8 @@ func (h *GitHandlers) computeGitCommits(ctx context.Context, req *GitCommitsRequ } return nil, fmt.Errorf("failed to get execution for session %s: %w", req.SessionID, err) } - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { return map[string]any{"commits": []any{}, "ready": false}, nil } @@ -927,7 +944,8 @@ func (h *GitHandlers) computeCumulativeDiff(ctx context.Context, req *Cumulative } return nil, fmt.Errorf("failed to get execution for session %s: %w", req.SessionID, err) } - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { return map[string]any{"cumulative_diff": nil, "ready": false}, nil } diff --git a/apps/backend/internal/agent/handlers/port_handlers.go b/apps/backend/internal/agent/handlers/port_handlers.go index 7ff29781ef..1077203508 100644 --- a/apps/backend/internal/agent/handlers/port_handlers.go +++ b/apps/backend/internal/agent/handlers/port_handlers.go @@ -65,7 +65,8 @@ func (h *PortHandlers) wsPortList(ctx context.Context, msg *ws.Message) (*ws.Mes return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "agentctl client not available", nil) } diff --git a/apps/backend/internal/agent/handlers/review_change_source.go b/apps/backend/internal/agent/handlers/review_change_source.go index 92e6b83612..40093c76d1 100644 --- a/apps/backend/internal/agent/handlers/review_change_source.go +++ b/apps/backend/internal/agent/handlers/review_change_source.go @@ -34,7 +34,12 @@ func (s *ReviewChangeSource) UncommittedFiles(ctx context.Context, sessionID str if err != nil { return nil, err } - status, err := execution.GetAgentCtlClient().GetGitStatus(ctx) + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return nil, fmt.Errorf("session %s workspace is not ready", sessionID) + } + status, err := client.GetGitStatus(ctx) if err != nil { return nil, fmt.Errorf("git status for session %s: %w", sessionID, err) } @@ -54,7 +59,11 @@ func (s *ReviewChangeSource) CommittedFiles(ctx context.Context, sessionID strin if err != nil { return nil, err } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return nil, fmt.Errorf("session %s workspace is not ready", sessionID) + } baseCommit, targetBranch := "", "" if s.sessionReader != nil { @@ -89,7 +98,12 @@ func (s *ReviewChangeSource) execution(ctx context.Context, sessionID string) (* if err != nil { return nil, fmt.Errorf("execution for session %s: %w", sessionID, err) } - if execution == nil || execution.GetAgentCtlClient() == nil { + if execution == nil { + return nil, fmt.Errorf("session %s workspace is not ready", sessionID) + } + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { return nil, fmt.Errorf("session %s workspace is not ready", sessionID) } return execution, nil diff --git a/apps/backend/internal/agent/handlers/shell_handlers.go b/apps/backend/internal/agent/handlers/shell_handlers.go index 3e96d3d99c..890a1cd359 100644 --- a/apps/backend/internal/agent/handlers/shell_handlers.go +++ b/apps/backend/internal/agent/handlers/shell_handlers.go @@ -122,7 +122,8 @@ func (h *ShellHandlers) wsShellStatus(ctx context.Context, msg *ws.Message) (*ws } // Get shell status from agentctl - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewResponse(msg.ID, msg.Action, map[string]interface{}{ "available": false, @@ -175,7 +176,9 @@ func (h *ShellHandlers) wsShellSubscribe(ctx context.Context, msg *ws.Message) ( // This ensures client gets current shell state without duplicate broadcasts // Shell output streaming is handled by the lifecycle manager via event bus buffer := "" - if client := execution.GetAgentCtlClient(); client != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client != nil { if b, err := client.ShellBuffer(ctx); err == nil { buffer = b } diff --git a/apps/backend/internal/agent/handlers/vscode_handlers.go b/apps/backend/internal/agent/handlers/vscode_handlers.go index 0df5adcb92..75a356eb29 100644 --- a/apps/backend/internal/agent/handlers/vscode_handlers.go +++ b/apps/backend/internal/agent/handlers/vscode_handlers.go @@ -79,7 +79,8 @@ func (h *VscodeHandlers) wsVscodeStart(ctx context.Context, msg *ws.Message) (*w return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "agentctl client not available", nil) } @@ -127,7 +128,8 @@ func (h *VscodeHandlers) wsVscodeStop(ctx context.Context, msg *ws.Message) (*ws return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "agentctl client not available", nil) } @@ -165,7 +167,8 @@ func (h *VscodeHandlers) wsVscodeStatus(ctx context.Context, msg *ws.Message) (* return ws.NewResponse(msg.ID, msg.Action, types.VscodeStatusResponse{Status: "stopped"}) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewResponse(msg.ID, msg.Action, types.VscodeStatusResponse{Status: "stopped"}) } @@ -225,7 +228,8 @@ func (h *VscodeHandlers) wsVscodeOpenFile(ctx context.Context, msg *ws.Message) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "session not found or no active execution: "+err.Error(), nil) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "agentctl client not available", nil) } diff --git a/apps/backend/internal/agent/handlers/workspace_file_handlers.go b/apps/backend/internal/agent/handlers/workspace_file_handlers.go index 941475aeff..3b616125b2 100644 --- a/apps/backend/internal/agent/handlers/workspace_file_handlers.go +++ b/apps/backend/internal/agent/handlers/workspace_file_handlers.go @@ -69,7 +69,8 @@ func (h *WorkspaceFileHandlers) wsGetFileTree(ctx context.Context, msg *ws.Messa } // Get agentctl client - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) } @@ -110,7 +111,8 @@ func (h *WorkspaceFileHandlers) wsGetFileContent(ctx context.Context, msg *ws.Me } // Get agentctl client - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) } @@ -166,7 +168,8 @@ func (h *WorkspaceFileHandlers) wsGetFileContentAtRef(ctx context.Context, msg * } // Get agentctl client - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) } @@ -219,7 +222,8 @@ func (h *WorkspaceFileHandlers) wsUpdateFileContent(ctx context.Context, msg *ws } // Get agentctl client - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) } @@ -238,7 +242,8 @@ func (h *WorkspaceFileHandlers) wsUpdateFileContent(ctx context.Context, msg *ws // and returns the agentctl client, or an error response. func (h *WorkspaceFileHandlers) resolveSessionFileClient( ctx context.Context, msg *ws.Message, -) (sessionID, path, repo string, client *agentctl.Client, errResp *ws.Message) { +) (sessionID, path, repo string, client *agentctl.Client, release func(), errResp *ws.Message) { + noRelease := func() {} var req struct { SessionID string `json:"session_id"` Path string `json:"path"` @@ -247,33 +252,34 @@ func (h *WorkspaceFileHandlers) resolveSessionFileClient( if err := msg.ParsePayload(&req); err != nil { errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "Invalid payload: "+err.Error(), nil) - return "", "", "", nil, errResp + return "", "", "", nil, noRelease, errResp } if req.SessionID == "" { errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "session_id is required", nil) - return "", "", "", nil, errResp + return "", "", "", nil, noRelease, errResp } if req.Path == "" { errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "path is required", nil) - return "", "", "", nil, errResp + return "", "", "", nil, noRelease, errResp } execution, resolveErr := h.lifecycle.GetOrEnsureExecution(ctx, req.SessionID) if resolveErr != nil { errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeNotFound, "No agent found for session: "+resolveErr.Error(), nil) - return "", "", "", nil, errResp + return "", "", "", nil, noRelease, errResp } - c := execution.GetAgentCtlClient() + c, releaseClient := execution.AcquireAgentCtlClient() if c == nil { errResp, _ = ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) - return "", "", "", nil, errResp + return "", "", "", nil, releaseClient, errResp } - return req.SessionID, req.Path, req.Repo, c, nil + return req.SessionID, req.Path, req.Repo, c, releaseClient, nil } // wsCreateFile handles workspace.file.create action func (h *WorkspaceFileHandlers) wsCreateFile(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - sessionID, path, repo, client, errResp := h.resolveSessionFileClient(ctx, msg) + sessionID, path, repo, client, releaseClient, errResp := h.resolveSessionFileClient(ctx, msg) + defer releaseClient() if errResp != nil { return errResp, nil } @@ -289,7 +295,8 @@ func (h *WorkspaceFileHandlers) wsCreateFile(ctx context.Context, msg *ws.Messag // wsDeleteFile handles workspace.file.delete action func (h *WorkspaceFileHandlers) wsDeleteFile(ctx context.Context, msg *ws.Message) (*ws.Message, error) { - sessionID, path, repo, client, errResp := h.resolveSessionFileClient(ctx, msg) + sessionID, path, repo, client, releaseClient, errResp := h.resolveSessionFileClient(ctx, msg) + defer releaseClient() if errResp != nil { return errResp, nil } @@ -326,7 +333,8 @@ func (h *WorkspaceFileHandlers) wsSearchFiles(ctx context.Context, msg *ws.Messa } // Get agentctl client - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) } @@ -379,7 +387,8 @@ func (h *WorkspaceFileHandlers) wsSearchContent( "No agent found for session: "+err.Error(), nil, ) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError( msg.ID, msg.Action, ws.ErrorCodeInternalError, @@ -439,7 +448,8 @@ func (h *WorkspaceFileHandlers) wsRenameFile(ctx context.Context, msg *ws.Messag } // Get agentctl client - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Agent client not available", nil) } diff --git a/apps/backend/internal/agent/kubernetes/admission.go b/apps/backend/internal/agent/kubernetes/admission.go new file mode 100644 index 0000000000..48c50a758d --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/admission.go @@ -0,0 +1,472 @@ +package kubernetes + +import ( + "errors" + "fmt" + "reflect" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ValidateAdmittedPod verifies the fields Kandev owns after API admission. +// Unrelated labels, annotations, scheduling selectors, and server metadata are +// intentionally allowed so normal admission/defaulting can still operate. +func ValidateAdmittedPod(admitted, desired *corev1.Pod, mainContainer string) error { + if admitted == nil || desired == nil { + return errors.New("admitted Pod validation requires both objects") + } + if err := validateAdmittedMetadata("Pod", admitted.ObjectMeta, desired.ObjectMeta); err != nil { + return err + } + if err := validateAdmittedPodSpec(admitted, desired); err != nil { + return err + } + desiredMain := admittedContainerByName(desired.Spec.Containers, mainContainer) + admittedMain := admittedContainerByName(admitted.Spec.Containers, mainContainer) + if desiredMain == nil || admittedMain == nil { + return errors.New("admitted Pod is missing the owned main container") + } + if !reflect.DeepEqual(admittedMain.Command, desiredMain.Command) || + !reflect.DeepEqual(admittedMain.Args, desiredMain.Args) || + admittedMain.WorkingDir != desiredMain.WorkingDir { + return errors.New("admitted Pod mutated the main-container bootstrap") + } + if err := validateAdmittedContainers(admitted, desiredMain, mainContainer); err != nil { + return err + } + return validateAdmittedVolumes(admitted.Spec.Volumes, desired.Spec.Volumes) +} + +// ValidateAdmittedPVC verifies the Kandev-owned shape of a managed claim while +// allowing ordinary API binding fields and default storage-class selection. +func ValidateAdmittedPVC(admitted, desired *corev1.PersistentVolumeClaim) error { + if admitted == nil || desired == nil { + return errors.New("admitted PVC validation requires both objects") + } + if err := validateAdmittedMetadata("PVC", admitted.ObjectMeta, desired.ObjectMeta); err != nil { + return err + } + if !samePersistentVolumeAccessModes(admitted.Spec.AccessModes, desired.Spec.AccessModes) || + !reflect.DeepEqual(admitted.Spec.VolumeMode, desired.Spec.VolumeMode) || + !sameVolumeResourceRequirements(admitted.Spec.Resources, desired.Spec.Resources) { + return errors.New("admitted PVC mutated owned storage requirements") + } + if desired.Spec.StorageClassName != nil && + !reflect.DeepEqual(admitted.Spec.StorageClassName, desired.Spec.StorageClassName) { + return errors.New("admitted PVC mutated the explicit storage class") + } + if !reflect.DeepEqual(admitted.Spec.VolumeAttributesClassName, desired.Spec.VolumeAttributesClassName) { + return errors.New("admitted PVC mutated the volume attributes class") + } + if !reflect.DeepEqual(admitted.Spec.Selector, desired.Spec.Selector) || + !reflect.DeepEqual(admitted.Spec.DataSource, desired.Spec.DataSource) || + !reflect.DeepEqual(admitted.Spec.DataSourceRef, desired.Spec.DataSourceRef) { + return errors.New("admitted PVC mutated selector or data-source semantics") + } + return nil +} + +// ValidateOwnershipLabels verifies all Kandev-owned standard and custom +// ownership labels while allowing unrelated labels added by the API server or +// cluster policy. +func ValidateOwnershipLabels(admitted, desired map[string]string) error { + return validateAdmittedOwnedLabels("resource", admitted, desired) +} + +func validateAdmittedMetadata(kind string, admitted, desired metav1.ObjectMeta) error { + if admitted.Name != desired.Name || admitted.Namespace != desired.Namespace { + return fmt.Errorf("admitted %s name or namespace changed", kind) + } + if len(admitted.OwnerReferences) != 0 { + return fmt.Errorf("admitted %s added an owner reference", kind) + } + if !admittedFinalizersAllowed(kind, admitted.Finalizers) { + return fmt.Errorf("admitted %s added a finalizer", kind) + } + if err := validateAdmittedOwnedLabels(kind, admitted.Labels, desired.Labels); err != nil { + return err + } + return validateAdmittedOwnedAnnotations(kind, admitted.Annotations, desired.Annotations) +} + +func validateAdmittedOwnedLabels(kind string, admitted, desired map[string]string) error { + for key, desiredValue := range desired { + if !isOwnedMetadataLabel(key) { + continue + } + if admitted[key] != desiredValue { + return fmt.Errorf("admitted %s ownership label %q changed", kind, key) + } + } + for key, admittedValue := range admitted { + if !isOwnedMetadataLabel(key) { + continue + } + desiredValue, exists := desired[key] + if !exists || admittedValue != desiredValue { + return fmt.Errorf("admitted %s added or changed ownership label %q", kind, key) + } + } + return nil +} + +func validateAdmittedOwnedAnnotations(kind string, admitted, desired map[string]string) error { + for key, desiredValue := range desired { + if !strings.HasPrefix(key, "kandev.ai/") { + continue + } + if admitted[key] != desiredValue { + return fmt.Errorf("admitted %s owned annotation %q changed", kind, key) + } + } + for key, admittedValue := range admitted { + if !strings.HasPrefix(key, "kandev.ai/") { + continue + } + desiredValue, exists := desired[key] + if !exists || admittedValue != desiredValue { + return fmt.Errorf("admitted %s added or changed owned annotation %q", kind, key) + } + } + return nil +} + +func isOwnedMetadataLabel(key string) bool { + return reservedLabelKeys[key] || strings.HasPrefix(key, "kandev.ai/") +} + +func admittedFinalizersAllowed(kind string, finalizers []string) bool { + if len(finalizers) == 0 { + return true + } + return kind == "PVC" && len(finalizers) == 1 && finalizers[0] == "kubernetes.io/pvc-protection" +} + +func validateAdmittedPodSpec(admitted, desired *corev1.Pod) error { + if admitted.Spec.RestartPolicy != desired.Spec.RestartPolicy || + admitted.Spec.NodeName != desired.Spec.NodeName || + !reflect.DeepEqual(admitted.Spec.OS, desired.Spec.OS) || + !reflect.DeepEqual(admitted.Spec.AutomountServiceAccountToken, desired.Spec.AutomountServiceAccountToken) { + return errors.New("admitted Pod mutated an owned Pod invariant") + } + for _, key := range []string{corev1.LabelOSStable, corev1.LabelArchStable} { + if admitted.Spec.NodeSelector[key] != desired.Spec.NodeSelector[key] { + return fmt.Errorf("admitted Pod mutated owned node selector %q", key) + } + } + return validateAdmittedNodeAffinity(admitted.Spec.Affinity, desired.Spec.Affinity, desired.Spec.NodeSelector) +} + +type admittedNodeRequirement struct { + requirement corev1.NodeSelectorRequirement + path string + preferred bool + matchField bool +} + +func validateAdmittedNodeAffinity( + admitted, desired *corev1.Affinity, + nodeSelector map[string]string, +) error { + desiredRequirements := platformNodeRequirements(desired) + matchedDesired := make([]bool, len(desiredRequirements)) + for _, admittedRequirement := range platformNodeRequirements(admitted) { + if matchDesiredNodeRequirement(admittedRequirement, desiredRequirements, matchedDesired) { + continue + } + if admittedRequirement.matchField || + !platformRequirementAllows(admittedRequirement.requirement, nodeSelector) { + return fmt.Errorf("admitted Pod added conflicting platform node affinity at %s", admittedRequirement.path) + } + } + for _, matched := range matchedDesired { + if !matched { + return errors.New("admitted Pod mutated desired platform node affinity") + } + } + return nil +} + +func matchDesiredNodeRequirement( + admitted admittedNodeRequirement, + desired []admittedNodeRequirement, + matched []bool, +) bool { + for index, candidate := range desired { + if matched[index] || admitted.preferred != candidate.preferred || admitted.matchField != candidate.matchField { + continue + } + if reflect.DeepEqual(admitted.requirement, candidate.requirement) { + matched[index] = true + return true + } + } + return false +} + +func platformNodeRequirements(affinity *corev1.Affinity) []admittedNodeRequirement { + if affinity == nil || affinity.NodeAffinity == nil { + return nil + } + var requirements []admittedNodeRequirement + nodeAffinity := affinity.NodeAffinity + if required := nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution; required != nil { + for termIndex, term := range required.NodeSelectorTerms { + requirements = appendPlatformNodeRequirements(requirements, term, false, termIndex) + } + } + for termIndex, term := range nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + requirements = appendPlatformNodeRequirements(requirements, term.Preference, true, termIndex) + } + return requirements +} + +func appendPlatformNodeRequirements( + result []admittedNodeRequirement, + term corev1.NodeSelectorTerm, + preferred bool, + termIndex int, +) []admittedNodeRequirement { + prefix := "required" + if preferred { + prefix = "preferred" + } + for index, requirement := range term.MatchExpressions { + if isPlatformNodeKey(requirement.Key) { + result = append(result, admittedNodeRequirement{ + requirement: requirement, + path: fmt.Sprintf("%s[%d].matchExpressions[%d]", prefix, termIndex, index), + preferred: preferred, + }) + } + } + for index, requirement := range term.MatchFields { + if isPlatformNodeKey(requirement.Key) { + result = append(result, admittedNodeRequirement{ + requirement: requirement, + path: fmt.Sprintf("%s[%d].matchFields[%d]", prefix, termIndex, index), + preferred: preferred, + matchField: true, + }) + } + } + return result +} + +func platformRequirementAllows(requirement corev1.NodeSelectorRequirement, nodeSelector map[string]string) bool { + want, ok := nodeSelector[requirement.Key] + if !ok || want == "" { + return false + } + switch requirement.Operator { + case corev1.NodeSelectorOpIn: + return stringSliceContains(requirement.Values, want) + case corev1.NodeSelectorOpNotIn: + return !stringSliceContains(requirement.Values, want) + case corev1.NodeSelectorOpExists: + return true + default: + return false + } +} + +func isPlatformNodeKey(key string) bool { + return key == corev1.LabelOSStable || key == corev1.LabelArchStable +} + +func stringSliceContains(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} + +func validateAdmittedContainers( + admitted *corev1.Pod, + desiredMain *corev1.Container, + mainContainer string, +) error { + for index := range admitted.Spec.Containers { + container := &admitted.Spec.Containers[index] + if err := validateAdmittedContainer(container, desiredMain, container.Name == mainContainer); err != nil { + return err + } + } + for index := range admitted.Spec.InitContainers { + if err := validateAdmittedContainer(&admitted.Spec.InitContainers[index], desiredMain, false); err != nil { + return err + } + } + for index := range admitted.Spec.EphemeralContainers { + container := corev1.Container(admitted.Spec.EphemeralContainers[index].EphemeralContainerCommon) + if err := validateAdmittedContainer(&container, desiredMain, false); err != nil { + return err + } + } + main := admittedContainerByName(admitted.Spec.Containers, mainContainer) + return validateRequiredMainContainerFields(main, desiredMain) +} + +func validateAdmittedContainer(container, desiredMain *corev1.Container, isMain bool) error { + if err := validateAdmittedContainerEnvironment(container); err != nil { + return err + } + if err := validateAdmittedContainerPorts(container, desiredMain, isMain); err != nil { + return err + } + if err := validateAdmittedContainerMounts(container, desiredMain, isMain); err != nil { + return err + } + return validateAdmittedContainerDevices(container) +} + +func validateAdmittedContainerEnvironment(container *corev1.Container) error { + if len(container.EnvFrom) > 0 { + return errors.New("admitted Pod uses envFrom, which can inject reserved environment keys") + } + for _, env := range container.Env { + if isReservedEnvironmentKey(env.Name) { + return errors.New("admitted Pod added a reserved environment key") + } + } + return nil +} + +func validateAdmittedContainerPorts(container, desiredMain *corev1.Container, isMain bool) error { + for _, port := range container.Ports { + if port.Name != AgentctlContainerPort && port.ContainerPort != DefaultAgentctlPort { + continue + } + if !isMain || countContainerPorts(desiredMain.Ports, port) != 1 { + return errors.New("admitted Pod mutated the agentctl port") + } + } + return nil +} + +func validateAdmittedContainerMounts(container, desiredMain *corev1.Container, isMain bool) error { + for _, mount := range container.VolumeMounts { + if !isReservedVolumeName(mount.Name) && !isReservedMountPath(mount.MountPath) { + continue + } + if !isMain || countVolumeMounts(desiredMain.VolumeMounts, mount) != 1 { + return errors.New("admitted Pod mutated a reserved volume mount") + } + } + return nil +} + +func validateAdmittedContainerDevices(container *corev1.Container) error { + for _, device := range container.VolumeDevices { + if isReservedVolumeName(device.Name) || isReservedMountPath(device.DevicePath) { + return errors.New("admitted Pod added a reserved volume device") + } + } + return nil +} + +func validateRequiredMainContainerFields(admitted, desired *corev1.Container) error { + if admitted == nil || desired == nil { + return errors.New("admitted Pod is missing the owned main container") + } + for _, required := range desired.Ports { + if required.Name == AgentctlContainerPort && countContainerPorts(admitted.Ports, required) != 1 { + return errors.New("admitted Pod mutated the agentctl port") + } + } + for _, required := range desired.VolumeMounts { + if isReservedVolumeName(required.Name) && countVolumeMounts(admitted.VolumeMounts, required) != 1 { + return errors.New("admitted Pod mutated a reserved volume mount") + } + } + return nil +} + +func validateAdmittedVolumes(admitted, desired []corev1.Volume) error { + for _, required := range desired { + if isReservedVolumeName(required.Name) && countVolumes(admitted, required) != 1 { + return errors.New("admitted Pod mutated a reserved volume") + } + } + for _, volume := range admitted { + if !isReservedVolumeName(volume.Name) { + continue + } + if countVolumes(desired, volume) != 1 { + return errors.New("admitted Pod added or mutated a reserved volume") + } + } + return nil +} + +func admittedContainerByName(containers []corev1.Container, name string) *corev1.Container { + for index := range containers { + if containers[index].Name == name { + return &containers[index] + } + } + return nil +} + +func countContainerPorts(values []corev1.ContainerPort, want corev1.ContainerPort) int { + count := 0 + for _, value := range values { + if reflect.DeepEqual(value, want) { + count++ + } + } + return count +} + +func countVolumeMounts(values []corev1.VolumeMount, want corev1.VolumeMount) int { + count := 0 + for _, value := range values { + if reflect.DeepEqual(value, want) { + count++ + } + } + return count +} + +func countVolumes(values []corev1.Volume, want corev1.Volume) int { + count := 0 + for _, value := range values { + if reflect.DeepEqual(value, want) { + count++ + } + } + return count +} + +func samePersistentVolumeAccessModes(left, right []corev1.PersistentVolumeAccessMode) bool { + leftSet := make(map[corev1.PersistentVolumeAccessMode]struct{}, len(left)) + for _, mode := range left { + leftSet[mode] = struct{}{} + } + rightSet := make(map[corev1.PersistentVolumeAccessMode]struct{}, len(right)) + for _, mode := range right { + rightSet[mode] = struct{}{} + } + return reflect.DeepEqual(leftSet, rightSet) +} + +func sameVolumeResourceRequirements(left, right corev1.VolumeResourceRequirements) bool { + return sameResourceList(left.Requests, right.Requests) && sameResourceList(left.Limits, right.Limits) +} + +func sameResourceList(left, right corev1.ResourceList) bool { + if len(left) != len(right) { + return false + } + for name, leftQuantity := range left { + rightQuantity, exists := right[name] + if !exists || leftQuantity.Cmp(rightQuantity) != 0 { + return false + } + } + return true +} diff --git a/apps/backend/internal/agent/kubernetes/admission_test.go b/apps/backend/internal/agent/kubernetes/admission_test.go new file mode 100644 index 0000000000..94a74bf27d --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/admission_test.go @@ -0,0 +1,310 @@ +package kubernetes + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func TestValidateAdmittedPodAllowsUnrelatedWebhookAdditions(t *testing.T) { + desired, admitted := admittedPodFixture(t) + if admitted.Labels == nil { + admitted.Labels = make(map[string]string) + } + if admitted.Annotations == nil { + admitted.Annotations = make(map[string]string) + } + if admitted.Spec.NodeSelector == nil { + admitted.Spec.NodeSelector = make(map[string]string) + } + admitted.Labels["mesh.example/injected"] = "true" + admitted.Annotations["mesh.example/status"] = "ready" + admitted.ManagedFields = []metav1.ManagedFieldsEntry{{Manager: "api-server"}} + admitted.Spec.NodeSelector["topology.kubernetes.io/zone"] = "zone-a" + admitted.Spec.Affinity = &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{{ + Key: "node.example/class", Operator: corev1.NodeSelectorOpIn, Values: []string{"agents"}, + }}, + }}}, + }} + + if err := ValidateAdmittedPod(admitted, desired, DefaultMainContainerName); err != nil { + t.Fatalf("ValidateAdmittedPod() error = %v", err) + } +} + +func TestValidateAdmittedPodAllowsCompatiblePlatformAffinity(t *testing.T) { + tests := []struct { + name string + key string + operator corev1.NodeSelectorOperator + values []string + preferred bool + }{ + {name: "required in includes linux", key: corev1.LabelOSStable, operator: corev1.NodeSelectorOpIn, values: []string{"linux", "other"}}, + {name: "required exists", key: corev1.LabelArchStable, operator: corev1.NodeSelectorOpExists}, + {name: "preferred not-in keeps amd64", key: corev1.LabelArchStable, operator: corev1.NodeSelectorOpNotIn, values: []string{"arm64"}, preferred: true}, + {name: "preferred in includes amd64", key: corev1.LabelArchStable, operator: corev1.NodeSelectorOpIn, values: []string{"amd64"}, preferred: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + desired, admitted := admittedPodFixture(t) + admitted.Spec.Affinity = nodeAffinityWithRequirement(corev1.NodeSelectorRequirement{ + Key: test.key, Operator: test.operator, Values: test.values, + }, test.preferred) + + if err := ValidateAdmittedPod(admitted, desired, DefaultMainContainerName); err != nil { + t.Fatalf("ValidateAdmittedPod() error = %v", err) + } + }) + } +} + +func TestValidateAdmittedPodRejectsOwnedMutations(t *testing.T) { + mutations := []struct { + name string + mutate func(*corev1.Pod) + }{ + {name: "name", mutate: func(p *corev1.Pod) { p.Name = "other" }}, + {name: "namespace", mutate: func(p *corev1.Pod) { p.Namespace = "other" }}, + {name: "ownership label", mutate: func(p *corev1.Pod) { p.Labels["kandev.ai/session-id"] = "other" }}, + {name: "standard ownership label", mutate: func(p *corev1.Pod) { + p.Labels["app.kubernetes.io/instance"] = "other" + }}, + {name: "extra ownership label", mutate: func(p *corev1.Pod) { p.Labels["kandev.ai/forged"] = "true" }}, + {name: "reserved annotation", mutate: func(p *corev1.Pod) { p.Annotations["kandev.ai/owner"] = "other" }}, + {name: "owner reference", mutate: func(p *corev1.Pod) { + p.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "owner", UID: types.UID("owner")}} + }}, + {name: "finalizer", mutate: func(p *corev1.Pod) { p.Finalizers = []string{"example.test/hold"} }}, + {name: "restart policy", mutate: func(p *corev1.Pod) { p.Spec.RestartPolicy = corev1.RestartPolicyNever }}, + {name: "os removed", mutate: func(p *corev1.Pod) { p.Spec.OS = nil }}, + {name: "os changed", mutate: func(p *corev1.Pod) { p.Spec.OS = &corev1.PodOS{Name: "windows"} }}, + {name: "automount", mutate: func(p *corev1.Pod) { value := true; p.Spec.AutomountServiceAccountToken = &value }}, + {name: "os selector", mutate: func(p *corev1.Pod) { p.Spec.NodeSelector[corev1.LabelOSStable] = "windows" }}, + {name: "arch selector", mutate: func(p *corev1.Pod) { p.Spec.NodeSelector[corev1.LabelArchStable] = "arm64" }}, + {name: "node name", mutate: func(p *corev1.Pod) { p.Spec.NodeName = "worker-a" }}, + {name: "required conflicting in affinity", mutate: func(p *corev1.Pod) { + p.Spec.Affinity = nodeAffinityWithRequirement(corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, Operator: corev1.NodeSelectorOpIn, Values: []string{"windows"}, + }, false) + }}, + {name: "preferred conflicting not-in affinity", mutate: func(p *corev1.Pod) { + p.Spec.Affinity = nodeAffinityWithRequirement(corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpNotIn, Values: []string{"amd64"}, + }, true) + }}, + {name: "required does-not-exist affinity", mutate: func(p *corev1.Pod) { + p.Spec.Affinity = nodeAffinityWithRequirement(corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, Operator: corev1.NodeSelectorOpDoesNotExist, + }, false) + }}, + {name: "preferred numeric affinity", mutate: func(p *corev1.Pod) { + p.Spec.Affinity = nodeAffinityWithRequirement(corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, Operator: corev1.NodeSelectorOpGt, Values: []string{"1"}, + }, true) + }}, + {name: "main command", mutate: func(p *corev1.Pod) { admittedMainContainer(p).Command = []string{"sleep"} }}, + {name: "main args", mutate: func(p *corev1.Pod) { admittedMainContainer(p).Args = []string{"infinity"} }}, + {name: "main workdir", mutate: func(p *corev1.Pod) { admittedMainContainer(p).WorkingDir = "/tmp" }}, + {name: "agentctl port", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).Ports[len(admittedMainContainer(p).Ports)-1].ContainerPort = 9999 + }}, + {name: "home env", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).Env = append(admittedMainContainer(p).Env, corev1.EnvVar{Name: "HOME", Value: "/tmp"}) + }}, + {name: "future agentctl env", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).Env = append(admittedMainContainer(p).Env, corev1.EnvVar{Name: "AGENTCTL_FUTURE", Value: "x"}) + }}, + {name: "future kandev env", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).Env = append(admittedMainContainer(p).Env, corev1.EnvVar{Name: "KANDEV_FUTURE", Value: "x"}) + }}, + {name: "main envFrom", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).EnvFrom = []corev1.EnvFromSource{{ + ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "runtime-env"}}, + }} + }}, + {name: "reserved mount", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).VolumeMounts[0].MountPath = "/tmp/runtime" + }}, + {name: "reserved volume", mutate: func(p *corev1.Pod) { p.Spec.Volumes[0].EmptyDir = nil }}, + {name: "sidecar reserved mount", mutate: func(p *corev1.Pod) { + p.Spec.Containers = append(p.Spec.Containers, corev1.Container{ + Name: "sidecar", Image: "example/sidecar", VolumeMounts: []corev1.VolumeMount{{Name: AuthVolumeName, MountPath: AuthMountPath}}, + }) + }}, + {name: "init reserved env", mutate: func(p *corev1.Pod) { + p.Spec.InitContainers = append(p.Spec.InitContainers, corev1.Container{ + Name: "init", Image: "example/init", Env: []corev1.EnvVar{{Name: "KANDEV_TASK_ID", Value: "forged"}}, + }) + }}, + {name: "ephemeral reserved mount", mutate: func(p *corev1.Pod) { + p.Spec.EphemeralContainers = append(p.Spec.EphemeralContainers, corev1.EphemeralContainer{ + EphemeralContainerCommon: corev1.EphemeralContainerCommon{ + Name: "debug", Image: "example/debug", VolumeMounts: []corev1.VolumeMount{{Name: RuntimeVolumeName, MountPath: RuntimeMountPath}}, + }, + }) + }}, + {name: "reserved volume device", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).VolumeDevices = []corev1.VolumeDevice{{Name: WorkspaceVolumeName, DevicePath: "/dev/workspace"}} + }}, + {name: "reserved volume device path", mutate: func(p *corev1.Pod) { + admittedMainContainer(p).VolumeDevices = []corev1.VolumeDevice{{Name: "foreign", DevicePath: "/opt/kandev/device"}} + }}, + } + + for _, mutation := range mutations { + t.Run(mutation.name, func(t *testing.T) { + desired, admitted := admittedPodFixture(t) + mutation.mutate(admitted) + if err := ValidateAdmittedPod(admitted, desired, DefaultMainContainerName); err == nil { + t.Fatal("ValidateAdmittedPod() error = nil") + } + }) + } +} + +func TestValidateAdmittedPVCAllowsBindingAndDefaultStorageClass(t *testing.T) { + desired, err := BuildPersistentVolumeClaim( + WorkspaceConfig{Mode: WorkspaceModeManagedPVC, Size: "1Gi", AccessModes: []string{"ReadWriteOnce", "ReadOnlyMany"}}, + PVCOptions{Name: "workspace", Namespace: "agents", Identity: podOptions().Identity}, + ) + if err != nil { + t.Fatal(err) + } + admitted := desired.DeepCopy() + admitted.UID = "pvc-uid" + admitted.ResourceVersion = "1" + admitted.ManagedFields = []metav1.ManagedFieldsEntry{{Manager: "api-server"}} + admitted.Finalizers = []string{"kubernetes.io/pvc-protection"} + admitted.Labels["storage.example/defaulted"] = "true" + admitted.Annotations = map[string]string{"storage.example/status": "bound"} + defaultClass := "default-class" + admitted.Spec.StorageClassName = &defaultClass + admitted.Spec.VolumeName = "pv-1" + admitted.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{ + corev1.ReadOnlyMany, corev1.ReadWriteOnce, + } + admitted.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("1024Mi") + admitted.Status.Phase = corev1.ClaimBound + + if err := ValidateAdmittedPVC(admitted, desired); err != nil { + t.Fatalf("ValidateAdmittedPVC() error = %v", err) + } +} + +func TestValidateAdmittedPVCRejectsOwnedMutations(t *testing.T) { + mutations := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + }{ + {name: "name", mutate: func(p *corev1.PersistentVolumeClaim) { p.Name = "other" }}, + {name: "namespace", mutate: func(p *corev1.PersistentVolumeClaim) { p.Namespace = "other" }}, + {name: "ownership label", mutate: func(p *corev1.PersistentVolumeClaim) { p.Labels["kandev.ai/session-id"] = "other" }}, + {name: "standard ownership label", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Labels["app.kubernetes.io/managed-by"] = "other" + }}, + {name: "extra ownership label", mutate: func(p *corev1.PersistentVolumeClaim) { p.Labels["kandev.ai/forged"] = "true" }}, + {name: "reserved annotation", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Annotations = map[string]string{"kandev.ai/owner": "forged"} + }}, + {name: "owner reference", mutate: func(p *corev1.PersistentVolumeClaim) { + p.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Pod", Name: "owner", UID: "owner"}} + }}, + {name: "finalizer", mutate: func(p *corev1.PersistentVolumeClaim) { p.Finalizers = []string{"example.test/hold"} }}, + {name: "extra finalizer", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Finalizers = []string{"kubernetes.io/pvc-protection", "example.test/hold"} + }}, + {name: "access modes", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }}, + {name: "volume mode", mutate: func(p *corev1.PersistentVolumeClaim) { mode := corev1.PersistentVolumeBlock; p.Spec.VolumeMode = &mode }}, + {name: "requests", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("2Gi") + }}, + {name: "explicit storage class", mutate: func(p *corev1.PersistentVolumeClaim) { value := "slow"; p.Spec.StorageClassName = &value }}, + {name: "volume attributes class", mutate: func(p *corev1.PersistentVolumeClaim) { + value := "mutated" + p.Spec.VolumeAttributesClassName = &value + }}, + {name: "selector", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"disk": "fast"}} + }}, + {name: "data source", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Spec.DataSource = &corev1.TypedLocalObjectReference{Kind: "PersistentVolumeClaim", Name: "source"} + }}, + {name: "data source ref", mutate: func(p *corev1.PersistentVolumeClaim) { + p.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "source"} + }}, + } + + for _, mutation := range mutations { + t.Run(mutation.name, func(t *testing.T) { + desired, err := BuildPersistentVolumeClaim( + WorkspaceConfig{Mode: WorkspaceModeManagedPVC, Size: "1Gi", StorageClass: "fast", AccessModes: []string{"ReadWriteOnce"}}, + PVCOptions{Name: "workspace", Namespace: "agents", Identity: podOptions().Identity}, + ) + if err != nil { + t.Fatal(err) + } + admitted := desired.DeepCopy() + admitted.UID = "pvc-uid" + mutation.mutate(admitted) + if err := ValidateAdmittedPVC(admitted, desired); err == nil { + t.Fatal("ValidateAdmittedPVC() error = nil") + } + }) + } +} + +func admittedPodFixture(t *testing.T) (*corev1.Pod, *corev1.Pod) { + t.Helper() + template, err := ParsePodTemplate(validPodTemplate("")) + if err != nil { + t.Fatal(err) + } + desired, _, err := ComposePod(template, validProfile(""), podOptions()) + if err != nil { + t.Fatal(err) + } + admitted := desired.DeepCopy() + admitted.UID = "pod-uid" + admitted.ResourceVersion = "1" + if admitted.Labels == nil { + admitted.Labels = make(map[string]string) + } + if admitted.Annotations == nil { + admitted.Annotations = make(map[string]string) + } + if admitted.Spec.NodeSelector == nil { + admitted.Spec.NodeSelector = make(map[string]string) + } + return desired, admitted +} + +func admittedMainContainer(pod *corev1.Pod) *corev1.Container { + for index := range pod.Spec.Containers { + if pod.Spec.Containers[index].Name == DefaultMainContainerName { + return &pod.Spec.Containers[index] + } + } + panic("main container missing") +} + +func nodeAffinityWithRequirement(requirement corev1.NodeSelectorRequirement, preferred bool) *corev1.Affinity { + nodeAffinity := &corev1.NodeAffinity{} + if preferred { + nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.PreferredSchedulingTerm{{ + Weight: 1, Preference: corev1.NodeSelectorTerm{MatchExpressions: []corev1.NodeSelectorRequirement{requirement}}, + }} + } else { + nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{ + MatchExpressions: []corev1.NodeSelectorRequirement{requirement}, + }}} + } + return &corev1.Affinity{NodeAffinity: nodeAffinity} +} diff --git a/apps/backend/internal/agent/kubernetes/client.go b/apps/backend/internal/agent/kubernetes/client.go new file mode 100644 index 0000000000..53bf08db26 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/client.go @@ -0,0 +1,100 @@ +package kubernetes + +import ( + "errors" + "time" + + kubeclient "k8s.io/client-go/kubernetes" + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + + "github.com/kandev/kandev/internal/agent/runtime/routingerr" +) + +const kubernetesUserAgent = "kandev-kubernetes-executor" + +type ConfigLoader struct { + Kubeconfig func(path, contextName string) (*rest.Config, error) + InCluster func() (*rest.Config, error) +} + +type ClientsetFactory func(*rest.Config) (kubeclient.Interface, error) + +type Client struct { + RESTConfig *rest.Config + Clientset kubeclient.Interface +} + +func DefaultConfigLoader() ConfigLoader { + return ConfigLoader{ + Kubeconfig: loadKubeconfig, + InCluster: rest.InClusterConfig, + } +} + +func NewClient(config ExecutorConfig, loader ConfigLoader, factory ClientsetFactory) (*Client, error) { + if err := config.Validate(); err != nil { + return nil, err + } + loader = withDefaultConfigLoader(loader) + restConfig, err := loadRESTConfig(config, loader) + if err != nil { + return nil, sanitizeClientError("load client configuration", err) + } + if restConfig == nil { + return nil, sanitizeClientError("load client configuration", errors.New("loader returned nil config")) + } + configured := rest.CopyConfig(restConfig) + configured.Timeout = time.Duration(config.RequestTimeoutSeconds) * time.Second + configured.UserAgent = kubernetesUserAgent + if factory == nil { + factory = func(config *rest.Config) (kubeclient.Interface, error) { + return kubeclient.NewForConfig(config) + } + } + clientset, err := factory(configured) + if err != nil { + return nil, sanitizeClientError("create client", err) + } + return &Client{RESTConfig: configured, Clientset: clientset}, nil +} + +func loadKubeconfig(path, contextName string) (*rest.Config, error) { + rules := clientcmd.NewDefaultClientConfigLoadingRules() + rules.ExplicitPath = path + overrides := &clientcmd.ConfigOverrides{CurrentContext: contextName} + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig() +} + +func withDefaultConfigLoader(loader ConfigLoader) ConfigLoader { + defaults := DefaultConfigLoader() + if loader.Kubeconfig == nil { + loader.Kubeconfig = defaults.Kubeconfig + } + if loader.InCluster == nil { + loader.InCluster = defaults.InCluster + } + return loader +} + +func loadRESTConfig(config ExecutorConfig, loader ConfigLoader) (*rest.Config, error) { + if config.AuthMode == AuthModeInCluster { + return loader.InCluster() + } + return loader.Kubeconfig(config.KubeconfigPath, config.KubeContext) +} + +type clientError struct { + operation string + cause error +} + +func (e *clientError) Error() string { + return "kubernetes " + e.operation + " failed: " + routingerr.Sanitize(e.cause.Error()) +} +func (e *clientError) Unwrap() error { return e.cause } + +func sanitizeClientError(operation string, cause error) error { + return &clientError{operation: operation, cause: cause} +} diff --git a/apps/backend/internal/agent/kubernetes/client_test.go b/apps/backend/internal/agent/kubernetes/client_test.go new file mode 100644 index 0000000000..33e3a3dfb2 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/client_test.go @@ -0,0 +1,186 @@ +package kubernetes + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" +) + +func TestNewClientLoadsKubeconfigLazilyAndCopiesRESTConfig(t *testing.T) { + t.Parallel() + + source := &rest.Config{Host: "https://cluster.example", BearerToken: "secret-token"} + var gotPath, gotContext string + loader := ConfigLoader{ + Kubeconfig: func(path, contextName string) (*rest.Config, error) { + gotPath, gotContext = path, contextName + return source, nil + }, + InCluster: func() (*rest.Config, error) { + t.Fatal("in-cluster loader called for kubeconfig auth") + return nil, nil + }, + } + var factoryConfig *rest.Config + factory := func(config *rest.Config) (kubeclient.Interface, error) { + factoryConfig = config + return fake.NewSimpleClientset(), nil + } + cfg := ExecutorConfig{ + AuthMode: AuthModeKubeconfig, KubeconfigPath: "/etc/kandev/config", + KubeContext: "production", Namespace: "agents", RequestTimeoutSeconds: 45, + } + + client, err := NewClient(cfg, loader, factory) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + if gotPath != cfg.KubeconfigPath || gotContext != cfg.KubeContext { + t.Fatalf("loader args = %q/%q", gotPath, gotContext) + } + if client.RESTConfig == source || factoryConfig != client.RESTConfig { + t.Fatal("NewClient() did not use one defensive REST config copy") + } + if client.RESTConfig.Timeout != 45*time.Second || source.Timeout != 0 { + t.Fatalf("timeouts = client %s, source %s", client.RESTConfig.Timeout, source.Timeout) + } + if client.RESTConfig.UserAgent != "kandev-kubernetes-executor" { + t.Fatalf("user agent = %q", client.RESTConfig.UserAgent) + } +} + +func TestNewClientUsesInClusterLoader(t *testing.T) { + t.Parallel() + + called := false + loader := ConfigLoader{ + Kubeconfig: func(string, string) (*rest.Config, error) { + t.Fatal("kubeconfig loader called for in-cluster auth") + return nil, nil + }, + InCluster: func() (*rest.Config, error) { + called = true + return &rest.Config{Host: "https://kubernetes.default.svc"}, nil + }, + } + _, err := NewClient( + ExecutorConfig{AuthMode: AuthModeInCluster, Namespace: "agents", RequestTimeoutSeconds: 30}, + loader, + func(*rest.Config) (kubeclient.Interface, error) { return fake.NewSimpleClientset(), nil }, + ) + if err != nil { + t.Fatalf("NewClient() error = %v", err) + } + if !called { + t.Fatal("in-cluster loader was not called") + } +} + +func TestNewClientRedactsLoaderErrorsAndPreservesCause(t *testing.T) { + t.Parallel() + + cause := errors.New("permission denied by https://cluster.example/private?token=kandev_pat_super-secret-value") + _, err := NewClient( + ExecutorConfig{AuthMode: AuthModeInCluster, Namespace: "agents", RequestTimeoutSeconds: 30}, + ConfigLoader{InCluster: func() (*rest.Config, error) { return nil, cause }}, + nil, + ) + if !errors.Is(err, cause) { + t.Fatalf("NewClient() error = %v, want wrapped cause", err) + } + if strings.Contains(err.Error(), "super-secret-value") { + t.Fatalf("NewClient() exposed credential: %q", err) + } + if !strings.Contains(err.Error(), "permission denied") || !strings.Contains(err.Error(), "https://cluster.example") { + t.Fatalf("NewClient() discarded safe causal diagnostics: %q", err) + } + if strings.Contains(err.Error(), "/private") { + t.Fatalf("NewClient() exposed URL path: %q", err) + } +} + +func TestDefaultConfigLoaderHonorsExplicitContext(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "config") + contents := []byte(`apiVersion: v1 +kind: Config +clusters: + - name: first + cluster: + server: https://first.example + - name: second + cluster: + server: https://second.example +contexts: + - name: first + context: + cluster: first + user: user + - name: second + context: + cluster: second + user: user +current-context: first +users: + - name: user + user: + token: fixture-token +`) + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatal(err) + } + + config, err := DefaultConfigLoader().Kubeconfig(path, "second") + if err != nil { + t.Fatalf("Kubeconfig() error = %v", err) + } + if config.Host != "https://second.example" || config.BearerToken != "fixture-token" { + t.Fatalf("REST config = host %q token %q", config.Host, config.BearerToken) + } +} + +func TestDefaultConfigLoaderSupportsOIDCAuthProvider(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "config") + contents := []byte(`apiVersion: v1 +kind: Config +clusters: + - name: cluster + cluster: + server: https://cluster.example +contexts: + - name: oidc + context: + cluster: cluster + user: oidc +current-context: oidc +users: + - name: oidc + user: + auth-provider: + name: oidc + config: + idp-issuer-url: https://issuer.example + client-id: kandev +`) + if err := os.WriteFile(path, contents, 0o600); err != nil { + t.Fatal(err) + } + + config, err := DefaultConfigLoader().Kubeconfig(path, "oidc") + if err != nil { + t.Fatalf("Kubeconfig() error = %v", err) + } + if _, err := rest.TransportFor(config); err != nil { + t.Fatalf("TransportFor() error = %v", err) + } +} diff --git a/apps/backend/internal/agent/kubernetes/compose.go b/apps/backend/internal/agent/kubernetes/compose.go new file mode 100644 index 0000000000..6c62cf6252 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/compose.go @@ -0,0 +1,256 @@ +package kubernetes + +import ( + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +// Runtime inventory metadata keys are shared by lifecycle persistence and +// higher-level fail-closed inventory checks. +const ( + MetadataKeyResourceExecutorID = "kubernetes_resource_executor_id" + MetadataKeyResourceProfileID = "kubernetes_resource_profile_id" + MetadataKeyResourceInstanceID = "kubernetes_resource_instance_id" + MetadataKeyResourceTaskID = "kubernetes_resource_task_id" + MetadataKeyResourceSessionID = "kubernetes_resource_session_id" + MetadataKeyResourceEnvironmentID = "kubernetes_resource_environment_id" +) + +type ResourceIdentity struct { + ExecutorID string + ProfileID string + InstanceID string + TaskID string + SessionID string + EnvironmentID string +} + +type PodOptions struct { + Name string + Namespace string + Identity ResourceIdentity + Command []string + Args []string + WorkingDir string + AgentctlPort int32 + ManagedPVCName string +} + +type Warning struct { + Path string `json:"path"` + Message string `json:"message"` +} + +func ComposePod(template *corev1.PodTemplate, profile ProfileConfig, options PodOptions) (*corev1.Pod, []Warning, error) { + if err := validateComposition(template, profile, options); err != nil { + return nil, nil, err + } + labels, err := OwnershipLabels(options.Identity) + if err != nil { + return nil, nil, err + } + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Pod"}, + ObjectMeta: *template.Template.ObjectMeta.DeepCopy(), + Spec: *template.Template.Spec.DeepCopy(), + } + pod.Name = options.Name + pod.Namespace = options.Namespace + if pod.Labels == nil { + pod.Labels = make(map[string]string, len(labels)) + } + for key, value := range labels { + pod.Labels[key] = value + } + applyPodInvariants(pod, profile.Platform) + applyMainContainer(pod, profile.MainContainer, options) + workspace, err := workspaceVolume(profile.Workspace, options.ManagedPVCName) + if err != nil { + return nil, nil, err + } + pod.Spec.Volumes = append(pod.Spec.Volumes, managedVolumes(workspace)...) + return pod, collectWarnings(pod), nil +} + +func OwnershipLabels(identity ResourceIdentity) (map[string]string, error) { + values := []struct { + path string + key string + value string + }{ + {"identity.executor_id", "kandev.ai/executor-id", identity.ExecutorID}, + {"identity.profile_id", "kandev.ai/profile-id", identity.ProfileID}, + {"identity.instance_id", "kandev.ai/instance-id", identity.InstanceID}, + {"identity.task_id", "kandev.ai/task-id", identity.TaskID}, + {"identity.session_id", "kandev.ai/session-id", identity.SessionID}, + {"identity.environment_id", "kandev.ai/environment-id", identity.EnvironmentID}, + } + labels := map[string]string{ + "app.kubernetes.io/name": "kandev-agent", + "app.kubernetes.io/component": "agent-session", + "app.kubernetes.io/managed-by": "kandev", + "app.kubernetes.io/instance": identity.InstanceID, + } + for _, item := range values { + if item.value == "" || len(validation.IsValidLabelValue(item.value)) > 0 { + return nil, fieldError(item.path, "must be a non-empty Kubernetes label value") + } + labels[item.key] = item.value + } + return labels, nil +} + +func validateComposition(template *corev1.PodTemplate, profile ProfileConfig, options PodOptions) error { + if len(validation.IsDNS1123Subdomain(options.Name)) > 0 { + return fieldError("pod.metadata.name", "must be a valid Pod name") + } + if len(validation.IsDNS1123Label(options.Namespace)) > 0 { + return fieldError("pod.metadata.namespace", "must be a valid namespace") + } + if options.AgentctlPort == 0 { + options.AgentctlPort = DefaultAgentctlPort + } + if options.AgentctlPort < 1 || options.AgentctlPort > 65535 { + return fieldError("pod.agentctl_port", "must be between 1 and 65535") + } + if len(options.Command) == 0 { + return fieldError("pod.command", "is required") + } + if !strings.HasPrefix(options.WorkingDir, "/") { + return fieldError("pod.working_dir", "must be absolute") + } + if profile.Platform != PlatformLinuxAMD64 && profile.Platform != PlatformLinuxARM64 { + return fieldError("config.platform", "must be linux/amd64 or linux/arm64") + } + if errs := validation.IsDNS1123Label(profile.MainContainer); len(errs) > 0 { + return fieldError("config.main_container", "must be a valid container name") + } + if err := profile.Workspace.validate(); err != nil { + return err + } + return ValidatePodTemplate(template, profile.MainContainer) +} + +func applyPodInvariants(pod *corev1.Pod, platform Platform) { + pod.Spec.RestartPolicy = corev1.RestartPolicyAlways + pod.Spec.OS = &corev1.PodOS{Name: corev1.Linux} + if pod.Spec.AutomountServiceAccountToken == nil { + value := false + pod.Spec.AutomountServiceAccountToken = &value + } + if pod.Spec.NodeSelector == nil { + pod.Spec.NodeSelector = make(map[string]string, 2) + } + architecture := strings.TrimPrefix(string(platform), "linux/") + pod.Spec.NodeSelector[corev1.LabelOSStable] = "linux" + pod.Spec.NodeSelector[corev1.LabelArchStable] = architecture +} + +func applyMainContainer(pod *corev1.Pod, mainContainer string, options PodOptions) { + port := options.AgentctlPort + if port == 0 { + port = DefaultAgentctlPort + } + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name != mainContainer { + continue + } + main := &pod.Spec.Containers[i] + main.Command = append([]string(nil), options.Command...) + main.Args = append([]string(nil), options.Args...) + main.WorkingDir = options.WorkingDir + main.Ports = append(main.Ports, corev1.ContainerPort{ + Name: AgentctlContainerPort, ContainerPort: port, Protocol: corev1.ProtocolTCP, + }) + main.VolumeMounts = append(main.VolumeMounts, managedVolumeMounts()...) + return + } +} + +func managedVolumeMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: RuntimeVolumeName, MountPath: RuntimeMountPath}, + {Name: AuthVolumeName, MountPath: AuthMountPath}, + {Name: WorkspaceVolumeName, MountPath: WorkspaceMountPath}, + } +} + +func managedVolumes(workspace corev1.VolumeSource) []corev1.Volume { + return []corev1.Volume{ + {Name: RuntimeVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: AuthVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}}, + {Name: WorkspaceVolumeName, VolumeSource: workspace}, + } +} + +func workspaceVolume(workspace WorkspaceConfig, managedPVCName string) (corev1.VolumeSource, error) { + switch workspace.Mode { + case WorkspaceModeEmptyDir: + return corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, nil + case WorkspaceModeExistingClaim: + return claimVolume(workspace.ClaimName), nil + case WorkspaceModeManagedPVC: + if len(validation.IsDNS1123Subdomain(managedPVCName)) > 0 { + return corev1.VolumeSource{}, fieldError("pod.managed_pvc_name", "must be a valid claim name") + } + return claimVolume(managedPVCName), nil + default: + return corev1.VolumeSource{}, fieldError("config.workspace.mode", "is unsupported") + } +} + +func claimVolume(name string) corev1.VolumeSource { + return corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}} +} + +func collectWarnings(pod *corev1.Pod) []Warning { + warnings := make([]Warning, 0) + if pod.Spec.AutomountServiceAccountToken != nil && *pod.Spec.AutomountServiceAccountToken { + warnings = append(warnings, warning( + ".template.spec.automountServiceAccountToken", + "service account token automount is enabled", + )) + } + if pod.Spec.HostNetwork { + warnings = append(warnings, warning(".template.spec.hostNetwork", "host networking is enabled")) + } + if pod.Spec.HostPID { + warnings = append(warnings, warning(".template.spec.hostPID", "host PID namespace is enabled")) + } + if pod.Spec.HostIPC { + warnings = append(warnings, warning(".template.spec.hostIPC", "host IPC namespace is enabled")) + } + for i, volume := range pod.Spec.Volumes { + if volume.HostPath != nil { + warnings = append(warnings, warning(fmt.Sprintf(".template.spec.volumes[%d].hostPath", i), "host filesystem access is enabled")) + } + } + for i := range pod.Spec.Containers { + warnings = append(warnings, containerWarnings(&pod.Spec.Containers[i], fmt.Sprintf(".template.spec.containers[%d]", i))...) + } + for i := range pod.Spec.InitContainers { + warnings = append(warnings, containerWarnings(&pod.Spec.InitContainers[i], fmt.Sprintf(".template.spec.initContainers[%d]", i))...) + } + return warnings +} + +func containerWarnings(container *corev1.Container, path string) []Warning { + warnings := make([]Warning, 0) + if container.SecurityContext != nil && container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged { + warnings = append(warnings, warning(path+".securityContext.privileged", "privileged container is enabled")) + } + for i, port := range container.Ports { + if port.HostPort != 0 { + warnings = append(warnings, warning(fmt.Sprintf("%s.ports[%d].hostPort", path, i), "host port is enabled")) + } + } + return warnings +} + +func warning(suffix, message string) Warning { + return Warning{Path: podTemplateFieldPrefix + suffix, Message: message} +} diff --git a/apps/backend/internal/agent/kubernetes/compose_test.go b/apps/backend/internal/agent/kubernetes/compose_test.go new file mode 100644 index 0000000000..f3b1398253 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/compose_test.go @@ -0,0 +1,236 @@ +package kubernetes + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestComposePodPreservesOperatorFieldsAndInjectsOwnedFields(t *testing.T) { + t.Parallel() + + template, err := ParsePodTemplate(`apiVersion: v1 +kind: PodTemplate +template: + metadata: + labels: + team: platform + annotations: + example.com/note: retained + spec: + serviceAccountName: workload + automountServiceAccountToken: true + imagePullSecrets: + - name: registry + affinity: + podAffinity: {} + tolerations: + - key: dedicated + operator: Exists + containers: + - name: kandev-agent + image: example/agent:v1 + resources: + requests: + cpu: 100m + - name: metrics + image: example/metrics:v2 +`) + if err != nil { + t.Fatalf("ParsePodTemplate() error = %v", err) + } + original := template.DeepCopy() + profile := validProfile("") + profile.Workspace = WorkspaceConfig{Mode: WorkspaceModeManagedPVC, Size: "10Gi", AccessModes: []string{"ReadWriteOnce"}} + + pod, warnings, err := ComposePod(template, profile, podOptions()) + if err != nil { + t.Fatalf("ComposePod() error = %v", err) + } + wantWarningPaths := []string{ + "config.pod_template_yaml.template.spec.automountServiceAccountToken", + } + if got := warningPaths(warnings); !reflect.DeepEqual(got, wantWarningPaths) { + t.Fatalf("ComposePod() warning paths = %v, want %v", got, wantWarningPaths) + } + if !reflect.DeepEqual(template, original) { + t.Fatal("ComposePod() mutated input template") + } + if pod.Name != "kandev-session-1" || pod.Namespace != "agents" { + t.Fatalf("pod identity = %s/%s", pod.Namespace, pod.Name) + } + if pod.Labels["team"] != "platform" || pod.Annotations["example.com/note"] != "retained" { + t.Fatalf("operator metadata not preserved: labels=%v annotations=%v", pod.Labels, pod.Annotations) + } + if pod.Labels["kandev.ai/session-id"] != "session-1" || pod.Labels["app.kubernetes.io/managed-by"] != "kandev" { + t.Fatalf("ownership labels = %v", pod.Labels) + } + if pod.Spec.RestartPolicy != corev1.RestartPolicyAlways { + t.Fatalf("restartPolicy = %q", pod.Spec.RestartPolicy) + } + if pod.Spec.OS == nil || pod.Spec.OS.Name != corev1.Linux { + t.Fatalf("spec.os = %#v, want linux", pod.Spec.OS) + } + if pod.Spec.NodeSelector[corev1.LabelOSStable] != "linux" || pod.Spec.NodeSelector[corev1.LabelArchStable] != "amd64" { + t.Fatalf("nodeSelector = %v", pod.Spec.NodeSelector) + } + if pod.Spec.AutomountServiceAccountToken == nil || !*pod.Spec.AutomountServiceAccountToken { + t.Fatal("explicit automountServiceAccountToken=true was not preserved") + } + if pod.Spec.ServiceAccountName != "workload" || len(pod.Spec.ImagePullSecrets) != 1 || len(pod.Spec.Tolerations) != 1 { + t.Fatalf("operator pod fields not preserved: %#v", pod.Spec) + } + main := pod.Spec.Containers[0] + if !reflect.DeepEqual(main.Command, []string{"/bin/sh"}) || !reflect.DeepEqual(main.Args, []string{"-ceu", "run-managed-entrypoint"}) || main.WorkingDir != WorkspaceMountPath { + t.Fatalf("main launch fields = command=%v args=%v workingDir=%q", main.Command, main.Args, main.WorkingDir) + } + if main.Resources.Requests.Cpu().Cmp(resource.MustParse("100m")) != 0 { + t.Fatalf("main resources not preserved: %v", main.Resources) + } + assertManagedPodVolumes(t, pod) +} + +func TestComposePodDefaultsServiceAccountTokenOff(t *testing.T) { + t.Parallel() + + template, err := ParsePodTemplate(validPodTemplate("")) + if err != nil { + t.Fatal(err) + } + profile := validProfile("") + pod, _, err := ComposePod(template, profile, podOptions()) + if err != nil { + t.Fatalf("ComposePod() error = %v", err) + } + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("automountServiceAccountToken default is not false") + } +} + +func TestComposePodWorkspaceVolumeModes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + workspace WorkspaceConfig + wantClaim string + wantEmpty bool + }{ + {name: "managed pvc", workspace: WorkspaceConfig{Mode: WorkspaceModeManagedPVC, Size: "10Gi", AccessModes: []string{"ReadWriteOnce"}}, wantClaim: "workspace-pvc"}, + {name: "existing claim", workspace: WorkspaceConfig{Mode: WorkspaceModeExistingClaim, ClaimName: "existing-pvc"}, wantClaim: "existing-pvc"}, + {name: "empty dir", workspace: WorkspaceConfig{Mode: WorkspaceModeEmptyDir}, wantEmpty: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + template, err := ParsePodTemplate(validPodTemplate("")) + if err != nil { + t.Fatal(err) + } + profile := validProfile("") + profile.Workspace = tt.workspace + pod, _, err := ComposePod(template, profile, podOptions()) + if err != nil { + t.Fatalf("ComposePod() error = %v", err) + } + workspace := findVolume(t, pod, WorkspaceVolumeName) + if tt.wantEmpty && workspace.EmptyDir == nil { + t.Fatalf("workspace volume = %#v, want emptyDir", workspace) + } + if tt.wantClaim != "" && (workspace.PersistentVolumeClaim == nil || workspace.PersistentVolumeClaim.ClaimName != tt.wantClaim) { + t.Fatalf("workspace volume = %#v, want claim %q", workspace, tt.wantClaim) + } + }) + } +} + +func TestComposePodWarnsWithoutStrippingPrivilegedHostFields(t *testing.T) { + t.Parallel() + + template, err := ParsePodTemplate(`apiVersion: v1 +kind: PodTemplate +template: + spec: + hostNetwork: true + volumes: + - name: host + hostPath: + path: /var/run + containers: + - name: kandev-agent + image: example/agent:v1 + securityContext: + privileged: true +`) + if err != nil { + t.Fatal(err) + } + pod, warnings, err := ComposePod(template, validProfile(""), podOptions()) + if err != nil { + t.Fatalf("ComposePod() error = %v", err) + } + if !pod.Spec.HostNetwork || pod.Spec.Volumes[0].HostPath == nil || pod.Spec.Containers[0].SecurityContext == nil || pod.Spec.Containers[0].SecurityContext.Privileged == nil || !*pod.Spec.Containers[0].SecurityContext.Privileged { + t.Fatal("privileged operator fields were stripped") + } + wantPaths := []string{ + "config.pod_template_yaml.template.spec.hostNetwork", + "config.pod_template_yaml.template.spec.volumes[0].hostPath", + "config.pod_template_yaml.template.spec.containers[0].securityContext.privileged", + } + if got := warningPaths(warnings); !reflect.DeepEqual(got, wantPaths) { + t.Fatalf("warning paths = %v, want %v", got, wantPaths) + } +} + +func podOptions() PodOptions { + return PodOptions{ + Name: "kandev-session-1", Namespace: "agents", + Identity: ResourceIdentity{ + ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "instance-1", + TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", + }, + Command: []string{"/bin/sh"}, Args: []string{"-ceu", "run-managed-entrypoint"}, + WorkingDir: WorkspaceMountPath, AgentctlPort: DefaultAgentctlPort, ManagedPVCName: "workspace-pvc", + } +} + +func assertManagedPodVolumes(t *testing.T, pod *corev1.Pod) { + t.Helper() + if findVolume(t, pod, RuntimeVolumeName).EmptyDir == nil { + t.Fatal("runtime volume is not emptyDir") + } + auth := findVolume(t, pod, AuthVolumeName) + if auth.EmptyDir == nil || auth.EmptyDir.Medium != corev1.StorageMediumMemory { + t.Fatal("auth volume is not memory-backed emptyDir") + } + workspace := findVolume(t, pod, WorkspaceVolumeName) + if workspace.PersistentVolumeClaim == nil || workspace.PersistentVolumeClaim.ClaimName != "workspace-pvc" { + t.Fatalf("workspace volume = %#v", workspace) + } + main := pod.Spec.Containers[0] + if len(main.VolumeMounts) != 3 || len(main.Ports) != 1 || main.Ports[0].ContainerPort != DefaultAgentctlPort { + t.Fatalf("main mounts/ports = %#v / %#v", main.VolumeMounts, main.Ports) + } +} + +func findVolume(t *testing.T, pod *corev1.Pod, name string) corev1.Volume { + t.Helper() + for _, volume := range pod.Spec.Volumes { + if volume.Name == name { + return volume + } + } + t.Fatalf("volume %q not found", name) + return corev1.Volume{} +} + +func warningPaths(warnings []Warning) []string { + paths := make([]string, len(warnings)) + for i, warning := range warnings { + paths[i] = warning.Path + } + return paths +} diff --git a/apps/backend/internal/agent/kubernetes/config.go b/apps/backend/internal/agent/kubernetes/config.go new file mode 100644 index 0000000000..79ed32430e --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/config.go @@ -0,0 +1,276 @@ +// Package kubernetes defines Kubernetes executor configuration and resource composition. +package kubernetes + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" +) + +const ( + DefaultRequestTimeoutSeconds = 30 + MaxRequestTimeoutSeconds = 300 + DefaultMainContainerName = "kandev-agent" +) + +type FieldError struct { + Path string + Message string +} + +func (e *FieldError) Error() string { return fmt.Sprintf("%s: %s", e.Path, e.Message) } + +type AuthMode string + +const ( + AuthModeKubeconfig AuthMode = "kubeconfig" + AuthModeInCluster AuthMode = "in_cluster" +) + +type ExecutorConfig struct { + AuthMode AuthMode `json:"auth_mode"` + KubeconfigPath string `json:"kubeconfig_path,omitempty"` + KubeContext string `json:"kube_context,omitempty"` + Namespace string `json:"namespace"` + RequestTimeoutSeconds int `json:"request_timeout_seconds"` +} + +func (c ExecutorConfig) Validate() error { + switch c.AuthMode { + case AuthModeKubeconfig: + if !filepath.IsAbs(c.KubeconfigPath) { + return fieldError("config.kubeconfig_path", "must be an absolute path for kubeconfig auth") + } + case AuthModeInCluster: + if c.KubeconfigPath != "" { + return fieldError("config.kubeconfig_path", "must be empty for in-cluster auth") + } + if c.KubeContext != "" { + return fieldError("config.kube_context", "must be empty for in-cluster auth") + } + default: + return fieldError("config.auth_mode", "must be kubeconfig or in_cluster") + } + if errs := validation.IsDNS1123Label(c.Namespace); len(errs) > 0 { + return fieldError("config.namespace", "must be a valid Kubernetes namespace") + } + if c.RequestTimeoutSeconds < 1 || c.RequestTimeoutSeconds > MaxRequestTimeoutSeconds { + return fieldError("config.request_timeout_seconds", "must be between 1 and 300") + } + return nil +} + +func ParseExecutorConfig(values map[string]string) (ExecutorConfig, error) { + timeout, err := parseTimeout(values["request_timeout_seconds"]) + if err != nil { + return ExecutorConfig{}, err + } + cfg := ExecutorConfig{ + AuthMode: AuthMode(strings.TrimSpace(values["auth_mode"])), + KubeconfigPath: strings.TrimSpace(values["kubeconfig_path"]), + KubeContext: strings.TrimSpace(values["kube_context"]), + Namespace: strings.TrimSpace(values["namespace"]), + RequestTimeoutSeconds: timeout, + } + if err := cfg.Validate(); err != nil { + return ExecutorConfig{}, err + } + return cfg, nil +} + +type Platform string + +const ( + PlatformLinuxAMD64 Platform = "linux/amd64" + PlatformLinuxARM64 Platform = "linux/arm64" +) + +type WorkspaceMode string + +const ( + WorkspaceModeManagedPVC WorkspaceMode = "managed_pvc" + WorkspaceModeEmptyDir WorkspaceMode = "empty_dir" + WorkspaceModeExistingClaim WorkspaceMode = "existing_claim" +) + +type WorkspaceConfig struct { + Mode WorkspaceMode `json:"mode"` + Size string `json:"size,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + AccessModes []string `json:"access_modes,omitempty"` + ClaimName string `json:"claim_name,omitempty"` +} + +type ProfileConfig struct { + Platform Platform `json:"platform"` + MainContainer string `json:"main_container"` + PodTemplateYAML string `json:"pod_template_yaml"` + Workspace WorkspaceConfig `json:"workspace"` +} + +func (c ProfileConfig) Validate() error { + if c.Platform != PlatformLinuxAMD64 && c.Platform != PlatformLinuxARM64 { + return fieldError("config.platform", "must be linux/amd64 or linux/arm64") + } + if errs := validation.IsDNS1123Label(c.MainContainer); len(errs) > 0 { + return fieldError("config.main_container", "must be a valid container name") + } + if strings.TrimSpace(c.PodTemplateYAML) == "" { + return fieldError("config.pod_template_yaml", "is required") + } + template, err := ParsePodTemplate(c.PodTemplateYAML) + if err != nil { + return err + } + if err := ValidatePodTemplate(template, c.MainContainer); err != nil { + return err + } + return c.Workspace.validate() +} + +func ParseProfileConfig(values map[string]string) (ProfileConfig, error) { + mainContainer := strings.TrimSpace(values["main_container"]) + if mainContainer == "" { + mainContainer = DefaultMainContainerName + } + accessModes, err := parseAccessModes(configValue(values, "workspace.access_modes", "workspace_access_modes")) + if err != nil { + return ProfileConfig{}, err + } + mode := WorkspaceMode(strings.TrimSpace(configValue(values, "workspace.mode", "workspace_mode"))) + if mode == WorkspaceModeManagedPVC && len(accessModes) == 0 { + accessModes = []string{"ReadWriteOnce"} + } + cfg := ProfileConfig{ + Platform: Platform(strings.TrimSpace(values["platform"])), + MainContainer: mainContainer, + PodTemplateYAML: values["pod_template_yaml"], + Workspace: WorkspaceConfig{ + Mode: mode, + Size: strings.TrimSpace(configValue(values, "workspace.size", "workspace_size")), + StorageClass: strings.TrimSpace(configValue(values, "workspace.storage_class", "workspace_storage_class")), + AccessModes: accessModes, + ClaimName: strings.TrimSpace(configValue(values, "workspace.claim_name", "workspace_claim_name")), + }, + } + if err := cfg.Validate(); err != nil { + return ProfileConfig{}, err + } + return cfg, nil +} + +func (c WorkspaceConfig) validate() error { + switch c.Mode { + case WorkspaceModeManagedPVC: + return c.validateManagedPVC() + case WorkspaceModeEmptyDir: + return c.validateEmptyDir() + case WorkspaceModeExistingClaim: + return c.validateExistingClaim() + default: + return fieldError("config.workspace.mode", "must be managed_pvc, empty_dir, or existing_claim") + } +} + +func (c WorkspaceConfig) validateManagedPVC() error { + quantity, err := resource.ParseQuantity(strings.TrimSpace(c.Size)) + if err != nil || quantity.Sign() <= 0 { + return fieldError("config.workspace.size", "must be a positive Kubernetes quantity") + } + if c.ClaimName != "" { + return fieldError("config.workspace.claim_name", "must be empty for managed_pvc") + } + if c.StorageClass != "" && len(validation.IsDNS1123Subdomain(c.StorageClass)) > 0 { + return fieldError("config.workspace.storage_class", "must be a valid storage class name") + } + if len(c.AccessModes) == 0 { + return fieldError("config.workspace.access_modes", "must contain at least one access mode") + } + for _, mode := range c.AccessModes { + if !validAccessMode(mode) { + return fieldError("config.workspace.access_modes", "contains an unsupported access mode") + } + } + return nil +} + +func (c WorkspaceConfig) validateEmptyDir() error { + switch { + case c.ClaimName != "": + return fieldError("config.workspace.claim_name", "must be empty for empty_dir") + case c.Size != "": + return fieldError("config.workspace.size", "must be empty for empty_dir") + case c.StorageClass != "": + return fieldError("config.workspace.storage_class", "must be empty for empty_dir") + case len(c.AccessModes) > 0: + return fieldError("config.workspace.access_modes", "must be empty for empty_dir") + default: + return nil + } +} + +func (c WorkspaceConfig) validateExistingClaim() error { + if errs := validation.IsDNS1123Subdomain(c.ClaimName); len(errs) > 0 { + return fieldError("config.workspace.claim_name", "must be a valid claim name") + } + switch { + case c.Size != "": + return fieldError("config.workspace.size", "must be empty for existing_claim") + case c.StorageClass != "": + return fieldError("config.workspace.storage_class", "must be empty for existing_claim") + case len(c.AccessModes) > 0: + return fieldError("config.workspace.access_modes", "must be empty for existing_claim") + default: + return nil + } +} + +func validAccessMode(mode string) bool { + switch mode { + case "ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany", "ReadWriteOncePod": + return true + default: + return false + } +} + +func parseTimeout(raw string) (int, error) { + if strings.TrimSpace(raw) == "" { + return DefaultRequestTimeoutSeconds, nil + } + timeout, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil { + return 0, fieldError("config.request_timeout_seconds", "must be an integer") + } + return timeout, nil +} + +func parseAccessModes(raw string) ([]string, error) { + if strings.TrimSpace(raw) == "" { + return nil, nil + } + var modes []string + if err := json.Unmarshal([]byte(raw), &modes); err != nil || modes == nil { + return nil, fieldError("config.workspace.access_modes", "must be a JSON array of access modes") + } + for i := range modes { + modes[i] = strings.TrimSpace(modes[i]) + } + return modes, nil +} + +func configValue(values map[string]string, preferred, fallback string) string { + if value, ok := values[preferred]; ok { + return value + } + return values[fallback] +} + +func fieldError(path, message string) error { + return &FieldError{Path: path, Message: message} +} diff --git a/apps/backend/internal/agent/kubernetes/config_test.go b/apps/backend/internal/agent/kubernetes/config_test.go new file mode 100644 index 0000000000..0231921d9f --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/config_test.go @@ -0,0 +1,171 @@ +package kubernetes + +import ( + "errors" + "reflect" + "testing" +) + +func TestExecutorConfigValidateReportsStableFieldPaths(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cfg ExecutorConfig + path string + }{ + {name: "auth mode", cfg: ExecutorConfig{Namespace: "agents", RequestTimeoutSeconds: 30}, path: "config.auth_mode"}, + {name: "kubeconfig path", cfg: ExecutorConfig{AuthMode: AuthModeKubeconfig, KubeconfigPath: "relative", Namespace: "agents", RequestTimeoutSeconds: 30}, path: "config.kubeconfig_path"}, + {name: "in-cluster path", cfg: ExecutorConfig{AuthMode: AuthModeInCluster, KubeconfigPath: "/secret/config", Namespace: "agents", RequestTimeoutSeconds: 30}, path: "config.kubeconfig_path"}, + {name: "namespace", cfg: ExecutorConfig{AuthMode: AuthModeInCluster, Namespace: "Bad_Namespace", RequestTimeoutSeconds: 30}, path: "config.namespace"}, + {name: "timeout", cfg: ExecutorConfig{AuthMode: AuthModeInCluster, Namespace: "agents", RequestTimeoutSeconds: 301}, path: "config.request_timeout_seconds"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := tt.cfg.Validate() + var fieldErr *FieldError + if !errors.As(err, &fieldErr) { + t.Fatalf("Validate() error = %v, want FieldError", err) + } + if fieldErr.Path != tt.path { + t.Fatalf("Validate() path = %q, want %q", fieldErr.Path, tt.path) + } + }) + } +} + +func TestParseExecutorConfigSupportsBothAuthModes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in map[string]string + want ExecutorConfig + }{ + { + name: "kubeconfig", + in: map[string]string{ + "auth_mode": "kubeconfig", "kubeconfig_path": "/etc/kandev/cluster.yaml", + "kube_context": "production", "namespace": "kandev-agents", "request_timeout_seconds": "45", + }, + want: ExecutorConfig{AuthMode: AuthModeKubeconfig, KubeconfigPath: "/etc/kandev/cluster.yaml", KubeContext: "production", Namespace: "kandev-agents", RequestTimeoutSeconds: 45}, + }, + { + name: "in cluster defaults timeout", + in: map[string]string{"auth_mode": "in_cluster", "namespace": "agents"}, + want: ExecutorConfig{AuthMode: AuthModeInCluster, Namespace: "agents", RequestTimeoutSeconds: DefaultRequestTimeoutSeconds}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := ParseExecutorConfig(tt.in) + if err != nil { + t.Fatalf("ParseExecutorConfig() error = %v", err) + } + if got != tt.want { + t.Fatalf("ParseExecutorConfig() = %#v, want %#v", got, tt.want) + } + }) + } +} + +func TestProfileConfigValidateSupportsAllWorkspaceModes(t *testing.T) { + t.Parallel() + + template := "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example/agent:latest\n" + tests := []WorkspaceConfig{ + {Mode: WorkspaceModeManagedPVC, Size: "20Gi", AccessModes: []string{"ReadWriteOnce"}}, + {Mode: WorkspaceModeEmptyDir}, + {Mode: WorkspaceModeExistingClaim, ClaimName: "shared-workspace"}, + } + + for _, workspace := range tests { + workspace := workspace + t.Run(string(workspace.Mode), func(t *testing.T) { + t.Parallel() + cfg := ProfileConfig{Platform: PlatformLinuxAMD64, MainContainer: "kandev-agent", PodTemplateYAML: template, Workspace: workspace} + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v", err) + } + }) + } +} + +func TestProfileConfigValidateRejectsModeSpecificFields(t *testing.T) { + t.Parallel() + + tests := []struct { + workspace WorkspaceConfig + path string + }{ + {workspace: WorkspaceConfig{Mode: WorkspaceModeManagedPVC}, path: "config.workspace.size"}, + { + workspace: WorkspaceConfig{ + Mode: WorkspaceModeManagedPVC, Size: "10Gi", + StorageClass: "Invalid_Storage_Class", AccessModes: []string{"ReadWriteOnce"}, + }, + path: "config.workspace.storage_class", + }, + {workspace: WorkspaceConfig{Mode: WorkspaceModeEmptyDir, ClaimName: "foreign"}, path: "config.workspace.claim_name"}, + {workspace: WorkspaceConfig{Mode: WorkspaceModeExistingClaim}, path: "config.workspace.claim_name"}, + } + + for _, tt := range tests { + cfg := ProfileConfig{Platform: PlatformLinuxARM64, MainContainer: "kandev-agent", PodTemplateYAML: validPodTemplate(""), Workspace: tt.workspace} + err := cfg.Validate() + var fieldErr *FieldError + if !errors.As(err, &fieldErr) || fieldErr.Path != tt.path { + t.Fatalf("Validate(%s) error = %v, want field %q", tt.workspace.Mode, err, tt.path) + } + } +} + +func TestParseProfileConfigBuildsTypedWorkspaceConfig(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + values map[string]string + workspace WorkspaceConfig + }{ + { + name: "managed pvc", + values: map[string]string{ + "workspace_mode": "managed_pvc", "workspace_size": "20Gi", + "workspace_storage_class": "fast", "workspace_access_modes": `["ReadWriteOnce","ReadWriteMany"]`, + }, + workspace: WorkspaceConfig{Mode: WorkspaceModeManagedPVC, Size: "20Gi", StorageClass: "fast", AccessModes: []string{"ReadWriteOnce", "ReadWriteMany"}}, + }, + {name: "empty dir", values: map[string]string{"workspace_mode": "empty_dir"}, workspace: WorkspaceConfig{Mode: WorkspaceModeEmptyDir}}, + {name: "existing claim", values: map[string]string{"workspace_mode": "existing_claim", "workspace_claim_name": "shared-pvc"}, workspace: WorkspaceConfig{Mode: WorkspaceModeExistingClaim, ClaimName: "shared-pvc"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tt.values["platform"] = "linux/arm64" + tt.values["pod_template_yaml"] = validPodTemplate("") + got, err := ParseProfileConfig(tt.values) + if err != nil { + t.Fatalf("ParseProfileConfig() error = %v", err) + } + if got.Platform != PlatformLinuxARM64 || got.MainContainer != "kandev-agent" || !reflect.DeepEqual(got.Workspace, tt.workspace) { + t.Fatalf("ParseProfileConfig() = %#v, want platform/main/workspace %#v", got, tt.workspace) + } + }) + } +} + +func TestParseProfileConfigRejectsMalformedAccessModes(t *testing.T) { + t.Parallel() + + _, err := ParseProfileConfig(map[string]string{ + "platform": "linux/amd64", "pod_template_yaml": validPodTemplate(""), + "workspace_mode": "managed_pvc", "workspace_size": "10Gi", "workspace_access_modes": "ReadWriteOnce", + }) + assertFieldPath(t, err, "config.workspace.access_modes") +} diff --git a/apps/backend/internal/agent/kubernetes/create.go b/apps/backend/internal/agent/kubernetes/create.go new file mode 100644 index 0000000000..75ec9ff1bc --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/create.go @@ -0,0 +1,58 @@ +package kubernetes + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io" + "net" + "net/url" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const CreateNonceAnnotation = "kandev.ai/create-nonce" + +// StampCreateNonce binds ambiguous-create reconciliation to this exact create +// attempt. The annotation is Kandev-owned, so the admission validators require +// the API result to preserve it exactly. +func StampCreateNonce(object metav1.Object) error { + bytes := make([]byte, 32) + if _, err := rand.Read(bytes); err != nil { + return err + } + annotations := object.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string, 1) + } + annotations[CreateNonceAnnotation] = hex.EncodeToString(bytes) + object.SetAnnotations(annotations) + return nil +} + +// IsAmbiguousCreateError reports whether a create call may have reached the API +// server before its result was lost. Definite admission/API rejections must +// never authorize adopting an exact-name object discovered after the error. +func IsAmbiguousCreateError(err error) bool { + if err == nil { + return false + } + if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || + errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + var statusErr apierrors.APIStatus + if errors.As(err, &statusErr) { + code := statusErr.Status().Code + return code >= 500 && code <= 599 + } + var transportErr *url.Error + if errors.As(err, &transportErr) { + return true + } + var networkErr net.Error + return errors.As(err, &networkErr) +} diff --git a/apps/backend/internal/agent/kubernetes/create_test.go b/apps/backend/internal/agent/kubernetes/create_test.go new file mode 100644 index 0000000000..42a8d1d0ee --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/create_test.go @@ -0,0 +1,44 @@ +package kubernetes + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +func TestIsAmbiguousCreateError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "transport reset", err: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, want: true}, + {name: "unexpected EOF", err: io.ErrUnexpectedEOF, want: true}, + {name: "caller deadline", err: context.DeadlineExceeded, want: true}, + {name: "server timeout", err: apierrors.NewServerTimeout(schema.GroupResource{Resource: "pods"}, "create", 1), want: true}, + {name: "internal server error", err: apierrors.NewInternalError(errors.New("apiserver unavailable")), want: true}, + {name: "already exists", err: apierrors.NewAlreadyExists(schema.GroupResource{Resource: "pods"}, "agent")}, + {name: "conflict", err: apierrors.NewConflict(schema.GroupResource{Resource: "pods"}, "agent", errors.New("conflict"))}, + {name: "too many requests", err: apierrors.NewTooManyRequests("rate limited", 1)}, + {name: "forbidden", err: apierrors.NewForbidden(schema.GroupResource{Resource: "pods"}, "agent", errors.New("denied"))}, + {name: "invalid", err: apierrors.NewInvalid(schema.GroupKind{Group: "", Kind: "Pod"}, "agent", nil)}, + {name: "unsupported media type", err: &apierrors.StatusError{ErrStatus: metav1.Status{ + Code: http.StatusUnsupportedMediaType, Reason: metav1.StatusReasonUnsupportedMediaType, + }}}, + {name: "ordinary error", err: errors.New("definite local failure")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.want, IsAmbiguousCreateError(test.err)) + }) + } +} diff --git a/apps/backend/internal/agent/kubernetes/portforward.go b/apps/backend/internal/agent/kubernetes/portforward.go new file mode 100644 index 0000000000..bb5ea38a51 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/portforward.go @@ -0,0 +1,518 @@ +package kubernetes + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" + + gwebsocket "github.com/gorilla/websocket" + "k8s.io/apimachinery/pkg/util/httpstream" + streamspdy "k8s.io/apimachinery/pkg/util/httpstream/spdy" + "k8s.io/apimachinery/pkg/util/httpstream/wsstream" + utilnet "k8s.io/apimachinery/pkg/util/net" + portforwardconstants "k8s.io/apimachinery/pkg/util/portforward" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport" + clientwebsocket "k8s.io/client-go/transport/websocket" +) + +// NewPortForwardDialer builds the maintained client-go WebSocket-primary, +// legacy-SPDY-fallback dialer. Each upgrade attempt is bounded by ctx and +// handshakeTimeout; successful upgraded connections are not tied to that +// request deadline. +func NewPortForwardDialer( + ctx context.Context, + config *rest.Config, + targetURL *url.URL, + handshakeTimeout time.Duration, +) (httpstream.Dialer, context.CancelFunc, error) { + if config == nil || targetURL == nil { + return nil, nil, errors.New("kubernetes port-forward dialer configuration is incomplete") + } + if handshakeTimeout <= 0 { + return nil, nil, errors.New("kubernetes port-forward handshake timeout must be positive") + } + if err := ctx.Err(); err != nil { + return nil, nil, err + } + dialContext, cancel := context.WithCancel(ctx) + dialConfig := rest.CopyConfig(config) + dialConfig.NextProtos = []string{"http/1.1"} + dialConfig.Wrap(func(next http.RoundTripper) http.RoundTripper { + return &portForwardContextRoundTripper{ + ctx: dialContext, timeout: handshakeTimeout, next: next, + } + }) + websocketDialer, err := newContextWebSocketDialer(targetURL, dialConfig) + if err != nil { + cancel() + return nil, nil, err + } + spdyTransport, err := rest.TransportFor(dialConfig) + if err != nil { + cancel() + return nil, nil, err + } + spdyDialer := &contextSPDYDialer{client: &http.Client{Transport: spdyTransport}, targetURL: targetURL} + return portforward.NewFallbackDialer( + websocketDialer, spdyDialer, ShouldFallbackStream, + ), cancel, nil +} + +type contextWebSocketDialer struct { + targetURL *url.URL + transport http.RoundTripper + holder clientwebsocket.ConnectionHolder +} + +func newContextWebSocketDialer(targetURL *url.URL, config *rest.Config) (httpstream.Dialer, error) { + transportConfig, err := config.TransportConfig() + if err != nil { + return nil, err + } + tlsConfig, err := transport.TLSConfigFor(transportConfig) + if err != nil { + return nil, err + } + proxy := config.Proxy + if proxy == nil { + proxy = utilnet.NewProxierWithNoProxyCIDR(http.ProxyFromEnvironment) + } + dialContext := (&net.Dialer{}).DialContext + if transportConfig.DialHolder != nil && transportConfig.DialHolder.Dial != nil { + dialContext = transportConfig.DialHolder.Dial + } + holder := &contextWebSocketRoundTripper{ + TLSConfig: tlsConfig, + Proxier: proxy, + DialContext: dialContext, + } + wrapper, err := transport.HTTPWrappersForConfig(transportConfig, holder) + if err != nil { + return nil, err + } + return &contextWebSocketDialer{targetURL: targetURL, transport: wrapper, holder: holder}, nil +} + +func (d *contextWebSocketDialer) Dial(protocols ...string) (httpstream.Connection, string, error) { + request, err := http.NewRequest(http.MethodGet, d.targetURL.String(), nil) + if err != nil { + return nil, "", err + } + tunnelingProtocols := make([]string, 0, len(protocols)) + for _, protocol := range protocols { + tunnelingProtocols = append( + tunnelingProtocols, portforwardconstants.WebsocketsSPDYTunnelingPrefix+protocol, + ) + } + connection, err := clientwebsocket.Negotiate( + d.transport, d.holder, request, tunnelingProtocols..., + ) + if err != nil { + return nil, "", err + } + protocol := strings.TrimPrefix( + connection.Subprotocol(), portforwardconstants.WebsocketsSPDYTunnelingPrefix, + ) + tunnel := portforward.NewTunnelingConnection("client", connection) + spdyConnection, err := streamspdy.NewClientConnectionWithPings( + tunnel, portforward.PingPeriod, + ) + if err != nil { + _ = tunnel.Close() + return nil, "", err + } + return spdyConnection, protocol, nil +} + +type contextWebSocketRoundTripper struct { + TLSConfig *tls.Config + Proxier func(*http.Request) (*url.URL, error) + DialContext func(context.Context, string, string) (net.Conn, error) + conn *gwebsocket.Conn +} + +func (r *contextWebSocketRoundTripper) TLSClientConfig() *tls.Config { return r.TLSConfig } +func (r *contextWebSocketRoundTripper) DataBufferSize() int { return 32 * 1024 } +func (r *contextWebSocketRoundTripper) Connection() *gwebsocket.Conn { return r.conn } + +func (r *contextWebSocketRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + protocols := request.Header[wsstream.WebSocketProtocolHeader] + delete(request.Header, wsstream.WebSocketProtocolHeader) + requestURL, err := webSocketUpgradeURL(request.URL) + if err != nil { + return nil, err + } + var handshakeConn *detachableHandshakeConn + dialer := r.handshakeDialer(protocols, &handshakeConn) + connection, response, err := dialer.DialContext( + request.Context(), requestURL.String(), request.Header, + ) + if err != nil { + return nil, webSocketDialError(request.Context(), handshakeConn, response, err) + } + if err := finishWebSocketHandshake(connection, response, handshakeConn, protocols); err != nil { + return nil, err + } + r.conn = connection + return response, nil +} + +func webSocketUpgradeURL(source *url.URL) (url.URL, error) { + requestURL := *source + switch requestURL.Scheme { + case "https": + requestURL.Scheme = "wss" + case "http": + requestURL.Scheme = "ws" + default: + return url.URL{}, fmt.Errorf("unknown websocket URL scheme %q", requestURL.Scheme) + } + return requestURL, nil +} + +func (r *contextWebSocketRoundTripper) handshakeDialer( + protocols []string, + handshakeConn **detachableHandshakeConn, +) gwebsocket.Dialer { + dialContext := r.DialContext + if dialContext == nil { + dialContext = (&net.Dialer{}).DialContext + } + return gwebsocket.Dialer{ + Proxy: r.Proxier, + TLSClientConfig: r.TLSConfig, + Subprotocols: protocols, + ReadBufferSize: r.DataBufferSize() + 1024, + WriteBufferSize: r.DataBufferSize() + 1024, + NetDialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + connection, err := dialContext(ctx, network, address) + if err != nil { + return nil, err + } + *handshakeConn = newDetachableHandshakeConn(ctx, connection) + return *handshakeConn, nil + }, + } +} + +func webSocketDialError( + ctx context.Context, + handshakeConn *detachableHandshakeConn, + response *http.Response, + dialErr error, +) error { + if handshakeConn != nil { + handshakeConn.close() + } + if contextErr := ctx.Err(); contextErr != nil { + closeHTTPResponse(response) + return contextErr + } + if !errors.Is(dialErr, gwebsocket.ErrBadHandshake) { + closeHTTPResponse(response) + return dialErr + } + cause := dialErr + if response != nil { + cause = fmt.Errorf("%w (%s)", dialErr, response.Status) + } + closeHTTPResponse(response) + return &httpstream.UpgradeFailureError{Cause: cause} +} + +func finishWebSocketHandshake( + connection *gwebsocket.Conn, + response *http.Response, + handshakeConn *detachableHandshakeConn, + protocols []string, +) error { + if !slicesContain(protocols, connection.Subprotocol()) { + closeWebSocketHandshake(connection, response, handshakeConn) + return &httpstream.UpgradeFailureError{Cause: fmt.Errorf( + "invalid websocket protocol, expected one of %q, got %q", + protocols, connection.Subprotocol(), + )} + } + if handshakeConn == nil { + closeWebSocketHandshake(connection, response, nil) + return errors.New("websocket handshake completed without a tracked connection") + } + if err := handshakeConn.detach(); err != nil { + closeWebSocketHandshake(connection, response, nil) + return err + } + return nil +} + +func slicesContain(values []string, target string) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} + +func closeWebSocketHandshake( + connection *gwebsocket.Conn, + response *http.Response, + handshakeConn *detachableHandshakeConn, +) { + if connection != nil { + _ = connection.Close() + } + if handshakeConn != nil { + handshakeConn.close() + } + closeHTTPResponse(response) +} + +func closeHTTPResponse(response *http.Response) { + if response != nil && response.Body != nil { + _ = response.Body.Close() + } +} + +type detachableHandshakeConn struct { + net.Conn + ctx context.Context + mu sync.Mutex + detached bool + closedByContext bool + stop func() bool +} + +func newDetachableHandshakeConn(ctx context.Context, connection net.Conn) *detachableHandshakeConn { + guarded := &detachableHandshakeConn{Conn: connection, ctx: ctx} + guarded.stop = context.AfterFunc(ctx, guarded.closeIfAttached) + return guarded +} + +func (c *detachableHandshakeConn) closeIfAttached() { + c.mu.Lock() + defer c.mu.Unlock() + if !c.detached { + c.closedByContext = true + _ = c.Close() + } +} + +func (c *detachableHandshakeConn) detach() error { + c.mu.Lock() + if c.closedByContext { + err := c.ctx.Err() + if err == nil { + err = context.Canceled + } + c.mu.Unlock() + return err + } + if err := c.ctx.Err(); err != nil { + c.closedByContext = true + _ = c.Close() + c.mu.Unlock() + return err + } + c.detached = true + stop := c.stop + c.mu.Unlock() + if stop != nil { + stop() + } + return nil +} + +func (c *detachableHandshakeConn) close() { + c.mu.Lock() + if c.stop != nil { + c.stop() + } + c.detached = false + _ = c.Close() + c.mu.Unlock() +} + +// ShouldFallbackStream keeps the WebSocket-to-SPDY compatibility policy shared +// by exec and port-forward callers. +func ShouldFallbackStream(err error) bool { + if httpstream.IsUpgradeFailure(err) || httpstream.IsHTTPSProxyError(err) { + return true + } + var timeoutErr net.Error + return errors.As(err, &timeoutErr) && timeoutErr.Timeout() +} + +type portForwardContextRoundTripper struct { + ctx context.Context + timeout time.Duration + next http.RoundTripper +} + +func (r *portForwardContextRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + requestContext := newDetachablePortForwardContext(r.ctx, r.timeout) + response, err := r.next.RoundTrip(request.Clone(requestContext)) + if err == nil && response != nil && response.StatusCode == http.StatusSwitchingProtocols { + requestContext.detach() + } else { + requestContext.cancelNow() + } + return response, err +} + +type detachablePortForwardContext struct { + parent context.Context + inner context.Context + cancel context.CancelCauseFunc + deadline time.Time + timer *time.Timer + stopParent func() bool + finishOnce sync.Once + stateMu sync.RWMutex + err error + detached bool +} + +func newDetachablePortForwardContext(parent context.Context, timeout time.Duration) *detachablePortForwardContext { + inner, cancel := context.WithCancelCause(context.Background()) + initialized := make(chan struct{}) + ctx := &detachablePortForwardContext{ + parent: parent, inner: inner, cancel: cancel, deadline: time.Now().Add(timeout), + } + ctx.stopParent = context.AfterFunc(parent, func() { + <-initialized + ctx.finish(context.Cause(parent), false) + }) + ctx.timer = time.AfterFunc(timeout, func() { + <-initialized + ctx.finish(context.DeadlineExceeded, false) + }) + close(initialized) + return ctx +} + +func (c *detachablePortForwardContext) Deadline() (time.Time, bool) { + c.stateMu.RLock() + defer c.stateMu.RUnlock() + if c.detached { + return time.Time{}, false + } + return c.deadline, true +} +func (c *detachablePortForwardContext) Done() <-chan struct{} { return c.inner.Done() } +func (c *detachablePortForwardContext) Err() error { + c.stateMu.RLock() + defer c.stateMu.RUnlock() + return c.err +} +func (c *detachablePortForwardContext) Value(key any) any { return c.parent.Value(key) } +func (c *detachablePortForwardContext) detach() { c.finish(nil, true) } +func (c *detachablePortForwardContext) cancelNow() { c.finish(context.Canceled, false) } +func (c *detachablePortForwardContext) finish(cause error, detach bool) { + c.finishOnce.Do(func() { + if c.stopParent != nil { + c.stopParent() + } + if c.timer != nil { + c.timer.Stop() + } + c.stateMu.Lock() + c.detached = detach + if !detach { + c.err = context.Canceled + if errors.Is(cause, context.DeadlineExceeded) { + c.err = context.DeadlineExceeded + } + } + err := c.err + c.stateMu.Unlock() + if !detach { + c.cancel(err) + } + }) +} + +// contextSPDYDialer uses the standard client-go REST transport for the HTTP +// upgrade. Unlike client-go's legacy SPDY dialer, that transport terminates a +// blocked response-header read when the request context is canceled. The +// upgraded byte stream is still handled by Kubernetes' maintained SPDY stack. +type contextSPDYDialer struct { + client *http.Client + targetURL *url.URL + newConnection func(net.Conn, time.Duration) (httpstream.Connection, error) +} + +func (d *contextSPDYDialer) Dial(protocols ...string) (httpstream.Connection, string, error) { + request, err := http.NewRequest(http.MethodPost, d.targetURL.String(), nil) + if err != nil { + return nil, "", err + } + request.Header.Set(httpstream.HeaderConnection, httpstream.HeaderUpgrade) + request.Header.Set(httpstream.HeaderUpgrade, streamspdy.HeaderSpdy31) + for _, protocol := range protocols { + request.Header.Add(httpstream.HeaderProtocolVersion, protocol) + } + response, err := d.client.Do(request) + if err != nil { + return nil, "", fmt.Errorf("send SPDY upgrade request: %w", err) + } + if !validSPDYUpgrade(response) { + defer func() { _ = response.Body.Close() }() + message, _ := io.ReadAll(io.LimitReader(response.Body, 64*1024)) + return nil, "", &httpstream.UpgradeFailureError{Cause: fmt.Errorf( + "SPDY upgrade rejected (%s): %s", response.Status, strings.TrimSpace(string(message)), + )} + } + stream, ok := response.Body.(io.ReadWriteCloser) + if !ok { + _ = response.Body.Close() + return nil, "", errors.New("SPDY upgrade response is not a writable stream") + } + newConnection := d.newConnection + if newConnection == nil { + newConnection = streamspdy.NewClientConnectionWithPings + } + connection, err := newConnection( + &upgradedResponseConn{ReadWriteCloser: stream, remote: d.targetURL.Host}, 5*time.Second, + ) + if err != nil { + _ = stream.Close() + return nil, "", err + } + return connection, response.Header.Get(httpstream.HeaderProtocolVersion), nil +} + +func validSPDYUpgrade(response *http.Response) bool { + if response == nil || response.StatusCode != http.StatusSwitchingProtocols { + return false + } + connection := strings.ToLower(response.Header.Get(httpstream.HeaderConnection)) + upgrade := strings.ToLower(response.Header.Get(httpstream.HeaderUpgrade)) + return strings.Contains(connection, strings.ToLower(httpstream.HeaderUpgrade)) && + strings.Contains(upgrade, strings.ToLower(streamspdy.HeaderSpdy31)) +} + +type upgradedResponseConn struct { + io.ReadWriteCloser + remote string +} + +func (c *upgradedResponseConn) LocalAddr() net.Addr { return portForwardAddr("local") } +func (c *upgradedResponseConn) RemoteAddr() net.Addr { return portForwardAddr(c.remote) } +func (c *upgradedResponseConn) SetDeadline(time.Time) error { return nil } +func (c *upgradedResponseConn) SetReadDeadline(time.Time) error { return nil } +func (c *upgradedResponseConn) SetWriteDeadline(time.Time) error { return nil } + +type portForwardAddr string + +func (a portForwardAddr) Network() string { return "tcp" } +func (a portForwardAddr) String() string { return string(a) } diff --git a/apps/backend/internal/agent/kubernetes/portforward_test.go b/apps/backend/internal/agent/kubernetes/portforward_test.go new file mode 100644 index 0000000000..7eedce7d31 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/portforward_test.go @@ -0,0 +1,348 @@ +package kubernetes + +import ( + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + gwebsocket "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/httpstream" + streamspdy "k8s.io/apimachinery/pkg/util/httpstream/spdy" + portforwardconstants "k8s.io/apimachinery/pkg/util/portforward" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" +) + +func TestPortForwardDialerBoundsBlockedWebSocketPrimaryAndUsesPOSTFallback(t *testing.T) { + getStarted := make(chan struct{}) + postStarted := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.Method { + case http.MethodGet: + close(getStarted) + <-request.Context().Done() + case http.MethodPost: + close(postStarted) + http.Error(response, "SPDY rejected", http.StatusBadRequest) + } + })) + t.Cleanup(server.Close) + target, err := url.Parse(server.URL + "/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + dialer, cancel, err := NewPortForwardDialer( + context.Background(), &rest.Config{Host: server.URL}, target, 40*time.Millisecond, + ) + require.NoError(t, err) + t.Cleanup(cancel) + + _, _, err = dialer.Dial(portforward.PortForwardProtocolV1Name) + + require.Error(t, err) + requireClosed(t, getStarted, "WebSocket GET primary was not attempted") + requireClosed(t, postStarted, "SPDY POST fallback was not attempted") +} + +func TestPortForwardDialerCancelClosesBlockedWebSocketResponseRead(t *testing.T) { + getStarted := make(chan struct{}) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet { + http.Error(response, "fallback unavailable", http.StatusBadRequest) + return + } + close(getStarted) + <-release + })) + t.Cleanup(func() { + close(release) + server.Close() + }) + target, err := url.Parse(server.URL + "/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + ctx, cancelContext := context.WithCancel(context.Background()) + dialer, cancelDialer, err := NewPortForwardDialer( + ctx, &rest.Config{Host: server.URL}, target, 5*time.Second, + ) + require.NoError(t, err) + result := make(chan error, 1) + go func() { + _, _, dialErr := dialer.Dial(portforward.PortForwardProtocolV1Name) + result <- dialErr + }() + requireClosed(t, getStarted, "WebSocket GET primary was not attempted") + + cancelContext() + cancelDialer() + + select { + case err := <-result: + require.Error(t, err) + require.ErrorIs(t, err, context.Canceled) + case <-time.After(250 * time.Millisecond): + t.Fatal("blocked WebSocket response-header read ignored cancellation") + } +} + +func TestContextWebSocketDialerUsesConfiguredRESTDial(t *testing.T) { + target, err := url.Parse("http://127.0.0.1:1/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + dialErr := errors.New("configured REST dial invoked") + dialer, err := newContextWebSocketDialer(target, &rest.Config{ + Host: target.Scheme + "://" + target.Host, + Dial: func(context.Context, string, string) (net.Conn, error) { + return nil, dialErr + }, + }) + require.NoError(t, err) + + _, _, err = dialer.Dial(portforward.PortForwardProtocolV1Name) + + require.ErrorIs(t, err, dialErr) +} + +func TestDetachableHandshakeConnRejectsDetachAfterCancellation(t *testing.T) { + client, server := net.Pipe() + t.Cleanup(func() { _ = server.Close() }) + ctx, cancel := context.WithCancel(context.Background()) + guarded := newDetachableHandshakeConn(ctx, client) + cancel() + + err := guarded.detach() + + require.ErrorIs(t, err, context.Canceled) + _, writeErr := guarded.Write([]byte("closed")) + require.Error(t, writeErr, "cancellation that wins detachment must close the raw connection") +} + +func TestPortForwardDialerCancelBoundsBlockedSPDYFallback(t *testing.T) { + postStarted := make(chan struct{}) + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodGet { + http.Error(response, "upgrade rejected", http.StatusBadRequest) + return + } + close(postStarted) + select { + case <-request.Context().Done(): + case <-release: + } + })) + t.Cleanup(func() { + close(release) + server.Close() + }) + target, err := url.Parse(server.URL + "/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + ctx, cancelContext := context.WithCancel(context.Background()) + dialer, cancelDialer, err := NewPortForwardDialer( + ctx, &rest.Config{Host: server.URL}, target, time.Second, + ) + require.NoError(t, err) + result := make(chan error, 1) + go func() { + _, _, dialErr := dialer.Dial(portforward.PortForwardProtocolV1Name) + result <- dialErr + }() + requireClosed(t, postStarted, "SPDY POST fallback was not attempted") + cancelContext() + cancelDialer() + + select { + case err := <-result: + require.Error(t, err) + case <-time.After(250 * time.Millisecond): + t.Fatal("cancel left the SPDY fallback handshake goroutine blocked") + } +} + +func TestPortForwardSPDYFallbackSurvivesHandshakeContextAfterUpgrade(t *testing.T) { + serverConnection := make(chan httpstream.Connection, 1) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodGet { + http.Error(response, "WebSocket unavailable", http.StatusBadRequest) + return + } + protocol := request.Header.Get(httpstream.HeaderProtocolVersion) + response.Header().Set(httpstream.HeaderProtocolVersion, protocol) + connection := streamspdy.NewResponseUpgrader().UpgradeResponse( + response, request, func(httpstream.Stream, <-chan struct{}) error { return nil }, + ) + serverConnection <- connection + })) + t.Cleanup(server.Close) + target, err := url.Parse(server.URL + "/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + handshakeTimeout := 25 * time.Millisecond + dialer, cancel, err := NewPortForwardDialer( + context.Background(), &rest.Config{Host: server.URL}, target, handshakeTimeout, + ) + require.NoError(t, err) + + connection, protocol, err := dialer.Dial(portforward.PortForwardProtocolV1Name) + + require.NoError(t, err) + require.Equal(t, portforward.PortForwardProtocolV1Name, protocol) + time.Sleep(2 * handshakeTimeout) + cancel() + select { + case <-connection.CloseChan(): + t.Fatal("successful SPDY fallback remained coupled to its handshake context") + case <-time.After(50 * time.Millisecond): + } + require.NoError(t, connection.Close()) + select { + case peer := <-serverConnection: + if peer != nil { + _ = peer.Close() + } + case <-time.After(time.Second): + t.Fatal("SPDY server did not complete upgrade") + } +} + +// Reviewer-requested contract coverage: successful WebSocket upgrade must be +// detached from both handshake timer and returned dialer cancellation. +func TestPortForwardWebSocketPrimarySurvivesHandshakeContextAfterUpgrade(t *testing.T) { + serverStreams := make(chan httpstream.Stream, 1) + serverErrors := make(chan error, 1) + stopServer := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + upgrader := gwebsocket.Upgrader{ + CheckOrigin: func(*http.Request) bool { return true }, + Subprotocols: []string{portforwardconstants.WebsocketsSPDYTunnelingPortForwardV1}, + } + websocketConnection, err := upgrader.Upgrade(response, request, nil) + if err != nil { + serverErrors <- err + return + } + tunnel := portforward.NewTunnelingConnection("server", websocketConnection) + serverConnection, err := streamspdy.NewServerConnection( + tunnel, func(stream httpstream.Stream, _ <-chan struct{}) error { + serverStreams <- stream + return nil + }, + ) + if err != nil { + serverErrors <- err + return + } + defer serverConnection.Close() //nolint:errcheck + <-stopServer + })) + t.Cleanup(func() { + close(stopServer) + server.Close() + }) + target, err := url.Parse(server.URL + "/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + handshakeTimeout := 25 * time.Millisecond + dialer, cancel, err := NewPortForwardDialer( + context.Background(), &rest.Config{Host: server.URL}, target, handshakeTimeout, + ) + require.NoError(t, err) + + connection, protocol, err := dialer.Dial(portforward.PortForwardProtocolV1Name) + require.NoError(t, err) + require.Equal(t, portforward.PortForwardProtocolV1Name, protocol) + time.Sleep(2 * handshakeTimeout) + cancel() + select { + case <-connection.CloseChan(): + t.Fatal("successful WebSocket primary remained coupled to handshake context") + default: + } + + stream, err := connection.CreateStream(http.Header{}) + require.NoError(t, err) + _, err = io.Copy(stream, strings.NewReader("still-alive")) + require.NoError(t, err) + require.NoError(t, stream.Close()) + select { + case serverStream := <-serverStreams: + payload, readErr := io.ReadAll(serverStream) + require.NoError(t, readErr) + require.Equal(t, "still-alive", string(payload)) + _ = serverStream.Close() + case serverErr := <-serverErrors: + t.Fatalf("WebSocket SPDY server failed: %v", serverErr) + case <-time.After(time.Second): + t.Fatal("detached WebSocket connection was not usable") + } + require.NoError(t, connection.Close()) +} + +func TestDetachablePortForwardContextPreservesDeadlineError(t *testing.T) { + ctx := newDetachablePortForwardContext(context.Background(), 10*time.Millisecond) + + select { + case <-ctx.Done(): + case <-time.After(time.Second): + t.Fatal("detachable context deadline did not fire") + } + require.ErrorIs(t, ctx.Err(), context.DeadlineExceeded) +} + +func TestContextSPDYDialerClosesUpgradeStreamWhenConnectionInitFails(t *testing.T) { + stream := &trackingUpgradedStream{} + target, err := url.Parse("https://cluster.example/api/v1/namespaces/agents/pods/agent/portforward") + require.NoError(t, err) + dialer := &contextSPDYDialer{ + targetURL: target, + client: &http.Client{Transport: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusSwitchingProtocols, + Status: "101 Switching Protocols", + Header: http.Header{ + httpstream.HeaderConnection: []string{httpstream.HeaderUpgrade}, + httpstream.HeaderUpgrade: []string{streamspdy.HeaderSpdy31}, + httpstream.HeaderProtocolVersion: []string{portforward.PortForwardProtocolV1Name}, + }, + Body: stream, + }, nil + })}, + newConnection: func(net.Conn, time.Duration) (httpstream.Connection, error) { + return nil, errors.New("SPDY init failed") + }, + } + + _, _, err = dialer.Dial(portforward.PortForwardProtocolV1Name) + + require.ErrorContains(t, err, "SPDY init failed") + require.True(t, stream.closed, "failed SPDY initialization must close upgraded response stream") +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +type trackingUpgradedStream struct { + closed bool +} + +func (*trackingUpgradedStream) Read([]byte) (int, error) { return 0, io.EOF } +func (*trackingUpgradedStream) Write(data []byte) (int, error) { return len(data), nil } +func (s *trackingUpgradedStream) Close() error { + s.closed = true + return nil +} + +func requireClosed(t *testing.T, channel <-chan struct{}, message string) { + t.Helper() + select { + case <-channel: + case <-time.After(time.Second): + t.Fatal(message) + } +} diff --git a/apps/backend/internal/agent/kubernetes/storage.go b/apps/backend/internal/agent/kubernetes/storage.go new file mode 100644 index 0000000000..8a8e47bfcc --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/storage.go @@ -0,0 +1,66 @@ +package kubernetes + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" +) + +type PVCOptions struct { + Name string + Namespace string + Identity ResourceIdentity +} + +func BuildPersistentVolumeClaim(workspace WorkspaceConfig, options PVCOptions) (*corev1.PersistentVolumeClaim, error) { + if workspace.Mode != WorkspaceModeManagedPVC { + return nil, fieldError("config.workspace.mode", "must be managed_pvc to create a claim") + } + if err := workspace.validateManagedPVC(); err != nil { + return nil, err + } + if len(validation.IsDNS1123Subdomain(options.Name)) > 0 { + return nil, fieldError("pvc.metadata.name", "must be a valid claim name") + } + if len(validation.IsDNS1123Label(options.Namespace)) > 0 { + return nil, fieldError("pvc.metadata.namespace", "must be a valid namespace") + } + labels, err := OwnershipLabels(options.Identity) + if err != nil { + return nil, err + } + quantity, err := resource.ParseQuantity(strings.TrimSpace(workspace.Size)) + if err != nil { + return nil, fieldError("config.workspace.size", "must be a positive Kubernetes quantity") + } + volumeMode := corev1.PersistentVolumeFilesystem + pvc := &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolumeClaim"}, + ObjectMeta: metav1.ObjectMeta{ + Name: options.Name, Namespace: options.Namespace, Labels: labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: accessModes(workspace.AccessModes), + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: quantity}, + }, + VolumeMode: &volumeMode, + }, + } + if workspace.StorageClass != "" { + storageClass := workspace.StorageClass + pvc.Spec.StorageClassName = &storageClass + } + return pvc, nil +} + +func accessModes(values []string) []corev1.PersistentVolumeAccessMode { + modes := make([]corev1.PersistentVolumeAccessMode, len(values)) + for i, value := range values { + modes[i] = corev1.PersistentVolumeAccessMode(value) + } + return modes +} diff --git a/apps/backend/internal/agent/kubernetes/storage_test.go b/apps/backend/internal/agent/kubernetes/storage_test.go new file mode 100644 index 0000000000..5ddd13e075 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/storage_test.go @@ -0,0 +1,64 @@ +package kubernetes + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestBuildPersistentVolumeClaimSynthesizesManagedStorage(t *testing.T) { + t.Parallel() + + workspace := WorkspaceConfig{ + Mode: WorkspaceModeManagedPVC, Size: " 25Gi ", StorageClass: "fast", + AccessModes: []string{"ReadWriteOnce", "ReadWriteMany"}, + } + options := PVCOptions{Name: "workspace-pvc", Namespace: "agents", Identity: podOptions().Identity} + + first, err := BuildPersistentVolumeClaim(workspace, options) + if err != nil { + t.Fatalf("BuildPersistentVolumeClaim() error = %v", err) + } + second, err := BuildPersistentVolumeClaim(workspace, options) + if err != nil { + t.Fatalf("second BuildPersistentVolumeClaim() error = %v", err) + } + if !reflect.DeepEqual(first, second) { + t.Fatal("BuildPersistentVolumeClaim() is not deterministic") + } + if first.APIVersion != "v1" || first.Kind != "PersistentVolumeClaim" || first.Name != options.Name || first.Namespace != options.Namespace { + t.Fatalf("PVC identity = %#v", first.TypeMeta) + } + if len(first.OwnerReferences) != 0 { + t.Fatalf("PVC ownerReferences = %v, want none", first.OwnerReferences) + } + if first.Labels["kandev.ai/session-id"] != options.Identity.SessionID { + t.Fatalf("PVC labels = %v", first.Labels) + } + if first.Spec.StorageClassName == nil || *first.Spec.StorageClassName != "fast" { + t.Fatalf("storageClassName = %v", first.Spec.StorageClassName) + } + wantModes := []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany} + if !reflect.DeepEqual(first.Spec.AccessModes, wantModes) { + t.Fatalf("accessModes = %v, want %v", first.Spec.AccessModes, wantModes) + } + if got := first.Spec.Resources.Requests.Storage(); got == nil || got.Cmp(resource.MustParse("25Gi")) != 0 { + t.Fatalf("storage request = %v", got) + } + if first.Spec.VolumeMode == nil || *first.Spec.VolumeMode != corev1.PersistentVolumeFilesystem { + t.Fatalf("volumeMode = %v", first.Spec.VolumeMode) + } +} + +func TestBuildPersistentVolumeClaimRejectsUnmanagedModes(t *testing.T) { + t.Parallel() + + for _, mode := range []WorkspaceMode{WorkspaceModeEmptyDir, WorkspaceModeExistingClaim} { + _, err := BuildPersistentVolumeClaim(WorkspaceConfig{Mode: mode, ClaimName: "existing"}, PVCOptions{ + Name: "workspace-pvc", Namespace: "agents", Identity: podOptions().Identity, + }) + assertFieldPath(t, err, "config.workspace.mode") + } +} diff --git a/apps/backend/internal/agent/kubernetes/stream.go b/apps/backend/internal/agent/kubernetes/stream.go new file mode 100644 index 0000000000..e29ac93964 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/stream.go @@ -0,0 +1,135 @@ +package kubernetes + +import ( + "context" + "errors" + "io" + + "k8s.io/apimachinery/pkg/util/validation" + + "github.com/kandev/kandev/internal/agent/runtime/routingerr" +) + +const loopbackAddress = "127.0.0.1" + +type ExecRequest struct { + Namespace string + Pod string + Container string + Command []string + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + TTY bool +} + +type PodExecutor interface { + Exec(context.Context, ExecRequest) error +} + +type PortForwardRequest struct { + Namespace string + Pod string + LocalAddress string + LocalPort uint16 + RemotePort uint16 +} + +type PortForwardSession interface { + LocalPort() uint16 + Ready() <-chan struct{} + Done() <-chan error + Close() error +} + +type PortForwarder interface { + Forward(context.Context, PortForwardRequest) (PortForwardSession, error) +} + +type StreamOperations struct { + executor PodExecutor + forwarder PortForwarder +} + +func NewStreamOperations(executor PodExecutor, forwarder PortForwarder) *StreamOperations { + return &StreamOperations{executor: executor, forwarder: forwarder} +} + +func (s *StreamOperations) Exec(ctx context.Context, request ExecRequest) error { + if err := validateExecRequest(s.executor, request); err != nil { + return err + } + if err := s.executor.Exec(ctx, request); err != nil { + return sanitizeStreamError("exec", err) + } + return nil +} + +func (s *StreamOperations) Forward(ctx context.Context, request PortForwardRequest) (PortForwardSession, error) { + if err := validateForwardRequest(s.forwarder, &request); err != nil { + return nil, err + } + session, err := s.forwarder.Forward(ctx, request) + if err != nil { + return nil, sanitizeStreamError("port forward", err) + } + if session == nil { + return nil, sanitizeStreamError("port forward", errors.New("forwarder returned nil session")) + } + return session, nil +} + +func validateExecRequest(executor PodExecutor, request ExecRequest) error { + if executor == nil { + return fieldError("exec.executor", "is not configured") + } + if len(validation.IsDNS1123Label(request.Namespace)) > 0 { + return fieldError("exec.namespace", "must be a valid namespace") + } + if len(validation.IsDNS1123Subdomain(request.Pod)) > 0 { + return fieldError("exec.pod", "must be a valid Pod name") + } + if len(validation.IsDNS1123Label(request.Container)) > 0 { + return fieldError("exec.container", "must be a valid container name") + } + if len(request.Command) == 0 { + return fieldError("exec.command", "is required") + } + return nil +} + +func validateForwardRequest(forwarder PortForwarder, request *PortForwardRequest) error { + if forwarder == nil { + return fieldError("port_forward.forwarder", "is not configured") + } + if len(validation.IsDNS1123Label(request.Namespace)) > 0 { + return fieldError("port_forward.namespace", "must be a valid namespace") + } + if len(validation.IsDNS1123Subdomain(request.Pod)) > 0 { + return fieldError("port_forward.pod", "must be a valid Pod name") + } + if request.LocalAddress == "" { + request.LocalAddress = loopbackAddress + } + if request.LocalAddress != loopbackAddress { + return fieldError("port_forward.local_address", "must be "+loopbackAddress) + } + if request.RemotePort == 0 { + return fieldError("port_forward.remote_port", "must be non-zero") + } + return nil +} + +type streamError struct { + operation string + cause error +} + +func (e *streamError) Error() string { + return "kubernetes " + e.operation + " failed: " + routingerr.Sanitize(e.cause.Error()) +} +func (e *streamError) Unwrap() error { return e.cause } + +func sanitizeStreamError(operation string, cause error) error { + return &streamError{operation: operation, cause: cause} +} diff --git a/apps/backend/internal/agent/kubernetes/stream_test.go b/apps/backend/internal/agent/kubernetes/stream_test.go new file mode 100644 index 0000000000..deea50f178 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/stream_test.go @@ -0,0 +1,123 @@ +package kubernetes + +import ( + "context" + "errors" + "strings" + "testing" +) + +type recordingPodExecutor struct { + request ExecRequest + err error + calls int +} + +func (e *recordingPodExecutor) Exec(_ context.Context, request ExecRequest) error { + e.calls++ + e.request = request + return e.err +} + +type recordingPortForwarder struct { + request PortForwardRequest + session PortForwardSession + err error + calls int +} + +func (f *recordingPortForwarder) Forward(_ context.Context, request PortForwardRequest) (PortForwardSession, error) { + f.calls++ + f.request = request + return f.session, f.err +} + +type fakeForwardSession struct{} + +func (fakeForwardSession) LocalPort() uint16 { return 43123 } +func (fakeForwardSession) Ready() <-chan struct{} { return make(chan struct{}) } +func (fakeForwardSession) Done() <-chan error { return make(chan error) } +func (fakeForwardSession) Close() error { return nil } + +func TestStreamOperationsExecDelegatesValidatedRequest(t *testing.T) { + t.Parallel() + + executor := &recordingPodExecutor{} + streams := NewStreamOperations(executor, nil) + request := ExecRequest{Namespace: "agents", Pod: "pod-1", Container: "kandev-agent", Command: []string{"mkdir", "-p", "/run/kandev"}} + if err := streams.Exec(context.Background(), request); err != nil { + t.Fatalf("Exec() error = %v", err) + } + if executor.calls != 1 || executor.request.Pod != request.Pod || executor.request.Container != request.Container { + t.Fatalf("executor request = %#v, calls %d", executor.request, executor.calls) + } +} + +func TestStreamOperationsForwardDefaultsLoopbackAndDelegates(t *testing.T) { + t.Parallel() + + forwarder := &recordingPortForwarder{session: fakeForwardSession{}} + streams := NewStreamOperations(nil, forwarder) + session, err := streams.Forward(context.Background(), PortForwardRequest{ + Namespace: "agents", Pod: "pod-1", RemotePort: uint16(DefaultAgentctlPort), + }) + if err != nil { + t.Fatalf("Forward() error = %v", err) + } + if session.LocalPort() != 43123 || forwarder.request.LocalAddress != "127.0.0.1" || forwarder.calls != 1 { + t.Fatalf("forward = session %v request %#v calls %d", session, forwarder.request, forwarder.calls) + } +} + +func TestStreamOperationsRejectsNonLoopbackForwardBeforeDelegation(t *testing.T) { + t.Parallel() + + forwarder := &recordingPortForwarder{} + streams := NewStreamOperations(nil, forwarder) + _, err := streams.Forward(context.Background(), PortForwardRequest{ + Namespace: "agents", Pod: "pod-1", LocalAddress: "0.0.0.0", RemotePort: uint16(DefaultAgentctlPort), + }) + assertFieldPath(t, err, "port_forward.local_address") + if forwarder.calls != 0 { + t.Fatal("Forward() delegated an unsafe bind address") + } +} + +func TestStreamOperationsRedactsBoundaryErrorsAndPreservesCause(t *testing.T) { + t.Parallel() + + cause := errors.New("exec forbidden at https://cluster.example/private Authorization: Bearer super-secret-token-value") + executor := &recordingPodExecutor{err: cause} + err := NewStreamOperations(executor, nil).Exec(context.Background(), ExecRequest{ + Namespace: "agents", Pod: "pod-1", Container: "kandev-agent", Command: []string{"true"}, + }) + if !errors.Is(err, cause) { + t.Fatalf("Exec() error = %v, want wrapped cause", err) + } + if strings.Contains(err.Error(), "super-secret-token-value") { + t.Fatalf("Exec() exposed credential: %q", err) + } + if !strings.Contains(err.Error(), "exec forbidden") || !strings.Contains(err.Error(), "https://cluster.example") { + t.Fatalf("Exec() discarded safe causal diagnostics: %q", err) + } + if strings.Contains(err.Error(), "/private") { + t.Fatalf("Exec() exposed URL path: %q", err) + } +} + +func TestStreamOperationsRedactsPortForwardErrorsAndPreservesCause(t *testing.T) { + t.Parallel() + + cause := errors.New("port forward forbidden at https://cluster.example/private?token=kandev_pat_super-secret-value") + forwarder := &recordingPortForwarder{err: cause} + _, err := NewStreamOperations(nil, forwarder).Forward(context.Background(), PortForwardRequest{ + Namespace: "agents", Pod: "pod-1", RemotePort: uint16(DefaultAgentctlPort), + }) + if !errors.Is(err, cause) { + t.Fatalf("Forward() error = %v, want wrapped cause", err) + } + if !strings.Contains(err.Error(), "port forward forbidden") || strings.Contains(err.Error(), "/private") || + strings.Contains(err.Error(), "super-secret-value") { + t.Fatalf("Forward() diagnostics were not safely preserved: %q", err) + } +} diff --git a/apps/backend/internal/agent/kubernetes/template.go b/apps/backend/internal/agent/kubernetes/template.go new file mode 100644 index 0000000000..14a2abe966 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/template.go @@ -0,0 +1,291 @@ +package kubernetes + +import ( + "fmt" + "io" + pathpkg "path" + "strings" + + yamlv3 "gopkg.in/yaml.v3" + corev1 "k8s.io/api/core/v1" + sigsyaml "sigs.k8s.io/yaml" +) + +const ( + MaxPodTemplateBytes = 256 * 1024 + DefaultAgentctlPort = int32(8765) + RuntimeVolumeName = "kandev-runtime" + AuthVolumeName = "kandev-auth" + WorkspaceVolumeName = "kandev-workspace" + RuntimeMountPath = "/opt/kandev" + AuthMountPath = "/run/kandev" + WorkspaceMountPath = "/workspace" + AgentctlContainerPort = "kandev-agentctl" + podTemplateFieldPrefix = "config.pod_template_yaml" +) + +var reservedLabelKeys = map[string]bool{ + "app.kubernetes.io/component": true, + "app.kubernetes.io/instance": true, + "app.kubernetes.io/managed-by": true, + "app.kubernetes.io/name": true, +} + +// ParsePodTemplate decodes exactly one strict core/v1 PodTemplate document. +func ParsePodTemplate(raw string) (*corev1.PodTemplate, error) { + if len(raw) > MaxPodTemplateBytes { + return nil, fieldError(podTemplateFieldPrefix, "must not exceed 256 KiB") + } + if strings.TrimSpace(raw) == "" { + return nil, fieldError(podTemplateFieldPrefix, "is required") + } + if err := requireSingleYAMLDocument(raw); err != nil { + return nil, err + } + var template corev1.PodTemplate + if err := sigsyaml.UnmarshalStrict([]byte(raw), &template); err != nil { + return nil, fieldError(podTemplateFieldPrefix, "must be strict PodTemplate YAML") + } + if template.APIVersion != "v1" || template.Kind != "PodTemplate" { + return nil, fieldError(podTemplateFieldPrefix, "must have apiVersion v1 and kind PodTemplate") + } + return &template, nil +} + +// ValidatePodTemplate rejects fields owned by Kandev before any API write. +func ValidatePodTemplate(template *corev1.PodTemplate, mainContainer string) error { + if template == nil { + return fieldError(podTemplateFieldPrefix, "is required") + } + if err := validateTemplateMetadata(template); err != nil { + return err + } + if err := validatePodSpecFields(&template.Template.Spec); err != nil { + return err + } + return validateTemplateContainers(&template.Template.Spec, mainContainer) +} + +func requireSingleYAMLDocument(raw string) error { + decoder := yamlv3.NewDecoder(strings.NewReader(raw)) + var document yamlv3.Node + if err := decoder.Decode(&document); err != nil { + return fieldError(podTemplateFieldPrefix, "must be valid YAML") + } + var extra yamlv3.Node + err := decoder.Decode(&extra) + if err == nil { + return fieldError(podTemplateFieldPrefix, "must contain exactly one YAML document") + } + if err != io.EOF { + return fieldError(podTemplateFieldPrefix, "must be valid YAML") + } + return nil +} + +func validateTemplateMetadata(template *corev1.PodTemplate) error { + meta := &template.Template.ObjectMeta + checks := []struct { + set bool + path string + }{ + {meta.Name != "", ".template.metadata.name"}, + {meta.GenerateName != "", ".template.metadata.generateName"}, + {meta.Namespace != "", ".template.metadata.namespace"}, + {meta.UID != "", ".template.metadata.uid"}, + {meta.ResourceVersion != "", ".template.metadata.resourceVersion"}, + {meta.Generation != 0, ".template.metadata.generation"}, + {!meta.CreationTimestamp.IsZero(), ".template.metadata.creationTimestamp"}, + {meta.DeletionTimestamp != nil, ".template.metadata.deletionTimestamp"}, + {len(meta.OwnerReferences) > 0, ".template.metadata.ownerReferences"}, + {len(meta.Finalizers) > 0, ".template.metadata.finalizers"}, + {len(meta.ManagedFields) > 0, ".template.metadata.managedFields"}, + } + for _, check := range checks { + if check.set { + return reservedField(check.path) + } + } + for key := range meta.Labels { + if reservedLabelKeys[key] || strings.HasPrefix(key, "kandev.ai/") { + return reservedField(".template.metadata.labels." + key) + } + } + for key := range meta.Annotations { + if strings.HasPrefix(key, "kandev.ai/") { + return reservedField(".template.metadata.annotations." + key) + } + } + return nil +} + +func validatePodSpecFields(spec *corev1.PodSpec) error { + if spec.RestartPolicy != "" { + return reservedField(".template.spec.restartPolicy") + } + if spec.NodeName != "" { + return reservedField(".template.spec.nodeName") + } + if spec.OS != nil { + return reservedField(".template.spec.os") + } + for _, key := range []string{corev1.LabelOSStable, corev1.LabelArchStable} { + if _, ok := spec.NodeSelector[key]; ok { + return reservedField(".template.spec.nodeSelector." + key) + } + } + if err := validateNodeAffinity(spec.Affinity); err != nil { + return err + } + for i, volume := range spec.Volumes { + if isReservedVolumeName(volume.Name) { + return reservedField(fmt.Sprintf(".template.spec.volumes[%d].name", i)) + } + } + return nil +} + +func validateTemplateContainers(spec *corev1.PodSpec, mainContainer string) error { + mainIndex := -1 + for i := range spec.Containers { + container := &spec.Containers[i] + path := fmt.Sprintf(".template.spec.containers[%d]", i) + if container.Image == "" { + return fieldError(podTemplateFieldPrefix+path+".image", "is required") + } + if container.Name == mainContainer { + if mainIndex >= 0 { + return fieldError(podTemplateFieldPrefix+path+".name", "duplicates the main container") + } + mainIndex = i + if err := validateMainContainerFields(container, path); err != nil { + return err + } + } + if err := validateContainerReservedFields(container, path); err != nil { + return err + } + } + if mainIndex < 0 { + return fieldError(podTemplateFieldPrefix+".template.spec.containers", "must contain the configured main container") + } + return validateAuxiliaryContainers(spec) +} + +func validateMainContainerFields(container *corev1.Container, path string) error { + switch { + case len(container.Command) > 0: + return reservedField(path + ".command") + case len(container.Args) > 0: + return reservedField(path + ".args") + case container.WorkingDir != "": + return reservedField(path + ".workingDir") + default: + return nil + } +} + +func validateContainerReservedFields(container *corev1.Container, path string) error { + if len(container.EnvFrom) > 0 { + return reservedField(path + ".envFrom") + } + for i, env := range container.Env { + if isReservedEnvironmentKey(env.Name) { + return reservedField(fmt.Sprintf("%s.env[%d].name", path, i)) + } + } + for i, port := range container.Ports { + if port.Name == AgentctlContainerPort || port.ContainerPort == DefaultAgentctlPort { + return reservedField(fmt.Sprintf("%s.ports[%d]", path, i)) + } + } + for i, mount := range container.VolumeMounts { + if isReservedVolumeName(mount.Name) || isReservedMountPath(mount.MountPath) { + return reservedField(fmt.Sprintf("%s.volumeMounts[%d]", path, i)) + } + } + for i, device := range container.VolumeDevices { + if isReservedVolumeName(device.Name) { + return reservedField(fmt.Sprintf("%s.volumeDevices[%d].name", path, i)) + } + if isReservedMountPath(device.DevicePath) { + return reservedField(fmt.Sprintf("%s.volumeDevices[%d].devicePath", path, i)) + } + } + return nil +} + +func isReservedEnvironmentKey(name string) bool { + return name == "HOME" || strings.HasPrefix(name, "AGENTCTL_") || strings.HasPrefix(name, "KANDEV_") +} + +func validateAuxiliaryContainers(spec *corev1.PodSpec) error { + for i := range spec.InitContainers { + path := fmt.Sprintf(".template.spec.initContainers[%d]", i) + if err := validateContainerReservedFields(&spec.InitContainers[i], path); err != nil { + return err + } + } + for i := range spec.EphemeralContainers { + path := fmt.Sprintf(".template.spec.ephemeralContainers[%d]", i) + container := corev1.Container(spec.EphemeralContainers[i].EphemeralContainerCommon) + if err := validateContainerReservedFields(&container, path); err != nil { + return err + } + } + return nil +} + +func validateNodeAffinity(affinity *corev1.Affinity) error { + if affinity == nil || affinity.NodeAffinity == nil { + return nil + } + nodeAffinity := affinity.NodeAffinity + if nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil { + for i, term := range nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + if path := reservedSelectorPath(term, fmt.Sprintf(".template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[%d]", i)); path != "" { + return reservedField(path) + } + } + } + for i, term := range nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + base := fmt.Sprintf(".template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution[%d].preference", i) + if path := reservedSelectorPath(term.Preference, base); path != "" { + return reservedField(path) + } + } + return nil +} + +func reservedSelectorPath(term corev1.NodeSelectorTerm, base string) string { + for i, requirement := range term.MatchExpressions { + if requirement.Key == corev1.LabelOSStable || requirement.Key == corev1.LabelArchStable { + return fmt.Sprintf("%s.matchExpressions[%d].key", base, i) + } + } + for i, requirement := range term.MatchFields { + if requirement.Key == corev1.LabelOSStable || requirement.Key == corev1.LabelArchStable { + return fmt.Sprintf("%s.matchFields[%d].key", base, i) + } + } + return "" +} + +func isReservedVolumeName(name string) bool { + return name == RuntimeVolumeName || name == AuthVolumeName || name == WorkspaceVolumeName +} + +func isReservedMountPath(path string) bool { + cleaned := pathpkg.Clean(path) + return isPathAtOrBelow(cleaned, RuntimeMountPath) || + isPathAtOrBelow(cleaned, AuthMountPath) || + isPathAtOrBelow(cleaned, WorkspaceMountPath) +} + +func isPathAtOrBelow(candidate, root string) bool { + return candidate == root || strings.HasPrefix(candidate, root+"/") +} + +func reservedField(suffix string) error { + return fieldError(podTemplateFieldPrefix+suffix, "is reserved for Kandev") +} diff --git a/apps/backend/internal/agent/kubernetes/template_validation_test.go b/apps/backend/internal/agent/kubernetes/template_validation_test.go new file mode 100644 index 0000000000..1dbc8cd272 --- /dev/null +++ b/apps/backend/internal/agent/kubernetes/template_validation_test.go @@ -0,0 +1,204 @@ +package kubernetes + +import ( + "errors" + "testing" +) + +func TestProfileConfigValidateRejectsInvalidPodTemplates(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + yaml string + path string + }{ + { + name: "unknown field", + yaml: validPodTemplate(" unknownField: true\n"), + path: "config.pod_template_yaml", + }, + { + name: "multiple documents", + yaml: validPodTemplate("") + "---\napiVersion: v1\nkind: PodTemplate\ntemplate: {}\n", + path: "config.pod_template_yaml", + }, + { + name: "wrong kind", + yaml: "apiVersion: v1\nkind: Pod\nmetadata: {}\nspec: {}\n", + path: "config.pod_template_yaml", + }, + { + name: "missing main image", + yaml: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n", + path: "config.pod_template_yaml.template.spec.containers[0].image", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := validProfile(tt.yaml).Validate() + assertFieldPath(t, err, tt.path) + }) + } +} + +func TestProfileConfigValidateRejectsReservedPodFields(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + yaml string + path string + }{ + { + name: "pod name", + yaml: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n metadata:\n name: foreign\n spec:\n containers:\n - name: kandev-agent\n image: example/agent:latest\n", + path: "config.pod_template_yaml.template.metadata.name", + }, + { + name: "identity label", + yaml: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n metadata:\n labels:\n kandev.ai/session-id: foreign\n spec:\n containers:\n - name: kandev-agent\n image: example/agent:latest\n", + path: "config.pod_template_yaml.template.metadata.labels.kandev.ai/session-id", + }, + { + name: "restart policy", + yaml: validPodTemplate(" restartPolicy: Never\n"), + path: "config.pod_template_yaml.template.spec.restartPolicy", + }, + { + name: "architecture selector", + yaml: validPodTemplate(" nodeSelector:\n kubernetes.io/arch: arm64\n"), + path: "config.pod_template_yaml.template.spec.nodeSelector.kubernetes.io/arch", + }, + { + name: "main command", + yaml: validMainContainerTemplate(" command: [\"sleep\"]\n"), + path: "config.pod_template_yaml.template.spec.containers[0].command", + }, + { + name: "reserved env", + yaml: validMainContainerTemplate(" env:\n - name: AGENTCTL_PORT\n value: \"9999\"\n"), + path: "config.pod_template_yaml.template.spec.containers[0].env[0].name", + }, + { + name: "reserved future agentctl env", + yaml: validMainContainerTemplate(" env:\n - name: AGENTCTL_FUTURE_CONTROL\n value: foreign\n"), + path: "config.pod_template_yaml.template.spec.containers[0].env[0].name", + }, + { + name: "reserved future kandev env", + yaml: validMainContainerTemplate(" env:\n - name: KANDEV_FUTURE_CONTROL\n value: foreign\n"), + path: "config.pod_template_yaml.template.spec.containers[0].env[0].name", + }, + { + name: "reserved home env", + yaml: validMainContainerTemplate(" env:\n - name: HOME\n value: /foreign\n"), + path: "config.pod_template_yaml.template.spec.containers[0].env[0].name", + }, + { + name: "reserved volume", + yaml: validPodTemplate(" volumes:\n - name: kandev-runtime\n emptyDir: {}\n"), + path: "config.pod_template_yaml.template.spec.volumes[0].name", + }, + { + name: "reserved runtime descendant mount", + yaml: validMainContainerTemplate(" volumeMounts:\n - name: foreign\n mountPath: /opt/kandev/agentctl\n"), + path: "config.pod_template_yaml.template.spec.containers[0].volumeMounts[0]", + }, + { + name: "reserved auth descendant mount", + yaml: validMainContainerTemplate(" volumeMounts:\n - name: foreign\n mountPath: /run/kandev/home\n"), + path: "config.pod_template_yaml.template.spec.containers[0].volumeMounts[0]", + }, + { + name: "reserved workspace descendant mount", + yaml: validMainContainerTemplate(" volumeMounts:\n - name: foreign\n mountPath: /workspace/.git\n"), + path: "config.pod_template_yaml.template.spec.containers[0].volumeMounts[0]", + }, + { + name: "reserved runtime descendant device", + yaml: validMainContainerTemplate(" volumeDevices:\n - name: foreign\n devicePath: /opt/kandev/device\n"), + path: "config.pod_template_yaml.template.spec.containers[0].volumeDevices[0].devicePath", + }, + { + name: "reserved architecture match field", + yaml: validPodTemplate(" affinity:\n nodeAffinity:\n requiredDuringSchedulingIgnoredDuringExecution:\n nodeSelectorTerms:\n - matchFields:\n - key: kubernetes.io/arch\n operator: In\n values: [amd64]\n"), + path: "config.pod_template_yaml.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchFields[0].key", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assertFieldPath(t, validProfile(tt.yaml).Validate(), tt.path) + }) + } +} + +func TestProfileConfigValidateRejectsEnvFromForEveryContainerKind(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + yaml string + path string + }{ + { + name: "main container", + yaml: validMainContainerTemplate(" envFrom:\n - configMapRef:\n name: app-env\n"), + path: "config.pod_template_yaml.template.spec.containers[0].envFrom", + }, + { + name: "sidecar", + yaml: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example/agent:latest\n - name: sidecar\n image: example/sidecar:latest\n envFrom:\n - secretRef:\n name: sidecar-env\n", + path: "config.pod_template_yaml.template.spec.containers[1].envFrom", + }, + { + name: "init container", + yaml: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n initContainers:\n - name: init\n image: example/init:latest\n envFrom:\n - configMapRef:\n name: init-env\n containers:\n - name: kandev-agent\n image: example/agent:latest\n", + path: "config.pod_template_yaml.template.spec.initContainers[0].envFrom", + }, + { + name: "ephemeral container", + yaml: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example/agent:latest\n ephemeralContainers:\n - name: debug\n image: example/debug:latest\n envFrom:\n - secretRef:\n name: debug-env\n", + path: "config.pod_template_yaml.template.spec.ephemeralContainers[0].envFrom", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assertFieldPath(t, validProfile(tt.yaml).Validate(), tt.path) + }) + } +} + +func validProfile(template string) ProfileConfig { + return ProfileConfig{ + Platform: PlatformLinuxAMD64, MainContainer: "kandev-agent", + PodTemplateYAML: template, Workspace: WorkspaceConfig{Mode: WorkspaceModeEmptyDir}, + } +} + +func validPodTemplate(extra string) string { + return "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n" + extra + + " containers:\n - name: kandev-agent\n image: example/agent:latest\n" +} + +func validMainContainerTemplate(extra string) string { + return "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n" + + " containers:\n - name: kandev-agent\n image: example/agent:latest\n" + extra +} + +func assertFieldPath(t *testing.T, err error, want string) { + t.Helper() + var fieldErr *FieldError + if !errors.As(err, &fieldErr) { + t.Fatalf("error = %v, want FieldError at %q", err, want) + } + if fieldErr.Path != want { + t.Fatalf("error path = %q, want %q", fieldErr.Path, want) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/default_scripts.go b/apps/backend/internal/agent/runtime/lifecycle/default_scripts.go index c92ac20b83..943bd41be4 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/default_scripts.go +++ b/apps/backend/internal/agent/runtime/lifecycle/default_scripts.go @@ -11,6 +11,8 @@ func DefaultPrepareScript(executorType string) string { return defaultWorktreePrepareScript case "local_docker", "remote_docker": return defaultDockerPrepareScript + case "k8s": + return defaultKubernetesPrepareScript case "sprites": return defaultSpritesPrepareScript case executorTypeSSH: @@ -130,6 +132,69 @@ git remote set-url origin "$(git remote get-url origin | sed 's|https://[^@]*@gi {{repository.setup_script}} ` +const defaultKubernetesPrepareScript = `#!/bin/sh +# Prepare a Kubernetes Pod workspace. The workspace may be a retained PVC +# mounted into a replacement Pod, so repository materialization is idempotent. + +set -eu + +workspace={{workspace.path}} +repository_url={{repository.clone_url}} +repository_branch={{repository.branch}} +clone_tmp=/opt/kandev/.workspace-clone + +# ---- Git identity and HTTPS authentication ---- +{{git.identity_setup}} +git config --global --add safe.directory '*' +git config --global url."https://github.com/".insteadOf "git@github.com:" +git config --global url."https://github.com/".insteadOf "ssh://git@github.com/" +{{github.auth_setup}} + +mkdir -p "$workspace" +if [ -n "$repository_url" ]; then + if git -C "$workspace" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + workspace_root=$(git -C "$workspace" rev-parse --show-toplevel) + if [ "$workspace_root" != "$workspace" ]; then + echo 'kandev: retained workspace repository root does not match the mount root' >&2 + exit 1 + fi + workspace_origin=$(git -C "$workspace" remote get-url origin 2>/dev/null || true) + expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|') + retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|') + if [ -z "$workspace_origin" ] || [ "$retained_origin" != "$expected_origin" ]; then + echo 'kandev: retained workspace repository origin does not match the configured repository' >&2 + exit 1 + fi + else + # A fresh filesystem-backed PVC may contain only lost+found. Preserve it, + # clone on the runtime emptyDir, then copy the checkout into the mount root. + if find "$workspace" -mindepth 1 -maxdepth 1 ! -name lost+found -print -quit | grep -q .; then + echo 'kandev: retained workspace is non-empty but is not a valid checkout' >&2 + exit 1 + fi + rm -rf "$clone_tmp" + trap 'rm -rf "$clone_tmp"' 0 1 2 15 + git clone --depth=1 --branch "$repository_branch" "$repository_url" "$clone_tmp" + cp -a "$clone_tmp"/. "$workspace"/ + rm -rf "$clone_tmp" + trap - 0 1 2 15 + fi + + cd "$workspace" + + # Strip embedded token from remote URL to avoid persisting credentials. + git remote set-url origin "$(git remote get-url origin | sed 's|https://[^@]*@github.com/|https://github.com/|')" 2>/dev/null || true + + # ---- Repository setup (if configured) ---- + {{repository.setup_script}} +else + cd "$workspace" +fi + +# ---- Pre-install agent CLI(s) ---- +{{kandev.agents.install}} +` + const defaultSpritesPrepareScript = `#!/bin/bash # Prepare Sprites.dev cloud sandbox # diff --git a/apps/backend/internal/agent/runtime/lifecycle/default_scripts_kubernetes_test.go b/apps/backend/internal/agent/runtime/lifecycle/default_scripts_kubernetes_test.go new file mode 100644 index 0000000000..52b829c73f --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/default_scripts_kubernetes_test.go @@ -0,0 +1,125 @@ +package lifecycle + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestDefaultPrepareScriptKubernetesUsesContainerWorkspaceContract(t *testing.T) { + script := DefaultPrepareScript("k8s") + if script == "" { + t.Fatal("DefaultPrepareScript(\"k8s\") returned empty") + } + for _, want := range []string{ + "{{repository.clone_url}}", + "{{repository.branch}}", + "{{workspace.path}}", + "{{repository.setup_script}}", + "{{kandev.agents.install}}", + } { + if !strings.Contains(script, want) { + t.Errorf("Kubernetes default prepare script missing %q", want) + } + } +} + +func TestDefaultPrepareScriptKubernetesReusesRetainedPVCWorkspace(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + _, origin := setupPostludeRepo(t, "main") + root := t.TempDir() + workspace := filepath.Join(root, "workspace") + if err := os.MkdirAll(filepath.Join(workspace, "lost+found"), 0o755); err != nil { + t.Fatal(err) + } + home := filepath.Join(root, "home") + if err := os.MkdirAll(home, 0o700); err != nil { + t.Fatal(err) + } + cloneTmp := filepath.Join(root, "runtime", "workspace-clone") + if err := os.MkdirAll(filepath.Dir(cloneTmp), 0o700); err != nil { + t.Fatal(err) + } + + script := strings.NewReplacer( + "{{git.identity_setup}}", ":", + "{{github.auth_setup}}", ":", + "{{repository.branch}}", shellQuote("main"), + "{{repository.clone_url}}", shellQuote(origin), + "{{workspace.path}}", shellQuote(workspace), + "{{repository.setup_script}}", ":", + "{{kandev.agents.install}}", ":", + "/opt/kandev/.workspace-clone", cloneTmp, + ).Replace(DefaultPrepareScript("k8s")) + run := func() { + t.Helper() + cmd := exec.Command("sh", "-eu", "-c", script) + cmd.Env = append(os.Environ(), "HOME="+home) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("Kubernetes prepare failed: %v\n%s", err, output) + } + } + + run() + localFile := filepath.Join(workspace, "local-untracked.txt") + if err := os.WriteFile(localFile, []byte("preserve me"), 0o600); err != nil { + t.Fatal(err) + } + run() + + if data, err := os.ReadFile(localFile); err != nil || string(data) != "preserve me" { + t.Fatalf("retained workspace data = %q, %v", data, err) + } + if output := runIn(t, workspace, "git", "branch", "--show-current"); strings.TrimSpace(string(output)) != "main" { + t.Fatalf("workspace branch = %q", output) + } +} + +func TestDefaultPrepareScriptKubernetesRejectsRetainedWorkspaceFromDifferentRepository(t *testing.T) { + if _, err := exec.LookPath("git"); err != nil { + t.Skip("git not available") + } + workspace, _ := setupPostludeRepo(t, "main") + _, expectedOrigin := setupPostludeRepo(t, "main") + root := t.TempDir() + home := filepath.Join(root, "home") + requireNoError(t, os.MkdirAll(home, 0o700)) + marker := filepath.Join(workspace, "local-untracked.txt") + requireNoError(t, os.WriteFile(marker, []byte("preserve me"), 0o600)) + + script := strings.NewReplacer( + "{{git.identity_setup}}", ":", + "{{github.auth_setup}}", ":", + "{{repository.branch}}", shellQuote("main"), + "{{repository.clone_url}}", shellQuote(expectedOrigin), + "{{workspace.path}}", shellQuote(workspace), + "{{repository.setup_script}}", ":", + "{{kandev.agents.install}}", ":", + "/opt/kandev/.workspace-clone", filepath.Join(root, "runtime", "workspace-clone"), + ).Replace(DefaultPrepareScript("k8s")) + cmd := exec.Command("sh", "-eu", "-c", script) + cmd.Env = append(os.Environ(), "HOME="+home) + output, err := cmd.CombinedOutput() + + if err == nil { + t.Fatalf("Kubernetes prepare unexpectedly reused a checkout from another repository") + } + if !strings.Contains(string(output), "origin does not match") { + t.Fatalf("prepare output = %q, want repository origin mismatch", output) + } + data, readErr := os.ReadFile(marker) + if readErr != nil || string(data) != "preserve me" { + t.Fatalf("retained workspace data = %q, %v", data, readErr) + } +} + +func requireNoError(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go b/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go index 01776804d0..e9d602991d 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go +++ b/apps/backend/internal/agent/runtime/lifecycle/diagnostic_materializer.go @@ -17,14 +17,19 @@ func (m *Manager) MaterializeDiagnosticBundle( source io.Reader, ) (DiagnosticMaterialization, error) { execution, exists := m.executionStore.GetBySessionID(sessionID) - if !exists || execution == nil || execution.agentctl == nil { + if !exists || execution == nil { + return DiagnosticMaterialization{}, fmt.Errorf("active task execution not found") + } + client, release := execution.AcquireAgentCtlClient() + defer release() + if client == nil { return DiagnosticMaterialization{}, fmt.Errorf("active task execution not found") } if execution.TaskID != taskID || execution.SessionID != sessionID { return DiagnosticMaterialization{}, fmt.Errorf("task and session do not match the active execution") } executionID := execution.ID - result, err := execution.agentctl.MaterializeDiagnosticBundle(ctx, bundleID, source) + result, err := client.MaterializeDiagnosticBundle(ctx, bundleID, source) if err != nil { return DiagnosticMaterialization{}, err } diff --git a/apps/backend/internal/agent/runtime/lifecycle/execution_metadata.go b/apps/backend/internal/agent/runtime/lifecycle/execution_metadata.go index e2d01dfd08..8d84c60ef5 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/execution_metadata.go +++ b/apps/backend/internal/agent/runtime/lifecycle/execution_metadata.go @@ -1,5 +1,7 @@ package lifecycle +import "reflect" + // AgentExecution.metadata is written and read from unrelated goroutines: the // launch path caches resolved profile env and MCP artifacts on it, the prompt // path stores model overrides, and the stop path (RemoveExecution -> @@ -93,3 +95,42 @@ func (e *AgentExecution) MetadataSnapshot() map[string]interface{} { } return snapshot } + +// mergeMetadataWithRollback applies a scoped metadata update and returns a +// compare-and-restore rollback. A key is restored only while it still contains +// the value applied here, so a concurrent writer is never overwritten by a +// failed remote refresh. +func (e *AgentExecution) mergeMetadataWithRollback(metadata map[string]interface{}) func() { + if e == nil || len(metadata) == 0 { + return func() {} + } + e.metadataMu.Lock() + if e.metadata == nil { + e.metadata = make(map[string]interface{}, len(metadata)) + } + prior := make(map[string]interface{}, len(metadata)) + priorExists := make(map[string]bool, len(metadata)) + applied := make(map[string]interface{}, len(metadata)) + for key, value := range metadata { + prior[key], priorExists[key] = e.metadata[key] + e.metadata[key] = value + applied[key] = value + } + e.metadataMu.Unlock() + + return func() { + e.metadataMu.Lock() + defer e.metadataMu.Unlock() + for key, appliedValue := range applied { + current, exists := e.metadata[key] + if !exists || !reflect.DeepEqual(current, appliedValue) { + continue + } + if priorExists[key] { + e.metadata[key] = prior[key] + } else { + delete(e.metadata, key) + } + } + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/execution_metadata_test.go b/apps/backend/internal/agent/runtime/lifecycle/execution_metadata_test.go index c4709c7e46..d057938491 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/execution_metadata_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/execution_metadata_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "sync" "testing" + + "github.com/stretchr/testify/require" ) // TestExecutionMetadataConcurrentAccessIsSafe pins the invariant behind the @@ -188,3 +190,23 @@ func TestExecutionMetadataAccessors(t *testing.T) { t.Fatalf("MetadataSnapshot on nil execution = %v, want nil", got) } } + +func TestExecutionMetadataScopedRollbackPreservesConcurrentWrites(t *testing.T) { + execution := &AgentExecution{metadata: map[string]interface{}{ + "refresh-owned": "before", + "unrelated": "before", + }} + rollback := execution.mergeMetadataWithRollback(map[string]interface{}{ + "refresh-owned": "applied", + "new-key": "applied", + }) + execution.setMetadataValue("refresh-owned", "newer-writer") + execution.setMetadataValue("unrelated", "newer-unrelated") + + rollback() + + require.Equal(t, "newer-writer", execution.metadataString("refresh-owned")) + require.Equal(t, "newer-unrelated", execution.metadataString("unrelated")) + _, exists := execution.metadataValue("new-key") + require.False(t, exists) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go b/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go index 1a5e573c19..2074d84ea2 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_backend.go @@ -10,6 +10,7 @@ import ( "github.com/kandev/kandev/internal/agent/agents" "github.com/kandev/kandev/internal/agent/executor" + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/agentctl/server/process" "github.com/kandev/kandev/internal/agentruntime" @@ -202,6 +203,51 @@ const ( MetadataKeySpriteCreatedAt = "sprite_created_at" MetadataKeyLocalPort = "local_port" + // Kubernetes executor connection metadata. These names mirror the persisted + // executor config parsed by internal/agent/kubernetes. + MetadataKeyKubernetesAuthMode = "auth_mode" + MetadataKeyKubernetesKubeconfigPath = "kubeconfig_path" + MetadataKeyKubernetesKubeContext = "kube_context" + MetadataKeyKubernetesConfigNamespace = "namespace" + MetadataKeyKubernetesRequestTimeoutSeconds = "request_timeout_seconds" + MetadataKeyKubernetesProfilePlatform = "platform" + MetadataKeyKubernetesProfileMainContainer = "main_container" + MetadataKeyKubernetesPodTemplateYAML = "pod_template_yaml" + MetadataKeyKubernetesWorkspaceMode = "workspace.mode" + MetadataKeyKubernetesWorkspaceSize = "workspace.size" + MetadataKeyKubernetesWorkspaceStorageClass = "workspace.storage_class" + MetadataKeyKubernetesWorkspaceAccessModes = "workspace.access_modes" + MetadataKeyKubernetesWorkspaceClaimName = "workspace.claim_name" + + // Kubernetes runtime inventory. The local forward port is deliberately not + // represented: every process reconnect must allocate a fresh loopback port. + MetadataKeyKubernetesNamespace = "kubernetes_namespace" + MetadataKeyKubernetesPodName = "kubernetes_pod_name" + MetadataKeyKubernetesPodUID = "kubernetes_pod_uid" + MetadataKeyKubernetesMainContainer = "kubernetes_main_container" + MetadataKeyKubernetesPlatform = "kubernetes_platform" + MetadataKeyKubernetesRuntimeWorkspaceMode = "kubernetes_workspace_mode" + MetadataKeyKubernetesPVCName = "kubernetes_pvc_name" + MetadataKeyKubernetesPVCUID = "kubernetes_pvc_uid" + MetadataKeyKubernetesPVCCreated = "kubernetes_pvc_created" + MetadataKeyKubernetesAgentctlRemotePort = "kubernetes_agentctl_remote_port" + MetadataKeyKubernetesAgentctlInstanceID = "kubernetes_agentctl_instance_id" + MetadataKeyKubernetesContainerRestartCount = "kubernetes_main_container_restart_count" + MetadataKeyKubernetesResourceInstanceID = agentkubernetes.MetadataKeyResourceInstanceID + MetadataKeyKubernetesResourceExecutorID = agentkubernetes.MetadataKeyResourceExecutorID + MetadataKeyKubernetesResourceProfileID = agentkubernetes.MetadataKeyResourceProfileID + MetadataKeyKubernetesResourceTaskID = agentkubernetes.MetadataKeyResourceTaskID + MetadataKeyKubernetesResourceSessionID = agentkubernetes.MetadataKeyResourceSessionID + MetadataKeyKubernetesResourceEnvironmentID = agentkubernetes.MetadataKeyResourceEnvironmentID + MetadataKeyKubernetesExecutorConfigHash = "kubernetes_executor_config_hash" + MetadataKeyKubernetesProfileConfigHash = "kubernetes_profile_config_hash" + MetadataKeyKubernetesTemplateHash = "kubernetes_template_hash" + MetadataKeyKubernetesProfileSnapshot = "kubernetes_profile_snapshot" + // MetadataKeyKubernetesInventoryState records how far Kubernetes admission + // progressed. Create-returned UIDs are checkpointed before validation so a + // failed rollback remains exactly recoverable after process restart. + MetadataKeyKubernetesInventoryState = "kubernetes_inventory_state" + // MetadataKeyModelOverride holds a user-requested model that overrides the // agent profile's configured model on the next launch. Set by SetSessionModel // for passthrough sessions, which restart the PTY to apply the new --model. @@ -278,6 +324,36 @@ var persistentMetadataKeys = map[string]bool{ MetadataKeySSHShell: true, MetadataKeySSHReclaimTaskDir: true, + // Kubernetes connection and exact resource inventory. + MetadataKeyKubernetesAuthMode: true, + MetadataKeyKubernetesKubeconfigPath: true, + MetadataKeyKubernetesKubeContext: true, + MetadataKeyKubernetesConfigNamespace: true, + MetadataKeyKubernetesRequestTimeoutSeconds: true, + MetadataKeyKubernetesNamespace: true, + MetadataKeyKubernetesPodName: true, + MetadataKeyKubernetesPodUID: true, + MetadataKeyKubernetesMainContainer: true, + MetadataKeyKubernetesPlatform: true, + MetadataKeyKubernetesRuntimeWorkspaceMode: true, + MetadataKeyKubernetesPVCName: true, + MetadataKeyKubernetesPVCUID: true, + MetadataKeyKubernetesPVCCreated: true, + MetadataKeyKubernetesAgentctlRemotePort: true, + MetadataKeyKubernetesAgentctlInstanceID: true, + MetadataKeyKubernetesContainerRestartCount: true, + MetadataKeyKubernetesResourceInstanceID: true, + MetadataKeyKubernetesResourceExecutorID: true, + MetadataKeyKubernetesResourceProfileID: true, + MetadataKeyKubernetesResourceTaskID: true, + MetadataKeyKubernetesResourceSessionID: true, + MetadataKeyKubernetesResourceEnvironmentID: true, + MetadataKeyKubernetesExecutorConfigHash: true, + MetadataKeyKubernetesProfileConfigHash: true, + MetadataKeyKubernetesTemplateHash: true, + MetadataKeyKubernetesProfileSnapshot: true, + MetadataKeyKubernetesInventoryState: true, + // Executor type marker MetadataKeyIsRemote: true, @@ -314,11 +390,39 @@ var persistentMetadataPrefixes = []string{ // the second session would try to attach to the first session's agentctl // process and end up sharing its ACP session and instance port. var sessionScopedMetadataKeys = map[string]bool{ - MetadataKeySSHRemoteSessionDir: true, - MetadataKeySSHRemoteAgentctlPort: true, - MetadataKeySSHRemoteAgentctlPID: true, - MetadataKeySSHLocalForwardPort: true, - MetadataKeySSHRemoteAgentctlURL: true, + MetadataKeySSHRemoteSessionDir: true, + MetadataKeySSHRemoteAgentctlPort: true, + MetadataKeySSHRemoteAgentctlPID: true, + MetadataKeySSHLocalForwardPort: true, + MetadataKeySSHRemoteAgentctlURL: true, + MetadataKeyKubernetesAuthMode: true, + MetadataKeyKubernetesKubeconfigPath: true, + MetadataKeyKubernetesKubeContext: true, + MetadataKeyKubernetesConfigNamespace: true, + MetadataKeyKubernetesRequestTimeoutSeconds: true, + MetadataKeyKubernetesNamespace: true, + MetadataKeyKubernetesPodName: true, + MetadataKeyKubernetesPodUID: true, + MetadataKeyKubernetesMainContainer: true, + MetadataKeyKubernetesPlatform: true, + MetadataKeyKubernetesRuntimeWorkspaceMode: true, + MetadataKeyKubernetesPVCName: true, + MetadataKeyKubernetesPVCUID: true, + MetadataKeyKubernetesPVCCreated: true, + MetadataKeyKubernetesAgentctlRemotePort: true, + MetadataKeyKubernetesAgentctlInstanceID: true, + MetadataKeyKubernetesContainerRestartCount: true, + MetadataKeyKubernetesResourceInstanceID: true, + MetadataKeyKubernetesResourceExecutorID: true, + MetadataKeyKubernetesResourceProfileID: true, + MetadataKeyKubernetesResourceTaskID: true, + MetadataKeyKubernetesResourceSessionID: true, + MetadataKeyKubernetesResourceEnvironmentID: true, + MetadataKeyKubernetesExecutorConfigHash: true, + MetadataKeyKubernetesProfileConfigHash: true, + MetadataKeyKubernetesTemplateHash: true, + MetadataKeyKubernetesProfileSnapshot: true, + MetadataKeyKubernetesInventoryState: true, } // ShouldPersistMetadataKey returns true if the given metadata key should @@ -429,6 +533,28 @@ type RemoteStatusProvider interface { GetRemoteStatus(ctx context.Context, instance *ExecutorInstance) (*RemoteStatus, error) } +// RemoteInstanceRefresh is a staged replacement for a live remote control +// connection. Commit installs it after lifecycle has durably persisted the +// replacement token/inventory; Abort closes only the staged resources. +type RemoteInstanceRefresh struct { + Instance *ExecutorInstance + AgentConfig agents.Agent + McpServers []McpServerConfig + AutoApprovePermissions bool + ProcessRestarted bool + // Commit installs the staged runtime connection and invokes publish before + // closing the prior connection. Callers use publish to atomically expose the + // matching lifecycle client under their client-replacement write lock. + Commit func(publish func()) error + Abort func() +} + +// RemoteInstanceRefresher is an optional capability for a tracked remote whose +// control process can restart independently of the backend process. +type RemoteInstanceRefresher interface { + RefreshRemoteInstance(ctx context.Context, instance *ExecutorInstance) (*RemoteInstanceRefresh, error) +} + // ExecutorCreateRequest contains parameters for creating an agentctl instance. type ExecutorCreateRequest struct { InstanceID string @@ -478,6 +604,14 @@ type ExecutorCreateRequest struct { // Executors that perform multi-step setup (e.g. Sprites, remote Docker) can // call this to report real-time progress to the frontend. OnProgress PrepareProgressCallback + + // CheckpointRuntimeInventory persists exact, non-secret resource inventory + // before a Kubernetes launch enters its next fallible phase. It is nil for + // runtimes and embedded tests that do not support provisional persistence. + CheckpointRuntimeInventory func(context.Context, map[string]interface{}) error + // ReleaseRuntimeInventory removes this launch's provisional row after every + // created resource was rolled back. Implementations must use execution CAS. + ReleaseRuntimeInventory func(context.Context) error } // ExecutorInstance represents an agentctl instance created by a runtime. @@ -507,15 +641,19 @@ type ExecutorInstance struct { AgentStopFailed bool // AuthToken is the agentctl auth token retrieved via handshake. - // Populated by Docker executor for encrypted storage in SecretStore. + // Populated by authenticated container/remote executors for encrypted storage in SecretStore. // Empty for standalone (launcher-owned token wired via cfg.Agent.StandaloneAuthToken) // and Sprites (no agentctl auth). AuthToken string - // BootstrapNonce is the one-time nonce injected into Docker container env. + // BootstrapNonce is the one-time nonce injected into the remote agentctl environment. // It is persisted so a restarted container can complete a fresh handshake // against the newly started agentctl process. BootstrapNonce string + + // ReleaseRuntimeInventory is carried into manager-owned post-create rollback + // paths. It must run only after exact runtime cleanup succeeds. + ReleaseRuntimeInventory func(context.Context) error } // ToAgentExecution converts a ExecutorInstance to an AgentExecution. diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_backend_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_backend_test.go index 80956d16bf..f09480cb97 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_backend_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_backend_test.go @@ -36,6 +36,42 @@ func TestShouldPersistMetadataKey(t *testing.T) { } } +func TestKubernetesRuntimeMetadataKeysPersistWithoutLocalForward(t *testing.T) { + persistent := []string{ + "auth_mode", + "kubeconfig_path", + "kube_context", + "namespace", + "request_timeout_seconds", + "kubernetes_namespace", + "kubernetes_pod_name", + "kubernetes_pod_uid", + "kubernetes_main_container", + "kubernetes_platform", + "kubernetes_workspace_mode", + "kubernetes_pvc_name", + "kubernetes_pvc_uid", + "kubernetes_pvc_created", + "kubernetes_agentctl_remote_port", + MetadataKeyKubernetesResourceExecutorID, + MetadataKeyKubernetesResourceProfileID, + MetadataKeyKubernetesResourceInstanceID, + MetadataKeyKubernetesResourceTaskID, + MetadataKeyKubernetesResourceSessionID, + MetadataKeyKubernetesResourceEnvironmentID, + "kubernetes_executor_config_hash", + "kubernetes_profile_config_hash", + "kubernetes_template_hash", + MetadataKeyKubernetesProfileSnapshot, + } + for _, key := range persistent { + require.True(t, ShouldPersistMetadataKey(key), "%s must survive same-session restart", key) + require.True(t, IsSessionScopedMetadataKey(key), "%s must not leak to a sibling session", key) + } + require.False(t, ShouldPersistMetadataKey("kubernetes_local_forward_port"), + "local forwards are process-local and must rotate after restart") +} + func TestFilterPersistentMetadata(t *testing.T) { t.Run("nil input returns nil", func(t *testing.T) { require.Nil(t, FilterPersistentMetadata(nil)) diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go new file mode 100644 index 0000000000..30122e4b7b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go @@ -0,0 +1,791 @@ +package lifecycle + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + + "github.com/kandev/kandev/internal/agent/executor" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/common/logger" +) + +const ( + kubernetesWorkspacePath = "/workspace" + kubernetesAgentctlPath = "/opt/kandev/agentctl" + kubernetesRuntimeEnvPath = "/opt/kandev/runtime.env" + kubernetesAuthEnvPath = "/run/kandev/auth.env" + kubernetesAuthHomePath = "/run/kandev/home" + kubernetesPreparePath = "/opt/kandev/prepare.sh" + kubernetesStartPath = "/opt/kandev/start" + kubernetesControlHost = "127.0.0.1" + kubernetesEnvInstanceID = "KANDEV_INSTANCE_ID" + kubernetesEnvSessionID = "KANDEV_SESSION_ID" + kubernetesEnvTaskID = "KANDEV_TASK_ID" + kubernetesEnvEnvironmentID = "KANDEV_TASK_ENVIRONMENT_ID" + kubernetesEnvAgentProfile = "KANDEV_AGENT_PROFILE_ID" + kubernetesEnvExecutionProfile = "KANDEV_EXECUTION_PROFILE_ID" + kubernetesRepositoryCloneURL = "repository.clone_url" +) + +var errKubernetesLifecycleRequestIncomplete = errors.New("kubernetes lifecycle request is incomplete") + +// KubernetesExecutor owns Kubernetes Pod/PVC lifecycle and process-local +// forwards. Cluster clients are initialized lazily from the selected executor. +type KubernetesExecutor struct { + agentctlResolver *AgentctlResolver + logger *logger.Logger + clientFactory kubernetesRuntimeClientFactory + resolveBinary kubernetesAgentctlBinaryResolver + healthRetryDelay time.Duration + + mu sync.Mutex + sessions map[string]*kubernetesSession + locks map[string]*kubernetesInstanceLock +} + +type kubernetesInstanceLock struct { + mutex sync.Mutex + refs int +} + +type kubernetesSession struct { + runtime *kubernetesRuntimeClient + forward kubeexecutor.PortForwardSession + client *agentctl.Client + request *ExecutorCreateRequest + restartCount int32 +} + +const ( + KubernetesInventoryStatePVCCreated = "pvc_created" + KubernetesInventoryStatePVCAdmitted = "pvc_admitted" + KubernetesInventoryStatePodCreated = "pod_created" + KubernetesInventoryStatePodAdmitted = "pod_admitted" + KubernetesInventoryStateReady = "ready" +) + +func NewKubernetesExecutor(agentctlResolver *AgentctlResolver, log *logger.Logger) *KubernetesExecutor { + runtime := &KubernetesExecutor{ + agentctlResolver: agentctlResolver, + logger: log, + clientFactory: newKubernetesRuntimeClient, + healthRetryDelay: agentctlHealthRetryDelay, + sessions: make(map[string]*kubernetesSession), + locks: make(map[string]*kubernetesInstanceLock), + } + if agentctlResolver != nil { + runtime.resolveBinary = func(platform kubeexecutor.Platform) ([]byte, error) { + arch := strings.TrimPrefix(string(platform), "linux/") + path, err := agentctlResolver.ResolveRemoteBinary(SSHRemotePlatform{GOOS: "linux", GOARCH: arch}) + if err != nil { + return nil, err + } + return os.ReadFile(path) + } + } + return runtime +} + +func (r *KubernetesExecutor) Name() executor.Name { return executor.NameKubernetes } + +func (r *KubernetesExecutor) HealthCheck(context.Context) error { return nil } + +func (r *KubernetesExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateRequest) (*ExecutorInstance, error) { + if req == nil || req.InstanceID == "" { + return nil, errKubernetesLifecycleRequestIncomplete + } + unlock := r.lockInstance(req.InstanceID) + defer unlock() + + executorConfig, err := kubernetesExecutorConfigFromMetadata(req.Metadata) + if err != nil { + return nil, err + } + reconnect := getMetadataString(req.Metadata, MetadataKeyKubernetesPodName) != "" + var profile kubeexecutor.ProfileConfig + if !reconnect { + profile, err = kubernetesProfileConfigFromMetadata(req.Metadata) + if err != nil { + return nil, err + } + } + if r.clientFactory == nil { + return nil, fmt.Errorf("kubernetes lifecycle: client factory is not configured") + } + runtime, err := r.clientFactory(executorConfig) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: create client: %w", err) + } + if runtime == nil || runtime.resources == nil || runtime.streams == nil { + return nil, fmt.Errorf("kubernetes lifecycle: client is incomplete") + } + + if reconnect { + return r.reconnect(ctx, runtime, req, executorConfig) + } + if r.resolveBinary == nil { + return nil, fmt.Errorf("kubernetes lifecycle: agentctl resolver is not configured") + } + return r.createFresh(ctx, runtime, req, executorConfig, profile) +} + +func (r *KubernetesExecutor) createFresh( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + executorConfig kubeexecutor.ExecutorConfig, + profile kubeexecutor.ProfileConfig, +) (_ *ExecutorInstance, returnedErr error) { + launch, err := newKubernetesFreshLaunch(r, runtime, req, executorConfig, profile) + if err != nil { + return nil, err + } + defer func() { + returnedErr = launch.rollbackAfterFailure(ctx, returnedErr) + }() + if err := launch.provisionWorkspace(ctx); err != nil { + return nil, err + } + runningPod, err := launch.createRunningPod(ctx) + if err != nil { + return nil, err + } + nonce, err := generateBootstrapNonce() + if err != nil { + return nil, err + } + binary, err := r.resolveBinary(profile.Platform) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: resolve agentctl for %s: %w", profile.Platform, err) + } + if err = r.bootstrapPod(ctx, runtime, req, runningPod, profile, nonce, binary); err != nil { + return nil, err + } + client, finalForward, token, remotePort, err := r.connectNewAgentctl(ctx, runtime, req, runningPod, nonce) + if err != nil { + return nil, err + } + return launch.complete(ctx, runningPod, client, finalForward, token, nonce, remotePort) +} + +type kubernetesFreshLaunch struct { + executor *KubernetesExecutor + runtime *kubernetesRuntimeClient + req *ExecutorCreateRequest + executorConfig kubeexecutor.ExecutorConfig + profile kubeexecutor.ProfileConfig + identity kubeexecutor.ResourceIdentity + podName string + pvcName string + workspace kubernetesWorkspaceProvision + createdPod *corev1.Pod +} + +func newKubernetesFreshLaunch( + executor *KubernetesExecutor, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + executorConfig kubeexecutor.ExecutorConfig, + profile kubeexecutor.ProfileConfig, +) (*kubernetesFreshLaunch, error) { + identity, err := kubernetesIdentity(req) + if err != nil { + return nil, err + } + podName, pvcName := kubernetesResourceNames(req.InstanceID) + return &kubernetesFreshLaunch{ + executor: executor, runtime: runtime, req: req, executorConfig: executorConfig, + profile: profile, identity: identity, podName: podName, pvcName: pvcName, + }, nil +} + +func (l *kubernetesFreshLaunch) checkpoint( + ctx context.Context, + state string, + pod *corev1.Pod, + pvc *corev1.PersistentVolumeClaim, + pvcCreated bool, + remotePort int, +) error { + metadata := kubernetesRuntimeMetadataAtState( + l.executorConfig, l.profile, l.identity, pod, pvc, pvcCreated, remotePort, state, + ) + return checkpointKubernetesRuntimeInventory(ctx, l.req, metadata) +} + +func (l *kubernetesFreshLaunch) provisionWorkspace(ctx context.Context) error { + workspace, err := provisionKubernetesWorkspace( + ctx, l.runtime.resources, l.executorConfig, l.profile, l.identity, l.pvcName, + func(created *corev1.PersistentVolumeClaim) error { + return l.checkpoint(ctx, KubernetesInventoryStatePVCCreated, nil, created, true, 0) + }, + ) + l.workspace = workspace + if err != nil { + return err + } + if workspace.createdClaim == nil { + return nil + } + return l.checkpoint(ctx, KubernetesInventoryStatePVCAdmitted, nil, workspace.claim, true, 0) +} + +func (l *kubernetesFreshLaunch) createRunningPod(ctx context.Context) (*corev1.Pod, error) { + desiredPod, err := composeKubernetesLifecyclePod( + l.profile, l.identity, l.podName, l.executorConfig.Namespace, l.workspace.claimName, + ) + if err != nil { + return nil, err + } + podWaitCtx, cancel := kubernetesRequestContext(ctx, l.executorConfig.RequestTimeoutSeconds) + defer cancel() + created, running, err := createAndWaitKubernetesPod( + podWaitCtx, l.runtime.resources, desiredPod, l.identity, l.profile.MainContainer, "Pod", + func(state string, pod *corev1.Pod) error { + return l.checkpoint( + ctx, state, pod, l.workspace.claim, l.workspace.createdClaim != nil, 0, + ) + }, + ) + l.createdPod = created + return running, err +} + +func (l *kubernetesFreshLaunch) complete( + ctx context.Context, + runningPod *corev1.Pod, + client *agentctl.Client, + forward kubeexecutor.PortForwardSession, + token, nonce string, + remotePort int, +) (*ExecutorInstance, error) { + metadata := kubernetesRuntimeMetadataAtState( + l.executorConfig, l.profile, l.identity, runningPod, l.workspace.claim, + l.workspace.createdClaim != nil, remotePort, KubernetesInventoryStateReady, + ) + // A fresh lifecycle-managed launch already has provisional Pod/PVC inventory. + // Manager persists the final ready row only after both required runtime secret + // references are durable, avoiding a restart-visible ready row that cannot + // reconnect. Standalone callers without a release callback keep the direct + // ready checkpoint behavior. + if l.req.ReleaseRuntimeInventory == nil { + if err := checkpointKubernetesRuntimeInventory(ctx, l.req, metadata); err != nil { + client.Close() + _ = forward.Close() + return nil, err + } + } + l.executor.replaceSession(l.req.InstanceID, &kubernetesSession{ + runtime: l.runtime, forward: forward, client: client, + request: cloneKubernetesCreateRequest(l.req), + restartCount: kubernetesMainContainerRestartCount(runningPod, l.profile.MainContainer), + }) + return &ExecutorInstance{ + InstanceID: l.req.InstanceID, TaskID: l.req.TaskID, SessionID: l.req.SessionID, + RuntimeName: l.executor.Name(), Client: client, WorkspacePath: kubernetesWorkspacePath, + Metadata: metadata, AuthToken: token, BootstrapNonce: nonce, + ReleaseRuntimeInventory: l.req.ReleaseRuntimeInventory, + }, nil +} + +func (l *kubernetesFreshLaunch) rollbackAfterFailure(ctx context.Context, launchErr error) error { + if launchErr == nil { + return nil + } + rollbackCtx, cancel := kubernetesDetachedRequestContext(ctx, l.executorConfig.RequestTimeoutSeconds) + defer cancel() + var rollbackErrors []error + if l.createdPod != nil { + if err := rollbackCreatedKubernetesPod(rollbackCtx, l.runtime.resources, l.createdPod); err != nil { + rollbackErrors = append(rollbackErrors, err) + } + } + if l.workspace.createdClaim != nil { + if err := rollbackCreatedKubernetesPVC(rollbackCtx, l.runtime.resources, l.workspace.createdClaim); err != nil { + rollbackErrors = append(rollbackErrors, err) + } + } + if len(rollbackErrors) == 0 && l.req.ReleaseRuntimeInventory != nil { + if err := l.req.ReleaseRuntimeInventory(rollbackCtx); err != nil { + rollbackErrors = append(rollbackErrors, fmt.Errorf( + "kubernetes lifecycle: release rolled-back runtime inventory: %w", err, + )) + } + } + return errors.Join(launchErr, errors.Join(rollbackErrors...)) +} + +func kubernetesDetachedRequestContext(ctx context.Context, requestTimeoutSeconds int) (context.Context, context.CancelFunc) { + if requestTimeoutSeconds <= 0 { + requestTimeoutSeconds = kubeexecutor.DefaultRequestTimeoutSeconds + } + return context.WithTimeout( + context.WithoutCancel(ctx), time.Duration(requestTimeoutSeconds)*time.Second, + ) +} + +func kubernetesAmbiguousCreateContext(ctx context.Context) (context.Context, context.CancelFunc) { + return kubernetesDetachedRequestContext(ctx, kubeexecutor.DefaultRequestTimeoutSeconds) +} + +func kubernetesRequestContext(ctx context.Context, requestTimeoutSeconds int) (context.Context, context.CancelFunc) { + if requestTimeoutSeconds <= 0 { + requestTimeoutSeconds = kubeexecutor.DefaultRequestTimeoutSeconds + } + return context.WithTimeout(ctx, time.Duration(requestTimeoutSeconds)*time.Second) +} + +func (r *KubernetesExecutor) connectNewAgentctl( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + nonce string, +) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) { + controlForward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod) + if err != nil { + return nil, nil, "", 0, err + } + defer func() { _ = controlForward.Close() }() + token, err := control.Handshake(ctx, nonce) + if err != nil { + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: nonce handshake: %w", err) + } + createRequest := buildReconnectCreateInstanceRequest(req, req.InstanceID) + createResponse, err := createOrReconcileKubernetesAgentctlInstance(ctx, control, createRequest) + if err != nil { + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: create agentctl instance: %w", err) + } + if createResponse.Port < 1 || createResponse.Port > 65535 { + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: invalid agentctl instance port %d", createResponse.Port) + } + forward, err := startKubernetesForward(ctx, runtime.streams, pod, uint16(createResponse.Port)) + if err != nil { + return nil, nil, "", 0, err + } + client := agentctl.NewClient(kubernetesControlHost, int(forward.LocalPort()), r.logger, + agentctl.WithExecutionID(req.InstanceID), agentctl.WithSessionID(req.SessionID), agentctl.WithAuthToken(token)) + if err := client.Health(ctx); err != nil { + client.Close() + _ = forward.Close() + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: instance health: %w", err) + } + return client, forward, token, createResponse.Port, nil +} + +type kubernetesAgentctlInstanceControl interface { + CreateInstance(context.Context, *agentctl.CreateInstanceRequest) (*agentctl.CreateInstanceResponse, error) + GetInstance(context.Context, string) (*agentctl.InstanceInfo, error) +} + +func createOrReconcileKubernetesAgentctlInstance( + ctx context.Context, + control kubernetesAgentctlInstanceControl, + request *agentctl.CreateInstanceRequest, +) (*agentctl.CreateInstanceResponse, error) { + response, createErr := control.CreateInstance(ctx, request) + if createErr == nil { + if err := validateKubernetesAgentctlInstanceResponse(response, request); err != nil { + return nil, err + } + return response, nil + } + if !kubeexecutor.IsAmbiguousCreateError(createErr) { + return nil, createErr + } + reconcileCtx, cancel := kubernetesAmbiguousCreateContext(ctx) + defer cancel() + info, reconcileErr := control.GetInstance(reconcileCtx, request.ID) + if reconcileErr != nil { + return nil, errors.Join(createErr, fmt.Errorf("reconcile ambiguous agentctl create: %w", reconcileErr)) + } + response = &agentctl.CreateInstanceResponse{ID: info.ID, Port: info.Port} + if err := validateKubernetesAgentctlInstanceResponse(response, request); err != nil { + return nil, errors.Join(createErr, fmt.Errorf("reconcile ambiguous agentctl create: %w", err)) + } + if info.WorkspacePath != request.WorkspacePath { + return nil, errors.Join(createErr, errors.New( + "reconcile ambiguous agentctl create: existing instance workspace does not match request", + )) + } + return response, nil +} + +func validateKubernetesAgentctlInstanceResponse( + response *agentctl.CreateInstanceResponse, + request *agentctl.CreateInstanceRequest, +) error { + if response == nil || request == nil || response.ID != request.ID { + return errors.New("kubernetes lifecycle: agentctl instance identity does not match request") + } + if response.Port < 1 || response.Port > 65535 { + return fmt.Errorf("kubernetes lifecycle: invalid agentctl instance port %d", response.Port) + } + return nil +} + +func (r *KubernetesExecutor) connectHealthyKubernetesControl( + ctx context.Context, + runtime *kubernetesRuntimeClient, + pod *corev1.Pod, +) (kubeexecutor.PortForwardSession, *agentctl.ControlClient, error) { + var lastForwardErr error + for { + forward, err := startKubernetesForward( + ctx, runtime.streams, pod, uint16(kubeexecutor.DefaultAgentctlPort), + ) + if err != nil { + return nil, nil, errors.Join(lastForwardErr, err) + } + control := agentctl.NewControlClient(kubernetesControlHost, int(forward.LocalPort()), r.logger) + stopped, healthErr := r.waitForKubernetesAgentctlHealthUntilForwardStops(ctx, control, forward.Done()) + if !stopped { + if healthErr != nil { + _ = forward.Close() + return nil, nil, fmt.Errorf("kubernetes lifecycle: control health: %w", errors.Join(lastForwardErr, healthErr)) + } + return forward, control, nil + } + _ = forward.Close() + lastForwardErr = healthErr + if err := ctx.Err(); err != nil { + return nil, nil, fmt.Errorf("kubernetes lifecycle: control health: %w", errors.Join(err, lastForwardErr)) + } + if err := r.waitForKubernetesControlForwardRetry(ctx); err != nil { + return nil, nil, fmt.Errorf("kubernetes lifecycle: control health: %w", errors.Join(err, lastForwardErr)) + } + } +} + +func (r *KubernetesExecutor) waitForKubernetesControlForwardRetry(ctx context.Context) error { + delay := r.healthRetryDelay + if delay <= 0 { + delay = agentctlHealthRetryDelay + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (r *KubernetesExecutor) waitForKubernetesAgentctlHealth(ctx context.Context, ctl healthChecker) error { + _, err := r.waitForKubernetesAgentctlHealthUntilForwardStops(ctx, ctl, nil) + return err +} + +func (r *KubernetesExecutor) waitForKubernetesAgentctlHealthUntilForwardStops( + ctx context.Context, + ctl healthChecker, + forwardDone <-chan error, +) (bool, error) { + delay := r.healthRetryDelay + if delay <= 0 { + delay = agentctlHealthRetryDelay + } + var lastErr error + for { + if err := ctl.Health(ctx); err == nil { + if forwardDone != nil { + select { + case forwardErr := <-forwardDone: + return true, kubernetesControlForwardStoppedError(forwardErr) + default: + } + } + return false, nil + } else { + lastErr = err + } + if ctx.Err() != nil { + return false, fmt.Errorf("agentctl did not become healthy: %w", errors.Join(ctx.Err(), lastErr)) + } + timer := time.NewTimer(delay) + select { + case forwardErr := <-forwardDone: + timer.Stop() + return true, kubernetesControlForwardStoppedError(forwardErr) + case <-ctx.Done(): + timer.Stop() + return false, fmt.Errorf("agentctl did not become healthy: %w", errors.Join(ctx.Err(), lastErr)) + case <-timer.C: + } + } +} + +func kubernetesControlForwardStoppedError(err error) error { + if err == nil { + err = errors.New("port forward stopped") + } + return fmt.Errorf("control port forward stopped before agentctl became healthy: %w", err) +} + +func startKubernetesForward( + ctx context.Context, + streams *kubeexecutor.StreamOperations, + pod *corev1.Pod, + remotePort uint16, +) (kubeexecutor.PortForwardSession, error) { + forward, err := streams.Forward(context.WithoutCancel(ctx), kubeexecutor.PortForwardRequest{ + Namespace: pod.Namespace, Pod: pod.Name, LocalAddress: kubernetesControlHost, + LocalPort: 0, RemotePort: remotePort, + }) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: forward port %d: %w", remotePort, err) + } + select { + case <-forward.Ready(): + if forward.LocalPort() == 0 { + _ = forward.Close() + return nil, fmt.Errorf("kubernetes lifecycle: port forward returned zero local port") + } + return forward, nil + case doneErr := <-forward.Done(): + _ = forward.Close() + if doneErr == nil { + doneErr = errors.New("port forward stopped before becoming ready") + } + return nil, fmt.Errorf("kubernetes lifecycle: port forward startup: %w", doneErr) + case <-ctx.Done(): + _ = forward.Close() + return nil, ctx.Err() + } +} + +func kubernetesProfileConfigFromMetadata(metadata map[string]interface{}) (kubeexecutor.ProfileConfig, error) { + values := map[string]string{ + MetadataKeyKubernetesProfilePlatform: getMetadataString(metadata, MetadataKeyKubernetesProfilePlatform), + MetadataKeyKubernetesProfileMainContainer: getMetadataString(metadata, MetadataKeyKubernetesProfileMainContainer), + MetadataKeyKubernetesPodTemplateYAML: getMetadataString(metadata, MetadataKeyKubernetesPodTemplateYAML), + MetadataKeyKubernetesWorkspaceMode: getMetadataString(metadata, MetadataKeyKubernetesWorkspaceMode), + MetadataKeyKubernetesWorkspaceSize: getMetadataString(metadata, MetadataKeyKubernetesWorkspaceSize), + MetadataKeyKubernetesWorkspaceStorageClass: getMetadataString(metadata, MetadataKeyKubernetesWorkspaceStorageClass), + MetadataKeyKubernetesWorkspaceAccessModes: getMetadataString(metadata, MetadataKeyKubernetesWorkspaceAccessModes), + MetadataKeyKubernetesWorkspaceClaimName: getMetadataString(metadata, MetadataKeyKubernetesWorkspaceClaimName), + } + return kubeexecutor.ParseProfileConfig(values) +} + +func (r *KubernetesExecutor) RecoverInstances(context.Context) ([]*ExecutorInstance, error) { + return nil, nil +} + +func (r *KubernetesExecutor) GetInteractiveRunner() *process.InteractiveRunner { return nil } + +func (r *KubernetesExecutor) RequiresCloneURL() bool { return true } +func (r *KubernetesExecutor) ShouldApplyPreferredShell() bool { return false } +func (r *KubernetesExecutor) IsAlwaysResumable() bool { return true } + +func kubernetesExecutorConfigFromMetadata(metadata map[string]interface{}) (kubeexecutor.ExecutorConfig, error) { + values := map[string]string{ + MetadataKeyKubernetesAuthMode: getMetadataString(metadata, MetadataKeyKubernetesAuthMode), + MetadataKeyKubernetesKubeconfigPath: getMetadataString(metadata, MetadataKeyKubernetesKubeconfigPath), + MetadataKeyKubernetesKubeContext: getMetadataString(metadata, MetadataKeyKubernetesKubeContext), + MetadataKeyKubernetesConfigNamespace: getMetadataString(metadata, MetadataKeyKubernetesConfigNamespace), + MetadataKeyKubernetesRequestTimeoutSeconds: getMetadataString(metadata, MetadataKeyKubernetesRequestTimeoutSeconds), + } + return kubeexecutor.ParseExecutorConfig(values) +} + +func kubernetesIdentity(req *ExecutorCreateRequest) (kubeexecutor.ResourceIdentity, error) { + identity := kubeexecutor.ResourceIdentity{ + ExecutorID: getMetadataString(req.Metadata, "executor_id"), ProfileID: getMetadataString(req.Metadata, MetadataKeyExecutorProfileID), + InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, EnvironmentID: req.TaskEnvironmentID, + } + if _, err := kubeexecutor.OwnershipLabels(identity); err != nil { + return kubeexecutor.ResourceIdentity{}, err + } + return identity, nil +} + +func kubernetesResourceNames(instanceID string) (string, string) { + sum := sha256.Sum256([]byte(instanceID)) + suffix := hex.EncodeToString(sum[:8]) + return "kandev-" + suffix, "kandev-" + suffix + "-workspace" +} + +func kubernetesRuntimeMetadata( + executorConfig kubeexecutor.ExecutorConfig, + profile kubeexecutor.ProfileConfig, + identity kubeexecutor.ResourceIdentity, + pod *corev1.Pod, + pvc *corev1.PersistentVolumeClaim, + pvcCreated bool, + remotePort int, +) map[string]interface{} { + metadata := map[string]interface{}{ + MetadataKeyKubernetesNamespace: executorConfig.Namespace, + MetadataKeyKubernetesMainContainer: profile.MainContainer, + MetadataKeyKubernetesPlatform: string(profile.Platform), + MetadataKeyKubernetesRuntimeWorkspaceMode: string(profile.Workspace.Mode), + MetadataKeyKubernetesPVCCreated: pvcCreated, + MetadataKeyKubernetesAgentctlInstanceID: identity.InstanceID, + MetadataKeyKubernetesResourceInstanceID: identity.InstanceID, + MetadataKeyKubernetesResourceExecutorID: identity.ExecutorID, + MetadataKeyKubernetesResourceProfileID: identity.ProfileID, + MetadataKeyKubernetesResourceTaskID: identity.TaskID, + MetadataKeyKubernetesResourceSessionID: identity.SessionID, + MetadataKeyKubernetesResourceEnvironmentID: identity.EnvironmentID, + MetadataKeyKubernetesExecutorConfigHash: kubernetesConfigHash(executorConfig), + MetadataKeyKubernetesProfileConfigHash: kubernetesConfigHash(profile), + MetadataKeyKubernetesTemplateHash: kubernetesStringHash(profile.PodTemplateYAML), + MetadataKeyKubernetesProfileSnapshot: kubernetesProfileSnapshot(profile), + MetadataKeyIsRemote: true, + "executor_id": identity.ExecutorID, + MetadataKeyExecutorProfileID: identity.ProfileID, + } + if pod != nil { + metadata[MetadataKeyKubernetesNamespace] = pod.Namespace + metadata[MetadataKeyKubernetesPodName] = pod.Name + metadata[MetadataKeyKubernetesPodUID] = string(pod.UID) + metadata[MetadataKeyKubernetesContainerRestartCount] = strconv.FormatInt( + int64(kubernetesMainContainerRestartCount(pod, profile.MainContainer)), 10, + ) + } + if pvc != nil { + metadata[MetadataKeyKubernetesPVCName] = pvc.Name + metadata[MetadataKeyKubernetesPVCUID] = string(pvc.UID) + } + if remotePort > 0 { + metadata[MetadataKeyKubernetesAgentctlRemotePort] = strconv.Itoa(remotePort) + } + return metadata +} + +func kubernetesRuntimeMetadataAtState( + executorConfig kubeexecutor.ExecutorConfig, + profile kubeexecutor.ProfileConfig, + identity kubeexecutor.ResourceIdentity, + pod *corev1.Pod, + pvc *corev1.PersistentVolumeClaim, + pvcCreated bool, + remotePort int, + state string, +) map[string]interface{} { + metadata := kubernetesRuntimeMetadata( + executorConfig, profile, identity, pod, pvc, pvcCreated, remotePort, + ) + metadata[MetadataKeyKubernetesInventoryState] = state + return metadata +} + +func checkpointKubernetesRuntimeInventory( + ctx context.Context, + req *ExecutorCreateRequest, + metadata map[string]interface{}, +) error { + if req == nil || req.CheckpointRuntimeInventory == nil { + return nil + } + if err := req.CheckpointRuntimeInventory(ctx, metadata); err != nil { + return fmt.Errorf("kubernetes lifecycle: checkpoint runtime inventory: %w", err) + } + return nil +} + +func kubernetesProfileSnapshot(profile kubeexecutor.ProfileConfig) string { + data, _ := json.Marshal(profile) + return string(data) +} + +func kubernetesConfigHash(value interface{}) string { + data, _ := json.Marshal(value) + return kubernetesStringHash(string(data)) +} + +func kubernetesStringHash(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +func (r *KubernetesExecutor) lockInstance(instanceID string) func() { + r.mu.Lock() + lock := r.locks[instanceID] + if lock == nil { + lock = &kubernetesInstanceLock{} + r.locks[instanceID] = lock + } + lock.refs++ + r.mu.Unlock() + lock.mutex.Lock() + return func() { + lock.mutex.Unlock() + r.mu.Lock() + lock.refs-- + if lock.refs == 0 && r.locks[instanceID] == lock { + delete(r.locks, instanceID) + } + r.mu.Unlock() + } +} + +func (r *KubernetesExecutor) replaceSession(instanceID string, session *kubernetesSession) { + r.mu.Lock() + old := r.sessions[instanceID] + r.sessions[instanceID] = session + r.mu.Unlock() + if old != nil { + if old.client != nil { + old.client.Close() + } + if old.forward != nil { + _ = old.forward.Close() + } + } +} + +func kubernetesMainContainerRestartCount(pod *corev1.Pod, mainContainer string) int32 { + if pod == nil { + return 0 + } + for index := range pod.Status.ContainerStatuses { + if pod.Status.ContainerStatuses[index].Name == mainContainer { + return pod.Status.ContainerStatuses[index].RestartCount + } + } + return 0 +} + +func cloneKubernetesCreateRequest(req *ExecutorCreateRequest) *ExecutorCreateRequest { + if req == nil { + return nil + } + cloned := *req + cloned.Env = cloneStringMap(req.Env) + cloned.Metadata = cloneKubernetesMetadata(req.Metadata) + cloned.WorkspaceSourceRoots = append([]string(nil), req.WorkspaceSourceRoots...) + cloned.ApprovedSecretEnvKeys = append([]string(nil), req.ApprovedSecretEnvKeys...) + cloned.McpServers = append([]McpServerConfig(nil), req.McpServers...) + cloned.McpProviders = append([]string(nil), req.McpProviders...) + return &cloned +} + +func kubernetesBootstrapCommand() string { + return `set -eu +while [ ! -f /opt/kandev/start ]; do sleep 1; done +set -a +. /opt/kandev/runtime.env +. /run/kandev/auth.env +set +a +if [ ! -f /opt/kandev/prepared ]; then + sh /opt/kandev/prepare.sh + : > /opt/kandev/prepared +fi +exec /opt/kandev/agentctl` +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_admission_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_admission_test.go new file mode 100644 index 0000000000..01f59049a1 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_admission_test.go @@ -0,0 +1,189 @@ +package lifecycle + +import ( + "context" + "os" + "path/filepath" + "testing" + + corev1 "k8s.io/api/core/v1" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/agent/agents" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestKubernetesCreateInstanceRollsBackAdmissionMutatedPodByCreatedUID(t *testing.T) { + resources := &fakeKubernetesResources{ + mutateCreatedPod: func(pod *corev1.Pod) { + pod.Labels["kandev.ai/instance-id"] = "mutated-by-admission" + }, + } + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + + _, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + + require.ErrorContains(t, err, "ownership label") + require.Equal(t, []string{"kandev-6212a06cac8b50f7:pod-uid"}, resources.deletedPods, + "the exact UID returned by Create authorizes rollback even when admission changed labels") +} + +func TestKubernetesCreateInstanceRollsBackAdmissionMutatedPVCByCreatedUID(t *testing.T) { + resources := &fakeKubernetesResources{ + mutateCreatedPVC: func(pvc *corev1.PersistentVolumeClaim) { + pvc.Labels["kandev.ai/instance-id"] = "mutated-by-admission" + }, + } + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "ownership label") + require.Equal(t, []string{"kandev-6212a06cac8b50f7-workspace:pvc-uid"}, resources.deletedPVCs) + require.Empty(t, resources.createdPods) +} + +func TestKubernetesReconnectRejectsExtraKandevOwnershipLabel(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + resources.mu.Lock() + resources.pod.Labels["kandev.ai/forged"] = "true" + resources.mu.Unlock() + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{41001: instancePort}) + + _, err = restarted.CreateInstance(context.Background(), kubernetesReconnectRequest(created)) + + require.ErrorContains(t, err, "ownership label") +} + +func TestKubernetesCleanupRejectsExtraKandevOwnershipLabels(t *testing.T) { + tests := []struct { + name string + mutate func(*fakeKubernetesResources) + }{ + {name: "Pod", mutate: func(resources *fakeKubernetesResources) { + resources.pod.Labels["kandev.ai/forged"] = "true" + }}, + {name: "managed PVC", mutate: func(resources *fakeKubernetesResources) { + resources.pvc.Labels["kandev.ai/forged"] = "true" + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + resources.mu.Lock() + test.mutate(resources) + resources.mu.Unlock() + + err = executor.StopInstance(context.Background(), instance, true) + + require.ErrorContains(t, err, "ownership label") + require.Empty(t, resources.deletedPods) + require.Empty(t, resources.deletedPVCs) + }) + } +} + +func TestVerifyCreatedKubernetesPodRejectsAdmissionMutatedAgentctlFields(t *testing.T) { + req := validKubernetesCreateRequest() + profile, err := kubernetesProfileConfigFromMetadata(req.Metadata) + require.NoError(t, err) + identity, err := kubernetesIdentity(req) + require.NoError(t, err) + template, err := kubeexecutor.ParsePodTemplate(profile.PodTemplateYAML) + require.NoError(t, err) + desired, _, err := kubeexecutor.ComposePod(template, profile, kubeexecutor.PodOptions{ + Name: "pod-1", Namespace: "kandev-agents", Identity: identity, + Command: []string{"sh", "-c"}, Args: []string{kubernetesBootstrapCommand()}, + WorkingDir: kubernetesWorkspacePath, AgentctlPort: kubeexecutor.DefaultAgentctlPort, + }) + require.NoError(t, err) + + t.Run("port", func(t *testing.T) { + created := desired.DeepCopy() + created.UID = "pod-uid" + main := findKubernetesContainer(created, profile.MainContainer) + require.NotNil(t, main) + main.Ports[len(main.Ports)-1].ContainerPort = 9999 + + err := verifyCreatedPod(created, desired, identity, profile.MainContainer) + + require.ErrorContains(t, err, "agentctl port") + }) + + t.Run("reserved environment", func(t *testing.T) { + created := desired.DeepCopy() + created.UID = "pod-uid" + main := findKubernetesContainer(created, profile.MainContainer) + require.NotNil(t, main) + main.Env = append(main.Env, corev1.EnvVar{Name: "AGENTCTL_PORT", Value: "9999"}) + + err := verifyCreatedPod(created, desired, identity, profile.MainContainer) + + require.ErrorContains(t, err, "reserved environment") + }) +} + +func TestKubernetesCreateInstanceMaterializesSkillsCredentialsAndPortableConfig(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".codex"), 0o700)) + require.NoError(t, os.MkdirAll(filepath.Join(home, ".mock-agent"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(home, ".codex", "auth.json"), []byte("auth-data"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(home, ".mock-agent", "settings.json"), []byte("settings-data"), 0o600)) + + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + execs := &recordingKubernetesExec{} + executor := NewKubernetesExecutor(nil, newTestLogger()) + executor.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{ + resources: &fakeKubernetesResources{}, + streams: kubeexecutor.NewStreamOperations(execs, &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }}), + }, nil + } + executor.resolveBinary = func(kubeexecutor.Platform) ([]byte, error) { return []byte("agentctl"), nil } + req := validKubernetesCreateRequest() + req.AgentConfig = agents.NewMockAgentWithID("codex-acp", "Mock Codex", "Mock Codex") + req.Metadata[MetadataKeyRemoteAuthHome] = "/home/untrusted-task-value" + req.Metadata["remote_credentials"] = `["agent:codex-acp:files:0"]` + req.Metadata[MetadataKeyAgentConfigBundles] = `["codex-acp.settings"]` + req.Metadata[MetadataKeySkillManifestJSON] = `{ + "Skills":[{"Slug":"review","Content":"# Review","Files":[{"Path":"refs/checklist.md","Content":"check"}]}], + "Instructions":[{"Filename":"AGENTS.md","Content":"office rules","IsEntry":true}], + "AgentTypeID":"codex-acp","WorkspaceSlug":"office","AgentID":"profile-1","ProjectSkillDir":".agents/skills" + }` + + _, err := executor.CreateInstance(context.Background(), req) + + require.NoError(t, err) + requireKubernetesUploadedFile(t, execs, "/workspace/.agents/skills/kandev-review/SKILL.md", "# Review") + requireKubernetesUploadedFile(t, execs, "/workspace/.agents/skills/kandev-review/refs/checklist.md", "check") + requireKubernetesUploadedFile(t, execs, "/opt/kandev/runtime/office/instructions/profile-1/AGENTS.md", "office rules") + requireKubernetesUploadedFile(t, execs, "/run/kandev/home/.codex/auth.json", "auth-data") + requireKubernetesUploadedFile(t, execs, "/run/kandev/home/.mock-agent/settings.json", "settings-data") + requireKubernetesPathNotUsed(t, execs, "/home/untrusted-task-value") +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go new file mode 100644 index 0000000000..8d3aca7fa3 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go @@ -0,0 +1,267 @@ +package lifecycle + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "regexp" + "sort" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/githubauth" + "github.com/kandev/kandev/internal/scriptengine" +) + +var kubernetesEnvironmentKey = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +func (r *KubernetesExecutor) bootstrapPod( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + profile kubeexecutor.ProfileConfig, + nonce string, + binary []byte, +) error { + if len(binary) == 0 { + return fmt.Errorf("kubernetes lifecycle: resolved agentctl binary is empty") + } + if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer, + kubernetesAgentctlPath, binary, 0o755); err != nil { + return fmt.Errorf("kubernetes lifecycle: upload agentctl: %w", err) + } + + runtimeEnv, err := kubernetesSerializeEnvironment(kubernetesRuntimeEnvironment(req)) + if err != nil { + return err + } + prepareScript, err := kubernetesPrepareScript(req) + if err != nil { + return fmt.Errorf("kubernetes lifecycle: resolve prepare script: %w", err) + } + if err := kubernetesWriteBootstrapFiles(ctx, runtime.streams, pod, profile.MainContainer, + runtimeEnv, []byte(prepareScript)); err != nil { + return fmt.Errorf("kubernetes lifecycle: upload runtime config: %w", err) + } + + authEnv := kubernetesAuthEnvironment(req.Env, nonce) + authData, err := kubernetesSerializeEnvironment(authEnv) + if err != nil { + return err + } + if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer, + kubernetesAuthEnvPath, authData, 0o600); err != nil { + return fmt.Errorf("kubernetes lifecycle: upload auth config: %w", err) + } + if err := r.materializeKubernetesRemoteFiles(ctx, runtime, req, pod, profile.MainContainer); err != nil { + return err + } + if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer, + kubernetesStartPath, nil, 0o600); err != nil { + return fmt.Errorf("kubernetes lifecycle: signal bootstrap: %w", err) + } + return nil +} + +func kubernetesRuntimeEnvironment(req *ExecutorCreateRequest) map[string]string { + return map[string]string{ + "AGENTCTL_LISTEN_HOST": kubernetesControlHost, + "AGENTCTL_PORT": strconv.Itoa(int(kubeexecutor.DefaultAgentctlPort)), + "AGENTCTL_INSTANCE_PORT_BASE": strconv.Itoa(dockerAgentctlInstancePortBase), + "AGENTCTL_INSTANCE_PORT_MAX": strconv.Itoa(dockerAgentctlInstancePortMax), + kubernetesEnvInstanceID: req.InstanceID, + kubernetesEnvSessionID: req.SessionID, + kubernetesEnvTaskID: req.TaskID, + kubernetesEnvEnvironmentID: req.TaskEnvironmentID, + kubernetesEnvAgentProfile: req.OfficeAgentProfileID, + kubernetesEnvExecutionProfile: req.AgentProfileID, + } +} + +func kubernetesAuthEnvironment(input map[string]string, nonce string) map[string]string { + values := make(map[string]string, len(input)+2) + for key, value := range input { + if !isKubernetesOwnedControlEnvironmentKey(key) { + values[key] = value + } + } + values["AGENTCTL_BOOTSTRAP_NONCE"] = nonce + values["HOME"] = kubernetesAuthHomePath + if hasManagedGitCredentialBrokerEnv(input) { + values[githubauth.CredentialHelperPathEnv] = kubernetesAgentctlPath + } + return values +} + +func isKubernetesOwnedControlEnvironmentKey(key string) bool { + if key == "HOME" || key == githubauth.CredentialHelperPathEnv || strings.HasPrefix(key, "AGENTCTL_") { + return true + } + switch key { + case kubernetesEnvInstanceID, + "KANDEV_EXECUTION_ID", + kubernetesEnvTaskID, + kubernetesEnvSessionID, + kubernetesEnvEnvironmentID, + kubernetesEnvAgentProfile, + kubernetesEnvExecutionProfile, + "KANDEV_OFFICE_AGENT_PROFILE_ID": + return true + default: + return false + } +} + +func kubernetesWriteFile( + ctx context.Context, + streams *kubeexecutor.StreamOperations, + pod *corev1.Pod, + container, destination string, + data []byte, + mode os.FileMode, +) error { + command := kubernetesWriteFileCommand(destination, mode, len(data) > 0) + var stdin io.Reader + if len(data) > 0 { + stdin = bytes.NewReader(data) + } + return streams.Exec(ctx, kubeexecutor.ExecRequest{ + Namespace: pod.Namespace, Pod: pod.Name, Container: container, + Command: []string{"sh", "-c", command}, Stdin: stdin, + Stdout: io.Discard, Stderr: io.Discard, + }) +} + +func kubernetesWriteFileCommand(destination string, mode os.FileMode, hasData bool) string { + directory := path.Dir(destination) + writeCommand := ":" + if hasData { + writeCommand = `cat > "$temporary"` + } + return fmt.Sprintf(`set -efu +umask 077 +directory=%s +destination=%s +current= +old_ifs=$IFS +IFS=/ +set -- $directory +IFS=$old_ifs +for component in "$@"; do + [ -n "$component" ] || continue + current="$current/$component" + [ ! -L "$current" ] || exit 73 + if [ -e "$current" ]; then + [ -d "$current" ] || exit 73 + else + mkdir "$current" + fi +done +[ ! -L "$destination" ] || exit 73 +[ ! -d "$destination" ] || exit 73 +temporary=$(mktemp "$directory/.kandev-upload.XXXXXX") +trap 'rm -f "$temporary"' EXIT HUP INT TERM +%s +chmod %04o "$temporary" +mv -f "$temporary" "$destination" +trap - EXIT HUP INT TERM`, shellQuote(directory), shellQuote(destination), writeCommand, mode.Perm()) +} + +func kubernetesWriteBootstrapFiles( + ctx context.Context, + streams *kubeexecutor.StreamOperations, + pod *corev1.Pod, + container string, + runtimeEnv, prepare []byte, +) error { + command := fmt.Sprintf( + "set -eu; umask 077; mkdir -p /opt/kandev %s; chmod 0700 %s; "+ + "dd bs=1 count=%d of=%s 2>/dev/null; dd bs=1 count=%d of=%s 2>/dev/null; "+ + "chmod 0600 %s; chmod 0700 %s", + shellQuote(kubernetesAuthHomePath), shellQuote(kubernetesAuthHomePath), + len(runtimeEnv), shellQuote(kubernetesRuntimeEnvPath), len(prepare), shellQuote(kubernetesPreparePath), + shellQuote(kubernetesRuntimeEnvPath), shellQuote(kubernetesPreparePath), + ) + data := append(append(make([]byte, 0, len(runtimeEnv)+len(prepare)), runtimeEnv...), prepare...) + return streams.Exec(ctx, kubeexecutor.ExecRequest{ + Namespace: pod.Namespace, Pod: pod.Name, Container: container, + Command: []string{"sh", "-c", command}, Stdin: bytes.NewReader(data), + Stdout: io.Discard, Stderr: io.Discard, + }) +} + +func kubernetesSerializeEnvironment(values map[string]string) ([]byte, error) { + keys := make([]string, 0, len(values)) + for key := range values { + if !kubernetesEnvironmentKey.MatchString(key) { + return nil, fmt.Errorf("kubernetes lifecycle: invalid environment key %q", key) + } + keys = append(keys, key) + } + sort.Strings(keys) + var output strings.Builder + for _, key := range keys { + output.WriteString(key) + output.WriteByte('=') + output.WriteString(shellQuote(values[key])) + output.WriteByte('\n') + } + return []byte(output.String()), nil +} + +func kubernetesPrepareScript(req *ExecutorCreateRequest) (string, error) { + script := getMetadataString(req.Metadata, MetadataKeySetupScript) + if script == "" { + script = DefaultPrepareScript("k8s") + } + if script == "" { + return ":\n", nil + } + script += KandevBranchCheckoutPostlude() + if binding, ok := req.RemoteContributions[""]; ok { + addition, err := scriptengine.RemoteContributionSetupScript(&binding) + if err != nil { + return "", err + } + script += addition + } + if destination, ok := req.ContributionDestinations[""]; ok { + addition, err := scriptengine.ContributionDestinationSetupScript(&destination) + if err != nil { + return "", err + } + script += addition + } + installScripts := []string(nil) + if req.AgentConfig != nil && strings.TrimSpace(req.AgentConfig.InstallScript()) != "" { + installScripts = append(installScripts, req.AgentConfig.InstallScript()) + } + resolver := scriptengine.NewResolver(). + WithStatic(map[string]string{kubernetesRepositoryCloneURL: shellQuote("")}). + WithProvider(scriptengine.WorkspaceProvider(kubernetesWorkspacePath)). + WithProvider(scriptengine.AgentctlProviderWithOptions( + int(kubeexecutor.DefaultAgentctlPort), + kubernetesWorkspacePath, + scriptengine.AgentctlProviderOptions{BinaryPath: kubernetesAgentctlPath, Start: false}, + )). + WithProvider(scriptengine.GitIdentityProvider(req.Metadata)). + WithProvider(scriptengine.GitHubAuthProvider(req.Env)). + WithProvider(scriptengine.AgentInstallProvider(installScripts)). + WithProvider(scriptengine.WorktreeProvider( + "", kubernetesWorkspacePath, + getMetadataString(req.Metadata, MetadataKeyWorktreeID), + getMetadataString(req.Metadata, MetadataKeyWorktreeBranch), + getMetadataString(req.Metadata, MetadataKeyBaseBranch), + )). + WithProvider(scriptengine.RepositoryProvider( + req.Metadata, req.Env, getGitRemoteURL, injectGitHubTokenIntoCloneURL, + )) + return resolver.Resolve(script), nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go new file mode 100644 index 0000000000..a61180592b --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go @@ -0,0 +1,257 @@ +package lifecycle + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/agent/agents" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/githubauth" +) + +func TestKubernetesWriteFileDoesNotOpenStdinForEmptySentinel(t *testing.T) { + execs := &recordingKubernetesExec{} + streams := kubeexecutor.NewStreamOperations(execs, nil) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "kandev-agents", Name: "agent-pod"}} + + err := kubernetesWriteFile( + context.Background(), streams, pod, "kandev-agent", kubernetesStartPath, nil, 0o600, + ) + + require.NoError(t, err) + require.Len(t, execs.requests, 1) + require.Nil(t, execs.requests[0].request.Stdin, + "empty start sentinel must not enable remotecommand stdin streaming") + command := strings.Join(execs.requests[0].request.Command, " ") + require.Contains(t, command, "temporary=$(mktemp") + require.Contains(t, command, "mv -f \"$temporary\" \"$destination\"") +} + +func TestKubernetesWriteFileCommandRejectsSymlinkedAncestor(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + require.NoError(t, os.Symlink(outside, filepath.Join(root, "redirect"))) + destination := filepath.ToSlash(filepath.Join(root, "redirect", "secret.json")) + command := exec.Command("sh", "-c", kubernetesWriteFileCommand(destination, 0o600, true)) + command.Stdin = strings.NewReader("secret") + + err := command.Run() + + require.Error(t, err) + _, statErr := os.Stat(filepath.Join(outside, "secret.json")) + require.ErrorIs(t, statErr, os.ErrNotExist) +} + +func TestKubernetesPodFileUploaderNormalizesWindowsSeparators(t *testing.T) { + execs := &recordingKubernetesExec{} + uploader := kubernetesPodFileUploader{ + streams: kubeexecutor.NewStreamOperations(execs, nil), + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "kandev-agents", Name: "agent-pod", + }}, + container: "kandev-agent", + } + + err := uploader.WriteFile( + context.Background(), `\run\kandev\home\.codex\auth.json`, []byte("secret"), 0o600, + ) + + require.NoError(t, err) + require.Len(t, execs.requests, 1) + remoteCommand := strings.Join(execs.requests[0].request.Command, " ") + require.Contains(t, remoteCommand, "/run/kandev/home/.codex/auth.json") + require.NotContains(t, remoteCommand, `\run\kandev`) +} + +func TestKubernetesBootstrapOwnsControlEnvironmentAndPreservesOfficeEnvironment(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, execs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + req.AgentProfileID = "execution-profile" + req.OfficeAgentProfileID = "office-profile" + req.Env = map[string]string{ + "HOME": "/tmp/attacker-home", + "AGENTCTL_AUTH_TOKEN": "attacker-token", + "AGENTCTL_BOOTSTRAP_NONCE": "attacker-nonce", + "AGENTCTL_LISTEN_HOST": "0.0.0.0", + "AGENTCTL_PORT": "9999", + "AGENTCTL_FUTURE_CONTROL": "attacker-control", + "KANDEV_INSTANCE_ID": "attacker-instance", + "KANDEV_EXECUTION_ID": "attacker-execution", + "KANDEV_TASK_ID": "attacker-task", + "KANDEV_SESSION_ID": "attacker-session", + "KANDEV_TASK_ENVIRONMENT_ID": "attacker-environment", + "KANDEV_AGENT_PROFILE_ID": "attacker-agent-profile", + "KANDEV_EXECUTION_PROFILE_ID": "attacker-execution-profile", + "KANDEV_API_KEY": "office-api-key", + "KANDEV_CLI": "/opt/kandev/bin/kandev", + "KANDEV_RUN_ID": "run-office-1", + } + + instance, err := executor.CreateInstance(context.Background(), req) + + require.NoError(t, err) + runtimeData := string(kubernetesRecordedUpload(t, execs, kubernetesRuntimeEnvPath)) + require.Contains(t, runtimeData, "KANDEV_INSTANCE_ID='instance-1'") + require.Contains(t, runtimeData, "KANDEV_TASK_ID='task-1'") + require.Contains(t, runtimeData, "KANDEV_SESSION_ID='session-1'") + require.Contains(t, runtimeData, "KANDEV_TASK_ENVIRONMENT_ID='environment-1'") + require.Contains(t, runtimeData, "KANDEV_AGENT_PROFILE_ID='office-profile'") + require.Contains(t, runtimeData, "KANDEV_EXECUTION_PROFILE_ID='execution-profile'") + require.NotContains(t, runtimeData, "KANDEV_EXECUTION_ID") + + authData := string(kubernetesRecordedUpload(t, execs, kubernetesAuthEnvPath)) + require.Contains(t, authData, "HOME='/run/kandev/home'") + require.Contains(t, authData, "AGENTCTL_BOOTSTRAP_NONCE='"+instance.BootstrapNonce+"'") + require.Contains(t, authData, "KANDEV_API_KEY='office-api-key'") + require.Contains(t, authData, "KANDEV_CLI='/opt/kandev/bin/kandev'") + require.Contains(t, authData, "KANDEV_RUN_ID='run-office-1'") + for _, hostile := range []string{ + "attacker-token", "attacker-nonce", "0.0.0.0", "9999", "attacker-control", + "attacker-instance", "attacker-execution", "attacker-task", "attacker-session", + "attacker-environment", "attacker-agent-profile", "attacker-execution-profile", + } { + require.NotContains(t, authData, hostile) + } +} + +func TestKubernetesBootstrapPublishesManagedCredentialHelperPath(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, execs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + req.Env = map[string]string{ + githubauth.CredentialBrokerURLEnv: "https://broker.example/resolve", + githubauth.CredentialLeaseEnv: "opaque-lease", + githubauth.CredentialHelperPathEnv: "/tmp/hostile-agentctl", + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.NoError(t, err) + authData := string(kubernetesRecordedUpload(t, execs, kubernetesAuthEnvPath)) + require.Contains(t, authData, + githubauth.CredentialHelperPathEnv+"='/opt/kandev/agentctl'", + ) + require.NotContains(t, authData, "/tmp/hostile-agentctl") + command := kubernetesBootstrapCommand() + require.Less(t, strings.Index(command, ". "+kubernetesAuthEnvPath), strings.Index(command, "sh "+kubernetesPreparePath)) + require.Less(t, strings.Index(command, ". "+kubernetesAuthEnvPath), strings.Index(command, "exec "+kubernetesAgentctlPath)) +} + +func TestKubernetesPrepareScriptUsesManagedAgentctlPathWithoutStartingSecondProcess(t *testing.T) { + req := validKubernetesCreateRequest() + req.Metadata[MetadataKeySetupScript] = "{{kandev.agentctl.install}}\n{{kandev.agentctl.start}}\n" + + script, err := kubernetesPrepareScript(req) + + require.NoError(t, err) + require.Contains(t, script, "chmod +x '/opt/kandev/agentctl'") + require.NotContains(t, script, "/usr/local/bin/agentctl") + require.NotContains(t, script, "nohup agentctl") +} + +func TestKubernetesDefaultPrepareScriptSupportsRepositorylessAgentInstall(t *testing.T) { + req := validKubernetesCreateRequest() + req.AgentConfig = &stubAgent{ + MockAgent: agents.NewMockAgent(), id: "office-agent", + installScript: "printf 'managed-agent-installed\\n'", + } + + script, err := kubernetesPrepareScript(req) + + require.NoError(t, err) + require.NotContains(t, script, "{{repository.clone_url}}") + require.NotContains(t, script, "{{kandev.agents.install}}") + require.Contains(t, script, "repository_url=''", "repo-less launch must resolve an explicit empty repository") + require.Contains(t, script, `if [ -n "$repository_url" ]; then`, "clone must be guarded for repo-less launches") + require.Contains(t, script, "managed-agent-installed") +} + +func TestKubernetesBootstrapCreatesPrivateAuthHomeWithoutCredentialFiles(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, execs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + + _, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + + require.NoError(t, err) + execs.mu.Lock() + defer execs.mu.Unlock() + homeIndex, authIndex := -1, -1 + for index, recorded := range execs.requests { + command := strings.Join(recorded.request.Command, " ") + if strings.Contains(command, kubernetesAuthHomePath) && strings.Contains(command, "chmod 0700") { + homeIndex = index + } + if strings.Contains(command, kubernetesAuthEnvPath) { + authIndex = index + } + } + require.NotEqual(t, -1, homeIndex, "bootstrap must create the fixed auth HOME even without config bundles") + require.NotEqual(t, -1, authIndex) + require.Less(t, homeIndex, authIndex, "auth HOME must exist before auth.env is written") +} + +func TestKubernetesSerializeEnvironmentPreservesExplicitEmptyValues(t *testing.T) { + serialized, err := kubernetesSerializeEnvironment(map[string]string{ + "FOO": "", + "BAR": "value", + }) + + require.NoError(t, err) + require.Equal(t, "BAR='value'\nFOO=''\n", string(serialized)) +} + +func TestKubernetesSkillManifestRejectsDotInstructionIdentity(t *testing.T) { + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, execs, nil) + req := validKubernetesCreateRequest() + req.Metadata[MetadataKeySkillManifestJSON] = `{ + "Instructions":[{"Filename":"agentctl","Content":"overwrite"}], + "WorkspaceSlug":"..","AgentID":"..","ProjectSkillDir":".agents/skills" + }` + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "unsafe path component") + for _, request := range execs.requests { + require.NotEqual(t, "overwrite", string(request.stdin), + "manifest traversal must not overwrite /opt/kandev/agentctl") + } +} + +func kubernetesRecordedUpload(t *testing.T, execs *recordingKubernetesExec, destination string) []byte { + t.Helper() + execs.mu.Lock() + defer execs.mu.Unlock() + for _, recorded := range execs.requests { + if strings.Contains(strings.Join(recorded.request.Command, " "), destination) { + return append([]byte(nil), recorded.stdin...) + } + } + t.Fatalf("upload for %s not found", destination) + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_checkpoint_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_checkpoint_test.go new file mode 100644 index 0000000000..950bacb509 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_checkpoint_test.go @@ -0,0 +1,709 @@ +package lifecycle + +import ( + "context" + "errors" + "net" + "regexp" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +func TestKubernetesCreateFreshLeavesDurableInventoryProvisionalUntilManagerPersistsSecrets(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + executor := newFakeKubernetesExecutor( + t, &fakeKubernetesResources{}, &recordingKubernetesExec{}, + map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }, + ) + req := validKubernetesCreateRequest() + var states []string + req.CheckpointRuntimeInventory = func(_ context.Context, metadata map[string]interface{}) error { + states = append(states, getMetadataString(metadata, MetadataKeyKubernetesInventoryState)) + return nil + } + req.ReleaseRuntimeInventory = func(context.Context) error { return nil } + + instance, err := executor.CreateInstance(context.Background(), req) + + require.NoError(t, err) + require.Equal(t, KubernetesInventoryStateReady, instance.Metadata[MetadataKeyKubernetesInventoryState]) + require.NotEmpty(t, states) + require.NotContains(t, states, KubernetesInventoryStateReady, + "manager must persist required runtime secret refs before marking durable inventory ready") +} + +func TestKubernetesCreateFreshReconcilesCommittedPodAfterAmbiguousCreateError(t *testing.T) { + createErr := &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET} + resources := &fakeKubernetesResources{createPodErr: createErr} + executor := newFakeKubernetesExecutor( + t, resources, &recordingKubernetesExec{err: errors.New("bootstrap failed")}, nil, + ) + req := validKubernetesCreateRequest() + var states []string + req.CheckpointRuntimeInventory = func(_ context.Context, metadata map[string]interface{}) error { + states = append(states, getMetadataString(metadata, MetadataKeyKubernetesInventoryState)) + return nil + } + releases := 0 + req.ReleaseRuntimeInventory = func(context.Context) error { releases++; return nil } + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "bootstrap") + require.Len(t, resources.createdPods, 1) + require.Equal(t, "kandev-agents/"+resources.createdPods[0].Name, resources.getPodRequests[0]) + require.Contains(t, states, KubernetesInventoryStatePodCreated) + require.Contains(t, states, KubernetesInventoryStatePodAdmitted) + require.Contains(t, resources.deletedPods, resources.createdPods[0].Name+":pod-uid") + require.Equal(t, 1, releases) +} + +func TestKubernetesCreateFreshReconcilesCommittedPVCAfterAmbiguousCreateError(t *testing.T) { + createErr := &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET} + resources := &fakeKubernetesResources{createPVCErr: createErr} + executor := newFakeKubernetesExecutor( + t, resources, &recordingKubernetesExec{err: errors.New("bootstrap failed")}, nil, + ) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + var states []string + req.CheckpointRuntimeInventory = func(_ context.Context, metadata map[string]interface{}) error { + states = append(states, getMetadataString(metadata, MetadataKeyKubernetesInventoryState)) + return nil + } + releases := 0 + req.ReleaseRuntimeInventory = func(context.Context) error { releases++; return nil } + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "bootstrap") + require.Len(t, resources.createdPVCs, 1) + require.Equal(t, "kandev-agents/"+resources.createdPVCs[0].Name, resources.getPVCRequests[0]) + require.Contains(t, states, KubernetesInventoryStatePVCCreated) + require.Contains(t, states, KubernetesInventoryStatePVCAdmitted) + require.Contains(t, resources.deletedPVCs, resources.createdPVCs[0].Name+":pvc-uid") + require.Equal(t, 1, releases) +} + +func TestKubernetesCreateStampsFreshNonceOnEachManagedResource(t *testing.T) { + resources := &fakeKubernetesResources{} + execs := &recordingKubernetesExec{err: errors.New("bootstrap failed")} + executor := newFakeKubernetesExecutor(t, resources, execs, nil) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "bootstrap") + require.Len(t, resources.createdPVCs, 1) + require.Len(t, resources.createdPods, 1) + pvcNonce := resources.createdPVCs[0].Annotations[kubeexecutor.CreateNonceAnnotation] + podNonce := resources.createdPods[0].Annotations[kubeexecutor.CreateNonceAnnotation] + require.Regexp(t, regexp.MustCompile(`^[0-9a-f]{64}$`), pvcNonce) + require.Regexp(t, regexp.MustCompile(`^[0-9a-f]{64}$`), podNonce) + require.NotEqual(t, pvcNonce, podNonce) +} + +func TestKubernetesAmbiguousCreateWithoutExactNonceFailsClosed(t *testing.T) { + tests := []struct { + name string + resources *fakeKubernetesResources + configure func(*ExecutorCreateRequest) + }{ + { + name: "Pod missing nonce", + resources: &fakeKubernetesResources{ + createPodErr: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, + mutateCreatedPod: func(pod *corev1.Pod) { + delete(pod.Annotations, kubeexecutor.CreateNonceAnnotation) + }, + }, + configure: func(*ExecutorCreateRequest) {}, + }, + { + name: "Pod different nonce", + resources: &fakeKubernetesResources{ + createPodErr: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, + mutateCreatedPod: func(pod *corev1.Pod) { + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + pod.Annotations[kubeexecutor.CreateNonceAnnotation] = "foreign" + }, + }, + configure: func(*ExecutorCreateRequest) {}, + }, + { + name: "PVC missing nonce", + resources: &fakeKubernetesResources{ + createPVCErr: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, + mutateCreatedPVC: func(pvc *corev1.PersistentVolumeClaim) { + delete(pvc.Annotations, kubeexecutor.CreateNonceAnnotation) + }, + }, + configure: setManagedKubernetesWorkspace, + }, + { + name: "PVC different nonce", + resources: &fakeKubernetesResources{ + createPVCErr: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, + mutateCreatedPVC: func(pvc *corev1.PersistentVolumeClaim) { + if pvc.Annotations == nil { + pvc.Annotations = make(map[string]string) + } + pvc.Annotations[kubeexecutor.CreateNonceAnnotation] = "foreign" + }, + }, + configure: setManagedKubernetesWorkspace, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, test.resources, execs, nil) + req := validKubernetesCreateRequest() + test.configure(req) + checkpoints := 0 + req.CheckpointRuntimeInventory = func(context.Context, map[string]interface{}) error { + checkpoints++ + return nil + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "owned annotation") + require.Zero(t, checkpoints) + require.Empty(t, execs.requests) + require.Empty(t, test.resources.deletedPods) + require.Empty(t, test.resources.deletedPVCs) + }) + } +} + +func TestKubernetesAmbiguousPodCreateNeverDeletesMismatchedObject(t *testing.T) { + resources := &fakeKubernetesResources{ + createPodErr: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, + mutateCreatedPod: func(pod *corev1.Pod) { + pod.Labels["kandev.ai/copied-owner"] = "other-session" + }, + } + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + var checkpoints int + req.CheckpointRuntimeInventory = func(context.Context, map[string]interface{}) error { + checkpoints++ + return nil + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "Pod ownership labels") + require.Zero(t, checkpoints) + require.Empty(t, resources.deletedPods) + require.NotNil(t, resources.pod, "foreign same-name Pod must remain untouched") +} + +func TestKubernetesAmbiguousPVCCreateNeverDeletesMismatchedObject(t *testing.T) { + resources := &fakeKubernetesResources{ + createPVCErr: &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNRESET}, + mutateCreatedPVC: func(pvc *corev1.PersistentVolumeClaim) { + pvc.Labels["kandev.ai/copied-owner"] = "other-session" + }, + } + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + var checkpoints int + req.CheckpointRuntimeInventory = func(context.Context, map[string]interface{}) error { + checkpoints++ + return nil + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "PVC ownership labels") + require.Zero(t, checkpoints) + require.Empty(t, resources.deletedPVCs) + require.NotNil(t, resources.pvc, "foreign same-name PVC must remain untouched") +} + +func TestKubernetesCreateAlreadyExistsNeverAdoptsVisiblePodOrPVC(t *testing.T) { + tests := []struct { + name string + resources *fakeKubernetesResources + configure func(*ExecutorCreateRequest) + }{ + { + name: "Pod", + resources: &fakeKubernetesResources{createPodErr: apierrors.NewAlreadyExists( + schema.GroupResource{Resource: "pods"}, "visible-pod", + )}, + configure: func(*ExecutorCreateRequest) {}, + }, + { + name: "PVC", + resources: &fakeKubernetesResources{createPVCErr: apierrors.NewAlreadyExists( + schema.GroupResource{Resource: "persistentvolumeclaims"}, "visible-pvc", + )}, + configure: setManagedKubernetesWorkspace, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newFakeKubernetesExecutor(t, test.resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + test.configure(req) + checkpoints := 0 + req.CheckpointRuntimeInventory = func(context.Context, map[string]interface{}) error { + checkpoints++ + return nil + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "already exists") + require.Zero(t, checkpoints, "known create rejection must not authorize visible-object adoption") + require.Empty(t, test.resources.getPodRequests) + require.Empty(t, test.resources.getPVCRequests) + require.Empty(t, test.resources.deletedPods) + require.Empty(t, test.resources.deletedPVCs) + }) + } +} + +func TestKubernetesRollbackUsesCurrentResourceVersionForMatchingCreateUID(t *testing.T) { + t.Run("Pod", func(t *testing.T) { + created := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "agents", Name: "pod", UID: types.UID("pod-uid"), ResourceVersion: "rv-create", + }} + resources := &fakeKubernetesResources{pod: created.DeepCopy()} + resources.pod.ResourceVersion = "rv-current" + resources.pod.Labels = map[string]string{"kandev.ai/admission-mutated": "true"} + + err := rollbackCreatedKubernetesPod(context.Background(), resources, created) + + require.NoError(t, err) + require.Equal(t, []string{"rv-current"}, resources.deletedPodResourceVersions) + }) + + t.Run("PVC", func(t *testing.T) { + created := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: "agents", Name: "pvc", UID: types.UID("pvc-uid"), ResourceVersion: "rv-create", + }} + resources := &fakeKubernetesResources{pvc: created.DeepCopy()} + resources.pvc.ResourceVersion = "rv-current" + resources.pvc.Labels = map[string]string{"kandev.ai/admission-mutated": "true"} + + err := rollbackCreatedKubernetesPVC(context.Background(), resources, created) + + require.NoError(t, err) + require.Equal(t, []string{"rv-current"}, resources.deletedPVCResourceVersions) + }) +} + +func TestKubernetesRollbackLeavesSameNameReplacementUntouched(t *testing.T) { + created := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "agents", Name: "pod", UID: types.UID("created-uid"), ResourceVersion: "rv-create", + }} + resources := &fakeKubernetesResources{pod: created.DeepCopy()} + resources.pod.UID = types.UID("replacement-uid") + resources.pod.ResourceVersion = "rv-replacement" + + require.NoError(t, rollbackCreatedKubernetesPod(context.Background(), resources, created)) + require.Empty(t, resources.deletedPods) + require.Equal(t, types.UID("replacement-uid"), resources.pod.UID) +} + +func TestKubernetesCreateNotFoundReconciliationPreservesOriginalCreateError(t *testing.T) { + tests := []struct { + name string + resources *fakeKubernetesResources + configure func(*ExecutorCreateRequest) + getCalls func(*fakeKubernetesResources) int + }{ + { + name: "Pod", + resources: &fakeKubernetesResources{createPodBeforeCommitErr: &net.OpError{ + Op: "write", Net: "tcp", Err: errors.New("Pod create transport failed"), + }}, + configure: func(*ExecutorCreateRequest) {}, + getCalls: func(resources *fakeKubernetesResources) int { return len(resources.getPodRequests) }, + }, + { + name: "PVC", + resources: &fakeKubernetesResources{createPVCBeforeCommitErr: &net.OpError{ + Op: "write", Net: "tcp", Err: errors.New("PVC create transport failed"), + }}, + configure: setManagedKubernetesWorkspace, + getCalls: func(resources *fakeKubernetesResources) int { return len(resources.getPVCRequests) }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newFakeKubernetesExecutor(t, test.resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + test.configure(req) + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, test.name+" create transport failed") + require.Equal(t, 1, test.getCalls(test.resources)) + require.Empty(t, test.resources.deletedPods) + require.Empty(t, test.resources.deletedPVCs) + }) + } +} + +func TestKubernetesAmbiguousCreateReconciliationOutlivesCanceledCreateContext(t *testing.T) { + t.Run("Pod", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + resources := &fakeKubernetesResources{ + createPodErr: context.Canceled, + rejectCanceledGetContexts: true, + mutateCreatedPod: func(*corev1.Pod) { cancel() }, + } + req := validKubernetesCreateRequest() + profile, err := kubernetesProfileConfigFromMetadata(req.Metadata) + require.NoError(t, err) + identity, err := kubernetesIdentity(req) + require.NoError(t, err) + desired, err := composeKubernetesLifecyclePod( + profile, identity, "pod", "kandev-agents", "", + ) + require.NoError(t, err) + + created, running, err := createAndWaitKubernetesPod( + ctx, resources, desired, identity, profile.MainContainer, "Pod", nil, + ) + + require.NoError(t, err) + require.NotNil(t, created) + require.NotNil(t, running) + }) + + t.Run("PVC", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + resources := &fakeKubernetesResources{ + createPVCErr: context.Canceled, + rejectCanceledGetContexts: true, + mutateCreatedPVC: func(*corev1.PersistentVolumeClaim) { cancel() }, + } + workspace := kubeexecutor.WorkspaceConfig{ + Mode: kubeexecutor.WorkspaceModeManagedPVC, Size: "1Gi", + AccessModes: []string{"ReadWriteOnce"}, + } + identity, err := kubernetesIdentity(validKubernetesCreateRequest()) + require.NoError(t, err) + + provision, err := createManagedKubernetesWorkspacePVC( + ctx, resources, "kandev-agents", workspace, identity, "workspace", nil, + ) + + require.NoError(t, err) + require.NotNil(t, provision.claim) + }) +} + +func TestKubernetesCreateFreshRetainsCheckpointWhenRollbackFails(t *testing.T) { + deleteErr := errors.New("delete denied") + resources := &fakeKubernetesResources{ + deletePodErr: deleteErr, + mutateCreatedPod: func(pod *corev1.Pod) { + pod.Labels["kandev.ai/injected"] = "unsafe" + }, + } + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + var checkpoints []map[string]interface{} + releases := 0 + req.CheckpointRuntimeInventory = func(_ context.Context, metadata map[string]interface{}) error { + checkpoints = append(checkpoints, cloneKubernetesMetadata(metadata)) + return nil + } + req.ReleaseRuntimeInventory = func(context.Context) error { + releases++ + return nil + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.Error(t, err) + require.ErrorIs(t, err, deleteErr, "rollback error must stay causal") + require.NotEmpty(t, checkpoints) + last := checkpoints[len(checkpoints)-1] + require.Equal(t, KubernetesInventoryStatePodCreated, last[MetadataKeyKubernetesInventoryState]) + require.Equal(t, "pod-uid", last[MetadataKeyKubernetesPodUID]) + require.Equal(t, "pvc-uid", last[MetadataKeyKubernetesPVCUID]) + require.Zero(t, releases, "failed rollback must retain exact durable inventory") + require.NotNil(t, resources.pod) + require.Nil(t, resources.pvc, "successful PVC rollback may complete independently") + + resources.deletePodErr = nil + cleanupMetadata := cloneKubernetesMetadata(req.Metadata) + for key, value := range last { + cleanupMetadata[key] = value + } + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + require.NoError(t, restarted.StopInstance(context.Background(), &ExecutorInstance{ + InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, + Metadata: cleanupMetadata, StopReason: StopReasonTaskDeleted, + }, true)) + require.Nil(t, resources.pod, "recorded Create UID must authorize retry after admission mutation") +} + +func TestKubernetesCreateFreshReleasesCheckpointAfterSuccessfulRollback(t *testing.T) { + resources := &fakeKubernetesResources{mutateCreatedPod: func(pod *corev1.Pod) { + pod.Labels["kandev.ai/injected"] = "unsafe" + }} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + checkpoints := 0 + releases := 0 + req.CheckpointRuntimeInventory = func(context.Context, map[string]interface{}) error { + checkpoints++ + return nil + } + req.ReleaseRuntimeInventory = func(context.Context) error { + releases++ + return nil + } + + _, err := executor.CreateInstance(context.Background(), req) + + require.Error(t, err) + require.Positive(t, checkpoints) + require.Equal(t, 1, releases) + require.Nil(t, resources.pod) + require.Nil(t, resources.pvc) +} + +func TestCheckpointKubernetesRuntimeInventoryPersistsBeforeExecutionRegistration(t *testing.T) { + writer := &checkpointExecutorRunningWriter{} + mgr := &Manager{runningWriter: writer, logger: newTestRegistryLogger()} + req := &ExecutorCreateRequest{ + InstanceID: "execution-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "agent-profile-1", + Metadata: map[string]interface{}{ + "executor_id": "executor-1", + MetadataKeyExecutorProfileID: "executor-profile-1", + MetadataKeyCleanupScript: "cleanup", + MetadataKeyKubernetesAuthMode: "in_cluster", + MetadataKeyKubernetesConfigNamespace: "kandev-agents", + "raw_secret_value": "must-not-persist", + }, + } + runtimeMetadata := map[string]interface{}{ + MetadataKeyKubernetesInventoryState: KubernetesInventoryStatePodCreated, + MetadataKeyKubernetesNamespace: "kandev-agents", + MetadataKeyKubernetesPodName: "pod-1", + MetadataKeyKubernetesPodUID: "pod-uid", + MetadataKeyKubernetesResourceInstanceID: "resource-1", + MetadataKeyKubernetesResourceExecutorID: "executor-1", + MetadataKeyKubernetesResourceProfileID: "executor-profile-1", + MetadataKeyKubernetesResourceTaskID: "task-1", + MetadataKeyKubernetesResourceSessionID: "session-1", + MetadataKeyKubernetesResourceEnvironmentID: "environment-1", + } + + err := mgr.checkpointKubernetesRuntimeInventory(context.Background(), req, runtimeMetadata) + + require.NoError(t, err) + require.NotNil(t, writer.running) + require.Equal(t, "execution-1", writer.running.AgentExecutionID) + require.Equal(t, "executor-1", writer.running.ExecutorID) + require.Equal(t, "agent-profile-1", writer.running.ExecutionProfileID) + require.Equal(t, agentruntime.RuntimeKubernetes, writer.running.Runtime) + require.Equal(t, models.ExecutorRunningStatusStarting, writer.running.Status) + require.Equal(t, "pod-uid", writer.running.Metadata[MetadataKeyKubernetesPodUID]) + require.Equal(t, "cleanup", writer.running.Metadata[MetadataKeyCleanupScript]) + require.NotContains(t, writer.running.Metadata, "raw_secret_value") + + require.NoError(t, mgr.releaseKubernetesRuntimeInventory(context.Background(), req)) + require.Equal(t, "session-1", writer.deletedSessionID) + require.Equal(t, "execution-1", writer.deletedExecutionID) +} + +func TestKubernetesInventoryPersistenceOutlivesCanceledLaunchContext(t *testing.T) { + writer := &checkpointExecutorRunningWriter{rejectCanceledCalls: true} + mgr := &Manager{runningWriter: writer, logger: newTestRegistryLogger()} + req := &ExecutorCreateRequest{ + InstanceID: "execution-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "agent-profile-1", + Metadata: map[string]interface{}{"executor_id": "executor-1"}, + } + runtimeMetadata := map[string]interface{}{ + MetadataKeyKubernetesInventoryState: KubernetesInventoryStatePodCreated, + MetadataKeyKubernetesPodName: "pod-1", + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + require.NoError(t, mgr.checkpointKubernetesRuntimeInventory(ctx, req, runtimeMetadata)) + require.NotNil(t, writer.running) + require.NoError(t, mgr.releaseKubernetesRuntimeInventory(ctx, req)) + require.Nil(t, writer.running) +} + +func TestRollbackLaunchExecutionReleasesInventoryOnlyAfterRuntimeCleanup(t *testing.T) { + tests := []struct { + name string + stopErr error + wantReleases int + }{ + {name: "cleanup succeeds", wantReleases: 1}, + {name: "cleanup fails", stopErr: errors.New("cleanup denied")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mgr := &Manager{logger: newTestRegistryLogger()} + backend := &mockStopTracker{name: agentruntime.RuntimeKubernetes, stopErr: test.stopErr} + releases := 0 + instance := &ExecutorInstance{ReleaseRuntimeInventory: func(context.Context) error { + releases++ + return nil + }} + execution := &AgentExecution{ID: "execution-1", SessionID: "session-1"} + + mgr.rollbackLaunchExecution(context.Background(), backend, instance, execution, "test rollback") + + require.Equal(t, test.wantReleases, releases) + }) + } +} + +func TestRollbackRegisteredLaunchPreservesInventoryUntilRuntimeCleanupSucceeds(t *testing.T) { + writer := &rollbackCheckpointWriter{checkpointExecutorRunningWriter: checkpointExecutorRunningWriter{ + running: &models.ExecutorRunning{ + SessionID: "session-1", AgentExecutionID: "execution-1", + Runtime: agentruntime.RuntimeKubernetes, + }, + }} + mgr := &Manager{ + runningWriter: writer, logger: newTestRegistryLogger(), stopCh: make(chan struct{}), + executionStore: NewExecutionStore(), + } + execution := &AgentExecution{ID: "execution-1", SessionID: "session-1"} + require.NoError(t, mgr.executionStore.Add(execution)) + backend := &mockStopTracker{ + name: agentruntime.RuntimeKubernetes, stopErr: errors.New("exact cleanup failed"), + } + + mgr.rollbackRegisteredLaunch(backend, &ExecutorInstance{}, execution, "test rollback") + + require.NotNil(t, writer.running, "cleanup failure must retain authoritative row") + _, exists := mgr.executionStore.Get("execution-1") + require.True(t, exists, "cleanup failure must retain retry ownership") + mgr.closeStopCh() + mgr.wg.Wait() +} + +func TestFreshKubernetesSiblingRollbackOwnsItsNewRuntime(t *testing.T) { + mgr := &Manager{logger: newTestRegistryLogger()} + backend := &mockStopTracker{name: agentruntime.RuntimeKubernetes} + releases := 0 + instance := &ExecutorInstance{ReleaseRuntimeInventory: func(context.Context) error { + releases++ + return nil + }} + execution := &AgentExecution{ + ID: "execution-new", SessionID: "session-new", RuntimeName: agentruntime.RuntimeKubernetes, + } + applyResumeIntent(execution, &ExecutorCreateRequest{PreviousExecutionID: ""}) + + err := mgr.stopRegisteredLaunchRuntime(backend, instance, execution) + + require.NoError(t, err) + require.False(t, execution.isResumedSession) + require.True(t, backend.stopForce, "fresh sibling Pod/PVC must be deleted on launch rollback") + require.Equal(t, 1, releases, "fresh provisional inventory must be released after exact cleanup") +} + +type checkpointExecutorRunningWriter struct { + running *models.ExecutorRunning + deletedSessionID string + deletedExecutionID string + rejectCanceledCalls bool +} + +type rollbackCheckpointWriter struct { + checkpointExecutorRunningWriter +} + +func (w *rollbackCheckpointWriter) DeleteExecutorRunningBySessionID(context.Context, string) error { + w.running = nil + return nil +} + +func (w *checkpointExecutorRunningWriter) UpsertExecutorRunning( + ctx context.Context, + running *models.ExecutorRunning, +) error { + if w.rejectCanceledCalls && ctx.Err() != nil { + return ctx.Err() + } + w.running = running + return nil +} + +func (w *checkpointExecutorRunningWriter) GetExecutorRunningBySessionID( + ctx context.Context, + _ string, +) (*models.ExecutorRunning, error) { + if w.rejectCanceledCalls && ctx.Err() != nil { + return nil, ctx.Err() + } + if w.running == nil { + return nil, models.ErrExecutorRunningNotFound + } + return w.running, nil +} + +func (*checkpointExecutorRunningWriter) DeleteExecutorRunningBySessionID(context.Context, string) error { + return errors.New("non-CAS delete must not be used") +} + +func (*checkpointExecutorRunningWriter) RepairExecutorRunningDead(context.Context, string) error { + return nil +} + +func (*checkpointExecutorRunningWriter) RepairExecutorRunningDeadIfCurrent( + context.Context, + string, + string, + time.Time, +) error { + return nil +} + +func (w *checkpointExecutorRunningWriter) DeleteExecutorRunningIfCurrent( + ctx context.Context, + sessionID string, + executionID string, + _ time.Time, +) error { + if w.rejectCanceledCalls && ctx.Err() != nil { + return ctx.Err() + } + w.deletedSessionID = sessionID + w.deletedExecutionID = executionID + w.running = nil + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_cleanup.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_cleanup.go new file mode 100644 index 0000000000..ae42473af9 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_cleanup.go @@ -0,0 +1,436 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + "github.com/kandev/kandev/internal/scriptengine" + "go.uber.org/zap" +) + +func (r *KubernetesExecutor) StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error { + if instance == nil { + return nil + } + unlock := r.lockInstance(instance.InstanceID) + defer unlock() + + session := r.closeKubernetesSession(instance.InstanceID) + if !force && !shouldRunExecutorCleanup(instance.StopReason) { + return nil + } + runtime, err := r.kubernetesCleanupRuntime(instance, session) + if err != nil { + return err + } + if runtime == nil || runtime.resources == nil { + return errors.New("kubernetes lifecycle: cleanup resource client is unavailable") + } + recorded, identity, err := kubernetesCleanupInventory(instance) + if err != nil { + return err + } + verifyCtx, cancelVerify := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + if err := verifyKubernetesCleanupTargets(verifyCtx, runtime.resources, recorded, identity); err != nil { + cancelVerify() + return err + } + cancelVerify() + r.runKubernetesTerminalCleanupScript(ctx, runtime.streams, instance, recorded) + deleteCtx, cancelDelete := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + defer cancelDelete() + return deleteKubernetesResources(deleteCtx, runtime.resources, recorded, identity) +} + +func (r *KubernetesExecutor) closeKubernetesSession(instanceID string) *kubernetesSession { + session := r.takeSession(instanceID) + _ = closeKubernetesSessionResources(session) + return session +} + +func closeKubernetesSessionResources(session *kubernetesSession) error { + if session == nil { + return nil + } + if session.client != nil { + session.client.Close() + } + if session.forward != nil { + return session.forward.Close() + } + return nil +} + +// Close is the backend-shutdown safety net. It closes process-local clients +// and forwards only; Kubernetes resources stay preserved for resume. +func (r *KubernetesExecutor) Close() error { + r.mu.Lock() + sessions := make([]*kubernetesSession, 0, len(r.sessions)) + for instanceID, session := range r.sessions { + sessions = append(sessions, session) + delete(r.sessions, instanceID) + } + r.mu.Unlock() + + var closeErrors []error + for _, session := range sessions { + if err := closeKubernetesSessionResources(session); err != nil { + closeErrors = append(closeErrors, err) + } + } + return errors.Join(closeErrors...) +} + +func (r *KubernetesExecutor) kubernetesCleanupRuntime( + instance *ExecutorInstance, + session *kubernetesSession, +) (*kubernetesRuntimeClient, error) { + executorConfig, err := kubernetesExecutorConfigFromMetadata(instance.Metadata) + if err != nil { + if session != nil && session.runtime != nil { + return session.runtime, nil + } + return nil, fmt.Errorf("kubernetes lifecycle: cleanup configuration: %w", err) + } + if r.clientFactory == nil { + return nil, errors.New("kubernetes lifecycle: cleanup client factory is not configured") + } + runtime, err := r.clientFactory(executorConfig) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: cleanup client: %w", err) + } + return runtime, nil +} + +func deleteKubernetesResources( + ctx context.Context, + resources kubernetesResourceClient, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if recorded.podUID != "" { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: recorded.namespace, Name: recorded.podName, UID: recorded.podUID, + }} + var err error + if recorded.podAdmissionVerified() { + err = deleteKubernetesPodIfExact(ctx, resources, pod, identity) + } else { + err = deleteKubernetesPodByRecordedUID(ctx, resources, pod) + } + if err != nil { + return err + } + } + if !recorded.pvcCreated { + return nil + } + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: recorded.namespace, Name: recorded.pvcName, UID: recorded.pvcUID, + }} + if recorded.pvcAdmissionVerified() { + return deleteKubernetesPVCIfExact(ctx, resources, pvc, identity) + } + return deleteKubernetesPVCByRecordedUID(ctx, resources, pvc) +} + +func (r *KubernetesExecutor) runKubernetesTerminalCleanupScript( + ctx context.Context, + streams *kubeexecutor.StreamOperations, + instance *ExecutorInstance, + recorded kubernetesRecordedState, +) { + if streams == nil || instance == nil || !shouldRunExecutorCleanup(instance.StopReason) || + recorded.podUID == "" || !recorded.podAdmissionVerified() { + return + } + script := strings.TrimSpace(getMetadataString(instance.Metadata, MetadataKeyCleanupScript)) + if script == "" { + return + } + resolved := scriptengine.NewResolver(). + WithStatic(map[string]string{kubernetesRepositoryCloneURL: shellQuote("")}). + WithProvider(scriptengine.WorkspaceProvider(kubernetesWorkspacePath)). + WithProvider(scriptengine.AgentctlProviderWithOptions( + int(kubeexecutor.DefaultAgentctlPort), kubernetesWorkspacePath, + scriptengine.AgentctlProviderOptions{BinaryPath: kubernetesAgentctlPath, Start: false}, + )). + WithProvider(scriptengine.GitIdentityProvider(instance.Metadata)). + WithProvider(scriptengine.RepositoryProvider( + instance.Metadata, nil, getGitRemoteURL, injectGitHubTokenIntoCloneURL, + )). + Resolve(script) + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute) + defer cancel() + err := streams.Exec(cleanupCtx, kubeexecutor.ExecRequest{ + Namespace: recorded.namespace, + Pod: recorded.podName, + Container: recorded.mainContainer, + Command: []string{"sh", "-c", resolved}, + Stdout: io.Discard, + Stderr: io.Discard, + }) + if err != nil { + r.logger.Warn("Kubernetes cleanup script failed", + zap.String("instance_id", instance.InstanceID), + zap.String("reason", instance.StopReason), + zap.Error(routingerr.SanitizeError(err))) + return + } + r.logger.Debug("Kubernetes cleanup script completed", + zap.String("instance_id", instance.InstanceID), + zap.String("reason", instance.StopReason)) +} + +func verifyKubernetesCleanupTargets( + ctx context.Context, + resources kubernetesResourceClient, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if err := verifyKubernetesCleanupPod(ctx, resources, recorded, identity); err != nil { + return err + } + return verifyKubernetesCleanupPVC(ctx, resources, recorded, identity) +} + +func verifyKubernetesCleanupPod( + ctx context.Context, + resources kubernetesResourceClient, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if recorded.podUID == "" { + return nil + } + pod, err := resources.GetPod(ctx, recorded.namespace, recorded.podName) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("kubernetes lifecycle: inspect Pod before cleanup: %w", err) + } + if err == nil { + verifyErr := verifyKubernetesCleanupPodIdentity(pod, recorded, identity) + if verifyErr != nil { + return verifyErr + } + } + return nil +} + +func verifyKubernetesCleanupPVC( + ctx context.Context, + resources kubernetesResourceClient, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if recorded.workspaceMode == kubeexecutor.WorkspaceModeEmptyDir || recorded.pvcName == "" { + return nil + } + pvc, err := resources.GetPersistentVolumeClaim(ctx, recorded.namespace, recorded.pvcName) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("kubernetes lifecycle: inspect PVC before cleanup: %w", err) + } + if pvc == nil || pvc.Namespace != recorded.namespace || pvc.Name != recorded.pvcName || pvc.UID != recorded.pvcUID { + return errors.New("kubernetes lifecycle: PVC UID or name does not match recorded inventory") + } + if !recorded.pvcCreated || !recorded.pvcAdmissionVerified() { + return nil + } + labels, err := kubeexecutor.OwnershipLabels(identity) + if err != nil { + return err + } + if err := kubeexecutor.ValidateOwnershipLabels(pvc.Labels, labels); err != nil { + return fmt.Errorf("kubernetes lifecycle: PVC ownership labels: %w", err) + } + return nil +} + +func (r *KubernetesExecutor) takeSession(instanceID string) *kubernetesSession { + r.mu.Lock() + defer r.mu.Unlock() + session := r.sessions[instanceID] + delete(r.sessions, instanceID) + return session +} + +func kubernetesCleanupInventory( + instance *ExecutorInstance, +) (kubernetesRecordedState, kubeexecutor.ResourceIdentity, error) { + if instance == nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, errors.New( + "kubernetes lifecycle: cleanup instance is nil", + ) + } + metadata := instance.Metadata + namespace := getMetadataString(metadata, MetadataKeyKubernetesNamespace) + executorConfig := kubeexecutor.ExecutorConfig{ + AuthMode: kubeexecutor.AuthModeInCluster, Namespace: namespace, + RequestTimeoutSeconds: kubeexecutor.DefaultRequestTimeoutSeconds, + } + req := &ExecutorCreateRequest{ + TaskID: instance.TaskID, + SessionID: instance.SessionID, + PreviousExecutionID: getMetadataString(metadata, MetadataKeyKubernetesResourceInstanceID), + Metadata: metadata, + } + return kubernetesRecordedCleanupInventory(req, executorConfig, false) +} + +func kubernetesRecordedCleanupInventory( + req *ExecutorCreateRequest, + executorConfig kubeexecutor.ExecutorConfig, + validateConnectionIdentity bool, +) (kubernetesRecordedState, kubeexecutor.ResourceIdentity, error) { + state := getMetadataString(req.Metadata, MetadataKeyKubernetesInventoryState) + if state == "" || state == KubernetesInventoryStateReady { + return kubernetesRecordedInventory(req, validateConnectionIdentity) + } + return kubernetesRecordedProvisionalInventory(req, executorConfig, validateConnectionIdentity, state) +} + +func kubernetesRecordedProvisionalInventory( + req *ExecutorCreateRequest, + executorConfig kubeexecutor.ExecutorConfig, + validateConnectionIdentity bool, + state string, +) (kubernetesRecordedState, kubeexecutor.ResourceIdentity, error) { + recorded := decodeKubernetesRecordedStateBase(req.Metadata) + identity, err := kubernetesRecordedResourceIdentity(req, validateConnectionIdentity) + if err != nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err + } + if err := validateKubernetesProvisionalInventory(recorded, identity, executorConfig.Namespace, state); err != nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err + } + return recorded, identity, nil +} + +func validateKubernetesProvisionalInventory( + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, + namespace, state string, +) error { + if recorded.agentctlInstanceID != identity.InstanceID { + return errors.New( + "kubernetes lifecycle: recorded agentctl instance does not match resource identity", + ) + } + if recorded.namespace == "" || recorded.namespace != namespace || recorded.mainContainer == "" { + return errors.New( + "kubernetes lifecycle: provisional namespace or container inventory is incomplete", + ) + } + if err := validateKubernetesRecordedWorkspace(recorded); err != nil { + return err + } + podComplete := recorded.podName != "" && recorded.podUID != "" + if (recorded.podName == "") != (recorded.podUID == "") { + return errors.New( + "kubernetes lifecycle: provisional Pod inventory is incomplete", + ) + } + return validateKubernetesProvisionalInventoryState(state, recorded.workspaceMode, podComplete) +} + +func validateKubernetesProvisionalInventoryState( + state string, + workspaceMode kubeexecutor.WorkspaceMode, + podComplete bool, +) error { + switch state { + case KubernetesInventoryStatePVCCreated, KubernetesInventoryStatePVCAdmitted: + if podComplete || workspaceMode != kubeexecutor.WorkspaceModeManagedPVC { + return errors.New( + "kubernetes lifecycle: provisional PVC inventory state is inconsistent", + ) + } + case KubernetesInventoryStatePodCreated, KubernetesInventoryStatePodAdmitted: + if !podComplete { + return errors.New( + "kubernetes lifecycle: provisional Pod inventory is incomplete", + ) + } + default: + return errors.New( + "kubernetes lifecycle: provisional inventory state is invalid", + ) + } + return nil +} + +func (recorded kubernetesRecordedState) podAdmissionVerified() bool { + return recorded.inventoryState != KubernetesInventoryStatePodCreated +} + +func (recorded kubernetesRecordedState) pvcAdmissionVerified() bool { + return recorded.inventoryState != KubernetesInventoryStatePVCCreated +} + +func verifyKubernetesCleanupPodIdentity( + pod *corev1.Pod, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if recorded.podAdmissionVerified() { + return verifyRecordedPod(pod, recorded.namespace, recorded.podName, recorded.podUID, identity) + } + if pod == nil || pod.Namespace != recorded.namespace || pod.Name != recorded.podName || + pod.UID != recorded.podUID || recorded.podUID == "" { + return errors.New("kubernetes lifecycle: Pod UID or name does not match recorded inventory") + } + return nil +} + +func deleteKubernetesPodByRecordedUID( + ctx context.Context, + resources kubernetesResourceClient, + recorded *corev1.Pod, +) error { + return deleteKubernetesResourceByRecordedUID(ctx, kubernetesPodDeletionTarget(resources, recorded)) +} + +func deleteKubernetesPVCByRecordedUID( + ctx context.Context, + resources kubernetesResourceClient, + recorded *corev1.PersistentVolumeClaim, +) error { + return deleteKubernetesResourceByRecordedUID(ctx, kubernetesPVCDeletionTarget(resources, recorded)) +} + +func deleteKubernetesResourceByRecordedUID(ctx context.Context, target kubernetesDeletionTarget) error { + current, err := target.get(ctx) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf( + "kubernetes lifecycle: inspect provisional %s before deletion: %w", target.kind, err, + ) + } + if current.namespace != target.namespace || current.name != target.name || + current.uid != target.uid || target.uid == "" { + return fmt.Errorf( + "kubernetes lifecycle: provisional %s UID or name does not match recorded inventory", target.kind, + ) + } + if err := target.delete(ctx, target.uid, current.resourceVersion); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("kubernetes lifecycle: delete provisional %s: %w", target.kind, err) + } + return target.wait(ctx, target.uid) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_client.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_client.go new file mode 100644 index 0000000000..6b32d2cc5d --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_client.go @@ -0,0 +1,531 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + kubernetesclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/tools/remotecommand" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" +) + +type kubernetesResourceClient interface { + CreatePersistentVolumeClaim(context.Context, string, *corev1.PersistentVolumeClaim) (*corev1.PersistentVolumeClaim, error) + GetPersistentVolumeClaim(context.Context, string, string) (*corev1.PersistentVolumeClaim, error) + DeletePersistentVolumeClaim(context.Context, string, string, types.UID, string) error + CreatePod(context.Context, string, *corev1.Pod) (*corev1.Pod, error) + GetPod(context.Context, string, string) (*corev1.Pod, error) + WaitForPodRunning(context.Context, string, string, string) (*corev1.Pod, error) + DeletePod(context.Context, string, string, types.UID, string) error +} + +type kubernetesRuntimeClient struct { + resources kubernetesResourceClient + streams *kubeexecutor.StreamOperations +} + +type kubernetesRuntimeClientFactory func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) +type kubernetesAgentctlBinaryResolver func(kubeexecutor.Platform) ([]byte, error) + +func newKubernetesRuntimeClient(config kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + client, err := kubeexecutor.NewClient(config, kubeexecutor.ConfigLoader{}, nil) + if err != nil { + return nil, err + } + streamingConfig := kubernetesStreamingRESTConfig(client.RESTConfig) + watchClient, err := kubernetesclient.NewForConfig(streamingConfig) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: create timeout-free watch client: %w", err) + } + resources := &liveKubernetesResources{client: client, watchClient: watchClient} + streams := kubeexecutor.NewStreamOperations( + &liveKubernetesExecutor{config: streamingConfig, resources: resources}, + &liveKubernetesForwarder{ + config: streamingConfig, resources: resources, + handshakeTimeout: kubernetesPortForwardHandshakeTimeout(config.RequestTimeoutSeconds), + }, + ) + return &kubernetesRuntimeClient{resources: resources, streams: streams}, nil +} + +func kubernetesStreamingRESTConfig(apiConfig *rest.Config) *rest.Config { + if apiConfig == nil { + return nil + } + streamingConfig := rest.CopyConfig(apiConfig) + streamingConfig.Timeout = 0 + return streamingConfig +} + +type liveKubernetesResources struct { + client *kubeexecutor.Client + watchClient kubernetesclient.Interface + watchRelistBackoff func(context.Context) error +} + +const kubernetesWatchRelistBackoff = 100 * time.Millisecond + +func (r *liveKubernetesResources) CreatePersistentVolumeClaim( + ctx context.Context, + namespace string, + pvc *corev1.PersistentVolumeClaim, +) (*corev1.PersistentVolumeClaim, error) { + return r.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Create(ctx, pvc, metav1.CreateOptions{}) +} + +func (r *liveKubernetesResources) GetPersistentVolumeClaim( + ctx context.Context, + namespace, name string, +) (*corev1.PersistentVolumeClaim, error) { + return r.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, name, metav1.GetOptions{}) +} + +func (r *liveKubernetesResources) DeletePersistentVolumeClaim( + ctx context.Context, + namespace, name string, + uid types.UID, + resourceVersion string, +) error { + propagation := metav1.DeletePropagationForeground + return r.client.Clientset.CoreV1().PersistentVolumeClaims(namespace).Delete(ctx, name, metav1.DeleteOptions{ + Preconditions: kubernetesDeletePreconditions(uid, resourceVersion), PropagationPolicy: &propagation, + }) +} + +func (r *liveKubernetesResources) CreatePod( + ctx context.Context, + namespace string, + pod *corev1.Pod, +) (*corev1.Pod, error) { + return r.client.Clientset.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) +} + +func (r *liveKubernetesResources) GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error) { + return r.client.Clientset.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) +} + +func (r *liveKubernetesResources) DeletePod( + ctx context.Context, + namespace, name string, + uid types.UID, + resourceVersion string, +) error { + grace := int64(0) + propagation := metav1.DeletePropagationForeground + return r.client.Clientset.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{ + GracePeriodSeconds: &grace, + Preconditions: kubernetesDeletePreconditions(uid, resourceVersion), + PropagationPolicy: &propagation, + }) +} + +func kubernetesDeletePreconditions(uid types.UID, resourceVersion string) *metav1.Preconditions { + preconditions := &metav1.Preconditions{UID: &uid} + if resourceVersion != "" { + preconditions.ResourceVersion = &resourceVersion + } + return preconditions +} + +func (r *liveKubernetesResources) WaitForPodRunning( + ctx context.Context, + namespace, name, mainContainer string, +) (*corev1.Pod, error) { + var latestDiagnostic error + for { + pod, err := r.GetPod(ctx, namespace, name) + if err != nil { + return nil, kubernetesPodWaitError(ctx, err, latestDiagnostic) + } + latestDiagnostic = kubernetesLatestPodDiagnostic(pod, latestDiagnostic) + if ready, terminalErr := kubernetesPodReady(pod, mainContainer); ready || terminalErr != nil { + return pod, terminalErr + } + outcome := r.watchKubernetesPodTransition( + ctx, namespace, name, mainContainer, pod.ResourceVersion, latestDiagnostic, + ) + latestDiagnostic = outcome.latestDiagnostic + if outcome.relist { + if err := r.waitBeforeWatchRelist(ctx); err != nil { + return nil, kubernetesPodWaitError(ctx, err, latestDiagnostic) + } + continue + } + return outcome.pod, outcome.err + } +} + +func (r *liveKubernetesResources) waitBeforeWatchRelist(ctx context.Context) error { + if r.watchRelistBackoff != nil { + return r.watchRelistBackoff(ctx) + } + timer := time.NewTimer(kubernetesWatchRelistBackoff) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +type kubernetesPodWatchOutcome struct { + pod *corev1.Pod + latestDiagnostic error + err error + relist bool + done bool +} + +func (r *liveKubernetesResources) watchKubernetesPodTransition( + ctx context.Context, + namespace, name, mainContainer, resourceVersion string, + latestDiagnostic error, +) kubernetesPodWatchOutcome { + watcher, err := r.watchPod(ctx, namespace, name, resourceVersion) + if err != nil { + return kubernetesPodWatchOutcome{ + latestDiagnostic: latestDiagnostic, + err: kubernetesPodWaitError(ctx, err, latestDiagnostic), + } + } + defer watcher.Stop() + for { + select { + case <-ctx.Done(): + return kubernetesPodWatchOutcome{ + latestDiagnostic: latestDiagnostic, + err: kubernetesPodWaitError(ctx, ctx.Err(), latestDiagnostic), + } + case event, ok := <-watcher.ResultChan(): + if !ok { + return kubernetesPodWatchOutcome{latestDiagnostic: latestDiagnostic, relist: true} + } + outcome := evaluateKubernetesPodWatchEvent( + ctx, event, namespace, name, mainContainer, latestDiagnostic, + ) + latestDiagnostic = outcome.latestDiagnostic + if outcome.done || outcome.relist { + return outcome + } + } + } +} + +func evaluateKubernetesPodWatchEvent( + ctx context.Context, + event watch.Event, + namespace, name, mainContainer string, + latestDiagnostic error, +) kubernetesPodWatchOutcome { + if event.Type == watch.Error { + eventErr := apierrors.FromObject(event.Object) + if apierrors.IsResourceExpired(eventErr) || apierrors.IsGone(eventErr) { + return kubernetesPodWatchOutcome{latestDiagnostic: latestDiagnostic, relist: true} + } + return kubernetesPodWatchOutcome{ + latestDiagnostic: latestDiagnostic, + err: kubernetesPodWaitError(ctx, eventErr, latestDiagnostic), + done: true, + } + } + current, ok := event.Object.(*corev1.Pod) + if !ok || current.Name != name || current.Namespace != namespace { + return kubernetesPodWatchOutcome{latestDiagnostic: latestDiagnostic} + } + latestDiagnostic = kubernetesLatestPodDiagnostic(current, latestDiagnostic) + if event.Type == watch.Deleted { + return kubernetesPodWatchOutcome{ + pod: current, latestDiagnostic: latestDiagnostic, + err: kubernetesDeletedPodError(current, mainContainer, latestDiagnostic), done: true, + } + } + ready, terminalErr := kubernetesPodReady(current, mainContainer) + return kubernetesPodWatchOutcome{ + pod: current, latestDiagnostic: latestDiagnostic, err: terminalErr, + done: ready || terminalErr != nil, + } +} + +func (r *liveKubernetesResources) watchPod( + ctx context.Context, + namespace, name, resourceVersion string, +) (watch.Interface, error) { + clientset := r.client.Clientset + if r.watchClient != nil { + clientset = r.watchClient + } + selector := fields.OneTermEqualSelector("metadata.name", name).String() + return clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{ + FieldSelector: selector, ResourceVersion: resourceVersion, + }) +} + +func kubernetesLatestPodDiagnostic(pod *corev1.Pod, previous error) error { + if diagnostic := kubernetesSchedulingError(pod); diagnostic != nil { + return diagnostic + } + return previous +} + +func kubernetesPodWaitError(ctx context.Context, err, diagnostic error) error { + if ctx.Err() != nil { + err = ctx.Err() + } else { + err = routingerr.SanitizeError(err) + } + if diagnostic == nil { + return err + } + return fmt.Errorf("%w: latest Pod diagnostic: %v", err, diagnostic) +} + +func kubernetesPodReady(pod *corev1.Pod, mainContainer string) (bool, error) { + if pod == nil { + return false, errors.New("kubernetes Pod is nil") + } + switch pod.Status.Phase { + case corev1.PodFailed: + return false, routingerr.SanitizeError(fmt.Errorf("pod failed: %s: %s", pod.Status.Reason, pod.Status.Message)) + case corev1.PodSucceeded: + return false, errors.New("pod completed before agentctl started") + } + for _, status := range pod.Status.InitContainerStatuses { + if err := kubernetesContainerImagePullError(status, "init container "+status.Name); err != nil { + return false, err + } + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name != mainContainer { + continue + } + if status.State.Terminated != nil { + return false, routingerr.SanitizeError(fmt.Errorf( + "main container terminated: %s: %s", + status.State.Terminated.Reason, + status.State.Terminated.Message, + )) + } + if err := kubernetesContainerImagePullError(status, "main container"); err != nil { + return false, err + } + if status.State.Running != nil { + return true, nil + } + break + } + return false, nil +} + +func kubernetesContainerImagePullError(status corev1.ContainerStatus, container string) error { + if status.State.Waiting == nil { + return nil + } + reason := status.State.Waiting.Reason + if reason != "ImagePullBackOff" && reason != "ErrImagePull" { + return nil + } + return routingerr.SanitizeError(fmt.Errorf( + "%s image pull failed: %s: %s", container, reason, status.State.Waiting.Message, + )) +} + +func kubernetesSchedulingError(pod *corev1.Pod) error { + for _, condition := range pod.Status.Conditions { + if condition.Type != corev1.PodScheduled || condition.Status != corev1.ConditionFalse { + continue + } + return routingerr.SanitizeError(fmt.Errorf( + "pod scheduling failed: %s: %s", condition.Reason, condition.Message, + )) + } + return nil +} + +func kubernetesDeletedPodError(pod *corev1.Pod, mainContainer string, latestDiagnostic error) error { + if _, diagnostic := kubernetesPodReady(pod, mainContainer); diagnostic != nil { + return fmt.Errorf("kubernetes Pod deleted before main container started: %w", diagnostic) + } + if latestDiagnostic != nil { + return fmt.Errorf("kubernetes Pod deleted before main container started: %w", latestDiagnostic) + } + return errors.New("kubernetes Pod deleted before main container started") +} + +type liveKubernetesExecutor struct { + config *rest.Config + resources *liveKubernetesResources +} + +func (e *liveKubernetesExecutor) Exec(ctx context.Context, request kubeexecutor.ExecRequest) error { + targetURL := e.resources.client.Clientset.CoreV1().RESTClient().Get(). + Resource("pods").Namespace(request.Namespace).Name(request.Pod).SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: request.Container, Command: request.Command, + Stdin: request.Stdin != nil, Stdout: request.Stdout != nil, Stderr: request.Stderr != nil, TTY: request.TTY, + }, scheme.ParameterCodec).URL() + executor, err := newKubernetesRemoteCommandExecutor(e.config, targetURL) + if err != nil { + return err + } + return executor.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: request.Stdin, Stdout: request.Stdout, Stderr: request.Stderr, Tty: request.TTY, + }) +} + +func newKubernetesRemoteCommandExecutor( + config *rest.Config, + targetURL *url.URL, +) (remotecommand.Executor, error) { + websocketExecutor, err := remotecommand.NewWebSocketExecutor(config, http.MethodGet, targetURL.String()) + if err != nil { + return nil, err + } + spdyExecutor, err := remotecommand.NewSPDYExecutor(config, http.MethodPost, targetURL) + if err != nil { + return nil, err + } + return remotecommand.NewFallbackExecutor( + websocketExecutor, spdyExecutor, kubeexecutor.ShouldFallbackStream, + ) +} + +type liveKubernetesForwarder struct { + config *rest.Config + resources *liveKubernetesResources + handshakeTimeout time.Duration +} + +func (f *liveKubernetesForwarder) Forward( + ctx context.Context, + request kubeexecutor.PortForwardRequest, +) (kubeexecutor.PortForwardSession, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + targetURL := f.resources.client.Clientset.CoreV1().RESTClient().Get(). + Resource("pods").Namespace(request.Namespace).Name(request.Pod).SubResource("portforward").URL() + dialer, cancelForward, err := kubeexecutor.NewPortForwardDialer( + ctx, f.config, targetURL, f.effectiveHandshakeTimeout(), + ) + if err != nil { + return nil, err + } + stop := make(chan struct{}) + internalReady := make(chan struct{}) + forwarder, err := portforward.NewOnAddresses( + dialer, []string{request.LocalAddress}, + []string{strconv.Itoa(int(request.LocalPort)) + ":" + strconv.Itoa(int(request.RemotePort))}, + stop, internalReady, io.Discard, io.Discard, + ) + if err != nil { + cancelForward() + return nil, err + } + session := &liveKubernetesForwardSession{ + stop: stop, ready: make(chan struct{}), done: make(chan error, 1), + cancel: cancelForward, finished: make(chan struct{}), + } + go session.run(ctx, forwarder, internalReady) + return session, nil +} + +func (f *liveKubernetesForwarder) effectiveHandshakeTimeout() time.Duration { + if f.handshakeTimeout > 0 { + return f.handshakeTimeout + } + return kubernetesPortForwardHandshakeTimeout(kubeexecutor.DefaultRequestTimeoutSeconds) +} + +func kubernetesPortForwardHandshakeTimeout(requestTimeoutSeconds int) time.Duration { + timeout := time.Duration(requestTimeoutSeconds) * time.Second + maximum := time.Duration(kubeexecutor.DefaultRequestTimeoutSeconds) * time.Second + if timeout <= 0 || timeout > maximum { + return maximum + } + return timeout +} + +type liveKubernetesForwardSession struct { + mu sync.RWMutex + closeOnce sync.Once + localPort uint16 + stop chan struct{} + ready chan struct{} + done chan error + cancel context.CancelFunc + finished chan struct{} +} + +func (s *liveKubernetesForwardSession) run(ctx context.Context, forwarder *portforward.PortForwarder, internalReady <-chan struct{}) { + defer close(s.finished) + forwardDone := make(chan error, 1) + go func() { forwardDone <- forwarder.ForwardPorts() }() + select { + case <-ctx.Done(): + s.signalClose() + <-forwardDone + s.done <- ctx.Err() + case err := <-forwardDone: + s.done <- err + case <-internalReady: + ports, err := forwarder.GetPorts() + if err != nil || len(ports) != 1 || ports[0].Local == 0 { + s.signalClose() + <-forwardDone + if err == nil { + err = errors.New("port-forward did not allocate one local port") + } + s.done <- err + return + } + s.mu.Lock() + s.localPort = ports[0].Local + s.mu.Unlock() + close(s.ready) + s.done <- <-forwardDone + } +} + +func (s *liveKubernetesForwardSession) LocalPort() uint16 { + s.mu.RLock() + defer s.mu.RUnlock() + return s.localPort +} + +func (s *liveKubernetesForwardSession) Ready() <-chan struct{} { return s.ready } +func (s *liveKubernetesForwardSession) Done() <-chan error { return s.done } +func (s *liveKubernetesForwardSession) Close() error { + s.signalClose() + if s.finished != nil { + <-s.finished + } + return nil +} + +func (s *liveKubernetesForwardSession) signalClose() { + s.closeOnce.Do(func() { + if s.cancel != nil { + s.cancel() + } + close(s.stop) + }) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_close_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_close_test.go new file mode 100644 index 0000000000..ddbafd46e5 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_close_test.go @@ -0,0 +1,34 @@ +package lifecycle + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestKubernetesExecutorCloseDropsForwardsWithoutDeletingResources(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }} + executor := newFakeKubernetesExecutorWithForwarder(resources, &recordingKubernetesExec{}, forwards) + _, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + + closer, ok := interface{}(executor).(Closeable) + require.True(t, ok, "Kubernetes executor must close process-local forwards during registry shutdown") + if !ok { + return + } + require.NoError(t, closer.Close()) + + require.True(t, forwards.lastSession().isClosed()) + require.Empty(t, resources.deletedPods) + require.Empty(t, resources.deletedPVCs) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_diagnostics_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_diagnostics_test.go new file mode 100644 index 0000000000..19d7d06055 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_diagnostics_test.go @@ -0,0 +1,283 @@ +package lifecycle + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + kubefake "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + + "github.com/stretchr/testify/require" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestKubernetesPodReadyRejectsImagePullFailureWithSanitizedDiagnostics(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "kandev-agent", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + Message: "pull https://registry.example/private/image?token=kandev_pat_supersecret", + }}, + }}, + }} + + ready, err := kubernetesPodReady(pod, "kandev-agent") + + require.False(t, ready) + require.ErrorContains(t, err, "ImagePullBackOff") + require.NotContains(t, err.Error(), "/private/image") + require.NotContains(t, err.Error(), "supersecret") +} + +func TestKubernetesWaitForPodRunningAllowsTransientUnschedulablePod(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodPending, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", + Message: "see https://cluster.example/schedule?token=kandev_pat_supersecret", + }}, + }} + + pod.Name = "agent" + pod.Namespace = "agents" + pod.ResourceVersion = "1" + clientset := kubefake.NewSimpleClientset(pod) + podWatch := watch.NewRaceFreeFake() + clientset.PrependWatchReactor("pods", func(clienttesting.Action) (bool, watch.Interface, error) { + return true, podWatch, nil + }) + resources := &liveKubernetesResources{client: &kubeexecutor.Client{Clientset: clientset}} + running := pod.DeepCopy() + running.ResourceVersion = "2" + running.Status.Conditions = nil + running.Status.Phase = corev1.PodRunning + running.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + go func() { + podWatch.Modify(pod.DeepCopy()) + podWatch.Modify(running) + }() + + got, err := resources.WaitForPodRunning(context.Background(), "agents", "agent", "kandev-agent") + + require.NoError(t, err) + require.Equal(t, corev1.PodRunning, got.Status.Phase) +} + +func TestKubernetesWaitForPodRunningSurfacesSchedulingDiagnosticAtDeadline(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "agent", Namespace: "agents", ResourceVersion: "1", + }, Status: corev1.PodStatus{ + Phase: corev1.PodPending, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", + Message: "see https://cluster.example/schedule?token=kandev_pat_supersecret", + }}, + }} + clientset := kubefake.NewSimpleClientset(pod) + podWatch := watch.NewRaceFreeFake() + clientset.PrependWatchReactor("pods", func(clienttesting.Action) (bool, watch.Interface, error) { + return true, podWatch, nil + }) + resources := &liveKubernetesResources{client: &kubeexecutor.Client{Clientset: clientset}} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + _, err := resources.WaitForPodRunning(ctx, "agents", "agent", "kandev-agent") + + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "Unschedulable") + require.NotContains(t, err.Error(), "/schedule") + require.NotContains(t, err.Error(), "supersecret") +} + +func TestKubernetesWaitForPodRunningRelistsAfterWatchCloses(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "agent", Namespace: "agents", ResourceVersion: "1", + }, Status: corev1.PodStatus{Phase: corev1.PodPending}} + clientset := kubefake.NewSimpleClientset(pod) + closedWatch := watch.NewRaceFreeFake() + closedWatch.Stop() + runningWatch := watch.NewRaceFreeFake() + watchCalls := 0 + clientset.PrependWatchReactor("pods", func(clienttesting.Action) (bool, watch.Interface, error) { + watchCalls++ + if watchCalls == 1 { + return true, closedWatch, nil + } + return true, runningWatch, nil + }) + running := pod.DeepCopy() + running.ResourceVersion = "2" + running.Status.Phase = corev1.PodRunning + running.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + go runningWatch.Modify(running) + resources := &liveKubernetesResources{client: &kubeexecutor.Client{Clientset: clientset}} + + got, err := resources.WaitForPodRunning(context.Background(), "agents", "agent", "kandev-agent") + + require.NoError(t, err) + require.Equal(t, corev1.PodRunning, got.Status.Phase) + require.Equal(t, 2, watchCalls) +} + +func TestKubernetesWaitForPodRunningBacksOffBeforeRelistingClosedWatch(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "agent", Namespace: "agents", ResourceVersion: "1", + }, Status: corev1.PodStatus{Phase: corev1.PodPending}} + clientset := kubefake.NewSimpleClientset(pod) + watchCalls := 0 + clientset.PrependWatchReactor("pods", func(clienttesting.Action) (bool, watch.Interface, error) { + watchCalls++ + closedWatch := watch.NewRaceFreeFake() + closedWatch.Stop() + return true, closedWatch, nil + }) + backoffEntered := make(chan struct{}) + resources := &liveKubernetesResources{ + client: &kubeexecutor.Client{Clientset: clientset}, + watchRelistBackoff: func(ctx context.Context) error { + close(backoffEntered) + <-ctx.Done() + return ctx.Err() + }, + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := resources.WaitForPodRunning(ctx, "agents", "agent", "kandev-agent") + done <- err + }() + <-backoffEntered + cancel() + + require.ErrorIs(t, <-done, context.Canceled) + require.Equal(t, 1, watchCalls, "closed watches must not trigger a tight relist loop") +} + +func TestKubernetesPodReadyRejectsInitContainerImagePullFailure(t *testing.T) { + pod := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodPending, + InitContainerStatuses: []corev1.ContainerStatus{{ + Name: "setup", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{ + Reason: "ErrImagePull", + Message: "pull https://registry.example/private/init?token=kandev_pat_supersecret", + }}, + }}, + }} + + ready, err := kubernetesPodReady(pod, "kandev-agent") + + require.False(t, ready) + require.ErrorContains(t, err, "init container setup image pull failed") + require.ErrorContains(t, err, "ErrImagePull") + require.NotContains(t, err.Error(), "/private/init") + require.NotContains(t, err.Error(), "supersecret") +} + +func TestKubernetesWaitForPodRunningFailsPromptlyWhenPodIsDeleted(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: "agent", Namespace: "agents", ResourceVersion: "1", + }, Status: corev1.PodStatus{ + Phase: corev1.PodPending, + }} + clientset := kubefake.NewSimpleClientset(pod) + podWatch := watch.NewRaceFreeFake() + clientset.PrependWatchReactor("pods", func(clienttesting.Action) (bool, watch.Interface, error) { + return true, podWatch, nil + }) + resources := &liveKubernetesResources{client: &kubeexecutor.Client{Clientset: clientset}} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go podWatch.Delete(pod.DeepCopy()) + + _, err := resources.WaitForPodRunning(ctx, "agents", "agent", "kandev-agent") + + require.ErrorContains(t, err, "deleted") + require.NotErrorIs(t, err, context.DeadlineExceeded) +} + +func TestKubernetesCreateBoundsPodWaitByConfiguredRequestTimeoutAndKeepsDiagnostic(t *testing.T) { + deadlineSeen := make(chan time.Time, 1) + resources := &fakeKubernetesResources{} + resources.waitForPodRunning = func(ctx context.Context, _, _, _ string) (*corev1.Pod, error) { + if deadline, ok := ctx.Deadline(); ok { + deadlineSeen <- deadline + } + select { + case <-ctx.Done(): + return nil, kubernetesPodWaitError(ctx, ctx.Err(), errors.New( + "pod scheduling failed: Unschedulable: no nodes match kubernetes.io/arch", + )) + case <-time.After(1500 * time.Millisecond): + return nil, errors.New("Pod wait context was unbounded") + } + } + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := validKubernetesCreateRequest() + req.Metadata[MetadataKeyKubernetesRequestTimeoutSeconds] = "1" + started := time.Now() + + _, err := executor.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "Unschedulable") + require.ErrorContains(t, err, "kubernetes.io/arch") + require.Less(t, time.Since(started), 1400*time.Millisecond) + select { + case deadline := <-deadlineSeen: + require.WithinDuration(t, started.Add(time.Second), deadline, 250*time.Millisecond) + default: + t.Fatal("configured Pod-wait deadline was not applied") + } +} + +func TestKubernetesRemoteStatusSanitizesConditionAndClientErrors(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + instance, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + resources.mu.Lock() + resources.pod.Status.Phase = corev1.PodPending + resources.pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{ + Reason: "ErrImagePull", + Message: "pull https://registry.example/private?token=kandev_pat_supersecret", + }}, + }} + resources.mu.Unlock() + + status, err := executor.GetRemoteStatus(context.Background(), instance) + + require.NoError(t, err) + require.Equal(t, "ErrImagePull", status.Details["reason"]) + message := status.Details["message"].(string) + require.NotContains(t, message, "/private") + require.NotContains(t, message, "supersecret") + + resources.mu.Lock() + resources.getPodErr = errors.New("GET https://cluster.example/api/v1/pods?token=kandev_pat_supersecret") + resources.mu.Unlock() + _, err = executor.GetRemoteStatus(context.Background(), instance) + require.Error(t, err) + require.False(t, strings.Contains(err.Error(), "/api/v1/pods"), err.Error()) + require.NotContains(t, err.Error(), "supersecret") +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go new file mode 100644 index 0000000000..58b94ffe03 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go @@ -0,0 +1,482 @@ +package lifecycle + +import ( + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + + "github.com/stretchr/testify/require" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func setManagedKubernetesWorkspace(req *ExecutorCreateRequest) { + req.Metadata[MetadataKeyKubernetesWorkspaceMode] = "managed_pvc" + req.Metadata[MetadataKeyKubernetesWorkspaceSize] = "1Gi" + req.Metadata[MetadataKeyKubernetesWorkspaceAccessModes] = `["ReadWriteOnce"]` +} + +func kubernetesReconnectRequest(created *ExecutorInstance) *ExecutorCreateRequest { + req := validKubernetesCreateRequest() + req.InstanceID = "new-execution-id" + req.PreviousExecutionID = created.InstanceID + req.AuthToken = created.AuthToken + req.BootstrapNonce = created.BootstrapNonce + for key, value := range created.Metadata { + req.Metadata[key] = value + } + return req +} + +func newFakeKubernetesExecutor( + t *testing.T, + resources *fakeKubernetesResources, + execs *recordingKubernetesExec, + ports map[uint16]uint16, +) *KubernetesExecutor { + t.Helper() + return newFakeKubernetesExecutorWithForwarder( + resources, execs, &recordingKubernetesForwarder{localPorts: ports}, + ) +} + +func newFakeKubernetesExecutorWithForwarder( + resources *fakeKubernetesResources, + execs *recordingKubernetesExec, + forwards *recordingKubernetesForwarder, +) *KubernetesExecutor { + executor := NewKubernetesExecutor(nil, newTestLogger()) + executor.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{ + resources: resources, + streams: kubeexecutor.NewStreamOperations(execs, forwards), + }, nil + } + executor.resolveBinary = func(kubeexecutor.Platform) ([]byte, error) { return []byte("agentctl"), nil } + return executor +} + +func requireKubernetesUploadedFile(t *testing.T, execs *recordingKubernetesExec, path, content string) { + t.Helper() + execs.mu.Lock() + defer execs.mu.Unlock() + for _, recorded := range execs.requests { + if strings.Contains(strings.Join(recorded.request.Command, " "), path) && string(recorded.stdin) == content { + return + } + } + t.Fatalf("remote file %s with content %q was not materialized; execs = %#v", path, content, execs.requests) +} + +func requireKubernetesPathNotUsed(t *testing.T, execs *recordingKubernetesExec, forbidden string) { + t.Helper() + execs.mu.Lock() + defer execs.mu.Unlock() + for _, recorded := range execs.requests { + if strings.Contains(strings.Join(recorded.request.Command, " "), forbidden) { + t.Fatalf("untrusted remote auth path %q reached Kubernetes exec: %#v", forbidden, recorded.request) + } + } +} + +func validKubernetesCreateRequest() *ExecutorCreateRequest { + return &ExecutorCreateRequest{ + InstanceID: "instance-1", + TaskID: "task-1", + SessionID: "session-1", + TaskEnvironmentID: "environment-1", + Metadata: map[string]interface{}{ + "executor_id": "executor-1", + "executor_profile_id": "profile-1", + "auth_mode": "in_cluster", + "namespace": "kandev-agents", + "request_timeout_seconds": "30", + "platform": "linux/amd64", + "main_container": "kandev-agent", + "pod_template_yaml": "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example.test/agent:latest\n", + "workspace.mode": "empty_dir", + }, + } +} + +type fakeKubernetesResources struct { + mu sync.Mutex + createdPVCs []*corev1.PersistentVolumeClaim + createdPods []*corev1.Pod + deletedPVCs []string + deletedPods []string + deletedPVCResourceVersions []string + deletedPodResourceVersions []string + deletePodContextDeadline time.Time + deletionOrder []string + nextPodUID types.UID + nextPVCUID types.UID + pvc *corev1.PersistentVolumeClaim + pod *corev1.Pod + getPodErr error + getPodRequests []string + getPVCErr error + getPVCRequests []string + createPodErr error + createPVCErr error + createPodBeforeCommitErr error + createPVCBeforeCommitErr error + deletePodErr error + deletePVCErr error + beforeDeletePod func(*corev1.Pod) + beforeDeletePVC func(*corev1.PersistentVolumeClaim) + retainPodAfterDelete bool + retainPVCAfterDelete bool + podDeleteIssued bool + pvcDeleteIssued bool + podPostDeleteGetsUntilAbsent int + pvcPostDeleteGetsUntilAbsent int + mutateCreatedPod func(*corev1.Pod) + mutateCreatedPVC func(*corev1.PersistentVolumeClaim) + waitForPodRunning func(context.Context, string, string, string) (*corev1.Pod, error) + rejectCanceledGetContexts bool +} + +func (f *fakeKubernetesResources) CreatePersistentVolumeClaim( + _ context.Context, + namespace string, + pvc *corev1.PersistentVolumeClaim, +) (*corev1.PersistentVolumeClaim, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.createPVCBeforeCommitErr != nil { + return nil, f.createPVCBeforeCommitErr + } + created := pvc.DeepCopy() + created.Namespace = namespace + created.UID = f.nextPVCUID + if created.UID == "" { + created.UID = types.UID("pvc-uid") + } + if created.ResourceVersion == "" { + created.ResourceVersion = "pvc-rv-1" + } + if f.mutateCreatedPVC != nil { + f.mutateCreatedPVC(created) + } + created.Status.Phase = corev1.ClaimBound + f.pvc = created.DeepCopy() + f.createdPVCs = append(f.createdPVCs, created.DeepCopy()) + return created, f.createPVCErr +} + +func (f *fakeKubernetesResources) GetPersistentVolumeClaim( + ctx context.Context, + namespace, name string, +) (*corev1.PersistentVolumeClaim, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.getPVCRequests = append(f.getPVCRequests, namespace+"/"+name) + if f.rejectCanceledGetContexts && ctx.Err() != nil { + return nil, ctx.Err() + } + if f.pvcDeleteIssued && f.retainPVCAfterDelete && f.pvcPostDeleteGetsUntilAbsent >= 0 { + if f.pvcPostDeleteGetsUntilAbsent == 0 { + f.pvc = nil + } else { + f.pvcPostDeleteGetsUntilAbsent-- + } + } + if f.getPVCErr != nil { + return nil, f.getPVCErr + } + if f.pvc == nil { + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "persistentvolumeclaims"}, "missing") + } + return f.pvc.DeepCopy(), nil +} + +func (f *fakeKubernetesResources) DeletePersistentVolumeClaim( + _ context.Context, + _, name string, + uid types.UID, + resourceVersion string, +) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.beforeDeletePVC != nil { + f.beforeDeletePVC(f.pvc) + } + if f.deletePVCErr != nil { + return f.deletePVCErr + } + if f.pvc != nil && (f.pvc.UID != uid || (resourceVersion != "" && f.pvc.ResourceVersion != resourceVersion)) { + return apierrors.NewConflict( + schema.GroupResource{Resource: "persistentvolumeclaims"}, name, errors.New("precondition failed"), + ) + } + f.deletedPVCs = append(f.deletedPVCs, name+":"+string(uid)) + f.deletedPVCResourceVersions = append(f.deletedPVCResourceVersions, resourceVersion) + f.deletionOrder = append(f.deletionOrder, "pvc") + f.pvcDeleteIssued = true + if !f.retainPVCAfterDelete { + f.pvc = nil + } + return nil +} + +func (f *fakeKubernetesResources) CreatePod( + _ context.Context, + namespace string, + pod *corev1.Pod, +) (*corev1.Pod, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.createPodBeforeCommitErr != nil { + return nil, f.createPodBeforeCommitErr + } + created := pod.DeepCopy() + created.Namespace = namespace + created.UID = f.nextPodUID + if created.UID == "" { + created.UID = types.UID("pod-uid") + } + if created.ResourceVersion == "" { + created.ResourceVersion = "pod-rv-1" + } + f.nextPodUID = "" + if f.mutateCreatedPod != nil { + f.mutateCreatedPod(created) + } + f.pod = runningKubernetesPod(created) + f.createdPods = append(f.createdPods, created.DeepCopy()) + return created, f.createPodErr +} + +func (f *fakeKubernetesResources) GetPod(ctx context.Context, namespace, name string) (*corev1.Pod, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.getPodRequests = append(f.getPodRequests, namespace+"/"+name) + if f.rejectCanceledGetContexts && ctx.Err() != nil { + return nil, ctx.Err() + } + if f.podDeleteIssued && f.retainPodAfterDelete && f.podPostDeleteGetsUntilAbsent >= 0 { + if f.podPostDeleteGetsUntilAbsent == 0 { + f.pod = nil + } else { + f.podPostDeleteGetsUntilAbsent-- + } + } + if f.getPodErr != nil { + return nil, f.getPodErr + } + if f.pod == nil { + return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, "missing") + } + return f.pod.DeepCopy(), nil +} + +func (f *fakeKubernetesResources) WaitForPodRunning( + ctx context.Context, + namespace string, + name string, + mainContainer string, +) (*corev1.Pod, error) { + f.mu.Lock() + hook := f.waitForPodRunning + pod := f.pod.DeepCopy() + f.mu.Unlock() + if hook != nil { + return hook(ctx, namespace, name, mainContainer) + } + return pod, nil +} + +func (f *fakeKubernetesResources) DeletePod( + ctx context.Context, + _, name string, + uid types.UID, + resourceVersion string, +) error { + f.mu.Lock() + defer f.mu.Unlock() + if deadline, ok := ctx.Deadline(); ok { + f.deletePodContextDeadline = deadline + } + if f.beforeDeletePod != nil { + f.beforeDeletePod(f.pod) + } + if f.deletePodErr != nil { + return f.deletePodErr + } + if f.pod != nil && (f.pod.UID != uid || (resourceVersion != "" && f.pod.ResourceVersion != resourceVersion)) { + return apierrors.NewConflict(schema.GroupResource{Resource: "pods"}, name, errors.New("precondition failed")) + } + f.deletedPods = append(f.deletedPods, name+":"+string(uid)) + f.deletedPodResourceVersions = append(f.deletedPodResourceVersions, resourceVersion) + f.deletionOrder = append(f.deletionOrder, "pod") + f.podDeleteIssued = true + if !f.retainPodAfterDelete { + f.pod = nil + } + return nil +} + +func runningKubernetesPod(pod *corev1.Pod) *corev1.Pod { + running := pod.DeepCopy() + running.Status.Phase = corev1.PodRunning + running.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", + Ready: true, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }} + return running +} + +type recordedKubernetesExec struct { + request kubeexecutor.ExecRequest + stdin []byte +} + +type recordingKubernetesExec struct { + mu sync.Mutex + requests []recordedKubernetesExec + err error +} + +func (r *recordingKubernetesExec) Exec(_ context.Context, request kubeexecutor.ExecRequest) error { + var data []byte + if request.Stdin != nil { + var err error + data, err = io.ReadAll(request.Stdin) + if err != nil { + return err + } + } + r.mu.Lock() + defer r.mu.Unlock() + r.requests = append(r.requests, recordedKubernetesExec{request: request, stdin: data}) + return r.err +} + +type recordingKubernetesForwarder struct { + mu sync.Mutex + localPorts map[uint16]uint16 + requests []kubeexecutor.PortForwardRequest + sessions []*fakeKubernetesForwardSession +} + +func (f *recordingKubernetesForwarder) Forward( + _ context.Context, + request kubeexecutor.PortForwardRequest, +) (kubeexecutor.PortForwardSession, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, request) + session := newFakeKubernetesForwardSession(f.localPorts[request.RemotePort]) + f.sessions = append(f.sessions, session) + return session, nil +} + +func (f *recordingKubernetesForwarder) lastSession() *fakeKubernetesForwardSession { + f.mu.Lock() + defer f.mu.Unlock() + return f.sessions[len(f.sessions)-1] +} + +func (f *recordingKubernetesForwarder) remotePorts() []uint16 { + f.mu.Lock() + defer f.mu.Unlock() + ports := make([]uint16, 0, len(f.requests)) + for _, request := range f.requests { + ports = append(ports, request.RemotePort) + } + return ports +} + +type fakeKubernetesForwardSession struct { + mu sync.Mutex + localPort uint16 + ready chan struct{} + done chan error + closed bool + closeErr error +} + +func newFakeKubernetesForwardSession(localPort uint16) *fakeKubernetesForwardSession { + ready := make(chan struct{}) + close(ready) + return &fakeKubernetesForwardSession{localPort: localPort, ready: ready, done: make(chan error, 1)} +} + +func (s *fakeKubernetesForwardSession) LocalPort() uint16 { return s.localPort } +func (s *fakeKubernetesForwardSession) Ready() <-chan struct{} { return s.ready } +func (s *fakeKubernetesForwardSession) Done() <-chan error { return s.done } +func (s *fakeKubernetesForwardSession) Close() error { + s.mu.Lock() + s.closed = true + err := s.closeErr + s.mu.Unlock() + return err +} +func (s *fakeKubernetesForwardSession) isClosed() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.closed +} + +func startKubernetesAgentctlServer(t *testing.T, control bool, instanceRemotePort int) uint16 { + return startKubernetesAgentctlServerWithToken(t, control, instanceRemotePort, "handshake-token") +} + +func startKubernetesAgentctlServerWithToken( + t *testing.T, + control bool, + instanceRemotePort int, + token string, +) uint16 { + t.Helper() + mux := http.NewServeMux() + mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + if control { + mux.HandleFunc("/auth/handshake", func(w http.ResponseWriter, request *http.Request) { + var body map[string]string + require.NoError(t, json.NewDecoder(request.Body).Decode(&body)) + require.NotEmpty(t, body["nonce"]) + _ = json.NewEncoder(w).Encode(map[string]string{"token": token}) + }) + mux.HandleFunc("/api/v1/instances", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]interface{}{"id": "instance-1", "port": instanceRemotePort}) + }) + } + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + _, rawPort, err := net.SplitHostPort(server.Listener.Addr().String()) + require.NoError(t, err) + port, err := strconv.Atoi(rawPort) + require.NoError(t, err) + return uint16(port) +} + +func startKubernetesUnhealthyServer(t *testing.T) uint16 { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + _, rawPort, err := net.SplitHostPort(server.Listener.Addr().String()) + require.NoError(t, err) + port, err := strconv.Atoi(rawPort) + require.NoError(t, err) + return uint16(port) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_files.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_files.go new file mode 100644 index 0000000000..7f61ffcdc1 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_files.go @@ -0,0 +1,285 @@ +package lifecycle + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path" + "regexp" + "runtime" + "strings" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + + "github.com/kandev/kandev/internal/agent/agents" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/remoteauth" +) + +var kubernetesSafePathComponent = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +type kubernetesPodFileUploader struct { + streams *kubeexecutor.StreamOperations + pod *corev1.Pod + container string +} + +func (u kubernetesPodFileUploader) WriteFile( + ctx context.Context, + destination string, + data []byte, + mode os.FileMode, +) error { + destination = strings.ReplaceAll(destination, "\\", "/") + if !kubernetesSafeAbsolutePath(destination) { + return fmt.Errorf("unsafe Kubernetes upload destination %q", destination) + } + return kubernetesWriteFile(ctx, u.streams, u.pod, u.container, destination, data, mode) +} + +func (r *KubernetesExecutor) materializeKubernetesRemoteFiles( + ctx context.Context, + runtimeClient *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + container string, +) error { + uploader := kubernetesPodFileUploader{streams: runtimeClient.streams, pod: pod, container: container} + if err := r.materializeKubernetesCredentials(ctx, uploader, runtimeClient, req, pod, container); err != nil { + return fmt.Errorf("kubernetes lifecycle: materialize agent configuration: %w", err) + } + if err := materializeKubernetesSkillManifest(ctx, uploader, req.Metadata); err != nil { + return fmt.Errorf("kubernetes lifecycle: materialize skill manifest: %w", err) + } + return nil +} + +func (r *KubernetesExecutor) materializeKubernetesCredentials( + ctx context.Context, + uploader kubernetesPodFileUploader, + runtimeClient *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + container string, +) error { + if req.AgentConfig == nil { + return nil + } + home := kubernetesAuthHomePath + catalog := remoteauth.BuildCatalogForHost([]agents.Agent{req.AgentConfig}, runtime.GOOS, mustUserHome()) + selectedMethods, err := kubernetesSelectedCredentialMethods(req.Metadata, catalog) + if err != nil { + return err + } + if err := UploadCredentialFiles(ctx, uploader, selectedMethods, home, r.logger); err != nil { + return err + } + warnings := UploadPortableConfigBundles( + ctx, uploader, req.AgentConfig, selectedPortableConfigBundleIDs(req.Metadata), home, r.logger, + ) + reportPortableConfigWarnings(req.OnProgress, warnings) + for _, spec := range catalog.Specs { + for _, method := range spec.Methods { + if method.Type != authMethodTypeEnv || method.EnvVar == "" || method.SetupScript == "" || req.Env[method.EnvVar] == "" { + continue + } + command := "set -eu; export HOME=" + shellQuote(home) + + "; set -a; . " + shellQuote(kubernetesAuthEnvPath) + "; set +a; " + method.SetupScript + if execErr := runtimeClient.streams.Exec(ctx, kubeexecutor.ExecRequest{ + Namespace: pod.Namespace, Pod: pod.Name, Container: container, + Command: []string{"sh", "-c", command}, + }); execErr != nil { + r.logger.Warn("Kubernetes agent auth setup script failed", + zap.String("method_id", method.MethodID), zap.Error(execErr)) + } + } + } + return nil +} + +func kubernetesSelectedCredentialMethods( + metadata map[string]interface{}, + catalog remoteauth.Catalog, +) ([]remoteauth.Method, error) { + raw := getMetadataString(metadata, "remote_credentials") + if strings.TrimSpace(raw) == "" { + return nil, nil + } + var ids []string + if err := json.Unmarshal([]byte(raw), &ids); err != nil { + return nil, fmt.Errorf("parse remote_credentials: %w", err) + } + methods := make([]remoteauth.Method, 0, len(ids)) + for _, id := range ids { + method, ok := catalog.FindMethod(id) + if !ok || method.Type != authMethodTypeFiles { + continue + } + methods = append(methods, method) + } + return methods, nil +} + +func mustUserHome() string { + home, _ := os.UserHomeDir() + return home +} + +type kubernetesManifestSupportFile struct { + Path string + Content string +} + +type kubernetesManifestSkill struct { + Slug string + Content string + Files []kubernetesManifestSupportFile +} + +type kubernetesManifestInstruction struct { + Filename string + Content string + IsEntry bool +} + +type kubernetesSkillManifest struct { + Skills []kubernetesManifestSkill + Instructions []kubernetesManifestInstruction + AgentTypeID string + WorkspaceSlug string + AgentID string + ProjectSkillDir string +} + +func materializeKubernetesSkillManifest( + ctx context.Context, + uploader kubernetesPodFileUploader, + metadata map[string]interface{}, +) error { + manifest, err := decodeKubernetesSkillManifest(metadata) + if err != nil || manifest == nil { + return err + } + if err := materializeKubernetesSkills(ctx, uploader, manifest); err != nil { + return err + } + instructionsRoot, err := kubernetesManifestInstructionsRoot(manifest) + if err != nil || instructionsRoot == "" { + return err + } + return materializeKubernetesInstructions(ctx, uploader, instructionsRoot, manifest.Instructions) +} + +func decodeKubernetesSkillManifest(metadata map[string]interface{}) (*kubernetesSkillManifest, error) { + raw := getMetadataString(metadata, MetadataKeySkillManifestJSON) + if strings.TrimSpace(raw) == "" { + return nil, nil + } + var manifest kubernetesSkillManifest + if err := json.Unmarshal([]byte(raw), &manifest); err != nil { + return nil, fmt.Errorf("decode manifest: %w", err) + } + return &manifest, nil +} + +func materializeKubernetesSkills( + ctx context.Context, + uploader kubernetesPodFileUploader, + manifest *kubernetesSkillManifest, +) error { + projectDir, ok := kubernetesSafeRelativePath(manifest.ProjectSkillDir) + if !ok { + if manifest.ProjectSkillDir != "" { + return fmt.Errorf("manifest project skill directory is unsafe") + } + projectDir = ".agents/skills" + } + for _, item := range manifest.Skills { + if !kubernetesSafeComponent(item.Slug) { + return fmt.Errorf("manifest skill slug contains an unsafe path component") + } + root := path.Join(kubernetesWorkspacePath, projectDir, "kandev-"+item.Slug) + if !kubernetesPathWithinRoot(root, kubernetesWorkspacePath) { + return fmt.Errorf("manifest skill root escapes the workspace") + } + if err := uploader.WriteFile(ctx, path.Join(root, "SKILL.md"), []byte(item.Content), 0o644); err != nil { + return err + } + for _, support := range item.Files { + relative, safe := kubernetesSafeRelativePath(support.Path) + if !safe || relative == "SKILL.md" { + return fmt.Errorf("manifest skill support path is unsafe") + } + destination := path.Join(root, relative) + if !kubernetesPathWithinRoot(destination, root) { + return fmt.Errorf("manifest skill support path escapes its skill root") + } + if err := uploader.WriteFile(ctx, destination, []byte(support.Content), 0o644); err != nil { + return err + } + } + } + return nil +} + +func kubernetesManifestInstructionsRoot(manifest *kubernetesSkillManifest) (string, error) { + if !kubernetesSafeComponent(manifest.WorkspaceSlug) || !kubernetesSafeComponent(manifest.AgentID) { + if len(manifest.Instructions) == 0 { + return "", nil + } + return "", fmt.Errorf("manifest instruction identity contains an unsafe path component") + } + root := path.Join("/opt/kandev/runtime", manifest.WorkspaceSlug, "instructions", manifest.AgentID) + if !kubernetesPathWithinRoot(root, "/opt/kandev/runtime") { + return "", fmt.Errorf("manifest instruction root escapes the runtime directory") + } + return root, nil +} + +func materializeKubernetesInstructions( + ctx context.Context, + uploader kubernetesPodFileUploader, + root string, + instructions []kubernetesManifestInstruction, +) error { + for _, instruction := range instructions { + if !kubernetesSafeComponent(instruction.Filename) { + return fmt.Errorf("manifest instruction filename contains an unsafe path component") + } + destination := path.Join(root, instruction.Filename) + if !kubernetesPathWithinRoot(destination, root) { + return fmt.Errorf("manifest instruction path escapes its instruction root") + } + if err := uploader.WriteFile(ctx, destination, []byte(instruction.Content), 0o644); err != nil { + return err + } + } + return nil +} + +func kubernetesSafeComponent(value string) bool { + return value != "." && value != ".." && kubernetesSafePathComponent.MatchString(value) +} + +func kubernetesPathWithinRoot(candidate, root string) bool { + candidate = path.Clean(candidate) + root = path.Clean(root) + return candidate == root || strings.HasPrefix(candidate, root+"/") +} + +func kubernetesSafeRelativePath(value string) (string, bool) { + if value == "" || strings.Contains(value, "\\") || strings.ContainsRune(value, '\x00') || path.IsAbs(value) { + return "", false + } + cleaned := path.Clean(value) + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", false + } + return cleaned, true +} + +func kubernetesSafeAbsolutePath(value string) bool { + return path.IsAbs(value) && !strings.ContainsRune(value, '\x00') && path.Clean(value) == value +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_identity.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_identity.go new file mode 100644 index 0000000000..e81fb4c141 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_identity.go @@ -0,0 +1,328 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +const kubernetesDeletionPollInterval = 100 * time.Millisecond + +func verifyCreatedPod(created, desired *corev1.Pod, identity kubeexecutor.ResourceIdentity, mainContainer string) error { + if created == nil || desired == nil || created.UID == "" { + return errors.New("kubernetes lifecycle: created Pod identity is incomplete") + } + if err := verifyRecordedPod(created, desired.Namespace, desired.Name, created.UID, identity); err != nil { + return err + } + if err := kubeexecutor.ValidateAdmittedPod(created, desired, mainContainer); err != nil { + return fmt.Errorf("kubernetes lifecycle: %w", err) + } + return nil +} + +func verifyRecordedPod( + pod *corev1.Pod, + namespace, name string, + uid types.UID, + identity kubeexecutor.ResourceIdentity, +) error { + if pod == nil || pod.Namespace != namespace || pod.Name != name || pod.UID != uid || uid == "" { + return errors.New("kubernetes lifecycle: Pod UID or name does not match recorded inventory") + } + want, err := kubeexecutor.OwnershipLabels(identity) + if err != nil { + return err + } + if err := kubeexecutor.ValidateOwnershipLabels(pod.Labels, want); err != nil { + return fmt.Errorf("kubernetes lifecycle: Pod ownership labels: %w", err) + } + return nil +} + +func verifyOwnedPVC(created, desired *corev1.PersistentVolumeClaim, identity kubeexecutor.ResourceIdentity) error { + if created == nil || desired == nil || created.UID == "" || + created.Namespace != desired.Namespace || created.Name != desired.Name { + return errors.New("kubernetes lifecycle: created PVC identity is incomplete") + } + want, err := kubeexecutor.OwnershipLabels(identity) + if err != nil { + return err + } + if err := kubeexecutor.ValidateOwnershipLabels(created.Labels, want); err != nil { + return fmt.Errorf("kubernetes lifecycle: PVC ownership labels: %w", err) + } + if err := kubeexecutor.ValidateAdmittedPVC(created, desired); err != nil { + return fmt.Errorf("kubernetes lifecycle: %w", err) + } + return nil +} + +func deleteKubernetesPodIfExact( + ctx context.Context, + resources kubernetesResourceClient, + recorded *corev1.Pod, + identity kubeexecutor.ResourceIdentity, +) error { + current, err := resources.GetPod(ctx, recorded.Namespace, recorded.Name) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("kubernetes lifecycle: inspect Pod before deletion: %w", err) + } + if err := verifyRecordedPod(current, recorded.Namespace, recorded.Name, recorded.UID, identity); err != nil { + return err + } + if err := resources.DeletePod( + ctx, recorded.Namespace, recorded.Name, recorded.UID, current.ResourceVersion, + ); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("kubernetes lifecycle: delete Pod: %w", err) + } + return waitForKubernetesPodDeletion(ctx, resources, recorded.Namespace, recorded.Name, recorded.UID) +} + +func deleteKubernetesPVCIfExact( + ctx context.Context, + resources kubernetesResourceClient, + recorded *corev1.PersistentVolumeClaim, + identity kubeexecutor.ResourceIdentity, +) error { + current, err := resources.GetPersistentVolumeClaim(ctx, recorded.Namespace, recorded.Name) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("kubernetes lifecycle: inspect PVC before deletion: %w", err) + } + if current == nil || current.Namespace != recorded.Namespace || current.Name != recorded.Name || current.UID != recorded.UID { + return errors.New("kubernetes lifecycle: PVC UID does not match recorded inventory") + } + labels, err := kubeexecutor.OwnershipLabels(identity) + if err != nil { + return err + } + if err := kubeexecutor.ValidateOwnershipLabels(current.Labels, labels); err != nil { + return fmt.Errorf("kubernetes lifecycle: PVC ownership labels: %w", err) + } + if err := resources.DeletePersistentVolumeClaim( + ctx, recorded.Namespace, recorded.Name, recorded.UID, current.ResourceVersion, + ); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("kubernetes lifecycle: delete PVC: %w", err) + } + return waitForKubernetesPVCDeletion(ctx, resources, recorded.Namespace, recorded.Name, recorded.UID) +} + +// rollbackCreatedKubernetesPod deletes only the UID returned by this launch's +// Create call. Unlike terminal cleanup, it deliberately does not require +// ownership labels: admission may have mutated those labels, and that is one +// of the failures rollback must remove. +func rollbackCreatedKubernetesPod( + ctx context.Context, + resources kubernetesResourceClient, + created *corev1.Pod, +) error { + return rollbackCreatedKubernetesResource(ctx, kubernetesPodDeletionTarget(resources, created)) +} + +func rollbackCreatedKubernetesPVC( + ctx context.Context, + resources kubernetesResourceClient, + created *corev1.PersistentVolumeClaim, +) error { + return rollbackCreatedKubernetesResource(ctx, kubernetesPVCDeletionTarget(resources, created)) +} + +type kubernetesCurrentResource struct { + namespace string + name string + uid types.UID + resourceVersion string +} + +type kubernetesDeletionTarget struct { + kind string + namespace string + name string + uid types.UID + get func(context.Context) (kubernetesCurrentResource, error) + delete func(context.Context, types.UID, string) error + wait func(context.Context, types.UID) error +} + +func kubernetesPodDeletionTarget( + resources kubernetesResourceClient, + pod *corev1.Pod, +) kubernetesDeletionTarget { + if pod == nil { + return kubernetesDeletionTarget{kind: "Pod"} + } + return newKubernetesDeletionTarget("Pod", pod, + func(ctx context.Context) (metav1.Object, error) { + return resources.GetPod(ctx, pod.Namespace, pod.Name) + }, + func(ctx context.Context, uid types.UID, rv string) error { + return resources.DeletePod(ctx, pod.Namespace, pod.Name, uid, rv) + }, + func(ctx context.Context, uid types.UID) error { + return waitForKubernetesPodDeletion(ctx, resources, pod.Namespace, pod.Name, uid) + }, + ) +} + +func kubernetesPVCDeletionTarget( + resources kubernetesResourceClient, + pvc *corev1.PersistentVolumeClaim, +) kubernetesDeletionTarget { + if pvc == nil { + return kubernetesDeletionTarget{kind: "PVC"} + } + return newKubernetesDeletionTarget("PVC", pvc, + func(ctx context.Context) (metav1.Object, error) { + return resources.GetPersistentVolumeClaim(ctx, pvc.Namespace, pvc.Name) + }, + func(ctx context.Context, uid types.UID, rv string) error { + return resources.DeletePersistentVolumeClaim(ctx, pvc.Namespace, pvc.Name, uid, rv) + }, + func(ctx context.Context, uid types.UID) error { + return waitForKubernetesPVCDeletion(ctx, resources, pvc.Namespace, pvc.Name, uid) + }, + ) +} + +func newKubernetesDeletionTarget( + kind string, + recorded metav1.Object, + get func(context.Context) (metav1.Object, error), + deleteResource func(context.Context, types.UID, string) error, + wait func(context.Context, types.UID) error, +) kubernetesDeletionTarget { + return kubernetesDeletionTarget{ + kind: kind, namespace: recorded.GetNamespace(), name: recorded.GetName(), uid: recorded.GetUID(), + get: func(ctx context.Context) (kubernetesCurrentResource, error) { + current, err := get(ctx) + if current == nil { + return kubernetesCurrentResource{}, err + } + return kubernetesCurrentResource{ + namespace: current.GetNamespace(), name: current.GetName(), uid: current.GetUID(), + resourceVersion: current.GetResourceVersion(), + }, err + }, + delete: deleteResource, + wait: wait, + } +} + +func rollbackCreatedKubernetesResource(ctx context.Context, target kubernetesDeletionTarget) error { + if target.namespace == "" || target.name == "" || target.uid == "" || target.get == nil { + return fmt.Errorf( + "kubernetes lifecycle: cannot roll back %s with incomplete created identity", target.kind, + ) + } + current, err := target.get(ctx) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return fmt.Errorf("kubernetes lifecycle: inspect created %s for rollback: %w", target.kind, err) + } + if current.namespace != target.namespace || current.name != target.name { + return fmt.Errorf( + "kubernetes lifecycle: cannot roll back %s with inconsistent current identity", target.kind, + ) + } + if current.uid != target.uid { + return nil + } + if err := target.delete(ctx, target.uid, current.resourceVersion); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("kubernetes lifecycle: roll back created %s: %w", target.kind, err) + } + return target.wait(ctx, target.uid) +} + +func waitForKubernetesPodDeletion( + ctx context.Context, + resources kubernetesResourceClient, + namespace, name string, + uid types.UID, +) error { + return waitForKubernetesResourceDeletion(ctx, "Pod", uid, func(ctx context.Context) (types.UID, error) { + pod, err := resources.GetPod(ctx, namespace, name) + if err != nil { + return "", err + } + if pod == nil { + return "", errors.New("kubernetes API returned an empty Pod") + } + return pod.UID, nil + }) +} + +func waitForKubernetesPVCDeletion( + ctx context.Context, + resources kubernetesResourceClient, + namespace, name string, + uid types.UID, +) error { + return waitForKubernetesResourceDeletion(ctx, "PVC", uid, func(ctx context.Context) (types.UID, error) { + pvc, err := resources.GetPersistentVolumeClaim(ctx, namespace, name) + if err != nil { + return "", err + } + if pvc == nil { + return "", errors.New("kubernetes API returned an empty PVC") + } + return pvc.UID, nil + }) +} + +func waitForKubernetesResourceDeletion( + ctx context.Context, + kind string, + recordedUID types.UID, + getUID func(context.Context) (types.UID, error), +) error { + ticker := time.NewTicker(kubernetesDeletionPollInterval) + defer ticker.Stop() + for { + currentUID, err := getUID(ctx) + if apierrors.IsNotFound(err) || (err == nil && currentUID != recordedUID) { + return nil + } + if err != nil { + return fmt.Errorf("kubernetes lifecycle: wait for %s deletion: %w", kind, err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("kubernetes lifecycle: wait for %s deletion: %w", kind, ctx.Err()) + case <-ticker.C: + } + } +} + +func findKubernetesContainer(pod *corev1.Pod, name string) *corev1.Container { + for index := range pod.Spec.Containers { + if pod.Spec.Containers[index].Name == name { + return &pod.Spec.Containers[index] + } + } + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_preparer.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_preparer.go new file mode 100644 index 0000000000..3db7405bfc --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_preparer.go @@ -0,0 +1,41 @@ +package lifecycle + +import ( + "context" + "time" + + "go.uber.org/zap" + + "github.com/kandev/kandev/internal/common/logger" +) + +// KubernetesPreparer emits launch progress; cluster/resource validation stays +// in KubernetesExecutor so it uses the same client and identity checks as launch. +type KubernetesPreparer struct { + logger *logger.Logger +} + +func NewKubernetesPreparer(log *logger.Logger) *KubernetesPreparer { + return &KubernetesPreparer{logger: log.WithFields(zap.String("component", "kubernetes-preparer"))} +} + +func (p *KubernetesPreparer) Name() string { return "k8s" } + +func (p *KubernetesPreparer) Prepare( + _ context.Context, + req *EnvPrepareRequest, + onProgress PrepareProgressCallback, +) (*EnvPrepareResult, error) { + started := time.Now() + step := beginStep("Validate Kubernetes executor configuration") + reportProgress(onProgress, step, 0, 1) + completeStepSuccess(&step) + reportProgress(onProgress, step, 1, 1) + result := &EnvPrepareResult{Success: true, Steps: []PrepareStep{step}, Duration: time.Since(started)} + if req != nil { + result.WorkspacePath = req.WorkspacePath + result.WorktreeBranch = nonWorktreeTaskBranch(req) + } + p.logger.Debug("prepared kubernetes environment") + return result, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_provision.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_provision.go new file mode 100644 index 0000000000..b733069c15 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_provision.go @@ -0,0 +1,190 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +type kubernetesWorkspaceProvision struct { + claimName string + claim *corev1.PersistentVolumeClaim + createdClaim *corev1.PersistentVolumeClaim +} + +func provisionKubernetesWorkspace( + ctx context.Context, + resources kubernetesResourceClient, + executorConfig kubeexecutor.ExecutorConfig, + profile kubeexecutor.ProfileConfig, + identity kubeexecutor.ResourceIdentity, + managedClaimName string, + onManagedCreated func(*corev1.PersistentVolumeClaim) error, +) (kubernetesWorkspaceProvision, error) { + switch profile.Workspace.Mode { + case kubeexecutor.WorkspaceModeManagedPVC: + return createManagedKubernetesWorkspacePVC( + ctx, resources, executorConfig.Namespace, profile.Workspace, identity, managedClaimName, + onManagedCreated, + ) + case kubeexecutor.WorkspaceModeExistingClaim: + return getExistingKubernetesWorkspacePVC( + ctx, resources, executorConfig.Namespace, profile.Workspace.ClaimName, + ) + case kubeexecutor.WorkspaceModeEmptyDir: + return kubernetesWorkspaceProvision{}, nil + default: + return kubernetesWorkspaceProvision{}, errors.New("kubernetes lifecycle: unsupported workspace mode") + } +} + +func createManagedKubernetesWorkspacePVC( + ctx context.Context, + resources kubernetesResourceClient, + namespace string, + workspace kubeexecutor.WorkspaceConfig, + identity kubeexecutor.ResourceIdentity, + claimName string, + onCreated func(*corev1.PersistentVolumeClaim) error, +) (kubernetesWorkspaceProvision, error) { + desired, err := kubeexecutor.BuildPersistentVolumeClaim(workspace, kubeexecutor.PVCOptions{ + Name: claimName, Namespace: namespace, Identity: identity, + }) + if err != nil { + return kubernetesWorkspaceProvision{}, err + } + if err := kubeexecutor.StampCreateNonce(desired); err != nil { + return kubernetesWorkspaceProvision{}, fmt.Errorf( + "kubernetes lifecycle: generate workspace PVC create nonce: %w", err, + ) + } + created, err := resources.CreatePersistentVolumeClaim(ctx, namespace, desired) + if err != nil { + createErr := fmt.Errorf("kubernetes lifecycle: create workspace PVC: %w", err) + if !kubeexecutor.IsAmbiguousCreateError(err) { + return kubernetesWorkspaceProvision{claimName: claimName}, createErr + } + reconcileCtx, cancel := kubernetesAmbiguousCreateContext(ctx) + created, err = resources.GetPersistentVolumeClaim(reconcileCtx, namespace, desired.Name) + cancel() + if apierrors.IsNotFound(err) { + return kubernetesWorkspaceProvision{claimName: claimName}, createErr + } + if err != nil { + return kubernetesWorkspaceProvision{claimName: claimName}, errors.Join( + createErr, fmt.Errorf("kubernetes lifecycle: reconcile ambiguous workspace PVC create: %w", err), + ) + } + if verifyErr := verifyOwnedPVC(created, desired, identity); verifyErr != nil { + return kubernetesWorkspaceProvision{claimName: claimName}, errors.Join( + createErr, fmt.Errorf("kubernetes lifecycle: reconcile ambiguous workspace PVC create: %w", verifyErr), + ) + } + } + provision := kubernetesWorkspaceProvision{claimName: claimName, claim: created, createdClaim: created} + if onCreated != nil { + if err := onCreated(created); err != nil { + return provision, err + } + } + if err := verifyOwnedPVC(created, desired, identity); err != nil { + return provision, err + } + return provision, nil +} + +func getExistingKubernetesWorkspacePVC( + ctx context.Context, + resources kubernetesResourceClient, + namespace, claimName string, +) (kubernetesWorkspaceProvision, error) { + claim, err := resources.GetPersistentVolumeClaim(ctx, namespace, claimName) + provision := kubernetesWorkspaceProvision{claimName: claimName, claim: claim} + if err != nil { + return provision, fmt.Errorf("kubernetes lifecycle: verify existing workspace PVC: %w", err) + } + if claim == nil || claim.UID == "" || claim.Name != claimName || claim.Namespace != namespace { + return provision, errors.New("kubernetes lifecycle: existing workspace PVC identity is incomplete") + } + return provision, nil +} + +func composeKubernetesLifecyclePod( + profile kubeexecutor.ProfileConfig, + identity kubeexecutor.ResourceIdentity, + name, namespace, claimName string, +) (*corev1.Pod, error) { + template, err := kubeexecutor.ParsePodTemplate(profile.PodTemplateYAML) + if err != nil { + return nil, err + } + pod, _, err := kubeexecutor.ComposePod(template, profile, kubeexecutor.PodOptions{ + Name: name, Namespace: namespace, Identity: identity, + Command: []string{"sh", "-c"}, Args: []string{kubernetesBootstrapCommand()}, + WorkingDir: kubernetesWorkspacePath, AgentctlPort: kubeexecutor.DefaultAgentctlPort, + ManagedPVCName: claimName, + }) + return pod, err +} + +func createAndWaitKubernetesPod( + ctx context.Context, + resources kubernetesResourceClient, + desired *corev1.Pod, + identity kubeexecutor.ResourceIdentity, + mainContainer, description string, + onCheckpoint func(string, *corev1.Pod) error, +) (*corev1.Pod, *corev1.Pod, error) { + if err := kubeexecutor.StampCreateNonce(desired); err != nil { + return nil, nil, fmt.Errorf("kubernetes lifecycle: generate %s create nonce: %w", description, err) + } + created, err := resources.CreatePod(ctx, desired.Namespace, desired) + if err != nil { + createErr := fmt.Errorf("kubernetes lifecycle: create %s: %w", description, err) + if !kubeexecutor.IsAmbiguousCreateError(err) { + return nil, nil, createErr + } + reconcileCtx, cancel := kubernetesAmbiguousCreateContext(ctx) + created, err = resources.GetPod(reconcileCtx, desired.Namespace, desired.Name) + cancel() + if apierrors.IsNotFound(err) { + return nil, nil, createErr + } + if err != nil { + return nil, nil, errors.Join( + createErr, fmt.Errorf("kubernetes lifecycle: reconcile ambiguous %s create: %w", description, err), + ) + } + if verifyErr := verifyCreatedPod(created, desired, identity, mainContainer); verifyErr != nil { + return nil, nil, errors.Join( + createErr, fmt.Errorf("kubernetes lifecycle: reconcile ambiguous %s create: %w", description, verifyErr), + ) + } + } + if onCheckpoint != nil { + if err := onCheckpoint(KubernetesInventoryStatePodCreated, created); err != nil { + return created, nil, err + } + } + if err := verifyCreatedPod(created, desired, identity, mainContainer); err != nil { + return created, nil, err + } + if onCheckpoint != nil { + if err := onCheckpoint(KubernetesInventoryStatePodAdmitted, created); err != nil { + return created, nil, err + } + } + running, err := resources.WaitForPodRunning(ctx, desired.Namespace, desired.Name, mainContainer) + if err != nil { + return created, nil, fmt.Errorf("kubernetes lifecycle: wait for %s: %w", description, err) + } + if err := verifyRecordedPod(running, desired.Namespace, desired.Name, created.UID, identity); err != nil { + return created, nil, err + } + return created, running, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_reconnect.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_reconnect.go new file mode 100644 index 0000000000..1b439a9b4a --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_reconnect.go @@ -0,0 +1,526 @@ +package lifecycle + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/common/logger" +) + +func (r *KubernetesExecutor) reconnect( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + executorConfig kubeexecutor.ExecutorConfig, +) (*ExecutorInstance, error) { + recorded, identity, err := kubernetesRecordedInventory(req, true) + if err != nil { + return nil, err + } + req = kubernetesRequestWithRecordedIdentity(req, identity) + pod, err := runtime.resources.GetPod(ctx, recorded.namespace, recorded.podName) + if err != nil { + if apierrors.IsNotFound(err) { + return r.recreateMissingKubernetesPod(ctx, runtime, req, executorConfig, recorded, identity) + } + return nil, fmt.Errorf("kubernetes lifecycle: inspect recorded Pod: %w", err) + } + if err := verifyRecordedPod(pod, recorded.namespace, recorded.podName, recorded.podUID, identity); err != nil { + return nil, err + } + if err := verifyKubernetesRecordedPVC(ctx, runtime.resources, recorded, identity); err != nil { + return nil, err + } + + client, forward, token, remotePort, err := r.reconnectKubernetesAgentctl(ctx, runtime, req, pod, recorded) + if err != nil { + return nil, err + } + r.replaceSession(req.InstanceID, &kubernetesSession{ + runtime: runtime, forward: forward, client: client, + request: cloneKubernetesCreateRequest(req), + restartCount: kubernetesMainContainerRestartCount(pod, recorded.mainContainer), + }) + metadata := cloneKubernetesMetadata(req.Metadata) + metadata[MetadataKeyKubernetesAgentctlRemotePort] = strconv.Itoa(remotePort) + metadata[MetadataKeyKubernetesAgentctlInstanceID] = recorded.agentctlInstanceID + metadata[MetadataKeyKubernetesContainerRestartCount] = strconv.FormatInt( + int64(kubernetesMainContainerRestartCount(pod, recorded.mainContainer)), 10, + ) + metadata[MetadataKeyKubernetesResourceInstanceID] = identity.InstanceID + metadata[MetadataKeyKubernetesExecutorConfigHash] = kubernetesConfigHash(executorConfig) + metadata[MetadataKeyIsRemote] = true + return &ExecutorInstance{ + InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, + RuntimeName: r.Name(), Client: client, WorkspacePath: kubernetesWorkspacePath, + Metadata: metadata, AuthToken: token, BootstrapNonce: req.BootstrapNonce, + }, nil +} + +type kubernetesRecordedState struct { + inventoryState string + namespace string + podName string + podUID types.UID + mainContainer string + workspaceMode kubeexecutor.WorkspaceMode + pvcName string + pvcUID types.UID + pvcCreated bool + remotePort int + agentctlInstanceID string +} + +func kubernetesRecordedInventory( + req *ExecutorCreateRequest, + validateConnectionIdentity bool, +) (kubernetesRecordedState, kubeexecutor.ResourceIdentity, error) { + recorded, err := decodeKubernetesRecordedState(req.Metadata) + if err != nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err + } + identity, err := kubernetesRecordedResourceIdentity(req, validateConnectionIdentity) + if err != nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err + } + if recorded.agentctlInstanceID != identity.InstanceID { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, errors.New( + "kubernetes lifecycle: recorded agentctl instance does not match resource identity", + ) + } + if err := validateKubernetesRecordedPod(recorded); err != nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err + } + if err := validateKubernetesRecordedWorkspace(recorded); err != nil { + return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err + } + return recorded, identity, nil +} + +func decodeKubernetesRecordedState(metadata map[string]interface{}) (kubernetesRecordedState, error) { + recorded := decodeKubernetesRecordedStateBase(metadata) + remotePort, err := strconv.Atoi(getMetadataString(metadata, MetadataKeyKubernetesAgentctlRemotePort)) + if err != nil || remotePort < 1 || remotePort > 65535 { + return kubernetesRecordedState{}, errors.New("kubernetes lifecycle: recorded agentctl port is invalid") + } + recorded.remotePort = remotePort + return recorded, nil +} + +func decodeKubernetesRecordedStateBase(metadata map[string]interface{}) kubernetesRecordedState { + return kubernetesRecordedState{ + inventoryState: getMetadataString(metadata, MetadataKeyKubernetesInventoryState), + namespace: getMetadataString(metadata, MetadataKeyKubernetesNamespace), + podName: getMetadataString(metadata, MetadataKeyKubernetesPodName), + podUID: types.UID(getMetadataString(metadata, MetadataKeyKubernetesPodUID)), + mainContainer: getMetadataString(metadata, MetadataKeyKubernetesMainContainer), + workspaceMode: kubeexecutor.WorkspaceMode(getMetadataString(metadata, MetadataKeyKubernetesRuntimeWorkspaceMode)), + pvcName: getMetadataString(metadata, MetadataKeyKubernetesPVCName), + pvcUID: types.UID(getMetadataString(metadata, MetadataKeyKubernetesPVCUID)), + pvcCreated: getMetadataBool(metadata, MetadataKeyKubernetesPVCCreated), + agentctlInstanceID: getMetadataString(metadata, MetadataKeyKubernetesAgentctlInstanceID), + } +} + +func kubernetesRecordedResourceIdentity( + req *ExecutorCreateRequest, + validateConnectionIdentity bool, +) (kubeexecutor.ResourceIdentity, error) { + identity := kubeexecutor.ResourceIdentity{ + ExecutorID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceExecutorID), + ProfileID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceProfileID), + InstanceID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceInstanceID), + TaskID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceTaskID), + SessionID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceSessionID), + EnvironmentID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceEnvironmentID), + } + if identity.ExecutorID == "" || identity.ProfileID == "" || identity.InstanceID == "" || + identity.TaskID == "" || identity.SessionID == "" || identity.EnvironmentID == "" { + return kubeexecutor.ResourceIdentity{}, errors.New("kubernetes lifecycle: recorded resource identity is incomplete") + } + if _, labelErr := kubeexecutor.OwnershipLabels(identity); labelErr != nil { + return kubeexecutor.ResourceIdentity{}, labelErr + } + if err := validateKubernetesRecordedIdentityField("task", req.TaskID, identity.TaskID); err != nil { + return kubeexecutor.ResourceIdentity{}, err + } + if err := validateKubernetesRecordedIdentityField("session", req.SessionID, identity.SessionID); err != nil { + return kubeexecutor.ResourceIdentity{}, err + } + if validateConnectionIdentity { + if err := validateKubernetesRecordedIdentityField( + "executor", getMetadataString(req.Metadata, "executor_id"), identity.ExecutorID, + ); err != nil { + return kubeexecutor.ResourceIdentity{}, err + } + if err := validateKubernetesRecordedIdentityField( + "profile", getMetadataString(req.Metadata, MetadataKeyExecutorProfileID), identity.ProfileID, + ); err != nil { + return kubeexecutor.ResourceIdentity{}, err + } + } + return identity, nil +} + +func validateKubernetesRecordedIdentityField(kind, requestValue, recordedValue string) error { + if requestValue == "" || requestValue != recordedValue { + return fmt.Errorf("kubernetes lifecycle: recorded resource %s identity does not match request", kind) + } + return nil +} + +func kubernetesRequestWithRecordedIdentity( + req *ExecutorCreateRequest, + identity kubeexecutor.ResourceIdentity, +) *ExecutorCreateRequest { + recorded := *req + recorded.TaskID = identity.TaskID + recorded.SessionID = identity.SessionID + recorded.TaskEnvironmentID = identity.EnvironmentID + return &recorded +} + +func validateKubernetesRecordedPod( + recorded kubernetesRecordedState, +) error { + if recorded.namespace == "" || + recorded.podName == "" || recorded.podUID == "" || recorded.mainContainer == "" || + recorded.agentctlInstanceID == "" { + return errors.New("kubernetes lifecycle: recorded Pod inventory is incomplete") + } + return nil +} + +func validateKubernetesRecordedWorkspace(recorded kubernetesRecordedState) error { + switch recorded.workspaceMode { + case kubeexecutor.WorkspaceModeEmptyDir: + if recorded.pvcName != "" || recorded.pvcUID != "" || recorded.pvcCreated { + return errors.New("kubernetes lifecycle: emptyDir inventory unexpectedly records a PVC") + } + case kubeexecutor.WorkspaceModeManagedPVC, kubeexecutor.WorkspaceModeExistingClaim: + if recorded.pvcName == "" || recorded.pvcUID == "" { + return errors.New("kubernetes lifecycle: recorded PVC inventory is incomplete") + } + if recorded.workspaceMode == kubeexecutor.WorkspaceModeManagedPVC && !recorded.pvcCreated { + return errors.New("kubernetes lifecycle: managed PVC inventory does not record ownership") + } + if recorded.workspaceMode == kubeexecutor.WorkspaceModeExistingClaim && recorded.pvcCreated { + return errors.New("kubernetes lifecycle: existing PVC inventory records Kandev ownership") + } + default: + return errors.New("kubernetes lifecycle: recorded workspace mode is invalid") + } + return nil +} + +func verifyKubernetesRecordedPVC( + ctx context.Context, + resources kubernetesResourceClient, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if recorded.workspaceMode == kubeexecutor.WorkspaceModeEmptyDir { + return nil + } + pvc, err := resources.GetPersistentVolumeClaim(ctx, recorded.namespace, recorded.pvcName) + if err != nil { + return fmt.Errorf("kubernetes lifecycle: inspect recorded PVC: %w", err) + } + if pvc == nil || pvc.Namespace != recorded.namespace || pvc.Name != recorded.pvcName || pvc.UID != recorded.pvcUID { + return errors.New("kubernetes lifecycle: PVC UID or name does not match recorded inventory") + } + if recorded.workspaceMode == kubeexecutor.WorkspaceModeManagedPVC { + labels, labelErr := kubeexecutor.OwnershipLabels(identity) + if labelErr != nil { + return labelErr + } + if labelErr := kubeexecutor.ValidateOwnershipLabels(pvc.Labels, labels); labelErr != nil { + return fmt.Errorf("kubernetes lifecycle: PVC ownership labels: %w", labelErr) + } + } + return nil +} + +func (r *KubernetesExecutor) reconnectKubernetesAgentctl( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + recorded kubernetesRecordedState, +) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) { + if req.AuthToken == "" { + return nil, nil, "", 0, errors.New("kubernetes lifecycle: recorded agentctl auth token is unavailable") + } + client, forward, err := r.reattachKubernetesAgentctl(ctx, runtime, req, pod, recorded) + if err == nil { + return client, forward, req.AuthToken, recorded.remotePort, nil + } + if req.BootstrapNonce == "" { + return nil, nil, "", 0, errors.New("kubernetes lifecycle: agentctl restart detected but bootstrap nonce is unavailable") + } + return r.connectRestartedKubernetesAgentctl(ctx, runtime, req, pod, recorded.agentctlInstanceID) +} + +func (r *KubernetesExecutor) reattachKubernetesAgentctl( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + recorded kubernetesRecordedState, +) (*agentctl.Client, kubeexecutor.PortForwardSession, error) { + forward, err := startKubernetesForward(ctx, runtime.streams, pod, uint16(recorded.remotePort)) + if err != nil { + return nil, nil, err + } + client := newKubernetesAgentctlClient(r.logger, req, recorded.agentctlInstanceID, req.AuthToken, forward.LocalPort()) + if healthErr := client.Health(ctx); healthErr != nil { + client.Close() + _ = forward.Close() + return nil, nil, fmt.Errorf("kubernetes lifecycle: recorded instance health: %w", healthErr) + } + return client, forward, nil +} + +func (r *KubernetesExecutor) connectRestartedKubernetesAgentctl( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + pod *corev1.Pod, + remoteInstanceID string, +) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) { + controlForward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod) + if err != nil { + return nil, nil, "", 0, err + } + defer func() { _ = controlForward.Close() }() + token, err := control.Handshake(ctx, req.BootstrapNonce) + if err != nil { + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: restarted nonce handshake: %w", err) + } + createRequest := buildReconnectCreateInstanceRequest(req, remoteInstanceID) + response, err := createOrReconcileKubernetesAgentctlInstance(ctx, control, createRequest) + if err != nil { + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: recreate agentctl instance: %w", err) + } + if response.Port < 1 || response.Port > 65535 { + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: invalid restarted agentctl port %d", response.Port) + } + forward, err := startKubernetesForward(ctx, runtime.streams, pod, uint16(response.Port)) + if err != nil { + return nil, nil, "", 0, err + } + client := newKubernetesAgentctlClient(r.logger, req, remoteInstanceID, token, forward.LocalPort()) + if err := client.Health(ctx); err != nil { + client.Close() + _ = forward.Close() + return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: restarted instance health: %w", err) + } + return client, forward, token, response.Port, nil +} + +func newKubernetesAgentctlClient( + log *logger.Logger, + req *ExecutorCreateRequest, + remoteInstanceID, token string, + localPort uint16, +) *agentctl.Client { + return agentctl.NewClient(kubernetesControlHost, int(localPort), log, + agentctl.WithExecutionID(remoteInstanceID), agentctl.WithSessionID(req.SessionID), agentctl.WithAuthToken(token)) +} + +func cloneKubernetesMetadata(metadata map[string]interface{}) map[string]interface{} { + cloned := make(map[string]interface{}, len(metadata)) + for key, value := range metadata { + cloned[key] = value + } + return cloned +} + +func getMetadataBool(metadata map[string]interface{}, key string) bool { + if metadata == nil { + return false + } + switch value := metadata[key].(type) { + case bool: + return value + case string: + parsed, _ := strconv.ParseBool(value) + return parsed + default: + return false + } +} + +func (r *KubernetesExecutor) recreateMissingKubernetesPod( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + executorConfig kubeexecutor.ExecutorConfig, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) (_ *ExecutorInstance, returnedErr error) { + profile, err := kubernetesProfileFromRecordedSnapshot(req.Metadata) + if err != nil { + return nil, err + } + if err := r.validateKubernetesPodReplacement(ctx, runtime, req, profile, recorded, identity); err != nil { + return nil, err + } + desired, err := composeKubernetesLifecyclePod( + profile, identity, recorded.podName, recorded.namespace, recorded.pvcName, + ) + if err != nil { + return nil, err + } + podWaitCtx, cancelPodWait := kubernetesRequestContext(ctx, executorConfig.RequestTimeoutSeconds) + created, pod, createErr := createAndWaitKubernetesPod( + podWaitCtx, runtime.resources, desired, identity, recorded.mainContainer, "replacement Pod", + func(state string, pod *corev1.Pod) error { + metadata := cloneKubernetesMetadata(req.Metadata) + metadata[MetadataKeyKubernetesInventoryState] = state + metadata[MetadataKeyKubernetesPodName] = pod.Name + metadata[MetadataKeyKubernetesPodUID] = string(pod.UID) + return checkpointKubernetesRuntimeInventory(ctx, req, metadata) + }, + ) + cancelPodWait() + defer func() { + if returnedErr != nil && created != nil { + rollbackCtx, cancel := kubernetesDetachedRequestContext(ctx, executorConfig.RequestTimeoutSeconds) + defer cancel() + rollbackErr := rollbackCreatedKubernetesPod(rollbackCtx, runtime.resources, created) + returnedErr = errors.Join(returnedErr, rollbackErr) + } + }() + if createErr != nil { + return nil, createErr + } + binary, err := r.resolveBinary(profile.Platform) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: resolve replacement agentctl: %w", err) + } + if err := r.bootstrapPod(ctx, runtime, req, pod, profile, req.BootstrapNonce, binary); err != nil { + return nil, err + } + client, forward, token, remotePort, err := r.connectRestartedKubernetesAgentctl( + ctx, runtime, req, pod, recorded.agentctlInstanceID, + ) + if err != nil { + return nil, err + } + metadata := cloneKubernetesMetadata(req.Metadata) + metadata[MetadataKeyKubernetesPodUID] = string(pod.UID) + metadata[MetadataKeyKubernetesAgentctlRemotePort] = strconv.Itoa(remotePort) + metadata[MetadataKeyKubernetesContainerRestartCount] = strconv.FormatInt( + int64(kubernetesMainContainerRestartCount(pod, recorded.mainContainer)), 10, + ) + metadata[MetadataKeyKubernetesExecutorConfigHash] = kubernetesConfigHash(executorConfig) + metadata[MetadataKeyKubernetesInventoryState] = KubernetesInventoryStateReady + metadata[MetadataKeyIsRemote] = true + if err := checkpointKubernetesRuntimeInventory(ctx, req, metadata); err != nil { + client.Close() + _ = forward.Close() + return nil, err + } + r.replaceSession(req.InstanceID, &kubernetesSession{ + runtime: runtime, forward: forward, client: client, + request: cloneKubernetesCreateRequest(req), + restartCount: kubernetesMainContainerRestartCount(pod, recorded.mainContainer), + }) + return &ExecutorInstance{ + InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, + RuntimeName: r.Name(), Client: client, WorkspacePath: kubernetesWorkspacePath, + Metadata: metadata, AuthToken: token, BootstrapNonce: req.BootstrapNonce, + }, nil +} + +func (r *KubernetesExecutor) validateKubernetesPodReplacement( + ctx context.Context, + runtime *kubernetesRuntimeClient, + req *ExecutorCreateRequest, + profile kubeexecutor.ProfileConfig, + recorded kubernetesRecordedState, + identity kubeexecutor.ResourceIdentity, +) error { + if recorded.workspaceMode == kubeexecutor.WorkspaceModeEmptyDir { + return errors.New("kubernetes lifecycle: recorded Pod is missing; emptyDir workspace is unrecoverable") + } + if profile.Workspace.Mode != recorded.workspaceMode || profile.MainContainer != recorded.mainContainer { + return errors.New("kubernetes lifecycle: recorded Pod profile does not match replacement configuration") + } + if err := verifyKubernetesRecordedPVC(ctx, runtime.resources, recorded, identity); err != nil { + return err + } + if req.BootstrapNonce == "" { + return errors.New("kubernetes lifecycle: cannot replace Pod without recorded bootstrap nonce") + } + if r.resolveBinary == nil { + return errors.New("kubernetes lifecycle: agentctl resolver is not configured") + } + return nil +} + +func kubernetesProfileFromRecordedSnapshot(metadata map[string]interface{}) (kubeexecutor.ProfileConfig, error) { + raw := getMetadataString(metadata, MetadataKeyKubernetesProfileSnapshot) + if raw == "" { + return kubeexecutor.ProfileConfig{}, errors.New("kubernetes lifecycle: recorded workload profile snapshot is missing") + } + var profile kubeexecutor.ProfileConfig + if err := json.Unmarshal([]byte(raw), &profile); err != nil { + return kubeexecutor.ProfileConfig{}, fmt.Errorf("kubernetes lifecycle: decode workload profile snapshot: %w", err) + } + if err := profile.Validate(); err != nil { + return kubeexecutor.ProfileConfig{}, fmt.Errorf("kubernetes lifecycle: validate workload profile snapshot: %w", err) + } + if recordedHash := getMetadataString(metadata, MetadataKeyKubernetesProfileConfigHash); recordedHash == "" || + recordedHash != kubernetesConfigHash(profile) { + return kubeexecutor.ProfileConfig{}, errors.New("kubernetes lifecycle: workload profile snapshot hash does not match") + } + if templateHash := getMetadataString(metadata, MetadataKeyKubernetesTemplateHash); templateHash == "" || + templateHash != kubernetesStringHash(profile.PodTemplateYAML) { + return kubeexecutor.ProfileConfig{}, errors.New("kubernetes lifecycle: workload template snapshot hash does not match") + } + return profile, nil +} + +// ValidateKubernetesResumeMetadata validates the durable runtime authority a +// caller must establish before stopping an active Kubernetes execution for an +// exact-Pod reconnect. It intentionally performs no API calls. +func ValidateKubernetesResumeMetadata( + metadata map[string]interface{}, + taskID, sessionID string, + executorConfigValues map[string]string, +) error { + if _, err := kubeexecutor.ParseExecutorConfig(executorConfigValues); err != nil { + return fmt.Errorf("kubernetes lifecycle: validate current executor config: %w", err) + } + req := &ExecutorCreateRequest{TaskID: taskID, SessionID: sessionID, Metadata: metadata} + recorded, _, err := kubernetesRecordedInventory(req, true) + if err != nil { + return err + } + if recorded.inventoryState != KubernetesInventoryStateReady { + return errors.New("kubernetes lifecycle: recorded runtime inventory is not ready") + } + profile, err := kubernetesProfileFromRecordedSnapshot(metadata) + if err != nil { + return err + } + if profile.MainContainer != recorded.mainContainer || profile.Workspace.Mode != recorded.workspaceMode { + return errors.New("kubernetes lifecycle: recorded workload snapshot does not match runtime inventory") + } + if getMetadataString(metadata, MetadataKeyAuthTokenSecret) == "" || + getMetadataString(metadata, MetadataKeyBootstrapNonceSecret) == "" { + return errors.New("kubernetes lifecycle: recorded runtime secret references are incomplete") + } + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_refresh.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_refresh.go new file mode 100644 index 0000000000..055b1efff8 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_refresh.go @@ -0,0 +1,249 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "strconv" + "sync" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + "go.uber.org/zap" +) + +// RefreshRemoteInstance stages a fresh agentctl instance connection when the +// tracked Pod's main container restarted or the tracked client died. Kubernetes +// resources and the durable bootstrap nonce remain unchanged. +func (r *KubernetesExecutor) RefreshRemoteInstance( + ctx context.Context, + instance *ExecutorInstance, +) (*RemoteInstanceRefresh, error) { + if instance == nil || instance.InstanceID == "" { + return nil, errKubernetesLifecycleRequestIncomplete + } + unlock := r.lockInstance(instance.InstanceID) + defer unlock() + current := r.currentKubernetesSession(instance.InstanceID) + if current == nil || current.runtime == nil || current.client == nil || current.request == nil { + return nil, nil + } + freshRuntime, err := r.newKubernetesRefreshRuntime(instance) + if err != nil { + return nil, err + } + inspection, refreshNeeded, err := r.inspectKubernetesRefresh(ctx, instance, current, freshRuntime) + if err != nil { + return nil, err + } + if !refreshNeeded { + if err := r.publishKubernetesRefreshRuntime(instance.InstanceID, current, freshRuntime); err != nil { + return nil, err + } + return nil, nil + } + req := kubernetesActiveRefreshRequest(current.request, instance, inspection.identity) + client, forward, token, remotePort, err := r.connectKubernetesRefresh( + ctx, freshRuntime, current, req, inspection, + ) + if err != nil { + return nil, err + } + metadata := cloneKubernetesMetadata(instance.Metadata) + metadata[MetadataKeyKubernetesAgentctlRemotePort] = strconv.Itoa(remotePort) + metadata[MetadataKeyKubernetesContainerRestartCount] = strconv.FormatInt(int64(inspection.restarts), 10) + metadata[MetadataKeyKubernetesInventoryState] = KubernetesInventoryStateReady + metadata[MetadataKeyIsRemote] = true + staged := &kubernetesSession{ + runtime: freshRuntime, forward: forward, client: client, + request: req, restartCount: inspection.restarts, + } + commit, abort := r.kubernetesRefreshFinalizers(instance.InstanceID, current, staged) + return &RemoteInstanceRefresh{ + Instance: &ExecutorInstance{ + InstanceID: instance.InstanceID, + TaskID: inspection.identity.TaskID, + SessionID: inspection.identity.SessionID, + RuntimeName: r.Name(), Client: client, WorkspacePath: kubernetesWorkspacePath, + Metadata: metadata, AuthToken: token, BootstrapNonce: instance.BootstrapNonce, + }, + AgentConfig: current.request.AgentConfig, + McpServers: append([]McpServerConfig(nil), current.request.McpServers...), + AutoApprovePermissions: current.request.AutoApprovePermissions, + ProcessRestarted: inspection.restarted, + Commit: commit, + Abort: abort, + }, nil +} + +type kubernetesRefreshInspection struct { + recorded kubernetesRecordedState + identity kubeexecutor.ResourceIdentity + pod *corev1.Pod + restarts int32 + restarted bool +} + +func (r *KubernetesExecutor) currentKubernetesSession(instanceID string) *kubernetesSession { + r.mu.Lock() + defer r.mu.Unlock() + return r.sessions[instanceID] +} + +func (r *KubernetesExecutor) inspectKubernetesRefresh( + ctx context.Context, + instance *ExecutorInstance, + current *kubernetesSession, + freshRuntime *kubernetesRuntimeClient, +) (kubernetesRefreshInspection, bool, error) { + recorded, identity, err := kubernetesCleanupInventory(instance) + if err != nil { + return kubernetesRefreshInspection{}, false, err + } + pod, err := freshRuntime.resources.GetPod(ctx, recorded.namespace, recorded.podName) + if err != nil { + if apierrors.IsNotFound(err) { + return kubernetesRefreshInspection{recorded: recorded, identity: identity}, false, nil + } + return kubernetesRefreshInspection{}, false, fmt.Errorf( + "kubernetes lifecycle: inspect active Pod for refresh: %w", err, + ) + } + if err := verifyRecordedPod(pod, recorded.namespace, recorded.podName, recorded.podUID, identity); err != nil { + return kubernetesRefreshInspection{}, false, err + } + if err := verifyKubernetesRecordedPVC(ctx, freshRuntime.resources, recorded, identity); err != nil { + return kubernetesRefreshInspection{}, false, err + } + restarts := kubernetesMainContainerRestartCount(pod, recorded.mainContainer) + restarted := restarts != current.restartCount + inspection := kubernetesRefreshInspection{ + recorded: recorded, identity: identity, pod: pod, restarts: restarts, restarted: restarted, + } + if !restarted && current.client.Health(ctx) == nil { + return inspection, false, nil + } + if restarted && instance.BootstrapNonce == "" { + return kubernetesRefreshInspection{}, false, errors.New( + "kubernetes lifecycle: active restart detected but bootstrap nonce is unavailable", + ) + } + return inspection, true, nil +} + +func (r *KubernetesExecutor) newKubernetesRefreshRuntime( + instance *ExecutorInstance, +) (*kubernetesRuntimeClient, error) { + if r.clientFactory == nil { + return nil, errors.New("kubernetes lifecycle: refresh client factory is not configured") + } + config, err := kubernetesExecutorConfigFromMetadata(instance.Metadata) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: refresh configuration: %w", err) + } + runtime, err := r.clientFactory(config) + if err != nil { + return nil, fmt.Errorf( + "kubernetes lifecycle: refresh client: %w", routingerr.SanitizeError(err), + ) + } + if runtime == nil || runtime.resources == nil || runtime.streams == nil { + return nil, errors.New("kubernetes lifecycle: refresh client is incomplete") + } + return runtime, nil +} + +func (r *KubernetesExecutor) publishKubernetesRefreshRuntime( + instanceID string, + current *kubernetesSession, + freshRuntime *kubernetesRuntimeClient, +) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.sessions[instanceID] != current { + return errors.New("kubernetes lifecycle: active session changed during credential refresh") + } + current.runtime = freshRuntime + return nil +} + +func kubernetesActiveRefreshRequest( + current *ExecutorCreateRequest, + instance *ExecutorInstance, + identity kubeexecutor.ResourceIdentity, +) *ExecutorCreateRequest { + req := cloneKubernetesCreateRequest(current) + req.InstanceID = instance.InstanceID + req.TaskID = identity.TaskID + req.SessionID = identity.SessionID + req.TaskEnvironmentID = identity.EnvironmentID + req.Metadata = cloneKubernetesMetadata(instance.Metadata) + req.AuthToken = instance.AuthToken + req.BootstrapNonce = instance.BootstrapNonce + return req +} + +func (r *KubernetesExecutor) connectKubernetesRefresh( + ctx context.Context, + freshRuntime *kubernetesRuntimeClient, + current *kubernetesSession, + req *ExecutorCreateRequest, + inspection kubernetesRefreshInspection, +) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) { + if inspection.restarted { + return r.connectRestartedKubernetesAgentctl( + ctx, freshRuntime, req, inspection.pod, inspection.recorded.agentctlInstanceID, + ) + } + client, forward, err := r.reattachKubernetesAgentctl( + ctx, freshRuntime, req, inspection.pod, inspection.recorded, + ) + return client, forward, req.AuthToken, inspection.recorded.remotePort, err +} + +func (r *KubernetesExecutor) kubernetesRefreshFinalizers( + instanceID string, + current, staged *kubernetesSession, +) (func(func()) error, func()) { + var finalizeMu sync.Mutex + finalized := false + commit := func(publish func()) error { + finalizeMu.Lock() + defer finalizeMu.Unlock() + if finalized { + return errors.New("kubernetes lifecycle: remote refresh was already finalized") + } + commitUnlock := r.lockInstance(instanceID) + defer commitUnlock() + r.mu.Lock() + if r.sessions[instanceID] != current { + r.mu.Unlock() + return errors.New("kubernetes lifecycle: active session changed during remote refresh") + } + r.sessions[instanceID] = staged + r.mu.Unlock() + finalized = true + if publish != nil { + publish() + } + if closeErr := closeKubernetesSessionResources(current); closeErr != nil { + r.logger.Warn("failed to close replaced Kubernetes session resources", + zap.String("instance_id", instanceID), zap.Error(closeErr)) + } + return nil + } + abort := func() { + finalizeMu.Lock() + defer finalizeMu.Unlock() + if finalized { + return + } + finalized = true + _ = closeKubernetesSessionResources(staged) + } + return commit, abort +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_restart_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_restart_test.go new file mode 100644 index 0000000000..9cff71bcda --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_restart_test.go @@ -0,0 +1,382 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestKubernetesRefreshRemoteInstanceUsesRotatedConnectionCredentials(t *testing.T) { + controlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }} + executor := newFakeKubernetesExecutorWithForwarder(resources, &recordingKubernetesExec{}, forwards) + req := validKubernetesCreateRequest() + created, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + + executor.mu.Lock() + initialSession := executor.sessions[created.InstanceID] + executor.mu.Unlock() + rotatedResources := copyFakeKubernetesResourceState(resources) + resources.mu.Lock() + resources.getPodErr = apierrors.NewUnauthorized("expired test credential") + resources.mu.Unlock() + var freshConfig kubeexecutor.ExecutorConfig + executor.clientFactory = func(config kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + freshConfig = config + return &kubernetesRuntimeClient{resources: rotatedResources, streams: initialSession.runtime.streams}, nil + } + instance := kubernetesRefreshInstance(created, req.Metadata) + instance.Metadata[MetadataKeyKubernetesAuthMode] = string(kubeexecutor.AuthModeKubeconfig) + instance.Metadata[MetadataKeyKubernetesKubeconfigPath] = "/etc/kandev/rotated.yaml" + instance.Metadata[MetadataKeyKubernetesKubeContext] = "rotated" + + refresh, err := executor.RefreshRemoteInstance(context.Background(), instance) + + require.NoError(t, err) + require.Nil(t, refresh, "healthy agentctl should not be replaced for a credential-only refresh") + require.Equal(t, kubeexecutor.AuthModeKubeconfig, freshConfig.AuthMode) + require.Equal(t, "/etc/kandev/rotated.yaml", freshConfig.KubeconfigPath) + require.Equal(t, "rotated", freshConfig.KubeContext) + executor.mu.Lock() + current := executor.sessions[created.InstanceID] + executor.mu.Unlock() + require.Same(t, rotatedResources, current.runtime.resources) + require.Same(t, initialSession.forward, current.forward) + require.Same(t, initialSession.client, current.client) + require.Len(t, resources.createdPods, 1) + require.Len(t, forwards.remotePorts(), 2, "credential refresh must not replace either retained forward") + status, err := executor.GetRemoteStatus(context.Background(), instance) + require.NoError(t, err) + require.Equal(t, kubernetesStatusRunning, status.State) +} + +func TestKubernetesRefreshRemoteInstanceRejectsFreshUnauthorizedOrForeignRuntime(t *testing.T) { + tests := []struct { + name string + mutate func(*fakeKubernetesResources) + }{ + { + name: "unauthorized", + mutate: func(resources *fakeKubernetesResources) { + resources.getPodErr = apierrors.NewUnauthorized("still unauthorized") + }, + }, + { + name: "foreign exact-name Pod", + mutate: func(resources *fakeKubernetesResources) { + resources.pod.Labels["kandev.ai/session-id"] = "foreign-session" + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + controlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + created, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + executor.mu.Lock() + initialRuntime := executor.sessions[created.InstanceID].runtime + executor.mu.Unlock() + freshResources := copyFakeKubernetesResourceState(resources) + test.mutate(freshResources) + executor.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{resources: freshResources, streams: initialRuntime.streams}, nil + } + + refresh, err := executor.RefreshRemoteInstance( + context.Background(), kubernetesRefreshInstance(created, req.Metadata), + ) + + require.Error(t, err) + require.Nil(t, refresh) + executor.mu.Lock() + currentRuntime := executor.sessions[created.InstanceID].runtime + executor.mu.Unlock() + require.Same(t, initialRuntime, currentRuntime, "failed validation must not publish the fresh runtime") + }) + } +} + +func TestKubernetesRefreshRemoteInstanceLeavesMissingPodForStatusProjection(t *testing.T) { + controlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + created, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + resources.mu.Lock() + resources.pod = nil + resources.mu.Unlock() + instance := kubernetesRefreshInstance(created, req.Metadata) + + refresh, err := executor.RefreshRemoteInstance(context.Background(), instance) + + require.NoError(t, err) + require.Nil(t, refresh) + status, err := executor.GetRemoteStatus(context.Background(), instance) + require.NoError(t, err) + require.Equal(t, "missing", status.State) +} + +func copyFakeKubernetesResourceState(source *fakeKubernetesResources) *fakeKubernetesResources { + source.mu.Lock() + defer source.mu.Unlock() + copy := &fakeKubernetesResources{} + if source.pod != nil { + copy.pod = source.pod.DeepCopy() + } + if source.pvc != nil { + copy.pvc = source.pvc.DeepCopy() + } + return copy +} + +func kubernetesRefreshInstance(created *ExecutorInstance, connection map[string]interface{}) *ExecutorInstance { + metadata := cloneKubernetesMetadata(connection) + for key, value := range created.Metadata { + metadata[key] = value + } + return &ExecutorInstance{ + InstanceID: created.InstanceID, TaskID: created.TaskID, SessionID: created.SessionID, + RuntimeName: created.RuntimeName, Metadata: metadata, + AuthToken: created.AuthToken, BootstrapNonce: created.BootstrapNonce, + } +} + +func TestKubernetesRefreshRemoteInstanceRehandshakesActiveRestart(t *testing.T) { + initialControlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + initialInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): initialControlPort, + 41001: initialInstancePort, + }} + executor := newFakeKubernetesExecutorWithForwarder( + resources, &recordingKubernetesExec{}, forwards, + ) + req := validKubernetesCreateRequest() + created, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + initialForward := forwards.lastSession() + + restartedControlPort := startKubernetesAgentctlServerWithToken(t, true, 41002, "rotated-token") + restartedInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources.mu.Lock() + resources.pod.Status.ContainerStatuses[0].RestartCount = 1 + resources.mu.Unlock() + forwards.mu.Lock() + forwards.localPorts[uint16(kubeexecutor.DefaultAgentctlPort)] = restartedControlPort + forwards.localPorts[41002] = restartedInstancePort + forwards.mu.Unlock() + + refresh, err := executor.RefreshRemoteInstance( + context.Background(), kubernetesRefreshInstance(created, req.Metadata), + ) + require.NoError(t, err) + require.NotNil(t, refresh) + require.Equal(t, "rotated-token", refresh.Instance.AuthToken) + require.Equal(t, "41002", refresh.Instance.Metadata[MetadataKeyKubernetesAgentctlRemotePort]) + require.Equal(t, created.Metadata[MetadataKeyKubernetesPodUID], refresh.Instance.Metadata[MetadataKeyKubernetesPodUID]) + require.False(t, initialForward.isClosed(), "old forward stays live until manager commits") + + require.NoError(t, refresh.Commit(nil)) + require.True(t, initialForward.isClosed()) + require.False(t, forwards.lastSession().isClosed()) + require.Len(t, resources.createdPods, 1) + + againInstance := kubernetesRefreshInstance(refresh.Instance, req.Metadata) + again, err := executor.RefreshRemoteInstance(context.Background(), againInstance) + require.NoError(t, err) + require.Nil(t, again, "recorded restart count prevents duplicate re-handshakes") +} + +func TestKubernetesRefreshRemoteInstanceReattachesDeadLocalClientWithoutHandshake(t *testing.T) { + initialControlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + initialInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): initialControlPort, + 41001: initialInstancePort, + }} + executor := newFakeKubernetesExecutorWithForwarder( + resources, &recordingKubernetesExec{}, forwards, + ) + req := validKubernetesCreateRequest() + created, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + initialRequestCount := len(forwards.remotePorts()) + + executor.mu.Lock() + executor.sessions[created.InstanceID].client = newKubernetesAgentctlClient( + executor.logger, req, created.InstanceID, created.AuthToken, 1, + ) + executor.mu.Unlock() + rotatedControlPort := startKubernetesAgentctlServerWithToken(t, true, 41002, "unexpected-rotated-token") + rotatedInstancePort := startKubernetesAgentctlServer(t, false, 0) + forwards.mu.Lock() + forwards.localPorts[uint16(kubeexecutor.DefaultAgentctlPort)] = rotatedControlPort + forwards.localPorts[41002] = rotatedInstancePort + forwards.mu.Unlock() + + refresh, err := executor.RefreshRemoteInstance( + context.Background(), kubernetesRefreshInstance(created, req.Metadata), + ) + + require.NoError(t, err) + require.NotNil(t, refresh) + require.Equal(t, created.AuthToken, refresh.Instance.AuthToken) + require.Equal(t, "41001", refresh.Instance.Metadata[MetadataKeyKubernetesAgentctlRemotePort]) + require.Equal(t, []uint16{41001}, forwards.remotePorts()[initialRequestCount:], + "dead local transport must re-forward the recorded instance port without using the nonce control port") + require.NoError(t, refresh.Commit(nil)) +} + +func TestKubernetesRefreshCommitDoesNotFailAfterInstallingReplacement(t *testing.T) { + initialControlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + initialInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): initialControlPort, + 41001: initialInstancePort, + }} + executor := newFakeKubernetesExecutorWithForwarder( + resources, &recordingKubernetesExec{}, forwards, + ) + req := validKubernetesCreateRequest() + created, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + initialForward := forwards.lastSession() + initialForward.closeErr = errors.New("old forward close failed") + + restartedControlPort := startKubernetesAgentctlServerWithToken(t, true, 41002, "rotated-token") + restartedInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources.mu.Lock() + resources.pod.Status.ContainerStatuses[0].RestartCount = 1 + resources.mu.Unlock() + forwards.mu.Lock() + forwards.localPorts[uint16(kubeexecutor.DefaultAgentctlPort)] = restartedControlPort + forwards.localPorts[41002] = restartedInstancePort + forwards.mu.Unlock() + + refresh, err := executor.RefreshRemoteInstance( + context.Background(), kubernetesRefreshInstance(created, req.Metadata), + ) + require.NoError(t, err) + require.NotNil(t, refresh) + + require.NoError(t, refresh.Commit(nil), "old-session close is post-commit best effort") + executor.mu.Lock() + installed := executor.sessions[created.InstanceID] + executor.mu.Unlock() + require.Same(t, refresh.Instance.Client, installed.client) +} + +func TestKubernetesPersistedInventorySupportsRestartStatusAndForceCleanup(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + created, err := initial.CreateInstance(context.Background(), req) + require.NoError(t, err) + require.NoError(t, initial.Close()) + + stored := cloneKubernetesMetadata(req.Metadata) + for key, value := range created.Metadata { + stored[key] = value + } + persisted := FilterPersistentMetadata(stored) + require.NotNil(t, persisted) + restartedInstance := &ExecutorInstance{ + InstanceID: "new-current-execution", TaskID: req.TaskID, SessionID: req.SessionID, + Metadata: persisted, StopReason: StopReasonTaskDeleted, + } + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + + status, err := restarted.GetRemoteStatus(context.Background(), restartedInstance) + + require.NoError(t, err) + require.Equal(t, created.Metadata[MetadataKeyKubernetesPodName], status.RemoteName) + require.Equal(t, "running", status.State) + require.NoError(t, restarted.StopInstance(context.Background(), restartedInstance, true)) + require.Equal(t, []string{created.Metadata[MetadataKeyKubernetesPodName].(string) + ":pod-uid"}, resources.deletedPods) + require.Equal(t, []string{created.Metadata[MetadataKeyKubernetesPVCName].(string) + ":pvc-uid"}, resources.deletedPVCs) +} + +func TestKubernetesPersistedInventoryRejectsMismatchedInstanceIdentity(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + created, err := initial.CreateInstance(context.Background(), req) + require.NoError(t, err) + require.NoError(t, initial.Close()) + + stored := cloneKubernetesMetadata(req.Metadata) + for key, value := range created.Metadata { + stored[key] = value + } + persisted := FilterPersistentMetadata(stored) + restartedInstance := &ExecutorInstance{ + InstanceID: "current-execution", TaskID: "other-task", SessionID: req.SessionID, + Metadata: persisted, StopReason: StopReasonTaskDeleted, + } + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + + _, statusErr := restarted.GetRemoteStatus(context.Background(), restartedInstance) + stopErr := restarted.StopInstance(context.Background(), restartedInstance, true) + + require.ErrorContains(t, statusErr, "does not match") + require.ErrorContains(t, stopErr, "does not match") + require.Empty(t, resources.deletedPods) + require.Empty(t, resources.deletedPVCs) +} + +func TestKubernetesLiveExecutionUsesRemoteContainerShellRouting(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + executor := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + + store := NewExecutionStore() + require.NoError(t, store.Add(instance.ToAgentExecution(req))) + manager := &Manager{executionStore: store} + + require.True(t, manager.IsRemoteSession(context.Background(), req.SessionID)) + require.True(t, manager.ShouldUseContainerShell(context.Background(), req.SessionID)) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_snapshot_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_snapshot_test.go new file mode 100644 index 0000000000..7bc6883314 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_snapshot_test.go @@ -0,0 +1,214 @@ +package lifecycle + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + + "github.com/stretchr/testify/require" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestKubernetesCreateInstanceReconnectsExistingPodWithoutParsingCurrentProfile(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + 41001: instancePort, + }) + reconnectRequest := kubernetesReconnectRequest(created) + reconnectRequest.Metadata[MetadataKeyKubernetesProfilePlatform] = "not-a-platform" + reconnectRequest.Metadata[MetadataKeyKubernetesPodTemplateYAML] = "not valid PodTemplate YAML" + + reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, created.Metadata[MetadataKeyKubernetesPodUID], reconnected.Metadata[MetadataKeyKubernetesPodUID]) + require.Len(t, resources.createdPods, 1) +} + +func TestKubernetesProfileSnapshotExcludesResolvedRuntimeMaterial(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + executor := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + req.Env = map[string]string{"OPENAI_API_KEY": "runtime-secret-marker"} + req.Metadata[MetadataKeySetupScript] = "echo setup-script-marker" + req.Metadata[MetadataKeyRepoSetupScript] = "echo repo-script-marker" + req.Metadata[MetadataKeyAgentConfigBundles] = "portable-config-marker" + + instance, err := executor.CreateInstance(context.Background(), req) + + require.NoError(t, err) + snapshot := instance.Metadata[MetadataKeyKubernetesProfileSnapshot].(string) + for _, forbidden := range []string{ + "runtime-secret-marker", "setup-script-marker", "repo-script-marker", "portable-config-marker", + } { + require.False(t, strings.Contains(snapshot, forbidden), "snapshot leaked %q", forbidden) + } +} + +func TestKubernetesCreateInstanceRebuildsLostPodFromRecordedSnapshotAfterProfileChange(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + initialRequest := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(initialRequest) + created, err := initial.CreateInstance(context.Background(), initialRequest) + require.NoError(t, err) + resources.mu.Lock() + resources.pod = nil + resources.nextPodUID = "replacement-pod-uid" + resources.mu.Unlock() + + restartControlPort := startKubernetesAgentctlServer(t, true, 41002) + restartInstancePort := startKubernetesAgentctlServer(t, false, 0) + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): restartControlPort, + 41002: restartInstancePort, + }) + reconnectRequest := kubernetesReconnectRequest(created) + setManagedKubernetesWorkspace(reconnectRequest) + reconnectRequest.Metadata[MetadataKeyKubernetesProfilePlatform] = "not-a-platform" + reconnectRequest.Metadata[MetadataKeyKubernetesPodTemplateYAML] = "not valid PodTemplate YAML" + + reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, "replacement-pod-uid", reconnected.Metadata[MetadataKeyKubernetesPodUID]) + require.Len(t, resources.createdPods, 2) + require.Equal(t, "amd64", resources.createdPods[1].Spec.NodeSelector[corev1.LabelArchStable], + "replacement must use the recorded launch platform, not the edited profile") + require.Equal(t, "example.test/agent:latest", resources.createdPods[1].Spec.Containers[0].Image, + "replacement must use the recorded Pod template") +} + +func TestKubernetesCreateInstanceRebuildsLostPodWhenCurrentProfileIsUnavailable(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + initialRequest := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(initialRequest) + created, err := initial.CreateInstance(context.Background(), initialRequest) + require.NoError(t, err) + resources.mu.Lock() + resources.pod = nil + resources.nextPodUID = "replacement-pod-uid" + resources.mu.Unlock() + + restartControlPort := startKubernetesAgentctlServer(t, true, 41002) + restartInstancePort := startKubernetesAgentctlServer(t, false, 0) + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): restartControlPort, + 41002: restartInstancePort, + }) + reconnectRequest := kubernetesReconnectRequest(created) + for _, key := range kubernetesProfileMetadataKeys { + delete(reconnectRequest.Metadata, key) + } + + reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, "replacement-pod-uid", reconnected.Metadata[MetadataKeyKubernetesPodUID]) + require.Len(t, resources.createdPods, 2) +} + +func TestKubernetesReplacementBootstrapUsesRecordedEnvironmentIdentity(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + initialRequest := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(initialRequest) + created, err := initial.CreateInstance(context.Background(), initialRequest) + require.NoError(t, err) + resources.mu.Lock() + resources.pod = nil + resources.nextPodUID = "replacement-pod-uid" + resources.mu.Unlock() + + restartControlPort := startKubernetesAgentctlServer(t, true, 41002) + restartInstancePort := startKubernetesAgentctlServer(t, false, 0) + replacementExecs := &recordingKubernetesExec{} + restarted := newFakeKubernetesExecutor(t, resources, replacementExecs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): restartControlPort, + 41002: restartInstancePort, + }) + reconnectRequest := kubernetesReconnectRequest(created) + reconnectRequest.TaskEnvironmentID = "drifted-environment" + + _, err = restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + replacementExecs.mu.Lock() + defer replacementExecs.mu.Unlock() + var bootstrapData string + for _, exec := range replacementExecs.requests { + if strings.Contains(strings.Join(exec.request.Command, " "), kubernetesRuntimeEnvPath) { + bootstrapData = string(exec.stdin) + break + } + } + require.Contains(t, bootstrapData, "KANDEV_TASK_ENVIRONMENT_ID='environment-1'") + require.NotContains(t, bootstrapData, "drifted-environment") +} + +func TestKubernetesReplacementRollbackUsesBoundedExecutorTimeout(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + created, err := initial.CreateInstance(context.Background(), req) + require.NoError(t, err) + + resources.mu.Lock() + resources.pod = nil + resources.nextPodUID = "replacement-pod-uid" + resources.deletePodErr = errors.New("rollback delete failed") + resources.mu.Unlock() + restarted := newFakeKubernetesExecutor( + t, resources, &recordingKubernetesExec{err: errors.New("replacement bootstrap failed")}, nil, + ) + reconnectRequest := kubernetesReconnectRequest(created) + reconnectRequest.Metadata[MetadataKeyKubernetesRequestTimeoutSeconds] = "1" + + _, err = restarted.CreateInstance(context.Background(), reconnectRequest) + + require.ErrorContains(t, err, "replacement bootstrap failed") + resources.mu.Lock() + deadline := resources.deletePodContextDeadline + resources.mu.Unlock() + require.False(t, deadline.IsZero(), "replacement rollback must not inherit an unbounded detached context") + require.WithinDuration(t, time.Now().Add(time.Second), deadline, 250*time.Millisecond) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_status.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_status.go new file mode 100644 index 0000000000..12bcc5a1df --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_status.go @@ -0,0 +1,210 @@ +package lifecycle + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/kandev/kandev/internal/agent/runtime/routingerr" +) + +const ( + kubernetesStatusStarting = "starting" + kubernetesStatusRunning = "running" + kubernetesStatusCompleted = "completed" + kubernetesStatusFailed = "failed" + kubernetesStatusUnknown = "unknown" + kubernetesStatusPVCName = "pvc_name" + kubernetesStatusWorkspace = "workspace_mode" +) + +func (r *KubernetesExecutor) GetRemoteStatus( + ctx context.Context, + instance *ExecutorInstance, +) (*RemoteStatus, error) { + checkedAt := time.Now().UTC() + if instance == nil { + return nil, fmt.Errorf("kubernetes lifecycle: instance is nil") + } + unlock := r.lockInstance(instance.InstanceID) + defer unlock() + + recorded, identity, err := kubernetesCleanupInventory(instance) + if err != nil { + return nil, err + } + if recorded.podUID == "" { + return &RemoteStatus{ + RuntimeName: r.Name(), State: kubernetesStatusStarting, LastCheckedAt: checkedAt, + Details: map[string]interface{}{ + MetadataKeyKubernetesConfigNamespace: recorded.namespace, + kubernetesStatusWorkspace: string(recorded.workspaceMode), + kubernetesStatusPVCName: recorded.pvcName, + "inventory_state": recorded.inventoryState, + }, + }, nil + } + runtime, err := r.kubernetesRuntimeForInstance(instance) + if err != nil { + return nil, err + } + pod, err := runtime.resources.GetPod(ctx, recorded.namespace, recorded.podName) + if apierrors.IsNotFound(err) { + return &RemoteStatus{ + RuntimeName: r.Name(), RemoteName: recorded.podName, State: "missing", + LastCheckedAt: checkedAt, + Details: map[string]interface{}{ + MetadataKeyKubernetesConfigNamespace: recorded.namespace, + "pod_name": recorded.podName, + kubernetesStatusWorkspace: string(recorded.workspaceMode), + kubernetesStatusPVCName: recorded.pvcName, + }, + }, nil + } + if err != nil { + return nil, fmt.Errorf( + "kubernetes lifecycle: query recorded Pod status: %w", + routingerr.SanitizeError(err), + ) + } + if err := verifyRecordedPod(pod, recorded.namespace, recorded.podName, recorded.podUID, identity); err != nil { + return nil, err + } + state, containerState, ready, restarts, reason, message := kubernetesRemotePodState(pod, recorded.mainContainer) + details := map[string]interface{}{ + MetadataKeyKubernetesConfigNamespace: recorded.namespace, + "pod_name": recorded.podName, + "pod_phase": string(pod.Status.Phase), + "main_container": recorded.mainContainer, + "container_state": containerState, + "container_ready": ready, + "restart_count": restarts, + kubernetesStatusWorkspace: string(recorded.workspaceMode), + kubernetesStatusPVCName: recorded.pvcName, + } + if reason != "" { + details["reason"] = reason + } + if message != "" { + details["message"] = message + } + var createdAt *time.Time + if !pod.CreationTimestamp.IsZero() { + value := pod.CreationTimestamp.UTC() + createdAt = &value + } + return &RemoteStatus{ + RuntimeName: r.Name(), RemoteName: recorded.podName, State: state, + CreatedAt: createdAt, LastCheckedAt: checkedAt, Details: details, + }, nil +} + +func (r *KubernetesExecutor) kubernetesRuntimeForInstance( + instance *ExecutorInstance, +) (*kubernetesRuntimeClient, error) { + r.mu.Lock() + session := r.sessions[instance.InstanceID] + r.mu.Unlock() + if session != nil && session.runtime != nil { + return session.runtime, nil + } + if r.clientFactory == nil { + return nil, fmt.Errorf("kubernetes lifecycle: status client factory is not configured") + } + config, err := kubernetesExecutorConfigFromMetadata(instance.Metadata) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: status configuration: %w", err) + } + runtime, err := r.clientFactory(config) + if err != nil { + return nil, fmt.Errorf("kubernetes lifecycle: status client: %w", routingerr.SanitizeError(err)) + } + if runtime == nil || runtime.resources == nil { + return nil, fmt.Errorf("kubernetes lifecycle: status resource client is unavailable") + } + return runtime, nil +} + +func kubernetesRemotePodState( + pod *corev1.Pod, + mainContainer string, +) (state, containerState string, ready bool, restarts int32, reason, message string) { + projection := kubernetesPodPhaseProjection(pod) + for index := range pod.Status.ContainerStatuses { + status := &pod.Status.ContainerStatuses[index] + if status.Name == mainContainer { + projection = kubernetesContainerProjection(pod.DeletionTimestamp != nil, status, projection) + break + } + } + return projection.state, projection.containerState, projection.ready, + projection.restarts, routingerr.Sanitize(projection.reason), routingerr.Sanitize(projection.message) +} + +type kubernetesPodStatusProjection struct { + state string + containerState string + ready bool + restarts int32 + reason string + message string +} + +func kubernetesPodPhaseProjection(pod *corev1.Pod) kubernetesPodStatusProjection { + projection := kubernetesPodStatusProjection{ + state: kubernetesStatusUnknown, reason: pod.Status.Reason, message: pod.Status.Message, + } + if pod.DeletionTimestamp != nil { + projection.state = "stopping" + return projection + } + switch pod.Status.Phase { + case corev1.PodPending, corev1.PodRunning: + projection.state = kubernetesStatusStarting + case corev1.PodSucceeded: + projection.state = kubernetesStatusCompleted + case corev1.PodFailed: + projection.state = kubernetesStatusFailed + } + return projection +} + +func kubernetesContainerProjection( + podDeleting bool, + status *corev1.ContainerStatus, + projection kubernetesPodStatusProjection, +) kubernetesPodStatusProjection { + projection.ready, projection.restarts = status.Ready, status.RestartCount + switch { + case status.State.Running != nil: + projection.containerState = kubernetesStatusRunning + if !podDeleting { + projection.state = kubernetesStatusStarting + if status.Ready { + projection.state = kubernetesStatusRunning + } + } + case status.State.Waiting != nil: + projection.containerState = "waiting" + projection.reason, projection.message = status.State.Waiting.Reason, status.State.Waiting.Message + if !podDeleting { + projection.state = kubernetesStatusStarting + } + case status.State.Terminated != nil: + projection.containerState = "terminated" + projection.reason, projection.message = status.State.Terminated.Reason, status.State.Terminated.Message + if !podDeleting && status.State.Terminated.ExitCode == 0 { + projection.state = kubernetesStatusCompleted + } else if !podDeleting { + projection.state = kubernetesStatusFailed + } + default: + projection.containerState = kubernetesStatusUnknown + } + return projection +} + +var _ RemoteStatusProvider = (*KubernetesExecutor)(nil) diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_stream_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_stream_test.go new file mode 100644 index 0000000000..bb63aadb8c --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_stream_test.go @@ -0,0 +1,314 @@ +package lifecycle + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "strconv" + "sync" + "testing" + "time" + + kubernetesclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/stretchr/testify/require" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestKubernetesStreamingConfigClearsRequestTimeoutWithoutMutatingAPIConfig(t *testing.T) { + apiConfig := &rest.Config{Host: "https://cluster.example", Timeout: 30 * time.Second} + + streamingConfig := kubernetesStreamingRESTConfig(apiConfig) + + require.NotSame(t, apiConfig, streamingConfig) + require.Equal(t, 30*time.Second, apiConfig.Timeout, "ordinary API calls retain their bounded timeout") + require.Zero(t, streamingConfig.Timeout, "long-lived streams must not inherit a whole-response deadline") +} + +func TestKubernetesPortForwardCancellationClosesBlockedWebSocketHandshake(t *testing.T) { + requestStarted := make(chan struct{}) + requestCanceled := make(chan struct{}) + releaseHandler := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + close(requestStarted) + select { + case <-request.Context().Done(): + close(requestCanceled) + case <-releaseHandler: + } + })) + t.Cleanup(func() { + close(releaseHandler) + server.Close() + }) + config := &rest.Config{Host: server.URL} + clientset, err := kubernetesclient.NewForConfig(config) + require.NoError(t, err) + forwarder := &liveKubernetesForwarder{ + config: config, handshakeTimeout: 50 * time.Millisecond, + resources: &liveKubernetesResources{client: &kubeexecutor.Client{ + RESTConfig: config, + Clientset: clientset, + }}, + } + ctx, cancel := context.WithCancel(context.Background()) + session, err := forwarder.Forward(ctx, kubeexecutor.PortForwardRequest{ + Namespace: "kandev-agents", Pod: "agent-pod", LocalAddress: "127.0.0.1", RemotePort: 8765, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, session.Close()) }) + + select { + case <-requestStarted: + case <-time.After(time.Second): + t.Fatal("WebSocket handshake did not reach test server") + } + require.NoError(t, session.Close()) + select { + case <-requestCanceled: + case <-time.After(time.Second): + t.Fatal("canceled port-forward startup left WebSocket handshake running") + } + select { + case <-session.Done(): + default: + t.Fatal("Close returned before the blocked port-forward startup goroutine joined") + } + cancel() +} + +func TestKubernetesPortForwardFallsBackToSPDYAfterWebSocketHandshakeTimeout(t *testing.T) { + webSocketStarted := make(chan struct{}) + spdyStarted := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + switch request.Method { + case http.MethodGet: + close(webSocketStarted) + <-request.Context().Done() + case http.MethodPost: + close(spdyStarted) + http.Error(response, "test SPDY rejection", http.StatusBadRequest) + default: + http.Error(response, "unexpected method", http.StatusMethodNotAllowed) + } + })) + t.Cleanup(server.Close) + config := &rest.Config{Host: server.URL} + clientset, err := kubernetesclient.NewForConfig(config) + require.NoError(t, err) + forwarder := &liveKubernetesForwarder{ + config: config, handshakeTimeout: 50 * time.Millisecond, + resources: &liveKubernetesResources{client: &kubeexecutor.Client{ + RESTConfig: config, + Clientset: clientset, + }}, + } + session, err := forwarder.Forward(context.Background(), kubeexecutor.PortForwardRequest{ + Namespace: "kandev-agents", Pod: "agent-pod", LocalAddress: "127.0.0.1", RemotePort: 8765, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, session.Close()) }) + + select { + case <-webSocketStarted: + case <-time.After(time.Second): + t.Fatal("WebSocket primary did not reach test server") + } + select { + case <-spdyStarted: + case <-time.After(time.Second): + t.Fatal("timed-out WebSocket primary did not fall back to SPDY") + } +} + +func TestKubernetesHealthWaitUsesCallerContextBeyondDockerRetryCap(t *testing.T) { + executor := NewKubernetesExecutor(nil, newTestLogger()) + executor.healthRetryDelay = time.Nanosecond + checker := &stubHealthChecker{ + becomesHealthyAfter: agentctlHealthMaxRetries + 1, + failErr: context.DeadlineExceeded, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + err := executor.waitForKubernetesAgentctlHealth(ctx, checker) + + require.NoError(t, err) + require.EqualValues(t, agentctlHealthMaxRetries+1, checker.calls) +} + +func TestKubernetesCreateRebuildsControlForwardThatDiesBeforeRemoteHealth(t *testing.T) { + firstHealth := make(chan struct{}) + unhealthy := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + select { + case <-firstHealth: + default: + close(firstHealth) + } + http.Error(response, "agentctl not listening remotely", http.StatusServiceUnavailable) + })) + t.Cleanup(unhealthy.Close) + _, unhealthyPort := testServerHostPort(t, unhealthy) + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + forwarder := &recoveringControlKubernetesForwarder{ + firstHealth: firstHealth, + controlPorts: []uint16{unhealthyPort, controlPort}, + instancePort: instancePort, + } + executor := NewKubernetesExecutor(nil, newTestLogger()) + executor.healthRetryDelay = 40 * time.Millisecond + executor.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{ + resources: &fakeKubernetesResources{}, + streams: kubeexecutor.NewStreamOperations( + &recordingKubernetesExec{}, forwarder, + ), + }, nil + } + executor.resolveBinary = func(kubeexecutor.Platform) ([]byte, error) { return []byte("agentctl"), nil } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + instance, err := executor.CreateInstance(ctx, validKubernetesCreateRequest()) + + require.NoError(t, err) + require.NotNil(t, instance.Client) + require.Equal(t, []uint16{ + uint16(kubeexecutor.DefaultAgentctlPort), + uint16(kubeexecutor.DefaultAgentctlPort), + 41001, + }, forwarder.remotePorts()) + require.True(t, forwarder.firstSession().isClosed()) + createdAt := forwarder.createdAtTimes() + require.Len(t, createdAt, 3) + require.GreaterOrEqual(t, createdAt[1].Sub(createdAt[0]), executor.healthRetryDelay, + "dead control-forward recreation must not hot-loop against the API server") +} + +func TestKubernetesRetainedForwardOutlivesCreateContext(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + forwarder := &contextBoundKubernetesForwarder{recordingKubernetesForwarder: recordingKubernetesForwarder{ + localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }, + }} + executor := NewKubernetesExecutor(nil, newTestLogger()) + executor.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{ + resources: &fakeKubernetesResources{}, + streams: kubeexecutor.NewStreamOperations( + &recordingKubernetesExec{}, forwarder, + ), + }, nil + } + executor.resolveBinary = func(kubeexecutor.Platform) ([]byte, error) { return []byte("agentctl"), nil } + ctx, cancel := context.WithCancel(context.Background()) + instance, err := executor.CreateInstance(ctx, validKubernetesCreateRequest()) + require.NoError(t, err) + require.NotNil(t, instance.Client) + retained := forwarder.lastSession() + + cancel() + time.Sleep(20 * time.Millisecond) + require.False(t, retained.isClosed(), "launch timeout cancellation must not tear down the retained forward") + + require.NoError(t, executor.Close()) + require.True(t, retained.isClosed(), "backend Close owns retained-forward shutdown") +} + +type contextBoundKubernetesForwarder struct { + recordingKubernetesForwarder +} + +type recoveringControlKubernetesForwarder struct { + mu sync.Mutex + firstHealth <-chan struct{} + controlPorts []uint16 + instancePort uint16 + requests []kubeexecutor.PortForwardRequest + sessions []*fakeKubernetesForwardSession + createdAt []time.Time +} + +func (f *recoveringControlKubernetesForwarder) Forward( + _ context.Context, + request kubeexecutor.PortForwardRequest, +) (kubeexecutor.PortForwardSession, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.requests = append(f.requests, request) + f.createdAt = append(f.createdAt, time.Now()) + localPort := f.instancePort + controlIndex := 0 + if request.RemotePort == uint16(kubeexecutor.DefaultAgentctlPort) { + controlIndex = len(f.sessions) + if controlIndex >= len(f.controlPorts) { + controlIndex = len(f.controlPorts) - 1 + } + localPort = f.controlPorts[controlIndex] + } + session := newFakeKubernetesForwardSession(localPort) + f.sessions = append(f.sessions, session) + if len(f.sessions) == 1 { + go func() { + <-f.firstHealth + session.done <- errors.New("remote port refused forwarded stream") + }() + } + return session, nil +} + +func (f *recoveringControlKubernetesForwarder) createdAtTimes() []time.Time { + f.mu.Lock() + defer f.mu.Unlock() + return append([]time.Time(nil), f.createdAt...) +} + +func (f *recoveringControlKubernetesForwarder) remotePorts() []uint16 { + f.mu.Lock() + defer f.mu.Unlock() + ports := make([]uint16, 0, len(f.requests)) + for _, request := range f.requests { + ports = append(ports, request.RemotePort) + } + return ports +} + +func (f *recoveringControlKubernetesForwarder) firstSession() *fakeKubernetesForwardSession { + f.mu.Lock() + defer f.mu.Unlock() + return f.sessions[0] +} + +func testServerHostPort(t *testing.T, server *httptest.Server) (string, uint16) { + t.Helper() + host, rawPort, err := net.SplitHostPort(server.Listener.Addr().String()) + require.NoError(t, err) + port, err := strconv.Atoi(rawPort) + require.NoError(t, err) + return host, uint16(port) +} + +func (f *contextBoundKubernetesForwarder) Forward( + ctx context.Context, + request kubeexecutor.PortForwardRequest, +) (kubeexecutor.PortForwardSession, error) { + session, err := f.recordingKubernetesForwarder.Forward(ctx, request) + if err != nil { + return nil, err + } + if done := ctx.Done(); done != nil { + go func() { + <-done + _ = session.Close() + }() + } + return session, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go new file mode 100644 index 0000000000..900c216ae8 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go @@ -0,0 +1,708 @@ +package lifecycle + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/require" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" +) + +type kubernetesAmbiguousCreateControl struct { + createErr error + info *agentctl.InstanceInfo + getErr error + getCalls int +} + +func (c *kubernetesAmbiguousCreateControl) CreateInstance( + context.Context, + *agentctl.CreateInstanceRequest, +) (*agentctl.CreateInstanceResponse, error) { + return nil, c.createErr +} + +func (c *kubernetesAmbiguousCreateControl) GetInstance( + ctx context.Context, + _ string, +) (*agentctl.InstanceInfo, error) { + c.getCalls++ + if err := ctx.Err(); err != nil { + return nil, err + } + return c.info, c.getErr +} + +func TestKubernetesCreateInstanceAcceptsCompleteTypedConfiguration(t *testing.T) { + executor := NewKubernetesExecutor(nil, newTestLogger()) + _, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + if errors.Is(err, errKubernetesLifecycleRequestIncomplete) { + t.Fatalf("CreateInstance() error = %v, complete typed configuration was not consumed", err) + } +} + +func TestKubernetesAgentctlCreateReconcilesAmbiguousResponse(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + control := &kubernetesAmbiguousCreateControl{ + createErr: context.Canceled, + info: &agentctl.InstanceInfo{ + ID: "instance-1", Port: 41001, WorkspacePath: dockerWorkspacePath, + }, + } + request := &agentctl.CreateInstanceRequest{ID: "instance-1", WorkspacePath: dockerWorkspacePath} + + response, err := createOrReconcileKubernetesAgentctlInstance(ctx, control, request) + + require.NoError(t, err) + require.Equal(t, &agentctl.CreateInstanceResponse{ID: "instance-1", Port: 41001}, response) + require.Equal(t, 1, control.getCalls) +} + +func TestKubernetesAgentctlCreateRejectsMismatchedReconciliation(t *testing.T) { + control := &kubernetesAmbiguousCreateControl{ + createErr: context.DeadlineExceeded, + info: &agentctl.InstanceInfo{ + ID: "instance-1", Port: 41001, WorkspacePath: "/foreign", + }, + } + request := &agentctl.CreateInstanceRequest{ID: "instance-1", WorkspacePath: dockerWorkspacePath} + + _, err := createOrReconcileKubernetesAgentctlInstance(context.Background(), control, request) + + require.ErrorContains(t, err, "workspace") +} + +func TestKubernetesInstanceLockSerializesSameInstanceAndCleansUp(t *testing.T) { + executor := NewKubernetesExecutor(nil, newTestLogger()) + firstUnlock := executor.lockInstance("instance-1") + secondAcquired := make(chan struct{}) + secondReleased := make(chan struct{}) + go func() { + secondUnlock := executor.lockInstance("instance-1") + close(secondAcquired) + secondUnlock() + close(secondReleased) + }() + + select { + case <-secondAcquired: + t.Fatal("same-instance operation was not serialized") + default: + } + firstUnlock() + select { + case <-secondReleased: + case <-time.After(time.Second): + t.Fatal("same-instance waiter did not acquire the released lock") + } + + executor.mu.Lock() + defer executor.mu.Unlock() + require.Empty(t, executor.locks) +} + +func TestKubernetesCreateInstanceProvisionsBootstrapsAndForwardsAgentctl(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + execs := &recordingKubernetesExec{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }} + executor := NewKubernetesExecutor(nil, newTestLogger()) + executor.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{ + resources: resources, + streams: kubeexecutor.NewStreamOperations(execs, forwards), + }, nil + } + executor.resolveBinary = func(kubeexecutor.Platform) ([]byte, error) { + return []byte("agentctl-binary"), nil + } + req := validKubernetesCreateRequest() + req.Metadata["workspace.mode"] = "managed_pvc" + req.Metadata["workspace.size"] = "1Gi" + req.Metadata["workspace.access_modes"] = `["ReadWriteOnce"]` + + instance, err := executor.CreateInstance(context.Background(), req) + + require.NoError(t, err) + require.Equal(t, "pod-uid", instance.Metadata[MetadataKeyKubernetesPodUID]) + require.Equal(t, "pvc-uid", instance.Metadata[MetadataKeyKubernetesPVCUID]) + require.Equal(t, true, instance.Metadata[MetadataKeyKubernetesPVCCreated]) + require.Equal(t, "41001", instance.Metadata[MetadataKeyKubernetesAgentctlRemotePort]) + require.Equal(t, "/workspace", instance.WorkspacePath) + require.Equal(t, true, instance.Metadata[MetadataKeyIsRemote]) + require.Equal(t, "handshake-token", instance.AuthToken) + require.NotEmpty(t, instance.BootstrapNonce) + require.NotNil(t, instance.Client) + require.Len(t, resources.createdPVCs, 1) + require.Len(t, resources.createdPods, 1) + require.Len(t, execs.requests, 4, "binary, runtime config, auth config, then start signal") + require.Equal(t, []uint16{uint16(kubeexecutor.DefaultAgentctlPort), 41001}, forwards.remotePorts()) + for _, request := range forwards.requests { + require.Equal(t, "127.0.0.1", request.LocalAddress) + } +} + +func TestKubernetesCreateInstanceReconnectsExactPodWithFreshForward(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initialExecs := &recordingKubernetesExec{} + initial := newFakeKubernetesExecutor(t, resources, initialExecs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + + reconnectExecs := &recordingKubernetesExec{} + reconnectForwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{41001: instancePort}} + restartedBackend := NewKubernetesExecutor(nil, newTestLogger()) + restartedBackend.clientFactory = func(kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + return &kubernetesRuntimeClient{ + resources: resources, + streams: kubeexecutor.NewStreamOperations(reconnectExecs, reconnectForwards), + }, nil + } + restartedBackend.resolveBinary = func(kubeexecutor.Platform) ([]byte, error) { return []byte("unused"), nil } + reconnectRequest := validKubernetesCreateRequest() + reconnectRequest.InstanceID = "new-execution-id" + reconnectRequest.PreviousExecutionID = created.InstanceID + reconnectRequest.AuthToken = created.AuthToken + reconnectRequest.BootstrapNonce = created.BootstrapNonce + for key, value := range created.Metadata { + reconnectRequest.Metadata[key] = value + } + + reconnected, err := restartedBackend.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, "new-execution-id", reconnected.InstanceID) + require.Equal(t, created.AuthToken, reconnected.AuthToken) + require.Equal(t, created.Metadata[MetadataKeyKubernetesPodUID], reconnected.Metadata[MetadataKeyKubernetesPodUID]) + require.Equal(t, created.InstanceID, reconnected.Metadata[MetadataKeyKubernetesAgentctlInstanceID]) + require.Empty(t, reconnectExecs.requests) + require.Len(t, resources.createdPods, 1) + require.Len(t, resources.createdPVCs, 0) + require.Equal(t, []uint16{41001}, reconnectForwards.remotePorts()) +} + +func TestKubernetesCreateInstanceRehandshakesAfterMainContainerRestart(t *testing.T) { + initialControlPort := startKubernetesAgentctlServerWithToken(t, true, 41001, "initial-token") + initialInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): initialControlPort, + 41001: initialInstancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + require.Equal(t, "initial-token", created.AuthToken) + + staleInstancePort := startKubernetesUnhealthyServer(t) + restartedControlPort := startKubernetesAgentctlServerWithToken(t, true, 41002, "rotated-token") + restartedInstancePort := startKubernetesAgentctlServer(t, false, 0) + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + 41001: staleInstancePort, + uint16(kubeexecutor.DefaultAgentctlPort): restartedControlPort, + 41002: restartedInstancePort, + }} + execs := &recordingKubernetesExec{} + restarted := newFakeKubernetesExecutorWithForwarder(resources, execs, forwards) + + reconnected, err := restarted.CreateInstance(context.Background(), kubernetesReconnectRequest(created)) + + require.NoError(t, err) + require.Equal(t, "rotated-token", reconnected.AuthToken) + require.Equal(t, "41002", reconnected.Metadata[MetadataKeyKubernetesAgentctlRemotePort]) + require.Equal(t, created.InstanceID, reconnected.Metadata[MetadataKeyKubernetesAgentctlInstanceID]) + require.Equal(t, []uint16{41001, uint16(kubeexecutor.DefaultAgentctlPort), 41002}, forwards.remotePorts()) + require.True(t, forwards.sessions[0].isClosed(), "the stale instance forward must close") + require.True(t, forwards.sessions[1].isClosed(), "the temporary control forward must close") + require.False(t, forwards.sessions[2].isClosed(), "the rotated instance forward must remain live") + require.Empty(t, execs.requests, "a container restart reuses Pod-scoped bootstrap files") +} + +func TestKubernetesCreateInstanceUsesCurrentConnectionConfigOnReconnect(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{41001: instancePort}} + restarted := newFakeKubernetesExecutorWithForwarder(resources, &recordingKubernetesExec{}, forwards) + reconnectRequest := kubernetesReconnectRequest(created) + reconnectRequest.Metadata[MetadataKeyKubernetesAuthMode] = "kubeconfig" + reconnectRequest.Metadata[MetadataKeyKubernetesKubeconfigPath] = "/etc/kandev/current-kubeconfig" + reconnectRequest.Metadata[MetadataKeyKubernetesKubeContext] = "current-context" + reconnectRequest.Metadata[MetadataKeyKubernetesRequestTimeoutSeconds] = "45" + + reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, "/etc/kandev/current-kubeconfig", reconnected.Metadata[MetadataKeyKubernetesKubeconfigPath]) + require.Equal(t, "current-context", reconnected.Metadata[MetadataKeyKubernetesKubeContext]) + require.Equal(t, "45", reconnected.Metadata[MetadataKeyKubernetesRequestTimeoutSeconds]) + require.NotEqual(t, created.Metadata[MetadataKeyKubernetesExecutorConfigHash], reconnected.Metadata[MetadataKeyKubernetesExecutorConfigHash]) + require.NotEmpty(t, resources.getPodRequests) + require.NotEmpty(t, forwards.requests) +} + +func TestKubernetesCreateInstanceUsesRecordedNamespaceAfterConnectionNamespaceChanges(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{41001: instancePort}) + reconnectRequest := kubernetesReconnectRequest(created) + reconnectRequest.Metadata[MetadataKeyKubernetesConfigNamespace] = "other-agents" + + reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, "other-agents", reconnected.Metadata[MetadataKeyKubernetesConfigNamespace]) + require.Equal(t, "kandev-agents", reconnected.Metadata[MetadataKeyKubernetesNamespace]) + require.Contains(t, resources.getPodRequests, "kandev-agents/"+created.Metadata[MetadataKeyKubernetesPodName].(string)) +} + +func TestKubernetesCreateInstanceRejectsIncompleteRecordedResourceIdentity(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{41001: instancePort}) + reconnectRequest := kubernetesReconnectRequest(created) + delete(reconnectRequest.Metadata, MetadataKeyKubernetesResourceInstanceID) + + _, err = restarted.CreateInstance(context.Background(), reconnectRequest) + + require.ErrorContains(t, err, "resource identity") +} + +func TestKubernetesCreateInstanceRejectsMismatchedRecordedAgentctlInstance(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + + reconnectRequest := kubernetesReconnectRequest(created) + reconnectRequest.Metadata[MetadataKeyKubernetesAgentctlInstanceID] = "redirected-agentctl-instance" + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + resources.mu.Lock() + requestsBefore := len(resources.getPodRequests) + resources.mu.Unlock() + + _, err = restarted.CreateInstance(context.Background(), reconnectRequest) + + require.ErrorContains(t, err, "agentctl instance") + resources.mu.Lock() + require.Len(t, resources.getPodRequests, requestsBefore, "identity mismatch must fail before Pod access") + resources.mu.Unlock() +} + +func TestKubernetesRecordedResourceIdentityRejectsRequestMismatch(t *testing.T) { + tests := []struct { + name string + mutate func(*ExecutorCreateRequest) + }{ + {name: "task", mutate: func(req *ExecutorCreateRequest) { req.TaskID = "other-task" }}, + {name: "session", mutate: func(req *ExecutorCreateRequest) { req.SessionID = "other-session" }}, + {name: "executor", mutate: func(req *ExecutorCreateRequest) { req.Metadata["executor_id"] = "other-executor" }}, + {name: "profile", mutate: func(req *ExecutorCreateRequest) { + req.Metadata[MetadataKeyExecutorProfileID] = "other-profile" + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := validKubernetesCreateRequest() + req.Metadata[MetadataKeyKubernetesResourceExecutorID] = "executor-1" + req.Metadata[MetadataKeyKubernetesResourceProfileID] = "profile-1" + req.Metadata[MetadataKeyKubernetesResourceInstanceID] = "resource-instance-1" + req.Metadata[MetadataKeyKubernetesResourceTaskID] = "task-1" + req.Metadata[MetadataKeyKubernetesResourceSessionID] = "session-1" + req.Metadata[MetadataKeyKubernetesResourceEnvironmentID] = "environment-1" + test.mutate(req) + + _, err := kubernetesRecordedResourceIdentity(req, true) + + require.ErrorContains(t, err, "does not match") + }) + } +} + +func TestKubernetesCreateInstanceRejectsLostEmptyDirPod(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + created, err := initial.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + resources.mu.Lock() + resources.pod = nil + resources.mu.Unlock() + + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + req := kubernetesReconnectRequest(created) + _, err = restarted.CreateInstance(context.Background(), req) + + require.ErrorContains(t, err, "emptyDir workspace is unrecoverable") + require.Len(t, resources.createdPods, 1) +} + +func TestKubernetesCreateInstanceRebuildsLostPodAgainstVerifiedManagedPVC(t *testing.T) { + initialControlPort := startKubernetesAgentctlServer(t, true, 41001) + initialInstancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): initialControlPort, + 41001: initialInstancePort, + }) + initialRequest := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(initialRequest) + created, err := initial.CreateInstance(context.Background(), initialRequest) + require.NoError(t, err) + created.Metadata[MetadataKeyKubernetesContainerRestartCount] = "7" + resources.mu.Lock() + resources.pod = nil + resources.nextPodUID = "replacement-pod-uid" + resources.mu.Unlock() + + restartControlPort := startKubernetesAgentctlServer(t, true, 41002) + restartInstancePort := startKubernetesAgentctlServer(t, false, 0) + reconnectExecs := &recordingKubernetesExec{} + restarted := newFakeKubernetesExecutor(t, resources, reconnectExecs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): restartControlPort, + 41002: restartInstancePort, + }) + reconnectRequest := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(reconnectRequest) + reconnectRequest.InstanceID = "new-execution-id" + reconnectRequest.PreviousExecutionID = created.InstanceID + reconnectRequest.AuthToken = created.AuthToken + reconnectRequest.BootstrapNonce = created.BootstrapNonce + for key, value := range created.Metadata { + reconnectRequest.Metadata[key] = value + } + + reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest) + + require.NoError(t, err) + require.Equal(t, "replacement-pod-uid", reconnected.Metadata[MetadataKeyKubernetesPodUID]) + require.Equal(t, "0", reconnected.Metadata[MetadataKeyKubernetesContainerRestartCount]) + require.Equal(t, "pvc-uid", reconnected.Metadata[MetadataKeyKubernetesPVCUID]) + require.Len(t, resources.createdPVCs, 1, "resume must reuse the recorded PVC") + require.Len(t, resources.createdPods, 2) + require.Len(t, reconnectExecs.requests, 4, "replacement Pod needs full bootstrap materialization") +} + +func TestKubernetesStopInstancePreservesOrdinaryStopAndForceCleansManagedResources(t *testing.T) { + t.Run("ordinary stop", func(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + forwards := &recordingKubernetesForwarder{localPorts: map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }} + executor := newFakeKubernetesExecutorWithForwarder(resources, &recordingKubernetesExec{}, forwards) + instance, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + + require.NoError(t, executor.StopInstance(context.Background(), instance, false)) + + require.Empty(t, resources.deletedPods) + require.Empty(t, resources.deletedPVCs) + require.True(t, forwards.lastSession().isClosed(), "ordinary stop must close the process-local forward") + }) + + t.Run("force cleanup", func(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + + require.NoError(t, executor.StopInstance(context.Background(), instance, true)) + + require.Equal(t, []string{instance.Metadata[MetadataKeyKubernetesPodName].(string) + ":pod-uid"}, resources.deletedPods) + require.Equal(t, []string{instance.Metadata[MetadataKeyKubernetesPVCName].(string) + ":pvc-uid"}, resources.deletedPVCs) + require.Equal(t, []string{"pod-rv-1"}, resources.deletedPodResourceVersions) + require.Equal(t, []string{"pvc-rv-1"}, resources.deletedPVCResourceVersions) + require.Equal(t, []string{"pod", "pvc"}, resources.deletionOrder) + }) +} + +func TestDeleteKubernetesResourcesWaitsForPodAndPVCRemoval(t *testing.T) { + resources := &fakeKubernetesResources{ + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "kandev-agents", Name: "pod-1", UID: "pod-uid", ResourceVersion: "pod-rv-1", + }}, + pvc: &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: "kandev-agents", Name: "pvc-1", UID: "pvc-uid", ResourceVersion: "pvc-rv-1", + }}, + retainPodAfterDelete: true, retainPVCAfterDelete: true, + podPostDeleteGetsUntilAbsent: 1, pvcPostDeleteGetsUntilAbsent: 1, + } + identity := kubeexecutor.ResourceIdentity{ + ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "instance-1", + TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", + } + labels, err := kubeexecutor.OwnershipLabels(identity) + require.NoError(t, err) + resources.pod.Labels = labels + resources.pvc.Labels = labels + recorded := kubernetesRecordedState{ + namespace: "kandev-agents", podName: "pod-1", podUID: "pod-uid", + pvcName: "pvc-1", pvcUID: "pvc-uid", pvcCreated: true, + inventoryState: KubernetesInventoryStateReady, + } + + require.NoError(t, deleteKubernetesResources(context.Background(), resources, recorded, identity)) + + resources.mu.Lock() + defer resources.mu.Unlock() + require.Nil(t, resources.pod, "successful cleanup must observe Pod absence") + require.Nil(t, resources.pvc, "successful cleanup must observe PVC absence") + require.Equal(t, []string{"pod", "pvc"}, resources.deletionOrder) +} + +func TestDeleteKubernetesPodFailsWhenTerminationDoesNotComplete(t *testing.T) { + resources := &fakeKubernetesResources{ + pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Namespace: "kandev-agents", Name: "pod-1", UID: "pod-uid", ResourceVersion: "pod-rv-1", + }}, + retainPodAfterDelete: true, podPostDeleteGetsUntilAbsent: -1, + } + identity := kubeexecutor.ResourceIdentity{ + ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "instance-1", + TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", + } + labels, err := kubeexecutor.OwnershipLabels(identity) + require.NoError(t, err) + resources.pod.Labels = labels + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + err = deleteKubernetesPodIfExact(ctx, resources, resources.pod.DeepCopy(), identity) + + require.ErrorIs(t, err, context.DeadlineExceeded) + resources.mu.Lock() + defer resources.mu.Unlock() + require.NotNil(t, resources.pod, "stalled deletion must remain tracked") +} + +func TestKubernetesStopInstanceRunsTerminalCleanupScriptAfterExactVerification(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, resources, execs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + instance.StopReason = StopReasonTaskDeleted + instance.Metadata[MetadataKeyCleanupScript] = "echo cleanup {{workspace.path}}" + verifiedBeforeCleanup := false + resources.beforeDeletePod = func(*corev1.Pod) { + execs.mu.Lock() + defer execs.mu.Unlock() + for _, recorded := range execs.requests { + command := strings.Join(recorded.request.Command, " ") + if strings.Contains(command, "echo cleanup") && strings.Contains(command, "/workspace") { + verifiedBeforeCleanup = true + } + } + } + + require.NoError(t, executor.StopInstance(context.Background(), instance, true)) + + require.True(t, verifiedBeforeCleanup, "cleanup script must execute after verification and before Pod deletion") + require.Equal(t, []string{"pod", "pvc"}, resources.deletionOrder) +} + +func TestKubernetesStopInstanceCleanupScriptFailureDoesNotBlockExactDeletion(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + execs := &recordingKubernetesExec{} + executor := newFakeKubernetesExecutor(t, resources, execs, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + instance.StopReason = StopReasonTaskDeleted + instance.Metadata[MetadataKeyCleanupScript] = "echo cleanup" + execs.err = errors.New("cleanup command failed") + + require.NoError(t, executor.StopInstance(context.Background(), instance, true)) + + execs.mu.Lock() + cleanupAttempted := false + for _, recorded := range execs.requests { + cleanupAttempted = cleanupAttempted || strings.Contains(strings.Join(recorded.request.Command, " "), "echo cleanup") + } + execs.mu.Unlock() + require.True(t, cleanupAttempted) + require.Equal(t, []string{"pod", "pvc"}, resources.deletionOrder) +} + +func TestKubernetesStopInstanceValidatesPVCBeforeDeletingPod(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + resources.mu.Lock() + resources.pvc.UID = "same-name-foreign-uid" + resources.mu.Unlock() + + err = executor.StopInstance(context.Background(), instance, true) + + require.ErrorContains(t, err, "PVC UID") + require.Empty(t, resources.deletedPods, "cleanup must validate every identity before deleting anything") + require.Empty(t, resources.deletedPVCs) +} + +func TestKubernetesStopInstanceFailsClosedOnConcurrentPodMutation(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + resources.mu.Lock() + resources.beforeDeletePod = func(pod *corev1.Pod) { pod.ResourceVersion = "pod-rv-concurrent" } + resources.mu.Unlock() + + err = executor.StopInstance(context.Background(), instance, true) + + require.ErrorContains(t, err, "precondition failed") + require.Empty(t, resources.deletedPods) + require.Empty(t, resources.deletedPVCs) +} + +func TestKubernetesStopInstanceNeverDeletesExistingClaim(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{pvc: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "shared-workspace", Namespace: "kandev-agents", UID: "shared-pvc-uid"}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + }} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + req.Metadata[MetadataKeyKubernetesWorkspaceMode] = "existing_claim" + req.Metadata[MetadataKeyKubernetesWorkspaceClaimName] = "shared-workspace" + instance, err := executor.CreateInstance(context.Background(), req) + require.NoError(t, err) + + require.NoError(t, executor.StopInstance(context.Background(), instance, true)) + + require.Len(t, resources.deletedPods, 1) + require.Empty(t, resources.deletedPVCs) + resources.mu.Lock() + defer resources.mu.Unlock() + require.NotNil(t, resources.pvc, "existing claims are never lifecycle-owned") +} + +func TestKubernetesRemoteStatusUsesExactRecordedPodAndContainerState(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + executor := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + instance, err := executor.CreateInstance(context.Background(), validKubernetesCreateRequest()) + require.NoError(t, err) + resources.mu.Lock() + resources.pod.Status.Phase = corev1.PodPending + resources.pod.Status.Reason = "Scheduling" + resources.pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", RestartCount: 2, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "ImagePullBackOff", Message: "pull delayed"}}, + }} + resources.getPodRequests = nil + resources.mu.Unlock() + + provider, ok := interface{}(executor).(RemoteStatusProvider) + require.True(t, ok, "Kubernetes executor must expose remote Pod status") + status, err := provider.GetRemoteStatus(context.Background(), instance) + + require.NoError(t, err) + require.Equal(t, "starting", status.State) + require.Equal(t, "ImagePullBackOff", status.Details["reason"]) + require.Equal(t, int32(2), status.Details["restart_count"]) + require.Equal(t, + []string{instance.Metadata[MetadataKeyKubernetesNamespace].(string) + "/" + instance.Metadata[MetadataKeyKubernetesPodName].(string)}, + resources.getPodRequests, + ) + + resources.mu.Lock() + resources.pod.UID = "same-name-foreign-uid" + resources.mu.Unlock() + _, err = provider.GetRemoteStatus(context.Background(), instance) + require.ErrorContains(t, err, "UID") +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/liveness.go b/apps/backend/internal/agent/runtime/lifecycle/liveness.go index 34f1df01de..c134638f98 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/liveness.go +++ b/apps/backend/internal/agent/runtime/lifecycle/liveness.go @@ -16,10 +16,9 @@ import ( // the actual host-local probe stays in this platform-split package. // // - local/standalone: judged by local_pid on this host. -// - SSH/remote: Unknown here. The agent process lives on another host and its -// liveness is owned by the SSH executor's remote `kill -0` path; the SSH -// `pid` column is a REMOTE pid and must never be probed with a local -// os.FindProcess. +// - SSH/Kubernetes/remote: Unknown here. The agent process lives elsewhere; +// SSH owns its remote `kill -0` probe and Kubernetes owns Pod status. Their +// process identifiers must never be probed with a local os.FindProcess. // - docker / anything else: Unknown for now (out of scope for this batch). func RowProcessLiveness(row *models.ExecutorRunning) models.ProcessLiveness { if row == nil { @@ -42,8 +41,9 @@ func RowProcessLiveness(row *models.ExecutorRunning) models.ProcessLiveness { // isLocalRuntime reports whether a row's runtime runs its agent process on THIS // host, so a local os.FindProcess check is meaningful. Standalone is the only -// local runtime; SSH/docker/remote-docker/sprites run the process elsewhere. -// An empty/unknown runtime is treated as non-local (Unknown), never probed. +// local runtime; SSH/Kubernetes/docker/remote-docker/sprites run the process +// elsewhere. An empty/unknown runtime is treated as non-local (Unknown), never +// probed. func isLocalRuntime(runtime agentruntime.Runtime) bool { return runtime == agentruntime.RuntimeStandalone } diff --git a/apps/backend/internal/agent/runtime/lifecycle/liveness_test.go b/apps/backend/internal/agent/runtime/lifecycle/liveness_test.go index de03fd952e..8a1c170ac9 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/liveness_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/liveness_test.go @@ -59,6 +59,7 @@ func TestRowProcessLivenessUnknownCases(t *testing.T) { }{ {"nil", nil}, {"docker", &models.ExecutorRunning{Runtime: agentruntime.RuntimeDocker, LocalPID: os.Getpid()}}, + {"kubernetes", &models.ExecutorRunning{Runtime: agentruntime.RuntimeKubernetes, PID: os.Getpid(), LocalPID: os.Getpid()}}, {"empty-runtime", &models.ExecutorRunning{Runtime: "", LocalPID: os.Getpid()}}, {"local-no-handle", &models.ExecutorRunning{Runtime: agentruntime.RuntimeStandalone, LocalPID: 0}}, } diff --git a/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go index b12e457b1f..6665619a63 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go +++ b/apps/backend/internal/agent/runtime/lifecycle/managed_runtime_startup.go @@ -27,12 +27,17 @@ func (m *Manager) managedRuntimeNpmStartupFailure( initErr error, packageSpec string, ) *routingerr.Error { - if execution == nil || execution.agentctl == nil || initErr == nil { + if execution == nil || initErr == nil { + return nil + } + client, release := execution.AcquireAgentCtlClient() + defer release() + if client == nil { return nil } stderrCtx, cancel := context.WithTimeout(ctx, managedRuntimeStartupStderrTimeout) defer cancel() - lines, err := execution.agentctl.GetAgentStderr(stderrCtx) + lines, err := client.GetAgentStderr(stderrCtx) if err != nil { return nil } @@ -117,7 +122,12 @@ func (m *Manager) prepareManagedRuntimeStartupRetry( initErr error, agentConfig agents.Agent, ) (*managedRuntimeStartupRetry, bool) { - if execution == nil || !supportsManagedRuntimeCacheRepair(execution.RuntimeName) || execution.agentctl == nil { + if execution == nil || !supportsManagedRuntimeCacheRepair(execution.RuntimeName) { + return nil, false + } + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { return nil, false } managed, ok := agentConfig.(agents.ManagedNPMRuntimeAgent) @@ -157,19 +167,24 @@ func (m *Manager) stopAndRepairManagedRuntime( execution *AgentExecution, retry *managedRuntimeStartupRetry, ) (needsFailure bool, err error) { - if stopErr := execution.agentctl.Stop(ctx); stopErr != nil { + client, release := execution.AcquireAgentCtlClient() + defer release() + if client == nil { + return false, errors.New("managed runtime agentctl client is unavailable") + } + if stopErr := client.Stop(ctx); stopErr != nil { if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { return false, aborted } m.logger.Warn("managed runtime process stop failed before retry; continuing", zap.String("execution_id", execution.ID), zap.Error(stopErr)) } - execution.agentctl.CloseUpdatesStream() + client.CloseUpdatesStream() if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { return false, aborted } - if err := execution.agentctl.RepairManagedRuntimeCache(ctx, retry.packageSpec); err != nil { + if err := client.RepairManagedRuntimeCache(ctx, retry.packageSpec); err != nil { if aborted := managedRuntimeRecoveryAborted(ctx, m); aborted != nil { return false, aborted } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager.go b/apps/backend/internal/agent/runtime/lifecycle/manager.go index 5b333aad36..4b85efbe23 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager.go @@ -104,6 +104,7 @@ type Manager struct { // singleflight deduplicates concurrent GetOrEnsureExecution calls for the same session ensureExecutionGroup singleflight.Group + remoteRefreshGroup singleflight.Group // Background remote status polling remoteStatusPollInterval time.Duration diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_auth_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_auth_test.go index bb628fd664..2f2a2df1a3 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_auth_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_auth_test.go @@ -4,18 +4,52 @@ import ( "context" "errors" "fmt" + "strings" + "sync" "testing" + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/secrets" "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" + "github.com/stretchr/testify/require" ) // inMemorySecretStore implements secrets.SecretStore for testing. type inMemorySecretStore struct { - store map[string]*secrets.SecretWithValue - err error - revealErr error + mu sync.RWMutex + store map[string]*secrets.SecretWithValue + err error + revealErr error + rejectCanceledCalls bool +} + +type failNthCreateSecretStore struct { + *inMemorySecretStore + failAt int + calls int +} + +func (s *failNthCreateSecretStore) Create(ctx context.Context, secret *secrets.SecretWithValue) error { + s.calls++ + if s.calls == s.failAt { + return errors.New("injected secret create failure") + } + return s.inMemorySecretStore.Create(ctx, secret) +} + +type stopTrackingExecutor struct { + MockExecutor + stopCalls int + forces []bool +} + +func (e *stopTrackingExecutor) StopInstance(_ context.Context, _ *ExecutorInstance, force bool) error { + e.stopCalls++ + e.forces = append(e.forces, force) + return nil } var _ secrets.SecretStore = (*inMemorySecretStore)(nil) @@ -27,7 +61,12 @@ func newInMemorySecretStore() *inMemorySecretStore { } // Create stores the secret, returning the injected error when set. -func (s *inMemorySecretStore) Create(_ context.Context, secret *secrets.SecretWithValue) error { +func (s *inMemorySecretStore) Create(ctx context.Context, secret *secrets.SecretWithValue) error { + if s.rejectCanceledCalls && ctx.Err() != nil { + return ctx.Err() + } + s.mu.Lock() + defer s.mu.Unlock() if s.err != nil { return s.err } @@ -39,31 +78,67 @@ func (s *inMemorySecretStore) Create(_ context.Context, secret *secrets.SecretWi } // Get returns the stored secret for the given ID, or an error when absent. -func (s *inMemorySecretStore) Get(_ context.Context, id string) (*secrets.Secret, error) { +func (s *inMemorySecretStore) Get(ctx context.Context, id string) (*secrets.Secret, error) { + if s.rejectCanceledCalls && ctx.Err() != nil { + return nil, ctx.Err() + } + s.mu.RLock() + defer s.mu.RUnlock() if sw, ok := s.store[id]; ok { return &sw.Secret, nil } - return nil, fmt.Errorf("not found") + return nil, fmt.Errorf("%w: %s", secrets.ErrNotFound, id) } // Reveal returns the plaintext value for the given ID, or an error when absent. -func (s *inMemorySecretStore) Reveal(_ context.Context, id string) (string, error) { +func (s *inMemorySecretStore) Reveal(ctx context.Context, id string) (string, error) { + if s.rejectCanceledCalls && ctx.Err() != nil { + return "", ctx.Err() + } + s.mu.RLock() + defer s.mu.RUnlock() if s.revealErr != nil { return "", s.revealErr } if sw, ok := s.store[id]; ok { return sw.Value, nil } - return "", fmt.Errorf("not found") + return "", fmt.Errorf("%w: %s", secrets.ErrNotFound, id) } -// Update is a no-op for the in-memory store. -func (s *inMemorySecretStore) Update(_ context.Context, _ string, _ *secrets.UpdateSecretRequest) error { +// Update changes the stored name/value. +func (s *inMemorySecretStore) Update(ctx context.Context, id string, req *secrets.UpdateSecretRequest) error { + if s.rejectCanceledCalls && ctx.Err() != nil { + return ctx.Err() + } + s.mu.Lock() + defer s.mu.Unlock() + stored, ok := s.store[id] + if !ok { + return fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + if req.Name != nil { + stored.Name = *req.Name + } + if req.Value != nil { + stored.Value = *req.Value + } return nil } -// Delete is a no-op for the in-memory store. -func (s *inMemorySecretStore) Delete(_ context.Context, _ string) error { return nil } +// Delete removes a stored secret. +func (s *inMemorySecretStore) Delete(ctx context.Context, id string) error { + if s.rejectCanceledCalls && ctx.Err() != nil { + return ctx.Err() + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.store[id]; !ok { + return fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + delete(s.store, id) + return nil +} // List returns no items for the in-memory store. func (s *inMemorySecretStore) List(_ context.Context) ([]*secrets.SecretListItem, error) { @@ -75,6 +150,8 @@ func (s *inMemorySecretStore) Close() error { return nil } // ListScoped returns stored secrets filtered by the requested scope and workspace. func (s *inMemorySecretStore) ListScoped(_ context.Context, opts secrets.SecretListOptions) ([]*secrets.SecretListItem, error) { + s.mu.RLock() + defer s.mu.RUnlock() items := make([]*secrets.SecretListItem, 0, len(s.store)) for _, stored := range s.store { scope := stored.Scope @@ -129,6 +206,8 @@ func (s *inMemorySecretStore) RevealForWorkspace(ctx context.Context, id, worksp // DeleteWorkspaceSecrets removes all secrets belonging to the given workspace. func (s *inMemorySecretStore) DeleteWorkspaceSecrets(_ context.Context, workspaceID string) error { + s.mu.Lock() + defer s.mu.Unlock() for id, stored := range s.store { if stored.Scope == secrets.ScopeWorkspace && stored.WorkspaceID == workspaceID { delete(s.store, id) @@ -411,3 +490,406 @@ func TestResolveLaunchAuthToken(t *testing.T) { } }) } + +func TestRegisterKubernetesExecutionPersistsRequiredSecretReferencesBeforeSuccess(t *testing.T) { + log, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + store := newInMemorySecretStore() + writer := &captureExecutorRunningWriter{} + mgr := newTestManager(t) + mgr.SetSecretStore(store) + mgr.SetExecutorRunningWriter(writer) + execution := &AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusStarting, agentctl: newReadyAgentctlClient(t, log), + metadata: map[string]interface{}{}, + } + instance := &ExecutorInstance{ + InstanceID: "execution-1", AuthToken: "agentctl-token", BootstrapNonce: "bootstrap-nonce", + } + + err := mgr.registerAndPublishExecution( + context.Background(), execution, + &MockExecutor{name: executor.NameKubernetes}, instance, execution.SessionID, + ) + if err != nil { + t.Fatalf("registerAndPublishExecution() error = %v", err) + } + if writer.running == nil { + t.Fatal("executors_running row was not persisted") + } + for _, key := range []string{MetadataKeyAuthTokenSecret, MetadataKeyBootstrapNonceSecret} { + secretID := getMetadataString(writer.running.Metadata, key) + if secretID == "" { + t.Fatalf("persisted %s = empty", key) + } + if !secrets.IsInternalID(secretID) { + t.Fatalf("persisted %s = %q, want internal runtime secret ID", key, secretID) + } + } +} + +func TestRegisterNonKubernetesExecutionRollsBackWhenRuntimeSecretPersistenceFails(t *testing.T) { + store := newInMemorySecretStore() + store.err = errors.New("secret store unavailable") + mgr := newTestManager(t) + mgr.SetSecretStore(store) + execution := &AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeDocker, + Status: v1.AgentStatusStarting, metadata: map[string]interface{}{}, + } + instance := &ExecutorInstance{InstanceID: execution.ID, AuthToken: "agentctl-token"} + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameDocker}} + + err := mgr.registerAndPublishExecution( + context.Background(), execution, backend, instance, execution.SessionID, + ) + + require.ErrorContains(t, err, "persist runtime secret") + require.Equal(t, 1, backend.stopCalls) + _, exists := mgr.executionStore.Get(execution.ID) + require.False(t, exists) +} + +func TestRegisterKubernetesExecutionRollsBackCreatedSecretsOnPersistenceFailure(t *testing.T) { + log, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + store := &failNthCreateSecretStore{inMemorySecretStore: newInMemorySecretStore(), failAt: 2} + mgr := newTestManager(t) + mgr.SetSecretStore(store) + mgr.SetExecutorRunningWriter(&captureExecutorRunningWriter{}) + execution := &AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusStarting, agentctl: newReadyAgentctlClient(t, log), + metadata: map[string]interface{}{}, + } + instance := &ExecutorInstance{ + InstanceID: "execution-1", AuthToken: "agentctl-token", BootstrapNonce: "bootstrap-nonce", + } + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + + err := mgr.registerAndPublishExecution(context.Background(), execution, backend, instance, execution.SessionID) + if err == nil || !strings.Contains(err.Error(), "bootstrap nonce") { + t.Fatalf("registerAndPublishExecution() error = %v, want bootstrap nonce persistence failure", err) + } + if backend.stopCalls != 1 { + t.Fatalf("StopInstance() calls = %d, want 1", backend.stopCalls) + } + if len(store.store) != 0 { + t.Fatalf("runtime secrets after rollback = %#v, want none", store.store) + } + if _, exists := mgr.executionStore.Get(execution.ID); exists { + t.Fatal("failed Kubernetes launch remained registered") + } +} + +func TestRegisterKubernetesExecutionRollsBackSecretsWhenDurableRowWriteFails(t *testing.T) { + log, _ := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + store := newInMemorySecretStore() + mgr := newTestManager(t) + mgr.SetSecretStore(store) + mgr.SetExecutorRunningWriter(&captureExecutorRunningWriter{upsertErr: errors.New("injected row failure")}) + execution := &AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusStarting, agentctl: newReadyAgentctlClient(t, log), + metadata: map[string]interface{}{}, + } + instance := &ExecutorInstance{ + InstanceID: "execution-1", AuthToken: "agentctl-token", BootstrapNonce: "bootstrap-nonce", + } + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + + err := mgr.registerAndPublishExecution(context.Background(), execution, backend, instance, execution.SessionID) + if err == nil || !strings.Contains(err.Error(), "persist execution registration") { + t.Fatalf("registerAndPublishExecution() error = %v, want row persistence failure", err) + } + if backend.stopCalls != 1 { + t.Fatalf("StopInstance() calls = %d, want 1", backend.stopCalls) + } + if len(store.store) != 0 { + t.Fatalf("runtime secrets after row rollback = %#v, want none", store.store) + } +} + +func TestCreateWorkspaceKubernetesExecutionPersistsRequiredSecretReferencesBeforeSuccess(t *testing.T) { + log := newTestLogger() + backend := &createInstanceExecutor{ + MockExecutor: MockExecutor{name: executor.NameKubernetes}, + client: newReadyAgentctlClient(t, log), + authToken: "agentctl-token", + nonce: "bootstrap-nonce", + } + executors := NewExecutorRegistry(log) + executors.Register(backend) + mgr := NewManager( + newTestRegistry(), &MockEventBus{}, executors, &MockCredentialsManager{}, + &MockProfileResolver{}, nil, ExecutorFallbackWarn, "", log, + ) + cleanupManagerStopCh(t, mgr) + mgr.SetSecretStore(newInMemorySecretStore()) + writer := &captureExecutorRunningWriter{} + mgr.SetExecutorRunningWriter(writer) + + _, err := mgr.createExecution(context.Background(), "task-1", &WorkspaceInfo{ + SessionID: "session-1", TaskEnvironmentID: "environment-1", + AgentID: "auggie", AgentProfileID: "profile-1", + ExecutorType: string(models.ExecutorTypeKubernetes), WorkspacePath: "/workspace", + }) + if err != nil { + t.Fatalf("createExecution() error = %v", err) + } + for _, key := range []string{MetadataKeyAuthTokenSecret, MetadataKeyBootstrapNonceSecret} { + if got := getMetadataString(writer.running.Metadata, key); got == "" { + t.Fatalf("persisted %s = empty", key) + } + } +} + +func TestStopKubernetesExecutionDeletesRuntimeSecretsAfterTerminalCleanup(t *testing.T) { + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:execution-1:agentctl-auth": "agentctl-token", + "kandev-runtime:execution-1:agentctl-bootstrap": "bootstrap-nonce", + } { + if err := store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + }); err != nil { + t.Fatalf("seed runtime secret: %v", err) + } + } + log := newTestLogger() + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + executors := NewExecutorRegistry(log) + executors.Register(backend) + mgr := NewManager( + newTestRegistry(), &MockEventBus{}, executors, nil, nil, nil, + ExecutorFallbackWarn, "", log, + ) + cleanupManagerStopCh(t, mgr) + mgr.SetSecretStore(store) + if err := mgr.executionStore.Add(&AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, Status: v1.AgentStatusRunning, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:execution-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:execution-1:agentctl-bootstrap", + }, + }); err != nil { + t.Fatalf("add execution: %v", err) + } + + err := mgr.StopAgentWithReason(context.Background(), "execution-1", StopReasonTaskDeleted, true) + + if err != nil { + t.Fatalf("StopAgentWithReason() error = %v", err) + } + if len(store.store) != 0 { + t.Fatalf("runtime secrets after cleanup = %#v, want none", store.store) + } +} + +func TestStopKubernetesExecutionDeletesRuntimeSecretsAfterRequestCancellation(t *testing.T) { + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:execution-1:agentctl-auth": "agentctl-token", + "kandev-runtime:execution-1:agentctl-bootstrap": "bootstrap-nonce", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + store.rejectCanceledCalls = true + log := newTestLogger() + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + executors := NewExecutorRegistry(log) + executors.Register(backend) + mgr := NewManager( + newTestRegistry(), &MockEventBus{}, executors, nil, nil, nil, + ExecutorFallbackWarn, "", log, + ) + cleanupManagerStopCh(t, mgr) + mgr.SetSecretStore(store) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, Status: v1.AgentStatusRunning, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:execution-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:execution-1:agentctl-bootstrap", + }, + })) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := mgr.StopAgentWithReason(ctx, "execution-1", StopReasonTaskDeleted, true) + + require.NoError(t, err) + require.Empty(t, store.store) +} + +func TestForceStopKubernetesExecutionDeletesRuntimeSecretsWithoutTerminalReason(t *testing.T) { + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:execution-1:agentctl-auth": "agentctl-token", + "kandev-runtime:execution-1:agentctl-bootstrap": "bootstrap-nonce", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + log := newTestLogger() + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + executors := NewExecutorRegistry(log) + executors.Register(backend) + mgr := NewManager( + newTestRegistry(), &MockEventBus{}, executors, nil, nil, nil, + ExecutorFallbackWarn, "", log, + ) + cleanupManagerStopCh(t, mgr) + mgr.SetSecretStore(store) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, Status: v1.AgentStatusRunning, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:execution-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:execution-1:agentctl-bootstrap", + }, + })) + + err := mgr.StopAgent(context.Background(), "execution-1", true) + + require.NoError(t, err) + require.Empty(t, store.store, "forced resource cleanup must also remove runtime secrets") +} + +func TestDeleteKubernetesRuntimeSecretsDerivesCanonicalRefsForProvisionalInventory(t *testing.T) { + for _, tc := range []struct { + name string + state string + wantDeleted bool + }{ + {name: "crash window", state: KubernetesInventoryStatePodAdmitted, wantDeleted: true}, + {name: "ready row without refs", state: KubernetesInventoryStateReady, wantDeleted: false}, + } { + t.Run(tc.name, func(t *testing.T) { + store := newInMemorySecretStore() + for _, id := range []string{ + "kandev-runtime:resource-instance-1:agentctl-auth", + "kandev-runtime:resource-instance-1:agentctl-bootstrap", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: "secret", + })) + } + mgr := newTestManager(t) + mgr.SetSecretStore(store) + + err := mgr.deleteKubernetesRuntimeSecrets(context.Background(), map[string]interface{}{ + MetadataKeyKubernetesInventoryState: tc.state, + MetadataKeyKubernetesResourceInstanceID: "resource-instance-1", + }) + + require.NoError(t, err) + store.mu.RLock() + defer store.mu.RUnlock() + if tc.wantDeleted { + require.Empty(t, store.store) + } else { + require.Len(t, store.store, 2) + } + }) + } +} + +func TestRegisteredKubernetesLaunchRollbackDeletesSecretsBeforeReleasingInventory(t *testing.T) { + store := newInMemorySecretStore() + for id := range map[string]struct{}{ + "kandev-runtime:execution-1:agentctl-auth": {}, "kandev-runtime:execution-1:agentctl-bootstrap": {}, + } { + if err := store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: "secret", + }); err != nil { + t.Fatalf("seed runtime secret: %v", err) + } + } + mgr := newTestManager(t) + mgr.SetSecretStore(store) + execution := &AgentExecution{ + ID: "execution-1", SessionID: "session-1", RuntimeName: agentruntime.RuntimeKubernetes, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:execution-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:execution-1:agentctl-bootstrap", + }, + } + releases := 0 + instance := &ExecutorInstance{ReleaseRuntimeInventory: func(context.Context) error { + releases++ + if len(store.store) != 0 { + return errors.New("inventory released before secrets were deleted") + } + return nil + }} + + err := mgr.stopRegisteredLaunchRuntime( + &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}}, + instance, + execution, + ) + + if err != nil { + t.Fatalf("stopRegisteredLaunchRuntime() error = %v", err) + } + if releases != 1 { + t.Fatalf("runtime inventory releases = %d, want 1", releases) + } +} + +func TestRegisteredKubernetesResumeRollbackPreservesRetainedRuntimeAndSecrets(t *testing.T) { + store := newInMemorySecretStore() + for id := range map[string]struct{}{ + "kandev-runtime:execution-1:agentctl-auth": {}, "kandev-runtime:execution-1:agentctl-bootstrap": {}, + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: "secret", + })) + } + mgr := newTestManager(t) + mgr.SetSecretStore(store) + execution := &AgentExecution{ + ID: "execution-1", SessionID: "session-1", RuntimeName: agentruntime.RuntimeKubernetes, + isResumedSession: true, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:execution-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:execution-1:agentctl-bootstrap", + }, + } + backend := &stopTrackingExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + + err := mgr.stopRegisteredLaunchRuntime(backend, &ExecutorInstance{}, execution) + + require.NoError(t, err) + require.Equal(t, []bool{false}, backend.forces, "resume rollback must only close its local client/forward") + require.Len(t, store.store, 2, "retained runtime secrets remain authoritative") +} + +func TestFinishedKubernetesResumeRollbackPreservesDurableInventoryRow(t *testing.T) { + mgr := newTestManager(t) + writer := &captureExecutorRunningWriter{prior: &models.ExecutorRunning{ + SessionID: "session-1", + AgentExecutionID: "execution-1", + Runtime: agentruntime.RuntimeKubernetes, + }} + mgr.SetExecutorRunningWriter(writer) + execution := &AgentExecution{ + ID: "execution-1", SessionID: "session-1", RuntimeName: agentruntime.RuntimeKubernetes, + isResumedSession: true, + } + require.NoError(t, mgr.executionStore.Add(execution)) + + mgr.finishRegisteredLaunchRollback(execution, true, true) + + require.Zero(t, writer.deleteCalls, "retained Kubernetes inventory must remain available to terminal cleanup") + _, exists := mgr.executionStore.Get(execution.ID) + require.False(t, exists) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_base_branches.go b/apps/backend/internal/agent/runtime/lifecycle/manager_base_branches.go index 3319f8e35c..e552ee282e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_base_branches.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_base_branches.go @@ -79,11 +79,13 @@ func (m *Manager) PushBaseBranchesForTask(ctx context.Context, taskID string, br if exec.TaskID != taskID { continue } - client := exec.GetAgentCtlClient() + client, releaseClient := exec.AcquireAgentCtlClient() if client == nil { continue } - if err := client.SetBaseBranches(ctx, branches); err != nil { + err := client.SetBaseBranches(ctx, branches) + releaseClient() + if err != nil { m.logger.Warn("failed to push base branches to agentctl", zap.String("task_id", taskID), zap.String("execution_id", exec.ID), diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_boot_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_boot_test.go index 5a9a9a5a0f..11a36ac504 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_boot_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_boot_test.go @@ -74,7 +74,7 @@ func TestFinalizeBootMessage_Success(t *testing.T) { } stopCh := make(chan struct{}) - mgr.finalizeBootMessage(nil, msg, stopCh, nil, "exited") + mgr.finalizeBootMessage(nil, msg, stopCh, "exited") // Verify stop channel was closed select { @@ -112,7 +112,7 @@ func TestFinalizeBootMessage_Failed(t *testing.T) { } stopCh := make(chan struct{}) - mgr.finalizeBootMessage(nil, msg, stopCh, nil, "failed") + mgr.finalizeBootMessage(nil, msg, stopCh, "failed") lastMsg := bootSvc.getLastUpdatedMessage() if lastMsg == nil { @@ -134,7 +134,7 @@ func TestFinalizeBootMessage_NilMessage(t *testing.T) { mgr.bootMessageService = bootSvc // Should not panic with nil message - mgr.finalizeBootMessage(nil, nil, nil, nil, "exited") + mgr.finalizeBootMessage(nil, nil, nil, "exited") bootSvc.mu.Lock() defer bootSvc.mu.Unlock() @@ -153,7 +153,7 @@ func TestFinalizeBootMessage_NilService(t *testing.T) { } // Should not panic with nil service - mgr.finalizeBootMessage(nil, msg, nil, nil, "exited") + mgr.finalizeBootMessage(nil, msg, nil, "exited") } func TestCreateBootMessage_MarksResumedSession(t *testing.T) { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_comparison_targets.go b/apps/backend/internal/agent/runtime/lifecycle/manager_comparison_targets.go index da3da9ac39..766e1a61c5 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_comparison_targets.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_comparison_targets.go @@ -56,13 +56,14 @@ func (m *Manager) PushComparisonTargetsForTask(ctx context.Context, taskID strin if exec.TaskID != taskID { continue } - client := exec.GetAgentCtlClient() + client, releaseClient := exec.AcquireAgentCtlClient() if client == nil { continue } pushCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) err := client.SetComparisonTargets(pushCtx, targets) cancel() + releaseClient() if err != nil { m.logger.Warn("failed to push comparison targets to agentctl", zap.String("task_id", taskID), diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go b/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go index e2c8cc2f9d..f5c4742167 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_debug.go @@ -4,8 +4,21 @@ import ( "context" "fmt" "io" + "sync" ) +type releaseOnCloseReadCloser struct { + io.ReadCloser + releaseOnce sync.Once + release func() +} + +func (r *releaseOnCloseReadCloser) Close() error { + err := r.ReadCloser.Close() + r.releaseOnce.Do(r.release) + return err +} + // ExportACPDebug fetches the selected execution's exact ACP conversation // export. The caller must perform task/session authorization before invoking // this method; lifecycle only resolves the in-memory execution seam. @@ -16,8 +29,22 @@ func (m *Manager) ExportACPDebug( return nil, fmt.Errorf("task session ID is required") } execution, ok := m.GetExecutionBySessionID(taskSessionID) - if !ok || execution == nil || execution.agentctl == nil || execution.ACPSessionID == "" { + if !ok || execution == nil || execution.ACPSessionID == "" { return nil, fmt.Errorf("ACP executor is unavailable") } - return execution.agentctl.ExportACPDebug(ctx, execution.ACPSessionID, maxBytes) + client, release := execution.AcquireAgentCtlClient() + if client == nil { + release() + return nil, fmt.Errorf("ACP executor is unavailable") + } + body, err := client.ExportACPDebug(ctx, execution.ACPSessionID, maxBytes) + if err != nil { + release() + return nil, err + } + if body == nil { + release() + return nil, fmt.Errorf("ACP debug export returned no body") + } + return &releaseOnCloseReadCloser{ReadCloser: body, release: release}, nil } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go index 898cb14468..21aa803824 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_debug_test.go @@ -1,6 +1,7 @@ package lifecycle import ( + "context" "io" "net/http" "net/http/httptest" @@ -13,6 +14,11 @@ import ( "go.uber.org/zap" ) +type debugExportResult struct { + body io.ReadCloser + err error +} + func TestManagerExportACPDebugUsesExecutionACPConversationID(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/api/v1/debug/acp/acp-session-1/export" { @@ -58,3 +64,109 @@ func TestManagerExportACPDebugUsesExecutionACPConversationID(t *testing.T) { t.Fatalf("body = %q", data) } } + +func TestManagerExportACPDebugHoldsClientLeaseDuringRequest(t *testing.T) { + requestEntered := make(chan struct{}) + releaseRequest := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + close(requestEntered) + <-releaseRequest + _, _ = w.Write([]byte("zip")) + })) + defer server.Close() + parsed, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(parsed.Port()) + if err != nil { + t.Fatal(err) + } + log, err := logger.NewFromZap(zap.NewNop()) + if err != nil { + t.Fatal(err) + } + execution := &AgentExecution{ + ID: "execution-1", SessionID: "task-session-1", ACPSessionID: "acp-session-1", + agentctl: agentctlclient.NewClient(parsed.Hostname(), port, log), + } + mgr := newTestManager(t) + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatal(err) + } + result := make(chan debugExportResult, 1) + go func() { + body, exportErr := mgr.ExportACPDebug(context.Background(), "task-session-1", 4096) + result <- debugExportResult{body: body, err: exportErr} + }() + <-requestEntered + if execution.agentctlLifecycleMu.TryLock() { + execution.agentctlLifecycleMu.Unlock() + close(releaseRequest) + got := <-result + if got.body != nil { + _ = got.body.Close() + } + t.Fatal("agentctl operation did not hold the client replacement lease") + } + close(releaseRequest) + got := <-result + if got.err != nil { + t.Fatal(got.err) + } + _ = got.body.Close() +} + +func TestManagerExportACPDebugHoldsClientLeaseUntilBodyClose(t *testing.T) { + headersSent := make(chan struct{}) + releaseBody := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + close(headersSent) + <-releaseBody + _, _ = w.Write([]byte("zip")) + })) + defer server.Close() + parsed, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(parsed.Port()) + if err != nil { + t.Fatal(err) + } + log, err := logger.NewFromZap(zap.NewNop()) + if err != nil { + t.Fatal(err) + } + execution := &AgentExecution{ + ID: "execution-1", SessionID: "task-session-1", ACPSessionID: "acp-session-1", + agentctl: agentctlclient.NewClient(parsed.Hostname(), port, log), + } + mgr := newTestManager(t) + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatal(err) + } + + body, err := mgr.ExportACPDebug(context.Background(), "task-session-1", 4096) + if err != nil { + t.Fatal(err) + } + <-headersSent + if execution.agentctlLifecycleMu.TryLock() { + execution.agentctlLifecycleMu.Unlock() + close(releaseBody) + _ = body.Close() + t.Fatal("agentctl replacement lease ended before the response body closed") + } + if err := body.Close(); err != nil { + t.Fatal(err) + } + if !execution.agentctlLifecycleMu.TryLock() { + close(releaseBody) + t.Fatal("agentctl replacement lease remained held after response body close") + } + execution.agentctlLifecycleMu.Unlock() + close(releaseBody) +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go b/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go index 3911a0f59f..1f9e478c68 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_execution.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "strings" "time" "github.com/google/uuid" @@ -321,7 +322,13 @@ func (m *Manager) ensureWorkspaceExecutionLocked(ctx context.Context, taskID, se waitCtx, cancel := appctx.Detached(ctx, m.stopCh, 60*time.Second) defer cancel() - if err := execution.agentctl.WaitForReady(waitCtx, 60*time.Second); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return + } + err := client.WaitForReady(waitCtx, 60*time.Second) + releaseClient() + if err != nil { m.logger.Error("agentctl not ready for workspace stream connection", zap.String("execution_id", execution.ID), zap.Error(err)) @@ -621,11 +628,21 @@ func (m *Manager) createExecution(ctx context.Context, taskID string, info *Work } return nil, fmt.Errorf("failed to register execution: %w", addErr) } + isKubernetes := execution.RuntimeName == agentruntime.RuntimeKubernetes + var createdRuntimeSecrets map[string]bool + if isKubernetes { + createdRuntimeSecrets, err = m.persistRequiredKubernetesRuntimeSecrets(ctx, runtimeInstance, execution) + if err != nil { + m.rollbackRegisteredLaunch(rt, runtimeInstance, execution, "Kubernetes runtime secret persistence failed") + return nil, err + } + } // Persist before the final session read so concurrent deletion cleanup can // inventory this execution even if it started between Add and validation. if err := m.persistExecutorRunningResult(ctx, execution); err != nil { + secretCleanupErr := m.deleteCreatedRuntimeSecrets(ctx, execution, createdRuntimeSecrets) m.rollbackRegisteredLaunchAfterPersistFailure(rt, runtimeInstance, execution) - return nil, fmt.Errorf("persist execution registration: %w", err) + return nil, errors.Join(fmt.Errorf("persist execution registration: %w", err), secretCleanupErr) } if err := m.ensureLaunchSessionStillActive(ctx, info.SessionID); err != nil { if errors.Is(err, errTaskCleanupActive) { @@ -635,7 +652,10 @@ func (m *Manager) createExecution(ctx context.Context, taskID string, info *Work } return nil, err } - m.publishCreatedExecution(ctx, runtimeInstance, execution, executionID, taskID) + if err := m.publishCreatedExecution(ctx, runtimeInstance, execution, executionID, taskID); err != nil { + m.rollbackRegisteredLaunch(rt, runtimeInstance, execution, "runtime secret persistence failed") + return nil, err + } return execution, nil } @@ -742,7 +762,7 @@ func (m *Manager) prepareExecutionCreateRequest( } } - return &executionCreatePreparation{ + preparation := &executionCreatePreparation{ request: &ExecutorCreateRequest{ InstanceID: executionID, TaskID: taskID, @@ -770,7 +790,9 @@ func (m *Manager) prepareExecutionCreateRequest( ComparisonTargets: comparisonTargets, }, profileInfo: profileInfo, - }, nil + } + m.wireKubernetesInventoryPersistence(preparation.request, info.ExecutorType) + return preparation, nil } func (m *Manager) resolveWorkspaceExecutionProfile(ctx context.Context, profileID string) *AgentProfileInfo { @@ -855,8 +877,9 @@ func (m *Manager) initializeCreatedExecution( } _, sessionSpan := tracing.TraceSessionStart(context.Background(), taskID, info.SessionID, executionID) execution.SetSessionSpan(sessionSpan) - if execution.agentctl != nil { - execution.agentctl.SetTraceContext(execution.SessionTraceContext()) + if client, releaseClient := execution.AcquireAgentCtlClient(); client != nil { + client.SetTraceContext(execution.SessionTraceContext()) + releaseClient() } return execution } @@ -867,12 +890,16 @@ func (m *Manager) publishCreatedExecution( execution *AgentExecution, executionID string, taskID string, -) { +) error { m.setRuntimeInterest(execution.SessionID, true) // Persist agentctl auth token only after the execution is tracked, so a // race-lost rollback never leaves an orphaned secret in the store. - m.persistRuntimeSecrets(ctx, runtimeInstance, execution) + if execution.RuntimeName != agentruntime.RuntimeKubernetes { + if err := m.persistRuntimeSecrets(ctx, runtimeInstance, execution); err != nil { + return err + } + } go m.pollOneRemoteStatus(context.Background(), execution) // Publish Starting BEFORE spawning waitForAgentctlReady so subscribers @@ -887,6 +914,7 @@ func (m *Manager) publishCreatedExecution( zap.String("task_id", taskID), zap.String("workspace_path", execution.WorkspacePath), zap.Stringer("runtime", execution.RuntimeName)) + return nil } func (m *Manager) reconcileWorkspaceWorktrees(ctx context.Context, taskID string, info *WorkspaceInfo) error { @@ -942,14 +970,16 @@ func (m *Manager) rollbackRacedExecution(ctx context.Context, rt ExecutorBackend zap.String("execution_id", execution.ID), zap.String("session_id", execution.SessionID)) if rt != nil && runtimeInstance != nil { - if stopErr := rt.StopInstance(ctx, runtimeInstance, true); stopErr != nil { + if stopErr := stopRuntimeInstanceAndRelease(ctx, rt, runtimeInstance, true); stopErr != nil { m.logger.Warn("failed to stop raced runtime instance during rollback", zap.String("execution_id", execution.ID), zap.Error(stopErr)) } } - if execution.agentctl != nil { - execution.agentctl.Close() + execution.agentctlLifecycleMu.Lock() + defer execution.agentctlLifecycleMu.Unlock() + if client := execution.currentAgentCtlClient(); client != nil { // protected by agentctlLifecycleMu + client.Close() } execution.EndSessionSpan() } @@ -966,10 +996,80 @@ const ( MetadataKeyContainerControlAuthSecret = "env_secret_id_CONTAINER_AGENTCTL_CONTROL_TOKEN" ) -func (m *Manager) persistRuntimeSecrets(ctx context.Context, instance *ExecutorInstance, execution *AgentExecution) { - m.persistAuthToken(ctx, instance, execution) - m.persistBootstrapNonce(ctx, instance, execution) - m.persistContainerControlAuthToken(ctx, instance, execution) +func (m *Manager) persistRuntimeSecrets( + ctx context.Context, + instance *ExecutorInstance, + execution *AgentExecution, +) error { + if instance == nil || execution == nil { + return errors.New("persist runtime secrets: runtime identity is unavailable") + } + secretValues := []struct { + metadataKey string + namePrefix string + value string + }{ + {MetadataKeyAuthTokenSecret, "agentctl-auth", instance.AuthToken}, + {MetadataKeyBootstrapNonceSecret, "agentctl-bootstrap", instance.BootstrapNonce}, + } + if instance.ContainerID != "" { + secretValues = append(secretValues, struct { + metadataKey string + namePrefix string + value string + }{MetadataKeyContainerControlAuthSecret, "agentctl-container-control", instance.AuthToken}) + } + created := make(map[string]bool, len(secretValues)) + for _, secretValue := range secretValues { + if secretValue.value == "" { + continue + } + wasCreated, err := m.persistRuntimeSecretResult( + ctx, instance, execution, secretValue.metadataKey, secretValue.namePrefix, secretValue.value, + ) + created[secretValue.metadataKey] = wasCreated + if err != nil { + cleanupErr := m.deleteCreatedRuntimeSecrets(ctx, execution, created) + return errors.Join( + fmt.Errorf("persist runtime secret %s: %w", secretValue.metadataKey, err), + cleanupErr, + ) + } + } + return nil +} + +func (m *Manager) persistRequiredKubernetesRuntimeSecrets( + ctx context.Context, + instance *ExecutorInstance, + execution *AgentExecution, +) (map[string]bool, error) { + if m.secretStore == nil { + return nil, errors.New("persist Kubernetes runtime secrets: secret store is unavailable") + } + if instance == nil || strings.TrimSpace(instance.AuthToken) == "" || strings.TrimSpace(instance.BootstrapNonce) == "" { + return nil, errors.New("persist Kubernetes runtime secrets: auth token and bootstrap nonce are required") + } + authCreated, err := m.persistRuntimeSecretResult( + ctx, instance, execution, MetadataKeyAuthTokenSecret, "agentctl-auth", instance.AuthToken, + ) + if err != nil { + return nil, fmt.Errorf("persist Kubernetes agentctl auth token: %w", err) + } + nonceCreated, err := m.persistRuntimeSecretResult( + ctx, instance, execution, MetadataKeyBootstrapNonceSecret, "agentctl-bootstrap", instance.BootstrapNonce, + ) + if err != nil { + cleanupErr := m.deleteCreatedRuntimeSecrets(ctx, execution, map[string]bool{ + MetadataKeyAuthTokenSecret: authCreated, + MetadataKeyBootstrapNonceSecret: nonceCreated, + }) + return nil, errors.Join(fmt.Errorf("persist Kubernetes bootstrap nonce: %w", err), cleanupErr) + } + return map[string]bool{ + MetadataKeyAuthTokenSecret: authCreated, + MetadataKeyBootstrapNonceSecret: nonceCreated, + }, nil } // persistAuthToken stores the agentctl handshake auth token in SecretStore @@ -1000,19 +1100,55 @@ func (m *Manager) persistRuntimeSecret( if value == "" || m.secretStore == nil { return } - - secret := &secrets.SecretWithValue{ - Secret: secrets.Secret{ - Name: fmt.Sprintf("%s-%s", secretNamePrefix, truncateID(instance.InstanceID, 12)), - }, - Value: value, - } - if err := m.secretStore.Create(ctx, secret); err != nil { + if _, err := m.persistRuntimeSecretResult( + ctx, instance, execution, metadataKey, secretNamePrefix, value, + ); err != nil { m.logger.Error("failed to persist runtime secret", zap.String("instance_id", instance.InstanceID), zap.String("metadata_key", metadataKey), zap.Error(err)) - return + } +} + +func (m *Manager) persistRuntimeSecretResult( + ctx context.Context, + instance *ExecutorInstance, + execution *AgentExecution, + metadataKey string, + secretNamePrefix string, + value string, +) (bool, error) { + if instance == nil || execution == nil || m.secretStore == nil { + return false, errors.New("runtime secret persistence is unavailable") + } + secretID := execution.metadataString(metadataKey) + if secretID == "" { + resourceInstanceID := execution.metadataString(MetadataKeyKubernetesResourceInstanceID) + if resourceInstanceID == "" { + resourceInstanceID = instance.InstanceID + } + secretID = kubernetesRuntimeSecretID(resourceInstanceID, secretNamePrefix) + } + name := fmt.Sprintf("%s-%s", secretNamePrefix, truncateID(instance.InstanceID, 12)) + if _, err := m.secretStore.Get(ctx, secretID); err == nil { + if err := m.secretStore.Update(ctx, secretID, &secrets.UpdateSecretRequest{Name: &name, Value: &value}); err != nil { + return false, err + } + execution.setMetadataValue(metadataKey, secretID) + return false, nil + } else if !errors.Is(err, secrets.ErrNotFound) { + return false, err + } + + secret := &secrets.SecretWithValue{ + Secret: secrets.Secret{ + ID: secretID, + Name: name, + }, + Value: value, + } + if err := m.secretStore.Create(ctx, secret); err != nil { + return false, err } execution.setMetadataValue(metadataKey, secret.ID) @@ -1020,6 +1156,86 @@ func (m *Manager) persistRuntimeSecret( m.logger.Debug("persisted runtime secret in secret store", zap.String("instance_id", instance.InstanceID), zap.String("metadata_key", metadataKey)) + return true, nil +} + +func (m *Manager) deleteCreatedRuntimeSecrets( + ctx context.Context, + execution *AgentExecution, + createdByMetadataKey map[string]bool, +) error { + var cleanupErrs []error + for metadataKey, created := range createdByMetadataKey { + if !created { + continue + } + secretID := execution.metadataString(metadataKey) + if secretID != "" { + if err := m.secretStore.Delete(ctx, secretID); err != nil && !errors.Is(err, secrets.ErrNotFound) { + cleanupErrs = append(cleanupErrs, fmt.Errorf("delete %s: %w", metadataKey, err)) + continue + } + } + execution.deleteMetadataValues(metadataKey) + } + return errors.Join(cleanupErrs...) +} + +func (m *Manager) deleteKubernetesRuntimeSecrets(ctx context.Context, metadata map[string]interface{}) error { + secretIDs := kubernetesRuntimeSecretIDs(metadata) + if secretIDs[0] == "" && secretIDs[1] == "" { + return nil + } + if m.secretStore == nil { + return errors.New("delete Kubernetes runtime secrets: secret store is unavailable") + } + var deleteErrs []error + for _, secretID := range secretIDs { + if secretID == "" { + continue + } + if err := m.secretStore.Delete(ctx, secretID); err != nil && !errors.Is(err, secrets.ErrNotFound) { + deleteErrs = append(deleteErrs, fmt.Errorf("delete runtime secret %q: %w", secretID, err)) + } + } + return errors.Join(deleteErrs...) +} + +func kubernetesRuntimeSecretIDs(metadata map[string]interface{}) []string { + secretIDs := []string{ + getMetadataString(metadata, MetadataKeyAuthTokenSecret), + getMetadataString(metadata, MetadataKeyBootstrapNonceSecret), + } + if !isProvisionalKubernetesInventory(metadata) { + return secretIDs + } + resourceInstanceID := getMetadataString(metadata, MetadataKeyKubernetesResourceInstanceID) + if resourceInstanceID == "" { + return secretIDs + } + if secretIDs[0] == "" { + secretIDs[0] = kubernetesRuntimeSecretID(resourceInstanceID, "agentctl-auth") + } + if secretIDs[1] == "" { + secretIDs[1] = kubernetesRuntimeSecretID(resourceInstanceID, "agentctl-bootstrap") + } + return secretIDs +} + +func kubernetesRuntimeSecretID(resourceInstanceID, kind string) string { + return fmt.Sprintf("kandev-runtime:%s:%s", strings.TrimSpace(resourceInstanceID), kind) +} + +func isProvisionalKubernetesInventory(metadata map[string]interface{}) bool { + switch getMetadataString(metadata, MetadataKeyKubernetesInventoryState) { + case KubernetesInventoryStatePVCCreated, + KubernetesInventoryStatePVCAdmitted, + KubernetesInventoryStatePodCreated, + KubernetesInventoryStatePodAdmitted: + return true + default: + return false + } } // revealRuntimeSecretValue reveals a configured runtime secret and preserves diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go index 8b963a9c89..ed5836fb48 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction.go @@ -189,7 +189,9 @@ func (m *Manager) CancelAgentForPrompt( func (m *Manager) cancelAgentExecution(ctx context.Context, execution *AgentExecution) error { executionID := execution.ID - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } @@ -198,7 +200,7 @@ func (m *Manager) cancelAgentExecution(ctx context.Context, execution *AgentExec zap.String("task_id", execution.TaskID), zap.String("session_id", execution.SessionID)) - cancelErr := execution.agentctl.Cancel(ctx) + cancelErr := client.Cancel(ctx) streamDisconnected := errors.Is(cancelErr, agentctlclient.ErrAgentStreamNotConnected) if cancelErr != nil && !errors.Is(cancelErr, agentctlclient.ErrTurnCancelNotAcknowledged) && @@ -351,13 +353,15 @@ func (m *Manager) SetSessionMode(ctx context.Context, executionID, _ string, mod if !exists { return fmt.Errorf("execution %q not found", executionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return fmt.Errorf("execution %q ACP session is not ready", executionID) } - return execution.agentctl.SetMode(ctx, execution.ACPSessionID, modeID) + return client.SetMode(ctx, execution.ACPSessionID, modeID) } // SetSessionModeBySessionID changes the session mode for a running agent by session ID. @@ -389,10 +393,12 @@ func (m *Manager) SetSessionModel(ctx context.Context, executionID, modelID stri return m.RestartAgentProcess(ctx, executionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } - return execution.agentctl.SetModel(ctx, modelID) + return client.SetModel(ctx, modelID) } // SetSessionModelBySessionID changes the session model for a running agent by session ID. @@ -410,13 +416,15 @@ func (m *Manager) SetSessionConfigOption(ctx context.Context, executionID, confi if !exists { return fmt.Errorf("execution %q not found", executionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return fmt.Errorf("execution %q ACP session is not ready", executionID) } - return execution.agentctl.SetConfigOption(ctx, configID, value) + return client.SetConfigOption(ctx, configID, value) } // SetSessionConfigOptionBySessionID changes an ACP session config option by task session ID. @@ -434,10 +442,12 @@ func (m *Manager) AuthenticateBySessionID(ctx context.Context, sessionID, method if !exists { return fmt.Errorf("no agent running for session %q", sessionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", execution.ID) } - return execution.agentctl.Authenticate(ctx, methodID) + return client.Authenticate(ctx, methodID) } // reapplySessionModeAfterReset re-applies the active session permission mode @@ -472,10 +482,12 @@ func (m *Manager) applySessionModeAfterReset( execution *AgentExecution, newSessionID, mode string, ) error { - if execution.agentctl == nil || mode == "" { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil || mode == "" { return nil } - if err := execution.agentctl.SetMode(ctx, newSessionID, mode); err != nil { + if err := client.SetMode(ctx, newSessionID, mode); err != nil { return fmt.Errorf("failed to restore session mode %q: %w", mode, err) } availableModes := []streams.SessionModeInfo(nil) @@ -502,13 +514,15 @@ func (m *Manager) reapplySessionModelAfterReset( execution *AgentExecution, newSessionID, modelID string, ) error { - if execution.agentctl == nil || modelID == "" { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil || modelID == "" { return nil } policy := m.resolveStartModelPolicy(ctx, execution.AgentProfileID) policy.Model = modelID decision, err := applyStartModelPolicy( - ctx, m.logger, execution.agentctl, execution.GetModelState(), policy, + ctx, m.logger, client, execution.GetModelState(), policy, ) if err != nil { m.logger.Warn("failed to re-apply session model after context reset", @@ -533,10 +547,15 @@ func (m *Manager) reapplySessionModelAfterReset( } func cacheFreshSessionModelState(execution *AgentExecution) bool { - if execution == nil || execution.agentctl == nil { + if execution == nil { return false } - state := execution.agentctl.GetLastSessionModelState() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return false + } + state := client.GetLastSessionModelState() if state == nil { return false } @@ -720,7 +739,12 @@ func (m *Manager) restoreSessionRuntimeConfig( sessionID string, config models.SessionRuntimeConfig, ) error { - if execution == nil || execution.agentctl == nil { + if execution == nil { + return fmt.Errorf("cannot restore session runtime configuration without an agentctl client") + } + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { return fmt.Errorf("cannot restore session runtime configuration without an agentctl client") } if err := m.reapplySessionModelAfterReset(ctx, execution, sessionID, config.Model); err != nil { @@ -737,7 +761,13 @@ func (m *Manager) restoreSessionRuntimeConfig( sort.Strings(optionIDs) for _, id := range optionIDs { value := options[id] - if err := execution.agentctl.SetConfigOption(ctx, id, value); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("restore session runtime option %q: agentctl client is unavailable", id) + } + err := client.SetConfigOption(ctx, id, value) + releaseClient() + if err != nil { return fmt.Errorf("restore session runtime option %q: %w", id, err) } } @@ -757,8 +787,14 @@ func (m *Manager) ResetAgentContext(ctx context.Context, executionID string) err if execution.PassthroughProcessID != "" { return m.RestartAgentProcess(ctx, executionID) } + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + if current, currentExists := m.executionStore.Get(executionID); !currentExists || current != execution { + return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound) + } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } @@ -777,6 +813,7 @@ func (m *Manager) ResetAgentContext(ctx context.Context, executionID string) err // Resolve agent config and MCP servers for session reset agentConfig, err := m.getAgentConfigForExecution(execution) if err != nil { + releaseClient() m.logger.Info("cannot resolve agent config for session reset, falling back to process restart", zap.String("execution_id", executionID), zap.Error(err)) return m.restartAgentProcess(ctx, executionID, &runtimeConfig) @@ -784,13 +821,15 @@ func (m *Manager) ResetAgentContext(ctx context.Context, executionID string) err mcpServers, err := m.resolveMcpServers(ctx, execution, agentConfig) if err != nil { + releaseClient() m.logger.Warn("cannot resolve MCP servers for session reset, falling back to process restart", zap.String("execution_id", executionID), zap.Error(err)) return m.restartAgentProcess(ctx, executionID, &runtimeConfig) } // Try session-level reset (only ACP adapters support this) - newSessionID, err := execution.agentctl.ResetSession(ctx, execution.WorkspacePath, mcpServers) + newSessionID, err := client.ResetSession(ctx, execution.WorkspacePath, mcpServers) + releaseClient() if err != nil { m.logger.Info("session reset not supported, falling back to process restart", zap.String("execution_id", executionID), zap.Error(err)) @@ -898,6 +937,15 @@ func (m *Manager) StopAgent(ctx context.Context, executionID string, force bool) func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, reason string, force bool) error { execution, exists := m.executionStore.Get(executionID) if !exists { + handled, err := m.stopPersistedKubernetesExecution(ctx, executionID, reason, force) + if handled { + return err + } + return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound) + } + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + if current, currentExists := m.executionStore.Get(executionID); !currentExists || current != execution { return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound) } activityLease, err := m.acquireActivity(ctx, activity.KindExecutionStopping) @@ -916,27 +964,8 @@ func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, r zap.Bool("force", force), zap.Stringer("runtime", execution.RuntimeName)) - // Try to gracefully stop via agentctl first, then always close connections - agentStopFailed := false - if execution.agentctl != nil { - if !force { - if err := execution.agentctl.Stop(ctx); err != nil { - agentStopFailed = true - // During shutdown the instance may already be stopping through - // another lifecycle path, so a failed HTTP call is expected. - if m.IsShuttingDown() { - m.logger.Debug("failed to stop agent via agentctl", - zap.String("execution_id", executionID), - zap.Error(err)) - } else { - m.logger.Warn("failed to stop agent via agentctl", - zap.String("execution_id", executionID), - zap.Error(err)) - } - } - } - execution.agentctl.Close() - } + // Try to gracefully stop via agentctl first, then always close connections. + agentStopFailed := m.stopExecutionAgentctl(ctx, executionID, execution, force) // Stop the agent execution via the runtime that created it. A failed stop // must remain tracked: removing it here would turn a retryable cleanup into @@ -944,6 +973,22 @@ func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, r if err := m.stopAgentViaBackend(ctx, executionID, execution, reason, force, agentStopFailed); err != nil { return fmt.Errorf("stop runtime for execution %q: %w", executionID, err) } + if execution.RuntimeName == executor.NameKubernetes && (force || shouldRunExecutorCleanup(reason)) { + cleanupCtx, cancelCleanup := kubernetesDurableContext(ctx) + err := m.deleteKubernetesRuntimeSecrets(cleanupCtx, execution.MetadataSnapshot()) + cancelCleanup() + if err != nil { + return fmt.Errorf("delete runtime secrets for execution %s: %w", executionID, err) + } + } + + // The runtime is now terminal. Detach agentctl under its own short critical + // section before any cleanup or event snapshot can acquire prompt state. + // Failed runtime stops intentionally retain the closed-but-reusable client so + // a later retry can issue the agentctl stop request again. + execution.agentctlLifecycleMu.Lock() + execution.detachAgentctlClient() + execution.agentctlLifecycleMu.Unlock() // Update execution status and remove from tracking _ = m.executionStore.WithLock(executionID, func(exec *AgentExecution) { @@ -968,6 +1013,37 @@ func (m *Manager) StopAgentWithReason(ctx context.Context, executionID string, r return nil } +func (m *Manager) stopExecutionAgentctl( + ctx context.Context, + executionID string, + execution *AgentExecution, + force bool, +) bool { + execution.agentctlLifecycleMu.Lock() + defer execution.agentctlLifecycleMu.Unlock() + client := execution.currentAgentCtlClient() // protected by agentctlLifecycleMu + if client == nil { + return false + } + defer client.Close() + if force { + return false + } + if err := client.Stop(ctx); err != nil { + // During shutdown the instance may already be stopping through another + // lifecycle path, so a failed HTTP call is expected. + if m.IsShuttingDown() { + m.logger.Debug("failed to stop agent via agentctl", + zap.String("execution_id", executionID), zap.Error(err)) + } else { + m.logger.Warn("failed to stop agent via agentctl", + zap.String("execution_id", executionID), zap.Error(err)) + } + return true + } + return false +} + // StopBySessionID stops the agent for a specific session func (m *Manager) StopBySessionID(ctx context.Context, sessionID string, force bool) error { execution, exists := m.executionStore.GetBySessionID(sessionID) @@ -982,6 +1058,15 @@ func (m *Manager) StopBySessionID(ctx context.Context, sessionID string, force b // conversation context. For ACP agents this restarts via agentctl with a new ACP session. // For passthrough (TUI) agents this kills the PTY process and relaunches without --resume. func (m *Manager) RestartAgentProcess(ctx context.Context, executionID string) error { + execution, exists := m.executionStore.Get(executionID) + if !exists { + return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound) + } + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + if current, currentExists := m.executionStore.Get(executionID); !currentExists || current != execution { + return fmt.Errorf("execution %q not found: %w", executionID, ErrExecutionNotFound) + } return m.restartAgentProcess(ctx, executionID, nil) } @@ -1003,9 +1088,11 @@ func (m *Manager) restartAgentProcess( return m.restartPassthroughProcess(ctx, execution) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } + releaseClient() preparation, err := m.prepareAgentRestart(ctx, execution, runtimeConfigOverride) if err != nil { @@ -1017,8 +1104,13 @@ func (m *Manager) restartAgentProcess( // barrier that flips the client into a closed state and would block // every StreamUpdates/StreamWorkspace call that this same restart path // makes a few lines below. - execution.agentctl.CloseUpdatesStream() - execution.agentctl.CloseWorkspaceStream() + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", executionID) + } + client.CloseUpdatesStream() + client.CloseWorkspaceStream() + releaseClient() // 2. Stop the agent subprocess via agentctl (keeps agentctl server alive) m.stopAgentProcessForRestart(ctx, execution) @@ -1027,7 +1119,13 @@ func (m *Manager) restartAgentProcess( m.resetAgentRestartState(executionID, preparation.commands) // 4. Wait for agentctl to be ready (it should still be running) - if err := execution.agentctl.WaitForReady(ctx, 30*time.Second); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", executionID) + } + err = client.WaitForReady(ctx, 30*time.Second) + releaseClient() + if err != nil { m.updateExecutionError(executionID, "agentctl not ready after restart: "+err.Error()) return fmt.Errorf("agentctl not ready after restart: %w", err) } @@ -1040,7 +1138,13 @@ func (m *Manager) restartAgentProcess( } // 6. Wait for agent process to initialize - if err := execution.agentctl.WaitForReady(ctx, 10*time.Second); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", executionID) + } + err = client.WaitForReady(ctx, 10*time.Second) + releaseClient() + if err != nil { m.logger.Warn("agent process slow to initialize after restart, continuing", zap.String("execution_id", executionID), zap.Error(err)) @@ -1080,7 +1184,12 @@ func (m *Manager) restartAgentProcess( } func (m *Manager) stopAgentProcessForRestart(ctx context.Context, execution *AgentExecution) { - if err := execution.agentctl.Stop(ctx); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return + } + if err := client.Stop(ctx); err != nil { m.logger.Warn("failed to stop agent subprocess during restart", zap.String("execution_id", execution.ID), zap.Error(err)) @@ -1178,9 +1287,14 @@ func (m *Manager) initializeACPSessionForRestart( } // Initialize ACP session (always session/new since ACPSessionID was cleared) + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } result, err := m.sessionManager.InitializeSession( ctx, - execution.agentctl, + client, agentConfig, "", // empty — force session/new execution.WorkspacePath, @@ -1409,11 +1523,13 @@ func (m *Manager) ProbeAgentRunningForSession(ctx context.Context, sessionID str } // Probe agentctl status to verify the agent process is running - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return false, fmt.Errorf("agentctl client is unavailable") } - status, err := execution.agentctl.GetStatus(ctx) + status, err := client.GetStatus(ctx) if err != nil { m.logger.Debug("failed to get agentctl status", zap.String("session_id", sessionID), @@ -1444,14 +1560,16 @@ func (m *Manager) IsAgentReadyForPrompt(ctx context.Context, sessionID string) b return m.IsAgentRunningForSession(ctx, sessionID) } - if execution.Status != v1.AgentStatusReady || execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if execution.Status != v1.AgentStatusReady || client == nil { return false } if !execution.isSessionInitialized() || execution.ACPSessionID == "" { return false } - return execution.agentctl.HasAgentStream() + return client.HasAgentStream() } func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string) error { @@ -1459,18 +1577,23 @@ func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string if !exists { return fmt.Errorf("session %q has no execution: %w", sessionID, ErrExecutionNotFound) } - if execution.PassthroughProcessID != "" || execution.IsPassthrough || execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if execution.PassthroughProcessID != "" || execution.IsPassthrough || client == nil { + releaseClient() return nil } // InitializeAndPrompt owns the first updates stream. Starting a recovery // stream before ACP initialization finishes creates competing consumers and // can split one prompt's events across them. if !execution.isSessionInitialized() || execution.ACPSessionID == "" { + releaseClient() return nil } - if execution.agentctl.HasAgentStream() { + if client.HasAgentStream() { + releaseClient() return nil } + releaseClient() if m.streamManager == nil { return fmt.Errorf("stream manager is not configured") } @@ -1484,7 +1607,13 @@ func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string case <-ctx.Done(): return ctx.Err() } - if !execution.agentctl.HasAgentStream() { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } + hasAgentStream := client.HasAgentStream() + releaseClient() + if !hasAgentStream { return fmt.Errorf("agent stream not connected") } if execution.Status == v1.AgentStatusFailed && execution.isSessionInitialized() && execution.ACPSessionID != "" { @@ -1494,7 +1623,12 @@ func (m *Manager) RecoverAgentPromptStream(ctx context.Context, sessionID string } func (m *Manager) restoreRecoveredFailedExecution(ctx context.Context, execution *AgentExecution) error { - status, err := execution.agentctl.GetStatus(ctx) + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } + status, err := client.GetStatus(ctx) if err != nil { return fmt.Errorf("failed to verify agent status after stream recovery: %w", err) } @@ -1985,7 +2119,9 @@ func (m *Manager) RespondToPermission(executionID, pendingID, optionID string, c return fmt.Errorf("agent execution not found: %s", executionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("agent execution has no agentctl client: %s", executionID) } @@ -1998,7 +2134,7 @@ func (m *Manager) RespondToPermission(executionID, pendingID, optionID string, c ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - return execution.agentctl.RespondToPermission(ctx, pendingID, optionID, cancelled) + return client.RespondToPermission(ctx, pendingID, optionID, cancelled) } // RespondToPermissionBySessionID sends a response to a permission request using session ID. @@ -2021,12 +2157,14 @@ func (m *Manager) ListPendingPermissionsBySessionID(ctx context.Context, session if !exists { return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return nil, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID) } requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return execution.agentctl.ListPendingPermissions(requestCtx) + return client.ListPendingPermissions(requestCtx) } // ResolvePermissionBySessionID selects one exact option on one exact request @@ -2036,12 +2174,14 @@ func (m *Manager) ResolvePermissionBySessionID(ctx context.Context, sessionID, r if !exists { return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return nil, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID) } requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return execution.agentctl.ResolvePermission(requestCtx, requestID, pendingID, optionID) + return client.ResolvePermission(requestCtx, requestID, pendingID, optionID) } func (m *Manager) CancelPermissionBySessionID(ctx context.Context, sessionID, requestID, pendingID string) (*streams.PermissionCancelResponse, error) { @@ -2049,12 +2189,14 @@ func (m *Manager) CancelPermissionBySessionID(ctx context.Context, sessionID, re if !exists { return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return nil, fmt.Errorf("agent execution has no agentctl client: %s", execution.ID) } requestCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() - return execution.agentctl.CancelPermission(requestCtx, requestID, pendingID) + return client.CancelPermission(requestCtx, requestID, pendingID) } // stopAgentViaBackend stops the agent execution via the runtime that created it. @@ -2074,6 +2216,7 @@ func (m *Manager) stopAgentViaBackend(ctx context.Context, executionID string, e runtimeInstance := &ExecutorInstance{ InstanceID: execution.ID, TaskID: execution.TaskID, + SessionID: execution.SessionID, ContainerID: execution.ContainerID, StandaloneInstanceID: execution.standaloneInstanceID, StandalonePort: execution.standalonePort, @@ -2081,6 +2224,13 @@ func (m *Manager) stopAgentViaBackend(ctx context.Context, executionID string, e StopReason: reason, AgentStopFailed: agentStopFailed, } + if execution.RuntimeName == executor.NameKubernetes { + metadata, resolveErr := m.currentKubernetesConnectionMetadata(ctx, runtimeInstance.Metadata) + if resolveErr != nil { + return fmt.Errorf("resolve current Kubernetes cleanup connection: %w", resolveErr) + } + runtimeInstance.Metadata = metadata + } if err := rt.StopInstance(ctx, runtimeInstance, force); err != nil { // During shutdown the runtime instance may already be stopping or // absent. Only surface this at WARN outside shutdown. diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go index 9b64dca4a2..ba4889c753 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_accessors_test.go @@ -9,9 +9,13 @@ import ( "github.com/stretchr/testify/require" "github.com/kandev/kandev/internal/agent/executor" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" "github.com/kandev/kandev/internal/agent/runtime/activity" "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/events" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" ) @@ -362,6 +366,164 @@ func TestStopAgentForcePassesForceToBackend(t *testing.T) { require.Equal(t, StopReasonBackendShutdown, stopTracker.stopReason) } +func TestStopAgentWithReasonRetainsExecutionWhenBackendCleanupFails(t *testing.T) { + log := newTestRegistryLogger() + execRegistry := NewExecutorRegistry(log) + cleanupErr := errors.New("exact Kubernetes cleanup failed") + stopTracker := &mockStopTracker{name: executor.NameKubernetes, stopErr: cleanupErr} + execRegistry.Register(stopTracker) + bus := &MockEventBus{} + mgr := NewManager(newTestRegistry(), bus, execRegistry, nil, nil, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + runtimeSecrets := newInMemorySecretStore() + for id := range map[string]struct{}{"legacy-auth-ref": {}, "legacy-nonce-ref": {}} { + require.NoError(t, runtimeSecrets.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: "secret", + })) + } + mgr.SetSecretStore(runtimeSecrets) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "exec-k8s", TaskID: "task-k8s", SessionID: "session-k8s", + RuntimeName: executor.NameKubernetes, Status: v1.AgentStatusRunning, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "legacy-auth-ref", + MetadataKeyBootstrapNonceSecret: "legacy-nonce-ref", + }, + })) + + err := mgr.StopAgentWithReason(context.Background(), "exec-k8s", StopReasonTaskDeleted, true) + + require.ErrorIs(t, err, cleanupErr) + require.Equal(t, "session-k8s", stopTracker.stoppedSessionID) + execution, exists := mgr.GetExecution("exec-k8s") + require.True(t, exists, "failed exact cleanup must remain retryable") + require.Equal(t, v1.AgentStatusRunning, execution.Status) + require.Len(t, runtimeSecrets.store, 2, "runtime cleanup failure must preserve recovery secrets") + for _, event := range bus.PublishedEvents { + require.NotEqual(t, events.AgentStopped, event.Type) + } +} + +func TestStopAgentWithReasonCleansPersistedKubernetesInventoryAfterRestart(t *testing.T) { + testPersistedKubernetesCleanupAfterRestart(t, StopReasonTaskDeleted, true) +} + +func TestForceStopAgentCleansPersistedKubernetesInventoryAfterRestartWithoutTerminalReason(t *testing.T) { + testPersistedKubernetesCleanupAfterRestart(t, "", true) +} + +func testPersistedKubernetesCleanupAfterRestart(t *testing.T, reason string, force bool) { + t.Helper() + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + resources := &fakeKubernetesResources{} + initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := initial.CreateInstance(context.Background(), req) + require.NoError(t, err) + metadata := cloneKubernetesMetadata(req.Metadata) + for key, value := range instance.Metadata { + metadata[key] = value + } + metadata[MetadataKeyKubernetesAuthMode] = "kubeconfig" + metadata[MetadataKeyKubernetesKubeconfigPath] = "/stale/kubeconfig" + metadata[MetadataKeyKubernetesKubeContext] = "stale-context" + metadata[MetadataKeyKubernetesRequestTimeoutSeconds] = "10" + runtimeSecrets := newInMemorySecretStore() + for id, value := range map[string]string{ + "legacy-auth-ref": "agentctl-token", "legacy-nonce-ref": "bootstrap-nonce", + } { + require.NoError(t, runtimeSecrets.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + metadata[MetadataKeyAuthTokenSecret] = "legacy-auth-ref" + metadata[MetadataKeyBootstrapNonceSecret] = "legacy-nonce-ref" + row := &models.ExecutorRunning{ + ID: instance.SessionID, SessionID: instance.SessionID, TaskID: instance.TaskID, + ExecutorID: "executor-1", ExecutionProfileID: "agent-profile-1", AgentExecutionID: instance.InstanceID, + Runtime: agentruntime.RuntimeKubernetes, Metadata: metadata, + } + store := &restartKubernetesInventoryStore{ + captureExecutorRunningWriter: captureExecutorRunningWriter{prior: row}, + rows: []*models.ExecutorRunning{row}, + executors: map[string]*models.Executor{"executor-1": { + ID: "executor-1", Type: models.ExecutorTypeKubernetes, + Config: map[string]string{ + MetadataKeyKubernetesAuthMode: "in_cluster", + MetadataKeyKubernetesConfigNamespace: "new-agents", + MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + }}, + } + restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, nil) + originalFactory := restarted.clientFactory + var cleanupConfig kubeexecutor.ExecutorConfig + restarted.clientFactory = func(config kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + cleanupConfig = config + return originalFactory(config) + } + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(restarted) + mgr := NewManager(nil, &MockEventBus{}, registry, nil, nil, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + mgr.SetExecutorRunningWriter(store) + mgr.SetSecretStore(runtimeSecrets) + + err = mgr.StopAgentWithReason(context.Background(), instance.InstanceID, reason, force) + + require.NoError(t, err) + require.Equal(t, []string{"pod", "pvc"}, resources.deletionOrder) + require.Equal(t, kubeexecutor.AuthModeInCluster, cleanupConfig.AuthMode) + require.Empty(t, cleanupConfig.KubeconfigPath) + require.Empty(t, cleanupConfig.KubeContext) + require.Equal(t, 45, cleanupConfig.RequestTimeoutSeconds) + require.Equal(t, "new-agents", cleanupConfig.Namespace) + require.Nil(t, store.running, "service cleanup owns row deletion after successful remote teardown") + require.Empty(t, runtimeSecrets.store, "destructive restart cleanup must remove recorded runtime secret refs") +} + +func TestStopAgentWithReasonPreservesPersistedKubernetesInventoryForBackendShutdown(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + stopTracker := &mockStopTracker{name: executor.NameKubernetes} + registry.Register(stopTracker) + store := &restartKubernetesInventoryStore{rows: []*models.ExecutorRunning{{ + AgentExecutionID: "instance-1", Runtime: agentruntime.RuntimeKubernetes, + }}} + mgr := NewManager(nil, &MockEventBus{}, registry, nil, nil, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + mgr.SetExecutorRunningWriter(store) + + err := mgr.StopAgentWithReason(context.Background(), "instance-1", StopReasonBackendShutdown, false) + + require.ErrorIs(t, err, ErrExecutionNotFound) + require.False(t, stopTracker.stopCalled) +} + +type restartKubernetesInventoryStore struct { + captureExecutorRunningWriter + rows []*models.ExecutorRunning + executors map[string]*models.Executor +} + +func (s *restartKubernetesInventoryStore) ListExecutorsRunning(context.Context) ([]*models.ExecutorRunning, error) { + return s.rows, nil +} + +func (s *restartKubernetesInventoryStore) GetExecutor(_ context.Context, id string) (*models.Executor, error) { + executor := s.executors[id] + if executor == nil { + return nil, models.ErrExecutorNotFound + } + return executor, nil +} + func TestStopAgentWithReason_BackendFailureKeepsExecutionRetryable(t *testing.T) { log := newTestRegistryLogger() execRegistry := NewExecutorRegistry(log) diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go index 27ff16a942..23b8912b35 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_interaction_test.go @@ -75,6 +75,60 @@ func TestStopAgentWithReason_MissingExecutionIsClassified(t *testing.T) { require.ErrorIs(t, err, ErrExecutionNotFound) } +func TestStopAgentWithReasonReleasesAgentctlBeforeStoppedEventSnapshot(t *testing.T) { + mgr := newTestManager(t) + execution := &AgentExecution{ + ID: "exec-stop-lock-order", + SessionID: "session-stop-lock-order", + agentctl: agentctl.NewClient("127.0.0.1", 12345, newTestLogger()), + } + require.NoError(t, mgr.executionStore.Add(execution)) + + // Hold the prompt lock so stop reaches the stopped-event snapshot and waits. + // It must release and detach agentctl first; prompt operations may need an + // agentctl read lease while this event snapshot is pending. + stopDone := make(chan error, 1) + execution.promptLifecycleMu.Lock() + promptLocked := true + stopJoined := false + defer func() { + if promptLocked { + execution.promptLifecycleMu.Unlock() + } + if !stopJoined { + select { + case <-stopDone: + case <-time.After(time.Second): + } + } + }() + go func() { + stopDone <- mgr.StopAgentWithReason( + context.Background(), execution.ID, StopReasonTaskDeleted, true, + ) + }() + + require.Eventually(t, func() bool { + _, exists := mgr.executionStore.Get(execution.ID) + return !exists + }, time.Second, time.Millisecond) + + agentctlUnlocked := execution.agentctlLifecycleMu.TryRLock() + var currentClient *agentctl.Client + if agentctlUnlocked { + currentClient = execution.currentAgentCtlClient() + execution.agentctlLifecycleMu.RUnlock() + } + execution.promptLifecycleMu.Unlock() + promptLocked = false + + stopErr := <-stopDone + stopJoined = true + require.NoError(t, stopErr) + require.True(t, agentctlUnlocked, "agentctl lifecycle lock remained held while publishing the stopped event") + require.Nil(t, currentClient, "terminal stop must detach the closed agentctl client") +} + func TestWaitForFreshSessionModelStateWaitsForAdvertisedCatalog(t *testing.T) { execution := &AgentExecution{ID: "exec-model-catalog"} execution.SetModelState(&CachedModelState{CurrentModelID: "mock-fast"}) diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_checkpoint.go b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_checkpoint.go new file mode 100644 index 0000000000..97120eb421 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_checkpoint.go @@ -0,0 +1,146 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +const kubernetesDurableWriteTimeout = time.Minute + +func kubernetesDurableContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + return context.WithTimeout(context.WithoutCancel(ctx), kubernetesDurableWriteTimeout) +} + +// wireKubernetesInventoryPersistence installs lifecycle-owned callbacks only +// for a fresh Kubernetes launch. Reconnect/replacement already has an +// authoritative row that must never be released as a provisional launch row. +func (m *Manager) wireKubernetesInventoryPersistence(req *ExecutorCreateRequest, executorType string) { + if req == nil || executorType != string(models.ExecutorTypeKubernetes) || m.runningWriter == nil { + return + } + req.CheckpointRuntimeInventory = func(ctx context.Context, metadata map[string]interface{}) error { + return m.checkpointKubernetesRuntimeInventory(ctx, req, metadata) + } + if req.PreviousExecutionID == "" && getMetadataString(req.Metadata, MetadataKeyKubernetesPodName) == "" { + req.ReleaseRuntimeInventory = func(ctx context.Context) error { + return m.releaseKubernetesRuntimeInventory(ctx, req) + } + } +} + +func (m *Manager) checkpointKubernetesRuntimeInventory( + ctx context.Context, + req *ExecutorCreateRequest, + runtimeMetadata map[string]interface{}, +) error { + if m.runningWriter == nil { + return nil + } + if req == nil || strings.TrimSpace(req.InstanceID) == "" || strings.TrimSpace(req.TaskID) == "" || + strings.TrimSpace(req.SessionID) == "" || strings.TrimSpace(req.AgentProfileID) == "" { + return errors.New("checkpoint Kubernetes runtime inventory: launch identity is incomplete") + } + persistCtx, cancelPersist := kubernetesDurableContext(ctx) + defer cancelPersist() + ctx = persistCtx + metadata := cloneKubernetesMetadata(req.Metadata) + for key, value := range runtimeMetadata { + metadata[key] = value + } + if strings.TrimSpace(getMetadataString(metadata, MetadataKeyKubernetesInventoryState)) == "" { + return errors.New("checkpoint Kubernetes runtime inventory: state is missing") + } + + var prior *models.ExecutorRunning + if reader, ok := m.runningWriter.(executorRunningReader); ok { + existing, err := reader.GetExecutorRunningBySessionID(ctx, req.SessionID) + switch { + case err == nil: + prior = existing + case errors.Is(err, models.ErrExecutorRunningNotFound): + default: + return fmt.Errorf("checkpoint Kubernetes runtime inventory: read prior row: %w", err) + } + } + execution := &AgentExecution{ + ID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, + TaskEnvironmentID: req.TaskEnvironmentID, AgentProfileID: req.AgentProfileID, + RuntimeName: agentruntime.RuntimeKubernetes, Status: v1.AgentStatusStarting, + metadata: metadata, + } + running := buildRunningFromExecution(execution, prior) + running.Runtime = agentruntime.RuntimeKubernetes + running.Status = models.ExecutorRunningStatusStarting + if err := m.runningWriter.UpsertExecutorRunning(ctx, running); err != nil { + return fmt.Errorf("checkpoint Kubernetes runtime inventory: upsert row: %w", err) + } + return nil +} + +func (m *Manager) releaseKubernetesRuntimeInventory( + ctx context.Context, + req *ExecutorCreateRequest, +) error { + if m.runningWriter == nil || req == nil { + return nil + } + persistCtx, cancelPersist := kubernetesDurableContext(ctx) + defer cancelPersist() + ctx = persistCtx + reader, hasReader := m.runningWriter.(executorRunningReader) + casWriter, hasCAS := m.runningWriter.(executorRunningCASWriter) + if !hasReader || !hasCAS { + return errors.New("release Kubernetes runtime inventory: exact CAS persistence is unavailable") + } + current, err := reader.GetExecutorRunningBySessionID(ctx, req.SessionID) + if errors.Is(err, models.ErrExecutorRunningNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("release Kubernetes runtime inventory: read row: %w", err) + } + if current == nil || current.AgentExecutionID != req.InstanceID { + return fmt.Errorf("release Kubernetes runtime inventory: %w", models.ErrExecutionRotated) + } + if err := casWriter.DeleteExecutorRunningIfCurrent( + ctx, req.SessionID, req.InstanceID, current.UpdatedAt, + ); err != nil && !errors.Is(err, models.ErrExecutorRunningNotFound) { + return fmt.Errorf("release Kubernetes runtime inventory: delete row: %w", err) + } + return nil +} + +func releaseExecutorInstanceRuntimeInventory(ctx context.Context, instance *ExecutorInstance) error { + if instance == nil || instance.ReleaseRuntimeInventory == nil { + return nil + } + return instance.ReleaseRuntimeInventory(ctx) +} + +func stopRuntimeInstanceAndRelease( + ctx context.Context, + runtime ExecutorBackend, + instance *ExecutorInstance, + force bool, +) error { + if runtime == nil || instance == nil { + return nil + } + if err := runtime.StopInstance(ctx, instance, force); err != nil { + return err + } + if err := releaseExecutorInstanceRuntimeInventory(ctx, instance); err != nil { + return fmt.Errorf("release stopped runtime inventory: %w", err) + } + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_cleanup_current_config_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_cleanup_current_config_test.go new file mode 100644 index 0000000000..21edb548f4 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_cleanup_current_config_test.go @@ -0,0 +1,136 @@ +package lifecycle + +import ( + "context" + "errors" + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/agent/executor" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +func TestStopAgentWithReasonUsesCurrentKubernetesConnectionOnRetry(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := ¤tConnectionStopBackend{ + MockExecutor: MockExecutor{name: executor.NameKubernetes}, + staleErr: errors.New("stale Kubernetes credentials"), + transientErr: errors.New("temporary Kubernetes API failure"), + } + registry.Register(backend) + mgr := NewManager(newTestRegistry(), &MockEventBus{}, registry, nil, nil, nil, ExecutorFallbackWarn, "", log) + cleanupManagerStopCh(t, mgr) + mgr.SetExecutorRunningWriter(&restartKubernetesInventoryStore{executors: map[string]*models.Executor{ + "executor-1": { + ID: "executor-1", Type: models.ExecutorTypeKubernetes, + Config: map[string]string{ + MetadataKeyKubernetesAuthMode: "in_cluster", + MetadataKeyKubernetesConfigNamespace: "new-agents", + MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + }, + }}) + require.NoError(t, mgr.executionStore.Add(&AgentExecution{ + ID: "execution-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: executor.NameKubernetes, Status: v1.AgentStatusRunning, + metadata: map[string]interface{}{ + "executor_id": "executor-1", + MetadataKeyKubernetesAuthMode: "kubeconfig", + MetadataKeyKubernetesKubeconfigPath: "/stale/kubeconfig", + MetadataKeyKubernetesKubeContext: "stale-context", + MetadataKeyKubernetesConfigNamespace: "old-agents", + MetadataKeyKubernetesRequestTimeoutSeconds: "10", + MetadataKeyKubernetesNamespace: "old-agents", + }, + })) + + err := mgr.StopAgentWithReason(context.Background(), "execution-1", StopReasonTaskDeleted, true) + require.ErrorIs(t, err, backend.transientErr) + _, exists := mgr.GetExecution("execution-1") + require.True(t, exists, "transient cleanup failure must retain the execution") + + require.NoError(t, mgr.StopAgentWithReason( + context.Background(), "execution-1", StopReasonTaskDeleted, true, + )) + require.Len(t, backend.calls, 2) + for _, metadata := range backend.calls { + require.Equal(t, "in_cluster", metadata[MetadataKeyKubernetesAuthMode]) + require.Empty(t, metadata[MetadataKeyKubernetesKubeconfigPath]) + require.Empty(t, metadata[MetadataKeyKubernetesKubeContext]) + require.Equal(t, "new-agents", metadata[MetadataKeyKubernetesConfigNamespace]) + require.Equal(t, "old-agents", metadata[MetadataKeyKubernetesNamespace]) + } +} + +func TestKubernetesStopUsesCurrentCleanupClientInsteadOfLaunchSessionClient(t *testing.T) { + controlPort := startKubernetesAgentctlServer(t, true, 41001) + instancePort := startKubernetesAgentctlServer(t, false, 0) + launchResources := &fakeKubernetesResources{} + executorBackend := newFakeKubernetesExecutor( + t, launchResources, &recordingKubernetesExec{}, map[uint16]uint16{ + uint16(kubeexecutor.DefaultAgentctlPort): controlPort, + 41001: instancePort, + }, + ) + req := validKubernetesCreateRequest() + setManagedKubernetesWorkspace(req) + instance, err := executorBackend.CreateInstance(context.Background(), req) + require.NoError(t, err) + + launchResources.mu.Lock() + currentResources := &fakeKubernetesResources{ + pod: launchResources.pod.DeepCopy(), pvc: launchResources.pvc.DeepCopy(), + } + launchResources.getPodErr = apierrors.NewUnauthorized("launch credentials expired") + launchResources.mu.Unlock() + var cleanupConfig kubeexecutor.ExecutorConfig + executorBackend.clientFactory = func(config kubeexecutor.ExecutorConfig) (*kubernetesRuntimeClient, error) { + cleanupConfig = config + return &kubernetesRuntimeClient{resources: currentResources}, nil + } + instance.Metadata[MetadataKeyKubernetesAuthMode] = "in_cluster" + instance.Metadata[MetadataKeyKubernetesKubeconfigPath] = "" + instance.Metadata[MetadataKeyKubernetesKubeContext] = "" + instance.Metadata[MetadataKeyKubernetesConfigNamespace] = "new-agents" + instance.Metadata[MetadataKeyKubernetesRequestTimeoutSeconds] = "45" + instance.StopReason = StopReasonTaskDeleted + + require.NoError(t, executorBackend.StopInstance(context.Background(), instance, true)) + require.Equal(t, kubeexecutor.AuthModeInCluster, cleanupConfig.AuthMode) + require.Equal(t, "new-agents", cleanupConfig.Namespace) + require.Equal(t, []string{"pod", "pvc"}, currentResources.deletionOrder) +} + +type currentConnectionStopBackend struct { + MockExecutor + staleErr error + transientErr error + currentAttempts int + calls []map[string]interface{} +} + +func (b *currentConnectionStopBackend) StopInstance( + _ context.Context, + instance *ExecutorInstance, + _ bool, +) error { + metadata := cloneKubernetesMetadata(instance.Metadata) + b.calls = append(b.calls, metadata) + if getMetadataString(metadata, MetadataKeyKubernetesAuthMode) != "in_cluster" || + getMetadataString(metadata, MetadataKeyKubernetesKubeconfigPath) != "" || + getMetadataString(metadata, MetadataKeyKubernetesKubeContext) != "" || + getMetadataString(metadata, MetadataKeyKubernetesConfigNamespace) != "new-agents" { + return b.staleErr + } + b.currentAttempts++ + if b.currentAttempts == 1 { + return b.transientErr + } + return nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_refresh.go b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_refresh.go new file mode 100644 index 0000000000..4887f324f1 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_refresh.go @@ -0,0 +1,284 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "strings" + + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + agentctltypes "github.com/kandev/kandev/internal/agentctl/types" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" +) + +func (m *Manager) refreshTrackedRemoteInstance( + ctx context.Context, + execution *AgentExecution, + refresher RemoteInstanceRefresher, +) error { + _, err, _ := m.remoteRefreshGroup.Do(execution.ID, func() (interface{}, error) { + return nil, m.refreshTrackedRemoteInstanceOnce(ctx, execution, refresher) + }) + return err +} + +func (m *Manager) refreshTrackedRemoteInstanceOnce( + ctx context.Context, + execution *AgentExecution, + refresher RemoteInstanceRefresher, +) error { + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + current, exists := m.executionStore.Get(execution.ID) + if !exists || current != execution { + return nil + } + instance, err := m.kubernetesRefreshStatusInstance(ctx, execution) + if err != nil { + return err + } + refresh, err := refresher.RefreshRemoteInstance(ctx, instance) + if err != nil || refresh == nil { + return err + } + return m.applyTrackedRemoteRefresh(ctx, execution, refresh) +} + +type kubernetesExecutorReader interface { + GetExecutor(ctx context.Context, id string) (*models.Executor, error) +} + +func (m *Manager) kubernetesRefreshStatusInstance( + ctx context.Context, + execution *AgentExecution, +) (*ExecutorInstance, error) { + instance := m.remoteStatusInstance(ctx, execution) + if execution == nil || execution.RuntimeName != agentruntime.RuntimeKubernetes { + return instance, nil + } + metadata, err := m.currentKubernetesConnectionMetadata(ctx, instance.Metadata) + if err != nil { + return nil, err + } + instance.Metadata = metadata + return instance, nil +} + +func (m *Manager) currentKubernetesConnectionMetadata( + ctx context.Context, + metadata map[string]interface{}, +) (map[string]interface{}, error) { + reader, ok := m.runningWriter.(kubernetesExecutorReader) + executorID := strings.TrimSpace(getMetadataString(metadata, "executor_id")) + if !ok || executorID == "" { + return metadata, nil + } + current, err := reader.GetExecutor(ctx, executorID) + if err != nil { + return nil, fmt.Errorf("load current Kubernetes executor %q: %w", executorID, err) + } + if current == nil || current.Type != models.ExecutorTypeKubernetes { + return nil, fmt.Errorf("current executor %q is not a Kubernetes executor", executorID) + } + config, err := kubeexecutor.ParseExecutorConfig(current.Config) + if err != nil { + return nil, fmt.Errorf("parse current Kubernetes executor %q: %w", executorID, err) + } + return overlayCurrentKubernetesConnectionMetadata(metadata, current.Config, config), nil +} + +func overlayCurrentKubernetesConnectionMetadata( + metadata map[string]interface{}, + configValues map[string]string, + config kubeexecutor.ExecutorConfig, +) map[string]interface{} { + resolved := cloneKubernetesMetadata(metadata) + for _, key := range kubernetesConnectionMetadataKeys { + resolved[key] = configValues[key] + } + resolved[MetadataKeyKubernetesExecutorConfigHash] = kubernetesConfigHash(config) + return resolved +} + +func (m *Manager) applyTrackedRemoteRefresh( + ctx context.Context, + execution *AgentExecution, + refresh *RemoteInstanceRefresh, +) error { + committed := false + defer func() { + if !committed && refresh.Abort != nil { + refresh.Abort() + } + }() + if refresh.Instance == nil || refresh.Instance.Client == nil { + return errors.New("remote refresh returned an incomplete agentctl instance") + } + newACPSessionID, err := m.prepareKubernetesRefreshProcess(ctx, execution, refresh) + if err != nil { + return err + } + rollbackPersistence, err := m.persistActiveKubernetesRefresh(ctx, execution, refresh.Instance) + if err != nil { + return err + } + committed, err = m.commitTrackedRemoteRefresh( + ctx, execution, refresh, newACPSessionID, rollbackPersistence, + ) + if err != nil { + return err + } + if m.streamManager != nil { + go m.streamManager.ReconnectAll(execution) + } + return nil +} + +func (m *Manager) prepareKubernetesRefreshProcess( + ctx context.Context, + execution *AgentExecution, + refresh *RemoteInstanceRefresh, +) (string, error) { + if !refresh.ProcessRestarted { + return "", nil + } + return m.prepareRestartedKubernetesAgentctl(ctx, execution, refresh) +} + +func (m *Manager) commitTrackedRemoteRefresh( + ctx context.Context, + execution *AgentExecution, + refresh *RemoteInstanceRefresh, + newACPSessionID string, + rollbackPersistence func(context.Context) error, +) (bool, error) { + if refresh.Commit == nil { + rollbackErr := rollbackPersistence(ctx) + return false, errors.Join(errors.New("remote refresh has no commit operation"), rollbackErr) + } + published := false + execution.agentctlLifecycleMu.Lock() + commitErr := refresh.Commit(func() { + refresh.Instance.Client.SetTraceContext(execution.SessionTraceContext()) + execution.replaceAgentctlClient(refresh.Instance.Client) + if newACPSessionID != "" { + execution.ACPSessionID = newACPSessionID + execution.setSessionInitialized(true) + } + published = true + }) + execution.agentctlLifecycleMu.Unlock() + if commitErr != nil && published { + return true, fmt.Errorf("remote refresh commit failed after publishing replacement: %w", commitErr) + } + if commitErr != nil { + return false, errors.Join(commitErr, rollbackPersistence(ctx)) + } + if !published { + return true, errors.New("remote refresh commit did not publish its replacement") + } + return true, nil +} + +func (m *Manager) prepareRestartedKubernetesAgentctl( + ctx context.Context, + execution *AgentExecution, + refresh *RemoteInstanceRefresh, +) (string, error) { + client := refresh.Instance.Client + env := execution.RuntimeEnvironment() + if env == nil { + env = runtimeEnvFromMetadata(execution.MetadataSnapshot()) + } + approvalPolicy := "untrusted" + if refresh.AutoApprovePermissions { + approvalPolicy = "never" + } + if execution.AgentCommand == "" { + return "", fmt.Errorf("execution %q has no recorded agent command for Kubernetes restart", execution.ID) + } + if err := client.ConfigureAgent( + ctx, execution.AgentCommand, execution.AgentArgs, env, approvalPolicy, + execution.ContinueCommand, execution.ContinueArgs, + ); err != nil { + return "", fmt.Errorf("configure agent after Kubernetes restart: %w", err) + } + if _, err := client.Start(ctx); err != nil { + return "", fmt.Errorf("start agent after Kubernetes restart: %w", err) + } + if refresh.AgentConfig == nil || m.sessionManager == nil { + return "", nil + } + result, err := m.sessionManager.InitializeSession( + ctx, client, refresh.AgentConfig, execution.ACPSessionID, + execution.WorkspacePath, kubernetesRefreshMcpServers(refresh.McpServers), + ) + if err != nil { + return "", fmt.Errorf("resume ACP session after Kubernetes restart: %w", err) + } + return result.SessionID, nil +} + +func kubernetesRefreshMcpServers(configs []McpServerConfig) []agentctltypes.McpServer { + servers := make([]agentctltypes.McpServer, 0, len(configs)) + for _, config := range configs { + servers = append(servers, agentctltypes.McpServer{ + Name: config.Name, Command: config.Command, Args: append([]string(nil), config.Args...), + URL: config.URL, Type: config.Type, Env: cloneStringMap(config.Env), + Headers: cloneStringMap(config.Headers), + }) + } + return servers +} + +func (m *Manager) persistActiveKubernetesRefresh( + ctx context.Context, + execution *AgentExecution, + instance *ExecutorInstance, +) (func(context.Context) error, error) { + if m.secretStore == nil || m.runningWriter == nil { + return nil, errors.New("persist Kubernetes remote refresh: durable stores are unavailable") + } + persistCtx, cancelPersist := kubernetesDurableContext(ctx) + defer cancelPersist() + ctx = persistCtx + oldMetadata := execution.MetadataSnapshot() + authID := getMetadataString(oldMetadata, MetadataKeyAuthTokenSecret) + nonceID := getMetadataString(oldMetadata, MetadataKeyBootstrapNonceSecret) + if authID == "" || nonceID == "" || instance.AuthToken == "" || instance.BootstrapNonce == "" { + return nil, errors.New("persist Kubernetes remote refresh: required secret references are incomplete") + } + oldToken, err := revealGlobalSecret(ctx, m.secretStore, authID) + if err != nil { + return nil, fmt.Errorf("reveal prior Kubernetes auth token: %w", err) + } + nonce, err := revealGlobalSecret(ctx, m.secretStore, nonceID) + if err != nil { + return nil, fmt.Errorf("reveal Kubernetes bootstrap nonce: %w", err) + } + if nonce != instance.BootstrapNonce { + return nil, errors.New("persist Kubernetes remote refresh: bootstrap nonce changed") + } + if err := m.secretStore.Update(ctx, authID, &secrets.UpdateSecretRequest{Value: &instance.AuthToken}); err != nil { + return nil, fmt.Errorf("rotate Kubernetes auth token: %w", err) + } + rollbackMetadata := execution.mergeMetadataWithRollback(instance.Metadata) + if err := m.persistExecutorRunningResult(ctx, execution); err != nil { + rollbackMetadata() + restoreErr := m.secretStore.Update(ctx, authID, &secrets.UpdateSecretRequest{Value: &oldToken}) + return nil, errors.Join(fmt.Errorf("persist Kubernetes refreshed inventory: %w", err), restoreErr) + } + rollback := func(rollbackCtx context.Context) error { + rollbackMetadata() + durableRollbackCtx, cancelRollback := kubernetesDurableContext(rollbackCtx) + defer cancelRollback() + secretErr := m.secretStore.Update( + durableRollbackCtx, authID, &secrets.UpdateSecretRequest{Value: &oldToken}, + ) + rowErr := m.persistExecutorRunningResult(durableRollbackCtx, execution) + return errors.Join(secretErr, rowErr) + } + return rollback, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_restart_cleanup.go b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_restart_cleanup.go new file mode 100644 index 0000000000..4401c26223 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_kubernetes_restart_cleanup.go @@ -0,0 +1,165 @@ +package lifecycle + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/kandev/kandev/internal/agent/executor" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/runtime/activity" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +// persistedKubernetesCleanupStore is intentionally narrower than the task +// repository. It lets terminal task cleanup recover exact Kubernetes inventory +// after a backend restart without making ordinary lifecycle code depend on the +// repository implementation. +type persistedKubernetesCleanupStore interface { + ListExecutorsRunning(ctx context.Context) ([]*models.ExecutorRunning, error) + GetExecutor(ctx context.Context, id string) (*models.Executor, error) +} + +// stopPersistedKubernetesExecution handles the restart-only case where the +// authoritative executors_running row survived but process-local lifecycle +// state did not. The caller that owns task cleanup remains responsible for +// deleting the row after this exact remote teardown succeeds. +func (m *Manager) stopPersistedKubernetesExecution( + ctx context.Context, + executionID string, + reason string, + force bool, +) (bool, error) { + if !force && !shouldRunExecutorCleanup(reason) { + return false, nil + } + store, ok := m.runningWriter.(persistedKubernetesCleanupStore) + if !ok { + return false, nil + } + row, err := loadPersistedKubernetesExecution(ctx, store, executionID) + if err != nil { + return true, err + } + if row == nil || row.Runtime != agentruntime.RuntimeKubernetes { + return false, nil + } + + activityLease, err := m.acquireActivity(ctx, activity.KindExecutionStopping) + if err != nil { + return true, err + } + defer activityLease.Release() + metadata, err := resolvePersistedKubernetesCleanupMetadata(ctx, store, row) + if err != nil { + return true, err + } + instance := &ExecutorInstance{ + InstanceID: row.AgentExecutionID, + TaskID: row.TaskID, + SessionID: row.SessionID, + RuntimeName: agentruntime.RuntimeKubernetes, + Metadata: metadata, + StopReason: reason, + } + if err := m.stopPersistedKubernetesBackend(ctx, instance, force); err != nil { + return true, fmt.Errorf("stop persisted Kubernetes runtime: %w", err) + } + if err := m.deleteKubernetesRuntimeSecrets(ctx, metadata); err != nil { + return true, fmt.Errorf("delete persisted Kubernetes runtime secrets: %w", err) + } + return true, nil +} + +func loadPersistedKubernetesExecution( + ctx context.Context, + store persistedKubernetesCleanupStore, + executionID string, +) (*models.ExecutorRunning, error) { + rows, err := store.ListExecutorsRunning(ctx) + if err != nil { + return nil, fmt.Errorf("load persisted runtime inventory: %w", err) + } + return uniquePersistedKubernetesExecution(rows, executionID) +} + +func resolvePersistedKubernetesCleanupMetadata( + ctx context.Context, + store persistedKubernetesCleanupStore, + row *models.ExecutorRunning, +) (map[string]interface{}, error) { + currentExecutor, err := store.GetExecutor(ctx, strings.TrimSpace(row.ExecutorID)) + if err != nil { + return nil, fmt.Errorf("load current Kubernetes executor %q: %w", row.ExecutorID, err) + } + if currentExecutor == nil { + return nil, fmt.Errorf("load current Kubernetes executor %q: %w", row.ExecutorID, models.ErrExecutorNotFound) + } + if currentExecutor.Type != models.ExecutorTypeKubernetes { + return nil, fmt.Errorf( + "persisted Kubernetes runtime executor %q now has type %q", row.ExecutorID, currentExecutor.Type, + ) + } + currentConfig, err := kubeexecutor.ParseExecutorConfig(currentExecutor.Config) + if err != nil { + return nil, fmt.Errorf("parse current Kubernetes executor %q: %w", row.ExecutorID, err) + } + metadata, err := persistedKubernetesCleanupMetadata(row, currentExecutor.Config, currentConfig) + if err != nil { + return nil, err + } + req := &ExecutorCreateRequest{TaskID: row.TaskID, SessionID: row.SessionID, Metadata: metadata} + if _, _, err := kubernetesRecordedCleanupInventory(req, currentConfig, true); err != nil { + return nil, fmt.Errorf("validate persisted Kubernetes runtime inventory: %w", err) + } + return metadata, nil +} + +func (m *Manager) stopPersistedKubernetesBackend( + ctx context.Context, + instance *ExecutorInstance, + force bool, +) error { + if m.executorRegistry == nil { + return errors.New("kubernetes runtime registry is unavailable") + } + backend, err := m.executorRegistry.GetBackend(executor.NameKubernetes) + if err != nil { + return fmt.Errorf("get Kubernetes runtime: %w", err) + } + return backend.StopInstance(ctx, instance, force) +} + +func uniquePersistedKubernetesExecution( + rows []*models.ExecutorRunning, + executionID string, +) (*models.ExecutorRunning, error) { + var found *models.ExecutorRunning + for _, row := range rows { + if row == nil || strings.TrimSpace(row.AgentExecutionID) != executionID { + continue + } + if found != nil { + return nil, fmt.Errorf("multiple persisted runtime rows match execution %q", executionID) + } + found = row + } + return found, nil +} + +func persistedKubernetesCleanupMetadata( + row *models.ExecutorRunning, + currentConfigValues map[string]string, + currentConfig kubeexecutor.ExecutorConfig, +) (map[string]interface{}, error) { + if row == nil || strings.TrimSpace(row.AgentExecutionID) == "" || + strings.TrimSpace(row.TaskID) == "" || strings.TrimSpace(row.SessionID) == "" || + strings.TrimSpace(row.ExecutorID) == "" { + return nil, errors.New("persisted Kubernetes runtime row identity is incomplete") + } + metadata := overlayCurrentKubernetesConnectionMetadata(row.Metadata, currentConfigValues, currentConfig) + metadata["executor_id"] = strings.TrimSpace(row.ExecutorID) + return metadata, nil +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go index b6ba8691e7..5cdba4d229 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch.go @@ -2,6 +2,7 @@ package lifecycle import ( "context" + "encoding/json" "errors" "fmt" "os" @@ -15,6 +16,7 @@ import ( "github.com/kandev/kandev/internal/agent/agents" "github.com/kandev/kandev/internal/agent/executor" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" "github.com/kandev/kandev/internal/agent/runtime/activity" "github.com/kandev/kandev/internal/agent/settings/cliflags" "github.com/kandev/kandev/internal/agentruntime" @@ -117,18 +119,160 @@ func applyRouteOverrideToProfile(profile *AgentProfileInfo, req *LaunchRequest) // task override them would allow pivoting an SSH launch to a different host // or bypassing the pinned host-key. var trustedExecutorConfigKeys = map[string]bool{ - MetadataKeySSHHost: true, - MetadataKeySSHHostAlias: true, - MetadataKeySSHPort: true, - MetadataKeySSHUser: true, - MetadataKeySSHHostFingerprint: true, - MetadataKeySSHIdentitySource: true, - MetadataKeySSHIdentityFile: true, - MetadataKeySSHProxyJump: true, + MetadataKeySSHHost: true, + MetadataKeySSHHostAlias: true, + MetadataKeySSHPort: true, + MetadataKeySSHUser: true, + MetadataKeySSHHostFingerprint: true, + MetadataKeySSHIdentitySource: true, + MetadataKeySSHIdentityFile: true, + MetadataKeySSHProxyJump: true, + MetadataKeyKubernetesAuthMode: true, + MetadataKeyKubernetesKubeconfigPath: true, + MetadataKeyKubernetesKubeContext: true, + MetadataKeyKubernetesConfigNamespace: true, + MetadataKeyKubernetesRequestTimeoutSeconds: true, } func isTrustedExecutorConfigKey(k string) bool { return trustedExecutorConfigKeys[k] } +var kubernetesProfileMetadataKeys = [...]string{ + MetadataKeyKubernetesProfilePlatform, + MetadataKeyKubernetesProfileMainContainer, + MetadataKeyKubernetesPodTemplateYAML, + MetadataKeyKubernetesWorkspaceMode, + MetadataKeyKubernetesWorkspaceSize, + MetadataKeyKubernetesWorkspaceStorageClass, + MetadataKeyKubernetesWorkspaceAccessModes, + MetadataKeyKubernetesWorkspaceClaimName, +} + +var kubernetesConnectionMetadataKeys = [...]string{ + MetadataKeyKubernetesAuthMode, + MetadataKeyKubernetesKubeconfigPath, + MetadataKeyKubernetesKubeContext, + MetadataKeyKubernetesConfigNamespace, + MetadataKeyKubernetesRequestTimeoutSeconds, +} + +// applyAuthoritativeKubernetesProfileConfig replaces fresh-launch Pod and +// workspace inputs with the stored executor-profile value. A retained launch +// keeps its immutable lifecycle-produced snapshot so profile edits or deletion +// cannot change or prevent recovery of the recorded workload. +func (m *Manager) applyAuthoritativeKubernetesProfileConfig( + ctx context.Context, + req *LaunchRequest, + metadata map[string]interface{}, +) error { + if req == nil || req.ExecutorType != string(models.ExecutorTypeKubernetes) { + return nil + } + if req.PreviousExecutionID != "" && hasCompleteKubernetesRecordedResumeMetadata(metadata) { + return nil + } + if m.executorProfileReader == nil { + return errors.New("load Kubernetes executor profile: profile reader is not configured") + } + profileID := strings.TrimSpace(getMetadataString(metadata, MetadataKeyExecutorProfileID)) + if profileID == "" { + return errors.New("load Kubernetes executor profile: profile ID is missing") + } + profile, err := m.executorProfileReader.GetExecutorProfile(ctx, profileID) + if err != nil { + return fmt.Errorf("load Kubernetes executor profile %q: %w", profileID, err) + } + if profile == nil { + return fmt.Errorf("load Kubernetes executor profile %q: not found", profileID) + } + if strings.TrimSpace(profile.ExecutorID) == "" { + return fmt.Errorf("load Kubernetes executor profile %q: executor ID is missing", profileID) + } + executorID := strings.TrimSpace(getMetadataString(metadata, "executor_id")) + if executorID != "" && executorID != profile.ExecutorID { + return fmt.Errorf( + "load Kubernetes executor profile %q: belongs to executor %q, not %q", + profileID, profile.ExecutorID, executorID, + ) + } + typedProfile, err := kubeexecutor.ParseProfileConfig(profile.Config) + if err != nil { + return fmt.Errorf("load Kubernetes executor profile %q: invalid config: %w", profileID, err) + } + metadata["executor_id"] = profile.ExecutorID + if err := applyKubernetesProfileConfigToMetadata(metadata, typedProfile); err != nil { + return fmt.Errorf("load Kubernetes executor profile %q: canonicalize config: %w", profileID, err) + } + return nil +} + +func applyKubernetesProfileConfigToMetadata( + metadata map[string]interface{}, + profile kubeexecutor.ProfileConfig, +) error { + accessModes := "" + if len(profile.Workspace.AccessModes) > 0 { + encoded, err := json.Marshal(profile.Workspace.AccessModes) + if err != nil { + return err + } + accessModes = string(encoded) + } + values := map[string]string{ + MetadataKeyKubernetesProfilePlatform: string(profile.Platform), + MetadataKeyKubernetesProfileMainContainer: profile.MainContainer, + MetadataKeyKubernetesPodTemplateYAML: profile.PodTemplateYAML, + MetadataKeyKubernetesWorkspaceMode: string(profile.Workspace.Mode), + MetadataKeyKubernetesWorkspaceSize: profile.Workspace.Size, + MetadataKeyKubernetesWorkspaceStorageClass: profile.Workspace.StorageClass, + MetadataKeyKubernetesWorkspaceAccessModes: accessModes, + MetadataKeyKubernetesWorkspaceClaimName: profile.Workspace.ClaimName, + } + for _, key := range kubernetesProfileMetadataKeys { + metadata[key] = values[key] + } + return nil +} + +func hasCompleteKubernetesRecordedResumeMetadata(metadata map[string]interface{}) bool { + required := []string{ + MetadataKeyKubernetesNamespace, + MetadataKeyKubernetesPodName, + MetadataKeyKubernetesPodUID, + MetadataKeyKubernetesMainContainer, + MetadataKeyKubernetesRuntimeWorkspaceMode, + MetadataKeyKubernetesAgentctlRemotePort, + MetadataKeyKubernetesAgentctlInstanceID, + MetadataKeyKubernetesResourceExecutorID, + MetadataKeyKubernetesResourceProfileID, + MetadataKeyKubernetesResourceInstanceID, + MetadataKeyKubernetesResourceTaskID, + MetadataKeyKubernetesResourceSessionID, + MetadataKeyKubernetesResourceEnvironmentID, + MetadataKeyKubernetesExecutorConfigHash, + MetadataKeyKubernetesProfileConfigHash, + MetadataKeyKubernetesTemplateHash, + MetadataKeyKubernetesProfileSnapshot, + } + for _, key := range required { + if getMetadataString(metadata, key) == "" { + return false + } + } + switch getMetadataString(metadata, MetadataKeyKubernetesRuntimeWorkspaceMode) { + case string(kubeexecutor.WorkspaceModeEmptyDir): + return true + case string(kubeexecutor.WorkspaceModeManagedPVC): + return getMetadataString(metadata, MetadataKeyKubernetesPVCName) != "" && + getMetadataString(metadata, MetadataKeyKubernetesPVCUID) != "" && + getMetadataBool(metadata, MetadataKeyKubernetesPVCCreated) + case string(kubeexecutor.WorkspaceModeExistingClaim): + return getMetadataString(metadata, MetadataKeyKubernetesPVCName) != "" && + getMetadataString(metadata, MetadataKeyKubernetesPVCUID) != "" + default: + return false + } +} + // buildLaunchMetadata builds runtime metadata for the Launch request. // // Per-executor config (host, fingerprint, ssh identity, …) is the trusted @@ -153,6 +297,11 @@ func buildLaunchMetadata(req *LaunchRequest, mainRepoGitDir, worktreeID, worktre metadata[k] = v } } + if req.ExecutorType == string(models.ExecutorTypeKubernetes) { + for _, key := range kubernetesConnectionMetadataKeys { + metadata[key] = req.ExecutorConfig[key] + } + } if mainRepoGitDir != "" { metadata[MetadataKeyMainRepoGitDir] = mainRepoGitDir } @@ -843,6 +992,9 @@ func (m *Manager) launchBuildExecutorRequest(ctx context.Context, executionID st } metadata := buildLaunchMetadata(reqWithWorktree, mainRepoGitDir, worktreeID, worktreeBranch) + if err := m.applyAuthoritativeKubernetesProfileConfig(ctx, reqWithWorktree, metadata); err != nil { + return nil, nil, nil, err + } remoteContributions, err := collectRemoteContributions(reqWithWorktree) if err != nil { return nil, nil, nil, err @@ -906,6 +1058,7 @@ func (m *Manager) launchBuildExecutorRequest(ctx context.Context, executionID st ContributionDestinations: contributionDestinations, ComparisonTargets: comparisonTargets, } + m.wireKubernetesInventoryPersistence(execReq, reqWithWorktree.ExecutorType) launchCtx, launchCancel := withLaunchPhaseTimeout(ctx) defer launchCancel() @@ -1270,10 +1423,12 @@ func (m *Manager) promoteWorkspaceExecution(ctx context.Context, execution *Agen } defer activityLease.Release() if len(req.McpProviders) > 0 { - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return nil, fmt.Errorf("execution %q has no agentctl client for MCP provider promotion", execution.ID) } - if err := execution.agentctl.SetMcpProviders(sharedCtx, req.McpProviders); err != nil { + if err := client.SetMcpProviders(sharedCtx, req.McpProviders); err != nil { return nil, fmt.Errorf("set MCP providers during workspace execution promotion: %w", err) } } @@ -1447,8 +1602,11 @@ func (m *Manager) launchInternal(ctx context.Context, req *LaunchRequest) (*Agen projectionErr = materializeWorkspaceRepositories(ctx, execInstance.Client, projection) } if projectionErr != nil { - _ = rt.StopInstance(context.WithoutCancel(ctx), execInstance, false) - err = fmt.Errorf("reconstruct remote workspace repositories: %w", projectionErr) + rollbackErr := stopRuntimeInstanceAndRelease(context.WithoutCancel(ctx), rt, execInstance, true) + err = errors.Join( + fmt.Errorf("reconstruct remote workspace repositories: %w", projectionErr), + rollbackErr, + ) m.publishLaunchPrepareCompleted(req, prepResult, progressRecorder, workspacePath, false, err) return nil, err } @@ -1479,7 +1637,7 @@ func (m *Manager) launchInternal(ctx context.Context, req *LaunchRequest) (*Agen // instance directly to avoid leaking it, then fail closed. if rt != nil && execInstance != nil { cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - if stopErr := rt.StopInstance(cleanupCtx, execInstance, true); stopErr != nil { + if stopErr := stopRuntimeInstanceAndRelease(cleanupCtx, rt, execInstance, true); stopErr != nil { m.logger.Warn("failed to stop runtime instance after command resolution error", zap.Error(stopErr)) } @@ -1570,12 +1728,23 @@ func (m *Manager) registerAndPublishExecution( } return fmt.Errorf("failed to register execution: %w", addErr) } + isKubernetes := execution.RuntimeName == agentruntime.RuntimeKubernetes + var createdRuntimeSecrets map[string]bool + if isKubernetes { + var err error + createdRuntimeSecrets, err = m.persistRequiredKubernetesRuntimeSecrets(ctx, execInstance, execution) + if err != nil { + m.rollbackRegisteredLaunch(rt, execInstance, execution, "Kubernetes runtime secret persistence failed") + return err + } + } // Make the execution visible to durable cleanup before the final session // read. This closes the precheck -> Add -> persist gap: deletion cleanup can // now inventory the row, while a deletion that already ran is caught below. if err := m.persistExecutorRunningResult(ctx, execution); err != nil { + secretCleanupErr := m.deleteCreatedRuntimeSecrets(ctx, execution, createdRuntimeSecrets) m.rollbackRegisteredLaunchAfterPersistFailure(rt, execInstance, execution) - return fmt.Errorf("persist execution registration: %w", err) + return errors.Join(fmt.Errorf("persist execution registration: %w", err), secretCleanupErr) } if err := m.ensureLaunchSessionStillActive(ctx, sessionID); err != nil { @@ -1588,7 +1757,12 @@ func (m *Manager) registerAndPublishExecution( } m.setRuntimeInterest(execution.SessionID, true) - m.persistRuntimeSecrets(ctx, execInstance, execution) + if !isKubernetes { + if err := m.persistRuntimeSecrets(ctx, execInstance, execution); err != nil { + m.rollbackRegisteredLaunch(rt, execInstance, execution, "runtime secret persistence failed") + return err + } + } go m.pollOneRemoteStatus(context.Background(), execution) @@ -1675,9 +1849,20 @@ func (m *Manager) rollbackRegisteredLaunchWithRetry( execution *AgentExecution, taskCleanupActive bool, reason string, +) { + m.rollbackRegisteredLaunchWithRetryMode(rt, execInstance, execution, taskCleanupActive, false, reason) +} + +func (m *Manager) rollbackRegisteredLaunchWithRetryMode( + rt ExecutorBackend, + execInstance *ExecutorInstance, + execution *AgentExecution, + taskCleanupActive bool, + discardDurable bool, + reason string, ) { if err := m.stopRegisteredLaunchRuntime(rt, execInstance, execution); err == nil { - m.finishRegisteredLaunchRollback(execution, taskCleanupActive) + m.finishRegisteredLaunchRollback(execution, taskCleanupActive, discardDurable) return } else { m.logger.Warn("registered launch rollback retained ownership after stop failure", @@ -1702,7 +1887,7 @@ func (m *Manager) rollbackRegisteredLaunchWithRetry( return } if err := m.stopRegisteredLaunchRuntime(rt, execInstance, execution); err == nil { - m.finishRegisteredLaunchRollback(execution, taskCleanupActive) + m.finishRegisteredLaunchRollback(execution, taskCleanupActive, discardDurable) return } else { m.logger.Warn("registered launch rollback retry failed", @@ -1723,36 +1908,67 @@ func (m *Manager) stopRegisteredLaunchRuntime( execInstance *ExecutorInstance, execution *AgentExecution, ) error { + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + execution.agentctlLifecycleMu.Lock() + defer execution.agentctlLifecycleMu.Unlock() if rt != nil && execInstance != nil { cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - err := rt.StopInstance(cleanupCtx, execInstance, true) + retainedKubernetesResume := execution.RuntimeName == agentruntime.RuntimeKubernetes && execution.isResumedSession + err := rt.StopInstance(cleanupCtx, execInstance, !retainedKubernetesResume) + if err == nil && execution.RuntimeName == agentruntime.RuntimeKubernetes && !retainedKubernetesResume { + err = m.deleteKubernetesRuntimeSecrets(cleanupCtx, execution.MetadataSnapshot()) + } + if err == nil && !retainedKubernetesResume { + err = releaseExecutorInstanceRuntimeInventory(cleanupCtx, execInstance) + if err != nil { + err = fmt.Errorf("release stopped runtime inventory: %w", err) + } + } cancel() if err != nil { return err } } - if execution.agentctl != nil { - execution.agentctl.Close() + if client := execution.currentAgentCtlClient(); client != nil { // protected by agentctlLifecycleMu + client.Close() } execution.EndSessionSpan() return nil } -func (m *Manager) finishRegisteredLaunchRollback(execution *AgentExecution, taskCleanupActive bool) { +func (m *Manager) finishRegisteredLaunchRollback(execution *AgentExecution, taskCleanupActive, discardDurable bool) { + if execution.RuntimeName == agentruntime.RuntimeKubernetes && execution.isResumedSession { + // A failed resume only owns the newly opened local client and forward. The + // recorded Pod/PVC inventory remains the authority for a later retry or + // terminal cleanup, so never discard its durable row here. + m.executionStore.Remove(execution.ID) + return + } cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if taskCleanupActive { + if !discardDurable || taskCleanupActive { m.deleteExecutorRunning(cleanupCtx, execution.SessionID) } else if reader, ok := m.runningWriter.(executorRunningReader); ok { running, err := reader.GetExecutorRunningBySessionID(cleanupCtx, execution.SessionID) if err == nil && running != nil && running.AgentExecutionID == execution.ID { - m.deleteExecutorRunning(cleanupCtx, execution.SessionID) + if err := m.deleteExecutorRunningRow(cleanupCtx, execution.SessionID, execution.ID); err != nil && + !errors.Is(err, models.ErrExecutorRunningNotFound) { + m.logger.Warn("failed to delete exact executor-running row after launch rollback", + zap.String("execution_id", execution.ID), + zap.String("session_id", execution.SessionID), + zap.Error(err)) + } } } m.executionStore.Remove(execution.ID) } func (m *Manager) rollbackLaunchExecution(_ context.Context, rt ExecutorBackend, execInstance *ExecutorInstance, execution *AgentExecution, reason string) { + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + execution.agentctlLifecycleMu.Lock() + defer execution.agentctlLifecycleMu.Unlock() m.logger.Warn("rolling back launch execution", zap.String("execution_id", execution.ID), zap.String("session_id", execution.SessionID), @@ -1760,36 +1976,24 @@ func (m *Manager) rollbackLaunchExecution(_ context.Context, rt ExecutorBackend, if rt != nil && execInstance != nil { cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - if stopErr := rt.StopInstance(cleanupCtx, execInstance, true); stopErr != nil { + if stopErr := stopRuntimeInstanceAndRelease(cleanupCtx, rt, execInstance, true); stopErr != nil { m.logger.Warn("failed to stop runtime instance during launch rollback", zap.String("execution_id", execution.ID), zap.Error(stopErr)) } } - if execution.agentctl != nil { - execution.agentctl.Close() + if client := execution.currentAgentCtlClient(); client != nil { // protected by agentctlLifecycleMu + client.Close() } execution.EndSessionSpan() } -// rollbackRegisteredLaunch removes both sides of an execution registration -// before stopping its runtime. This path intentionally deletes the durable row -// without the normal resume-token repair: the owning session was just proven -// terminal or absent, so leaving a repaired row would expose a phantom runtime. +// rollbackRegisteredLaunch stops the runtime before removing either side of +// the execution registration. A failed stop retains both in-memory and durable +// ownership for retry; successful terminal rollback removes the exact row +// without normal resume-token repair. func (m *Manager) rollbackRegisteredLaunch(rt ExecutorBackend, execInstance *ExecutorInstance, execution *AgentExecution, reason string) { - m.executionStore.Remove(execution.ID) - if m.runningWriter != nil && execution.SessionID != "" { - cleanupCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := m.runningWriter.DeleteExecutorRunningBySessionID(cleanupCtx, execution.SessionID); err != nil && - !errors.Is(err, models.ErrExecutorRunningNotFound) { - m.logger.Warn("failed to delete executor-running row during launch rollback", - zap.String("execution_id", execution.ID), - zap.String("session_id", execution.SessionID), - zap.Error(err)) - } - cancel() - } - m.rollbackLaunchExecution(context.Background(), rt, execInstance, execution, reason) + m.rollbackRegisteredLaunchWithRetryMode(rt, execInstance, execution, false, true, reason) } // SetExecutionDescription updates the task description stored in an execution's metadata. @@ -1844,10 +2048,12 @@ func (m *Manager) SetMcpMode(ctx context.Context, executionID string, mode strin if !exists { return fmt.Errorf("execution %q not found", executionID) } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } - return execution.agentctl.SetMcpMode(ctx, mode) + return client.SetMcpMode(ctx, mode) } // SetMcpProvidersForSession replaces the task-mode MCP provider capabilities @@ -1865,10 +2071,12 @@ func (m *Manager) SetMcpProvidersForSession(ctx context.Context, sessionID strin zap.String("session_id", sessionID)) return nil } - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", execution.ID) } - if err := execution.agentctl.SetMcpProviders(ctx, providers); err != nil { + if err := client.SetMcpProviders(ctx, providers); err != nil { return fmt.Errorf("set MCP providers for session %s: %w", sessionID, err) } return nil @@ -1880,10 +2088,16 @@ func (m *Manager) SetMcpProvidersForSession(ctx context.Context, sessionID strin func (m *Manager) SetPluginToolsForAllExecutions(ctx context.Context, snapshot plugintools.Snapshot) error { var refreshErr error for _, execution := range m.ListExecutions() { - if execution == nil || execution.agentctl == nil { + if execution == nil { continue } - if err := execution.agentctl.SetPluginTools(ctx, snapshot); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + continue + } + err := client.SetPluginTools(ctx, snapshot) + releaseClient() + if err != nil { refreshErr = errors.Join(refreshErr, fmt.Errorf("refresh execution %s plugin tools: %w", execution.ID, err)) } } @@ -1944,7 +2158,7 @@ func (m *Manager) createBootMessage(ctx context.Context, execution *AgentExecuti return nil, nil } bootStopCh := make(chan struct{}) - go m.pollAgentStderr(execution, execution.agentctl, bootMsg, bootStopCh) + go m.pollAgentStderr(execution, bootMsg, bootStopCh) return bootMsg, bootStopCh } @@ -1982,12 +2196,17 @@ func (m *Manager) configureAndStartAgent(ctx context.Context, execution *AgentEx m.updateExecutionError(execution.ID, "failed to prepare agent env: "+err.Error()) return "", fmt.Errorf("failed to prepare agent env: %w", err) } + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return "", fmt.Errorf("execution %q has no agentctl client", execution.ID) + } - if err := execution.agentctl.ConfigureAgent(ctx, execution.AgentCommand, execution.AgentArgs, env, approvalPolicy, execution.ContinueCommand, execution.ContinueArgs); err != nil { + if err := client.ConfigureAgent(ctx, execution.AgentCommand, execution.AgentArgs, env, approvalPolicy, execution.ContinueCommand, execution.ContinueArgs); err != nil { return "", fmt.Errorf("failed to configure agent: %w", err) } - fullCommand, err := execution.agentctl.Start(ctx) + fullCommand, err := client.Start(ctx) if err != nil { m.updateExecutionError(execution.ID, "failed to start agent: "+err.Error()) return "", fmt.Errorf("failed to start agent: %w", err) @@ -2030,13 +2249,13 @@ func (m *Manager) initializeAgentSession(ctx context.Context, execution *AgentEx agentConfig, err := m.getAgentConfigForExecution(execution) if err != nil { - m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, "failed") + m.finalizeBootMessage(execution, bootMsg, bootStopCh, "failed") return fmt.Errorf("failed to get agent config: %w", err) } mcpServers, err := m.resolveMcpServers(ctx, execution, agentConfig) if err != nil { - m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, "failed") + m.finalizeBootMessage(execution, bootMsg, bootStopCh, "failed") m.updateExecutionError(execution.ID, "failed to resolve MCP config: "+err.Error()) return fmt.Errorf("failed to resolve MCP config: %w", err) } @@ -2055,19 +2274,19 @@ func (m *Manager) initializeAgentSession(ctx context.Context, execution *AgentEx ) if attempted { if retryErr == nil { - m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, containerStateExited) + m.finalizeBootMessage(execution, bootMsg, bootStopCh, containerStateExited) return nil } err = retryErr } else if retryErr != nil { err = retryErr } - m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, "failed") + m.finalizeBootMessage(execution, bootMsg, bootStopCh, "failed") m.updateExecutionError(execution.ID, "failed to initialize ACP: "+err.Error()) return fmt.Errorf("failed to initialize ACP: %w", err) } - m.finalizeBootMessage(execution, bootMsg, bootStopCh, execution.agentctl, containerStateExited) + m.finalizeBootMessage(execution, bootMsg, bootStopCh, containerStateExited) return nil } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_launch_metadata_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_launch_metadata_test.go index 81c7b81fc8..2533fb4ddc 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_launch_metadata_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_launch_metadata_test.go @@ -64,6 +64,230 @@ func TestBuildLaunchMetadataExecutorConfigWinsForTrustedKeys(t *testing.T) { "untrusted executor-config keys still fill in when the caller supplied nothing") } +func TestBuildLaunchMetadataExecutorConfigWinsForKubernetesConnectionKeys(t *testing.T) { + req := &LaunchRequest{ + ExecutorType: string(models.ExecutorTypeKubernetes), + Metadata: map[string]interface{}{ + "auth_mode": "in_cluster", + "kubeconfig_path": "/tmp/attacker-kubeconfig", + "kube_context": "attacker", + "namespace": "attacker", + "request_timeout_seconds": "300", + }, + ExecutorConfig: map[string]string{ + "auth_mode": "kubeconfig", + "kubeconfig_path": "/etc/kandev/cluster.yaml", + "namespace": "kandev-agents", + "request_timeout_seconds": "30", + }, + } + + metadata := buildLaunchMetadata(req, "", "", "") + + require.Equal(t, "kubeconfig", metadata["auth_mode"]) + require.Equal(t, "/etc/kandev/cluster.yaml", metadata["kubeconfig_path"]) + require.Empty(t, metadata["kube_context"], + "an omitted trusted context must clear task metadata instead of selecting another context") + require.Equal(t, "kandev-agents", metadata["namespace"]) + require.Equal(t, "30", metadata["request_timeout_seconds"]) +} + +func TestLaunchBuildExecutorRequestUsesAuthoritativeKubernetesProfileConfig(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + registry.Register(backend) + reader := &fakeExecutorProfileReader{profiles: map[string]*models.ExecutorProfile{ + "profile-1": { + ID: "profile-1", + ExecutorID: "executor-1", + Config: map[string]string{ + MetadataKeyKubernetesProfilePlatform: "linux/arm64", + MetadataKeyKubernetesProfileMainContainer: "trusted-main", + MetadataKeyKubernetesPodTemplateYAML: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: trusted-main\n image: example.test/agent:latest\n", + MetadataKeyKubernetesWorkspaceMode: "existing_claim", + MetadataKeyKubernetesWorkspaceClaimName: "trusted-claim", + }, + }, + }} + mgr := &Manager{ + executorRegistry: registry, + executorFallbackPolicy: ExecutorFallbackDeny, + executorProfileReader: reader, + logger: log, + } + req := &LaunchRequest{ + ExecutorType: string(models.ExecutorTypeKubernetes), + EnvironmentFinalized: true, + Metadata: map[string]interface{}{ + "executor_id": "executor-1", + MetadataKeyExecutorProfileID: "profile-1", + MetadataKeyKubernetesProfilePlatform: "linux/amd64", + MetadataKeyKubernetesProfileMainContainer: "attacker-main", + MetadataKeyKubernetesPodTemplateYAML: "attacker-template", + MetadataKeyKubernetesWorkspaceMode: "managed_pvc", + MetadataKeyKubernetesWorkspaceSize: "1Ti", + MetadataKeyKubernetesWorkspaceStorageClass: "attacker-class", + MetadataKeyKubernetesWorkspaceAccessModes: "ReadWriteOnce", + MetadataKeyKubernetesWorkspaceClaimName: "attacker-claim", + }, + } + + _, _, _, err := mgr.launchBuildExecutorRequest( + context.Background(), "instance-1", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil, + ) + + require.NoError(t, err) + require.Equal(t, []string{"profile-1"}, reader.profileArgs) + require.NotNil(t, backend.lastRequest) + metadata := backend.lastRequest.Metadata + require.Equal(t, "linux/arm64", metadata[MetadataKeyKubernetesProfilePlatform]) + require.Equal(t, "trusted-main", metadata[MetadataKeyKubernetesProfileMainContainer]) + require.Contains(t, metadata[MetadataKeyKubernetesPodTemplateYAML], "name: trusted-main") + require.Equal(t, "existing_claim", metadata[MetadataKeyKubernetesWorkspaceMode]) + require.Equal(t, "trusted-claim", metadata[MetadataKeyKubernetesWorkspaceClaimName]) + require.Empty(t, metadata[MetadataKeyKubernetesWorkspaceAccessModes]) + require.Empty(t, metadata[MetadataKeyKubernetesWorkspaceStorageClass]) + require.Empty(t, getMetadataString(metadata, MetadataKeyKubernetesWorkspaceSize), + "an omitted profile field must erase a task-supplied value") +} + +func TestLaunchBuildExecutorRequestCanonicalizesLegacyKubernetesWorkspaceKeys(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + registry.Register(backend) + reader := &fakeExecutorProfileReader{profiles: map[string]*models.ExecutorProfile{ + "profile-legacy": { + ID: "profile-legacy", ExecutorID: "executor-1", + Config: map[string]string{ + "platform": "linux/amd64", + "main_container": "kandev-agent", + "pod_template_yaml": validKubernetesProfilePodTemplate, + "workspace_mode": "managed_pvc", + "workspace_size": "8Gi", + "workspace_storage_class": "fast", + "workspace_access_modes": `["ReadWriteOnce","ReadOnlyMany"]`, + "workspace.claim_name": "", + }, + }, + }} + mgr := &Manager{ + executorRegistry: registry, executorFallbackPolicy: ExecutorFallbackDeny, + executorProfileReader: reader, logger: log, + } + req := &LaunchRequest{ + ExecutorType: string(models.ExecutorTypeKubernetes), EnvironmentFinalized: true, + Metadata: map[string]interface{}{ + "executor_id": "executor-1", MetadataKeyExecutorProfileID: "profile-legacy", + MetadataKeyKubernetesWorkspaceMode: "empty_dir", + }, + } + + _, _, _, err := mgr.launchBuildExecutorRequest( + context.Background(), "instance-legacy", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil, + ) + + require.NoError(t, err) + require.NotNil(t, backend.lastRequest) + metadata := backend.lastRequest.Metadata + require.Equal(t, "managed_pvc", metadata[MetadataKeyKubernetesWorkspaceMode]) + require.Equal(t, "8Gi", metadata[MetadataKeyKubernetesWorkspaceSize]) + require.Equal(t, "fast", metadata[MetadataKeyKubernetesWorkspaceStorageClass]) + require.Equal(t, `["ReadWriteOnce","ReadOnlyMany"]`, metadata[MetadataKeyKubernetesWorkspaceAccessModes]) +} + +const validKubernetesProfilePodTemplate = "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example.test/agent:latest\n" + +func TestLaunchBuildExecutorRequestUsesRecordedKubernetesSnapshotWhenProfileWasDeleted(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + backend := &createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}} + registry.Register(backend) + reader := &fakeExecutorProfileReader{profiles: map[string]*models.ExecutorProfile{}} + mgr := &Manager{ + executorRegistry: registry, + executorFallbackPolicy: ExecutorFallbackDeny, + executorProfileReader: reader, + logger: log, + } + snapshot := `{"platform":"linux/amd64","main_container":"kandev-agent","pod_template_yaml":"apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example.test/agent:latest\n","workspace":{"mode":"managed_pvc","size":"1Gi","access_modes":["ReadWriteOnce"]}}` + req := &LaunchRequest{ + ExecutorType: string(models.ExecutorTypeKubernetes), + EnvironmentFinalized: true, + PreviousExecutionID: "previous-instance", + Metadata: completeKubernetesRecordedResumeMetadata(snapshot), + } + req.Metadata[MetadataKeyExecutorProfileID] = "deleted-profile" + req.Metadata[MetadataKeyKubernetesProfilePlatform] = "linux/arm64" + req.Metadata[MetadataKeyKubernetesPodTemplateYAML] = "attacker-template" + req.Metadata[MetadataKeyKubernetesWorkspaceMode] = "empty_dir" + + _, _, _, err := mgr.launchBuildExecutorRequest( + context.Background(), "instance-2", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil, + ) + + require.NoError(t, err) + require.Empty(t, reader.profileArgs, "retained workloads must not depend on the current profile row") + require.NotNil(t, backend.lastRequest) + require.Equal(t, snapshot, backend.lastRequest.Metadata[MetadataKeyKubernetesProfileSnapshot]) +} + +func TestLaunchBuildExecutorRequestDoesNotBypassProfileForIncompleteKubernetesInventory(t *testing.T) { + log := newTestRegistryLogger() + registry := NewExecutorRegistry(log) + registry.Register(&createInstanceExecutor{MockExecutor: MockExecutor{name: executor.NameKubernetes}}) + reader := &fakeExecutorProfileReader{profiles: map[string]*models.ExecutorProfile{}} + mgr := &Manager{ + executorRegistry: registry, + executorFallbackPolicy: ExecutorFallbackDeny, + executorProfileReader: reader, + logger: log, + } + req := &LaunchRequest{ + ExecutorType: string(models.ExecutorTypeKubernetes), + EnvironmentFinalized: true, + PreviousExecutionID: "previous-instance", + Metadata: map[string]interface{}{ + "executor_id": "executor-1", + MetadataKeyExecutorProfileID: "missing-profile", + MetadataKeyKubernetesPodName: "kandev-recorded", + MetadataKeyKubernetesProfileSnapshot: `{}`, + }, + } + + _, _, _, err := mgr.launchBuildExecutorRequest( + context.Background(), "instance-2", req, &testAgent{id: "agent"}, &AgentProfileInfo{}, "", "", "", nil, + ) + + require.Error(t, err) + require.Equal(t, []string{"missing-profile"}, reader.profileArgs) +} + +func completeKubernetesRecordedResumeMetadata(snapshot string) map[string]interface{} { + return map[string]interface{}{ + "executor_id": "executor-1", + MetadataKeyExecutorProfileID: "profile-1", + MetadataKeyKubernetesNamespace: "kandev-agents", + MetadataKeyKubernetesPodName: "kandev-recorded", + MetadataKeyKubernetesPodUID: "pod-uid", + MetadataKeyKubernetesMainContainer: "kandev-agent", + MetadataKeyKubernetesRuntimeWorkspaceMode: "empty_dir", + MetadataKeyKubernetesAgentctlRemotePort: "41001", + MetadataKeyKubernetesAgentctlInstanceID: "previous-instance", + MetadataKeyKubernetesResourceExecutorID: "executor-1", + MetadataKeyKubernetesResourceProfileID: "profile-1", + MetadataKeyKubernetesResourceInstanceID: "previous-instance", + MetadataKeyKubernetesResourceTaskID: "task-1", + MetadataKeyKubernetesResourceSessionID: "session-1", + MetadataKeyKubernetesResourceEnvironmentID: "environment-1", + MetadataKeyKubernetesExecutorConfigHash: "executor-hash", + MetadataKeyKubernetesProfileConfigHash: "profile-hash", + MetadataKeyKubernetesTemplateHash: "template-hash", + MetadataKeyKubernetesProfileSnapshot: snapshot, + } +} + func TestIsTrustedExecutorConfigKey(t *testing.T) { for _, key := range []string{ MetadataKeySSHHost, MetadataKeySSHHostAlias, MetadataKeySSHPort, MetadataKeySSHUser, diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go index 334844d61f..51b04b3959 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go @@ -71,8 +71,8 @@ func (m *Manager) Start(ctx context.Context) error { context.Background(), execution.TaskID, execution.SessionID, execution.ID, ) execution.SetSessionSpan(recoverySpan) - if execution.agentctl != nil { - execution.agentctl.SetTraceContext(execution.SessionTraceContext()) + if ri.Client != nil { + ri.Client.SetTraceContext(execution.SessionTraceContext()) } // Create short-lived init span so recovery-phase operations are visible @@ -88,8 +88,8 @@ func (m *Manager) Start(ctx context.Context) error { zap.String("execution_id", execution.ID), zap.String("session_id", execution.SessionID), zap.Error(err)) - if execution.agentctl != nil { - execution.agentctl.Close() + if ri.Client != nil { + ri.Client.Close() } execution.EndSessionSpan() initSpan.End() @@ -108,9 +108,10 @@ func (m *Manager) Start(ctx context.Context) error { // without this its trackers keep whatever map they had — or none, // if it was created by a path that could not supply one — and // their diff stats stay pinned to an integration-branch fallback. - if client := execution.GetAgentCtlClient(); client != nil { + if client, releaseClient := execution.AcquireAgentCtlClient(); client != nil { m.pushTaskBaseBranches(ctx, execution.TaskID, execution.ID, client) m.pushTaskComparisonTargets(ctx, execution.TaskID, execution.ID, client) + releaseClient() } // Reconnect to workspace streams (shell, git, file changes) in background @@ -308,6 +309,10 @@ func (m *Manager) cleanupStaleExecution(ctx context.Context, execution *AgentExe if execution == nil { return nil } + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + execution.agentctlLifecycleMu.Lock() + defer execution.agentctlLifecycleMu.Unlock() sessionID := execution.SessionID m.logger.Info("cleaning up stale agent execution", @@ -327,8 +332,8 @@ func (m *Manager) cleanupStaleExecution(ctx context.Context, execution *AgentExe execution.EndSessionSpan() // Close agentctl connection if it exists - if execution.agentctl != nil { - execution.agentctl.Close() + if client := execution.currentAgentCtlClient(); client != nil { // protected by agentctlLifecycleMu + client.Close() } // Remove from execution store @@ -353,7 +358,7 @@ func (m *Manager) cleanupStaleExecution(ctx context.Context, execution *AgentExe // - Removes the execution from the in-memory store // - Makes the sessionID available for new executions // - Does NOT stop the agent process (call StopAgent first) -// - Does NOT close the agentctl client (call execution.agentctl.Close() first) +// - Does NOT close the agentctl client (use the execution lifecycle lock first) // - Does NOT clean up resources (worktrees, containers, etc.) // // After calling this, the executionID and sessionID can no longer be used to query diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go index cf7b36b24e..c6395be026 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle_test.go @@ -239,7 +239,9 @@ type mockStopTracker struct { name executor.Name stopCalled bool stoppedInstanceID string + stoppedSessionID string stopReason string + stopForce bool stopErr error } @@ -253,7 +255,9 @@ func (m *mockStopTracker) CreateInstance(ctx context.Context, req *ExecutorCreat func (m *mockStopTracker) StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error { m.stopCalled = true m.stoppedInstanceID = instance.InstanceID + m.stoppedSessionID = instance.SessionID m.stopReason = instance.StopReason + m.stopForce = force return m.stopErr } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough.go b/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough.go index f9887bd766..31fbd0c821 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_passthrough.go @@ -226,10 +226,12 @@ func (m *Manager) buildPassthroughEnv(ctx context.Context, execution *AgentExecu // startPassthroughShell starts the shell session for a passthrough execution. // Non-fatal errors are logged with the provided warning message. func (m *Manager) startPassthroughShell(ctx context.Context, execution *AgentExecution, shellWarnMsg string) { - if execution.agentctl == nil { + client, release := execution.AcquireAgentCtlClient() + defer release() + if client == nil { return } - if err := execution.agentctl.StartShell(ctx); err != nil { + if err := client.StartShell(ctx); err != nil { m.logger.Warn(shellWarnMsg, zap.String("execution_id", execution.ID), zap.Error(err)) @@ -827,12 +829,15 @@ func (m *Manager) startPassthroughSession(ctx context.Context, execution *AgentE m.eventPublisher.PublishAgentctlEvent(ctx, events.AgentctlReady, execution, "") m.startPassthroughShell(ctx, execution, "failed to start shell for passthrough session") - if m.streamManager != nil && execution.agentctl != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + hasAgentStream := client != nil && client.HasAgentStream() + releaseClient() + if m.streamManager != nil && client != nil { m.streamManager.ConnectWorkspaceStream(execution, nil) // Also open the agent updates stream so the agentctl instance can proxy // kandev MCP tool calls to the backend (passthrough has no ACP stream // otherwise, so MCP tool calls would hang). - if !execution.agentctl.HasAgentStream() { + if !hasAgentStream { m.streamManager.ConnectMCPStream(execution) } } @@ -1091,11 +1096,14 @@ func (m *Manager) resumePassthroughSession(ctx context.Context, sessionID, expec // Connect to workspace stream for shell/git/file features. // Only connect if not already connected (process restart reuses the same agentctl). - if m.streamManager != nil && execution.agentctl != nil && execution.GetWorkspaceStream() == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + hasAgentStream := client != nil && client.HasAgentStream() + releaseClient() + if m.streamManager != nil && client != nil && execution.GetWorkspaceStream() == nil { m.streamManager.ConnectWorkspaceStream(execution, nil) } // Re-open the MCP proxy stream too (drains kandev MCP tool calls). - if m.streamManager != nil && execution.agentctl != nil && !execution.agentctl.HasAgentStream() { + if m.streamManager != nil && client != nil && !hasAgentStream { m.streamManager.ConnectMCPStream(execution) } @@ -1426,10 +1434,13 @@ func (m *Manager) attemptResumeFallbackForProcess(execution *AgentExecution, run // shell/git/file features come back too — without this the main terminal // works but the user's shell session and workspace stream stay torn down. m.startPassthroughShell(ctx, execution, "failed to start shell after passthrough resume fallback") - if m.streamManager != nil && execution.agentctl != nil && execution.GetWorkspaceStream() == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + hasAgentStream := client != nil && client.HasAgentStream() + releaseClient() + if m.streamManager != nil && client != nil && execution.GetWorkspaceStream() == nil { m.streamManager.ConnectWorkspaceStream(execution, nil) } - if m.streamManager != nil && execution.agentctl != nil && !execution.agentctl.HasAgentStream() { + if m.streamManager != nil && client != nil && !hasAgentStream { m.streamManager.ConnectMCPStream(execution) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status.go b/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status.go index 024e470fbc..3352ec2911 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status.go @@ -44,17 +44,20 @@ func (m *Manager) pollOneRemoteStatus(ctx context.Context, execution *AgentExecu return } - instance := &ExecutorInstance{ - InstanceID: execution.ID, - TaskID: execution.TaskID, - SessionID: execution.SessionID, - RuntimeName: execution.RuntimeName, - ContainerID: execution.ContainerID, - ContainerIP: execution.ContainerIP, - WorkspacePath: execution.WorkspacePath, - StandaloneInstanceID: execution.standaloneInstanceID, - StandalonePort: execution.standalonePort, - Metadata: execution.MetadataSnapshot(), + instance := m.remoteStatusInstance(ctx, execution) + if refresher, refreshable := rt.(RemoteInstanceRefresher); refreshable { + if refreshErr := m.refreshTrackedRemoteInstance(ctx, execution, refresher); refreshErr != nil { + m.storeRemoteStatus(execution.SessionID, &RemoteStatus{ + RuntimeName: execution.RuntimeName, LastCheckedAt: time.Now().UTC(), + ErrorMessage: refreshErr.Error(), + }) + m.logger.Debug("remote control refresh failed", + zap.String("session_id", execution.SessionID), + zap.String("execution_id", execution.ID), + zap.Error(refreshErr)) + return + } + instance = m.remoteStatusInstance(ctx, execution) } status, statusErr := provider.GetRemoteStatus(ctx, instance) if statusErr != nil { @@ -74,6 +77,11 @@ func (m *Manager) pollOneRemoteStatus(ctx context.Context, execution *AgentExecu if status == nil { return } + // Providers may reuse a cached status object across concurrent polls. Copy it + // before filling lifecycle-owned defaults so polling never mutates provider + // state (or races with another caller holding the same result pointer). + statusCopy := *status + status = &statusCopy if status.RuntimeName == "" { status.RuntimeName = execution.RuntimeName } @@ -83,6 +91,24 @@ func (m *Manager) pollOneRemoteStatus(ctx context.Context, execution *AgentExecu m.storeRemoteStatus(execution.SessionID, status) } +func (m *Manager) remoteStatusInstance(ctx context.Context, execution *AgentExecution) *ExecutorInstance { + metadata := execution.MetadataSnapshot() + return &ExecutorInstance{ + InstanceID: execution.ID, + TaskID: execution.TaskID, + SessionID: execution.SessionID, + RuntimeName: execution.RuntimeName, + ContainerID: execution.ContainerID, + ContainerIP: execution.ContainerIP, + WorkspacePath: execution.WorkspacePath, + StandaloneInstanceID: execution.standaloneInstanceID, + StandalonePort: execution.standalonePort, + Metadata: metadata, + AuthToken: m.revealRuntimeSecret(ctx, metadata, MetadataKeyAuthTokenSecret), + BootstrapNonce: m.revealRuntimeSecret(ctx, metadata, MetadataKeyBootstrapNonceSecret), + } +} + func (m *Manager) storeRemoteStatus(sessionID string, status *RemoteStatus) { if sessionID == "" || status == nil { return @@ -126,6 +152,7 @@ func (m *Manager) GetRemoteStatusBySessionID(ctx context.Context, sessionID stri // RemoteStatusPollRecord contains the minimal information needed to poll remote status // for a session that is not currently tracked in the in-memory execution store. type RemoteStatusPollRecord struct { + TaskID string SessionID string Runtime agentruntime.Runtime AgentExecutionID string @@ -155,6 +182,7 @@ func (m *Manager) PollRemoteStatusForRecords(ctx context.Context, records []Remo instance := &ExecutorInstance{ InstanceID: rec.AgentExecutionID, + TaskID: rec.TaskID, SessionID: rec.SessionID, RuntimeName: rec.Runtime, ContainerID: rec.ContainerID, diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status_test.go index 4b014bd5cf..e48d1de740 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_remote_status_test.go @@ -3,6 +3,10 @@ package lifecycle import ( "context" "errors" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" "testing" "time" @@ -10,6 +14,9 @@ import ( "github.com/kandev/kandev/internal/agent/executor" "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" ) // statusProviderExecutor is an ExecutorBackend that also implements @@ -22,6 +29,101 @@ type statusProviderExecutor struct { observed []*ExecutorInstance } +type refreshingStatusExecutor struct { + statusProviderExecutor + refresh *RemoteInstanceRefresh + refreshEntered chan struct{} + releaseRefresh chan struct{} + once sync.Once + mu sync.Mutex + refreshCalls int + committed bool + stopEntered chan struct{} + stopOnce sync.Once + onStop func() +} + +func (e *refreshingStatusExecutor) RefreshRemoteInstance( + _ context.Context, + _ *ExecutorInstance, +) (*RemoteInstanceRefresh, error) { + e.mu.Lock() + if e.committed { + e.mu.Unlock() + return nil, nil + } + e.refreshCalls++ + e.mu.Unlock() + e.once.Do(func() { close(e.refreshEntered) }) + <-e.releaseRefresh + return e.refresh, nil +} + +func (e *refreshingStatusExecutor) GetRemoteStatus(_ context.Context, instance *ExecutorInstance) (*RemoteStatus, error) { + e.mu.Lock() + defer e.mu.Unlock() + e.observed = append(e.observed, instance) + return e.status, e.err +} + +func (e *refreshingStatusExecutor) StopInstance(context.Context, *ExecutorInstance, bool) error { + if e.stopEntered != nil { + e.stopOnce.Do(func() { close(e.stopEntered) }) + } + if e.onStop != nil { + e.onStop() + } + return nil +} + +type synchronizedRunningWriter struct { + mu sync.Mutex + running *models.ExecutorRunning +} + +func (w *synchronizedRunningWriter) GetExecutorRunningBySessionID( + context.Context, string, +) (*models.ExecutorRunning, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.running == nil { + return nil, models.ErrExecutorRunningNotFound + } + return w.running, nil +} + +func (w *synchronizedRunningWriter) UpsertExecutorRunning( + _ context.Context, running *models.ExecutorRunning, +) error { + w.mu.Lock() + w.running = running + w.mu.Unlock() + return nil +} + +func (w *synchronizedRunningWriter) DeleteExecutorRunningBySessionID(context.Context, string) error { + w.mu.Lock() + w.running = nil + w.mu.Unlock() + return nil +} + +func (w *synchronizedRunningWriter) RepairExecutorRunningDead(context.Context, string) error { + return nil +} + +func (w *synchronizedRunningWriter) clear() { + w.mu.Lock() + w.running = nil + w.mu.Unlock() +} + +func (w *synchronizedRunningWriter) snapshot() *models.ExecutorRunning { + w.mu.Lock() + defer w.mu.Unlock() + return w.running +} + func (e *statusProviderExecutor) GetRemoteStatus(_ context.Context, instance *ExecutorInstance) (*RemoteStatus, error) { e.observed = append(e.observed, instance) return e.status, e.err @@ -81,6 +183,42 @@ func TestPollOneRemoteStatusProjectsExecutionIntoInstance(t *testing.T) { "an unstamped status is stamped so the UI can show staleness") } +func TestKubernetesRefreshStatusInstanceOverlaysCurrentConnectionSettings(t *testing.T) { + store := &restartKubernetesInventoryStore{ + executors: map[string]*models.Executor{"executor-1": { + ID: "executor-1", Type: models.ExecutorTypeKubernetes, + Config: map[string]string{ + MetadataKeyKubernetesAuthMode: "kubeconfig", + MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/current.yaml", + MetadataKeyKubernetesKubeContext: "current", + MetadataKeyKubernetesConfigNamespace: "new-agents", + MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + }}, + } + mgr := &Manager{runningWriter: store} + execution := &AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, + metadata: map[string]interface{}{ + "executor_id": "executor-1", + MetadataKeyKubernetesAuthMode: "in_cluster", + MetadataKeyKubernetesConfigNamespace: "old-agents", + MetadataKeyKubernetesNamespace: "old-agents", + MetadataKeyKubernetesRequestTimeoutSeconds: "30", + }, + } + + instance, err := mgr.kubernetesRefreshStatusInstance(context.Background(), execution) + + require.NoError(t, err) + require.Equal(t, "kubeconfig", instance.Metadata[MetadataKeyKubernetesAuthMode]) + require.Equal(t, "/etc/kandev/current.yaml", instance.Metadata[MetadataKeyKubernetesKubeconfigPath]) + require.Equal(t, "current", instance.Metadata[MetadataKeyKubernetesKubeContext]) + require.Equal(t, "new-agents", instance.Metadata[MetadataKeyKubernetesConfigNamespace]) + require.Equal(t, "old-agents", instance.Metadata[MetadataKeyKubernetesNamespace]) +} + func TestPollOneRemoteStatusStoresProviderError(t *testing.T) { provider := &statusProviderExecutor{ MockExecutor: MockExecutor{name: executor.NameSprites}, @@ -98,6 +236,327 @@ func TestPollOneRemoteStatusStoresProviderError(t *testing.T) { require.Equal(t, agentruntime.RuntimeSprites, status.RuntimeName) } +func TestPollOneRemoteStatusAtomicallyRefreshesRestartedKubernetesAgentctl(t *testing.T) { + var configureCalls atomic.Int32 + var startCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/health": + w.WriteHeader(http.StatusOK) + case "/api/v1/agent/configure": + configureCalls.Add(1) + _, _ = w.Write([]byte(`{"success":true}`)) + case "/api/v1/start": + startCalls.Add(1) + _, _ = w.Write([]byte(`{"success":true,"command":"agent"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + newClient := newTestAgentctlClient(t, server.URL, newTestLogger()) + oldClient := newReadyAgentctlClient(t, newTestLogger()) + + refresh := &RemoteInstanceRefresh{Instance: &ExecutorInstance{ + InstanceID: "exec-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, Client: newClient, + AuthToken: "rotated-token", BootstrapNonce: "durable-nonce", + Metadata: map[string]interface{}{ + MetadataKeyKubernetesAgentctlRemotePort: "41002", + }, + }, ProcessRestarted: true} + provider := &refreshingStatusExecutor{ + statusProviderExecutor: statusProviderExecutor{ + MockExecutor: MockExecutor{name: executor.NameKubernetes}, + status: &RemoteStatus{State: kubernetesStatusRunning}, + }, + refresh: refresh, refreshEntered: make(chan struct{}), releaseRefresh: make(chan struct{}), + } + refresh.Commit = func(publish func()) error { + provider.mu.Lock() + provider.committed = true + provider.mu.Unlock() + if publish != nil { + publish() + } + return nil + } + refresh.Abort = func() { t.Error("successful refresh was aborted") } + mgr := newRemoteStatusManager(t, provider) + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:exec-1:agentctl-auth": "old-token", + "kandev-runtime:exec-1:agentctl-bootstrap": "durable-nonce", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + mgr.SetSecretStore(store) + writer := &captureExecutorRunningWriter{} + mgr.SetExecutorRunningWriter(writer) + execution := &AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusRunning, AgentCommand: "agent", agentctl: oldClient, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:exec-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:exec-1:agentctl-bootstrap", + MetadataKeyKubernetesAgentctlRemotePort: "41001", + }, + } + execution.setRuntimeEnvironment(map[string]string{"FOO": "bar"}) + require.NoError(t, mgr.executionStore.Add(execution)) + + firstDone := make(chan struct{}) + go func() { + mgr.pollOneRemoteStatus(context.Background(), execution) + close(firstDone) + }() + <-provider.refreshEntered + if execution.remoteInstanceLifecycleMu.TryLock() { + execution.remoteInstanceLifecycleMu.Unlock() + close(provider.releaseRefresh) + <-firstDone + t.Fatal("active remote refresh did not own the execution lifecycle boundary") + } + secondDone := make(chan struct{}) + go func() { + mgr.pollOneRemoteStatus(context.Background(), execution) + close(secondDone) + }() + close(provider.releaseRefresh) + <-firstDone + <-secondDone + + provider.mu.Lock() + refreshCalls := provider.refreshCalls + committed := provider.committed + provider.mu.Unlock() + require.Equal(t, 1, refreshCalls, "concurrent polls must share one refresh") + require.True(t, committed) + client, releaseClient := execution.AcquireAgentCtlClient() + require.Same(t, newClient, client) + releaseClient() + require.Equal(t, int32(1), configureCalls.Load()) + require.Equal(t, int32(1), startCalls.Load()) + revealed, err := store.Reveal(context.Background(), "kandev-runtime:exec-1:agentctl-auth") + require.NoError(t, err) + require.Equal(t, "rotated-token", revealed) + store.mu.RLock() + secretCount := len(store.store) + store.mu.RUnlock() + require.Equal(t, 2, secretCount, "token rotation must reuse the internal secret reference") + require.NotNil(t, writer.running) + require.Equal(t, "41002", getMetadataString(writer.running.Metadata, MetadataKeyKubernetesAgentctlRemotePort)) +} + +func TestPollOneRemoteStatusReattachesKubernetesClientWithoutRestartingAgent(t *testing.T) { + var configureCalls atomic.Int32 + var startCalls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + switch request.URL.Path { + case "/health": + w.WriteHeader(http.StatusOK) + case "/api/v1/agent/configure": + configureCalls.Add(1) + _, _ = w.Write([]byte(`{"success":true}`)) + case "/api/v1/start": + startCalls.Add(1) + _, _ = w.Write([]byte(`{"success":true,"command":"agent"}`)) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(server.Close) + newClient := newTestAgentctlClient(t, server.URL, newTestLogger()) + oldClient := newReadyAgentctlClient(t, newTestLogger()) + releaseRefresh := make(chan struct{}) + close(releaseRefresh) + refresh := &RemoteInstanceRefresh{Instance: &ExecutorInstance{ + InstanceID: "exec-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, Client: newClient, + AuthToken: "old-token", BootstrapNonce: "durable-nonce", + Metadata: map[string]interface{}{ + MetadataKeyKubernetesAgentctlRemotePort: "41001", + }, + }} + provider := &refreshingStatusExecutor{ + statusProviderExecutor: statusProviderExecutor{ + MockExecutor: MockExecutor{name: executor.NameKubernetes}, + status: &RemoteStatus{State: kubernetesStatusRunning}, + }, + refresh: refresh, refreshEntered: make(chan struct{}), releaseRefresh: releaseRefresh, + } + refresh.Commit = func(publish func()) error { + provider.mu.Lock() + provider.committed = true + provider.mu.Unlock() + if publish != nil { + publish() + } + return nil + } + refresh.Abort = func() { t.Error("successful refresh was aborted") } + mgr := newRemoteStatusManager(t, provider) + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:exec-1:agentctl-auth": "old-token", + "kandev-runtime:exec-1:agentctl-bootstrap": "durable-nonce", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + mgr.SetSecretStore(store) + mgr.SetExecutorRunningWriter(&captureExecutorRunningWriter{}) + execution := &AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusRunning, AgentCommand: "agent", agentctl: oldClient, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:exec-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:exec-1:agentctl-bootstrap", + MetadataKeyKubernetesAgentctlRemotePort: "41001", + }, + } + require.NoError(t, mgr.executionStore.Add(execution)) + + mgr.pollOneRemoteStatus(context.Background(), execution) + + client, releaseClient := execution.AcquireAgentCtlClient() + require.Same(t, newClient, client) + releaseClient() + require.Zero(t, configureCalls.Load(), "local transport reattachment must not restart the remote agent") + require.Zero(t, startCalls.Load(), "local transport reattachment must not restart the remote agent") +} + +func TestPersistActiveKubernetesRefreshOutlivesCanceledPollContext(t *testing.T) { + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:exec-1:agentctl-auth": "old-token", + "kandev-runtime:exec-1:agentctl-bootstrap": "durable-nonce", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + store.rejectCanceledCalls = true + mgr := newRemoteStatusManager(t, &MockExecutor{name: executor.NameKubernetes}) + mgr.SetSecretStore(store) + mgr.SetExecutorRunningWriter(&captureExecutorRunningWriter{}) + execution := &AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusRunning, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:exec-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:exec-1:agentctl-bootstrap", + }, + } + instance := &ExecutorInstance{ + AuthToken: "rotated-token", BootstrapNonce: "durable-nonce", + Metadata: map[string]interface{}{MetadataKeyKubernetesAgentctlRemotePort: "41002"}, + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + rollback, err := mgr.persistActiveKubernetesRefresh(ctx, execution, instance) + require.NoError(t, err) + require.NotNil(t, rollback) + rotated, err := store.Reveal(context.Background(), "kandev-runtime:exec-1:agentctl-auth") + require.NoError(t, err) + require.Equal(t, "rotated-token", rotated) + require.NoError(t, rollback(ctx)) + restored, err := store.Reveal(context.Background(), "kandev-runtime:exec-1:agentctl-auth") + require.NoError(t, err) + require.Equal(t, "old-token", restored) +} + +func TestStopAgentSerializesWithActiveKubernetesRefresh(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path == "/health" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(server.Close) + newClient := newTestAgentctlClient(t, server.URL, newTestLogger()) + oldClient := newReadyAgentctlClient(t, newTestLogger()) + writer := &synchronizedRunningWriter{} + refresh := &RemoteInstanceRefresh{Instance: &ExecutorInstance{ + InstanceID: "exec-1", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, Client: newClient, + AuthToken: "old-token", BootstrapNonce: "durable-nonce", + Metadata: map[string]interface{}{ + MetadataKeyKubernetesAgentctlRemotePort: "41001", + }, + }} + provider := &refreshingStatusExecutor{ + statusProviderExecutor: statusProviderExecutor{ + MockExecutor: MockExecutor{name: executor.NameKubernetes}, + status: &RemoteStatus{State: kubernetesStatusRunning}, + }, + refresh: refresh, refreshEntered: make(chan struct{}), releaseRefresh: make(chan struct{}), + stopEntered: make(chan struct{}), onStop: writer.clear, + } + refresh.Commit = func(publish func()) error { + provider.mu.Lock() + provider.committed = true + provider.mu.Unlock() + if publish != nil { + publish() + } + return nil + } + refresh.Abort = func() { t.Error("successful refresh was aborted") } + mgr := newRemoteStatusManager(t, provider) + store := newInMemorySecretStore() + for id, value := range map[string]string{ + "kandev-runtime:exec-1:agentctl-auth": "old-token", + "kandev-runtime:exec-1:agentctl-bootstrap": "durable-nonce", + } { + require.NoError(t, store.Create(context.Background(), &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: id, Name: id}, Value: value, + })) + } + mgr.SetSecretStore(store) + mgr.SetExecutorRunningWriter(writer) + execution := &AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + AgentProfileID: "profile-1", RuntimeName: agentruntime.RuntimeKubernetes, + Status: v1.AgentStatusRunning, AgentCommand: "agent", agentctl: oldClient, + metadata: map[string]interface{}{ + MetadataKeyAuthTokenSecret: "kandev-runtime:exec-1:agentctl-auth", + MetadataKeyBootstrapNonceSecret: "kandev-runtime:exec-1:agentctl-bootstrap", + MetadataKeyKubernetesAgentctlRemotePort: "41001", + }, + } + require.NoError(t, mgr.executionStore.Add(execution)) + + refreshDone := make(chan struct{}) + go func() { + mgr.pollOneRemoteStatus(context.Background(), execution) + close(refreshDone) + }() + <-provider.refreshEntered + stopDone := make(chan error, 1) + go func() { + stopDone <- mgr.StopAgentWithReason(context.Background(), execution.ID, StopReasonTaskDeleted, true) + }() + close(provider.releaseRefresh) + <-refreshDone + require.NoError(t, <-stopDone) + + require.Nil(t, writer.snapshot(), "terminal stop must remain the final persistence owner") + _, exists := mgr.executionStore.Get(execution.ID) + require.False(t, exists) + store.mu.RLock() + defer store.mu.RUnlock() + require.Empty(t, store.store) +} + func TestPollOneRemoteStatusSkipsUnsupportedExecutions(t *testing.T) { provider := &statusProviderExecutor{ MockExecutor: MockExecutor{name: executor.NameSprites}, @@ -228,6 +687,7 @@ func TestPollRemoteStatusForRecordsPopulatesCacheBeforeResume(t *testing.T) { {SessionID: "session-no-runtime"}, // skipped {SessionID: "session-unregistered", Runtime: agentruntime.RuntimeSSH}, // skipped { + TaskID: "task-1", SessionID: "session-1", Runtime: agentruntime.RuntimeSprites, AgentExecutionID: "exec-1", @@ -238,6 +698,7 @@ func TestPollRemoteStatusForRecordsPopulatesCacheBeforeResume(t *testing.T) { require.Len(t, provider.observed, 1, "only the complete, registered record is polled") require.Equal(t, "exec-1", provider.observed[0].InstanceID) + require.Equal(t, "task-1", provider.observed[0].TaskID) require.Equal(t, "container-1", provider.observed[0].ContainerID) require.Equal(t, "kandev-abc", provider.observed[0].Metadata[MetadataKeySpriteName]) diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go b/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go index 7f38cd34e2..466c25d5a6 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_startup.go @@ -12,7 +12,6 @@ import ( "github.com/kandev/kandev/internal/agent/agents" "github.com/kandev/kandev/internal/agent/runtime/activity" - agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" "github.com/kandev/kandev/internal/common/appctx" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/task/models" @@ -138,8 +137,9 @@ func (m *Manager) startAgentProcess(ctx context.Context, executionID string) (re return m.startPassthroughExecution(operationCtx, execution, profileInfo) } execution.beginStartupAttempt() - - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { return fmt.Errorf("execution %q has no agentctl client", executionID) } @@ -153,7 +153,13 @@ func (m *Manager) startAgentProcess(ctx context.Context, executionID string) (re } // Wait for agentctl to be ready - if err := execution.agentctl.WaitForReady(operationCtx, 60*time.Second); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", executionID) + } + err = client.WaitForReady(operationCtx, 60*time.Second) + releaseClient() + if err != nil { m.updateExecutionError(executionID, "agentctl not ready: "+err.Error()) return fmt.Errorf("agentctl not ready: %w", err) } @@ -200,7 +206,12 @@ func (m *Manager) startAgentProcess(ctx context.Context, executionID string) (re } func (m *Manager) preflightRemoteContributionPushes(ctx context.Context, execution *AgentExecution) error { - if execution == nil || execution.agentctl == nil { + if execution == nil { + return nil + } + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return nil } bindings, err := remoteContributionsFromMetadata(execution.MetadataSnapshot()) @@ -216,7 +227,7 @@ func (m *Manager) preflightRemoteContributionPushes(ctx context.Context, executi } sort.Strings(keys) for _, key := range keys { - result, err := execution.agentctl.GitPushPreflight(ctx, key) + result, err := client.GitPushPreflight(ctx, key) if err != nil { return fmt.Errorf("repository %q: %w", key, err) } @@ -232,7 +243,7 @@ func (m *Manager) preflightRemoteContributionPushes(ctx context.Context, executi } // pollAgentStderr polls the agent's stderr buffer every 2 seconds and updates the boot message. -func (m *Manager) pollAgentStderr(execution *AgentExecution, client *agentctl.Client, msg *models.Message, stopCh chan struct{}) { +func (m *Manager) pollAgentStderr(execution *AgentExecution, msg *models.Message, stopCh chan struct{}) { ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() @@ -245,7 +256,13 @@ func (m *Manager) pollAgentStderr(execution *AgentExecution, client *agentctl.Cl case <-ticker.C: baseCtx := execution.SessionTraceContext() ctx, cancel := context.WithTimeout(baseCtx, 5*time.Second) + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + cancel() + continue + } lines, err := client.GetAgentStderr(ctx) + releaseClient() cancel() if err != nil { m.logger.Debug("failed to poll agent stderr", zap.Error(err)) @@ -268,7 +285,7 @@ func (m *Manager) pollAgentStderr(execution *AgentExecution, client *agentctl.Cl } // finalizeBootMessage stops the polling goroutine and updates the boot message with final status. -func (m *Manager) finalizeBootMessage(execution *AgentExecution, msg *models.Message, stopCh chan struct{}, client *agentctl.Client, status string) { +func (m *Manager) finalizeBootMessage(execution *AgentExecution, msg *models.Message, stopCh chan struct{}, status string) { if msg == nil || m.bootMessageService == nil { return } @@ -279,10 +296,16 @@ func (m *Manager) finalizeBootMessage(execution *AgentExecution, msg *models.Mes } // Final stderr fetch - if client != nil && execution != nil { + if execution != nil { baseCtx := execution.SessionTraceContext() ctx, cancel := context.WithTimeout(baseCtx, 5*time.Second) - lines, err := client.GetAgentStderr(ctx) + client, releaseClient := execution.AcquireAgentCtlClient() + var lines []string + var err error + if client != nil { + lines, err = client.GetAgentStderr(ctx) + releaseClient() + } cancel() if err == nil && len(lines) > 0 { msg.Content = strings.Join(lines, "\n") @@ -424,11 +447,18 @@ func (m *Manager) waitForAgentctlReady(execution *AgentExecution) { ctx, cancel := appctx.Detached(context.Background(), m.stopCh, 60*time.Second) defer cancel() + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + m.updateExecutionError(execution.ID, "agentctl client is unavailable") + return + } m.logger.Debug("waiting for agentctl to be ready", zap.String("execution_id", execution.ID), - zap.String("url", execution.agentctl.BaseURL())) + zap.String("url", client.BaseURL())) - if err := execution.agentctl.WaitForReady(ctx, 60*time.Second); err != nil { + err := client.WaitForReady(ctx, 60*time.Second) + releaseClient() + if err != nil { m.logger.Error("agentctl not ready", zap.String("execution_id", execution.ID), zap.Duration("duration", time.Since(opStart)), @@ -460,8 +490,12 @@ func (m *Manager) waitForAgentctlReady(execution *AgentExecution) { // agent starting on an already-prepared workspace, or by lazy recovery // after a restart, would otherwise have none — and their branch diff stat // would silently fall back to an integration branch. - m.pushTaskBaseBranches(ctx, execution.TaskID, execution.ID, execution.GetAgentCtlClient()) - m.pushTaskComparisonTargets(ctx, execution.TaskID, execution.ID, execution.GetAgentCtlClient()) + client, releaseClient = execution.AcquireAgentCtlClient() + if client != nil { + m.pushTaskBaseBranches(ctx, execution.TaskID, execution.ID, client) + m.pushTaskComparisonTargets(ctx, execution.TaskID, execution.ID, client) + releaseClient() + } // Use the timeout context for event publishing instead of a fresh Background context m.eventPublisher.PublishAgentctlEvent(ctx, events.AgentctlReady, execution, "") } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_subscription.go b/apps/backend/internal/agent/runtime/lifecycle/manager_subscription.go index 77837f7dee..8c1aa28741 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_subscription.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_subscription.go @@ -6,8 +6,6 @@ import ( "time" "go.uber.org/zap" - - agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl" ) // WorkspacePollMode mirrors process.PollMode for the lifecycle layer. Defined @@ -78,10 +76,11 @@ type workspacePollAggregator struct { pushInFlight map[string]bool } -// workspacePushTarget bundles the queued mode and the agentctl client captured at enqueue time. +// workspacePushTarget bundles the queued mode and execution resolved at enqueue time. +// The worker pins that execution's current client only while issuing the RPC. type workspacePushTarget struct { - mode WorkspacePollMode - client *agentctl.Client + mode WorkspacePollMode + execution *AgentExecution } // newWorkspacePollAggregator wires an aggregator to the lifecycle manager. @@ -288,12 +287,13 @@ func (a *workspacePollAggregator) recordRuntimeAndCompute(sessionID string, acti // pushAsync queues the latest mode and ensures exactly one pusher goroutine per workspace drains it (last-write-wins). func (a *workspacePollAggregator) pushAsync(execution *AgentExecution, workspacePath string, mode WorkspacePollMode) { - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() if client == nil { return } + releaseClient() a.mu.Lock() - a.pendingPush[workspacePath] = workspacePushTarget{mode: mode, client: client} + a.pendingPush[workspacePath] = workspacePushTarget{mode: mode, execution: execution} if a.pushInFlight[workspacePath] { a.mu.Unlock() return @@ -316,9 +316,14 @@ func (a *workspacePollAggregator) pushLoop(workspacePath string) { delete(a.pendingPush, workspacePath) a.mu.Unlock() + client, releaseClient := target.execution.AcquireAgentCtlClient() + if client == nil { + continue + } ctx, cancel := context.WithTimeout(context.Background(), pushPollModeTimeout) - err := target.client.SetWorkspacePollMode(ctx, string(target.mode)) + err := client.SetWorkspacePollMode(ctx, string(target.mode)) cancel() + releaseClient() if err != nil { a.mgr.logger.Warn("failed to push workspace poll mode", zap.String("workspace", workspacePath), diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_subscription_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_subscription_test.go index 0fd6047e90..005e6290e7 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_subscription_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_subscription_test.go @@ -18,7 +18,7 @@ import ( // newTestManagerForAggregator builds a minimal Manager with just the bits the // aggregator needs: an executionStore and a logger. We don't need the real // dependency graph because the aggregator only reads execution metadata and -// dispatches to AgentExecution.GetAgentCtlClient (which we leave nil — pushAsync +// acquires the AgentExecution agentctl client (which we leave nil — pushAsync // will then no-op, and the test inspects sessionModes/lastPushed directly). func newTestManagerForAggregator(t *testing.T) *Manager { t.Helper() diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_branch.go b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_branch.go index 563acbd8c3..181ef734b5 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_branch.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_branch.go @@ -87,7 +87,8 @@ func (m *Manager) renameBranchForSession( if execution == nil { return nil, fmt.Errorf("execution for session %s is unavailable", sessionID) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return nil, fmt.Errorf("agentctl client for session %s is unavailable", sessionID) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go index c3277d42da..309bd48144 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rebind.go @@ -32,7 +32,11 @@ func (m *Manager) RebindWorkspaceForSession(ctx context.Context, sessionID, work // failing a batch because an old session is no longer live. return nil } - if execution.IsPassthrough || execution.agentctl == nil || execution.ACPSessionID == "" { + execution.remoteInstanceLifecycleMu.Lock() + defer execution.remoteInstanceLifecycleMu.Unlock() + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if execution.IsPassthrough || client == nil || execution.ACPSessionID == "" { return fmt.Errorf("workspace rebind is unsupported for this session; start a new session after attaching sources") } execution.promptLifecycleMu.Lock() @@ -49,16 +53,25 @@ func (m *Manager) RebindWorkspaceForSession(ctx context.Context, sessionID, work // Stop before changing agentctl's workdir: a successful rebind must never // leave a child running in the old CWD. If stop cannot be proven, do not // attempt adoption or start a duplicate process. - if err := execution.agentctl.Stop(ctx); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + err := client.Stop(ctx) + releaseClient() + if err != nil { execution.Status = v1.AgentStatusReady return fmt.Errorf("stop agent before workspace rebind: %w", err) } execution.WorkspaceSourceRoots = newRoots - if err := execution.agentctl.RebindWorkspace(ctx, workspacePath, newRoots); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + err = client.RebindWorkspace(ctx, workspacePath, newRoots) + releaseClient() + if err != nil { return m.rollbackWorkspaceRebind(ctx, execution, oldPath, oldRoots, acpID, fmt.Errorf("rebind agentctl workspace: %w", err)) } execution.WorkspacePath = workspacePath - if _, err := execution.agentctl.Start(ctx); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + _, err = client.Start(ctx) + releaseClient() + if err != nil { return m.rollbackWorkspaceRebind(ctx, execution, oldPath, oldRoots, acpID, fmt.Errorf("restart agent after workspace rebind: %w", err)) } if err := m.restoreReboundACPSession(ctx, execution, acpID, startNewSession); err != nil { @@ -73,16 +86,34 @@ func (m *Manager) rollbackWorkspaceRebind(ctx context.Context, execution *AgentE // Restore the authoritative in-memory policy first, even if an I/O failure // prevents the best-effort child rollback below. execution.WorkspaceSourceRoots = append([]string(nil), oldRoots...) - if err := execution.agentctl.Stop(rollbackCtx); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("%w; rollback agentctl client is unavailable", cause) + } + err := client.Stop(rollbackCtx) + releaseClient() + if err != nil { m.executionStore.UpdateError(execution.ID, fmt.Sprintf("%v; rollback stop failed: %v", cause, err)) return fmt.Errorf("%w; rollback stop failed: %v", cause, err) } - if err := execution.agentctl.RebindWorkspace(rollbackCtx, oldPath, oldRoots); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("%w; rollback agentctl client is unavailable", cause) + } + err = client.RebindWorkspace(rollbackCtx, oldPath, oldRoots) + releaseClient() + if err != nil { m.executionStore.UpdateError(execution.ID, fmt.Sprintf("%v; rollback rebind failed: %v", cause, err)) return fmt.Errorf("%w; rollback rebind failed: %v", cause, err) } execution.WorkspacePath = oldPath - if _, err := execution.agentctl.Start(rollbackCtx); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("%w; rollback agentctl client is unavailable", cause) + } + _, err = client.Start(rollbackCtx) + releaseClient() + if err != nil { m.executionStore.UpdateError(execution.ID, fmt.Sprintf("%v; rollback restart failed: %v", cause, err)) return fmt.Errorf("%w; rollback restart failed: %v", cause, err) } @@ -106,13 +137,25 @@ func (m *Manager) restoreReboundACPSession(ctx context.Context, execution *Agent if err := waitForReboundUpdatesStream(ctx, updatesReady); err != nil { return err } - if _, err := execution.agentctl.Initialize(ctx, "kandev", "1.0.0"); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } + _, err := client.Initialize(ctx, "kandev", "1.0.0") + releaseClient() + if err != nil { return fmt.Errorf("initialize restarted ACP adapter: %w", err) } if startNewSession { return m.createReboundACPSession(ctx, execution) } - if err := execution.agentctl.LoadSession(ctx, acpID, nil); err != nil { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } + err = client.LoadSession(ctx, acpID, nil) + releaseClient() + if err != nil { return err } return nil @@ -142,7 +185,12 @@ func (m *Manager) createReboundACPSession(ctx context.Context, execution *AgentE previousModel := execution.GetModelState() previousModelID := m.effectiveSessionModelForReset(ctx, execution) execution.SetModelState(nil) - newSessionID, err := execution.agentctl.NewSession(ctx, execution.WorkspacePath, mcpServers) + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } + newSessionID, err := client.NewSession(ctx, execution.WorkspacePath, mcpServers) + releaseClient() if err != nil { execution.SetModelState(previousModel) return fmt.Errorf("create ACP session in rebound workspace: %w", err) @@ -173,9 +221,14 @@ func (m *Manager) reapplyReboundSessionConfig( mode *CachedModeState, ) error { if modelID != "" { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } policy := m.resolveStartModelPolicy(ctx, execution.AgentProfileID) policy.Model = modelID - decision, err := applyStartModelPolicy(ctx, m.logger, execution.agentctl, execution.GetModelState(), policy) + decision, err := applyStartModelPolicy(ctx, m.logger, client, execution.GetModelState(), policy) + releaseClient() if err != nil { m.logger.Warn("failed to re-apply model after workspace rebind", zap.String("execution_id", execution.ID), @@ -194,7 +247,13 @@ func (m *Manager) reapplyReboundSessionConfig( strings.EqualFold(option.Category, "mode") { continue } - if err := execution.agentctl.SetConfigOption(ctx, option.ID, option.CurrentValue); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("agentctl client is unavailable") + } + err := client.SetConfigOption(ctx, option.ID, option.CurrentValue) + releaseClient() + if err != nil { m.logger.Warn("failed to re-apply config option after workspace rebind", zap.String("execution_id", execution.ID), zap.String("config_id", option.ID), @@ -213,7 +272,12 @@ func waitForReboundAgentReady(ctx context.Context, execution *AgentExecution) er defer ticker.Stop() var lastErr error for { - status, err := execution.agentctl.GetStatus(readyCtx) + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("wait for restarted agent readiness: agentctl client is unavailable") + } + status, err := client.GetStatus(readyCtx) + releaseClient() if err == nil && status.AgentStatus == agentctlProcessStatusRunning { return nil } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan.go b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan.go index d06fcc03f0..dfdd830680 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_workspace_rescan.go @@ -99,7 +99,8 @@ func (m *Manager) RescanWorkspaceForSession(ctx context.Context, sessionID, work zap.String("session_id", sessionID)) return nil } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { m.logger.Debug("rescan skipped: execution has no agentctl client", zap.String("session_id", sessionID), diff --git a/apps/backend/internal/agent/runtime/lifecycle/persistence.go b/apps/backend/internal/agent/runtime/lifecycle/persistence.go index 89a8e4fe2c..14c2b000a2 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/persistence.go +++ b/apps/backend/internal/agent/runtime/lifecycle/persistence.go @@ -5,6 +5,7 @@ import ( "errors" "net/url" "strconv" + "strings" "time" "go.uber.org/zap" @@ -76,6 +77,7 @@ func buildRunningFromExecution(execution *AgentExecution, prior *models.Executor ID: execution.SessionID, SessionID: execution.SessionID, TaskID: execution.TaskID, + ExecutorID: strings.TrimSpace(getMetadataString(metadata, "executor_id")), ExecutionProfileID: execution.AgentProfileID, Runtime: execution.RuntimeName, Status: executorRunningStatusFromExecution(execution), @@ -92,7 +94,9 @@ func buildRunningFromExecution(execution *AgentExecution, prior *models.Executor LastSeenAt: lastSeenAt, } if prior != nil { - running.ExecutorID = prior.ExecutorID + if strings.TrimSpace(prior.ExecutorID) != "" { + running.ExecutorID = prior.ExecutorID + } if prior.ExecutionProfileID == execution.AgentProfileID { running.ResumeToken = prior.ResumeToken running.LastMessageUUID = prior.LastMessageUUID diff --git a/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go b/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go index 2810575bb6..bb66aa3c3a 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/persistence_test.go @@ -12,9 +12,10 @@ import ( ) type captureExecutorRunningWriter struct { - prior *models.ExecutorRunning - running *models.ExecutorRunning - upsertErr error + prior *models.ExecutorRunning + running *models.ExecutorRunning + upsertErr error + deleteCalls int } func (w *captureExecutorRunningWriter) GetExecutorRunningBySessionID( @@ -32,6 +33,7 @@ func (w *captureExecutorRunningWriter) UpsertExecutorRunning(_ context.Context, } func (w *captureExecutorRunningWriter) DeleteExecutorRunningBySessionID(_ context.Context, _ string) error { + w.deleteCalls++ return nil } @@ -67,6 +69,28 @@ func TestBuildRunningFromExecutionPersistsLiveAgentctlEndpoint(t *testing.T) { } } +func TestBuildRunningFromExecutionPersistsFreshExecutorIdentity(t *testing.T) { + running := buildRunningFromExecution(&AgentExecution{ + ID: "exec-1", TaskID: "task-1", SessionID: "session-1", + metadata: map[string]interface{}{"executor_id": " executor-1 "}, + }, nil) + + if running.ExecutorID != "executor-1" { + t.Fatalf("ExecutorID = %q, want executor-1", running.ExecutorID) + } +} + +func TestBuildRunningFromExecutionPreservesPriorExecutorIdentity(t *testing.T) { + running := buildRunningFromExecution(&AgentExecution{ + ID: "exec-2", TaskID: "task-1", SessionID: "session-1", + metadata: map[string]interface{}{"executor_id": "hostile-replacement"}, + }, &models.ExecutorRunning{ExecutorID: "recorded-executor"}) + + if running.ExecutorID != "recorded-executor" { + t.Fatalf("ExecutorID = %q, want recorded-executor", running.ExecutorID) + } +} + func TestBuildRunningFromExecutionBindsResumeTokenToExecutionProfile(t *testing.T) { prior := &models.ExecutorRunning{ ExecutionProfileID: "codex-profile", diff --git a/apps/backend/internal/agent/runtime/lifecycle/process_runner.go b/apps/backend/internal/agent/runtime/lifecycle/process_runner.go index 91379cd54d..725ead1a3a 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/process_runner.go +++ b/apps/backend/internal/agent/runtime/lifecycle/process_runner.go @@ -27,7 +27,8 @@ func (m *Manager) StartProcess(ctx context.Context, req StartProcessRequest) (*a if !ok { return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, req.SessionID) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return nil, fmt.Errorf("agentctl client not available for session %s", req.SessionID) } @@ -67,7 +68,8 @@ func (m *Manager) WaitForAgentctlReadyForSession(ctx context.Context, sessionID if !ok { return fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return fmt.Errorf("agentctl client not available for session %s", sessionID) } @@ -79,16 +81,20 @@ func (m *Manager) StopProcess(ctx context.Context, processID string) error { return fmt.Errorf("process_id is required") } for _, exec := range m.executionStore.List() { - client := exec.GetAgentCtlClient() + client, releaseClient := exec.AcquireAgentCtlClient() if client == nil { + releaseClient() continue } if _, err := client.GetProcess(ctx, processID, false); err != nil { + releaseClient() continue } if err := client.StopProcess(ctx, processID); err != nil { + releaseClient() return err } + releaseClient() m.releaseActivity(processActivityKey(processID)) return nil } @@ -129,7 +135,8 @@ func (m *Manager) StopProcessForSession(ctx context.Context, sessionID, processI if !ok { return fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return fmt.Errorf("agentctl client not available for session %s", sessionID) } @@ -148,7 +155,8 @@ func (m *Manager) ListProcesses(ctx context.Context, sessionID string) ([]agentc if !ok { return nil, fmt.Errorf("%w: %s", ErrNoExecutionForSession, sessionID) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return nil, fmt.Errorf("agentctl client not available for session %s", sessionID) } @@ -160,10 +168,13 @@ func (m *Manager) GetProcess(ctx context.Context, processID string, includeOutpu return nil, fmt.Errorf("process_id is required") } for _, exec := range m.executionStore.List() { - if exec.GetAgentCtlClient() == nil { + client, releaseClient := exec.AcquireAgentCtlClient() + if client == nil { + releaseClient() continue } - proc, err := exec.GetAgentCtlClient().GetProcess(ctx, processID, includeOutput) + proc, err := client.GetProcess(ctx, processID, includeOutput) + releaseClient() if err == nil { return proc, nil } @@ -176,13 +187,15 @@ func (m *Manager) StopAllProcesses(ctx context.Context) error { executions := m.executionStore.List() var errs []error for _, exec := range executions { - client := exec.GetAgentCtlClient() + client, releaseClient := exec.AcquireAgentCtlClient() if client == nil { + releaseClient() continue } procs, err := client.ListProcesses(ctx, exec.SessionID) if err != nil { errs = append(errs, err) + releaseClient() continue } for _, proc := range procs { @@ -192,6 +205,7 @@ func (m *Manager) StopAllProcesses(ctx context.Context) error { m.releaseActivity(processActivityKey(proc.ID)) } } + releaseClient() } return errors.Join(errs...) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/session.go b/apps/backend/internal/agent/runtime/lifecycle/session.go index 46839c8814..0f096385b3 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/session.go +++ b/apps/backend/internal/agent/runtime/lifecycle/session.go @@ -480,7 +480,9 @@ func (sm *SessionManager) applyStartModelPolicyToEffectiveModel( if runtimeModel != "" { effective.model = runtimeModel } - if effective.model == "" || execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if effective.model == "" || client == nil { return effective } // The policy owns this model decision even when it deliberately continues @@ -489,7 +491,7 @@ func (sm *SessionManager) applyStartModelPolicyToEffectiveModel( effective.handled = true startModelPolicy.Model = effective.model decision, policyErr := applyStartModelPolicy( - ctx, sm.logger, execution.agentctl, + ctx, sm.logger, client, execution.GetModelState(), startModelPolicy, ) if policyErr != nil { @@ -521,15 +523,20 @@ func (sm *SessionManager) initializeACPConnection( ) execution.SetSessionSpan(sessionSpan) ctx = trace.ContextWithSpan(ctx, sessionSpan) - if execution.agentctl != nil { - execution.agentctl.SetTraceContext(execution.SessionTraceContext()) + client, releaseClient := execution.AcquireAgentCtlClient() + if client != nil { + client.SetTraceContext(execution.SessionTraceContext()) + } else { + return ctx, nil, fmt.Errorf("execution %q has no agentctl client", execution.ID) } ctx, initSpan := tracing.TraceSessionInit(ctx, execution.TaskID, execution.SessionID, execution.ID) defer initSpan.End() rt := agentConfig.Runtime() + agentctlURL := client.BaseURL() + releaseClient() sm.logger.Info("initializing ACP session", - zap.String("execution_id", execution.ID), zap.String("agentctl_url", execution.agentctl.BaseURL()), + zap.String("execution_id", execution.ID), zap.String("agentctl_url", agentctlURL), zap.String("agent_type", agentConfig.ID()), zap.String("existing_acp_session_id", execution.ACPSessionID), zap.Bool("native_session_resume", rt.SessionConfig.NativeSessionResume)) if sm.streamManager != nil { @@ -542,7 +549,12 @@ func (sm *SessionManager) initializeACPConnection( return ctx, nil, fmt.Errorf("timeout waiting for agent stream to connect") } } - result, err := sm.InitializeSession(ctx, execution.agentctl, agentConfig, execution.ACPSessionID, execution.WorkspacePath, mcpServers) + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return ctx, nil, fmt.Errorf("execution %q has no agentctl client", execution.ID) + } + result, err := sm.InitializeSession(ctx, client, agentConfig, execution.ACPSessionID, execution.WorkspacePath, mcpServers) + releaseClient() if err != nil { // loadSession already logged the root cause. context.Canceled is // caller teardown, so keep this outer boundary at WARN too — @@ -573,7 +585,9 @@ func (sm *SessionManager) applyProfileSessionLayers( modelPolicyHandled bool, policyAppliedModel string, ) (string, string, string, map[string]string) { - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return "", "", "", nil } finalConfigID := "" @@ -595,7 +609,7 @@ func (sm *SessionManager) applyProfileSessionLayers( sm.logger.Info("profile model applied by start-model policy", zap.String("execution_id", execution.ID), zap.String("model", policyAppliedModel)) } - } else if err := execution.agentctl.SetModel(ctx, profileModel); err != nil { + } else if err := client.SetModel(ctx, profileModel); err != nil { sm.logger.Warn("failed to set profile model via ACP", zap.String("execution_id", execution.ID), zap.String("model", profileModel), zap.Error(err)) } else { @@ -606,7 +620,7 @@ func (sm *SessionManager) applyProfileSessionLayers( } } if profileMode != "" { - if err := execution.agentctl.SetMode(ctx, acpSessionID, profileMode); err != nil { + if err := client.SetMode(ctx, acpSessionID, profileMode); err != nil { sm.logger.Warn("failed to set profile mode via ACP", zap.String("execution_id", execution.ID), zap.String("mode", profileMode), zap.Error(err)) } else { @@ -618,7 +632,7 @@ func (sm *SessionManager) applyProfileSessionLayers( sanitizedOptions := profileconfig.SanitizeConfigOptions(profileConfigOptions) for _, configID := range sortedConfigOptionKeys(sanitizedOptions) { value := sanitizedOptions[configID] - if err := execution.agentctl.SetConfigOption(ctx, configID, value); err != nil { + if err := client.SetConfigOption(ctx, configID, value); err != nil { sm.logger.Warn("failed to set profile config option via ACP", zap.String("execution_id", execution.ID), zap.String("config_id", configID), zap.String("value", value), zap.Error(err)) @@ -644,11 +658,13 @@ func (sm *SessionManager) applyRuntimeSessionLayers( runtimeConfigOptions map[string]string, ) (string, []string) { failed := make([]string, 0) - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return finalConfigID, failed } if runtimeModel != "" && runtimeModel != profileModel { - if err := execution.agentctl.SetModel(ctx, runtimeModel); err != nil { + if err := client.SetModel(ctx, runtimeModel); err != nil { failed = append(failed, "model") sm.logger.Warn("failed to set runtime model via ACP", zap.String("execution_id", execution.ID), zap.String("model", runtimeModel), zap.Error(err)) @@ -662,7 +678,7 @@ func (sm *SessionManager) applyRuntimeSessionLayers( } } if runtimeMode != "" && runtimeMode != profileMode { - if err := execution.agentctl.SetMode(ctx, acpSessionID, runtimeMode); err != nil { + if err := client.SetMode(ctx, acpSessionID, runtimeMode); err != nil { failed = append(failed, "mode") sm.logger.Warn("failed to set runtime mode via ACP", zap.String("execution_id", execution.ID), zap.String("mode", runtimeMode), zap.Error(err)) @@ -681,7 +697,7 @@ func (sm *SessionManager) applyRuntimeSessionLayers( } for _, configID := range sortedConfigOptionKeys(sanitizedOptions) { value := sanitizedOptions[configID] - if err := execution.agentctl.SetConfigOption(ctx, configID, value); err != nil { + if err := client.SetConfigOption(ctx, configID, value); err != nil { failed = append(failed, configID) sm.logger.Warn("failed to set runtime config option via ACP", zap.String("execution_id", execution.ID), zap.String("config_id", configID), @@ -1238,7 +1254,9 @@ func (sm *SessionManager) SendPromptSteerWithDispatchCallback( dispatchOnly bool, onDispatched func(), ) (*PromptResult, error) { - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { return nil, fmt.Errorf("execution %q has no agentctl client", execution.ID) } if validateStatus && execution.Status != v1.AgentStatusRunning && execution.Status != v1.AgentStatusReady { @@ -1381,7 +1399,9 @@ func (sm *SessionManager) sendPrompt( callbacks sendPromptCallbacks, steer bool, ) (*PromptResult, error) { - if execution.agentctl == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { err := fmt.Errorf("execution %q has no agentctl client", execution.ID) sm.reportPromptFailure(execution, 0, err, callbacks.onFailure) return nil, err @@ -1500,7 +1520,12 @@ func (sm *SessionManager) materializeAttachments( if err != nil { return nil, fmt.Errorf("open attachment %q: %w", attachment.AttachmentID, err) } - materialized, materializeErr := execution.agentctl.MaterializeAttachment( + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + _ = reader.Close() + return nil, fmt.Errorf("execution %q has no agentctl client", execution.ID) + } + materialized, materializeErr := client.MaterializeAttachment( ctx, execution.ACPSessionID, attachment.AttachmentID, @@ -1509,6 +1534,7 @@ func (sm *SessionManager) materializeAttachments( sizeBytes, reader, ) + releaseClient() closeErr := reader.Close() if materializeErr != nil { return nil, fmt.Errorf("materialize attachment %q: %w", attachment.AttachmentID, materializeErr) @@ -1675,10 +1701,15 @@ func (sm *SessionManager) callAgentctlPrompt( promptGeneration uint64, steer bool, ) error { - if steer { - return execution.agentctl.PromptSteer(ctx, prompt, attachments, promptGeneration) + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", execution.ID) } - return execution.agentctl.Prompt(ctx, prompt, attachments, promptGeneration) + if steer { + return client.PromptSteer(ctx, prompt, attachments, promptGeneration) + } + return client.Prompt(ctx, prompt, attachments, promptGeneration) } func (sm *SessionManager) dispatchPrompt( @@ -1730,7 +1761,13 @@ func (sm *SessionManager) retryPromptAfterReconnect( var lastErr error for { - if !execution.agentctl.HasAgentStream() { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", execution.ID) + } + hasAgentStream := client.HasAgentStream() + releaseClient() + if !hasAgentStream { ready := make(chan struct{}) sm.streamManager.connectUpdatesStreamAsync(execution, ready) @@ -1744,7 +1781,13 @@ func (sm *SessionManager) retryPromptAfterReconnect( } } - if execution.agentctl.HasAgentStream() { + client, releaseClient = execution.AcquireAgentCtlClient() + if client == nil { + return fmt.Errorf("execution %q has no agentctl client", execution.ID) + } + hasAgentStream = client.HasAgentStream() + releaseClient() + if hasAgentStream { if err := sm.callAgentctlPrompt( reconnectCtx, execution, prompt, attachments, promptGeneration, steer, ); err == nil { diff --git a/apps/backend/internal/agent/runtime/lifecycle/skill/delivery.go b/apps/backend/internal/agent/runtime/lifecycle/skill/delivery.go index f728f7ddfe..4c10335cb5 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/skill/delivery.go +++ b/apps/backend/internal/agent/runtime/lifecycle/skill/delivery.go @@ -21,13 +21,19 @@ func (d *Deployer) deliver(_ context.Context, manifest *Manifest, executorType, switch executorType { case "sprites": return d.deliverSprites(manifest) - default: + case "k8s": + return d.deliverKubernetes(manifest) + case "", "local", "local_pc", "worktree", "local_docker", "remote_docker", "ssh", "mock_remote": // local_pc and local_docker share the same delivery: the // worktree IS the agent's CWD inside the executor (Docker // bind-mounts it; local_pc runs the agent in it directly), so // writing skills under //kandev- // gets them in front of the agent's project-skill discovery. return d.deliverLocal(manifest, worktreePath) + default: + d.logger.Warn("skill delivery skipped for unsupported executor", + zap.String("executor_type", executorType)) + return DeployResult{} } } @@ -50,17 +56,26 @@ func (d *Deployer) deliverLocal(manifest *Manifest, worktreePath string) DeployR return DeployResult{InstructionsDir: instructionsDir} } -// deliverSprites serialises the manifest as JSON and stashes it on -// the launch metadata. The Sprites executor reads the JSON during -// post-create setup and uploads files into the sprite. We do NOT -// write files to the host because the sprite runs in a remote sandbox. +// deliverSprites serialises the manifest into launch metadata for upload by +// the remote executor. It never writes remote runtime files on the host. func (d *Deployer) deliverSprites(manifest *Manifest) DeployResult { dir := spritesInstructionsDir(manifest.WorkspaceSlug, manifest.AgentID) + return d.deliverRemote(manifest, dir, "sprites") +} + +func (d *Deployer) deliverKubernetes(manifest *Manifest) DeployResult { + dir := kubernetesInstructionsDir(manifest.WorkspaceSlug, manifest.AgentID) + return d.deliverRemote(manifest, dir, "k8s") +} + +func (d *Deployer) deliverRemote(manifest *Manifest, dir, executorType string) DeployResult { rewriteManifestRefs(manifest, dir) normalizeManifestSkills(manifest) data, err := json.Marshal(manifest) if err != nil { - d.logger.Warn("failed to marshal skill manifest for sprites", zap.Error(err)) + d.logger.Warn("failed to marshal remote skill manifest", + zap.String("executor_type", executorType), + zap.Error(err)) return DeployResult{} } return DeployResult{ diff --git a/apps/backend/internal/agent/runtime/lifecycle/skill/deployer.go b/apps/backend/internal/agent/runtime/lifecycle/skill/deployer.go index b89a51f5a6..6e3be8fab9 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/skill/deployer.go +++ b/apps/backend/internal/agent/runtime/lifecycle/skill/deployer.go @@ -11,10 +11,9 @@ import ( ) // MetadataKeySkillManifestJSON mirrors lifecycle.MetadataKeySkillManifestJSON -// to avoid an import cycle. It carries the JSON-serialised Manifest the -// Sprites executor consumes during post-create setup to upload skill + -// instruction files into the sprite. The lifecycle test suite asserts -// the same string value. +// to avoid an import cycle. It carries the JSON-serialised Manifest that +// remote workspace executors consume during setup to upload skill and +// instruction files. The lifecycle test suite asserts the same string value. const MetadataKeySkillManifestJSON = "skill_manifest_json" // Config holds Deployer dependencies. All fields except Logger are diff --git a/apps/backend/internal/agent/runtime/lifecycle/skill/deployer_kubernetes_test.go b/apps/backend/internal/agent/runtime/lifecycle/skill/deployer_kubernetes_test.go new file mode 100644 index 0000000000..28cef7f833 --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/skill/deployer_kubernetes_test.go @@ -0,0 +1,77 @@ +package skill_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle/skill" + settingsmodels "github.com/kandev/kandev/internal/agent/settings/models" +) + +func TestDeployKubernetesReturnsRemoteManifestWithoutHostWrites(t *testing.T) { + base := t.TempDir() + reader := &fakeSkillReader{skills: map[string]*skill.Skill{ + "sk-foo": {Slug: "sk-foo", Content: "# foo", SourceType: "inline"}, + }} + lister := &fakeInstructionLister{files: map[string][]*skill.InstructionFile{ + "p1": {{Filename: "AGENTS.md", Content: "# inst"}}, + }} + deployer := newDeployer(t, base, reader, lister) + + result, err := deployer.Deploy(context.Background(), skill.Request{ + Profile: &settingsmodels.AgentProfile{ + ID: "p1", AgentID: "claude-acp", SkillIDs: `["sk-foo"]`, + }, + ExecutorType: "k8s", + }) + if err != nil { + t.Fatalf("Deploy: %v", err) + } + if !strings.HasPrefix(result.InstructionsDir, "/opt/kandev/runtime/") { + t.Errorf("instructions dir = %q, want Kubernetes runtime volume", result.InstructionsDir) + } + raw, ok := result.Metadata[skill.MetadataKeySkillManifestJSON].(string) + if !ok || raw == "" { + t.Fatalf("metadata missing skill_manifest_json: %#v", result.Metadata) + } + var decoded skill.Manifest + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + t.Fatalf("unmarshal manifest: %v", err) + } + if len(decoded.Skills) != 1 || len(decoded.Instructions) != 1 { + t.Fatalf("manifest = %#v, want one skill and one instruction", decoded) + } + if _, err := os.Stat(filepath.Join(base, "runtime")); !os.IsNotExist(err) { + t.Errorf("host runtime tree should not exist for Kubernetes delivery") + } +} + +func TestDeployUnknownExecutorFailsClosed(t *testing.T) { + base := t.TempDir() + worktree := t.TempDir() + reader := &fakeSkillReader{skills: map[string]*skill.Skill{ + "sk-foo": {Slug: "sk-foo", Content: "# foo", SourceType: "inline"}, + }} + deployer := newDeployer(t, base, reader, &fakeInstructionLister{}) + + result, err := deployer.Deploy(context.Background(), skill.Request{ + Profile: &settingsmodels.AgentProfile{ + ID: "p1", AgentID: "claude-acp", SkillIDs: `["sk-foo"]`, + }, + ExecutorType: "future_executor", + WorkspacePath: worktree, + }) + if err != nil { + t.Fatalf("Deploy: %v", err) + } + if result.InstructionsDir != "" || len(result.Metadata) != 0 { + t.Fatalf("result = %#v, want empty fail-closed result", result) + } + if _, err := os.Stat(filepath.Join(worktree, ".claude", "skills", "kandev-sk-foo")); !os.IsNotExist(err) { + t.Fatal("unknown executor wrote skills into host worktree") + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/skill/paths.go b/apps/backend/internal/agent/runtime/lifecycle/skill/paths.go index 2ba654d62e..079c1d7969 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/skill/paths.go +++ b/apps/backend/internal/agent/runtime/lifecycle/skill/paths.go @@ -43,10 +43,15 @@ func cleanRelativeSkillFilePath(p string) (string, bool) { return cleaned, true } -// SpritesRuntimeBase is the on-sprite path where runtime instruction -// files are uploaded. Skills no longer live under this tree; they go -// directly into the sprite's worktree (/workspace/). -const SpritesRuntimeBase = "/root/.kandev/runtime" +const ( + // SpritesRuntimeBase is the on-sprite path where runtime instruction + // files are uploaded. Skills no longer live under this tree; they go + // directly into the sprite's worktree (/workspace/). + SpritesRuntimeBase = "/root/.kandev/runtime" + // KubernetesRuntimeBase is inside the reserved Pod runtime volume. + // The Kubernetes lifecycle uploads the serialized manifest there. + KubernetesRuntimeBase = "/opt/kandev/runtime" +) // instructionsDirHost returns the on-host directory where a manifest's // instruction files are written. @@ -59,3 +64,7 @@ func instructionsDirHost(kandevBase, workspaceSlug, agentID string) string { func spritesInstructionsDir(workspaceSlug, agentID string) string { return SpritesRuntimeBase + "/" + workspaceSlug + "/instructions/" + agentID } + +func kubernetesInstructionsDir(workspaceSlug, agentID string) string { + return KubernetesRuntimeBase + "/" + workspaceSlug + "/instructions/" + agentID +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/streams.go b/apps/backend/internal/agent/runtime/lifecycle/streams.go index 174a715f83..0a470a5776 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/streams.go +++ b/apps/backend/internal/agent/runtime/lifecycle/streams.go @@ -233,7 +233,14 @@ func (sm *StreamManager) ReconnectAll(execution *AgentExecution) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - if err := execution.agentctl.WaitForReady(ctx, 10*time.Second); err != nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + sm.logger.Warn("agentctl unavailable for stream reconnection", zap.String("instance_id", execution.ID)) + return + } + err := client.WaitForReady(ctx, 10*time.Second) + releaseClient() + if err != nil { sm.logger.Warn("agentctl not ready for stream reconnection", zap.String("instance_id", execution.ID), zap.Error(err)) @@ -288,8 +295,15 @@ func (sm *StreamManager) streamContext(execution *AgentExecution) context.Contex func (sm *StreamManager) connectUpdatesStream(execution *AgentExecution, ready chan<- struct{}) { ctx := sm.streamContext(execution) startupGeneration := execution.startupAttemptSnapshot() + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + if ready != nil { + close(ready) + } + return + } - err := execution.agentctl.StreamUpdates(ctx, func(event agentctl.AgentEvent) { + err := client.StreamUpdates(ctx, func(event agentctl.AgentEvent) { if sm.callbacks.OnAgentEventWithGeneration != nil { sm.callbacks.OnAgentEventWithGeneration(execution, event, startupGeneration) } else if sm.callbacks.OnAgentEvent != nil { @@ -300,6 +314,7 @@ func (sm *StreamManager) connectUpdatesStream(execution *AgentExecution, ready c sm.handleUpdatesDisconnectWithGeneration(execution, disconnectErr, startupGeneration) } }) + releaseClient() // Signal that the stream connection attempt is complete (success or failure) // StreamUpdates returns immediately after establishing the WebSocket connection @@ -356,13 +371,18 @@ func (sm *StreamManager) handleUpdatesDisconnectWithGeneration( // closing this stream is expected, not an error. func (sm *StreamManager) connectMCPStream(execution *AgentExecution) { ctx := sm.streamContext(execution) - err := execution.agentctl.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, sm.mcpHandlerFor(execution), func(disconnectErr error) { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return + } + err := client.StreamUpdates(ctx, func(agentctl.AgentEvent) {}, sm.mcpHandlerFor(execution), func(disconnectErr error) { if disconnectErr != nil { sm.logger.Debug("passthrough MCP stream disconnected", zap.String("execution_id", execution.ID), zap.Error(disconnectErr)) } }) + releaseClient() if err != nil { sm.logger.Error("failed to connect passthrough MCP stream", zap.String("execution_id", execution.ID), @@ -470,8 +490,12 @@ func (sm *StreamManager) connectWorkspaceStream(execution *AgentExecution, ready } callbacks := sm.buildWorkspaceCallbacks(execution) - - ws, err := execution.agentctl.StreamWorkspace(ctx, callbacks) + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + return + } + ws, err := client.StreamWorkspace(ctx, callbacks) + releaseClient() if err != nil { sm.logger.Debug("workspace stream connection failed, retrying", zap.String("instance_id", execution.ID), diff --git a/apps/backend/internal/agent/runtime/lifecycle/types.go b/apps/backend/internal/agent/runtime/lifecycle/types.go index 45fc32b486..48fd99c975 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/types.go +++ b/apps/backend/internal/agent/runtime/lifecycle/types.go @@ -100,7 +100,10 @@ type AgentExecution struct { PrepareResult *EnvPrepareResult `json:"-"` // agentctl client for this execution - agentctl *agentctl.Client + agentctl *agentctl.Client + agentctlOverride atomic.Pointer[agentctl.Client] + agentctlLifecycleMu sync.RWMutex + remoteInstanceLifecycleMu sync.Mutex // agentctlReady records the successful health check independently of the // agent process and workspace stream lifecycles. Prepared sessions have a // healthy agentctl before either of those is started or attached. @@ -483,11 +486,58 @@ func (e *AgentExecution) setPromptTurnID(turnID string) { e.promptLifecycleMu.Unlock() } -// GetAgentCtlClient returns the agentctl client for this execution -func (ae *AgentExecution) GetAgentCtlClient() *agentctl.Client { +// currentAgentCtlClient returns the unpinned client snapshot. Callers must +// already hold agentctlLifecycleMu or be implementing the scoped lease. +func (ae *AgentExecution) currentAgentCtlClient() *agentctl.Client { + if ae == nil { + return nil + } + if current := ae.agentctlOverride.Load(); current != nil { + return current + } return ae.agentctl } +// AcquireAgentCtlClient pins the current client for one bounded operation. +// Remote replacement takes the write side before publishing a client, while +// terminal stop takes it while closing and detaching the client. An in-flight +// operation therefore never observes its client being closed underneath it. +// Callers must invoke the returned release function as soon as the bounded +// client operation finishes. +func (ae *AgentExecution) AcquireAgentCtlClient() (*agentctl.Client, func()) { + if ae == nil { + return nil, func() {} + } + ae.agentctlLifecycleMu.RLock() + client := ae.currentAgentCtlClient() + if client == nil { + ae.agentctlLifecycleMu.RUnlock() + return nil, func() {} + } + return client, ae.agentctlLifecycleMu.RUnlock +} + +// replaceAgentctlClient atomically publishes a replacement connection while +// retaining the construction-time field for test/source compatibility. +func (ae *AgentExecution) replaceAgentctlClient(client *agentctl.Client) *agentctl.Client { + if ae == nil || client == nil { + return nil + } + previous := ae.currentAgentCtlClient() + ae.agentctlOverride.Store(client) + return previous +} + +// detachAgentctlClient clears every lookup tier after terminal teardown. +// Callers must hold agentctlLifecycleMu for writing. +func (ae *AgentExecution) detachAgentctlClient() { + if ae == nil { + return + } + ae.agentctl = nil + ae.agentctlOverride.Store(nil) +} + // MarkAgentctlReady records that agentctl passed its startup health check. func (ae *AgentExecution) MarkAgentctlReady() { ae.agentctlReady.Store(true) @@ -502,10 +552,12 @@ func (ae *AgentExecution) IsAgentctlReady() bool { // execution. Returns an empty string when no agentctl client is set (e.g. // before the execution has been wired to an agentctl instance). func (ae *AgentExecution) AgentctlURL() string { - if ae.agentctl == nil { + client, release := ae.AcquireAgentCtlClient() + defer release() + if client == nil { return "" } - return ae.agentctl.BaseURL() + return client.BaseURL() } // SetWorkspaceStream sets the unified workspace stream for this execution diff --git a/apps/backend/internal/agent/runtime/lifecycle/types_agentctl_url_test.go b/apps/backend/internal/agent/runtime/lifecycle/types_agentctl_url_test.go index e741b858af..3bd6cf368f 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/types_agentctl_url_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/types_agentctl_url_test.go @@ -38,3 +38,25 @@ func TestAgentExecution_AgentctlURL_WithClient(t *testing.T) { t.Errorf("AgentctlURL() = %q, want %q", got, want) } } + +func TestAgentExecution_AcquireAgentCtlClientPinsReplacement(t *testing.T) { + t.Parallel() + client := agentctl.NewClient("127.0.0.1", 12345, newNopLogger(t)) + exec := &AgentExecution{ID: "exec-lease", agentctl: client} + + acquired, release := exec.AcquireAgentCtlClient() + if acquired != client { + t.Fatalf("AcquireAgentCtlClient() = %p, want %p", acquired, client) + } + if exec.agentctlLifecycleMu.TryLock() { + exec.agentctlLifecycleMu.Unlock() + release() + t.Fatal("replacement lock acquired while client lease was active") + } + + release() + if !exec.agentctlLifecycleMu.TryLock() { + t.Fatal("replacement lock remained blocked after client lease release") + } + exec.agentctlLifecycleMu.Unlock() +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/utility.go b/apps/backend/internal/agent/runtime/lifecycle/utility.go index ac7fa71845..7875da346b 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/utility.go +++ b/apps/backend/internal/agent/runtime/lifecycle/utility.go @@ -42,7 +42,8 @@ func (m *Manager) ExecuteInferencePrompt(ctx context.Context, sessionID, agentID return nil, fmt.Errorf("no execution available for session %s: %w", sessionID, err) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return nil, fmt.Errorf("agentctl client not available for session %s", sessionID) } @@ -102,7 +103,8 @@ func (m *Manager) ExecuteInferenceProfilePrompt(ctx context.Context, sessionID, if err != nil { return nil, fmt.Errorf("no execution available for session %s: %w", sessionID, err) } - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if client == nil { return nil, fmt.Errorf("agentctl client not available for session %s", sessionID) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/workspace_materialization.go b/apps/backend/internal/agent/runtime/lifecycle/workspace_materialization.go index 6c6fb2b3be..a4dd1c45b6 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/workspace_materialization.go +++ b/apps/backend/internal/agent/runtime/lifecycle/workspace_materialization.go @@ -71,6 +71,7 @@ func (m *Manager) MaterializeRepositoriesForEnvironment(ctx context.Context, tas return nil, err } clients := distinctWorkspaceRepositoryClients(executions) + defer releaseWorkspaceRepositoryClients(clients) if len(clients) == 0 { return nil, fmt.Errorf("workspace execution has no agentctl client") } @@ -112,6 +113,7 @@ func materializeWorkspaceRepositories(ctx context.Context, client workspaceRepos type workspaceRepositoryExecution struct { sessionID string client workspaceRepositoryClient + release func() sourceRoots []string } @@ -121,7 +123,12 @@ func (m *Manager) liveWorkspaceExecutionsForEnvironment(ctx context.Context, tas } executions := make([]*AgentExecution, 0) for _, execution := range m.executionStore.List() { - if execution != nil && execution.TaskEnvironmentID == taskEnvironmentID && execution.GetAgentCtlClient() != nil { + if execution == nil || execution.TaskEnvironmentID != taskEnvironmentID { + continue + } + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client != nil { executions = append(executions, execution) } } @@ -130,7 +137,12 @@ func (m *Manager) liveWorkspaceExecutionsForEnvironment(ctx context.Context, tas if err != nil { return nil, fmt.Errorf("ensure workspace execution: %w", err) } - if execution == nil || execution.GetAgentCtlClient() == nil { + if execution == nil { + return nil, fmt.Errorf("workspace execution has no agentctl client") + } + client, releaseClient := execution.AcquireAgentCtlClient() + releaseClient() + if client == nil { return nil, fmt.Errorf("workspace execution has no agentctl client") } executions = append(executions, execution) @@ -143,23 +155,33 @@ func distinctWorkspaceRepositoryClients(executions []*AgentExecution) []workspac clients := make([]workspaceRepositoryExecution, 0, len(executions)) seen := make(map[*agentctl.Client]struct{}, len(executions)) for _, execution := range executions { - client := execution.GetAgentCtlClient() + client, releaseClient := execution.AcquireAgentCtlClient() if client == nil { continue } if _, exists := seen[client]; exists { + releaseClient() continue } seen[client] = struct{}{} clients = append(clients, workspaceRepositoryExecution{ sessionID: execution.SessionID, client: client, + release: releaseClient, sourceRoots: append([]string(nil), execution.WorkspaceSourceRoots...), }) } return clients } +func releaseWorkspaceRepositoryClients(executions []workspaceRepositoryExecution) { + for _, execution := range executions { + if execution.release != nil { + execution.release() + } + } +} + func workspaceRepositorySessionIDs(executions []*AgentExecution) []string { ids := make([]string, 0, len(executions)) seen := make(map[string]struct{}, len(executions)) diff --git a/apps/backend/internal/agentruntime/runtime.go b/apps/backend/internal/agentruntime/runtime.go index aa6fc1aef7..2219623ba7 100644 --- a/apps/backend/internal/agentruntime/runtime.go +++ b/apps/backend/internal/agentruntime/runtime.go @@ -17,6 +17,7 @@ const ( RuntimeRemoteDocker Runtime = "remote_docker" RuntimeSprites Runtime = "sprites" RuntimeSSH Runtime = "ssh" + RuntimeKubernetes Runtime = "k8s" ) // IsContainerized reports whether the runtime hosts the agent @@ -25,7 +26,7 @@ const ( // decision gets reviewed; new constants default to host-mode. func (r Runtime) IsContainerized() bool { switch r { - case RuntimeDocker, RuntimeRemoteDocker, RuntimeSprites: + case RuntimeDocker, RuntimeRemoteDocker, RuntimeSprites, RuntimeKubernetes: return true default: return false diff --git a/apps/backend/internal/agentruntime/runtime_test.go b/apps/backend/internal/agentruntime/runtime_test.go index 54aa0b7468..8ade055165 100644 --- a/apps/backend/internal/agentruntime/runtime_test.go +++ b/apps/backend/internal/agentruntime/runtime_test.go @@ -13,6 +13,7 @@ func TestRuntimeIsContainerized(t *testing.T) { {RuntimeDocker, true}, {RuntimeRemoteDocker, true}, {RuntimeSprites, true}, + {RuntimeKubernetes, true}, {Runtime(""), false}, {Runtime("unknown"), false}, } @@ -38,6 +39,7 @@ func TestRuntimeString(t *testing.T) { {RuntimeDocker, "docker"}, {RuntimeRemoteDocker, "remote_docker"}, {RuntimeSprites, "sprites"}, + {RuntimeKubernetes, "k8s"}, {Runtime(""), ""}, } diff --git a/apps/backend/internal/backendapp/adapters.go b/apps/backend/internal/backendapp/adapters.go index b5c76e6221..7875f4b030 100644 --- a/apps/backend/internal/backendapp/adapters.go +++ b/apps/backend/internal/backendapp/adapters.go @@ -668,6 +668,7 @@ func (a *lifecycleAdapter) PollRemoteStatusForRecords(ctx context.Context, recor lcRecords := make([]lifecycle.RemoteStatusPollRecord, len(records)) for i, r := range records { lcRecords[i] = lifecycle.RemoteStatusPollRecord{ + TaskID: r.TaskID, SessionID: r.SessionID, Runtime: r.Runtime, AgentExecutionID: r.AgentExecutionID, @@ -744,7 +745,8 @@ func (a *lifecycleAdapter) GetGitLog(ctx context.Context, sessionID, baseCommit if !ok { return nil, nil // No execution, not an error } - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { return nil, nil } @@ -757,7 +759,8 @@ func (a *lifecycleAdapter) GetCumulativeDiff(ctx context.Context, sessionID, bas if !ok { return nil, nil // No execution, not an error } - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { return nil, nil } @@ -773,7 +776,8 @@ func (a *lifecycleAdapter) GetGitStatus(ctx context.Context, sessionID string) ( if !ok { return nil, nil // No execution, not an error } - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { return nil, nil } @@ -786,7 +790,8 @@ func (a *lifecycleAdapter) GetGitStatusFresh(ctx context.Context, sessionID stri if !ok { return nil, nil } - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { return nil, nil } diff --git a/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go b/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go new file mode 100644 index 0000000000..a03b9a5ce9 --- /dev/null +++ b/apps/backend/internal/backendapp/adapters_kubernetes_launch_test.go @@ -0,0 +1,246 @@ +package backendapp + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" + + "github.com/kandev/kandev/internal/agent/agents" + agentexecutor "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agent/registry" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/events/bus" + orchestratorexecutor "github.com/kandev/kandev/internal/orchestrator/executor" + "github.com/kandev/kandev/internal/task/models" + sqlitetaskrepo "github.com/kandev/kandev/internal/task/repository/sqlite" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +var errKubernetesLaunchCaptured = errors.New("kubernetes launch captured") + +type capturingKubernetesBackend struct { + req *lifecycle.ExecutorCreateRequest +} + +func (b *capturingKubernetesBackend) Name() agentexecutor.Name { return agentexecutor.NameKubernetes } +func (b *capturingKubernetesBackend) HealthCheck(context.Context) error { return nil } +func (b *capturingKubernetesBackend) CreateInstance( + _ context.Context, + req *lifecycle.ExecutorCreateRequest, +) (*lifecycle.ExecutorInstance, error) { + b.req = req + return nil, errKubernetesLaunchCaptured +} +func (b *capturingKubernetesBackend) StopInstance(context.Context, *lifecycle.ExecutorInstance, bool) error { + return nil +} +func (b *capturingKubernetesBackend) RecoverInstances(context.Context) ([]*lifecycle.ExecutorInstance, error) { + return nil, nil +} +func (b *capturingKubernetesBackend) GetInteractiveRunner() *process.InteractiveRunner { return nil } +func (b *capturingKubernetesBackend) RequiresCloneURL() bool { return true } +func (b *capturingKubernetesBackend) ShouldApplyPreferredShell() bool { return false } +func (b *capturingKubernetesBackend) IsAlwaysResumable() bool { return true } + +type kubernetesLaunchProfileResolver struct{} + +func (kubernetesLaunchProfileResolver) ResolveProfile( + context.Context, + string, +) (*lifecycle.AgentProfileInfo, error) { + return &lifecycle.AgentProfileInfo{ + ProfileID: "agent-profile", AgentID: "mock-agent", AgentName: "mock-agent", + }, nil +} + +type kubernetesLaunchProfileReader struct{} + +func (kubernetesLaunchProfileReader) GetTaskSession(context.Context, string) (*models.TaskSession, error) { + return &models.TaskSession{ID: "session-1", TaskID: "task-1", ExecutorProfileID: "profile-k8s"}, nil +} +func (kubernetesLaunchProfileReader) HasActiveTaskResourceCleanupJob(context.Context, string) (bool, error) { + return false, nil +} +func (kubernetesLaunchProfileReader) GetTaskEnvironment(context.Context, string) (*models.TaskEnvironment, error) { + return nil, nil +} +func (kubernetesLaunchProfileReader) GetExecutorProfile(context.Context, string) (*models.ExecutorProfile, error) { + return &models.ExecutorProfile{ + ID: "profile-k8s", ExecutorID: "executor-k8s", + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesProfilePlatform: "linux/amd64", + lifecycle.MetadataKeyKubernetesProfileMainContainer: "kandev-agent", + lifecycle.MetadataKeyKubernetesPodTemplateYAML: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example.test/agent:latest\n", + lifecycle.MetadataKeyKubernetesWorkspaceMode: "empty_dir", + }, + }, nil +} + +// Reviewer-requested boundary contract: orchestrator request -> backend adapter +// -> lifecycle runtime request must keep authoritative Kubernetes connection keys. +func TestKubernetesLaunchCarriesConnectionConfigThroughAdapterAndLifecycle(t *testing.T) { + log := newTestLogger() + agentRegistry := registry.NewRegistry(log) + mockAgent := agents.NewMockAgentWithID("mock-agent", "Mock", "Mock") + mockAgent.SetEnabled(true) + require.NoError(t, agentRegistry.Register(mockAgent)) + backend := &capturingKubernetesBackend{} + executorRegistry := lifecycle.NewExecutorRegistry(log) + executorRegistry.Register(backend) + eventBus := bus.NewMemoryEventBus(log) + t.Cleanup(eventBus.Close) + mgr := lifecycle.NewManager( + agentRegistry, eventBus, executorRegistry, nil, kubernetesLaunchProfileResolver{}, nil, + lifecycle.ExecutorFallbackWarn, "", log, + ) + mgr.SetExecutorProfileReader(kubernetesLaunchProfileReader{}) + t.Cleanup(func() { require.NoError(t, mgr.Stop()) }) + adapter := newLifecycleAdapter(mgr, agentRegistry, log) + wantConfig := map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/kind.yaml", + lifecycle.MetadataKeyKubernetesKubeContext: "kind-kandev", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + } + + _, err := adapter.LaunchAgent(context.Background(), &orchestratorexecutor.LaunchAgentRequest{ + TaskID: "task-1", WorkspaceID: "workspace-1", SessionID: "session-1", + AgentProfileID: "agent-profile", ExecutorType: string(models.ExecutorTypeKubernetes), + ExecutorConfig: wantConfig, + Metadata: map[string]interface{}{ + lifecycle.MetadataKeyExecutorProfileID: "profile-k8s", + "executor_id": "executor-k8s", + }, + }) + + require.ErrorIs(t, err, errKubernetesLaunchCaptured) + require.NotNil(t, backend.req) + for key, want := range wantConfig { + require.Equal(t, want, backend.req.Metadata[key], key) + } +} + +// Reviewer-requested full-path contract: persisted executor selection must +// survive orchestrator request construction, backend adapter conversion, and +// lifecycle metadata projection before the Kubernetes client is constructed. +func TestPreparedKubernetesLaunchCarriesPersistedConnectionConfigToLifecycle(t *testing.T) { + tests := []struct { + name string + executorConfig map[string]string + wantMetadata map[string]string + }{ + { + name: "kubeconfig", + executorConfig: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/kind.yaml", + lifecycle.MetadataKeyKubernetesKubeContext: "kind-kandev", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + }, + { + name: "in cluster clears kubeconfig fields", + executorConfig: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "30", + }, + wantMetadata: map[string]string{ + lifecycle.MetadataKeyKubernetesKubeconfigPath: "", + lifecycle.MetadataKeyKubernetesKubeContext: "", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + wantMetadata := make(map[string]string, len(test.wantMetadata)+len(test.executorConfig)) + for key, value := range test.wantMetadata { + wantMetadata[key] = value + } + for key, value := range test.executorConfig { + wantMetadata[key] = value + } + testPreparedKubernetesLaunchCarriesConnectionConfig( + t, test.executorConfig, wantMetadata, + ) + }) + } +} + +func testPreparedKubernetesLaunchCarriesConnectionConfig( + t *testing.T, + executorConfig map[string]string, + wantMetadata map[string]string, +) { + t.Helper() + ctx := context.Background() + dbConn, err := db.OpenSQLite(filepath.Join(t.TempDir(), "kubernetes-launch.db")) + require.NoError(t, err) + sqlxDB := sqlx.NewDb(dbConn, "sqlite3") + t.Cleanup(func() { require.NoError(t, sqlxDB.Close()) }) + repo, err := sqlitetaskrepo.NewWithDB(sqlxDB, sqlxDB, nil) + require.NoError(t, err) + require.NoError(t, repo.CreateWorkspace(ctx, &models.Workspace{ID: "workspace-1", Name: "Workspace"})) + require.NoError(t, repo.CreateTask(ctx, &models.Task{ + ID: "task-1", WorkspaceID: "workspace-1", Title: "Kubernetes launch", State: v1.TaskStateCreated, + })) + require.NoError(t, repo.CreateExecutor(ctx, &models.Executor{ + ID: "executor-k8s", Name: "Kind", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: executorConfig, + })) + require.NoError(t, repo.CreateExecutorProfile(ctx, &models.ExecutorProfile{ + ID: "profile-k8s", ExecutorID: "executor-k8s", Name: "Kind workload", + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesProfilePlatform: "linux/amd64", + lifecycle.MetadataKeyKubernetesProfileMainContainer: "kandev-agent", + lifecycle.MetadataKeyKubernetesPodTemplateYAML: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n containers:\n - name: kandev-agent\n image: example.test/agent:latest\n", + lifecycle.MetadataKeyKubernetesWorkspaceMode: "empty_dir", + }, + })) + now := time.Now().UTC() + require.NoError(t, repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-1", TaskID: "task-1", AgentProfileID: "agent-profile", + ExecutorID: "executor-k8s", ExecutorProfileID: "profile-k8s", + State: models.TaskSessionStateCreated, StartedAt: now, UpdatedAt: now, + })) + + log := newTestLogger() + agentRegistry := registry.NewRegistry(log) + mockAgent := agents.NewMockAgentWithID("mock-agent", "Mock", "Mock") + mockAgent.SetEnabled(true) + require.NoError(t, agentRegistry.Register(mockAgent)) + backend := &capturingKubernetesBackend{} + executorRegistry := lifecycle.NewExecutorRegistry(log) + executorRegistry.Register(backend) + eventBus := bus.NewMemoryEventBus(log) + t.Cleanup(eventBus.Close) + mgr := lifecycle.NewManager( + agentRegistry, eventBus, executorRegistry, nil, kubernetesLaunchProfileResolver{}, nil, + lifecycle.ExecutorFallbackWarn, "", log, + ) + mgr.SetExecutorProfileReader(repo) + t.Cleanup(func() { require.NoError(t, mgr.Stop()) }) + adapter := newLifecycleAdapter(mgr, agentRegistry, log) + orchestrator := orchestratorexecutor.NewExecutor( + adapter, repo, log, orchestratorexecutor.ExecutorConfig{}, + ) + + _, err = orchestrator.LaunchPreparedSession(ctx, &v1.Task{ + ID: "task-1", WorkspaceID: "workspace-1", Title: "Kubernetes launch", + }, "session-1", orchestratorexecutor.LaunchOptions{AgentProfileID: "agent-profile"}) + + require.ErrorIs(t, err, errKubernetesLaunchCaptured) + require.NotNil(t, backend.req) + for key, want := range wantMetadata { + require.Equal(t, want, backend.req.Metadata[key], key) + } +} diff --git a/apps/backend/internal/backendapp/agents.go b/apps/backend/internal/backendapp/agents.go index a7280e094e..9abafbe673 100644 --- a/apps/backend/internal/backendapp/agents.go +++ b/apps/backend/internal/backendapp/agents.go @@ -30,7 +30,7 @@ func provideLifecycleManager( eventBus bus.EventBus, agentSettingsRepo settingsstore.Repository, agentRegistry *registry.Registry, - secretStore secrets.SecretStore, + rawSecretStore secrets.SecretStore, baseBranchProvider lifecycle.BaseBranchProvider, comparisonTargetProvider lifecycle.ComparisonTargetProvider, managedRuntimeSelections managedruntime.SelectionReader, @@ -38,6 +38,7 @@ func provideLifecycleManager( mcpPrincipalScoper lifecycle.MCPPrincipalScoper, ) (*lifecycle.Manager, error) { log.Info("Initializing Agent Manager...") + secretStores := newLifecycleSecretStores(rawSecretStore) // Create runtime registry to manage multiple runtimes executorRegistry := lifecycle.NewExecutorRegistry(log) @@ -80,18 +81,19 @@ func provideLifecycleManager( // Register Sprites runtime (remote sandboxes via Sprites.dev) agentctlResolver := lifecycle.NewAgentctlResolver(log) - spritesExec := lifecycle.NewSpritesExecutor(secretStore, agentRegistry, agentctlResolver, 8765, log) + spritesExec := lifecycle.NewSpritesExecutor(secretStores.credentials, agentRegistry, agentctlResolver, 8765, log) executorRegistry.Register(spritesExec) log.Info("Sprites runtime registered") // Register SSH runtime (run an agent on any Linux box reachable over SSH). - sshExec := lifecycle.NewSSHExecutor(secretStore, agentRegistry, agentctlResolver, log) + sshExec := lifecycle.NewSSHExecutor(secretStores.credentials, agentRegistry, agentctlResolver, log) executorRegistry.Register(sshExec) log.Info("SSH runtime registered") + registerKubernetesBackend(executorRegistry, agentctlResolver, log) credsMgr := credentials.NewManager(log) - if secretStore != nil { - credsMgr.AddProvider(secrets.NewSecretStoreProvider(secretStore)) + if secretStores.credentials != nil { + credsMgr.AddProvider(secrets.NewSecretStoreProvider(secretStores.credentials)) } credsMgr.AddProvider(credentials.NewEnvProvider("KANDEV_")) credsMgr.AddProvider(credentials.NewAugmentSessionProvider()) @@ -125,8 +127,9 @@ func provideLifecycleManager( preparerRegistry.Register(models.ExecutorTypeLocalDocker, lifecycle.NewDockerPreparer(log)) preparerRegistry.Register(models.ExecutorTypeSprites, lifecycle.NewSpritesPreparer(log)) preparerRegistry.Register(models.ExecutorTypeSSH, lifecycle.NewSSHPreparer(log)) + registerKubernetesPreparer(preparerRegistry, log) lifecycleMgr.SetPreparerRegistry(preparerRegistry) - lifecycleMgr.SetSecretStore(secretStore) + lifecycleMgr.SetSecretStore(secretStores.runtime) if err := lifecycleMgr.SetAgentctlStartupConfig(cfg.ManagedAgentctlStartupConfig()); err != nil { return nil, fmt.Errorf("configure managed agentctl startup: %w", err) } @@ -170,6 +173,32 @@ func provideLifecycleManager( return lifecycleMgr, nil } +type lifecycleSecretStores struct { + runtime secrets.SecretStore + credentials secrets.SecretStore +} + +func newLifecycleSecretStores(raw secrets.SecretStore) lifecycleSecretStores { + return lifecycleSecretStores{ + runtime: raw, + credentials: secrets.NewUserVisibleStore(raw), + } +} + +func registerKubernetesBackend( + registry *lifecycle.ExecutorRegistry, + agentctlResolver *lifecycle.AgentctlResolver, + log *logger.Logger, +) { + registry.Register(lifecycle.NewKubernetesExecutor(agentctlResolver, log)) + log.Info("Kubernetes runtime registered (lazy initialization)") +} + +func registerKubernetesPreparer(registry *lifecycle.PreparerRegistry, log *logger.Logger) { + registry.Register(models.ExecutorTypeKubernetes, lifecycle.NewKubernetesPreparer(log)) + log.Info("Kubernetes preparer registered") +} + func credentialFilePath(cfg *config.Config) string { if cfg != nil { return strings.TrimSpace(cfg.Credentials.File) diff --git a/apps/backend/internal/backendapp/agents_kubernetes_test.go b/apps/backend/internal/backendapp/agents_kubernetes_test.go new file mode 100644 index 0000000000..cfa8ec07da --- /dev/null +++ b/apps/backend/internal/backendapp/agents_kubernetes_test.go @@ -0,0 +1,175 @@ +package backendapp + +import ( + "context" + "errors" + "fmt" + "sync" + "testing" + + "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" +) + +func TestRegisterKubernetesBackendPreventsStandaloneFallback(t *testing.T) { + t.Parallel() + + log := newTestLogger() + registry := lifecycle.NewExecutorRegistry(log) + registerKubernetesBackend(registry, lifecycle.NewAgentctlResolver(log), log) + manager := lifecycle.NewManager(nil, nil, registry, nil, nil, nil, lifecycle.ExecutorFallbackWarn, "", log) + if !manager.RequiresCloneURL(string(models.ExecutorTypeKubernetes)) { + t.Fatal("k8s executor resolved as missing/standalone instead of its registered remote backend") + } + backend, err := registry.GetBackend(executor.NameKubernetes) + if err != nil { + t.Fatalf("GetBackend(k8s) error = %v", err) + } + if _, err := backend.CreateInstance(context.Background(), &lifecycle.ExecutorCreateRequest{}); err == nil { + t.Fatal("Kubernetes backend accepted incomplete configuration") + } +} + +func TestRegisterKubernetesPreparerUsesRealLifecyclePreparer(t *testing.T) { + t.Parallel() + + log := newTestLogger() + registry := lifecycle.NewPreparerRegistry(log) + registerKubernetesPreparer(registry, log) + preparer := registry.Get(models.ExecutorTypeKubernetes) + if preparer == nil { + t.Fatal("k8s preparer was not registered") + } + result, err := preparer.Prepare(context.Background(), &lifecycle.EnvPrepareRequest{}, nil) + if err != nil { + t.Fatalf("Prepare() error = %v", err) + } + if result == nil || !result.Success { + t.Fatalf("Prepare() = %#v, want successful Kubernetes validation result", result) + } +} + +func TestLifecycleSecretStoresSeparateRuntimeAndUserCredentialSecrets(t *testing.T) { + t.Parallel() + + ctx := context.Background() + rawStore := newBackendappSecretStore() + stores := newLifecycleSecretStores(rawStore) + internal := &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: "kandev-runtime:instance-1:agentctl-auth", Name: "runtime auth"}, + Value: "runtime-token", + } + visible := &secrets.SecretWithValue{ + Secret: secrets.Secret{ID: "user-api-key", Name: "user API key"}, + Value: "user-token", + } + if err := stores.runtime.Create(ctx, internal); err != nil { + t.Fatal(err) + } + if err := stores.credentials.Create(ctx, visible); err != nil { + t.Fatal(err) + } + if got, err := stores.runtime.Reveal(ctx, internal.ID); err != nil || got != internal.Value { + t.Fatalf("runtime secret reveal = %q, %v; want raw internal value", got, err) + } + if got, err := stores.credentials.Reveal(ctx, internal.ID); !errors.Is(err, secrets.ErrNotFound) { + t.Fatalf("internal runtime credential was revealed to agents: %q", got) + } + if got, err := stores.credentials.Reveal(ctx, visible.ID); err != nil || got != visible.Value { + t.Fatalf("user credential reveal = %q, %v", got, err) + } + items, err := stores.credentials.List(ctx) + if err != nil { + t.Fatalf("list user credentials: %v", err) + } + if len(items) != 1 || items[0].ID != visible.ID { + t.Fatalf("visible credentials = %#v; want only %q", items, visible.ID) + } + for _, item := range items { + if item.ID == internal.ID { + t.Fatalf("internal runtime credential appeared in agent credential list: %q", item.ID) + } + } +} + +type backendappSecretStore struct { + mu sync.Mutex + secrets map[string]*secrets.SecretWithValue +} + +func newBackendappSecretStore() *backendappSecretStore { + return &backendappSecretStore{secrets: make(map[string]*secrets.SecretWithValue)} +} + +func (s *backendappSecretStore) Create(_ context.Context, secret *secrets.SecretWithValue) error { + s.mu.Lock() + defer s.mu.Unlock() + copy := *secret + s.secrets[secret.ID] = © + return nil +} + +func (s *backendappSecretStore) Get(_ context.Context, id string) (*secrets.Secret, error) { + s.mu.Lock() + defer s.mu.Unlock() + secret := s.secrets[id] + if secret == nil { + return nil, fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + copy := secret.Secret + return ©, nil +} + +func (s *backendappSecretStore) Reveal(_ context.Context, id string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + secret := s.secrets[id] + if secret == nil { + return "", fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + return secret.Value, nil +} + +func (s *backendappSecretStore) Update( + _ context.Context, + id string, + req *secrets.UpdateSecretRequest, +) error { + s.mu.Lock() + defer s.mu.Unlock() + secret := s.secrets[id] + if secret == nil { + return fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + if req.Name != nil { + secret.Name = *req.Name + } + if req.Value != nil { + secret.Value = *req.Value + } + return nil +} + +func (s *backendappSecretStore) Delete(_ context.Context, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.secrets[id]; !ok { + return fmt.Errorf("%w: %s", secrets.ErrNotFound, id) + } + delete(s.secrets, id) + return nil +} + +func (s *backendappSecretStore) List(context.Context) ([]*secrets.SecretListItem, error) { + s.mu.Lock() + defer s.mu.Unlock() + items := make([]*secrets.SecretListItem, 0, len(s.secrets)) + for _, secret := range s.secrets { + items = append(items, &secrets.SecretListItem{ID: secret.ID, Name: secret.Name, HasValue: true}) + } + return items, nil +} + +func (*backendappSecretStore) Close() error { return nil } diff --git a/apps/backend/internal/backendapp/helpers.go b/apps/backend/internal/backendapp/helpers.go index 114ad72f00..6d138892db 100644 --- a/apps/backend/internal/backendapp/helpers.go +++ b/apps/backend/internal/backendapp/helpers.go @@ -54,6 +54,7 @@ import ( "github.com/kandev/kandev/internal/improvekandev" "github.com/kandev/kandev/internal/integrations/workspacescope" "github.com/kandev/kandev/internal/jira" + kuberneteshandlers "github.com/kandev/kandev/internal/kubernetes" "github.com/kandev/kandev/internal/linear" lspinstaller "github.com/kandev/kandev/internal/lsp/installer" mcphandlers "github.com/kandev/kandev/internal/mcp/handlers" @@ -372,7 +373,8 @@ func executionMatchesGitStatusSource(sources *gitStatusSources, execution *lifec } func tryGetLiveGitStatusFromExecution(ctx context.Context, execution *lifecycle.AgentExecution, requestedSessionID, sourceSessionID, taskEnvironmentID string, log *logger.Logger) []*ws.Message { - agentClient := execution.GetAgentCtlClient() + agentClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentClient == nil { log.Debug("no agentctl client available for live git status", zap.String("source_session_id", sourceSessionID)) @@ -1374,6 +1376,15 @@ func registerSecondaryRoutes( } if p.taskRepo != nil { + kuberneteshandlers.RegisterRoutes( + p.router, + p.gateway.Dispatcher, + p.taskRepo, + p.services.Task, + p.log, + ) + p.log.Debug("Registered Kubernetes handlers (HTTP + WebSocket)") + sshhandlers.RegisterRoutes( p.router, p.gateway.Dispatcher, diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index 64cba290ea..8e3e287092 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -514,7 +514,7 @@ func startAgentInfrastructure( eventBus, repos.AgentSettings, agentRegistry, - userSecretStore, + repos.Secrets, services.Task.TaskBaseBranches, services.Task.TaskComparisonTargets, services.ManagedRuntimeSelections, diff --git a/apps/backend/internal/backendapp/metrics_provider.go b/apps/backend/internal/backendapp/metrics_provider.go index 6475389001..adefe2a600 100644 --- a/apps/backend/internal/backendapp/metrics_provider.go +++ b/apps/backend/internal/backendapp/metrics_provider.go @@ -1,6 +1,8 @@ package backendapp import ( + "context" + "errors" "fmt" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" @@ -16,6 +18,23 @@ type metricExecutionLister interface { ListExecutions() []*lifecycle.AgentExecution } +type lifecycleMetricClient struct { + execution *lifecycle.AgentExecution +} + +func (c lifecycleMetricClient) SystemMetrics( + ctx context.Context, + metricIDs []string, + diskPath string, +) (*metrics.SourceSnapshot, error) { + client, releaseClient := c.execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + return nil, errors.New("agentctl client is unavailable") + } + return client.SystemMetrics(ctx, metricIDs, diskPath) +} + func (p lifecycleMetricProvider) MetricExecutions() []metrics.ExecutionSource { if p.manager == nil { return nil @@ -23,7 +42,13 @@ func (p lifecycleMetricProvider) MetricExecutions() []metrics.ExecutionSource { executions := p.manager.ListExecutions() sources := make([]metrics.ExecutionSource, 0, len(executions)) for _, execution := range executions { - if execution == nil || execution.GetAgentCtlClient() == nil || !shouldCollectExecutionMetrics(execution.RuntimeName) { + if execution == nil || !shouldCollectExecutionMetrics(execution.RuntimeName) { + continue + } + client, releaseClient := execution.AcquireAgentCtlClient() + hasClient := client != nil + releaseClient() + if !hasClient { continue } label := fmt.Sprintf("Execution %s", execution.SessionID) @@ -36,17 +61,12 @@ func (p lifecycleMetricProvider) MetricExecutions() []metrics.ExecutionSource { ExecutorType: execution.RuntimeName.String(), SessionID: execution.SessionID, TaskID: execution.TaskID, - Client: execution.GetAgentCtlClient(), + Client: lifecycleMetricClient{execution: execution}, }) } return sources } func shouldCollectExecutionMetrics(runtime agentruntime.Runtime) bool { - switch runtime { - case agentruntime.RuntimeDocker, agentruntime.RuntimeRemoteDocker, agentruntime.RuntimeSprites, agentruntime.RuntimeSSH: - return true - default: - return false - } + return runtime.IsContainerized() || runtime == agentruntime.RuntimeSSH } diff --git a/apps/backend/internal/backendapp/metrics_provider_kubernetes_test.go b/apps/backend/internal/backendapp/metrics_provider_kubernetes_test.go new file mode 100644 index 0000000000..e519ae145d --- /dev/null +++ b/apps/backend/internal/backendapp/metrics_provider_kubernetes_test.go @@ -0,0 +1,26 @@ +package backendapp + +import ( + "testing" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" +) + +func TestLifecycleMetricProviderIncludesKubernetesExecution(t *testing.T) { + execution := executionWithAgentCtl(t, &lifecycle.AgentExecution{ + ID: "exec-k8s", TaskID: "task-1", SessionID: "session-1", + RuntimeName: agentruntime.RuntimeKubernetes, + }) + provider := lifecycleMetricProvider{manager: metricExecutionListStub{ + executions: []*lifecycle.AgentExecution{execution}, + }} + + sources := provider.MetricExecutions() + if len(sources) != 1 { + t.Fatalf("execution sources = %d, want 1", len(sources)) + } + if sources[0].ExecutorType != string(agentruntime.RuntimeKubernetes) { + t.Fatalf("executor type = %q, want k8s", sources[0].ExecutorType) + } +} diff --git a/apps/backend/internal/backendapp/workspace_source_materializer.go b/apps/backend/internal/backendapp/workspace_source_materializer.go index 88b054d2cd..22741bfcea 100644 --- a/apps/backend/internal/backendapp/workspace_source_materializer.go +++ b/apps/backend/internal/backendapp/workspace_source_materializer.go @@ -118,6 +118,13 @@ func (m *workspaceSourceMaterializer) MaterializeWorkspaceSources(ctx context.Co return &taskservice.WorkspaceSourceMaterializationResult{}, nil } if !isHostWorkspaceExecutor(state.environment.ExecutorType) { + if !models.IsRemoteExecutorType(models.ExecutorType(state.environment.ExecutorType)) { + return nil, fmt.Errorf( + "%w: executor %q cannot materialize workspace sources", + taskservice.ErrUnsupportedWorkspaceSource, + state.environment.ExecutorType, + ) + } return m.materializeRemoteWorkspaceSources(ctx, taskID, state, batch) } return m.materializeHostWorkspaceSources(ctx, taskID, state, batch) diff --git a/apps/backend/internal/backendapp/workspace_source_materializer_kubernetes_test.go b/apps/backend/internal/backendapp/workspace_source_materializer_kubernetes_test.go new file mode 100644 index 0000000000..fc3fbd0328 --- /dev/null +++ b/apps/backend/internal/backendapp/workspace_source_materializer_kubernetes_test.go @@ -0,0 +1,76 @@ +package backendapp + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/kandev/kandev/internal/task/models" + taskservice "github.com/kandev/kandev/internal/task/service" +) + +func TestWorkspaceSourceMaterializerKubernetesUsesRemoteMaterialization(t *testing.T) { + ctx := context.Background() + repo := newMaterializerRepo(t) + seedWorkspaceSourceTask(t, repo, t.TempDir()) + seedRemoteMaterializerRepositories(t, repo) + env, err := repo.GetTaskEnvironmentByTaskID(ctx, "task-1") + if err != nil { + t.Fatal(err) + } + env.ExecutorType = string(models.ExecutorTypeKubernetes) + if err := repo.UpdateTaskEnvironment(ctx, env); err != nil { + t.Fatal(err) + } + remote := &remoteWorkspaceMaterializerStub{ids: []string{"session-1"}} + materializer := &workspaceSourceMaterializer{ + repo: repo, worktreeMgr: newMaterializerWorktreeMgr(t, filepath.Join(t.TempDir(), "task-1")), + remoteMaterializer: remote, logger: newTestLogger(), + } + + result, err := materializer.MaterializeWorkspaceSources(ctx, "task-1", &models.WorkspaceSourceBatch{ + TaskID: "task-1", + Sources: []models.WorkspaceSource{{Repository: &models.TaskRepository{ + RepositoryID: "repo-added", BaseBranch: "main", + }}}, + }) + if err != nil { + t.Fatal(err) + } + if len(result.SessionIDs) != 1 || result.SessionIDs[0] != "session-1" { + t.Fatalf("materialization result = %#v", result) + } + if len(remote.calls) != 1 || len(remote.calls[0]) != 1 || remote.calls[0][0].Destination != "added-main" { + t.Fatalf("remote projection = %+v, want added-main", remote.calls) + } +} + +func TestWorkspaceSourceMaterializerRejectsUnknownExecutor(t *testing.T) { + ctx := context.Background() + repo := newMaterializerRepo(t) + seedWorkspaceSourceTask(t, repo, t.TempDir()) + env, err := repo.GetTaskEnvironmentByTaskID(ctx, "task-1") + if err != nil { + t.Fatal(err) + } + env.ExecutorType = "future_executor" + if err := repo.UpdateTaskEnvironment(ctx, env); err != nil { + t.Fatal(err) + } + remote := &remoteWorkspaceMaterializerStub{} + materializer := &workspaceSourceMaterializer{ + repo: repo, worktreeMgr: newMaterializerWorktreeMgr(t, filepath.Join(t.TempDir(), "task-1")), + remoteMaterializer: remote, logger: newTestLogger(), + } + + _, err = materializer.MaterializeWorkspaceSources( + ctx, "task-1", &models.WorkspaceSourceBatch{TaskID: "task-1"}, + ) + if !errors.Is(err, taskservice.ErrUnsupportedWorkspaceSource) { + t.Fatalf("error = %v, want unsupported workspace source", err) + } + if len(remote.calls) != 0 { + t.Fatalf("remote materializer called for unknown executor: %+v", remote.calls) + } +} diff --git a/apps/backend/internal/editors/capabilities/capabilities.go b/apps/backend/internal/editors/capabilities/capabilities.go index 4defbf8f74..2bd3ba150d 100644 --- a/apps/backend/internal/editors/capabilities/capabilities.go +++ b/apps/backend/internal/editors/capabilities/capabilities.go @@ -16,7 +16,7 @@ func SupportsEmbeddedVscode(executorType models.ExecutorType, hostOS string) boo case models.ExecutorTypeLocal, models.ExecutorTypeWorktree: return hostOS == hostOSLinux || hostOS == hostOSDarwin case models.ExecutorTypeLocalDocker, models.ExecutorTypeRemoteDocker, - models.ExecutorTypeSprites, models.ExecutorTypeSSH: + models.ExecutorTypeSprites, models.ExecutorTypeSSH, models.ExecutorTypeKubernetes: return true default: return false diff --git a/apps/backend/internal/editors/capabilities/capabilities_test.go b/apps/backend/internal/editors/capabilities/capabilities_test.go index d6928bf168..a085890f57 100644 --- a/apps/backend/internal/editors/capabilities/capabilities_test.go +++ b/apps/backend/internal/editors/capabilities/capabilities_test.go @@ -25,6 +25,7 @@ func TestSupportsEmbeddedVscode(t *testing.T) { {name: "remote Docker", executorType: models.ExecutorTypeRemoteDocker, hostOS: "windows", want: true}, {name: "Sprites", executorType: models.ExecutorTypeSprites, hostOS: "windows", want: true}, {name: "SSH", executorType: models.ExecutorTypeSSH, hostOS: "windows", want: true}, + {name: "Kubernetes", executorType: models.ExecutorTypeKubernetes, hostOS: "windows", want: true}, {name: "mock remote fails closed", executorType: models.ExecutorTypeMockRemote, hostOS: "linux", want: false}, {name: "unknown executor fails closed", executorType: models.ExecutorType("future"), hostOS: "linux", want: false}, } diff --git a/apps/backend/internal/gateway/websocket/lsp_handler.go b/apps/backend/internal/gateway/websocket/lsp_handler.go index 92e5e9742a..8219188407 100644 --- a/apps/backend/internal/gateway/websocket/lsp_handler.go +++ b/apps/backend/internal/gateway/websocket/lsp_handler.go @@ -127,14 +127,16 @@ func (h *LSPHandler) HandleLSPConnection(c *gin.Context) { return } defer h.capacity.Release() - agentctlClient := execution.GetAgentCtlClient() + agentctlClient, releaseClient := execution.AcquireAgentCtlClient() if agentctlClient == nil { + releaseClient() h.logger.Warn("LSP: execution has no agentctl client", zap.String("session_id", sessionID)) h.closeWithCode(c, lspCloseSessionNotFound, "agentctl unavailable") return } browserConn, upgradeErr := lspUpgrader.Upgrade(c.Writer, c.Request, nil) if upgradeErr != nil { + releaseClient() h.logger.Error("LSP: failed to upgrade to WebSocket", zap.String("session_id", sessionID), zap.Error(upgradeErr)) @@ -144,6 +146,7 @@ func (h *LSPHandler) HandleLSPConnection(c *gin.Context) { autoInstall := h.shouldAutoInstall(c.Request.Context(), language) upstreamConn, resp, dialErr := agentctlClient.DialLSP(c.Request.Context(), language, autoInstall) + releaseClient() if dialErr != nil { status := 0 if resp != nil { diff --git a/apps/backend/internal/gateway/websocket/port_proxy.go b/apps/backend/internal/gateway/websocket/port_proxy.go index 82410daacb..69811f78ab 100644 --- a/apps/backend/internal/gateway/websocket/port_proxy.go +++ b/apps/backend/internal/gateway/websocket/port_proxy.go @@ -236,7 +236,8 @@ func (h *PortProxyHandler) resolveProxy(c *gin.Context, sessionID string, port i return nil, fmt.Errorf("session not found") } - agentctlClient := execution.GetAgentCtlClient() + agentctlClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentctlClient == nil { c.JSON(http.StatusServiceUnavailable, gin.H{"error": "agentctl client not available"}) return nil, fmt.Errorf("agentctl client not available") diff --git a/apps/backend/internal/gateway/websocket/port_tunnel.go b/apps/backend/internal/gateway/websocket/port_tunnel.go index 1d9236b993..fdf8678bf9 100644 --- a/apps/backend/internal/gateway/websocket/port_tunnel.go +++ b/apps/backend/internal/gateway/websocket/port_tunnel.go @@ -200,7 +200,8 @@ func (m *TunnelManager) resolveAndBind(sessionID string, tunnelPort int) (*url.U return nil, "", nil, fmt.Errorf("session not found or no active execution") } - agentctlClient := execution.GetAgentCtlClient() + agentctlClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentctlClient == nil { return nil, "", nil, fmt.Errorf("agentctl client not available") } diff --git a/apps/backend/internal/gateway/websocket/terminal_handler.go b/apps/backend/internal/gateway/websocket/terminal_handler.go index 262950d7f5..718d52265e 100644 --- a/apps/backend/internal/gateway/websocket/terminal_handler.go +++ b/apps/backend/internal/gateway/websocket/terminal_handler.go @@ -179,14 +179,16 @@ func (h *TerminalHandler) handleRemoteUserShellWS( terminalID string, ) { sessionID := execution.SessionID - if execution.GetAgentCtlClient() == nil { + client, releaseClient := execution.AcquireAgentCtlClient() + if client == nil { + releaseClient() c.JSON(http.StatusNotFound, gin.H{"error": "remote execution not available"}) return } - client := execution.GetAgentCtlClient() _, initialCommand, httpErr := h.resolveShellLabel(c) if httpErr != "" { + releaseClient() c.JSON(http.StatusBadRequest, gin.H{"error": httpErr}) return } @@ -200,6 +202,7 @@ func (h *TerminalHandler) handleRemoteUserShellWS( // Create a per-terminal shell on agentctl (idempotent if already exists) if err := client.StartShellTerminal(c.Request.Context(), terminalID, 80, 24); err != nil { + releaseClient() h.logger.Error("failed to start remote terminal shell", zap.String("session_id", sessionID), zap.String("terminal_id", terminalID), @@ -210,6 +213,7 @@ func (h *TerminalHandler) handleRemoteUserShellWS( // Open binary WS to the per-terminal shell on agentctl agentctlConn, err := client.StreamShellTerminal(c.Request.Context(), terminalID) + releaseClient() if err != nil { h.logger.Error("failed to connect to remote terminal shell stream", zap.String("session_id", sessionID), diff --git a/apps/backend/internal/gateway/websocket/terminal_sessions.go b/apps/backend/internal/gateway/websocket/terminal_sessions.go index 33df6b797c..bd70ecdd91 100644 --- a/apps/backend/internal/gateway/websocket/terminal_sessions.go +++ b/apps/backend/internal/gateway/websocket/terminal_sessions.go @@ -67,7 +67,12 @@ func (h *TerminalHandler) remoteExecutionWithClientReady( deadline time.Time, ) (*lifecycle.AgentExecution, bool) { execution, exists := h.lifecycleMgr.GetExecutionBySessionID(sessionID) - if !exists || execution.GetAgentCtlClient() == nil { + if !exists { + return nil, false + } + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { return nil, false } remaining := time.Until(deadline) @@ -81,7 +86,7 @@ func (h *TerminalHandler) remoteExecutionWithClientReady( } probeCtx, cancel := context.WithTimeout(ctx, probeTimeout) defer cancel() - if err := execution.GetAgentCtlClient().Health(probeCtx); err != nil { + if err := client.Health(probeCtx); err != nil { h.logger.Debug("remote execution agentctl client not ready", zap.String("session_id", sessionID), zap.String("execution_id", execution.ID), diff --git a/apps/backend/internal/gateway/websocket/vscode_proxy.go b/apps/backend/internal/gateway/websocket/vscode_proxy.go index 7ad1f01321..40d0dc2bea 100644 --- a/apps/backend/internal/gateway/websocket/vscode_proxy.go +++ b/apps/backend/internal/gateway/websocket/vscode_proxy.go @@ -126,7 +126,8 @@ func (h *VscodeProxyHandler) resolveProxy(c *gin.Context, sessionID string) (*ht return nil, fmt.Errorf("session not found") } - agentctlClient := execution.GetAgentCtlClient() + agentctlClient, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() if agentctlClient == nil { c.JSON(http.StatusServiceUnavailable, gin.H{"error": "agentctl client not available"}) return nil, fmt.Errorf("agentctl client not available") diff --git a/apps/backend/internal/kubernetes/admission_test.go b/apps/backend/internal/kubernetes/admission_test.go new file mode 100644 index 0000000000..8ce9507e41 --- /dev/null +++ b/apps/backend/internal/kubernetes/admission_test.go @@ -0,0 +1,331 @@ +package kubernetes + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestRunPodDryRunRejectsAdmissionMutationOfOwnedInvariants(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.Pod) + }{ + {name: "restart policy", mutate: func(pod *corev1.Pod) { + pod.Spec.RestartPolicy = corev1.RestartPolicyNever + }}, + {name: "operating system", mutate: func(pod *corev1.Pod) { + pod.Spec.OS = &corev1.PodOS{Name: corev1.Windows} + }}, + {name: "operating system selector", mutate: func(pod *corev1.Pod) { + pod.Spec.NodeSelector[corev1.LabelOSStable] = "windows" + }}, + {name: "architecture selector", mutate: func(pod *corev1.Pod) { + pod.Spec.NodeSelector[corev1.LabelArchStable] = "arm64" + }}, + {name: "node name", mutate: func(pod *corev1.Pod) { + pod.Spec.NodeName = "kind-worker" + }}, + {name: "required operating system affinity", mutate: func(pod *corev1.Pod) { + pod.Spec.Affinity = reservedNodeAffinity(corev1.LabelOSStable, false) + }}, + {name: "preferred architecture affinity", mutate: func(pod *corev1.Pod) { + pod.Spec.Affinity = reservedNodeAffinity(corev1.LabelArchStable, true) + }}, + {name: "service account token automount", mutate: func(pod *corev1.Pod) { + value := true + pod.Spec.AutomountServiceAccountToken = &value + }}, + {name: "main command", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Command = []string{"false"} + }}, + {name: "main arguments", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Args = []string{"changed"} + }}, + {name: "main working directory", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].WorkingDir = "/tmp" + }}, + {name: "agentctl port", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Ports[0].ContainerPort++ + }}, + {name: "reserved mount", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].VolumeMounts[0].MountPath = "/tmp/runtime" + }}, + {name: "reserved volume", mutate: func(pod *corev1.Pod) { + pod.Spec.Volumes[0].EmptyDir = nil + pod.Spec.Volumes[0].ConfigMap = &corev1.ConfigMapVolumeSource{} + }}, + {name: "AGENTCTL prefix environment", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{Name: "AGENTCTL_FUTURE"}) + }}, + {name: "KANDEV prefix environment", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{Name: "KANDEV_FUTURE"}) + }}, + {name: "HOME environment", mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{Name: "HOME"}) + }}, + {name: "owner reference", mutate: func(pod *corev1.Pod) { + pod.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "ConfigMap", Name: "foreign"}} + }}, + {name: "finalizer", mutate: func(pod *corev1.Pod) { + pod.Finalizers = []string{"example.com/hold"} + }}, + {name: "extra Kandev label", mutate: func(pod *corev1.Pod) { + pod.Labels["kandev.ai/foreign"] = "mutated" + }}, + {name: "extra Kandev annotation", mutate: func(pod *corev1.Pod) { + pod.Annotations = map[string]string{"kandev.ai/foreign": "mutated"} + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + step := runMutatedPodDryRun(t, test.mutate) + + require.False(t, step.Success) + require.Equal(t, "Dry-run admission failed", step.Detail) + require.Contains(t, step.Error, "admitted Pod") + }) + } +} + +func TestRunPodDryRunAllowsUnrelatedAdmissionAdditions(t *testing.T) { + step := runMutatedPodDryRun(t, func(pod *corev1.Pod) { + pod.Labels["example.com/injected"] = "true" + pod.Annotations = map[string]string{"example.com/injected": "true"} + pod.Spec.NodeSelector["example.com/pool"] = "agents" + pod.Spec.Affinity = reservedNodeAffinity("example.com/pool", false) + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "injected", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }) + pod.Spec.Containers = append(pod.Spec.Containers, corev1.Container{ + Name: "injected", Image: "example.test/sidecar:latest", + Env: []corev1.EnvVar{{Name: "EXAMPLE", Value: "true"}}, + }) + }) + + require.True(t, step.Success, step.Error) +} + +func TestRunPodDryRunRejectsMissingServerUID(t *testing.T) { + step := runMutatedPodDryRun(t, func(pod *corev1.Pod) { + pod.UID = "" + }) + + require.False(t, step.Success) + require.Contains(t, step.Error, "UID") +} + +func TestRunPVCDryRunRejectsAdmissionMutationOfStorageSemantics(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + }{ + {name: "explicit storage class", mutate: func(claim *corev1.PersistentVolumeClaim) { + value := "slow" + claim.Spec.StorageClassName = &value + }}, + {name: "access modes", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + }}, + {name: "volume mode", mutate: func(claim *corev1.PersistentVolumeClaim) { + value := corev1.PersistentVolumeBlock + claim.Spec.VolumeMode = &value + }}, + {name: "storage request", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.Resources.Requests[corev1.ResourceStorage] = resource.MustParse("20Gi") + }}, + {name: "selector", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"foreign": "true"}} + }}, + {name: "data source", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.DataSource = &corev1.TypedLocalObjectReference{Kind: "PersistentVolumeClaim", Name: "foreign"} + }}, + {name: "data source ref", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.DataSourceRef = &corev1.TypedObjectReference{Kind: "PersistentVolumeClaim", Name: "foreign"} + }}, + {name: "volume name", mutate: func(claim *corev1.PersistentVolumeClaim) { + claim.Spec.VolumeName = "foreign-volume" + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + step := runMutatedPVCDryRun(t, "fast", test.mutate) + + require.False(t, step.Success) + require.Equal(t, "Dry-run admission failed", step.Detail) + require.Contains(t, step.Error, "admitted PVC") + }) + } +} + +func TestRunPVCDryRunAllowsStorageClassDefaultingWhenNotRequested(t *testing.T) { + step := runMutatedPVCDryRun(t, "", func(claim *corev1.PersistentVolumeClaim) { + value := "cluster-default" + claim.Spec.StorageClassName = &value + claim.Finalizers = []string{"kubernetes.io/pvc-protection"} + claim.Annotations = map[string]string{"volume.kubernetes.io/storage-provisioner": "example.test"} + }) + + require.True(t, step.Success, step.Error) +} + +func TestRunPVCDryRunRejectsMissingServerUID(t *testing.T) { + step := runMutatedPVCDryRun(t, "fast", func(claim *corev1.PersistentVolumeClaim) { + claim.UID = "" + }) + + require.False(t, step.Success) + require.Contains(t, step.Error, "UID") +} + +func TestLiveStreamingProbeRejectsAdmissionMutationAndCleansExactPod(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.Pod) + }{ + {name: "operating system", mutate: func(pod *corev1.Pod) { + pod.Spec.OS = &corev1.PodOS{Name: corev1.Windows} + }}, + {name: "node name", mutate: func(pod *corev1.Pod) { + pod.Spec.NodeName = "kind-worker" + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + state := installStreamingPodReactors(t, clientset) + state.mutateLivePod = test.mutate + profile, err := agentkubernetes.ParseProfileConfig(validHandlerProfileConfig()) + require.NoError(t, err) + probeCalls := 0 + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + probeCalls++ + return nil + }} + + err = handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", profile) + + require.ErrorContains(t, err, "admitted Pod") + require.Zero(t, probeCalls) + requireStreamingPodCleaned(t, state) + }) + } +} + +func TestLiveStreamingProbeCleansReturnedIdentityAfterAdmissionMismatch(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + state := installStreamingPodReactors(t, clientset) + state.mutateLivePod = func(pod *corev1.Pod) { + pod.Name = "admitted-probe-name" + pod.Namespace = "admitted-probe-namespace" + } + profile, err := agentkubernetes.ParseProfileConfig(validHandlerProfileConfig()) + require.NoError(t, err) + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + return nil + }} + + err = handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", profile) + + require.ErrorContains(t, err, "name or namespace") + require.Equal(t, state.actualPod.Namespace, state.deletedNamespace) + require.Equal(t, state.actualPod.Name, state.deletedName) + requireStreamingPodCleaned(t, state) +} + +func runMutatedPodDryRun(t *testing.T, mutate func(*corev1.Pod)) TestStep { + t.Helper() + clientset := kubernetesfake.NewSimpleClientset() + clientset.PrependReactor("create", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + pod := action.(k8stesting.CreateAction).GetObject().(*corev1.Pod).DeepCopy() + pod.UID = "dry-run-pod-uid" + mutate(pod) + return true, pod, nil + }) + profile, err := agentkubernetes.ParseProfileConfig(validHandlerProfileConfig()) + require.NoError(t, err) + result := &TestResult{} + + runPodDryRun( + context.Background(), clientset, "kandev", "admission-probe", + admissionTestIdentity(), profile, result, + ) + + require.Len(t, result.Steps, 1) + return result.Steps[0] +} + +func runMutatedPVCDryRun( + t *testing.T, + storageClass string, + mutate func(*corev1.PersistentVolumeClaim), +) TestStep { + t.Helper() + clientset := kubernetesfake.NewSimpleClientset() + clientset.PrependReactor("create", "persistentvolumeclaims", func(action k8stesting.Action) (bool, runtime.Object, error) { + claim := action.(k8stesting.CreateAction).GetObject().(*corev1.PersistentVolumeClaim).DeepCopy() + claim.UID = "dry-run-pvc-uid" + mutate(claim) + return true, claim, nil + }) + values := validHandlerProfileConfig() + values["workspace.mode"] = "managed_pvc" + values["workspace.size"] = "10Gi" + values["workspace.storage_class"] = storageClass + profile, err := agentkubernetes.ParseProfileConfig(values) + require.NoError(t, err) + result := &TestResult{} + + runPVCDryRun( + context.Background(), clientset, "kandev", "admission-probe", + admissionTestIdentity(), profile, result, + ) + + require.Len(t, result.Steps, 1) + return result.Steps[0] +} + +func admissionTestIdentity() agentkubernetes.ResourceIdentity { + return agentkubernetes.ResourceIdentity{ + ExecutorID: connectionTestIdentity, ProfileID: connectionTestIdentity, + InstanceID: connectionTestIdentity, TaskID: connectionTestIdentity, + SessionID: connectionTestIdentity, EnvironmentID: connectionTestIdentity, + } +} + +func reservedNodeAffinity(key string, preferred bool) *corev1.Affinity { + requirement := corev1.NodeSelectorRequirement{ + Key: key, Operator: corev1.NodeSelectorOpIn, Values: []string{"foreign"}, + } + term := corev1.NodeSelectorTerm{MatchExpressions: []corev1.NodeSelectorRequirement{requirement}} + nodeAffinity := &corev1.NodeAffinity{} + if preferred { + nodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution = []corev1.PreferredSchedulingTerm{{ + Weight: 1, Preference: term, + }} + } else { + nodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{term}, + } + } + return &corev1.Affinity{NodeAffinity: nodeAffinity} +} diff --git a/apps/backend/internal/kubernetes/handlers.go b/apps/backend/internal/kubernetes/handlers.go new file mode 100644 index 0000000000..67bd01b062 --- /dev/null +++ b/apps/backend/internal/kubernetes/handlers.go @@ -0,0 +1,635 @@ +// Package kubernetes exposes Kubernetes executor diagnostics and active-session status. +package kubernetes + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" + ws "github.com/kandev/kandev/pkg/websocket" +) + +var ( + errAuthenticationRequired = errors.New("authentication required") + errAdminRequired = errors.New("administrator identity required") + errExecutorIDRequired = errors.New("executor id required") + errTaskIDRequired = errors.New("task id is required when session id is provided") + errExecutorNotKubernetes = errors.New("executor is not a Kubernetes executor") + errSessionStatusNotWired = errors.New("kubernetes session status is unavailable") + errStreamingProbeCleanup = errors.New("streaming probe cleanup failed") +) + +const ( + errorJSONKey = "error" + verbGet = "get" + verbCreate = "create" + resourcePods = "pods" + resourcePersistentClaims = "persistentvolumeclaims" + connectionTestIdentity = "connection-test" + metadataNamespace = "kubernetes_namespace" + metadataPodName = "kubernetes_pod_name" + metadataPodUID = "kubernetes_pod_uid" + metadataMainContainer = "kubernetes_main_container" + metadataWorkspaceMode = "kubernetes_workspace_mode" + metadataResourceExecutor = agentkubernetes.MetadataKeyResourceExecutorID + metadataResourceProfile = agentkubernetes.MetadataKeyResourceProfileID + metadataResourceInstance = agentkubernetes.MetadataKeyResourceInstanceID + metadataResourceTask = agentkubernetes.MetadataKeyResourceTaskID + metadataResourceSession = agentkubernetes.MetadataKeyResourceSessionID + metadataResourceEnv = agentkubernetes.MetadataKeyResourceEnvironmentID +) + +type ResourceRepository interface { + ListExecutorsRunning(ctx context.Context) ([]*models.ExecutorRunning, error) + GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error) + GetExecutor(ctx context.Context, id string) (*models.Executor, error) +} + +type AccessChecker interface { + AuthorizeTaskAccess(ctx context.Context, taskID string) error +} + +type ClientFactory func(config agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) + +type StreamingProbe func( + ctx context.Context, + client *agentkubernetes.Client, + namespace string, + podName string, + mainContainer string, +) error + +type Handler struct { + repo ResourceRepository + access AccessChecker + clients ClientFactory + probeStreaming StreamingProbe +} + +type TestRequest struct { + Config map[string]string `json:"config"` + ProfileConfig map[string]string `json:"profile_config,omitempty"` +} + +type TestStep struct { + Key string `json:"key"` + Success bool `json:"success"` + DurationMS int64 `json:"duration_ms"` + Detail string `json:"detail"` + Error string `json:"error,omitempty"` +} + +type TestResult struct { + Success bool `json:"success"` + ServerVersion string `json:"server_version,omitempty"` + Namespace string `json:"namespace,omitempty"` + Steps []TestStep `json:"steps"` + Warnings []agentkubernetes.Warning `json:"warnings"` + Error string `json:"error,omitempty"` +} + +type requiredPermission struct { + verb string + resource string + subresource string +} + +func NewHandler(repo ResourceRepository, access AccessChecker, clients ClientFactory) *Handler { + if clients == nil { + clients = func(config agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return agentkubernetes.NewClient( + config, + agentkubernetes.DefaultConfigLoader(), + nil, + ) + } + } + return &Handler{ + repo: repo, access: access, clients: clients, + probeStreaming: probeStreamingTransport, + } +} + +// RegisterRoutes wires Kubernetes executor diagnostics. +func RegisterRoutes( + router *gin.Engine, + dispatcher *ws.Dispatcher, + repo ResourceRepository, + access AccessChecker, + _ *logger.Logger, +) { + handler := NewHandler(repo, access, nil) + handler.registerHTTP(router) + handler.registerWS(dispatcher) +} + +func (h *Handler) registerHTTP(router *gin.Engine) { + api := router.Group("/api/v1/kubernetes") + api.POST("/test", h.httpTest) + api.GET("/executors/:id/sessions", h.httpListSessions) + api.GET("/executors/:id/session-impact", h.httpSessionImpact) +} + +func (h *Handler) registerWS(dispatcher *ws.Dispatcher) { + dispatcher.RegisterFunc("kubernetes.test", h.wsTest) + dispatcher.RegisterFunc("kubernetes.sessions.list", h.wsListSessions) + dispatcher.RegisterFunc("kubernetes.sessions.impact", h.wsSessionImpact) +} + +func (h *Handler) httpTest(c *gin.Context) { + if err := requireAdmin(c.Request.Context()); err != nil { + writeAdminError(c, err) + return + } + var request TestRequest + if err := c.ShouldBindJSON(&request); err != nil { + c.JSON(http.StatusBadRequest, gin.H{errorJSONKey: "invalid request body"}) + return + } + c.JSON(http.StatusOK, h.testConnection(c.Request.Context(), request)) +} + +func (h *Handler) wsTest(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + if err := requireAdmin(ctx); err != nil { + return adminWSError(msg, err) + } + var request TestRequest + if err := msg.ParsePayload(&request); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "invalid payload", nil) + } + return ws.NewResponse(msg.ID, msg.Action, h.testConnection(ctx, request)) +} + +func requireAdmin(ctx context.Context) error { + identity, ok := authn.IdentityFromContext(ctx) + if !ok { + return errAuthenticationRequired + } + if !identity.IsAdmin() { + return errAdminRequired + } + return nil +} + +func writeAdminError(c *gin.Context, err error) { + if errors.Is(err, errAuthenticationRequired) { + c.JSON(http.StatusUnauthorized, gin.H{errorJSONKey: err.Error()}) + return + } + c.JSON(http.StatusForbidden, gin.H{errorJSONKey: err.Error()}) +} + +func adminWSError(msg *ws.Message, err error) (*ws.Message, error) { + code := ws.ErrorCodeForbidden + if errors.Is(err, errAuthenticationRequired) { + code = ws.ErrorCodeUnauthorized + } + return ws.NewError(msg.ID, msg.Action, code, err.Error(), nil) +} + +func (h *Handler) httpListSessions(c *gin.Context) { + executorID := strings.TrimSpace(c.Param("id")) + filter := SessionFilter{ + TaskID: strings.TrimSpace(c.Query("task_id")), + SessionID: strings.TrimSpace(c.Query("session_id")), + } + rows, err := h.listSessions(c.Request.Context(), executorID, filter) + if err != nil { + status, message := sessionHTTPError(err) + c.JSON(status, gin.H{errorJSONKey: message}) + return + } + c.JSON(http.StatusOK, rows) +} + +func (h *Handler) wsListSessions(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + var payload struct { + ExecutorID string `json:"executor_id"` + TaskID string `json:"task_id,omitempty"` + SessionID string `json:"session_id,omitempty"` + } + if err := msg.ParsePayload(&payload); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "invalid payload", nil) + } + rows, err := h.listSessions(ctx, strings.TrimSpace(payload.ExecutorID), SessionFilter{ + TaskID: strings.TrimSpace(payload.TaskID), + SessionID: strings.TrimSpace(payload.SessionID), + }) + if err != nil { + code, message := sessionWSError(err) + return ws.NewError(msg.ID, msg.Action, code, message, nil) + } + return ws.NewResponse(msg.ID, msg.Action, rows) +} + +func (h *Handler) httpSessionImpact(c *gin.Context) { + if err := requireAdmin(c.Request.Context()); err != nil { + writeAdminError(c, err) + return + } + impact, err := h.sessionImpact(c.Request.Context(), strings.TrimSpace(c.Param("id"))) + if err != nil { + status, message := sessionHTTPError(err) + c.JSON(status, gin.H{errorJSONKey: message}) + return + } + c.JSON(http.StatusOK, impact) +} + +func (h *Handler) wsSessionImpact(ctx context.Context, msg *ws.Message) (*ws.Message, error) { + if err := requireAdmin(ctx); err != nil { + return adminWSError(msg, err) + } + var payload struct { + ExecutorID string `json:"executor_id"` + } + if err := msg.ParsePayload(&payload); err != nil { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "invalid payload", nil) + } + impact, err := h.sessionImpact(ctx, strings.TrimSpace(payload.ExecutorID)) + if err != nil { + code, message := sessionWSError(err) + return ws.NewError(msg.ID, msg.Action, code, message, nil) + } + return ws.NewResponse(msg.ID, msg.Action, impact) +} + +func sessionHTTPError(err error) (int, string) { + switch { + case errors.Is(err, errExecutorIDRequired), errors.Is(err, errTaskIDRequired), + errors.Is(err, errExecutorNotKubernetes): + return http.StatusBadRequest, err.Error() + case errors.Is(err, models.ErrExecutorNotFound): + return http.StatusNotFound, "Kubernetes executor not found" + default: + return http.StatusInternalServerError, "Kubernetes session status is unavailable" + } +} + +func sessionWSError(err error) (string, string) { + switch { + case errors.Is(err, errExecutorIDRequired), errors.Is(err, errTaskIDRequired), + errors.Is(err, errExecutorNotKubernetes): + return ws.ErrorCodeBadRequest, err.Error() + case errors.Is(err, models.ErrExecutorNotFound): + return ws.ErrorCodeNotFound, "Kubernetes executor not found" + default: + return ws.ErrorCodeInternalError, "Kubernetes session status is unavailable" + } +} + +func (h *Handler) testConnection(ctx context.Context, request TestRequest) *TestResult { + result := &TestResult{Steps: make([]TestStep, 0, 6), Warnings: []agentkubernetes.Warning{}} + config, err := agentkubernetes.ParseExecutorConfig(request.Config) + if err != nil { + result.addFailedStep("configuration", time.Now(), "Executor configuration is invalid", err) + return result + } + profile, err := parseOptionalProfile(request.ProfileConfig) + if err != nil { + result.addFailedStep("configuration", time.Now(), "Profile configuration is invalid", err) + return result + } + result.Namespace = config.Namespace + client, err := h.clients(config) + if err != nil || client == nil || client.Clientset == nil { + if err == nil { + err = errors.New("kubernetes client is unavailable") + } + result.addFailedStep("discovery", time.Now(), "Kubernetes API discovery failed", err) + return result + } + testCtx, cancel := context.WithTimeout(ctx, time.Duration(config.RequestTimeoutSeconds)*time.Second) + defer cancel() + if !runDiscoveryStep(testCtx, client.Clientset, result) { + return result + } + runNamespaceStep(config.Namespace, result) + basePermissions := []requiredPermission{ + {verb: verbGet, resource: resourcePods}, + {verb: verbCreate, resource: resourcePods}, + {verb: "delete", resource: resourcePods}, + {verb: "watch", resource: resourcePods}, + } + basePermissions = append(basePermissions, workspacePermissions(profile)...) + rbacStep := runPermissionStep( + testCtx, client.Clientset, config.Namespace, "rbac", basePermissions, + ) + result.Steps = append(result.Steps, rbacStep) + admissionSuccess := true + if profile != nil { + admissionSuccess = false + if rbacStep.Success { + admissionSuccess = runAdmissionSteps( + testCtx, client.Clientset, config.Namespace, *profile, result, + ) + } + } + streamingPermissions := []requiredPermission{ + {verb: verbGet, resource: resourcePods, subresource: "exec"}, + {verb: verbCreate, resource: resourcePods, subresource: "exec"}, + {verb: verbGet, resource: resourcePods, subresource: "portforward"}, + {verb: verbCreate, resource: resourcePods, subresource: "portforward"}, + } + streamingStep := h.runStreamingStep( + testCtx, client, config.Namespace, streamingPermissions, + profile, rbacStep.Success && admissionSuccess, + ) + result.Steps = append(result.Steps, streamingStep) + result.finalize() + return result +} + +func parseOptionalProfile(values map[string]string) (*agentkubernetes.ProfileConfig, error) { + if len(values) == 0 { + return nil, nil + } + profile, err := agentkubernetes.ParseProfileConfig(values) + if err != nil { + return nil, err + } + return &profile, nil +} + +func workspacePermissions(profile *agentkubernetes.ProfileConfig) []requiredPermission { + if profile == nil { + return nil + } + switch profile.Workspace.Mode { + case agentkubernetes.WorkspaceModeManagedPVC: + return []requiredPermission{ + {verb: verbGet, resource: resourcePersistentClaims}, + {verb: verbCreate, resource: resourcePersistentClaims}, + {verb: "delete", resource: resourcePersistentClaims}, + } + case agentkubernetes.WorkspaceModeExistingClaim: + return []requiredPermission{{verb: verbGet, resource: resourcePersistentClaims}} + default: + return nil + } +} + +func runAdmissionSteps( + ctx context.Context, + client kubeclient.Interface, + namespace string, + profile agentkubernetes.ProfileConfig, + result *TestResult, +) bool { + firstStep := len(result.Steps) + resourceName := "kandev-admission-probe-" + uuid.NewString()[:8] + identity := agentkubernetes.ResourceIdentity{ + ExecutorID: connectionTestIdentity, ProfileID: connectionTestIdentity, InstanceID: connectionTestIdentity, + TaskID: connectionTestIdentity, SessionID: connectionTestIdentity, EnvironmentID: connectionTestIdentity, + } + switch profile.Workspace.Mode { + case agentkubernetes.WorkspaceModeManagedPVC: + runPVCDryRun(ctx, client, namespace, resourceName, identity, profile, result) + case agentkubernetes.WorkspaceModeExistingClaim: + runExistingClaimCheck(ctx, client, namespace, profile.Workspace.ClaimName, result) + } + runPodDryRun(ctx, client, namespace, resourceName, identity, profile, result) + for _, step := range result.Steps[firstStep:] { + if !step.Success { + return false + } + } + return true +} + +func runExistingClaimCheck( + ctx context.Context, + client kubeclient.Interface, + namespace string, + claimName string, + result *TestResult, +) { + started := time.Now() + claim, err := client.CoreV1().PersistentVolumeClaims(namespace).Get( + ctx, claimName, metav1.GetOptions{}, + ) + if err == nil && (claim == nil || claim.Namespace != namespace || claim.Name != claimName || claim.UID == "") { + err = errors.New("existing workspace claim identity is incomplete or mismatched") + } + if err != nil { + result.Steps = append(result.Steps, failedStep( + "storage.existing_claim", started, "Existing workspace claim lookup failed", err, + )) + return + } + result.Steps = append(result.Steps, TestStep{ + Key: "storage.existing_claim", Success: true, + DurationMS: time.Since(started).Milliseconds(), + Detail: "Existing workspace claim is available", + }) +} + +func runPVCDryRun( + ctx context.Context, + client kubeclient.Interface, + namespace string, + name string, + identity agentkubernetes.ResourceIdentity, + profile agentkubernetes.ProfileConfig, + result *TestResult, +) { + started := time.Now() + claim, err := agentkubernetes.BuildPersistentVolumeClaim(profile.Workspace, agentkubernetes.PVCOptions{ + Name: name, Namespace: namespace, Identity: identity, + }) + if err == nil { + var admitted *corev1.PersistentVolumeClaim + admitted, err = client.CoreV1().PersistentVolumeClaims(namespace).Create( + ctx, claim, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}}, + ) + if err == nil { + err = agentkubernetes.ValidateAdmittedPVC(admitted, claim) + } + if err == nil && admitted.UID == "" { + err = errors.New("admitted PVC dry-run UID is unavailable") + } + if err == nil && admitted.Spec.VolumeName != claim.Spec.VolumeName { + err = errors.New("admitted PVC dry-run changed the volume name") + } + } + appendAdmissionStep(result, "admission.pvc", started, "PVC dry-run admission succeeded", err) +} + +func runPodDryRun( + ctx context.Context, + client kubeclient.Interface, + namespace string, + name string, + identity agentkubernetes.ResourceIdentity, + profile agentkubernetes.ProfileConfig, + result *TestResult, +) { + started := time.Now() + template, err := agentkubernetes.ParsePodTemplate(profile.PodTemplateYAML) + var pod *corev1.Pod + var warnings []agentkubernetes.Warning + if err == nil { + pod, warnings, err = agentkubernetes.ComposePod(template, profile, agentkubernetes.PodOptions{ + Name: name, Namespace: namespace, Identity: identity, + Command: []string{"sh"}, Args: []string{"-c", "sleep 1"}, + WorkingDir: "/workspace", AgentctlPort: agentkubernetes.DefaultAgentctlPort, + ManagedPVCName: name, + }) + } + if err == nil { + result.Warnings = append(result.Warnings, warnings...) + var admitted *corev1.Pod + admitted, err = client.CoreV1().Pods(namespace).Create( + ctx, pod, metav1.CreateOptions{DryRun: []string{metav1.DryRunAll}}, + ) + if err == nil { + err = agentkubernetes.ValidateAdmittedPod(admitted, pod, profile.MainContainer) + } + if err == nil && admitted.UID == "" { + err = errors.New("admitted Pod dry-run UID is unavailable") + } + } + appendAdmissionStep(result, "admission.pod", started, "Pod dry-run admission succeeded", err) +} + +func appendAdmissionStep(result *TestResult, key string, started time.Time, detail string, err error) { + if err != nil { + result.Steps = append(result.Steps, failedStep(key, started, "Dry-run admission failed", err)) + return + } + result.Steps = append(result.Steps, TestStep{ + Key: key, Success: true, DurationMS: time.Since(started).Milliseconds(), Detail: detail, + }) +} + +func runDiscoveryStep(ctx context.Context, client kubeclient.Interface, result *TestResult) bool { + started := time.Now() + serverVersion, err := client.Discovery().ServerVersion() + if err != nil { + result.addFailedStep("discovery", started, "Kubernetes API discovery failed", err) + return false + } + result.ServerVersion = serverVersion.GitVersion + result.Steps = append(result.Steps, TestStep{ + Key: "discovery", Success: true, DurationMS: time.Since(started).Milliseconds(), + Detail: "Kubernetes API discovery succeeded", + }) + return true +} + +func runNamespaceStep(namespace string, result *TestResult) { + started := time.Now() + result.Steps = append(result.Steps, TestStep{ + Key: "namespace", Success: true, DurationMS: time.Since(started).Milliseconds(), + Detail: "Using configured namespace " + namespace, + }) +} + +func runPermissionStep( + ctx context.Context, + client kubeclient.Interface, + namespace string, + key string, + permissions []requiredPermission, +) TestStep { + started := time.Now() + denied := make([]string, 0) + for _, permission := range permissions { + allowed, err := checkPermission(ctx, client, namespace, permission) + if err != nil { + return failedStep(key, started, "Permission check failed", err) + } + if !allowed { + denied = append(denied, permission.String()) + } + } + if len(denied) > 0 { + return TestStep{ + Key: key, Success: false, DurationMS: time.Since(started).Milliseconds(), + Detail: "Denied permissions: " + strings.Join(denied, ", "), + Error: "required Kubernetes permissions are denied", + } + } + return TestStep{ + Key: key, Success: true, DurationMS: time.Since(started).Milliseconds(), + Detail: fmt.Sprintf("All %d required permissions are allowed", len(permissions)), + } +} + +func checkPermission( + ctx context.Context, + client kubeclient.Interface, + namespace string, + permission requiredPermission, +) (bool, error) { + review := &authorizationv1.SelfSubjectAccessReview{Spec: authorizationv1.SelfSubjectAccessReviewSpec{ + ResourceAttributes: &authorizationv1.ResourceAttributes{ + Namespace: namespace, Verb: permission.verb, + Resource: permission.resource, Subresource: permission.subresource, + }, + }} + created, err := client.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return false, err + } + return created.Status.Allowed, nil +} + +func (p requiredPermission) String() string { + resource := p.resource + if p.subresource != "" { + resource += "/" + p.subresource + } + return p.verb + " " + resource +} + +func (r *TestResult) addFailedStep(key string, started time.Time, detail string, err error) { + step := failedStep(key, started, detail, err) + r.Steps = append(r.Steps, step) + r.Error = step.Error +} + +func failedStep(key string, started time.Time, detail string, err error) TestStep { + return TestStep{ + Key: key, Success: false, DurationMS: time.Since(started).Milliseconds(), + Detail: detail, Error: sanitizeError(err), + } +} + +func (r *TestResult) finalize() { + r.Success = true + for _, step := range r.Steps { + if !step.Success { + r.Success = false + if r.Error == "" { + r.Error = step.Error + } + } + } +} + +func sanitizeError(err error) string { + if err == nil { + return "Kubernetes request failed" + } + var fieldErr *agentkubernetes.FieldError + if errors.As(err, &fieldErr) { + return fieldErr.Error() + } + return routingerr.SanitizeError(err).Error() +} diff --git a/apps/backend/internal/kubernetes/handlers_test.go b/apps/backend/internal/kubernetes/handlers_test.go new file mode 100644 index 0000000000..5c355b3723 --- /dev/null +++ b/apps/backend/internal/kubernetes/handlers_test.go @@ -0,0 +1,771 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/version" + "k8s.io/client-go/discovery/fake" + kubeclient "k8s.io/client-go/kubernetes" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/rest" + k8stesting "k8s.io/client-go/testing" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + "github.com/kandev/kandev/internal/auth/authn" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestRegisterRoutesWiresHTTPAndWebSocket(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + dispatcher := ws.NewDispatcher() + + RegisterRoutes(router, dispatcher, nil, nil, nil) + + routes := make(map[string]bool) + for _, route := range router.Routes() { + routes[route.Method+" "+route.Path] = true + } + require.True(t, routes["POST /api/v1/kubernetes/test"]) + require.True(t, routes["GET /api/v1/kubernetes/executors/:id/sessions"]) + require.True(t, routes["GET /api/v1/kubernetes/executors/:id/session-impact"]) + require.True(t, dispatcher.HasHandler("kubernetes.test")) + require.True(t, dispatcher.HasHandler("kubernetes.sessions.list")) + require.True(t, dispatcher.HasHandler("kubernetes.sessions.impact")) +} + +func TestHTTPTestConnectionRejectsMember(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "member-1", Role: authn.RoleMember}) + c.Next() + }) + RegisterRoutes(router, ws.NewDispatcher(), nil, nil, nil) + req := httptest.NewRequest(http.MethodPost, "/api/v1/kubernetes/test", strings.NewReader(`{"config":{}}`)) + req.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, req) + + require.Equal(t, http.StatusForbidden, recorder.Code) +} + +func TestWebSocketTestConnectionRejectsMember(t *testing.T) { + dispatcher := ws.NewDispatcher() + RegisterRoutes(gin.New(), dispatcher, nil, nil, nil) + msg, err := ws.NewRequest("req-1", "kubernetes.test", map[string]any{"config": map[string]string{}}) + require.NoError(t, err) + ctx := authn.WithIdentity(context.Background(), authn.Identity{UserID: "member-1", Role: authn.RoleMember}) + + response, err := dispatcher.Dispatch(ctx, msg) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeError, response.Type) + var payload ws.ErrorPayload + require.NoError(t, json.Unmarshal(response.Payload, &payload)) + require.Equal(t, ws.ErrorCodeForbidden, payload.Code) +} + +func TestHTTPTestConnectionReportsMandatoryChecks(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "kandev"}, + }) + discovery := clientset.Discovery().(*fake.FakeDiscovery) + discovery.FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + handler := NewHandler(nil, nil, func(config agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + probeCalls := 0 + handler.probeStreaming = func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + probeCalls++ + return nil + } + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "admin-1", Role: authn.RoleAdmin}) + c.Next() + }) + handler.registerHTTP(router) + req := httptest.NewRequest(http.MethodPost, "/api/v1/kubernetes/test", strings.NewReader(`{ + "config": { + "auth_mode": "kubeconfig", + "kubeconfig_path": "/etc/kandev/kubeconfig", + "namespace": "kandev", + "request_timeout_seconds": "30" + } + }`)) + req.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, req) + + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var result testConnectionResponse + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &result)) + require.True(t, result.Success) + require.Equal(t, "v1.30.1", result.ServerVersion) + require.Equal(t, "kandev", result.Namespace) + require.Equal(t, []string{"discovery", "namespace", "rbac", "streaming"}, testStepKeys(result.Steps)) + require.True(t, result.Steps[0].Success) + require.True(t, result.Steps[1].Success) + require.True(t, result.Steps[2].Success) + require.True(t, result.Steps[3].Success) + require.Contains(t, result.Steps[3].Detail, "live transport probe was not run") + require.Zero(t, probeCalls) + permissions := accessReviewPermissions(clientset.Actions()) + for _, permission := range []string{ + "get pods/exec", "create pods/exec", + "get pods/portforward", "create pods/portforward", + } { + require.Contains(t, permissions, permission) + } + for _, action := range clientset.Actions() { + require.NotEqual(t, "namespaces", action.GetResource().Resource, + "namespace validation must not require cluster-scoped Namespace RBAC") + if action.GetResource().Resource == "pods" { + require.NotEqual(t, "create", action.GetVerb(), "executor-only test must not create a probe Pod") + } + } +} + +func TestHTTPTestConnectionDryRunsPodAdmission(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset(&corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "kandev"}, + }) + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + probeState := installStreamingPodReactors(t, clientset) + handler := NewHandler(nil, nil, func(config agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "admin-1", Role: authn.RoleAdmin}) + c.Next() + }) + handler.registerHTTP(router) + body, err := json.Marshal(TestRequest{ + Config: map[string]string{ + "auth_mode": "kubeconfig", "kubeconfig_path": "/etc/kandev/kubeconfig", + "namespace": "kandev", "request_timeout_seconds": "30", + }, + ProfileConfig: validHandlerProfileConfig(), + }) + require.NoError(t, err) + req := httptest.NewRequest(http.MethodPost, "/api/v1/kubernetes/test", strings.NewReader(string(body))) + req.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, req) + + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var result testConnectionResponse + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &result)) + require.True(t, result.Success) + require.Contains(t, testStepKeys(result.Steps), "admission.pod") + require.Equal(t, []string{metav1.DryRunAll}, probeState.podDryRun.DryRun) + requireStreamingPodCleaned(t, probeState) +} + +func TestConnectionWaitsForExactProbeUIDToDisappear(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + probeState := installStreamingPodReactors(t, clientset) + probeState.deleteVisibilityGets = 2 + handler := NewHandler(nil, nil, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: validHandlerProfileConfig(), + }) + + require.True(t, result.Success) + require.GreaterOrEqual(t, probeState.getsAfterDelete, 3) + requireStreamingPodCleaned(t, probeState) +} + +func TestConnectionStreamingFailureStillDeletesExactProbePod(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + probeState := installStreamingPodReactors(t, clientset) + handler := NewHandler(nil, nil, func(config agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + probeCalls := 0 + handler.probeStreaming = func( + _ context.Context, + _ *agentkubernetes.Client, + _, podName, mainContainer string, + ) error { + probeCalls++ + require.Equal(t, probeState.actualPod.Name, podName) + require.Equal(t, "kandev-agent", mainContainer) + return errors.New("proxy refused upgrade: token=secret-token") + } + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: validHandlerProfileConfig(), + }) + + require.Equal(t, 1, probeCalls) + require.True(t, connectionStep(t, result.Steps, "discovery").Success) + require.True(t, connectionStep(t, result.Steps, "rbac").Success) + streaming := connectionStep(t, result.Steps, "streaming") + require.False(t, streaming.Success) + require.NotContains(t, streaming.Error, "secret-token") + requireStreamingPodCleaned(t, probeState) +} + +func TestConnectionReportsExactProbePodCleanupFailure(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + probeState := installStreamingPodReactors(t, clientset) + probeState.deleteErr = errors.New("delete failed: token=secret-token") + handler := NewHandler(nil, nil, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: validHandlerProfileConfig(), + }) + + streaming := connectionStep(t, result.Steps, "streaming") + require.False(t, streaming.Success) + require.Contains(t, streaming.Detail, "cleanup") + require.NotContains(t, streaming.Error, "secret-token") + require.True(t, probeState.live) + require.Equal(t, probeState.actualPod.Name, probeState.deletedName) + require.Equal(t, probeState.actualPod.UID, *probeState.deleteOptions.Preconditions.UID) +} + +func TestSanitizeErrorPreservesCauseAndRedactsSensitiveDetails(t *testing.T) { + err := errors.New( + "admission denied: quota exceeded; Bearer abcdefghijklmnopqrstuvwxyz; " + + "endpoint https://user:password@cluster.example/apis/private?token=secret; " + + "file /home/alice/.kube/config; " + strings.Repeat("x", routingerr.MaxRawExcerptBytes), + ) + + got := sanitizeError(err) + + require.Contains(t, got, "admission denied: quota exceeded") + require.Contains(t, got, "https://cluster.example") + require.NotContains(t, got, "abcdefghijklmnopqrstuvwxyz") + require.NotContains(t, got, "user:password") + require.NotContains(t, got, "/apis/private") + require.NotContains(t, got, "/home/alice/") + require.LessOrEqual(t, len(got), routingerr.MaxRawExcerptBytes) +} + +func TestStreamingRESTConfigClearsTimeoutWithoutMutatingClient(t *testing.T) { + original := &rest.Config{Host: "https://cluster.example", Timeout: 30 * time.Second} + client := &agentkubernetes.Client{RESTConfig: original} + + streamingConfig, err := streamingRESTConfig(client) + + require.NoError(t, err) + require.NotSame(t, original, streamingConfig) + require.Zero(t, streamingConfig.Timeout) + require.Equal(t, 30*time.Second, original.Timeout) +} + +func TestStreamingTransportsUseWebSocketGETThenSPDYPOSTFallback(t *testing.T) { + tests := []struct { + name string + path string + transport func(context.Context, *agentkubernetes.Client) error + }{ + { + name: "exec", path: "/exec", + transport: func(ctx context.Context, client *agentkubernetes.Client) error { + return probeExecUpgrade(ctx, client, "kandev", "probe-pod", "kandev-agent") + }, + }, + { + name: "port-forward", path: "/portforward", + transport: func(ctx context.Context, client *agentkubernetes.Client) error { + return probePortForwardUpgrade(ctx, client, "kandev", "probe-pod") + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + methods := make([]string, 0, 2) + paths := make([]string, 0, 2) + commands := make([][]string, 0, 2) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + methods = append(methods, request.Method) + paths = append(paths, request.URL.Path) + commands = append(commands, request.URL.Query()["command"]) + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusBadGateway) + _, _ = w.Write([]byte("proxy rejected streaming upgrade")) + })) + t.Cleanup(server.Close) + config := &rest.Config{Host: server.URL, Timeout: time.Second} + clientset, err := kubeclient.NewForConfig(config) + require.NoError(t, err) + client := &agentkubernetes.Client{RESTConfig: config, Clientset: clientset} + + err = test.transport(context.Background(), client) + + require.Error(t, err) + require.Equal(t, []string{http.MethodGet, http.MethodPost}, methods) + require.Len(t, paths, 2) + require.Contains(t, paths[0], test.path) + require.Contains(t, paths[1], test.path) + if test.name == "exec" { + require.Equal(t, []string{"sh", "-c", ":"}, commands[0]) + require.Equal(t, commands[0], commands[1]) + } + require.Equal(t, time.Second, config.Timeout) + }) + } +} + +func TestStreamingTransportDoesNotTreatPodNotFoundAsCompatible(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{ + "kind":"Status","apiVersion":"v1","status":"Failure", + "message":"pods probe-pod not found","reason":"NotFound","code":404 + }`)) + })) + t.Cleanup(server.Close) + config := &rest.Config{Host: server.URL, Timeout: time.Second} + clientset, err := kubeclient.NewForConfig(config) + require.NoError(t, err) + client := &agentkubernetes.Client{RESTConfig: config, Clientset: clientset} + + err = probeExecUpgrade(context.Background(), client, "kandev", "probe-pod", "kandev-agent") + + require.Error(t, err) + require.True(t, apierrors.IsNotFound(err), err) +} + +func TestWaitForStreamingProbePodRejectsReplacementWithSameName(t *testing.T) { + current := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "probe-pod", Namespace: "kandev", UID: types.UID("original-uid"), + Labels: map[string]string{"kandev.ai/managed-by": "kandev"}, + }, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + } + replacement := current.DeepCopy() + replacement.UID = types.UID("replacement-uid") + replacement.Status.Phase = corev1.PodRunning + replacement.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}, + }, + }} + clientset := kubernetesfake.NewSimpleClientset(replacement) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := waitForStreamingProbePod( + ctx, clientset, "kandev", "probe-pod", "kandev-agent", current, + ) + + require.ErrorContains(t, err, "identity changed") +} + +func TestHTTPTestConnectionChecksManagedPVCPermissionsAndDryRunsAdmission(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + permissions := make([]string, 0) + clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SelfSubjectAccessReview).DeepCopy() + attributes := review.Spec.ResourceAttributes + permissions = append(permissions, attributes.Verb+" "+attributes.Resource+"/"+attributes.Subresource) + review.Status.Allowed = true + return true, review, nil + }) + var pvcCreateOptions metav1.CreateOptions + clientset.PrependReactor("create", "persistentvolumeclaims", func(action k8stesting.Action) (bool, runtime.Object, error) { + create := action.(k8stesting.CreateAction) + pvcCreateOptions = action.(k8stesting.CreateActionImpl).GetCreateOptions() + claim := create.GetObject().(*corev1.PersistentVolumeClaim).DeepCopy() + claim.UID = "dry-run-pvc-uid" + return true, claim, nil + }) + probeState := installStreamingPodReactors(t, clientset) + handler := NewHandler(nil, nil, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + profile := validHandlerProfileConfig() + profile["workspace.mode"] = "managed_pvc" + profile["workspace.size"] = "10Gi" + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: profile, + }) + + require.True(t, result.Success) + require.True(t, connectionStep(t, result.Steps, "admission.pvc").Success) + require.True(t, connectionStep(t, result.Steps, "admission.pod").Success) + require.Equal(t, []string{metav1.DryRunAll}, pvcCreateOptions.DryRun) + for _, permission := range []string{ + "get persistentvolumeclaims/", "create persistentvolumeclaims/", + "delete persistentvolumeclaims/", + } { + require.Contains(t, permissions, permission) + } + require.NotContains(t, permissions, "watch persistentvolumeclaims/") + require.NotNil(t, probeState.actualPod.Spec.Volumes[0].EmptyDir) + require.Nil(t, probeState.actualPod.Spec.Volumes[0].PersistentVolumeClaim) + require.Equal(t, []string{"sh"}, probeState.actualPod.Spec.Containers[0].Command) + requireStreamingPodCleaned(t, probeState) +} + +func TestConnectionGetsConfiguredExistingClaimWithoutMutation(t *testing.T) { + claim := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "shared-workspace", Namespace: "kandev", UID: types.UID("shared-workspace-uid"), + }} + clientset := kubernetesfake.NewSimpleClientset(claim) + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + probeState := installStreamingPodReactors(t, clientset) + handler := NewHandler(nil, nil, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + profile := validHandlerProfileConfig() + profile["workspace.mode"] = "existing_claim" + profile["workspace.claim_name"] = claim.Name + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: profile, + }) + + require.True(t, result.Success) + require.True(t, connectionStep(t, result.Steps, "storage.existing_claim").Success) + requireOnlyExactClaimGet(t, clientset.Actions(), claim.Name) + requireStreamingPodCleaned(t, probeState) +} + +func TestConnectionRejectsMismatchedExistingClaimResponseWithoutMutation(t *testing.T) { + tests := []struct { + name string + claim *corev1.PersistentVolumeClaim + }{ + { + name: "name", + claim: &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "other-workspace", Namespace: "kandev", UID: types.UID("claim-uid"), + }}, + }, + { + name: "namespace", + claim: &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "shared-workspace", Namespace: "other", UID: types.UID("claim-uid"), + }}, + }, + { + name: "uid", + claim: &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "shared-workspace", Namespace: "kandev", + }}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + clientset.PrependReactor("get", "persistentvolumeclaims", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, test.claim.DeepCopy(), nil + }) + probeState := installStreamingPodReactors(t, clientset) + handler := NewHandler(nil, nil, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + profile := validHandlerProfileConfig() + profile["workspace.mode"] = "existing_claim" + profile["workspace.claim_name"] = "shared-workspace" + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: profile, + }) + + require.False(t, result.Success) + storage := connectionStep(t, result.Steps, "storage.existing_claim") + require.False(t, storage.Success) + require.Contains(t, storage.Error, "identity") + requireOnlyExactClaimGet(t, clientset.Actions(), "shared-workspace") + require.Nil(t, probeState.actualPod, "invalid claim identity must skip live probe creation") + }) + } +} + +func TestConnectionReportsMissingExistingClaimWithoutMutation(t *testing.T) { + clientset := kubernetesfake.NewSimpleClientset() + clientset.Discovery().(*fake.FakeDiscovery).FakedServerVersion = &version.Info{GitVersion: "v1.30.1"} + clientset.PrependReactor("create", "selfsubjectaccessreviews", allowAccessReview) + probeState := installStreamingPodReactors(t, clientset) + handler := NewHandler(nil, nil, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.probeStreaming = successfulStreamingProbe(t, probeState) + profile := validHandlerProfileConfig() + profile["workspace.mode"] = "existing_claim" + profile["workspace.claim_name"] = "missing-workspace" + + result := handler.testConnection(context.Background(), TestRequest{ + Config: validHandlerExecutorConfig(), ProfileConfig: profile, + }) + + require.False(t, result.Success) + storage := connectionStep(t, result.Steps, "storage.existing_claim") + require.False(t, storage.Success) + require.Contains(t, strings.ToLower(storage.Error), "not found") + requireOnlyExactClaimGet(t, clientset.Actions(), "missing-workspace") + require.Nil(t, probeState.actualPod, "failed storage validation must skip live probe creation") +} + +type testConnectionResponse struct { + Success bool `json:"success"` + ServerVersion string `json:"server_version"` + Namespace string `json:"namespace"` + Steps []testStepResponse `json:"steps"` +} + +type testStepResponse struct { + Key string `json:"key"` + Success bool `json:"success"` + Detail string `json:"detail"` +} + +func allowAccessReview(action k8stesting.Action) (bool, runtime.Object, error) { + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SelfSubjectAccessReview).DeepCopy() + review.Status.Allowed = true + return true, review, nil +} + +func accessReviewPermissions(actions []k8stesting.Action) []string { + permissions := make([]string, 0) + for _, action := range actions { + if action.GetResource().Resource != "selfsubjectaccessreviews" { + continue + } + review := action.(k8stesting.CreateAction).GetObject().(*authorizationv1.SelfSubjectAccessReview) + attributes := review.Spec.ResourceAttributes + resource := attributes.Resource + if attributes.Subresource != "" { + resource += "/" + attributes.Subresource + } + permissions = append(permissions, attributes.Verb+" "+resource) + } + return permissions +} + +func requireOnlyExactClaimGet(t *testing.T, actions []k8stesting.Action, claimName string) { + t.Helper() + gets := 0 + for _, action := range actions { + if action.GetResource().Resource != "persistentvolumeclaims" { + continue + } + require.Equal(t, "get", action.GetVerb(), "existing claims must never be listed or mutated") + get, ok := action.(k8stesting.GetAction) + require.True(t, ok) + require.Equal(t, claimName, get.GetName()) + gets++ + } + require.Equal(t, 1, gets) +} + +func testStepKeys(steps []testStepResponse) []string { + keys := make([]string, 0, len(steps)) + for _, step := range steps { + keys = append(keys, step.Key) + } + return keys +} + +func validHandlerProfileConfig() map[string]string { + return map[string]string{ + "platform": "linux/amd64", + "main_container": "kandev-agent", + "pod_template_yaml": `apiVersion: v1 +kind: PodTemplate +template: + spec: + containers: + - name: kandev-agent + image: alpine:3.20 +`, + "workspace.mode": "empty_dir", + } +} + +func validHandlerExecutorConfig() map[string]string { + return map[string]string{ + "auth_mode": "kubeconfig", "kubeconfig_path": "/etc/kandev/kubeconfig", + "namespace": "kandev", "request_timeout_seconds": "30", + } +} + +type streamingPodState struct { + podDryRun metav1.CreateOptions + requestedPod *corev1.Pod + actualPod *corev1.Pod + deletedNamespace string + deletedName string + deleteOptions metav1.DeleteOptions + deleteErr error + deleteIssued bool + deleteVisibilityGets int + getsAfterDelete int + live bool + mutateLivePod func(*corev1.Pod) + createErr error + commitOnCreateError bool + getErr error +} + +func installStreamingPodReactors( + t *testing.T, + clientset *kubernetesfake.Clientset, +) *streamingPodState { + t.Helper() + state := &streamingPodState{} + clientset.PrependReactor("create", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + create := action.(k8stesting.CreateAction) + options := action.(k8stesting.CreateActionImpl).GetCreateOptions() + pod := create.GetObject().(*corev1.Pod).DeepCopy() + if len(options.DryRun) > 0 { + state.podDryRun = options + pod.UID = types.UID("dry-run-pod-uid") + return true, pod, nil + } + state.requestedPod = pod.DeepCopy() + pod.UID = types.UID("probe-uid") + pod.Status.Phase = corev1.PodRunning + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}, + }, + }} + if state.mutateLivePod != nil { + state.mutateLivePod(pod) + } + if state.createErr != nil { + if state.commitOnCreateError { + state.actualPod = pod + state.live = true + } + return true, nil, state.createErr + } + state.actualPod = pod + state.live = true + return true, pod.DeepCopy(), nil + }) + clientset.PrependReactor("get", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + get := action.(k8stesting.GetAction) + name := get.GetName() + namespace := get.GetNamespace() + if state.getErr != nil && !state.deleteIssued { + return true, nil, state.getErr + } + if state.deleteIssued && name == state.deletedName && namespace == state.deletedNamespace { + state.getsAfterDelete++ + if state.actualPod != nil && state.actualPod.Name == name && state.actualPod.Namespace == namespace && + state.deleteVisibilityGets > 0 { + state.deleteVisibilityGets-- + return true, state.actualPod.DeepCopy(), nil + } + if state.actualPod != nil && state.actualPod.Name == name && state.actualPod.Namespace == namespace { + state.live = false + } + return true, nil, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, name) + } + if state.live && state.actualPod != nil && state.actualPod.Name == name && state.actualPod.Namespace == namespace { + return true, state.actualPod.DeepCopy(), nil + } + return true, nil, apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, name) + }) + clientset.PrependReactor("delete", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + deletion := action.(k8stesting.DeleteAction) + state.deletedNamespace = deletion.GetNamespace() + state.deletedName = deletion.GetName() + state.deleteOptions = deletion.GetDeleteOptions() + if state.deleteErr != nil { + return true, nil, state.deleteErr + } + state.deleteIssued = true + if state.deleteVisibilityGets == 0 && state.actualPod != nil && + state.actualPod.Name == state.deletedName && state.actualPod.Namespace == state.deletedNamespace { + state.live = false + } + return true, nil, nil + }) + return state +} + +func successfulStreamingProbe(t *testing.T, state *streamingPodState) StreamingProbe { + t.Helper() + return func( + _ context.Context, + _ *agentkubernetes.Client, + _, podName, mainContainer string, + ) error { + require.NotNil(t, state.actualPod) + require.True(t, state.live) + require.Equal(t, state.actualPod.Name, podName) + require.Equal(t, "kandev-agent", mainContainer) + return nil + } +} + +func requireStreamingPodCleaned(t *testing.T, state *streamingPodState) { + t.Helper() + require.NotNil(t, state.actualPod) + require.False(t, state.live) + require.Equal(t, state.actualPod.Namespace, state.deletedNamespace) + require.Equal(t, state.actualPod.Name, state.deletedName) + require.NotNil(t, state.deleteOptions.Preconditions) + require.NotNil(t, state.deleteOptions.Preconditions.UID) + require.Equal(t, state.actualPod.UID, *state.deleteOptions.Preconditions.UID) +} + +func connectionStep(t *testing.T, steps []TestStep, key string) TestStep { + t.Helper() + for _, step := range steps { + if step.Key == key { + return step + } + } + t.Fatalf("step %q missing from %+v", key, steps) + return TestStep{} +} diff --git a/apps/backend/internal/kubernetes/sessions.go b/apps/backend/internal/kubernetes/sessions.go new file mode 100644 index 0000000000..cc599ac3e0 --- /dev/null +++ b/apps/backend/internal/kubernetes/sessions.go @@ -0,0 +1,338 @@ +package kubernetes + +import ( + "context" + "errors" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository/repoerrors" +) + +// SessionRow is a sanitized projection of one Kubernetes-backed running executor. +type SessionRow struct { + SessionID string `json:"session_id"` + TaskID string `json:"task_id"` + PodName string `json:"pod_name,omitempty"` + PodPhase string `json:"pod_phase,omitempty"` + ContainerState string `json:"container_state,omitempty"` + Restarts int32 `json:"restarts"` + WorkspaceKind string `json:"workspace_kind,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + FailureReason string `json:"failure_reason,omitempty"` +} + +// SessionImpact is the authoritative mutation impact of one Kubernetes executor. +type SessionImpact struct { + ActiveSessionCount int `json:"active_session_count"` +} + +// SessionFilter narrows an executor's authoritative running inventory before +// task authorization or Kubernetes API access. +type SessionFilter struct { + TaskID string + SessionID string +} + +func (h *Handler) sessionImpact(ctx context.Context, executorID string) (SessionImpact, error) { + if executorID == "" { + return SessionImpact{}, errExecutorIDRequired + } + if h.repo == nil { + return SessionImpact{}, errSessionStatusNotWired + } + executor, err := h.repo.GetExecutor(ctx, executorID) + if err != nil { + return SessionImpact{}, err + } + if executor == nil { + return SessionImpact{}, models.ErrExecutorNotFound + } + if executor.Type != models.ExecutorTypeKubernetes { + return SessionImpact{}, errExecutorNotKubernetes + } + runs, err := h.repo.ListExecutorsRunning(ctx) + if err != nil { + return SessionImpact{}, err + } + impact := SessionImpact{} + for _, run := range runs { + if run != nil && run.Runtime == agentruntime.RuntimeKubernetes && run.ExecutorID == executorID && + isActiveExecutorRunningStatus(run.Status) { + impact.ActiveSessionCount++ + } + } + return impact, nil +} + +func isActiveExecutorRunningStatus(status string) bool { + switch status { + case models.ExecutorRunningStatusFailed, + models.ExecutorRunningStatusStopped, + models.ExecutorRunningStatusComplete: + return false + default: + return true + } +} + +func (h *Handler) listSessions( + ctx context.Context, + executorID string, + filter SessionFilter, +) ([]SessionRow, error) { + if filter.SessionID != "" && filter.TaskID == "" { + return nil, errTaskIDRequired + } + _, client, runs, err := h.sessionStatusSource(ctx, executorID) + if err != nil { + return nil, err + } + rows := make([]SessionRow, 0, len(runs)) + for _, run := range runs { + if !filter.matches(run) { + continue + } + row, visible, rowErr := h.sessionRow(ctx, client, executorID, run) + if rowErr != nil { + return nil, rowErr + } + if visible { + rows = append(rows, row) + } + } + return rows, nil +} + +func (f SessionFilter) matches(run *models.ExecutorRunning) bool { + if run == nil { + return f.TaskID == "" && f.SessionID == "" + } + if f.TaskID != "" && run.TaskID != f.TaskID { + return false + } + return f.SessionID == "" || run.SessionID == f.SessionID +} + +func (h *Handler) sessionStatusSource( + ctx context.Context, + executorID string, +) (agentkubernetes.ExecutorConfig, kubeclient.Interface, []*models.ExecutorRunning, error) { + if executorID == "" { + return agentkubernetes.ExecutorConfig{}, nil, nil, errExecutorIDRequired + } + if h.repo == nil || h.access == nil || h.clients == nil { + return agentkubernetes.ExecutorConfig{}, nil, nil, errSessionStatusNotWired + } + executor, err := h.repo.GetExecutor(ctx, executorID) + if err != nil { + return agentkubernetes.ExecutorConfig{}, nil, nil, err + } + if executor == nil { + return agentkubernetes.ExecutorConfig{}, nil, nil, models.ErrExecutorNotFound + } + if executor.Type != models.ExecutorTypeKubernetes { + return agentkubernetes.ExecutorConfig{}, nil, nil, errExecutorNotKubernetes + } + config, err := agentkubernetes.ParseExecutorConfig(executor.Config) + if err != nil { + return agentkubernetes.ExecutorConfig{}, nil, nil, errSessionStatusNotWired + } + client, err := h.clients(config) + if err != nil || client == nil || client.Clientset == nil { + return agentkubernetes.ExecutorConfig{}, nil, nil, errSessionStatusNotWired + } + runs, err := h.repo.ListExecutorsRunning(ctx) + if err != nil { + return agentkubernetes.ExecutorConfig{}, nil, nil, err + } + return config, client.Clientset, runs, nil +} + +func (h *Handler) sessionRow( + ctx context.Context, + client kubeclient.Interface, + executorID string, + run *models.ExecutorRunning, +) (SessionRow, bool, error) { + if run == nil || run.Runtime != agentruntime.RuntimeKubernetes { + return SessionRow{}, false, nil + } + if run.ExecutorID != executorID { + return SessionRow{}, false, nil + } + session, err := h.repo.GetTaskSession(ctx, run.SessionID) + if err != nil { + if errors.Is(err, models.ErrTaskSessionNotFound) { + return SessionRow{}, false, nil + } + return SessionRow{}, false, err + } + if session == nil || session.ID != run.SessionID || session.TaskID != run.TaskID { + return SessionRow{}, false, nil + } + if err := h.access.AuthorizeTaskAccess(ctx, run.TaskID); err != nil { + if errors.Is(err, repoerrors.ErrTaskNotFound) { + return SessionRow{}, false, nil + } + return SessionRow{}, false, err + } + row := newInventorySessionRow(run) + if inventoryFailure := validateSessionInventory(run, executorID, row); inventoryFailure != "" { + row.FailureReason = inventoryFailure + return row, true, nil + } + namespace := metadataString(run.Metadata, metadataNamespace) + pod, err := client.CoreV1().Pods(namespace).Get(ctx, row.PodName, metav1.GetOptions{}) + if err != nil { + row.FailureReason = podLookupFailure(err) + return row, true, nil + } + if !matchesSessionIdentity(pod, run) { + row.FailureReason = "Pod identity does not match runtime inventory" + return row, true, nil + } + populatePodStatus(&row, pod, metadataString(run.Metadata, metadataMainContainer)) + return row, true, nil +} + +func newInventorySessionRow(run *models.ExecutorRunning) SessionRow { + row := SessionRow{ + SessionID: run.SessionID, + TaskID: run.TaskID, + PodName: metadataString(run.Metadata, metadataPodName), + WorkspaceKind: metadataString(run.Metadata, metadataWorkspaceMode), + } + if !run.CreatedAt.IsZero() { + row.CreatedAt = run.CreatedAt.UTC().Format(time.RFC3339) + } + return row +} + +func validateSessionInventory( + run *models.ExecutorRunning, + executorID string, + row SessionRow, +) string { + identity, validIdentity := recordedResourceIdentity(run.Metadata) + if !validIdentity || run.ID != run.SessionID || run.ExecutorID != executorID || + identity.ExecutorID != executorID || identity.TaskID != run.TaskID || + identity.SessionID != run.SessionID || metadataString(run.Metadata, metadataNamespace) == "" || + row.PodName == "" || metadataString(run.Metadata, metadataPodUID) == "" || + metadataString(run.Metadata, metadataMainContainer) == "" || row.WorkspaceKind == "" { + return "Kubernetes runtime inventory is incomplete" + } + return "" +} + +func matchesSessionIdentity(pod *corev1.Pod, run *models.ExecutorRunning) bool { + if pod == nil || pod.Name != metadataString(run.Metadata, metadataPodName) || + pod.Namespace != metadataString(run.Metadata, metadataNamespace) || + string(pod.UID) != metadataString(run.Metadata, metadataPodUID) { + return false + } + identity, ok := recordedResourceIdentity(run.Metadata) + if !ok { + return false + } + expected, err := agentkubernetes.OwnershipLabels(identity) + if err != nil { + return false + } + for key, value := range expected { + if pod.Labels[key] != value { + return false + } + } + for key, value := range pod.Labels { + if !strings.HasPrefix(key, "kandev.ai/") { + continue + } + expectedValue, exists := expected[key] + if !exists || value != expectedValue { + return false + } + } + return true +} + +func recordedResourceIdentity(metadata map[string]interface{}) (agentkubernetes.ResourceIdentity, bool) { + identity := agentkubernetes.ResourceIdentity{ + ExecutorID: metadataString(metadata, metadataResourceExecutor), + ProfileID: metadataString(metadata, metadataResourceProfile), + InstanceID: metadataString(metadata, metadataResourceInstance), + TaskID: metadataString(metadata, metadataResourceTask), + SessionID: metadataString(metadata, metadataResourceSession), + EnvironmentID: metadataString(metadata, metadataResourceEnv), + } + if _, err := agentkubernetes.OwnershipLabels(identity); err != nil { + return agentkubernetes.ResourceIdentity{}, false + } + return identity, true +} + +func populatePodStatus(row *SessionRow, pod *corev1.Pod, mainContainer string) { + row.PodPhase = string(pod.Status.Phase) + row.ContainerState = "unknown" + failureReason := pod.Status.Reason + for _, status := range pod.Status.ContainerStatuses { + if status.Name != mainContainer { + continue + } + row.Restarts = status.RestartCount + switch { + case status.State.Running != nil: + row.ContainerState = "running" + case status.State.Waiting != nil: + row.ContainerState = "waiting" + failureReason = status.State.Waiting.Reason + case status.State.Terminated != nil: + row.ContainerState = "terminated" + failureReason = status.State.Terminated.Reason + } + break + } + row.FailureReason = sanitizedPodReason(failureReason) +} + +func metadataString(metadata map[string]interface{}, key string) string { + value, _ := metadata[key].(string) + return strings.TrimSpace(value) +} + +func podLookupFailure(err error) string { + switch { + case apierrors.IsNotFound(err): + return "Pod not found" + case apierrors.IsForbidden(err), apierrors.IsUnauthorized(err): + return "Pod status access is denied" + default: + return "Pod status is unavailable" + } +} + +func sanitizedPodReason(reason string) string { + reason = strings.TrimSpace(reason) + if reason == "" { + return "" + } + if len(reason) > 80 { + return "Kubernetes workload reported a failure" + } + for _, char := range reason { + if (char >= 'a' && char <= 'z') || (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || strings.ContainsRune("._-", char) { + continue + } + return "Kubernetes workload reported a failure" + } + return reason +} diff --git a/apps/backend/internal/kubernetes/sessions_namespace_test.go b/apps/backend/internal/kubernetes/sessions_namespace_test.go new file mode 100644 index 0000000000..aed0fc5b28 --- /dev/null +++ b/apps/backend/internal/kubernetes/sessions_namespace_test.go @@ -0,0 +1,53 @@ +package kubernetes + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/task/models" +) + +func TestListSessionsUsesRecordedNamespaceAfterExecutorNamespaceChange(t *testing.T) { + createdAt := time.Date(2026, time.September, 2, 9, 0, 0, 0, time.UTC) + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", createdAt, + ) + run.Metadata[metadataNamespace] = "old-agents" + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1") + pod := kubernetesOwnedPod("pod-1", "pod-uid-1", "instance-1", session) + pod.Namespace = "old-agents" + config := validHandlerExecutorConfig() + config["namespace"] = "new-agents" + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: config, + }, + runs: []*models.ExecutorRunning{run}, + sessions: map[string]*models.TaskSession{"session-1": session}, + } + clientset := kubernetesfake.NewSimpleClientset(pod) + handler := NewHandler(repo, &fakeAccessChecker{}, func(got agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + require.Equal(t, "new-agents", got.Namespace, "current config must build the client") + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + + rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{}) + + require.NoError(t, err) + require.Equal(t, []SessionRow{{ + SessionID: "session-1", TaskID: "task-1", PodName: "pod-1", + PodPhase: "Running", ContainerState: "running", Restarts: 2, + WorkspaceKind: "empty_dir", CreatedAt: createdAt.Format(time.RFC3339), + }}, rows) + require.Len(t, clientset.Actions(), 1) + getAction, ok := clientset.Actions()[0].(k8stesting.GetAction) + require.True(t, ok) + require.Equal(t, "old-agents", getAction.GetNamespace()) + require.Equal(t, "pod-1", getAction.GetName()) +} diff --git a/apps/backend/internal/kubernetes/sessions_test.go b/apps/backend/internal/kubernetes/sessions_test.go new file mode 100644 index 0000000000..5e07a05cf7 --- /dev/null +++ b/apps/backend/internal/kubernetes/sessions_test.go @@ -0,0 +1,747 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository/repoerrors" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestHTTPListSessionsAllowsMemberAndGetsOnlyAuthorizedRecordedPods(t *testing.T) { + createdAt := time.Date(2026, time.August, 24, 9, 30, 0, 0, time.UTC) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{ + kubernetesRunningRow("instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", createdAt), + kubernetesRunningRow("instance-2", "session-2", "task-2", "pod-2", "pod-uid-2", createdAt), + {ID: "ssh-instance", Runtime: agentruntime.RuntimeSSH, SessionID: "ssh-session"}, + }, + sessions: map[string]*models.TaskSession{ + "session-1": kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1"), + "session-2": kubernetesTaskSession("session-2", "task-2", "executor-1", "profile-1"), + }, + } + clientset := kubernetesfake.NewSimpleClientset(kubernetesOwnedPod( + "pod-1", "pod-uid-1", "instance-1", repo.sessions["session-1"], + )) + access := &fakeAccessChecker{denied: map[string]error{"task-2": repoerrors.ErrTaskNotFound}} + handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "member-1", Role: authn.RoleMember}) + c.Next() + }) + handler.registerHTTP(router) + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, httptest.NewRequest( + http.MethodGet, "/api/v1/kubernetes/executors/executor-1/sessions", nil, + )) + + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var rows []SessionRow + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &rows)) + require.Equal(t, []SessionRow{{ + SessionID: "session-1", TaskID: "task-1", PodName: "pod-1", + PodPhase: "Running", ContainerState: "running", Restarts: 2, + WorkspaceKind: "empty_dir", CreatedAt: createdAt.Format(time.RFC3339), + }}, rows) + require.Equal(t, []string{"task-1", "task-2"}, access.calls) + require.Len(t, clientset.Actions(), 1) + getAction, ok := clientset.Actions()[0].(k8stesting.GetAction) + require.True(t, ok) + require.Equal(t, "pods", getAction.GetResource().Resource) + require.Equal(t, "pod-1", getAction.GetName()) +} + +func TestWebSocketListSessionsAllowsMember(t *testing.T) { + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + sessions: map[string]*models.TaskSession{}, + } + dispatcher := ws.NewDispatcher() + handler := NewHandler(repo, &fakeAccessChecker{}, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: kubernetesfake.NewSimpleClientset()}, nil + }) + handler.registerWS(dispatcher) + msg, err := ws.NewRequest("req-1", "kubernetes.sessions.list", map[string]string{ + "executor_id": "executor-1", + }) + require.NoError(t, err) + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", Role: authn.RoleMember, + }) + + response, err := dispatcher.Dispatch(ctx, msg) + + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type) + require.JSONEq(t, `[]`, string(response.Payload)) +} + +func TestHTTPListSessionsFiltersTaskSessionBeforeAuthorizationAndPodLookup(t *testing.T) { + createdAt := time.Date(2026, time.August, 25, 9, 30, 0, 0, time.UTC) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{ + kubernetesRunningRow("instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", createdAt), + kubernetesRunningRow("instance-2", "session-2", "task-2", "pod-2", "pod-uid-2", createdAt), + }, + sessions: map[string]*models.TaskSession{ + "session-1": kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1"), + "session-2": kubernetesTaskSession("session-2", "task-2", "executor-1", "profile-1"), + }, + } + clientset := kubernetesfake.NewSimpleClientset( + kubernetesOwnedPod("pod-1", "pod-uid-1", "instance-1", repo.sessions["session-1"]), + kubernetesOwnedPod("pod-2", "pod-uid-2", "instance-2", repo.sessions["session-2"]), + ) + access := &fakeAccessChecker{} + handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + router := gin.New() + handler.registerHTTP(router) + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, httptest.NewRequest( + http.MethodGet, + "/api/v1/kubernetes/executors/executor-1/sessions?task_id=task-1&session_id=session-1", + nil, + )) + + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var rows []SessionRow + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &rows)) + require.Len(t, rows, 1) + require.Equal(t, "session-1", rows[0].SessionID) + require.Equal(t, []string{"task-1"}, access.calls) + require.Len(t, clientset.Actions(), 1) + getAction, ok := clientset.Actions()[0].(k8stesting.GetAction) + require.True(t, ok) + require.Equal(t, "pod-1", getAction.GetName()) + + clientset.ClearActions() + access.calls = nil + mismatchRecorder := httptest.NewRecorder() + router.ServeHTTP(mismatchRecorder, httptest.NewRequest( + http.MethodGet, + "/api/v1/kubernetes/executors/executor-1/sessions?task_id=task-1&session_id=session-2", + nil, + )) + require.Equal(t, http.StatusOK, mismatchRecorder.Code, mismatchRecorder.Body.String()) + require.JSONEq(t, `[]`, mismatchRecorder.Body.String()) + require.Empty(t, access.calls) + require.Empty(t, clientset.Actions()) +} + +func TestHTTPListSessionsRejectsSessionFilterWithoutTask(t *testing.T) { + handler := NewHandler(&fakeResourceRepository{}, &fakeAccessChecker{}, nil) + router := gin.New() + handler.registerHTTP(router) + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, httptest.NewRequest( + http.MethodGet, + "/api/v1/kubernetes/executors/executor-1/sessions?session_id=session-1", + nil, + )) + + require.Equal(t, http.StatusBadRequest, recorder.Code, recorder.Body.String()) +} + +func TestWebSocketListSessionsFiltersTaskSessionAndRejectsSessionOnly(t *testing.T) { + createdAt := time.Date(2026, time.August, 25, 9, 30, 0, 0, time.UTC) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{ + kubernetesRunningRow("instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", createdAt), + kubernetesRunningRow("instance-2", "session-2", "task-2", "pod-2", "pod-uid-2", createdAt), + }, + sessions: map[string]*models.TaskSession{ + "session-1": kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1"), + "session-2": kubernetesTaskSession("session-2", "task-2", "executor-1", "profile-1"), + }, + } + clientset := kubernetesfake.NewSimpleClientset( + kubernetesOwnedPod("pod-1", "pod-uid-1", "instance-1", repo.sessions["session-1"]), + kubernetesOwnedPod("pod-2", "pod-uid-2", "instance-2", repo.sessions["session-2"]), + ) + access := &fakeAccessChecker{} + dispatcher := ws.NewDispatcher() + handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + handler.registerWS(dispatcher) + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", Role: authn.RoleMember, + }) + filtered, err := ws.NewRequest("req-1", "kubernetes.sessions.list", map[string]string{ + "executor_id": "executor-1", + "task_id": "task-1", + "session_id": "session-1", + }) + require.NoError(t, err) + + response, err := dispatcher.Dispatch(ctx, filtered) + + require.NoError(t, err) + require.Equal(t, ws.MessageTypeResponse, response.Type) + var rows []SessionRow + require.NoError(t, json.Unmarshal(response.Payload, &rows)) + require.Len(t, rows, 1) + require.Equal(t, "session-1", rows[0].SessionID) + require.Equal(t, []string{"task-1"}, access.calls) + require.Len(t, clientset.Actions(), 1) + + sessionOnly, err := ws.NewRequest("req-2", "kubernetes.sessions.list", map[string]string{ + "executor_id": "executor-1", + "session_id": "session-1", + }) + require.NoError(t, err) + response, err = dispatcher.Dispatch(ctx, sessionOnly) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeError, response.Type) + var wsError ws.ErrorPayload + require.NoError(t, json.Unmarshal(response.Payload, &wsError)) + require.Equal(t, ws.ErrorCodeBadRequest, wsError.Code) +} + +func TestHTTPSessionImpactCountsEveryExactKubernetesInventoryRowForAdmin(t *testing.T) { + repo := &fakeResourceRepository{ + executor: &models.Executor{ID: "executor-1", Type: models.ExecutorTypeKubernetes}, + runs: []*models.ExecutorRunning{ + {ExecutorID: "executor-1", Runtime: agentruntime.RuntimeKubernetes}, + {ExecutorID: "executor-1", Runtime: agentruntime.RuntimeKubernetes, Metadata: map[string]interface{}{}}, + {ExecutorID: "executor-1", Runtime: agentruntime.RuntimeKubernetes, Status: models.ExecutorRunningStatusFailed}, + {ExecutorID: "executor-1", Runtime: agentruntime.RuntimeKubernetes, Status: models.ExecutorRunningStatusStopped}, + {ExecutorID: "executor-1", Runtime: agentruntime.RuntimeKubernetes, Status: models.ExecutorRunningStatusComplete}, + {ExecutorID: "executor-1", Runtime: agentruntime.RuntimeDocker}, + {ExecutorID: "executor-other", Runtime: agentruntime.RuntimeKubernetes}, + nil, + }, + } + handler := NewHandler(repo, nil, nil) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "admin-1", Role: authn.RoleAdmin}) + c.Next() + }) + handler.registerHTTP(router) + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, httptest.NewRequest( + http.MethodGet, "/api/v1/kubernetes/executors/executor-1/session-impact", nil, + )) + + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + require.JSONEq(t, `{"active_session_count":2}`, recorder.Body.String()) +} + +func TestSessionImpactForbidsMembersOverHTTPAndWebSocket(t *testing.T) { + repo := &fakeResourceRepository{ + executor: &models.Executor{ID: "executor-1", Type: models.ExecutorTypeKubernetes}, + } + handler := NewHandler(repo, nil, nil) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "member-1", Role: authn.RoleMember}) + c.Next() + }) + handler.registerHTTP(router) + httpRecorder := httptest.NewRecorder() + router.ServeHTTP(httpRecorder, httptest.NewRequest( + http.MethodGet, "/api/v1/kubernetes/executors/executor-1/session-impact", nil, + )) + require.Equal(t, http.StatusForbidden, httpRecorder.Code) + + dispatcher := ws.NewDispatcher() + handler.registerWS(dispatcher) + msg, err := ws.NewRequest("req-1", "kubernetes.sessions.impact", map[string]string{ + "executor_id": "executor-1", + }) + require.NoError(t, err) + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", Role: authn.RoleMember, + }) + response, err := dispatcher.Dispatch(ctx, msg) + require.NoError(t, err) + require.Equal(t, ws.MessageTypeError, response.Type) + var payload ws.ErrorPayload + require.NoError(t, json.Unmarshal(response.Payload, &payload)) + require.Equal(t, ws.ErrorCodeForbidden, payload.Code) +} + +func TestHTTPSessionImpactValidatesExecutorAndPropagatesRepositoryFailures(t *testing.T) { + repositoryFailure := errors.New("inventory unavailable") + tests := []struct { + name string + repo *fakeResourceRepository + wantStatus int + }{ + { + name: "executor not found", repo: &fakeResourceRepository{}, + wantStatus: http.StatusNotFound, + }, + { + name: "wrong executor type", + repo: &fakeResourceRepository{executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeLocal, + }}, + wantStatus: http.StatusBadRequest, + }, + { + name: "executor repository failure", + repo: &fakeResourceRepository{ + executorErr: repositoryFailure, + }, + wantStatus: http.StatusInternalServerError, + }, + { + name: "inventory repository failure", + repo: &fakeResourceRepository{ + executor: &models.Executor{ID: "executor-1", Type: models.ExecutorTypeKubernetes}, + runsErr: repositoryFailure, + }, + wantStatus: http.StatusInternalServerError, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + handler := NewHandler(test.repo, nil, nil) + router := gin.New() + router.Use(func(c *gin.Context) { + authn.SetOnGin(c, authn.Identity{UserID: "admin-1", Role: authn.RoleAdmin}) + c.Next() + }) + handler.registerHTTP(router) + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, httptest.NewRequest( + http.MethodGet, "/api/v1/kubernetes/executors/executor-1/session-impact", nil, + )) + + require.Equal(t, test.wantStatus, recorder.Code, recorder.Body.String()) + }) + } +} + +func TestHTTPListSessionsReturns500ForUnexpectedAccessError(t *testing.T) { + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, + Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{run}, + sessions: map[string]*models.TaskSession{ + "session-1": kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1"), + }, + } + handler := NewHandler( + repo, + &fakeAccessChecker{denied: map[string]error{ + "task-1": errors.New("workspace database unavailable"), + }}, + func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: kubernetesfake.NewSimpleClientset()}, nil + }, + ) + router := gin.New() + handler.registerHTTP(router) + recorder := httptest.NewRecorder() + + router.ServeHTTP(recorder, httptest.NewRequest( + http.MethodGet, "/api/v1/kubernetes/executors/executor-1/sessions", nil, + )) + + require.Equal(t, http.StatusInternalServerError, recorder.Code, recorder.Body.String()) + require.JSONEq(t, `{"error":"Kubernetes session status is unavailable"}`, recorder.Body.String()) +} + +func TestListSessionsPropagatesUnexpectedSessionAndAccessErrors(t *testing.T) { + tests := []struct { + name string + sessionErr error + accessErr error + }{ + {name: "session repository", sessionErr: errors.New("session database unavailable")}, + {name: "access repository", accessErr: errors.New("workspace database unavailable")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, + Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{run}, + sessions: map[string]*models.TaskSession{ + "session-1": kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1"), + }, + sessionErrs: map[string]error{"session-1": test.sessionErr}, + } + access := &fakeAccessChecker{denied: map[string]error{"task-1": test.accessErr}} + handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: kubernetesfake.NewSimpleClientset()}, nil + }) + + rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{}) + + require.Error(t, err) + require.Empty(t, rows) + if test.sessionErr != nil { + require.ErrorIs(t, err, test.sessionErr) + } else { + require.ErrorIs(t, err, test.accessErr) + } + }) + } +} + +func TestListSessionsFailsClosedOnPodIdentityMismatch(t *testing.T) { + createdAt := time.Date(2026, time.August, 24, 9, 30, 0, 0, time.UTC) + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1") + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{ + kubernetesRunningRow("instance-1", "session-1", "task-1", "pod-1", "recorded-uid", createdAt), + }, + sessions: map[string]*models.TaskSession{"session-1": session}, + } + pod := kubernetesOwnedPod("pod-1", "different-uid", "instance-1", session) + handler := NewHandler(repo, &fakeAccessChecker{}, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: kubernetesfake.NewSimpleClientset(pod)}, nil + }) + + rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{}) + + require.NoError(t, err) + require.Equal(t, []SessionRow{{ + SessionID: "session-1", TaskID: "task-1", PodName: "pod-1", + WorkspaceKind: "empty_dir", CreatedAt: createdAt.Format(time.RFC3339), + FailureReason: "Pod identity does not match runtime inventory", + }}, rows) +} + +func TestListSessionsUsesAuthoritativeRunningExecutorAfterSessionRepoint(t *testing.T) { + createdAt := time.Date(2026, time.August, 24, 9, 30, 0, 0, time.UTC) + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", createdAt, + ) + session := kubernetesTaskSession("session-1", "task-1", "repointed-executor", "current-profile") + clientset := kubernetesfake.NewSimpleClientset(kubernetesOwnedPodForIdentity( + "pod-1", "pod-uid-1", agentkubernetes.ResourceIdentity{ + ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "instance-1", + TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", + }, + )) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{run}, + sessions: map[string]*models.TaskSession{"session-1": session}, + } + handler := NewHandler(repo, &fakeAccessChecker{}, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + + rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{}) + + require.NoError(t, err) + require.Equal(t, []SessionRow{{ + SessionID: "session-1", TaskID: "task-1", PodName: "pod-1", + PodPhase: "Running", ContainerState: "running", Restarts: 2, + WorkspaceKind: "empty_dir", CreatedAt: createdAt.Format(time.RFC3339), + }}, rows) +} + +func TestListSessionsOmitsKubernetesRowWithoutRecordedExecutorBeforeAuthorization(t *testing.T) { + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + run.ExecutorID = "" + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{run}, + sessions: map[string]*models.TaskSession{ + "session-1": kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1"), + }, + } + clientset := kubernetesfake.NewSimpleClientset() + access := &fakeAccessChecker{} + handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + + rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{}) + + require.NoError(t, err) + require.Empty(t, rows, "an unassociated row must not appear under every Kubernetes executor") + require.Empty(t, access.calls, "an unassociated row must stop before authorization") + require.Empty(t, clientset.Actions(), "an unassociated row must stop before Kubernetes API access") +} + +func TestListSessionsRejectsUncorrelatedTaskSessionBeforeAuthorization(t *testing.T) { + tests := []struct { + name string + mutate func(*models.TaskSession) + }{ + {name: "session id", mutate: func(session *models.TaskSession) { session.ID = "other-session" }}, + {name: "task id", mutate: func(session *models.TaskSession) { session.TaskID = "other-task" }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1") + test.mutate(session) + repo := &fakeResourceRepository{ + executor: &models.Executor{ + ID: "executor-1", Type: models.ExecutorTypeKubernetes, + Config: validHandlerExecutorConfig(), + }, + runs: []*models.ExecutorRunning{run}, + sessions: map[string]*models.TaskSession{ + run.SessionID: session, + }, + } + clientset := kubernetesfake.NewSimpleClientset() + access := &fakeAccessChecker{} + handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) { + return &agentkubernetes.Client{Clientset: clientset}, nil + }) + + rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{}) + + require.NoError(t, err) + require.Empty(t, rows) + require.Empty(t, access.calls, "uncorrelated row must stop before authorization") + require.Empty(t, clientset.Actions(), "uncorrelated row must stop before Kubernetes API access") + }) + } +} + +func TestMatchesSessionIdentityRejectsRecordedNameOrNamespaceMismatch(t *testing.T) { + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1") + run := kubernetesRunningRow( + "instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + pod := kubernetesOwnedPod("pod-1", "pod-uid-1", "instance-1", session) + pod.Name = "different-pod" + + require.False(t, matchesSessionIdentity(pod, run)) + pod.Name = "pod-1" + pod.Namespace = "different-namespace" + require.False(t, matchesSessionIdentity(pod, run)) +} + +func TestMatchesSessionIdentityUsesPersistedResourceInstanceID(t *testing.T) { + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1") + run := kubernetesRunningRow( + "resource-instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + pod := kubernetesOwnedPod("pod-1", "pod-uid-1", "resource-instance-1", session) + + require.Equal(t, "session-1", run.ID) + require.True(t, matchesSessionIdentity(pod, run)) +} + +func TestMatchesSessionIdentityUsesEntireRecordedIdentityAfterSessionEdit(t *testing.T) { + run := kubernetesRunningRow( + "resource-instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "current-profile") + session.TaskEnvironmentID = "current-environment" + pod := kubernetesOwnedPodForIdentity("pod-1", "pod-uid-1", agentkubernetes.ResourceIdentity{ + ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "resource-instance-1", + TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", + }) + + require.NotEqual(t, session.ExecutorProfileID, "profile-1") + require.NotEqual(t, session.TaskEnvironmentID, "environment-1") + require.True(t, matchesSessionIdentity(pod, run)) +} + +func TestMatchesSessionIdentityRejectsAdditionalKandevOwnershipLabel(t *testing.T) { + run := kubernetesRunningRow( + "resource-instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + pod := kubernetesOwnedPodForIdentity("pod-1", "pod-uid-1", agentkubernetes.ResourceIdentity{ + ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "resource-instance-1", + TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", + }) + pod.Labels["kandev.ai/forged"] = "true" + + require.False(t, matchesSessionIdentity(pod, run)) + delete(pod.Labels, "kandev.ai/forged") + pod.Labels["example.com/injected"] = "true" + require.True(t, matchesSessionIdentity(pod, run), "unrelated admission labels remain allowed") +} + +func TestValidateSessionInventoryRequiresEntireRecordedIdentity(t *testing.T) { + keys := []string{ + metadataResourceExecutor, + metadataResourceProfile, + metadataResourceInstance, + metadataResourceTask, + metadataResourceSession, + metadataResourceEnv, + } + for _, key := range keys { + t.Run(key, func(t *testing.T) { + run := kubernetesRunningRow( + "resource-instance-1", "session-1", "task-1", "pod-1", "pod-uid-1", time.Now(), + ) + delete(run.Metadata, key) + session := kubernetesTaskSession("session-1", "task-1", "executor-1", "profile-1") + + failure := validateSessionInventory(run, session.ExecutorID, newInventorySessionRow(run)) + + require.Equal(t, "Kubernetes runtime inventory is incomplete", failure) + }) + } +} + +type fakeResourceRepository struct { + executor *models.Executor + executorErr error + runs []*models.ExecutorRunning + runsErr error + sessions map[string]*models.TaskSession + sessionErrs map[string]error +} + +func (r *fakeResourceRepository) ListExecutorsRunning(context.Context) ([]*models.ExecutorRunning, error) { + return r.runs, r.runsErr +} + +func (r *fakeResourceRepository) GetTaskSession(_ context.Context, id string) (*models.TaskSession, error) { + if err := r.sessionErrs[id]; err != nil { + return nil, err + } + session := r.sessions[id] + if session == nil { + return nil, errors.New("session not found") + } + return session, nil +} + +func (r *fakeResourceRepository) GetExecutor(context.Context, string) (*models.Executor, error) { + if r.executorErr != nil { + return nil, r.executorErr + } + if r.executor == nil { + return nil, models.ErrExecutorNotFound + } + return r.executor, nil +} + +type fakeAccessChecker struct { + denied map[string]error + calls []string +} + +func (c *fakeAccessChecker) AuthorizeTaskAccess(_ context.Context, taskID string) error { + c.calls = append(c.calls, taskID) + return c.denied[taskID] +} + +func kubernetesRunningRow( + instanceID, sessionID, taskID, podName, podUID string, + createdAt time.Time, +) *models.ExecutorRunning { + return &models.ExecutorRunning{ + ID: sessionID, Runtime: agentruntime.RuntimeKubernetes, + SessionID: sessionID, TaskID: taskID, ExecutorID: "executor-1", CreatedAt: createdAt, + Metadata: map[string]interface{}{ + metadataNamespace: "kandev", metadataPodName: podName, metadataPodUID: podUID, + metadataMainContainer: "kandev-agent", metadataWorkspaceMode: "empty_dir", + metadataResourceExecutor: "executor-1", + metadataResourceProfile: "profile-1", + metadataResourceInstance: instanceID, + metadataResourceTask: taskID, + metadataResourceSession: sessionID, + metadataResourceEnv: "environment-1", + }, + } +} + +func kubernetesTaskSession(sessionID, taskID, executorID, profileID string) *models.TaskSession { + return &models.TaskSession{ + ID: sessionID, TaskID: taskID, ExecutorID: executorID, + ExecutorProfileID: profileID, TaskEnvironmentID: "environment-1", + } +} + +func kubernetesOwnedPod( + podName, podUID, instanceID string, + session *models.TaskSession, +) *corev1.Pod { + return kubernetesOwnedPodForIdentity(podName, podUID, agentkubernetes.ResourceIdentity{ + ExecutorID: session.ExecutorID, ProfileID: session.ExecutorProfileID, + InstanceID: instanceID, TaskID: session.TaskID, SessionID: session.ID, + EnvironmentID: session.TaskEnvironmentID, + }) +} + +func kubernetesOwnedPodForIdentity( + podName, podUID string, + identity agentkubernetes.ResourceIdentity, +) *corev1.Pod { + labels, err := agentkubernetes.OwnershipLabels(identity) + if err != nil { + panic(err) + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, Namespace: "kandev", UID: types.UID(podUID), Labels: labels, + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "kandev-agent", RestartCount: 2, + State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}, + }}, + }, + } +} diff --git a/apps/backend/internal/kubernetes/streaming.go b/apps/backend/internal/kubernetes/streaming.go new file mode 100644 index 0000000000..6c1af19311 --- /dev/null +++ b/apps/backend/internal/kubernetes/streaming.go @@ -0,0 +1,416 @@ +package kubernetes + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "time" + + "github.com/google/uuid" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + kubeclient "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/tools/remotecommand" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func probeStreamingTransport( + ctx context.Context, + client *agentkubernetes.Client, + namespace string, + podName string, + mainContainer string, +) error { + if client == nil || client.RESTConfig == nil || client.Clientset == nil { + return errors.New("streaming transport is unavailable") + } + if err := probeExecUpgrade(ctx, client, namespace, podName, mainContainer); err != nil { + return fmt.Errorf("pods/exec upgrade failed: %w", err) + } + if err := probePortForwardUpgrade(ctx, client, namespace, podName); err != nil { + return fmt.Errorf("pods/portforward upgrade failed: %w", err) + } + return nil +} + +func probeExecUpgrade( + ctx context.Context, + client *agentkubernetes.Client, + namespace string, + podName string, + mainContainer string, +) error { + streamingConfig, err := streamingRESTConfig(client) + if err != nil { + return err + } + requestURL := client.Clientset.CoreV1().RESTClient().Post(). + Namespace(namespace).Resource("pods").Name(podName).SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: mainContainer, Command: []string{"sh", "-c", ":"}, Stdout: true, + }, scheme.ParameterCodec).URL() + websocketExecutor, err := remotecommand.NewWebSocketExecutor( + streamingConfig, http.MethodGet, requestURL.String(), + ) + if err != nil { + return err + } + spdyExecutor, err := remotecommand.NewSPDYExecutor( + streamingConfig, http.MethodPost, requestURL, + ) + if err != nil { + return err + } + executor, err := remotecommand.NewFallbackExecutor( + websocketExecutor, spdyExecutor, shouldFallbackStreaming, + ) + if err != nil { + return err + } + return executor.StreamWithContext(ctx, remotecommand.StreamOptions{Stdout: io.Discard}) +} + +func probePortForwardUpgrade( + ctx context.Context, + client *agentkubernetes.Client, + namespace string, + podName string, +) error { + streamingConfig, err := streamingRESTConfig(client) + if err != nil { + return err + } + requestURL := client.Clientset.CoreV1().RESTClient().Get(). + Namespace(namespace).Resource("pods").Name(podName).SubResource("portforward").URL() + dialer, cancelDialer, err := agentkubernetes.NewPortForwardDialer( + ctx, streamingConfig, requestURL, streamingPortForwardHandshakeTimeout(ctx), + ) + if err != nil { + return err + } + defer cancelDialer() + connection, protocol, err := dialer.Dial(portforward.PortForwardProtocolV1Name) + if connection != nil { + _ = connection.Close() + } + if err != nil { + return err + } + if protocol != portforward.PortForwardProtocolV1Name { + return fmt.Errorf("unexpected port-forward protocol %q", protocol) + } + return nil +} + +func streamingPortForwardHandshakeTimeout(ctx context.Context) time.Duration { + maximum := time.Duration(agentkubernetes.DefaultRequestTimeoutSeconds) * time.Second + deadline, ok := ctx.Deadline() + if !ok { + return maximum + } + remaining := time.Until(deadline) + if remaining <= 0 { + return time.Nanosecond + } + if remaining < maximum { + return remaining + } + return maximum +} + +func streamingRESTConfig(client *agentkubernetes.Client) (*rest.Config, error) { + if client == nil || client.RESTConfig == nil { + return nil, errors.New("streaming REST configuration is unavailable") + } + config := rest.CopyConfig(client.RESTConfig) + config.Timeout = 0 + return config, nil +} + +func shouldFallbackStreaming(err error) bool { + return agentkubernetes.ShouldFallbackStream(err) +} + +func (h *Handler) runStreamingStep( + ctx context.Context, + client *agentkubernetes.Client, + namespace string, + permissions []requiredPermission, + profile *agentkubernetes.ProfileConfig, + prerequisitesPassed bool, +) TestStep { + step := runPermissionStep(ctx, client.Clientset, namespace, "streaming", permissions) + if !step.Success { + return step + } + if profile == nil { + step.Detail = "Streaming permissions succeeded; live transport probe was not run without a profile" + return step + } + if !prerequisitesPassed { + step.Success = false + step.Detail = "Pod permissions and dry-run admission must succeed before the live streaming probe" + step.Error = "streaming probe prerequisites failed" + return step + } + started := time.Now() + if err := h.runLiveStreamingProbe(ctx, client, namespace, *profile); err != nil { + detail := "Streaming upgrade transport is incompatible" + if errors.Is(err, errStreamingProbeCleanup) { + detail = "Streaming probe cleanup failed" + } + return failedStep("streaming", started, detail, err) + } + step.DurationMS += time.Since(started).Milliseconds() + step.Detail = "Exec and port-forward transports succeeded against a disposable Pod" + return step +} + +func (h *Handler) runLiveStreamingProbe( + ctx context.Context, + client *agentkubernetes.Client, + namespace string, + profile agentkubernetes.ProfileConfig, +) (returnErr error) { + probeName := "kandev-stream-probe-" + uuid.NewString()[:8] + pod, err := buildStreamingProbePod(namespace, probeName, profile) + if err != nil { + return err + } + if err := agentkubernetes.StampCreateNonce(pod); err != nil { + return fmt.Errorf("generate streaming probe create nonce: %w", err) + } + created, err := client.Clientset.CoreV1().Pods(namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + if !agentkubernetes.IsAmbiguousCreateError(err) { + return err + } + return reconcileStreamingProbeCreateError( + ctx, client.Clientset, pod, profile.MainContainer, err, + ) + } + if created == nil || created.Namespace == "" || created.Name == "" || created.UID == "" { + return fmt.Errorf("%w: created Pod identity is incomplete", errStreamingProbeCleanup) + } + createdNamespace := created.Namespace + createdName := created.Name + uid := created.UID + defer func() { + if cleanupErr := deleteStreamingProbePod( + ctx, client.Clientset, createdNamespace, createdName, uid, + ); cleanupErr != nil { + cleanupErr = fmt.Errorf("%w: %v", errStreamingProbeCleanup, cleanupErr) + returnErr = errors.Join(returnErr, cleanupErr) + } + }() + if err := agentkubernetes.ValidateAdmittedPod(created, pod, profile.MainContainer); err != nil { + return err + } + running, err := waitForStreamingProbePod( + ctx, client.Clientset, createdNamespace, createdName, profile.MainContainer, created, + ) + if err != nil { + return err + } + return h.probeStreaming(ctx, client, running.Namespace, running.Name, profile.MainContainer) +} + +func reconcileStreamingProbeCreateError( + ctx context.Context, + client kubeclient.Interface, + desired *corev1.Pod, + mainContainer string, + createErr error, +) error { + reconcileCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + admitted, getErr := client.CoreV1().Pods(desired.Namespace).Get( + reconcileCtx, desired.Name, metav1.GetOptions{}, + ) + switch { + case apierrors.IsNotFound(getErr): + return createErr + case getErr != nil: + return errors.Join( + createErr, + fmt.Errorf("reconcile streaming probe create: inspect exact Pod: %w", getErr), + ) + } + if admitted == nil || admitted.Namespace == "" || admitted.Name == "" || admitted.UID == "" { + return errors.Join( + createErr, + errors.New("reconcile streaming probe create: returned Pod identity is incomplete"), + ) + } + if err := agentkubernetes.ValidateAdmittedPod(admitted, desired, mainContainer); err != nil { + return errors.Join( + createErr, + fmt.Errorf("reconcile streaming probe create: %w", err), + ) + } + if err := deleteStreamingProbePod( + reconcileCtx, client, admitted.Namespace, admitted.Name, admitted.UID, + ); err != nil { + return errors.Join( + createErr, + fmt.Errorf("%w: %v", errStreamingProbeCleanup, err), + ) + } + return createErr +} + +func buildStreamingProbePod( + namespace string, + name string, + profile agentkubernetes.ProfileConfig, +) (*corev1.Pod, error) { + probeProfile := profile + probeProfile.Workspace = agentkubernetes.WorkspaceConfig{Mode: agentkubernetes.WorkspaceModeEmptyDir} + template, err := agentkubernetes.ParsePodTemplate(probeProfile.PodTemplateYAML) + if err != nil { + return nil, err + } + identity := agentkubernetes.ResourceIdentity{ + ExecutorID: connectionTestIdentity, ProfileID: connectionTestIdentity, + InstanceID: name, TaskID: connectionTestIdentity, + SessionID: name, EnvironmentID: connectionTestIdentity, + } + pod, _, err := agentkubernetes.ComposePod(template, probeProfile, agentkubernetes.PodOptions{ + Name: name, Namespace: namespace, Identity: identity, + Command: []string{"sh"}, Args: []string{"-c", "while :; do sleep 30; done"}, + WorkingDir: "/workspace", AgentctlPort: agentkubernetes.DefaultAgentctlPort, + }) + return pod, err +} + +func matchesStreamingProbePod(current, admitted *corev1.Pod, mainContainer string) bool { + if current == nil || admitted == nil { + return false + } + runtimePod := current + if current.Spec.NodeName != admitted.Spec.NodeName { + if admitted.Spec.NodeName != "" || current.Spec.NodeName == "" { + return false + } + runtimePod = current.DeepCopy() + runtimePod.Spec.NodeName = "" + } + return agentkubernetes.ValidateAdmittedPod(runtimePod, admitted, mainContainer) == nil +} + +func waitForStreamingProbePod( + ctx context.Context, + client kubeclient.Interface, + namespace string, + name string, + mainContainer string, + current *corev1.Pod, +) (*corev1.Pod, error) { + if current == nil || current.UID == "" { + return nil, errors.New("streaming probe Pod identity is unavailable") + } + expected := current.DeepCopy() + assignedNodeName := expected.Spec.NodeName + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + if current.Spec.NodeName != "" { + if assignedNodeName == "" { + assignedNodeName = current.Spec.NodeName + } else if current.Spec.NodeName != assignedNodeName { + return nil, errors.New("streaming probe Pod identity changed while waiting") + } + } else if assignedNodeName != "" { + return nil, errors.New("streaming probe Pod identity changed while waiting") + } + if current.UID != expected.UID || !matchesStreamingProbePod(current, expected, mainContainer) { + return nil, errors.New("streaming probe Pod identity changed while waiting") + } + ready, err := streamingProbePodReady(current, mainContainer) + if ready || err != nil { + return current, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ticker.C: + } + current, err = client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + } +} + +func streamingProbePodReady(pod *corev1.Pod, mainContainer string) (bool, error) { + if pod == nil { + return false, errors.New("streaming probe Pod is unavailable") + } + switch pod.Status.Phase { + case corev1.PodFailed, corev1.PodSucceeded: + return false, errors.New("streaming probe Pod terminated before transport checks") + } + if pod.Status.Phase != corev1.PodRunning { + return false, nil + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name != mainContainer { + continue + } + if status.State.Terminated != nil { + return false, errors.New("streaming probe main container terminated") + } + return status.State.Running != nil, nil + } + return false, nil +} + +func deleteStreamingProbePod( + ctx context.Context, + client kubeclient.Interface, + namespace string, + name string, + uid types.UID, +) error { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + grace := int64(0) + propagation := metav1.DeletePropagationForeground + err := client.CoreV1().Pods(namespace).Delete(cleanupCtx, name, metav1.DeleteOptions{ + GracePeriodSeconds: &grace, + PropagationPolicy: &propagation, + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + pod, getErr := client.CoreV1().Pods(namespace).Get(cleanupCtx, name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(getErr): + return nil + case getErr != nil: + return getErr + case pod == nil || pod.UID != uid: + return nil + } + select { + case <-cleanupCtx.Done(): + return cleanupCtx.Err() + case <-ticker.C: + } + } +} diff --git a/apps/backend/internal/kubernetes/streaming_cancel_test.go b/apps/backend/internal/kubernetes/streaming_cancel_test.go new file mode 100644 index 0000000000..d6fabb4a6f --- /dev/null +++ b/apps/backend/internal/kubernetes/streaming_cancel_test.go @@ -0,0 +1,86 @@ +package kubernetes + +import ( + "context" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestProbePortForwardUpgradeCancelsBlockedWebSocketHandshake(t *testing.T) { + testProbePortForwardUpgradeCancelsBlockedHandshake(t, http.MethodGet) +} + +func TestProbePortForwardUpgradeCancelsBlockedSPDYFallbackHandshake(t *testing.T) { + testProbePortForwardUpgradeCancelsBlockedHandshake(t, http.MethodPost) +} + +func testProbePortForwardUpgradeCancelsBlockedHandshake(t *testing.T, blockedMethod string) { + t.Helper() + requestStarted := make(chan struct{}) + releaseRequest := make(chan struct{}) + var startOnce sync.Once + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(releaseRequest) }) } + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) { + if blockedMethod == http.MethodPost && request.Method == http.MethodGet { + http.Error(response, "WebSocket upgrade refused", http.StatusBadRequest) + return + } + if request.Method != blockedMethod { + http.Error(response, "unexpected method", http.StatusMethodNotAllowed) + return + } + startOnce.Do(func() { close(requestStarted) }) + select { + case <-request.Context().Done(): + case <-releaseRequest: + } + })) + t.Cleanup(func() { + release() + server.Close() + }) + config := &rest.Config{Host: server.URL} + clientset, err := kubernetes.NewForConfig(config) + require.NoError(t, err) + client := &agentkubernetes.Client{RESTConfig: config, Clientset: clientset} + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + result := make(chan error, 1) + go func() { + result <- probePortForwardUpgrade(ctx, client, "kandev", "probe-pod") + }() + + select { + case <-requestStarted: + case err := <-result: + release() + require.FailNow(t, "port-forward probe returned before blocked handshake", "error: %v", err) + case <-time.After(time.Second): + release() + select { + case <-result: + case <-time.After(time.Second): + } + require.FailNow(t, "blocked port-forward handshake did not start") + } + cancel() + + select { + case err := <-result: + require.Error(t, err) + case <-time.After(300 * time.Millisecond): + release() + err := <-result + require.FailNow(t, "blocked port-forward handshake ignored cancellation", "eventual error: %v", err) + } +} diff --git a/apps/backend/internal/kubernetes/streaming_reconcile_test.go b/apps/backend/internal/kubernetes/streaming_reconcile_test.go new file mode 100644 index 0000000000..c1c86c7ba8 --- /dev/null +++ b/apps/backend/internal/kubernetes/streaming_reconcile_test.go @@ -0,0 +1,246 @@ +package kubernetes + +import ( + "context" + "errors" + "net" + "regexp" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestLiveStreamingProbeReconcilesCommittedPodAfterCreateError(t *testing.T) { + createErr := ambiguousStreamingCreateError("create response was lost") + clientset, state := createErrorStreamingClient(t, createErr, true) + profile := streamingProbeProfile(t) + probeCalls := 0 + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + probeCalls++ + return nil + }} + + err := handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", profile) + + require.ErrorIs(t, err, createErr) + require.EqualError(t, err, createErr.Error()) + require.Zero(t, probeCalls) + require.Regexp(t, regexp.MustCompile(`^[0-9a-f]{64}$`), + state.requestedPod.Annotations[agentkubernetes.CreateNonceAnnotation]) + require.Equal(t, state.requestedPod.Annotations[agentkubernetes.CreateNonceAnnotation], + state.actualPod.Annotations[agentkubernetes.CreateNonceAnnotation]) + requireStreamingPodCleaned(t, state) + requireOnlyExactProbePodActions(t, clientset.Actions(), state.requestedPod.Name) +} + +func TestLiveStreamingProbeCreateErrorNotFoundPreservesOriginalError(t *testing.T) { + createErr := ambiguousStreamingCreateError("create connection reset") + clientset, state := createErrorStreamingClient(t, createErr, false) + profile := streamingProbeProfile(t) + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + return nil + }} + + err := handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", profile) + + require.EqualError(t, err, createErr.Error()) + require.False(t, state.deleteIssued) + requireOnlyExactProbePodActions(t, clientset.Actions(), state.requestedPod.Name) + require.Equal(t, []string{"create", "get"}, podActionVerbs(clientset.Actions())) +} + +func TestLiveStreamingProbeCreateErrorFailsClosedOnAmbiguousPod(t *testing.T) { + tests := []struct { + name string + configure func(*streamingPodState) + want string + }{ + { + name: "read error", + configure: func(state *streamingPodState) { + state.getErr = errors.New("read denied: Bearer abcdefghijklmnopqrstuvwxyz") + }, + want: "inspect exact Pod", + }, + { + name: "missing uid", + configure: func(state *streamingPodState) { + state.mutateLivePod = func(pod *corev1.Pod) { pod.UID = "" } + }, + want: "identity is incomplete", + }, + { + name: "ownership mismatch", + configure: func(state *streamingPodState) { + state.mutateLivePod = func(pod *corev1.Pod) { + pod.Labels["kandev.ai/session-id"] = "foreign" + } + }, + want: "ownership label", + }, + { + name: "missing create nonce", + configure: func(state *streamingPodState) { + state.mutateLivePod = func(pod *corev1.Pod) { + delete(pod.Annotations, agentkubernetes.CreateNonceAnnotation) + } + }, + want: "owned annotation", + }, + { + name: "different create nonce", + configure: func(state *streamingPodState) { + state.mutateLivePod = func(pod *corev1.Pod) { + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) + } + pod.Annotations[agentkubernetes.CreateNonceAnnotation] = "foreign" + } + }, + want: "owned annotation", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + createErr := ambiguousStreamingCreateError("create result is ambiguous") + clientset, state := createErrorStreamingClient(t, createErr, true) + test.configure(state) + profile := streamingProbeProfile(t) + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + return nil + }} + + err := handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", profile) + + require.ErrorIs(t, err, createErr) + require.ErrorContains(t, err, test.want) + require.False(t, state.deleteIssued, "ambiguous identity must never authorize deletion") + requireOnlyExactProbePodActions(t, clientset.Actions(), state.requestedPod.Name) + if test.name == "read error" { + require.NotContains(t, sanitizeError(err), "abcdefghijklmnopqrstuvwxyz") + } + }) + } +} + +func TestLiveStreamingProbeCreateErrorJoinsCleanupFailure(t *testing.T) { + createErr := ambiguousStreamingCreateError("create response was lost") + clientset, state := createErrorStreamingClient(t, createErr, true) + state.deleteErr = errors.New("cleanup denied") + profile := streamingProbeProfile(t) + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + return nil + }} + + err := handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", profile) + + require.ErrorIs(t, err, createErr) + require.ErrorContains(t, err, "cleanup denied") + require.ErrorIs(t, err, errStreamingProbeCleanup) + requireOnlyExactProbePodActions(t, clientset.Actions(), state.requestedPod.Name) +} + +func TestLiveStreamingProbeDefiniteCreateErrorsNeverAdoptVisiblePod(t *testing.T) { + pods := schema.GroupResource{Resource: "pods"} + tests := []struct { + name string + createErr error + }{ + {name: "already exists", createErr: apierrors.NewAlreadyExists(pods, "probe-pod")}, + {name: "conflict", createErr: apierrors.NewConflict(pods, "probe-pod", errors.New("conflict"))}, + {name: "too many requests", createErr: apierrors.NewTooManyRequests("rate limited", 1)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + clientset, state := createErrorStreamingClient(t, test.createErr, true) + handler := &Handler{probeStreaming: func( + context.Context, *agentkubernetes.Client, string, string, string, + ) error { + return nil + }} + + err := handler.runLiveStreamingProbe(context.Background(), &agentkubernetes.Client{ + Clientset: clientset, + }, "kandev", streamingProbeProfile(t)) + + require.ErrorIs(t, err, test.createErr) + require.False(t, state.deleteIssued) + require.True(t, state.live, "a visible same-name Pod must not be adopted after a definite rejection") + require.Equal(t, []string{"create"}, podActionVerbs(clientset.Actions())) + }) + } +} + +func createErrorStreamingClient( + t *testing.T, + createErr error, + committed bool, +) (*kubernetesfake.Clientset, *streamingPodState) { + t.Helper() + clientset := kubernetesfake.NewSimpleClientset() + state := installStreamingPodReactors(t, clientset) + state.createErr = createErr + state.commitOnCreateError = committed + return clientset, state +} + +func ambiguousStreamingCreateError(message string) error { + return &net.OpError{Op: "create", Net: "tcp", Err: errors.New(message)} +} + +func streamingProbeProfile(t *testing.T) agentkubernetes.ProfileConfig { + t.Helper() + profile, err := agentkubernetes.ParseProfileConfig(validHandlerProfileConfig()) + require.NoError(t, err) + return profile +} + +func requireOnlyExactProbePodActions(t *testing.T, actions []k8stesting.Action, name string) { + t.Helper() + for _, action := range actions { + if action.GetResource().Resource != "pods" { + continue + } + require.Equal(t, "kandev", action.GetNamespace()) + require.NotEqual(t, "list", action.GetVerb()) + switch typed := action.(type) { + case k8stesting.DeleteAction: + require.Equal(t, name, typed.GetName()) + case k8stesting.GetAction: + require.Equal(t, name, typed.GetName()) + } + } +} + +func podActionVerbs(actions []k8stesting.Action) []string { + verbs := make([]string, 0, len(actions)) + for _, action := range actions { + if action.GetResource().Resource == "pods" { + verbs = append(verbs, action.GetVerb()) + } + } + return verbs +} diff --git a/apps/backend/internal/kubernetes/streaming_test.go b/apps/backend/internal/kubernetes/streaming_test.go new file mode 100644 index 0000000000..47a8ce6f36 --- /dev/null +++ b/apps/backend/internal/kubernetes/streaming_test.go @@ -0,0 +1,13 @@ +package kubernetes + +import ( + "net" + "testing" +) + +func TestShouldFallbackStreamingIncludesTimeoutErrors(t *testing.T) { + err := &net.DNSError{Err: "upgrade timed out", IsTimeout: true} + if !shouldFallbackStreaming(err) { + t.Fatal("streaming transport timeout did not permit SPDY fallback") + } +} diff --git a/apps/backend/internal/kubernetes/streaming_wait_test.go b/apps/backend/internal/kubernetes/streaming_wait_test.go new file mode 100644 index 0000000000..f3d2a5242f --- /dev/null +++ b/apps/backend/internal/kubernetes/streaming_wait_test.go @@ -0,0 +1,98 @@ +package kubernetes + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" +) + +func TestWaitForStreamingProbePodAllowsSchedulerAssignedNodeName(t *testing.T) { + admitted := streamingWaitProbePod(t) + scheduled := admitted.DeepCopy() + scheduled.Spec.NodeName = "kind-worker" + markStreamingProbeRunning(scheduled) + clientset := kubernetesfake.NewSimpleClientset(scheduled) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + running, err := waitForStreamingProbePod( + ctx, clientset, admitted.Namespace, admitted.Name, "kandev-agent", admitted, + ) + + require.NoError(t, err) + require.Equal(t, "kind-worker", running.Spec.NodeName) +} + +func TestWaitForStreamingProbePodRejectsOwnedMutationAfterScheduling(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.Pod) + }{ + { + name: "ownership label", + mutate: func(pod *corev1.Pod) { + pod.Labels["kandev.ai/session-id"] = "foreign" + }, + }, + { + name: "main bootstrap", + mutate: func(pod *corev1.Pod) { + pod.Spec.Containers[0].Command = []string{"false"} + }, + }, + { + name: "platform selector", + mutate: func(pod *corev1.Pod) { + pod.Spec.NodeSelector[corev1.LabelOSStable] = "windows" + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + admitted := streamingWaitProbePod(t) + scheduled := admitted.DeepCopy() + scheduled.Spec.NodeName = "kind-worker" + test.mutate(scheduled) + markStreamingProbeRunning(scheduled) + clientset := kubernetesfake.NewSimpleClientset(scheduled) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := waitForStreamingProbePod( + ctx, clientset, admitted.Namespace, admitted.Name, "kandev-agent", admitted, + ) + + require.ErrorContains(t, err, "identity changed") + }) + } +} + +func streamingWaitProbePod(t *testing.T) *corev1.Pod { + t.Helper() + profile, err := agentkubernetes.ParseProfileConfig(validHandlerProfileConfig()) + require.NoError(t, err) + pod, err := buildStreamingProbePod("kandev", "probe-pod", profile) + require.NoError(t, err) + pod.UID = types.UID("probe-uid") + pod.CreationTimestamp = metav1.Now() + pod.Status.Phase = corev1.PodPending + return pod +} + +func markStreamingProbeRunning(pod *corev1.Pod) { + pod.Status.Phase = corev1.PodRunning + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{ + Name: "kandev-agent", + State: corev1.ContainerState{ + Running: &corev1.ContainerStateRunning{StartedAt: metav1.Now()}, + }, + }} +} diff --git a/apps/backend/internal/office/service/env_builder.go b/apps/backend/internal/office/service/env_builder.go index b1f5cb2cae..f291f44427 100644 --- a/apps/backend/internal/office/service/env_builder.go +++ b/apps/backend/internal/office/service/env_builder.go @@ -37,12 +37,13 @@ func (si *SchedulerIntegration) buildEnvVars( } // injectKandevCLI overrides KANDEV_CLI for remote executor types where the -// host binary path does not apply. For Docker and Sprites, the agentctl -// binary is baked into the image or uploaded, so we use the container-side path. +// host binary path does not apply. Each value is the executor-visible path. func (si *SchedulerIntegration) injectKandevCLI(env map[string]string, executorType string) { switch executorType { case "local_docker", "sprites": env["KANDEV_CLI"] = "/usr/local/bin/agentctl" + case "k8s": + env["KANDEV_CLI"] = "/opt/kandev/agentctl" } } diff --git a/apps/backend/internal/office/service/kubernetes_parity_test.go b/apps/backend/internal/office/service/kubernetes_parity_test.go new file mode 100644 index 0000000000..b760e7eec3 --- /dev/null +++ b/apps/backend/internal/office/service/kubernetes_parity_test.go @@ -0,0 +1,28 @@ +package service + +import "testing" + +func TestInjectKandevCLIKubernetesUsesRuntimeVolumeBinary(t *testing.T) { + integration := &SchedulerIntegration{} + env := map[string]string{"KANDEV_CLI": "/host/bin/agentctl"} + + integration.injectKandevCLI(env, "k8s") + + if got := env["KANDEV_CLI"]; got != "/opt/kandev/agentctl" { + t.Fatalf("KANDEV_CLI = %q, want Kubernetes runtime binary", got) + } +} + +func TestInstructionsDirForKubernetesUsesRuntimeVolume(t *testing.T) { + got := instructionsDirForExecutor("/host/kandev", "workspace", "agent", "k8s") + want := "/opt/kandev/runtime/workspace/instructions/agent" + if got != want { + t.Fatalf("instructions dir = %q, want %q", got, want) + } +} + +func TestInstructionsDirForUnknownExecutorFailsClosed(t *testing.T) { + if got := instructionsDirForExecutor("/host/kandev", "workspace", "agent", "future_executor"); got != "" { + t.Fatalf("instructions dir = %q, want empty path for unknown executor", got) + } +} diff --git a/apps/backend/internal/office/service/skill_lookup.go b/apps/backend/internal/office/service/skill_lookup.go index 9eb2fa1592..bea11a9f53 100644 --- a/apps/backend/internal/office/service/skill_lookup.go +++ b/apps/backend/internal/office/service/skill_lookup.go @@ -23,6 +23,10 @@ const runtimeSubdir = "runtime" // rationale as runtimeSubdir. const spritesRuntimeBase = "/root/.kandev/runtime" +// kubernetesRuntimeBase mirrors +// internal/agent/runtime/lifecycle/skill.KubernetesRuntimeBase. +const kubernetesRuntimeBase = "/opt/kandev/runtime" + // resolveInstructionsForPrompt returns the path the agent will see at // instructionsDir (host or sprite-side, depending on executor) and // the AGENTS.md content for prompt embedding. Pure data — no disk I/O. @@ -50,13 +54,20 @@ func (si *SchedulerIntegration) resolveInstructionsForPrompt(manifest *SkillMani // will materialise instruction files at, given an executor type. // // - sprites → /root/.kandev/runtime//instructions/ +// - k8s → /opt/kandev/runtime//instructions/ // - local_pc/docker → /runtime//instructions/ // // The returned path is used both by office's prompt builder (embeds // it as a hint string) and matches what skill.Deployer writes to. func instructionsDirForExecutor(basePath, workspaceSlug, agentID, executorType string) string { - if executorType == "sprites" { + switch executorType { + case "sprites": return fmt.Sprintf("%s/%s/instructions/%s", spritesRuntimeBase, workspaceSlug, agentID) + case "k8s": + return fmt.Sprintf("%s/%s/instructions/%s", kubernetesRuntimeBase, workspaceSlug, agentID) + case "", "local", "local_pc", "worktree", "local_docker", "remote_docker", "ssh", "mock_remote": + return filepath.Join(basePath, runtimeSubdir, workspaceSlug, "instructions", agentID) + default: + return "" } - return filepath.Join(basePath, runtimeSubdir, workspaceSlug, "instructions", agentID) } diff --git a/apps/backend/internal/orchestrator/executor/executor.go b/apps/backend/internal/orchestrator/executor/executor.go index c3148d6e8b..8dd2f0099c 100644 --- a/apps/backend/internal/orchestrator/executor/executor.go +++ b/apps/backend/internal/orchestrator/executor/executor.go @@ -349,6 +349,7 @@ type RemoteRuntimeStatus struct { // RemoteStatusPollRequest contains the fields from ExecutorRunning needed for remote status polling. type RemoteStatusPollRequest struct { + TaskID string SessionID string Runtime agentruntime.Runtime AgentExecutionID string diff --git a/apps/backend/internal/orchestrator/executor/executor_environment_reuse.go b/apps/backend/internal/orchestrator/executor/executor_environment_reuse.go index f44ffbfbc3..a362667d5b 100644 --- a/apps/backend/internal/orchestrator/executor/executor_environment_reuse.go +++ b/apps/backend/internal/orchestrator/executor/executor_environment_reuse.go @@ -6,6 +6,7 @@ import ( "sort" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/worktree" "go.uber.org/zap" @@ -506,7 +507,14 @@ func executorRunningMatchesEnvironment(running *models.ExecutorRunning, env *mod } func applyExecutorRunningMetadata(req *LaunchAgentRequest, running *models.ExecutorRunning) { - if running.AgentExecutionID != "" && req.PreviousExecutionID == "" { + requestIsKubernetes := models.ExecutorType(req.ExecutorType) == models.ExecutorTypeKubernetes + runningIsKubernetes := running.Runtime == agentruntime.RuntimeKubernetes + mayReuseExecution := true + if requestIsKubernetes || runningIsKubernetes { + mayReuseExecution = requestIsKubernetes && runningIsKubernetes && + req.SessionID != "" && running.SessionID == req.SessionID + } + if running.AgentExecutionID != "" && req.PreviousExecutionID == "" && mayReuseExecution { req.PreviousExecutionID = running.AgentExecutionID } var metadata map[string]interface{} diff --git a/apps/backend/internal/orchestrator/executor/executor_environment_test.go b/apps/backend/internal/orchestrator/executor/executor_environment_test.go index a9012f951e..a55aa06c49 100644 --- a/apps/backend/internal/orchestrator/executor/executor_environment_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_environment_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" @@ -1004,6 +1005,86 @@ func TestApplyExecutorRunningMetadata_SkipsSessionScopedKeys(t *testing.T) { } } +func TestApplyExecutorRunningMetadata_DoesNotResumeSiblingKubernetesSession(t *testing.T) { + req := &LaunchAgentRequest{ + TaskID: "task-1", SessionID: "session-new", ExecutorType: string(models.ExecutorTypeKubernetes), + } + running := &models.ExecutorRunning{ + SessionID: "session-old", TaskID: "task-1", AgentExecutionID: "execution-old", + Runtime: agentruntime.RuntimeKubernetes, + Metadata: recordedKubernetesResumeMetadataFor("task-1", "session-old", "execution-old"), + } + + applyExecutorRunningMetadata(req, running) + + if req.PreviousExecutionID != "" { + t.Fatalf("PreviousExecutionID = %q, want empty for fresh sibling Kubernetes launch", req.PreviousExecutionID) + } +} + +func TestApplyExecutorRunningMetadata_KubernetesResumeRequiresExactSessionAuthority(t *testing.T) { + tests := []struct { + name string + requestType string + requestSession string + runtime agentruntime.Runtime + runningSession string + wantPrevious string + }{ + { + name: "exact non-empty Kubernetes session resumes", requestType: string(models.ExecutorTypeKubernetes), + requestSession: "session-1", runtime: agentruntime.RuntimeKubernetes, + runningSession: "session-1", wantPrevious: "execution-old", + }, + { + name: "sibling Kubernetes session starts fresh", requestType: string(models.ExecutorTypeKubernetes), + requestSession: "session-new", runtime: agentruntime.RuntimeKubernetes, + runningSession: "session-old", + }, + { + name: "missing recorded Kubernetes session starts fresh", requestType: string(models.ExecutorTypeKubernetes), + requestSession: "session-new", runtime: agentruntime.RuntimeKubernetes, + }, + { + name: "missing request Kubernetes session starts fresh", requestType: string(models.ExecutorTypeKubernetes), + runtime: agentruntime.RuntimeKubernetes, runningSession: "session-old", + }, + { + name: "Kubernetes request rejects a mismatched recorded runtime", requestType: string(models.ExecutorTypeKubernetes), + requestSession: "session-1", runtime: agentruntime.RuntimeSSH, + runningSession: "session-1", + }, + { + name: "non-Kubernetes request rejects a recorded Kubernetes runtime", requestType: string(models.ExecutorTypeLocal), + requestSession: "session-1", runtime: agentruntime.RuntimeKubernetes, + runningSession: "session-1", + }, + { + name: "legacy runtime keeps cross-session environment reuse", requestType: string(models.ExecutorTypeLocal), + requestSession: "session-new", runtime: agentruntime.RuntimeStandalone, + runningSession: "session-old", wantPrevious: "execution-old", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + req := &LaunchAgentRequest{ + TaskID: "task-1", SessionID: test.requestSession, ExecutorType: test.requestType, + } + running := &models.ExecutorRunning{ + SessionID: test.runningSession, TaskID: "task-1", + AgentExecutionID: "execution-old", Runtime: test.runtime, + } + + applyExecutorRunningMetadata(req, running) + + if req.PreviousExecutionID != test.wantPrevious { + t.Fatalf("PreviousExecutionID = %q, want %q", req.PreviousExecutionID, test.wantPrevious) + } + }) + } +} + // TestApplyExecutorRunningMetadata_RequestKeysWin documents that an explicit // value already on the request (e.g. set by the caller from launch options) // is not overwritten by the previous ExecutorRunning record. This applies to diff --git a/apps/backend/internal/orchestrator/executor/executor_execute.go b/apps/backend/internal/orchestrator/executor/executor_execute.go index feecb68806..794c1cb277 100644 --- a/apps/backend/internal/orchestrator/executor/executor_execute.go +++ b/apps/backend/internal/orchestrator/executor/executor_execute.go @@ -101,16 +101,12 @@ func (e *Executor) resolveTaskSessionMCPProfile(ctx context.Context, taskID stri } // isContainerizedExecutor returns true for executor types that run agents in -// containers or remote sandboxes (Docker variants + Sprites). These are the -// same executors that need explicitly configured remote credentials and the -// kandev-managed feature branch propagated through env metadata. +// containers or remote sandboxes (Docker variants, Sprites, and Kubernetes). +// These are the same executors that need explicitly configured remote +// credentials and the kandev-managed feature branch propagated through env +// metadata. func isContainerizedExecutor(executorType string) bool { - switch models.ExecutorType(executorType) { - case models.ExecutorTypeLocalDocker, models.ExecutorTypeRemoteDocker, models.ExecutorTypeSprites: - return true - default: - return false - } + return models.IsContainerizedExecutorType(models.ExecutorType(executorType)) } // executorNeedsResolvedCredentials reports whether an executor runs the agent @@ -1159,6 +1155,13 @@ func (e *Executor) LaunchPreparedSession(ctx context.Context, task *v1.Task, ses if session.TaskID != task.ID { return nil, fmt.Errorf("session does not belong to task") } + if strings.TrimSpace(executorID) == "" { + // PrepareSession already persisted the selected executor. Launch calls + // that omit the option must retain that selection so the authoritative + // connection config reaches lifecycle instead of falling back to the + // workspace default (or an empty config). + executorID = strings.TrimSpace(session.ExecutorID) + } if opts.McpMode == "" { opts.McpMode, err = e.resolveTaskSessionMCPMode(ctx, task.ID, session, opts.StartAgent) if err != nil { @@ -1173,7 +1176,10 @@ func (e *Executor) LaunchPreparedSession(ctx context.Context, task *v1.Task, ses opts.McpProfile = &profileContext } - running, _ := e.repo.GetExecutorRunningBySessionID(ctx, sessionID) + running, runningErr := e.repo.GetExecutorRunningBySessionID(ctx, sessionID) + if runningErr != nil && !errors.Is(runningErr, models.ErrExecutorRunningNotFound) { + return nil, fmt.Errorf("load runtime inventory for session %q: %w", sessionID, runningErr) + } if running != nil && running.ExecutionProfileID != "" && running.ExecutionProfileID != agentProfileID { if running.AgentExecutionID != "" { @@ -1212,7 +1218,10 @@ func (e *Executor) LaunchPreparedSession(ctx context.Context, task *v1.Task, ses // execution was lost (e.g. backend restart). The full LaunchAgent path below // will create a new execution and lifecycle.persistExecutorRunning will // overwrite the stale row. - hasRunning, _ := e.repo.HasExecutorRunningRow(ctx, sessionID) + hasRunning, hasRunningErr := e.repo.HasExecutorRunningRow(ctx, sessionID) + if hasRunningErr != nil { + return nil, fmt.Errorf("check runtime inventory for session %q: %w", sessionID, hasRunningErr) + } if hasRunning { result, err := e.startAgentOnExistingWorkspace(ctx, task, session, prompt, startAgent, opts.McpMode, opts.Env, opts.TurnID) if !errors.Is(err, ErrStaleExecution) && !errors.Is(err, ErrAgentCommandMissing) { diff --git a/apps/backend/internal/orchestrator/executor/executor_interaction.go b/apps/backend/internal/orchestrator/executor/executor_interaction.go index f066b5e995..bf61e75b97 100644 --- a/apps/backend/internal/orchestrator/executor/executor_interaction.go +++ b/apps/backend/internal/orchestrator/executor/executor_interaction.go @@ -14,6 +14,7 @@ import ( "github.com/kandev/kandev/internal/agent/runtime/lifecycle" agentctlshared "github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared" "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" "go.uber.org/zap" @@ -624,12 +625,20 @@ func (e *Executor) SwitchModel(ctx context.Context, taskID, sessionID, newModel, e.logger.Debug("in-place model switch not available, falling back to agent restart", zap.String("session_id", sessionID)) - session, task, acpSessionID, existingRunning, err := e.prepareModelSwitch(ctx, taskID, sessionID) + session, task, acpSessionID, executionID, existingRunning, err := e.prepareModelSwitch(ctx, taskID, sessionID) if err != nil { return nil, err } - execConfig := e.resolveExecutorConfig(ctx, session.ExecutorID, task.WorkspaceID, nil) + execConfig, err := e.resolveModelSwitchExecutorConfig(ctx, task, session, existingRunning) + if err != nil { + return nil, err + } + if err := validateKubernetesModelSwitchAuthority( + task, session, executionID, existingRunning, execConfig, + ); err != nil { + return nil, err + } req, err := e.buildSwitchModelRequest(ctx, task, session, sessionID, newModel, prompt, acpSessionID, execConfig, existingRunning) if err != nil { @@ -637,6 +646,9 @@ func (e *Executor) SwitchModel(ctx context.Context, taskID, sessionID, newModel, } req.Env = e.applyPreferredShellEnv(ctx, req.ExecutorType, req.Env) + if err := e.stopPreparedModelSwitchAgent(ctx, executionID); err != nil { + return nil, err + } e.logger.Info("launching new agent with model override", zap.String("task_id", task.ID), @@ -659,43 +671,153 @@ func (e *Executor) SwitchModel(ctx context.Context, taskID, sessionID, newModel, }, nil } -// prepareModelSwitch validates the session/task and stops the current agent. -// Returns the session, task, ACP session ID, existing ExecutorRunning record, and any error. -func (e *Executor) prepareModelSwitch(ctx context.Context, taskID, sessionID string) (*models.TaskSession, *models.Task, string, *models.ExecutorRunning, error) { +// prepareModelSwitch loads the authoritative state needed to build and +// validate a replacement request. It deliberately does not stop the current +// agent; Kubernetes must prove exact recorded authority first. +func (e *Executor) prepareModelSwitch(ctx context.Context, taskID, sessionID string) (*models.TaskSession, *models.Task, string, string, *models.ExecutorRunning, error) { session, err := e.repo.GetTaskSession(ctx, sessionID) if err != nil { - return nil, nil, "", nil, fmt.Errorf("failed to get session: %w", err) + return nil, nil, "", "", nil, fmt.Errorf("failed to get session: %w", err) } if session.TaskID != taskID { - return nil, nil, "", nil, fmt.Errorf("session %s does not belong to task %s", sessionID, taskID) + return nil, nil, "", "", nil, fmt.Errorf("session %s does not belong to task %s", sessionID, taskID) } executionID, err := e.agentManager.GetExecutionIDForSession(ctx, sessionID) if err != nil || executionID == "" { - return nil, nil, "", nil, ErrExecutionNotFound + return nil, nil, "", "", nil, ErrExecutionNotFound } task, err := e.repo.GetTask(ctx, taskID) if err != nil { - return nil, nil, "", nil, fmt.Errorf("failed to get task: %w", err) + return nil, nil, "", "", nil, fmt.Errorf("failed to get task: %w", err) } - var acpSessionID string - var existingRunning *models.ExecutorRunning - if running, runErr := e.repo.GetExecutorRunningBySessionID(ctx, sessionID); runErr == nil && running != nil { - existingRunning = running - acpSessionID = running.ResumeToken + existingRunning, runErr := e.repo.GetExecutorRunningBySessionID(ctx, sessionID) + if runErr != nil && !errors.Is(runErr, models.ErrExecutorRunningNotFound) { + probe := e.resolveExecutorConfig( + ctx, session.ExecutorID, task.WorkspaceID, cloneMetadata(session.Metadata), + ) + if models.ExecutorType(probe.ExecutorType) == models.ExecutorTypeKubernetes || + hasKubernetesRuntimeMetadata(session.Metadata) { + return nil, nil, "", "", nil, fmt.Errorf("load model-switch runtime inventory: %w", runErr) + } + existingRunning = nil } + acpSessionID := "" + if existingRunning != nil { + acpSessionID = existingRunning.ResumeToken + } + return session, task, acpSessionID, executionID, existingRunning, nil +} +func hasKubernetesRuntimeMetadata(metadata map[string]interface{}) bool { + for key := range metadata { + if strings.HasPrefix(key, "kubernetes_") { + return true + } + } + return false +} + +func (e *Executor) resolveModelSwitchExecutorConfig( + ctx context.Context, + task *models.Task, + session *models.TaskSession, + running *models.ExecutorRunning, +) (executorConfig, error) { + if running == nil || running.Runtime != agentruntime.RuntimeKubernetes { + return e.resolveExecutorConfig(ctx, session.ExecutorID, task.WorkspaceID, nil), nil + } + metadata := cloneMetadata(session.Metadata) + if metadata == nil { + metadata = make(map[string]interface{}) + } + if session.ExecutorProfileID != "" { + metadata[lifecycle.MetadataKeyExecutorProfileID] = session.ExecutorProfileID + } + applyAuthoritativeKubernetesRunningMetadata(metadata, running.Metadata) + config, err := e.applyRecordedKubernetesExecutorConfigToResumeRequest( + ctx, &LaunchAgentRequest{}, session, metadata, running, + ) + if err != nil { + return executorConfig{}, err + } + return config, nil +} + +func validateKubernetesModelSwitchAuthority( + task *models.Task, + session *models.TaskSession, + executionID string, + running *models.ExecutorRunning, + config executorConfig, +) error { + resolvedKubernetes := models.ExecutorType(config.ExecutorType) == models.ExecutorTypeKubernetes + recordedKubernetes := running != nil && running.Runtime == agentruntime.RuntimeKubernetes + if !resolvedKubernetes && !recordedKubernetes { + return nil + } + if err := validateKubernetesModelSwitchRunningIdentity(task, session, executionID, running); err != nil { + return err + } + return validateKubernetesModelSwitchConfig(task, session, running, config) +} + +func validateKubernetesModelSwitchRunningIdentity( + task *models.Task, + session *models.TaskSession, + executionID string, + running *models.ExecutorRunning, +) error { + if running == nil { + return errors.New("switch Kubernetes model: authoritative running inventory is missing") + } + if running.Runtime != agentruntime.RuntimeKubernetes { + return fmt.Errorf("switch Kubernetes model: recorded runtime is %q", running.Runtime) + } + if running.TaskID == "" || running.TaskID != task.ID { + return errors.New("switch Kubernetes model: recorded task identity does not match") + } + if running.SessionID == "" || running.SessionID != session.ID { + return errors.New("switch Kubernetes model: recorded session identity does not match") + } + if running.AgentExecutionID == "" || running.AgentExecutionID != executionID { + return errors.New("switch Kubernetes model: recorded execution identity does not match") + } + return nil +} + +func validateKubernetesModelSwitchConfig( + task *models.Task, + session *models.TaskSession, + running *models.ExecutorRunning, + config executorConfig, +) error { + if config.ExecutorType != string(models.ExecutorTypeKubernetes) || + config.ExecutorID == "" || config.ExecutorID != running.ExecutorID { + return errors.New("switch Kubernetes model: recorded executor is not an available Kubernetes executor") + } + if err := lifecycle.ValidateKubernetesResumeMetadata( + config.Metadata, task.ID, session.ID, config.ExecutorCfg, + ); err != nil { + return fmt.Errorf("switch Kubernetes model: validate recorded runtime: %w", err) + } + return nil +} + +func (e *Executor) stopPreparedModelSwitchAgent( + ctx context.Context, + executionID string, +) error { e.logger.Info("stopping current agent for model switch", zap.String("agent_execution_id", executionID)) if err := e.agentManager.StopAgent(ctx, executionID, false); err != nil { e.logger.Warn("failed to stop agent for model switch", zap.Error(err), zap.String("agent_execution_id", executionID)) - return nil, nil, "", nil, fmt.Errorf("failed to stop agent for model switch: %w", err) + return fmt.Errorf("failed to stop agent for model switch: %w", err) } - - return session, task, acpSessionID, existingRunning, nil + return nil } // launchModelSwitchAgent launches the new agent, persists state, and starts the process. @@ -752,11 +874,16 @@ func (e *Executor) buildSwitchModelRequest(ctx context.Context, task *models.Tas ModelOverride: newModel, ACPSessionID: acpSessionID, ExecutorType: execConfig.ExecutorType, + ExecutorConfig: execConfig.ExecutorCfg, Metadata: execConfig.Metadata, + SetupScript: execConfig.SetupScript, IsEphemeral: task.IsEphemeral, IsPassthrough: session.IsPassthrough, TaskEnvironmentID: session.TaskEnvironmentID, } + if running != nil && running.Runtime == agentruntime.RuntimeKubernetes { + req.PreviousExecutionID = running.AgentExecutionID + } mcpMode, err := e.resolveTaskSessionMCPMode(ctx, task.ID, session, true) if err != nil { diff --git a/apps/backend/internal/orchestrator/executor/executor_interaction_test.go b/apps/backend/internal/orchestrator/executor/executor_interaction_test.go index 3271992b70..85300ac615 100644 --- a/apps/backend/internal/orchestrator/executor/executor_interaction_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_interaction_test.go @@ -485,32 +485,24 @@ func TestStopExecution_PreservesRuntimeFailureClassification(t *testing.T) { } } -func TestPrepareModelSwitch_StopsBeforeReplacementWhenTeardownFails(t *testing.T) { +func TestStopPreparedModelSwitchAgent_ReturnsTeardownFailure(t *testing.T) { stopErr := errors.New("runtime teardown failed") - repo := newMockRepository() - repo.sessions["session-model-switch"] = &models.TaskSession{ - ID: "session-model-switch", TaskID: "task-model-switch", - } - repo.tasks["task-model-switch"] = &models.Task{ID: "task-model-switch"} manager := &mockAgentManager{ - getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { - return "execution-model-switch", nil - }, - stopAgentFunc: func(context.Context, string, bool) error { + stopAgentFunc: func(_ context.Context, executionID string, force bool) error { + if executionID != "execution-model-switch" || force { + t.Fatalf("StopAgent = (%q, %v), want (execution-model-switch, false)", executionID, force) + } return stopErr }, } - exec := newTestExecutor(t, manager, repo) + exec := newTestExecutor(t, manager, newMockRepository()) - session, task, acpSessionID, existingRunning, err := exec.prepareModelSwitch( - context.Background(), "task-model-switch", "session-model-switch", + err := exec.stopPreparedModelSwitchAgent( + context.Background(), "execution-model-switch", ) if !errors.Is(err, stopErr) { - t.Fatalf("prepareModelSwitch error = %v, want %v", err, stopErr) - } - if session != nil || task != nil || acpSessionID != "" || existingRunning != nil { - t.Fatalf("prepareModelSwitch returned replacement inputs after stop failure: session=%#v task=%#v acp=%q running=%#v", session, task, acpSessionID, existingRunning) + t.Fatalf("stopPreparedModelSwitchAgent error = %v, want %v", err, stopErr) } } diff --git a/apps/backend/internal/orchestrator/executor/executor_kubernetes_model_switch_test.go b/apps/backend/internal/orchestrator/executor/executor_kubernetes_model_switch_test.go new file mode 100644 index 0000000000..3dc9ad9e35 --- /dev/null +++ b/apps/backend/internal/orchestrator/executor/executor_kubernetes_model_switch_test.go @@ -0,0 +1,292 @@ +package executor + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +func TestSwitchModelFallbackReusesAuthoritativeKubernetesRuntime(t *testing.T) { + repo := newMockRepository() + repo.tasks["task-k8s"] = &models.Task{ + ID: "task-k8s", WorkspaceID: "workspace-1", Title: "Kubernetes task", + } + repo.sessions["session-k8s"] = &models.TaskSession{ + ID: "session-k8s", TaskID: "task-k8s", AgentProfileID: "agent-profile", + ExecutorID: "repointed-executor", ExecutorProfileID: "edited-profile", + State: models.TaskSessionStateRunning, + Metadata: map[string]interface{}{ + lifecycle.MetadataKeyKubernetesPodUID: "hostile-pod-uid", + lifecycle.MetadataKeyKubernetesProfileSnapshot: "hostile-snapshot", + }, + } + wantConfig := map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/current-kubeconfig", + lifecycle.MetadataKeyKubernetesKubeContext: "current-context", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + } + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: wantConfig, + } + repo.executors["repointed-executor"] = &models.Executor{ + ID: "repointed-executor", Type: models.ExecutorTypeLocal, + Status: models.ExecutorStatusActive, + } + repo.executorsRunning["session-k8s"] = &models.ExecutorRunning{ + ID: "running-k8s", SessionID: "session-k8s", TaskID: "task-k8s", + ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", ResumeToken: "resume-token", + Metadata: recordedKubernetesResumeMetadataFor("task-k8s", "session-k8s", "recorded-execution"), + } + var launched *LaunchAgentRequest + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "recorded-execution", nil + }, + launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) { + launched = req + return &LaunchAgentResponse{AgentExecutionID: "replacement-execution"}, nil + }, + } + exec := newTestExecutor(t, manager, repo) + + _, err := exec.SwitchModel( + context.Background(), "task-k8s", "session-k8s", "new-model", "continue", + ) + + if err != nil { + t.Fatalf("SwitchModel() error = %v", err) + } + if launched == nil { + t.Fatal("Kubernetes model-switch fallback did not launch") + } + if launched.ExecutorType != string(models.ExecutorTypeKubernetes) { + t.Fatalf("ExecutorType = %q, want k8s", launched.ExecutorType) + } + if launched.PreviousExecutionID != "recorded-execution" { + t.Fatalf("PreviousExecutionID = %q, want recorded-execution", launched.PreviousExecutionID) + } + if got := launched.Metadata[lifecycle.MetadataKeyKubernetesPodUID]; got != "recorded-pod-uid" { + t.Fatalf("Pod UID = %v, want recorded-pod-uid", got) + } + if got := launched.Metadata[lifecycle.MetadataKeyKubernetesProfileSnapshot]; got != repo.executorsRunning["session-k8s"].Metadata[lifecycle.MetadataKeyKubernetesProfileSnapshot] { + t.Fatalf("profile snapshot = %v, want recorded workload snapshot", got) + } + if fmt.Sprint(launched.ExecutorConfig) != fmt.Sprint(wantConfig) { + t.Fatalf("ExecutorConfig = %#v, want %#v", launched.ExecutorConfig, wantConfig) + } +} + +func TestSwitchModelFallbackStopsOnKubernetesTeardownFailure(t *testing.T) { + repo := newMockRepository() + repo.tasks["task-k8s"] = &models.Task{ID: "task-k8s", WorkspaceID: "workspace-1"} + repo.sessions["session-k8s"] = &models.TaskSession{ + ID: "session-k8s", TaskID: "task-k8s", AgentProfileID: "agent-profile", + ExecutorID: "recorded-kubernetes-executor", + State: models.TaskSessionStateRunning, + } + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "30", + }, + } + repo.executorsRunning["session-k8s"] = &models.ExecutorRunning{ + SessionID: "session-k8s", TaskID: "task-k8s", Runtime: agentruntime.RuntimeKubernetes, + ExecutorID: "recorded-kubernetes-executor", AgentExecutionID: "recorded-execution", + Metadata: recordedKubernetesResumeMetadataFor("task-k8s", "session-k8s", "recorded-execution"), + } + teardownErr := errors.New("close Kubernetes forward failed") + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "recorded-execution", nil + }, + stopAgentFunc: func(context.Context, string, bool) error { return teardownErr }, + } + exec := newTestExecutor(t, manager, repo) + + _, err := exec.SwitchModel( + context.Background(), "task-k8s", "session-k8s", "new-model", "continue", + ) + + if !errors.Is(err, teardownErr) { + t.Fatalf("SwitchModel() error = %v, want teardown cause", err) + } + if manager.launchAgentCallCount != 0 { + t.Fatalf("LaunchAgent calls = %d, want zero after failed Kubernetes teardown", manager.launchAgentCallCount) + } +} + +func TestSwitchModelFallbackValidatesKubernetesAuthorityBeforeStop(t *testing.T) { + readErr := errors.New("running inventory unavailable") + tests := []struct { + name string + mutate func(*mockRepository) + }{ + { + name: "running row read failure", + mutate: func(repo *mockRepository) { + repo.getExecutorRunningFunc = func(context.Context, string) (*models.ExecutorRunning, error) { + return nil, readErr + } + }, + }, + {name: "missing Kubernetes row", mutate: func(repo *mockRepository) { delete(repo.executorsRunning, "session-k8s") }}, + {name: "task mismatch", mutate: func(repo *mockRepository) { repo.executorsRunning["session-k8s"].TaskID = "other-task" }}, + {name: "session mismatch", mutate: func(repo *mockRepository) { repo.executorsRunning["session-k8s"].SessionID = "other-session" }}, + {name: "execution mismatch", mutate: func(repo *mockRepository) { repo.executorsRunning["session-k8s"].AgentExecutionID = "other-execution" }}, + {name: "wrong recorded runtime", mutate: func(repo *mockRepository) { repo.executorsRunning["session-k8s"].Runtime = agentruntime.RuntimeSSH }}, + {name: "missing recorded executor", mutate: func(repo *mockRepository) { repo.executorsRunning["session-k8s"].ExecutorID = "deleted-executor" }}, + {name: "recorded executor changed type", mutate: func(repo *mockRepository) { + repo.executors["recorded-kubernetes-executor"].Type = models.ExecutorTypeLocal + }}, + {name: "incomplete workload snapshot", mutate: func(repo *mockRepository) { + delete(repo.executorsRunning["session-k8s"].Metadata, lifecycle.MetadataKeyKubernetesProfileSnapshot) + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := newMockRepository() + repo.tasks["task-k8s"] = &models.Task{ID: "task-k8s", WorkspaceID: "workspace-1"} + repo.sessions["session-k8s"] = &models.TaskSession{ + ID: "session-k8s", TaskID: "task-k8s", AgentProfileID: "agent-profile", + ExecutorID: "recorded-kubernetes-executor", State: models.TaskSessionStateRunning, + } + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "30", + }, + } + repo.executorsRunning["session-k8s"] = &models.ExecutorRunning{ + ID: "running-k8s", SessionID: "session-k8s", TaskID: "task-k8s", + ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", ResumeToken: "resume-token", + Metadata: recordedKubernetesResumeMetadataFor("task-k8s", "session-k8s", "recorded-execution"), + } + test.mutate(repo) + stopCalls := 0 + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "recorded-execution", nil + }, + stopAgentFunc: func(context.Context, string, bool) error { stopCalls++; return nil }, + } + exec := newTestExecutor(t, manager, repo) + + _, err := exec.SwitchModel(context.Background(), "task-k8s", "session-k8s", "new-model", "continue") + + if err == nil { + t.Fatal("SwitchModel() error = nil, want fail-closed Kubernetes authority error") + } + if stopCalls != 0 { + t.Fatalf("StopAgent calls = %d, want zero before Kubernetes request validation", stopCalls) + } + if manager.launchAgentCallCount != 0 { + t.Fatalf("LaunchAgent calls = %d, want zero", manager.launchAgentCallCount) + } + }) + } +} + +func TestSwitchModelFallbackKeepsLegacyBehaviorWhenNonKubernetesInventoryReadFails(t *testing.T) { + repo := newMockRepository() + repo.tasks["task-local"] = &models.Task{ID: "task-local", WorkspaceID: "workspace-1"} + repo.sessions["session-local"] = &models.TaskSession{ + ID: "session-local", TaskID: "task-local", AgentProfileID: "agent-profile", + ExecutorID: "local-executor", State: models.TaskSessionStateRunning, + } + repo.executors["local-executor"] = &models.Executor{ + ID: "local-executor", Type: models.ExecutorTypeLocal, Status: models.ExecutorStatusActive, + } + repo.getExecutorRunningFunc = func(context.Context, string) (*models.ExecutorRunning, error) { + return nil, errors.New("running inventory unavailable") + } + stopCalls := 0 + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "local-execution", nil + }, + stopAgentFunc: func(context.Context, string, bool) error { stopCalls++; return nil }, + } + exec := newTestExecutor(t, manager, repo) + + _, err := exec.SwitchModel(context.Background(), "task-local", "session-local", "new-model", "continue") + + if err != nil { + t.Fatalf("SwitchModel() error = %v", err) + } + if stopCalls != 1 || manager.launchAgentCallCount != 1 { + t.Fatalf("stop/launch calls = %d/%d, want 1/1", stopCalls, manager.launchAgentCallCount) + } +} + +// Reviewer-requested contract coverage: request construction remains before +// destructive Kubernetes stop, so any late resolver error leaves live runtime intact. +func TestSwitchModelFallbackBuildFailureLeavesKubernetesAgentRunning(t *testing.T) { + repo := newMockRepository() + task := &models.Task{ID: "task-k8s", WorkspaceID: "workspace-1"} + repo.tasks[task.ID] = task + repo.sessions["session-k8s"] = &models.TaskSession{ + ID: "session-k8s", TaskID: task.ID, AgentProfileID: "agent-profile", + ExecutorID: "recorded-kubernetes-executor", State: models.TaskSessionStateRunning, + } + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "30", + }, + } + repo.executorsRunning["session-k8s"] = &models.ExecutorRunning{ + ID: "running-k8s", SessionID: "session-k8s", TaskID: task.ID, + ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", ResumeToken: "resume-token", + Metadata: recordedKubernetesResumeMetadataFor(task.ID, "session-k8s", "recorded-execution"), + } + buildErr := errors.New("MCP request projection unavailable") + taskReads := 0 + repo.getTaskFunc = func(context.Context, string) (*models.Task, error) { + taskReads++ + if taskReads > 1 { + return nil, buildErr + } + return task, nil + } + stopCalls := 0 + manager := &mockAgentManager{ + getExecutionIDForSessionFunc: func(context.Context, string) (string, error) { + return "recorded-execution", nil + }, + stopAgentFunc: func(context.Context, string, bool) error { stopCalls++; return nil }, + } + exec := newTestExecutor(t, manager, repo) + + _, err := exec.SwitchModel(context.Background(), task.ID, "session-k8s", "new-model", "continue") + + if !errors.Is(err, buildErr) { + t.Fatalf("SwitchModel() error = %v, want request-build cause", err) + } + if stopCalls != 0 { + t.Fatalf("StopAgent calls = %d, want zero before complete request build", stopCalls) + } + if manager.launchAgentCallCount != 0 { + t.Fatalf("LaunchAgent calls = %d, want zero", manager.launchAgentCallCount) + } +} diff --git a/apps/backend/internal/orchestrator/executor/executor_kubernetes_parity_test.go b/apps/backend/internal/orchestrator/executor/executor_kubernetes_parity_test.go new file mode 100644 index 0000000000..13c0729967 --- /dev/null +++ b/apps/backend/internal/orchestrator/executor/executor_kubernetes_parity_test.go @@ -0,0 +1,80 @@ +package executor + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/task/models" + v1 "github.com/kandev/kandev/pkg/api/v1" +) + +func TestKubernetesExecutorUsesRemoteContainerCredentialPolicy(t *testing.T) { + executorType := string(models.ExecutorTypeKubernetes) + if !isContainerizedExecutor(executorType) { + t.Fatal("Kubernetes executor must be classified as containerized") + } + if !executorNeedsResolvedCredentials(executorType) { + t.Fatal("Kubernetes executor must resolve selected profile credentials") + } + if shouldUseWorktree(executorType) { + t.Fatal("Kubernetes executor must not use host git worktrees") + } +} + +func TestKubernetesCredentialBrokerRequiresHTTPS(t *testing.T) { + err := validateGitHubCredentialBrokerURL( + "http://127.0.0.1:8080/api/github/credentials/resolve", + string(models.ExecutorTypeKubernetes), + ) + if !errors.Is(err, ErrGitHubCredentialBrokerURL) { + t.Fatalf("validateGitHubCredentialBrokerURL() error = %v, want broker URL error", err) + } +} + +func TestKubernetesProfileTokenSkipsManagedCredentialPreflight(t *testing.T) { + repo := newMockRepository() + seedPreflightTaskRepository(repo, "task-1", "repo-1", &models.Repository{ + ID: "repo-1", SourceType: sourceTypeLocal, Provider: "acme-forge", + RemoteURL: "https://forge.example/acme/widgets.git", + }) + repo.executors["exec-k8s"] = &models.Executor{ + ID: "exec-k8s", Type: models.ExecutorTypeKubernetes, + } + repo.executorProfiles["profile-token"] = &models.ExecutorProfile{ + ID: "profile-token", ExecutorID: "exec-k8s", + Config: map[string]string{profileKeyRemoteAuthSecrets: `{"gh_cli_env":"secret-gh"}`}, + } + exec := newPreflightTestExecutor(t, repo) + + task := &v1.Task{ID: "task-1", WorkspaceID: "workspace-1"} + if _, err := exec.PrepareSession( + context.Background(), task, "agent-profile", "exec-k8s", "profile-token", "", + ); err != nil { + t.Fatalf("PrepareSession() error = %v, want Kubernetes profile token override", err) + } + if len(repo.createTaskSessionCalls) != 1 { + t.Fatalf("CreateTaskSession calls = %d, want 1", len(repo.createTaskSessionCalls)) + } +} + +func TestKubernetesEnvironmentReuseCarriesFeatureBranch(t *testing.T) { + exec := newEnvTestExecutor(t) + req := &LaunchAgentRequest{ + TaskID: "task-1", + ExecutorType: string(models.ExecutorTypeKubernetes), + } + env := &models.TaskEnvironment{ + ExecutorType: string(models.ExecutorTypeKubernetes), + Repos: []*models.TaskEnvironmentRepo{ + {WorktreeBranch: "feature/existing-task-abc"}, + }, + } + + exec.reuseExistingEnvironment(context.Background(), req, env) + + if got := req.Metadata[lifecycle.MetadataKeyWorktreeBranch]; got != "feature/existing-task-abc" { + t.Fatalf("metadata worktree_branch = %v, want existing branch", got) + } +} diff --git a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go index a7885ba0f3..0535bb5e58 100644 --- a/apps/backend/internal/orchestrator/executor/executor_mocks_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_mocks_test.go @@ -280,6 +280,8 @@ type mockRepository struct { getTaskFunc func(ctx context.Context, id string) (*models.Task, error) getTaskEnvironmentFunc func(ctx context.Context, id string) (*models.TaskEnvironment, error) getTaskEnvironmentByTaskIDFunc func(ctx context.Context, taskID string) (*models.TaskEnvironment, error) + getExecutorRunningFunc func(ctx context.Context, sessionID string) (*models.ExecutorRunning, error) + hasExecutorRunningFunc func(ctx context.Context, sessionID string) (bool, error) createTaskEnvironmentRepoErr error finalizeTaskEnvironmentErr error createTaskSessionFunc func(ctx context.Context, session *models.TaskSession) error @@ -1128,6 +1130,9 @@ func (m *mockRepository) ListExecutorsRunning(ctx context.Context) ([]*models.Ex return nil, nil } func (m *mockRepository) GetExecutorRunningBySessionID(ctx context.Context, sessionID string) (*models.ExecutorRunning, error) { + if m.getExecutorRunningFunc != nil { + return m.getExecutorRunningFunc(ctx, sessionID) + } if running, ok := m.executorsRunning[sessionID]; ok { return running, nil } @@ -1137,6 +1142,9 @@ func (m *mockRepository) DeleteExecutorRunningBySessionID(ctx context.Context, s return nil } func (m *mockRepository) HasExecutorRunningRow(ctx context.Context, sessionID string) (bool, error) { + if m.hasExecutorRunningFunc != nil { + return m.hasExecutorRunningFunc(ctx, sessionID) + } if m.executorsRunning != nil { _, ok := m.executorsRunning[sessionID] return ok, nil diff --git a/apps/backend/internal/orchestrator/executor/executor_resume.go b/apps/backend/internal/orchestrator/executor/executor_resume.go index 8e3c59e503..b3f7e98cec 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume.go @@ -11,6 +11,7 @@ import ( "time" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/common/gitref" "github.com/kandev/kandev/internal/integrations/cloneauth" "github.com/kandev/kandev/internal/repoclone" @@ -1141,7 +1142,33 @@ func (e *Executor) buildResumeRequestAtCredentialBoundaryWithOptions( options ResumeOptions, ) (*LaunchAgentRequest, string, executorConfig, *models.TaskEnvironment, *models.ExecutorRunning, error) { req, metadata := newResumeLaunchRequest(task, session, startAgent, options) - execConfig := e.applyExecutorConfigToResumeRequest(ctx, req, task, session, metadata) + existingRunning, runningErr := e.repo.GetExecutorRunningBySessionID(ctx, session.ID) + if runningErr != nil && !errors.Is(runningErr, models.ErrExecutorRunningNotFound) { + return nil, "", executorConfig{}, nil, nil, + fmt.Errorf("load runtime inventory for session %q: %w", session.ID, runningErr) + } + recordedKubernetes := existingRunning != nil && existingRunning.Runtime == agentruntime.RuntimeKubernetes + if recordedKubernetes { + applyAuthoritativeKubernetesRunningMetadata(metadata, existingRunning.Metadata) + } + var execConfig executorConfig + if recordedKubernetes { + var configErr error + execConfig, configErr = e.applyRecordedKubernetesExecutorConfigToResumeRequest( + ctx, req, session, metadata, existingRunning, + ) + if configErr != nil { + return nil, "", executorConfig{}, nil, existingRunning, configErr + } + if err := lifecycle.ValidateKubernetesResumeMetadata( + metadata, task.ID, session.ID, execConfig.ExecutorCfg, + ); err != nil { + return nil, "", executorConfig{}, nil, existingRunning, + fmt.Errorf("validate recorded Kubernetes runtime for resume: %w", err) + } + } else { + execConfig = e.applyExecutorConfigToResumeRequest(ctx, req, task, session, metadata) + } repositoryID, existingEnv, allRepos, err := e.prepareResumeRepositorySettings( ctx, task, session, req, ) @@ -1152,7 +1179,7 @@ func (e *Executor) buildResumeRequestAtCredentialBoundaryWithOptions( return nil, "", execConfig, existingEnv, nil, err } - existingRunning := e.applyRunningRecordToResumeRequest(ctx, req, task, session, startAgent) + existingRunning = e.applyRunningRecordToResumeRequest(req, task, session, startAgent, existingRunning) if err := e.applyResumeWorkspaceFolders(ctx, task.ID, req); err != nil { return nil, "", execConfig, existingEnv, existingRunning, err } @@ -1281,6 +1308,60 @@ func (e *Executor) applyResumeMCPSettings( return nil } +func applyAuthoritativeKubernetesRunningMetadata( + metadata map[string]interface{}, + recorded map[string]interface{}, +) { + for key := range metadata { + if isKubernetesRecordedMetadataKey(key) { + delete(metadata, key) + } + } + for key, value := range recorded { + if isKubernetesRecordedMetadataKey(key) && lifecycle.ShouldPersistMetadataKey(key) { + metadata[key] = value + } + } +} + +func isKubernetesRecordedMetadataKey(key string) bool { + return strings.HasPrefix(key, "kubernetes_") || + key == lifecycle.MetadataKeyIsRemote || + key == lifecycle.MetadataKeyExecutorProfileID || + key == lifecycle.MetadataKeyAuthTokenSecret || + key == lifecycle.MetadataKeyBootstrapNonceSecret +} + +func (e *Executor) applyRecordedKubernetesExecutorConfigToResumeRequest( + ctx context.Context, + req *LaunchAgentRequest, + session *models.TaskSession, + metadata map[string]interface{}, + running *models.ExecutorRunning, +) (executorConfig, error) { + executorID := strings.TrimSpace(running.ExecutorID) + if executorID == "" { + return executorConfig{}, errors.New("resume Kubernetes executor: recorded executor ID is missing") + } + current, err := e.repo.GetExecutor(ctx, executorID) + if err != nil { + return executorConfig{}, fmt.Errorf("resume Kubernetes executor %q: %w", executorID, err) + } + if current == nil || current.Type != models.ExecutorTypeKubernetes { + return executorConfig{}, fmt.Errorf("resume Kubernetes executor %q: current Kubernetes executor is unavailable", executorID) + } + metadata["executor_id"] = executorID + config := executorConfig{ + ExecutorID: executorID, ExecutorType: string(current.Type), ExecutorCfg: current.Config, + Metadata: metadata, Resumable: current.Resumable, RuntimeName: string(current.Type), + } + session.ExecutorID = executorID + req.ExecutorType = config.ExecutorType + req.ExecutorConfig = config.ExecutorCfg + req.Metadata = metadata + return config, nil +} + func (e *Executor) applyResumeWorkspaceFolders( ctx context.Context, taskID string, @@ -1417,9 +1498,14 @@ func isArchiveCancelledResumeSession(session *models.TaskSession) bool { // applyRunningRecordToResumeRequest loads the ExecutorRunning record and applies // resume-related fields (remote reconnect, resume token) to the request. -func (e *Executor) applyRunningRecordToResumeRequest(ctx context.Context, req *LaunchAgentRequest, task *v1.Task, session *models.TaskSession, startAgent bool) *models.ExecutorRunning { - running, runErr := e.repo.GetExecutorRunningBySessionID(ctx, session.ID) - if runErr != nil || running == nil { +func (e *Executor) applyRunningRecordToResumeRequest( + req *LaunchAgentRequest, + task *v1.Task, + session *models.TaskSession, + startAgent bool, + running *models.ExecutorRunning, +) *models.ExecutorRunning { + if running == nil { // Archive cleanup tears down the executors_running row entirely, so an // archive-cancelled session reaches this point with running == nil — // exactly the shape GetTaskSessionStatus's resumeReasonArchiveCancelledResumable diff --git a/apps/backend/internal/orchestrator/executor/executor_resume_test.go b/apps/backend/internal/orchestrator/executor/executor_resume_test.go index acee6ea8c1..ea6dadcf2c 100644 --- a/apps/backend/internal/orchestrator/executor/executor_resume_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_resume_test.go @@ -2,6 +2,7 @@ package executor import ( "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -9,7 +10,9 @@ import ( "testing" "time" + kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/gitcredentials" "github.com/kandev/kandev/internal/task/models" v1 "github.com/kandev/kandev/pkg/api/v1" @@ -106,6 +109,194 @@ func setupLiveResumeTestFixture(repo *mockRepository) { } } +func TestBuildResumeRequestUsesAuthoritativeKubernetesRunningRecord(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + session := repo.sessions["sess-1"] + session.ExecutorID = "repointed-executor" + session.ExecutorProfileID = "deleted-or-invalid-profile" + session.Metadata = map[string]interface{}{ + lifecycle.MetadataKeyKubernetesPodUID: "hostile-pod-uid", + lifecycle.MetadataKeyKubernetesProfileSnapshot: "hostile-snapshot", + lifecycle.MetadataKeyKubernetesNamespace: "hostile-namespace", + lifecycle.MetadataKeyAuthTokenSecret: "stale-session-auth-secret", + lifecycle.MetadataKeyBootstrapNonceSecret: "stale-session-nonce-secret", + } + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, Resumable: true, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/current-kubeconfig", + lifecycle.MetadataKeyKubernetesKubeContext: "current-context", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + } + repo.executors["repointed-executor"] = &models.Executor{ + ID: "repointed-executor", Type: models.ExecutorTypeLocal, + } + repo.executorsRunning["sess-1"] = &models.ExecutorRunning{ + ID: "sess-1", SessionID: "sess-1", TaskID: "task-1", + ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", Metadata: recordedKubernetesResumeMetadata(), + } + exec := newTestExecutor(t, &mockAgentManager{}, repo) + + req, _, config, _, running, err := exec.buildResumeRequest( + context.Background(), &v1.Task{ID: "task-1", WorkspaceID: "workspace-1"}, session, true, + ) + + if err != nil { + t.Fatalf("buildResumeRequest() error = %v", err) + } + if running != repo.executorsRunning["sess-1"] { + t.Fatal("buildResumeRequest() did not retain the authoritative running row") + } + if req.ExecutorType != string(models.ExecutorTypeKubernetes) || config.ExecutorID != "recorded-kubernetes-executor" { + t.Fatalf("executor = type %q id %q", req.ExecutorType, config.ExecutorID) + } + if req.PreviousExecutionID != "recorded-execution" { + t.Fatalf("PreviousExecutionID = %q", req.PreviousExecutionID) + } + if got := req.Metadata[lifecycle.MetadataKeyKubernetesPodUID]; got != "recorded-pod-uid" { + t.Fatalf("Pod UID = %v, want recorded-pod-uid", got) + } + if got := req.Metadata[lifecycle.MetadataKeyKubernetesProfileSnapshot]; got != recordedKubernetesResumeMetadata()[lifecycle.MetadataKeyKubernetesProfileSnapshot] { + t.Fatalf("snapshot = %v, want recorded workload snapshot", got) + } + if got := req.Metadata[lifecycle.MetadataKeyExecutorProfileID]; got != "recorded-profile" { + t.Fatalf("executor profile = %v, want recorded-profile", got) + } + if got := req.Metadata[lifecycle.MetadataKeyAuthTokenSecret]; got != "recorded-auth-secret" { + t.Fatalf("auth secret reference = %v, want recorded-auth-secret", got) + } + if got := req.Metadata[lifecycle.MetadataKeyBootstrapNonceSecret]; got != "recorded-nonce-secret" { + t.Fatalf("nonce secret reference = %v, want recorded-nonce-secret", got) + } + if config.ExecutorCfg[lifecycle.MetadataKeyKubernetesKubeContext] != "current-context" || + config.ExecutorCfg[lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds] != "45" { + t.Fatalf("current connection config was not retained: %#v", config.ExecutorCfg) + } + if session.ExecutorID != "recorded-kubernetes-executor" { + t.Fatalf("session executor = %q, want recorded Kubernetes executor", session.ExecutorID) + } +} + +func TestBuildResumeRequestFailsClosedWhenRecordedKubernetesExecutorChangedType(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + session := repo.sessions["sess-1"] + session.ExecutorID = "repointed-executor" + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeLocal, + } + repo.executorsRunning["sess-1"] = &models.ExecutorRunning{ + ID: "sess-1", SessionID: "sess-1", TaskID: "task-1", + ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", Metadata: recordedKubernetesResumeMetadata(), + } + exec := newTestExecutor(t, &mockAgentManager{}, repo) + + _, _, _, _, _, err := exec.buildResumeRequest( + context.Background(), &v1.Task{ID: "task-1", WorkspaceID: "workspace-1"}, session, true, + ) + + if err == nil || !strings.Contains(err.Error(), "current Kubernetes executor is unavailable") { + t.Fatalf("buildResumeRequest() error = %v", err) + } +} + +func TestBuildResumeRequestFailsClosedWhenRecordedKubernetesInventoryIsIncomplete(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + session := repo.sessions["sess-1"] + session.ExecutorID = "recorded-kubernetes-executor" + repo.executors["recorded-kubernetes-executor"] = &models.Executor{ + ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, Resumable: true, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + } + incomplete := recordedKubernetesResumeMetadata() + delete(incomplete, lifecycle.MetadataKeyKubernetesPodUID) + repo.executorsRunning["sess-1"] = &models.ExecutorRunning{ + ID: "sess-1", SessionID: "sess-1", TaskID: "task-1", + ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", Metadata: incomplete, + } + exec := newTestExecutor(t, &mockAgentManager{}, repo) + + _, _, _, _, _, err := exec.buildResumeRequest( + context.Background(), &v1.Task{ID: "task-1", WorkspaceID: "workspace-1"}, session, true, + ) + + if err == nil || !strings.Contains(err.Error(), "validate recorded Kubernetes runtime") { + t.Fatalf("buildResumeRequest() error = %v, want incomplete inventory rejection", err) + } +} + +func TestBuildResumeRequestFailsClosedWhenRuntimeInventoryReadFails(t *testing.T) { + repo := newMockRepository() + setupLiveResumeTestFixture(repo) + repo.getExecutorRunningFunc = func(context.Context, string) (*models.ExecutorRunning, error) { + return nil, errors.New("database unavailable") + } + exec := newTestExecutor(t, &mockAgentManager{}, repo) + + _, _, _, _, _, err := exec.buildResumeRequest( + context.Background(), + &v1.Task{ID: "task-1", WorkspaceID: "workspace-1"}, + repo.sessions["sess-1"], + true, + ) + + if err == nil || !strings.Contains(err.Error(), "load runtime inventory") { + t.Fatalf("buildResumeRequest() error = %v, want runtime inventory read failure", err) + } +} + +func recordedKubernetesResumeMetadata() map[string]interface{} { + return recordedKubernetesResumeMetadataFor("task-1", "sess-1", "recorded-execution") +} + +func recordedKubernetesResumeMetadataFor(taskID, sessionID, executionID string) map[string]interface{} { + profile := kubeexecutor.ProfileConfig{ + Platform: kubeexecutor.PlatformLinuxAMD64, + MainContainer: kubeexecutor.DefaultMainContainerName, + PodTemplateYAML: "apiVersion: v1\nkind: PodTemplate\ntemplate:\n spec:\n" + + " containers:\n - name: kandev-agent\n image: example.test/agent:latest\n", + Workspace: kubeexecutor.WorkspaceConfig{Mode: kubeexecutor.WorkspaceModeEmptyDir}, + } + snapshot, _ := json.Marshal(profile) + profileHash := sha256.Sum256(snapshot) + templateHash := sha256.Sum256([]byte(profile.PodTemplateYAML)) + return map[string]interface{}{ + lifecycle.MetadataKeyExecutorProfileID: "recorded-profile", + lifecycle.MetadataKeyKubernetesInventoryState: lifecycle.KubernetesInventoryStateReady, + lifecycle.MetadataKeyKubernetesNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesPodName: "recorded-pod", + lifecycle.MetadataKeyKubernetesPodUID: "recorded-pod-uid", + lifecycle.MetadataKeyKubernetesMainContainer: "kandev-agent", + lifecycle.MetadataKeyKubernetesRuntimeWorkspaceMode: "empty_dir", + lifecycle.MetadataKeyKubernetesAgentctlRemotePort: "41001", + lifecycle.MetadataKeyKubernetesAgentctlInstanceID: executionID, + lifecycle.MetadataKeyKubernetesResourceExecutorID: "recorded-kubernetes-executor", + lifecycle.MetadataKeyKubernetesResourceProfileID: "recorded-profile", + lifecycle.MetadataKeyKubernetesResourceInstanceID: executionID, + lifecycle.MetadataKeyKubernetesResourceTaskID: taskID, + lifecycle.MetadataKeyKubernetesResourceSessionID: sessionID, + lifecycle.MetadataKeyKubernetesResourceEnvironmentID: "environment-1", + lifecycle.MetadataKeyKubernetesExecutorConfigHash: "recorded-executor-hash", + lifecycle.MetadataKeyKubernetesProfileConfigHash: fmt.Sprintf("%x", profileHash), + lifecycle.MetadataKeyKubernetesTemplateHash: fmt.Sprintf("%x", templateHash), + lifecycle.MetadataKeyKubernetesProfileSnapshot: string(snapshot), + lifecycle.MetadataKeyAuthTokenSecret: "recorded-auth-secret", + lifecycle.MetadataKeyBootstrapNonceSecret: "recorded-nonce-secret", + } +} + func TestBuildResumeRequestWithOptions_ForwardsBranchReplacementPermission(t *testing.T) { repo := newMockRepository() setupLiveResumeTestFixture(repo) diff --git a/apps/backend/internal/orchestrator/executor/executor_state.go b/apps/backend/internal/orchestrator/executor/executor_state.go index 21adce9f3f..2bf75cddd3 100644 --- a/apps/backend/internal/orchestrator/executor/executor_state.go +++ b/apps/backend/internal/orchestrator/executor/executor_state.go @@ -214,7 +214,7 @@ func (e *Executor) applyProfile(ctx context.Context, profileID string, cfg *exec if policyJSON := strings.TrimSpace(profile.McpPolicy); policyJSON != "" { metadata["executor_mcp_policy"] = policyJSON } - applyProfileConfigToMetadata(profile.Config, metadata) + applyProfileConfigToMetadata(cfg.ExecutorType, profile.Config, metadata) } // Profile.Config / metadata keys shared with executor_credentials.go. @@ -265,10 +265,21 @@ var profileConfigAuthoritativeKeys = []string{ lifecycle.MetadataKeySSHReclaimTaskDir, } +var kubernetesProfileConfigAuthoritativeKeys = []string{ + lifecycle.MetadataKeyKubernetesProfilePlatform, + lifecycle.MetadataKeyKubernetesProfileMainContainer, + lifecycle.MetadataKeyKubernetesPodTemplateYAML, + lifecycle.MetadataKeyKubernetesWorkspaceMode, + lifecycle.MetadataKeyKubernetesWorkspaceSize, + lifecycle.MetadataKeyKubernetesWorkspaceStorageClass, + lifecycle.MetadataKeyKubernetesWorkspaceAccessModes, + lifecycle.MetadataKeyKubernetesWorkspaceClaimName, +} + // applyProfileConfigToMetadata projects profile.Config keys into the // launch metadata. Pulled out so the policy (passthrough vs rename vs // authoritative) is declarative and testable in isolation. -func applyProfileConfigToMetadata(profileConfig map[string]string, metadata map[string]interface{}) { +func applyProfileConfigToMetadata(executorType string, profileConfig map[string]string, metadata map[string]interface{}) { for _, k := range profileConfigPassthroughKeys { if v := profileConfig[k]; v != "" { metadata[k] = v @@ -285,6 +296,13 @@ func applyProfileConfigToMetadata(profileConfig map[string]string, metadata map[ // to a default handles the empty case. metadata[k] = profileConfig[k] } + if models.ExecutorType(executorType) == models.ExecutorTypeKubernetes { + for _, k := range kubernetesProfileConfigAuthoritativeKeys { + // Kubernetes profile config is admin-owned launch policy. Empty + // mode-inapplicable values must also clear task metadata. + metadata[k] = profileConfig[k] + } + } } func cloneMetadata(src map[string]interface{}) map[string]interface{} { diff --git a/apps/backend/internal/orchestrator/executor/executor_state_reclaim_test.go b/apps/backend/internal/orchestrator/executor/executor_state_reclaim_test.go index c86c7dc4c0..6f3808bd67 100644 --- a/apps/backend/internal/orchestrator/executor/executor_state_reclaim_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_state_reclaim_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/task/models" ) // Reclaiming a remote task directory is irreversible and happens on a machine @@ -55,7 +56,7 @@ func TestApplyProfileConfigToMetadata_TaskCannotEnableReclaim(t *testing.T) { for k, v := range tc.taskMetadata { metadata[k] = v } - applyProfileConfigToMetadata(tc.profileConfig, metadata) + applyProfileConfigToMetadata(string(models.ExecutorTypeSSH), tc.profileConfig, metadata) got, _ := metadata[lifecycle.MetadataKeySSHReclaimTaskDir].(string) if got != tc.want { diff --git a/apps/backend/internal/orchestrator/executor/executor_state_test.go b/apps/backend/internal/orchestrator/executor/executor_state_test.go index c8a479821d..b0df076dac 100644 --- a/apps/backend/internal/orchestrator/executor/executor_state_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_state_test.go @@ -2,6 +2,7 @@ package executor import ( "context" + "reflect" "testing" "github.com/kandev/kandev/internal/agent/runtime/lifecycle" @@ -137,3 +138,79 @@ func TestResolveExecutorConfig_PassthroughKeys_DontClobberTaskMetadata(t *testin t.Fatalf("metadata[git_user_name] = %q, want task value to survive empty profile", got) } } + +func TestResolveExecutorConfig_KubernetesProfileConfigClobbersTaskMetadata(t *testing.T) { + repo := newMockRepository() + exec := newTestExecutor(t, &mockAgentManager{}, repo) + repo.executors["exec-k8s"] = &models.Executor{ID: "exec-k8s", Type: models.ExecutorTypeKubernetes} + profileConfig := map[string]string{ + lifecycle.MetadataKeyKubernetesProfilePlatform: "linux/arm64", + lifecycle.MetadataKeyKubernetesProfileMainContainer: "trusted-agent", + lifecycle.MetadataKeyKubernetesPodTemplateYAML: "trusted pod template", + lifecycle.MetadataKeyKubernetesWorkspaceMode: "existing_claim", + lifecycle.MetadataKeyKubernetesWorkspaceSize: "", + lifecycle.MetadataKeyKubernetesWorkspaceStorageClass: "", + lifecycle.MetadataKeyKubernetesWorkspaceAccessModes: "", + lifecycle.MetadataKeyKubernetesWorkspaceClaimName: "trusted-claim", + } + repo.executorProfiles["profile-k8s"] = &models.ExecutorProfile{ + ID: "profile-k8s", ExecutorID: "exec-k8s", Config: profileConfig, + } + metadata := map[string]interface{}{"executor_profile_id": "profile-k8s"} + for key := range profileConfig { + metadata[key] = "hostile-task-override" + } + + cfg := exec.resolveExecutorConfig(context.Background(), "exec-k8s", "ws-1", metadata) + got := make(map[string]string, len(profileConfig)) + for key := range profileConfig { + got[key], _ = cfg.Metadata[key].(string) + } + + if !reflect.DeepEqual(got, profileConfig) { + t.Fatalf("Kubernetes launch metadata = %#v, want authoritative profile config %#v", got, profileConfig) + } +} + +func TestResolveExecutorConfig_KubernetesProfileKeysDoNotClobberNonKubernetesMetadata(t *testing.T) { + repo := newMockRepository() + exec := newTestExecutor(t, &mockAgentManager{}, repo) + repo.executors["exec-docker"] = &models.Executor{ID: "exec-docker", Type: models.ExecutorTypeLocalDocker} + profileConfig := map[string]string{ + lifecycle.MetadataKeyKubernetesProfilePlatform: "linux/arm64", + lifecycle.MetadataKeyKubernetesProfileMainContainer: "profile-agent", + lifecycle.MetadataKeyKubernetesPodTemplateYAML: "profile pod template", + lifecycle.MetadataKeyKubernetesWorkspaceMode: "empty_dir", + lifecycle.MetadataKeyKubernetesWorkspaceSize: "", + lifecycle.MetadataKeyKubernetesWorkspaceStorageClass: "", + lifecycle.MetadataKeyKubernetesWorkspaceAccessModes: "", + lifecycle.MetadataKeyKubernetesWorkspaceClaimName: "", + } + repo.executorProfiles["profile-docker"] = &models.ExecutorProfile{ + ID: "profile-docker", ExecutorID: "exec-docker", Config: profileConfig, + } + want := map[string]string{ + lifecycle.MetadataKeyKubernetesProfilePlatform: "task-platform", + lifecycle.MetadataKeyKubernetesProfileMainContainer: "task-container", + lifecycle.MetadataKeyKubernetesPodTemplateYAML: "task pod template", + lifecycle.MetadataKeyKubernetesWorkspaceMode: "task-mode", + lifecycle.MetadataKeyKubernetesWorkspaceSize: "task-size", + lifecycle.MetadataKeyKubernetesWorkspaceStorageClass: "task-storage-class", + lifecycle.MetadataKeyKubernetesWorkspaceAccessModes: "task-access-modes", + lifecycle.MetadataKeyKubernetesWorkspaceClaimName: "task-claim", + } + metadata := map[string]interface{}{"executor_profile_id": "profile-docker"} + for key, value := range want { + metadata[key] = value + } + + cfg := exec.resolveExecutorConfig(context.Background(), "exec-docker", "ws-1", metadata) + got := make(map[string]string, len(want)) + for key := range want { + got[key], _ = cfg.Metadata[key].(string) + } + + if !reflect.DeepEqual(got, want) { + t.Fatalf("non-Kubernetes launch metadata = %#v, want task metadata unchanged %#v", got, want) + } +} diff --git a/apps/backend/internal/orchestrator/executor/executor_test.go b/apps/backend/internal/orchestrator/executor/executor_test.go index 5f7b8a1122..5927af7a61 100644 --- a/apps/backend/internal/orchestrator/executor/executor_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os/exec" + "reflect" "strings" "sync" "sync/atomic" @@ -604,6 +605,59 @@ func TestLaunchPreparedSession_Success(t *testing.T) { } } +func TestLaunchPreparedSessionFailsClosedWhenRuntimeInventoryReadFails(t *testing.T) { + tests := []struct { + name string + setup func(*mockRepository) + }{ + { + name: "row read", + setup: func(repo *mockRepository) { + repo.getExecutorRunningFunc = func(context.Context, string) (*models.ExecutorRunning, error) { + return nil, errors.New("database unavailable") + } + }, + }, + { + name: "presence check", + setup: func(repo *mockRepository) { + repo.hasExecutorRunningFunc = func(context.Context, string) (bool, error) { + return false, errors.New("database unavailable") + } + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-inventory-error"] = &models.TaskSession{ + ID: "session-inventory-error", TaskID: "task-inventory-error", + AgentProfileID: "profile-123", State: models.TaskSessionStateCreated, + StartedAt: time.Now(), UpdatedAt: time.Now(), + } + test.setup(repo) + launchCalled := false + exec := newTestExecutor(t, &mockAgentManager{ + launchAgentFunc: func(context.Context, *LaunchAgentRequest) (*LaunchAgentResponse, error) { + launchCalled = true + return &LaunchAgentResponse{}, nil + }, + }, repo) + + _, err := exec.LaunchPreparedSession(context.Background(), &v1.Task{ + ID: "task-inventory-error", WorkspaceID: "workspace-123", + }, "session-inventory-error", LaunchOptions{AgentProfileID: "profile-123"}) + + if err == nil || !strings.Contains(err.Error(), "runtime inventory") { + t.Fatalf("LaunchPreparedSession() error = %v, want runtime inventory failure", err) + } + if launchCalled { + t.Fatal("runtime launched after authoritative inventory read failure") + } + }) + } +} + func TestLaunchPreparedSession_PropagatesTaskEnvironmentPersistenceFailure(t *testing.T) { repo := newMockRepository() persistErr := errors.New("inventory write failed") @@ -722,6 +776,50 @@ func TestLaunchPreparedSession_PersistsResolvedExecutorID(t *testing.T) { } } +func TestLaunchPreparedSession_UsesPersistedKubernetesExecutorConnectionConfig(t *testing.T) { + repo := newMockRepository() + repo.sessions["session-k8s"] = &models.TaskSession{ + ID: "session-k8s", TaskID: "task-k8s", AgentProfileID: "profile-123", + ExecutorID: "executor-k8s", State: models.TaskSessionStateCreated, + StartedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC(), + } + wantConfig := map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/kind.yaml", + lifecycle.MetadataKeyKubernetesKubeContext: "kind-kandev", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + } + repo.executors["executor-k8s"] = &models.Executor{ + ID: "executor-k8s", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: wantConfig, + } + var launched *LaunchAgentRequest + exec := newTestExecutor(t, &mockAgentManager{ + launchAgentFunc: func(_ context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) { + launched = req + return &LaunchAgentResponse{AgentExecutionID: "execution-k8s", Status: v1.AgentStatusStarting}, nil + }, + }, repo) + + _, err := exec.LaunchPreparedSession(context.Background(), &v1.Task{ + ID: "task-k8s", WorkspaceID: "workspace-1", + }, "session-k8s", LaunchOptions{AgentProfileID: "profile-123"}) + + if err != nil { + t.Fatalf("LaunchPreparedSession() error = %v", err) + } + if launched == nil { + t.Fatal("LaunchAgent was not called") + } + if launched.ExecutorType != string(models.ExecutorTypeKubernetes) { + t.Fatalf("ExecutorType = %q, want k8s", launched.ExecutorType) + } + if !reflect.DeepEqual(launched.ExecutorConfig, wantConfig) { + t.Fatalf("ExecutorConfig = %#v, want %#v", launched.ExecutorConfig, wantConfig) + } +} + func TestLaunchPreparedSession_AbortsWhenStartingPersistenceFails(t *testing.T) { repo := newMockRepository() session := &models.TaskSession{ diff --git a/apps/backend/internal/orchestrator/service.go b/apps/backend/internal/orchestrator/service.go index a3e34091eb..fab56becb1 100644 --- a/apps/backend/internal/orchestrator/service.go +++ b/apps/backend/internal/orchestrator/service.go @@ -2631,6 +2631,7 @@ func (s *Service) reconcileExecutorSessionsOnStartup(ctx context.Context) { for _, running := range runningExecutors { if models.IsRemoteExecutorType(models.ExecutorType(running.Runtime)) { remoteRecords = append(remoteRecords, executor.RemoteStatusPollRequest{ + TaskID: running.TaskID, SessionID: running.SessionID, Runtime: running.Runtime, AgentExecutionID: running.AgentExecutionID, diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index b707ce80ff..b65c45d131 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -2966,9 +2966,7 @@ func (s *Service) applyRemoteRuntimeStatus(ctx context.Context, sessionID string } resp.RemoteState = status.State resp.RemoteName = status.RemoteName - if status.ErrorMessage != "" { - resp.RemoteStatusErr = status.ErrorMessage - } + resp.RemoteStatusErr = publicRemoteStatusError(status.ErrorMessage) if status.CreatedAt != nil && !status.CreatedAt.IsZero() { resp.RemoteCreatedAt = status.CreatedAt.UTC().Format(time.RFC3339) } @@ -2977,6 +2975,15 @@ func (s *Service) applyRemoteRuntimeStatus(ctx context.Context, sessionID string } } +const remoteStatusUnavailable = "remote executor status is unavailable" + +func publicRemoteStatusError(errorMessage string) string { + if errorMessage == "" { + return "" + } + return remoteStatusUnavailable +} + // populateWorktreeInfo copies worktree path and branch into the response if present. func canRestoreWorkspace(resp *dto.TaskSessionStatusResponse) bool { return resp != nil && resp.WorktreePath != nil && *resp.WorktreePath != "" diff --git a/apps/backend/internal/orchestrator/task_operations_remote_status_test.go b/apps/backend/internal/orchestrator/task_operations_remote_status_test.go new file mode 100644 index 0000000000..b81cc5e02e --- /dev/null +++ b/apps/backend/internal/orchestrator/task_operations_remote_status_test.go @@ -0,0 +1,16 @@ +package orchestrator + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPublicRemoteStatusErrorDoesNotExposeProviderDetails(t *testing.T) { + require.Empty(t, publicRemoteStatusError("")) + require.Equal( + t, + remoteStatusUnavailable, + publicRemoteStatusError("request failed at /home/operator/.kube/config?token=secret"), + ) +} diff --git a/apps/backend/internal/scriptengine/placeholders.go b/apps/backend/internal/scriptengine/placeholders.go index 0bc23c563f..947cfd85d8 100644 --- a/apps/backend/internal/scriptengine/placeholders.go +++ b/apps/backend/internal/scriptengine/placeholders.go @@ -8,126 +8,135 @@ type PlaceholderInfo struct { ExecutorTypes []string `json:"executor_types"` } +const ( + executorTypeLocal = "local" + executorTypeWorktree = "worktree" + executorTypeLocalDocker = "local_docker" + executorTypeRemoteDocker = "remote_docker" + executorTypeSprites = "sprites" + executorTypeKubernetes = "k8s" +) + // DefaultPlaceholders is the registry of all available script template placeholders. var DefaultPlaceholders = []PlaceholderInfo{ { Key: "repository.path", Description: "Local repository path on the host machine", Example: "/Users/dev/myapp", - ExecutorTypes: []string{"local", "worktree"}, + ExecutorTypes: []string{executorTypeLocal, executorTypeWorktree}, }, { Key: "repository.name", Description: "Repository name", Example: "myapp", - ExecutorTypes: []string{"local", "worktree", "local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocal, executorTypeWorktree, executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "repository.clone_url", Description: "HTTPS clone URL (with auth token injected if available)", Example: "https://token@github.com/org/myapp.git", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "repository.ssh_url", Description: "SSH clone URL", Example: "git@github.com:org/myapp.git", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "repository.branch", Description: "Target branch name", Example: "main", - ExecutorTypes: []string{"local", "worktree", "local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocal, executorTypeWorktree, executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "repository.setup_script", Description: "Repository-level setup script (if configured in repo settings)", Example: "npm install", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "git.user_name", Description: "Git author name configured for remote executor", Example: "Jane Developer", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "git.user_email", Description: "Git author email configured for remote executor", Example: "jane@example.com", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "git.identity_setup", Description: "Expands to git config commands when name/email are provided", Example: "git config --global user.name 'Jane Developer'", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "workspace.path", Description: "Working directory for the current executor", Example: "/workspace", - ExecutorTypes: []string{"local", "worktree", "local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocal, executorTypeWorktree, executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "worktree.base_path", Description: "Base directory where worktrees are stored", Example: "/Users/dev/.kandev/worktrees", - ExecutorTypes: []string{"worktree"}, + ExecutorTypes: []string{executorTypeWorktree}, }, { Key: "worktree.path", Description: "Resolved worktree directory path for this session", Example: "/Users/dev/.kandev/worktrees/fix-bug_ab12cd34", - ExecutorTypes: []string{"worktree"}, + ExecutorTypes: []string{executorTypeWorktree}, }, { Key: "worktree.id", Description: "Worktree ID for this session", Example: "f4db4fa6-82f4-4d8d-b29c-6ffbd44f57de", - ExecutorTypes: []string{"worktree"}, + ExecutorTypes: []string{executorTypeWorktree}, }, { Key: "worktree.branch", Description: "Created/reused worktree branch name", Example: "feature/fix-login-abc", - ExecutorTypes: []string{"worktree"}, + ExecutorTypes: []string{executorTypeWorktree}, }, { Key: "worktree.base_branch", Description: "Base branch used for worktree creation", Example: "main", - ExecutorTypes: []string{"worktree"}, + ExecutorTypes: []string{executorTypeWorktree}, }, { Key: "kandev.agents.install", Description: "Pre-installs agent CLIs globally for all agents configured on the profile", Example: "npm install -g @anthropic-ai/claude-code@2.1.50", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "kandev.agentctl.install", Description: "Expands to full agentctl binary upload and install commands", Example: "# (multi-line: upload binary, chmod, verify)", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "kandev.agentctl.start", Description: "Expands to agentctl start command with configured flags", Example: "agentctl --port 8765 --workspace /workspace &", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "kandev.agentctl.port", Description: "Agentctl port number", Example: "8765", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, { Key: "github.auth_setup", Description: "Expands to commands that configure gh CLI and git credential helper for GitHub token auth", Example: "git config --global credential.https://github.com.helper '...'", - ExecutorTypes: []string{"local_docker", "remote_docker", "sprites"}, + ExecutorTypes: []string{executorTypeLocalDocker, executorTypeRemoteDocker, executorTypeSprites, executorTypeKubernetes}, }, } diff --git a/apps/backend/internal/scriptengine/placeholders_kubernetes_test.go b/apps/backend/internal/scriptengine/placeholders_kubernetes_test.go new file mode 100644 index 0000000000..08bc9247ce --- /dev/null +++ b/apps/backend/internal/scriptengine/placeholders_kubernetes_test.go @@ -0,0 +1,21 @@ +package scriptengine + +import ( + "slices" + "testing" +) + +func TestKubernetesAdvertisesDockerCompatiblePlaceholders(t *testing.T) { + for _, placeholder := range DefaultPlaceholders { + dockerCompatible := slices.Contains(placeholder.ExecutorTypes, "local_docker") + kubernetesCompatible := slices.Contains(placeholder.ExecutorTypes, "k8s") + if dockerCompatible != kubernetesCompatible { + t.Errorf( + "placeholder %q: local_docker compatibility = %v, k8s compatibility = %v", + placeholder.Key, + dockerCompatible, + kubernetesCompatible, + ) + } + } +} diff --git a/apps/backend/internal/scriptengine/providers.go b/apps/backend/internal/scriptengine/providers.go index 2c17ef1db0..73683543d4 100644 --- a/apps/backend/internal/scriptengine/providers.go +++ b/apps/backend/internal/scriptengine/providers.go @@ -188,17 +188,46 @@ func RepositoryProvider( } } -// AgentctlProvider returns kandev agentctl-related placeholders. +// AgentctlProviderOptions configures executor-specific agentctl placeholders. +type AgentctlProviderOptions struct { + BinaryPath string + Start bool +} + +// AgentctlProvider returns the legacy agentctl placeholders used by existing +// callers. func AgentctlProvider(agentctlPort int, workspacePath string) PlaceholderProvider { + return agentctlProvider(agentctlPort, workspacePath, "/usr/local/bin/agentctl", "agentctl") +} + +// AgentctlProviderWithOptions returns agentctl placeholders for an +// executor-visible binary path. +func AgentctlProviderWithOptions( + agentctlPort int, + workspacePath string, + options AgentctlProviderOptions, +) PlaceholderProvider { + startCommand := "" + if options.Start { + startCommand = options.BinaryPath + } + return agentctlProvider(agentctlPort, workspacePath, options.BinaryPath, startCommand) +} + +func agentctlProvider(agentctlPort int, workspacePath, binaryPath, startCommand string) PlaceholderProvider { return func() map[string]string { portStr := fmt.Sprintf("%d", agentctlPort) + startScript := "" + if startCommand != "" { + startScript = fmt.Sprintf( + "nohup %s --port %s --workdir %s > /tmp/agentctl.log 2>&1 &\nsleep 1", + shellQuote(startCommand), portStr, shellQuote(workspacePath), + ) + } return map[string]string{ "kandev.agentctl.port": portStr, - "kandev.agentctl.install": "chmod +x /usr/local/bin/agentctl", - "kandev.agentctl.start": fmt.Sprintf( - "nohup agentctl --port %s --workdir %s > /tmp/agentctl.log 2>&1 &\nsleep 1", - portStr, workspacePath, - ), + "kandev.agentctl.install": "chmod +x " + shellQuote(binaryPath), + "kandev.agentctl.start": startScript, } } } diff --git a/apps/backend/internal/scriptengine/providers_test.go b/apps/backend/internal/scriptengine/providers_test.go index 4c3b416e69..965549cc77 100644 --- a/apps/backend/internal/scriptengine/providers_test.go +++ b/apps/backend/internal/scriptengine/providers_test.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "reflect" "strings" "testing" @@ -59,6 +60,60 @@ func TestAgentInstallProvider(t *testing.T) { }) } +func TestAgentctlProviderSupportsExecutorBinaryPath(t *testing.T) { + vars := AgentctlProviderWithOptions(8765, "/workspace", AgentctlProviderOptions{ + BinaryPath: "/opt/kandev/agentctl", + Start: true, + })() + + if got, want := vars["kandev.agentctl.install"], "chmod +x '/opt/kandev/agentctl'"; got != want { + t.Fatalf("kandev.agentctl.install = %q, want %q", got, want) + } +} + +func TestAgentctlProviderShellQuotesExecutorBinaryAndWorkspacePaths(t *testing.T) { + binaryPath := "/opt/kandev agentctl'$(touch /tmp/pwned)" + workspacePath := "/workspace/project name'$(touch /tmp/workspace-pwned)" + vars := AgentctlProviderWithOptions(8765, workspacePath, AgentctlProviderOptions{ + BinaryPath: binaryPath, + Start: true, + })() + + if got, want := vars["kandev.agentctl.install"], "chmod +x "+shellQuote(binaryPath); got != want { + t.Fatalf("kandev.agentctl.install = %q, want %q", got, want) + } + wantStart := "nohup " + shellQuote(binaryPath) + + " --port 8765 --workdir " + shellQuote(workspacePath) + + " > /tmp/agentctl.log 2>&1 &\nsleep 1" + if got := vars["kandev.agentctl.start"]; got != wantStart { + t.Fatalf("kandev.agentctl.start = %q, want %q", got, wantStart) + } +} + +func TestAgentctlProviderCanLeaveStartToManagedBootstrap(t *testing.T) { + vars := AgentctlProviderWithOptions(8765, "/workspace", AgentctlProviderOptions{ + BinaryPath: "/opt/kandev/agentctl", + Start: false, + })() + + if got := vars["kandev.agentctl.start"]; got != "" { + t.Fatalf("kandev.agentctl.start = %q, want empty managed-bootstrap expansion", got) + } +} + +func TestAgentctlProviderPreservesLegacyExpansions(t *testing.T) { + got := AgentctlProvider(8765, "/workspace")() + want := map[string]string{ + "kandev.agentctl.port": "8765", + "kandev.agentctl.install": "chmod +x '/usr/local/bin/agentctl'", + "kandev.agentctl.start": "nohup 'agentctl' --port 8765 --workdir '/workspace' > /tmp/agentctl.log 2>&1 &\nsleep 1", + } + + if !reflect.DeepEqual(got, want) { + t.Fatalf("AgentctlProvider() = %#v, want %#v", got, want) + } +} + func TestRepositoryProvider_UsesRepositorySetupScriptKey(t *testing.T) { provider := RepositoryProvider(map[string]any{ "repository_path": "/tmp/repo", diff --git a/apps/backend/internal/secrets/user_visible_store.go b/apps/backend/internal/secrets/user_visible_store.go index 72e533ffde..b45d8473fc 100644 --- a/apps/backend/internal/secrets/user_visible_store.go +++ b/apps/backend/internal/secrets/user_visible_store.go @@ -8,12 +8,17 @@ import ( "github.com/jmoiron/sqlx" ) -const internalGitHubSecretPrefix = "github:" +const ( + internalGitHubSecretPrefix = "github:" + internalRuntimeSecretPrefix = "kandev-runtime:" +) // IsInternalID reports whether a secret is owned by backend infrastructure // and must never be listed, revealed, or selected as an agent credential. func IsInternalID(id string) bool { - return strings.HasPrefix(strings.ToLower(strings.TrimSpace(id)), internalGitHubSecretPrefix) + normalized := strings.ToLower(strings.TrimSpace(id)) + return strings.HasPrefix(normalized, internalGitHubSecretPrefix) || + strings.HasPrefix(normalized, internalRuntimeSecretPrefix) } // UserVisibleStore restricts a SecretStore to user-managed credentials. The diff --git a/apps/backend/internal/secrets/user_visible_store_test.go b/apps/backend/internal/secrets/user_visible_store_test.go index b4c949257c..a959c11420 100644 --- a/apps/backend/internal/secrets/user_visible_store_test.go +++ b/apps/backend/internal/secrets/user_visible_store_test.go @@ -54,6 +54,17 @@ func TestUserVisibleStoreRejectsInternalCreate(t *testing.T) { } } +func TestIsInternalIDRecognizesKubernetesRuntimeSecrets(t *testing.T) { + t.Parallel() + + if !IsInternalID("kandev-runtime:execution-1:agentctl-auth-token") { + t.Fatal("Kubernetes runtime secret ID must be internal") + } + if IsInternalID("legacy-random-runtime-secret-id") { + t.Fatal("legacy random secret ID must remain user-visible") + } +} + func TestUserVisibleStoreWorkspaceOperationsFilterInternalSecrets(t *testing.T) { store := newTestSQLiteStore(t) ctx := context.Background() diff --git a/apps/backend/internal/task/handlers/executor_handlers.go b/apps/backend/internal/task/handlers/executor_handlers.go index 7703eeb9f5..ed357df992 100644 --- a/apps/backend/internal/task/handlers/executor_handlers.go +++ b/apps/backend/internal/task/handlers/executor_handlers.go @@ -115,6 +115,10 @@ func (h *ExecutorHandlers) httpCreateExecutor(c *gin.Context) { Config: body.Config, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } if errors.Is(err, service.ErrInvalidExecutorConfig) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return @@ -168,10 +172,18 @@ func (h *ExecutorHandlers) httpUpdateExecutor(c *gin.Context) { Config: body.Config, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } if errors.Is(err, service.ErrInvalidExecutorConfig) { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } + if errors.Is(err, service.ErrActiveTaskSessions) { + c.JSON(http.StatusConflict, gin.H{"error": "executor is used by an active agent session"}) + return + } h.logger.Error("failed to update executor", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{"error": "executor not updated"}) return @@ -181,6 +193,10 @@ func (h *ExecutorHandlers) httpUpdateExecutor(c *gin.Context) { func (h *ExecutorHandlers) httpDeleteExecutor(c *gin.Context) { if err := h.service.DeleteExecutor(c.Request.Context(), c.Param("id")); err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } if errors.Is(err, service.ErrActiveTaskSessions) { c.JSON(http.StatusConflict, gin.H{"error": "executor is used by an active agent session"}) return @@ -235,8 +251,11 @@ func (h *ExecutorHandlers) wsCreateExecutor(ctx context.Context, msg *ws.Message Config: req.Config, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, service.ErrKubernetesAdminRequired.Error(), nil) + } if errors.Is(err, service.ErrInvalidExecutorConfig) { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, err.Error(), nil) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, service.ErrInvalidExecutorConfig.Error(), nil) } h.logger.Error("failed to create executor", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to create executor", nil) @@ -298,8 +317,15 @@ func (h *ExecutorHandlers) wsUpdateExecutor(ctx context.Context, msg *ws.Message Config: req.Config, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, service.ErrKubernetesAdminRequired.Error(), nil) + } if errors.Is(err, service.ErrInvalidExecutorConfig) { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, err.Error(), nil) + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, service.ErrInvalidExecutorConfig.Error(), nil) + } + if errors.Is(err, service.ErrActiveTaskSessions) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, + "executor is used by an active agent session", nil) } h.logger.Error("failed to update executor", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to update executor", nil) diff --git a/apps/backend/internal/task/handlers/executor_handlers_test.go b/apps/backend/internal/task/handlers/executor_handlers_test.go index 8e8c4c363c..c92cb491d3 100644 --- a/apps/backend/internal/task/handlers/executor_handlers_test.go +++ b/apps/backend/internal/task/handlers/executor_handlers_test.go @@ -11,6 +11,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/require" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/task/dto" "github.com/kandev/kandev/internal/task/models" "github.com/kandev/kandev/internal/task/service" @@ -34,6 +35,7 @@ type executorRepo struct { updateErr error deleteErr error hasActiveSessions bool + running []*models.ExecutorRunning created []*models.Executor updated []*models.Executor @@ -85,6 +87,10 @@ func (r *executorRepo) HasActiveTaskSessionsByExecutor(context.Context, string) return r.hasActiveSessions, nil } +func (r *executorRepo) ListExecutorsRunning(context.Context) ([]*models.ExecutorRunning, error) { + return r.running, nil +} + func (r *executorRepo) ListAllExecutorProfiles(context.Context) ([]*models.ExecutorProfile, error) { if r.listProfilesErr != nil { return nil, r.listProfilesErr @@ -365,6 +371,24 @@ func TestHTTPUpdateExecutorRejectsInvalidMCPPolicy(t *testing.T) { require.Empty(t, repo.updated) } +func TestHTTPUpdateExecutorReturnsConflictForRetainedKubernetesInventory(t *testing.T) { + repo := kubernetesHandlerFixture() + repo.running = []*models.ExecutorRunning{{ + SessionID: "session-retained", TaskID: "task-retained", + ExecutorID: "ex-kubernetes", Runtime: agentruntime.RuntimeKubernetes, + }} + h := newExecutorHandlers(t, repo) + c, rec := newWorkflowRequest(t, http.MethodPatch, "/api/v1/executors/ex-kubernetes", `{"type":"local"}`) + c.Params = gin.Params{{Key: "id", Value: "ex-kubernetes"}} + c.Request = c.Request.WithContext(kubernetesAdminContext()) + + h.httpUpdateExecutor(c) + + require.Equal(t, http.StatusConflict, rec.Code) + require.JSONEq(t, `{"error":"executor is used by an active agent session"}`, rec.Body.String()) + require.Empty(t, repo.updated) +} + // TestHTTPUpdateExecutorRefusesToRenameSystemExecutor covers the service-side // system guard reaching the handler as a plain 500 rather than a 400 — the // error is not an ErrInvalidExecutorConfig, so it takes the generic branch. @@ -485,7 +509,7 @@ func TestWSCreateExecutorRejectsInvalidMCPPolicy(t *testing.T) { require.NoError(t, err) payload := wsWorkflowError(t, resp) require.Equal(t, string(ws.ErrorCodeValidation), payload.Code) - require.Contains(t, payload.Message, "mcp_policy must be valid JSON") + require.Equal(t, service.ErrInvalidExecutorConfig.Error(), payload.Message) require.Empty(t, repo.created) } @@ -554,6 +578,26 @@ func TestWSUpdateExecutorSurfacesRepositoryFailure(t *testing.T) { require.Equal(t, "Failed to update executor", payload.Message) } +func TestWSUpdateExecutorReturnsValidationForRetainedKubernetesInventory(t *testing.T) { + repo := kubernetesHandlerFixture() + repo.running = []*models.ExecutorRunning{{ + SessionID: "session-retained", TaskID: "task-retained", + ExecutorID: "ex-kubernetes", Runtime: agentruntime.RuntimeKubernetes, + }} + h := newExecutorHandlers(t, repo) + + resp, err := h.wsUpdateExecutor(kubernetesAdminContext(), + wsWorkflowRequest(t, ws.ActionExecutorUpdate, map[string]any{ + "id": "ex-kubernetes", "type": string(models.ExecutorTypeLocal), + })) + + require.NoError(t, err) + payload := wsWorkflowError(t, resp) + require.Equal(t, string(ws.ErrorCodeValidation), payload.Code) + require.Equal(t, "executor is used by an active agent session", payload.Message) + require.Empty(t, repo.updated) +} + func TestWSDeleteExecutorDeletesAndReportsActiveSessions(t *testing.T) { repo := executorFixture() h := newExecutorHandlers(t, repo) diff --git a/apps/backend/internal/task/handlers/executor_kubernetes_authz_test.go b/apps/backend/internal/task/handlers/executor_kubernetes_authz_test.go new file mode 100644 index 0000000000..931e6ed545 --- /dev/null +++ b/apps/backend/internal/task/handlers/executor_kubernetes_authz_test.go @@ -0,0 +1,383 @@ +package handlers + +import ( + "context" + "net/http" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest/observer" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/service" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestHTTPKubernetesExecutorMutationsReturnForbiddenWithoutInternalLog(t *testing.T) { + tests := []struct { + name string + method string + target string + body string + params gin.Params + invoke func(*ExecutorHandlers, *gin.Context) + }{ + { + name: "create", method: http.MethodPost, target: "/api/v1/executors", + body: `{"name":"Kubernetes","type":"k8s","config":{"auth_mode":"kubeconfig","kubeconfig_path":"/etc/kandev/kubeconfig","namespace":"kandev","request_timeout_seconds":"30"}}`, + invoke: (*ExecutorHandlers).httpCreateExecutor, + }, + { + name: "update", method: http.MethodPatch, target: "/api/v1/executors/ex-kubernetes", + body: `{"name":"Renamed"}`, params: gin.Params{{Key: "id", Value: "ex-kubernetes"}}, + invoke: (*ExecutorHandlers).httpUpdateExecutor, + }, + { + name: "delete", method: http.MethodDelete, target: "/api/v1/executors/ex-kubernetes", + params: gin.Params{{Key: "id", Value: "ex-kubernetes"}}, + invoke: (*ExecutorHandlers).httpDeleteExecutor, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + handler := NewExecutorHandlers(newExecutorService(t, repo), log) + ctx, recorder := newWorkflowRequest(t, test.method, test.target, test.body) + ctx.Params = test.params + ctx.Request = ctx.Request.WithContext(kubernetesMemberContext()) + + test.invoke(handler, ctx) + + require.Equal(t, http.StatusForbidden, recorder.Code) + require.JSONEq(t, `{"error":"administrator identity required for Kubernetes settings"}`, recorder.Body.String()) + require.Empty(t, repo.created) + require.Empty(t, repo.updated) + require.Empty(t, repo.deleted) + require.Zero(t, observed.Len(), "expected authorization denial must not be logged as internal") + }) + } +} + +func TestWSKubernetesExecutorMutationsReturnForbiddenWithoutInternalLog(t *testing.T) { + tests := []struct { + name string + action string + payload map[string]any + invoke func(*ExecutorHandlers, context.Context, *ws.Message) (*ws.Message, error) + }{ + { + name: "create", action: ws.ActionExecutorCreate, + payload: map[string]any{ + "name": "Kubernetes", "type": string(models.ExecutorTypeKubernetes), + "config": validHandlerKubernetesExecutorConfig(), + }, + invoke: (*ExecutorHandlers).wsCreateExecutor, + }, + { + name: "update", action: ws.ActionExecutorUpdate, + payload: map[string]any{"id": "ex-kubernetes", "name": "Renamed"}, + invoke: (*ExecutorHandlers).wsUpdateExecutor, + }, + { + name: "delete", action: ws.ActionExecutorDelete, + payload: map[string]any{"id": "ex-kubernetes"}, + invoke: (*ExecutorHandlers).wsDeleteExecutor, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + handler := NewExecutorHandlers(newExecutorService(t, repo), log) + + response, err := test.invoke( + handler, kubernetesMemberContext(), wsWorkflowRequest(t, test.action, test.payload), + ) + + require.NoError(t, err) + payload := wsWorkflowError(t, response) + require.Equal(t, string(ws.ErrorCodeForbidden), payload.Code) + require.Equal(t, service.ErrKubernetesAdminRequired.Error(), payload.Message) + require.Empty(t, repo.created) + require.Empty(t, repo.updated) + require.Empty(t, repo.deleted) + require.Zero(t, observed.Len(), "expected authorization denial must not be logged as internal") + }) + } +} + +func TestHTTPKubernetesProfileMutationsReturnForbiddenWithoutInternalLog(t *testing.T) { + tests := []struct { + name string + method string + target string + body string + params gin.Params + invoke func(*ExecutorProfileHandlers, *gin.Context) + }{ + { + name: "create", method: http.MethodPost, + target: "/api/v1/executors/ex-kubernetes/profiles", + body: `{"name":"Default"}`, + params: gin.Params{{Key: "id", Value: "ex-kubernetes"}}, + invoke: (*ExecutorProfileHandlers).httpCreateProfile, + }, + { + name: "update", method: http.MethodPatch, + target: "/api/v1/executors/ex-kubernetes/profiles/pr-kubernetes", + body: `{"name":"Renamed"}`, + params: gin.Params{{Key: "id", Value: "ex-kubernetes"}, {Key: "profileId", Value: "pr-kubernetes"}}, + invoke: (*ExecutorProfileHandlers).httpUpdateProfile, + }, + { + name: "delete", method: http.MethodDelete, + target: "/api/v1/executor-profiles/pr-kubernetes", + params: gin.Params{{Key: "profileId", Value: "pr-kubernetes"}}, + invoke: (*ExecutorProfileHandlers).httpDeleteProfile, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + handler := NewExecutorProfileHandlers(newExecutorService(t, repo), nil, log) + ctx, recorder := newWorkflowRequest(t, test.method, test.target, test.body) + ctx.Params = test.params + ctx.Request = ctx.Request.WithContext(kubernetesMemberContext()) + + test.invoke(handler, ctx) + + require.Equal(t, http.StatusForbidden, recorder.Code) + require.JSONEq(t, `{"error":"administrator identity required for Kubernetes settings"}`, recorder.Body.String()) + require.Empty(t, repo.createdProfile) + require.Empty(t, repo.updatedProfile) + require.Empty(t, repo.deletedProfile) + require.Zero(t, observed.Len(), "expected authorization denial must not be logged as internal") + }) + } +} + +func TestWSKubernetesProfileMutationsReturnForbiddenWithoutInternalLog(t *testing.T) { + tests := []struct { + name string + action string + payload map[string]any + invoke func(*ExecutorProfileHandlers, context.Context, *ws.Message) (*ws.Message, error) + }{ + { + name: "create", action: ws.ActionExecutorProfileCreate, + payload: map[string]any{"executor_id": "ex-kubernetes", "name": "Default"}, + invoke: (*ExecutorProfileHandlers).wsCreateProfile, + }, + { + name: "update", action: ws.ActionExecutorProfileUpdate, + payload: map[string]any{"id": "pr-kubernetes", "name": "Renamed"}, + invoke: (*ExecutorProfileHandlers).wsUpdateProfile, + }, + { + name: "delete", action: ws.ActionExecutorProfileDelete, + payload: map[string]any{"id": "pr-kubernetes"}, + invoke: (*ExecutorProfileHandlers).wsDeleteProfile, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + handler := NewExecutorProfileHandlers(newExecutorService(t, repo), nil, log) + + response, err := test.invoke( + handler, kubernetesMemberContext(), wsWorkflowRequest(t, test.action, test.payload), + ) + + require.NoError(t, err) + payload := wsWorkflowError(t, response) + require.Equal(t, string(ws.ErrorCodeForbidden), payload.Code) + require.Equal(t, service.ErrKubernetesAdminRequired.Error(), payload.Message) + require.Empty(t, repo.createdProfile) + require.Empty(t, repo.updatedProfile) + require.Empty(t, repo.deletedProfile) + require.Zero(t, observed.Len(), "expected authorization denial must not be logged as internal") + }) + } +} + +func TestHTTPKubernetesProfileValidationReturnsBadRequestWithoutInternalLog(t *testing.T) { + tests := []struct { + name string + method string + target string + params gin.Params + invoke func(*ExecutorProfileHandlers, *gin.Context) + }{ + { + name: "create", method: http.MethodPost, + target: "/api/v1/executors/ex-kubernetes/profiles", + params: gin.Params{{Key: "id", Value: "ex-kubernetes"}}, + invoke: (*ExecutorProfileHandlers).httpCreateProfile, + }, + { + name: "update", method: http.MethodPatch, + target: "/api/v1/executors/ex-kubernetes/profiles/pr-kubernetes", + params: gin.Params{{Key: "id", Value: "ex-kubernetes"}, {Key: "profileId", Value: "pr-kubernetes"}}, + invoke: (*ExecutorProfileHandlers).httpUpdateProfile, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + handler := NewExecutorProfileHandlers(newExecutorService(t, repo), nil, log) + ctx, recorder := newWorkflowRequest( + t, test.method, test.target, + `{"name":"Invalid","config":{"platform":"windows/amd64"}}`, + ) + ctx.Params = test.params + ctx.Request = ctx.Request.WithContext(kubernetesAdminContext()) + + test.invoke(handler, ctx) + + require.Equal(t, http.StatusBadRequest, recorder.Code) + require.Contains(t, recorder.Body.String(), service.ErrInvalidExecutorConfig.Error()) + require.Contains(t, recorder.Body.String(), "config.platform") + require.Empty(t, repo.createdProfile) + require.Empty(t, repo.updatedProfile) + require.Zero(t, observed.Len(), "expected validation denial must not be logged as internal") + }) + } +} + +func TestWSKubernetesProfileValidationReturnsValidationWithoutInternalLog(t *testing.T) { + tests := []struct { + name string + action string + payload map[string]any + invoke func(*ExecutorProfileHandlers, context.Context, *ws.Message) (*ws.Message, error) + }{ + { + name: "create", action: ws.ActionExecutorProfileCreate, + payload: map[string]any{ + "executor_id": "ex-kubernetes", "name": "Invalid", + "config": map[string]string{"platform": "windows/amd64"}, + }, + invoke: (*ExecutorProfileHandlers).wsCreateProfile, + }, + { + name: "update", action: ws.ActionExecutorProfileUpdate, + payload: map[string]any{ + "id": "pr-kubernetes", "config": map[string]string{"platform": "windows/amd64"}, + }, + invoke: (*ExecutorProfileHandlers).wsUpdateProfile, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + handler := NewExecutorProfileHandlers(newExecutorService(t, repo), nil, log) + + response, err := test.invoke( + handler, kubernetesAdminContext(), wsWorkflowRequest(t, test.action, test.payload), + ) + + require.NoError(t, err) + payload := wsWorkflowError(t, response) + require.Equal(t, string(ws.ErrorCodeValidation), payload.Code) + require.Equal(t, service.ErrInvalidExecutorConfig.Error(), payload.Message) + require.Empty(t, repo.createdProfile) + require.Empty(t, repo.updatedProfile) + require.Zero(t, observed.Len(), "expected validation denial must not be logged as internal") + }) + } +} + +func TestKubernetesExecutorAndProfileReadsRemainAvailableToMemberHandlers(t *testing.T) { + repo := kubernetesHandlerFixture() + log, observed := observedErrorLogger(t) + executorHandler := NewExecutorHandlers(newExecutorService(t, repo), log) + profileHandler := NewExecutorProfileHandlers(newExecutorService(t, repo), nil, log) + + executorCtx, executorRecorder := newWorkflowRequest( + t, http.MethodGet, "/api/v1/executors/ex-kubernetes", "", + ) + executorCtx.Params = gin.Params{{Key: "id", Value: "ex-kubernetes"}} + executorCtx.Request = executorCtx.Request.WithContext(kubernetesMemberContext()) + executorHandler.httpGetExecutor(executorCtx) + + profileCtx, profileRecorder := newWorkflowRequest( + t, http.MethodGet, "/api/v1/executors/ex-kubernetes/profiles/pr-kubernetes", "", + ) + profileCtx.Params = gin.Params{{Key: "profileId", Value: "pr-kubernetes"}} + profileCtx.Request = profileCtx.Request.WithContext(kubernetesMemberContext()) + profileHandler.httpGetProfile(profileCtx) + + require.Equal(t, http.StatusOK, executorRecorder.Code) + require.Equal(t, http.StatusOK, profileRecorder.Code) + require.Zero(t, observed.Len()) +} + +func kubernetesHandlerFixture() *executorRepo { + repo := executorFixture() + executor := &models.Executor{ + ID: "ex-kubernetes", Name: "Kubernetes", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validHandlerKubernetesExecutorConfig(), + } + profile := &models.ExecutorProfile{ + ID: "pr-kubernetes", ExecutorID: executor.ID, Name: "Default", + Config: validHandlerKubernetesProfileConfig(), + } + repo.executors = append(repo.executors, executor) + repo.byID[executor.ID] = executor + repo.profiles = append(repo.profiles, profile) + repo.profilesByExecutor[executor.ID] = []*models.ExecutorProfile{profile} + repo.profileByID[profile.ID] = profile + return repo +} + +func validHandlerKubernetesExecutorConfig() map[string]string { + return map[string]string{ + "auth_mode": "kubeconfig", "kubeconfig_path": "/etc/kandev/kubeconfig", + "namespace": "kandev", "request_timeout_seconds": "30", + } +} + +func validHandlerKubernetesProfileConfig() map[string]string { + return map[string]string{ + "platform": "linux/amd64", + "main_container": "kandev-agent", + "pod_template_yaml": `apiVersion: v1 +kind: PodTemplate +template: + spec: + containers: + - name: kandev-agent + image: alpine:3.20 +`, + "workspace.mode": "empty_dir", + } +} + +func kubernetesMemberContext() context.Context { + return authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", Role: authn.RoleMember, + }) +} + +func kubernetesAdminContext() context.Context { + return authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "admin-1", Role: authn.RoleAdmin, + }) +} + +func observedErrorLogger(t *testing.T) (*logger.Logger, *observer.ObservedLogs) { + t.Helper() + core, observed := observer.New(zap.ErrorLevel) + log, err := logger.NewFromZap(zap.New(core)) + require.NoError(t, err) + return log, observed +} diff --git a/apps/backend/internal/task/handlers/executor_profile_handlers.go b/apps/backend/internal/task/handlers/executor_profile_handlers.go index 120e36226a..98a3e5b9a8 100644 --- a/apps/backend/internal/task/handlers/executor_profile_handlers.go +++ b/apps/backend/internal/task/handlers/executor_profile_handlers.go @@ -2,6 +2,7 @@ package handlers import ( "context" + "errors" "net/http" "strings" @@ -213,6 +214,14 @@ func (h *ExecutorProfileHandlers) httpCreateProfile(c *gin.Context) { EnvVars: body.EnvVars, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } + if errors.Is(err, service.ErrInvalidExecutorConfig) { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } h.logger.Error("failed to create executor profile", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{"error": "profile not created"}) return @@ -253,6 +262,14 @@ func (h *ExecutorProfileHandlers) httpUpdateProfile(c *gin.Context) { EnvVars: body.EnvVars, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } + if errors.Is(err, service.ErrInvalidExecutorConfig) { + c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + return + } h.logger.Error("failed to update executor profile", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{"error": "profile not updated"}) return @@ -262,6 +279,10 @@ func (h *ExecutorProfileHandlers) httpUpdateProfile(c *gin.Context) { func (h *ExecutorProfileHandlers) httpDeleteProfile(c *gin.Context) { if err := h.service.DeleteExecutorProfile(c.Request.Context(), c.Param("profileId")); err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + c.JSON(http.StatusForbidden, gin.H{"error": err.Error()}) + return + } h.logger.Error("failed to delete executor profile", zap.Error(err)) c.JSON(http.StatusBadRequest, gin.H{"error": "profile not deleted"}) return @@ -351,6 +372,12 @@ func (h *ExecutorProfileHandlers) wsCreateProfile(ctx context.Context, msg *ws.M EnvVars: req.EnvVars, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, service.ErrKubernetesAdminRequired.Error(), nil) + } + if errors.Is(err, service.ErrInvalidExecutorConfig) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, service.ErrInvalidExecutorConfig.Error(), nil) + } h.logger.Error("failed to create executor profile", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to create profile", nil) } @@ -403,6 +430,12 @@ func (h *ExecutorProfileHandlers) wsUpdateProfile(ctx context.Context, msg *ws.M EnvVars: req.EnvVars, }) if err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, service.ErrKubernetesAdminRequired.Error(), nil) + } + if errors.Is(err, service.ErrInvalidExecutorConfig) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, service.ErrInvalidExecutorConfig.Error(), nil) + } h.logger.Error("failed to update executor profile", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to update profile", nil) } @@ -418,6 +451,9 @@ func (h *ExecutorProfileHandlers) wsDeleteProfile(ctx context.Context, msg *ws.M return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "id is required", nil) } if err := h.service.DeleteExecutorProfile(ctx, req.ID); err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, service.ErrKubernetesAdminRequired.Error(), nil) + } h.logger.Error("failed to delete executor profile", zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to delete profile", nil) } diff --git a/apps/backend/internal/task/handlers/process_metrics_handlers.go b/apps/backend/internal/task/handlers/process_metrics_handlers.go index 20947b9164..03b5e4c199 100644 --- a/apps/backend/internal/task/handlers/process_metrics_handlers.go +++ b/apps/backend/internal/task/handlers/process_metrics_handlers.go @@ -9,6 +9,8 @@ import ( agentctlmetrics "github.com/kandev/kandev/internal/system/metrics" ) +const agentctlNotReadyError = "agentctl not ready" + var sessionAgentctlDiagnosticMetricIDs = []string{ agentctlmetrics.MetricAgentctlGoroutines, agentctlmetrics.MetricAgentctlGitPollMillis, @@ -32,12 +34,18 @@ func (h *ProcessHandlers) httpGetAgentctlMetrics(c *gin.Context) { } execution, found := h.lifecycleMgr.GetExecutionBySessionID(sessionID) - if !found || execution == nil || execution.GetAgentCtlClient() == nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "agentctl not ready"}) + if !found || execution == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": agentctlNotReadyError}) + return + } + client, releaseClient := execution.AcquireAgentCtlClient() + defer releaseClient() + if client == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": agentctlNotReadyError}) return } - snapshot, err := execution.GetAgentCtlClient().SystemMetrics( + snapshot, err := client.SystemMetrics( c.Request.Context(), sessionAgentctlDiagnosticMetricIDs, "", ) if err != nil { diff --git a/apps/backend/internal/task/handlers/ws_helpers.go b/apps/backend/internal/task/handlers/ws_helpers.go index 3c7de674e1..2493d363b6 100644 --- a/apps/backend/internal/task/handlers/ws_helpers.go +++ b/apps/backend/internal/task/handlers/ws_helpers.go @@ -33,6 +33,9 @@ func wsDeleteWithActiveSessionCheck( return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "id is required", nil) } if err := deleteFn(ctx, req.ID); err != nil { + if errors.Is(err, service.ErrKubernetesAdminRequired) { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeForbidden, service.ErrKubernetesAdminRequired.Error(), nil) + } log.Error("failed to delete "+resourceName, zap.Error(err)) if errors.Is(err, service.ErrActiveTaskSessions) { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, resourceName+" is used by an active agent session", nil) diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index 8fb9e4f7b5..35986f9c28 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -1825,6 +1825,7 @@ const ( ExecutorTypeRemoteDocker ExecutorType = "remote_docker" ExecutorTypeSprites ExecutorType = "sprites" ExecutorTypeSSH ExecutorType = "ssh" + ExecutorTypeKubernetes ExecutorType = "k8s" ExecutorTypeMockRemote ExecutorType = "mock_remote" ) @@ -1833,7 +1834,7 @@ const ( // These environments run shells inside the container/VM, not on the host. func IsRemoteExecutorType(t ExecutorType) bool { switch t { - case ExecutorTypeSprites, ExecutorTypeRemoteDocker, ExecutorTypeLocalDocker, ExecutorTypeSSH, ExecutorTypeMockRemote: + case ExecutorTypeSprites, ExecutorTypeRemoteDocker, ExecutorTypeLocalDocker, ExecutorTypeSSH, ExecutorTypeKubernetes, ExecutorTypeMockRemote: return true default: return false @@ -1857,6 +1858,8 @@ func (t ExecutorType) Runtime() agentruntime.Runtime { return agentruntime.RuntimeSprites case ExecutorTypeSSH: return agentruntime.RuntimeSSH + case ExecutorTypeKubernetes: + return agentruntime.RuntimeKubernetes default: return agentruntime.RuntimeStandalone } @@ -1874,7 +1877,7 @@ func IsContainerizedExecutorType(t ExecutorType) bool { // IsAlwaysResumableRuntime reports whether the given runtime represents // an executor that can always be resumed even without an explicit resume token. func IsAlwaysResumableRuntime(runtime agentruntime.Runtime) bool { - return runtime == agentruntime.RuntimeSprites || runtime == agentruntime.RuntimeSSH + return runtime == agentruntime.RuntimeSprites || runtime == agentruntime.RuntimeSSH || runtime == agentruntime.RuntimeKubernetes } const ( diff --git a/apps/backend/internal/task/models/models_test.go b/apps/backend/internal/task/models/models_test.go index f31a24544b..9cfe9d135d 100644 --- a/apps/backend/internal/task/models/models_test.go +++ b/apps/backend/internal/task/models/models_test.go @@ -448,6 +448,7 @@ func TestExecutorTypeRuntime(t *testing.T) { {ExecutorTypeLocalDocker, agentruntime.RuntimeDocker}, {ExecutorTypeRemoteDocker, agentruntime.RuntimeRemoteDocker}, {ExecutorTypeSprites, agentruntime.RuntimeSprites}, + {ExecutorTypeKubernetes, agentruntime.RuntimeKubernetes}, } for _, tc := range cases { t.Run(string(tc.in), func(t *testing.T) { @@ -468,3 +469,26 @@ func TestExecutorTypeRuntime(t *testing.T) { } }) } + +func TestKubernetesExecutorClassification(t *testing.T) { + t.Parallel() + + executorType := ExecutorTypeKubernetes + if got := executorType.Runtime(); got != agentruntime.RuntimeKubernetes { + t.Fatalf("ExecutorType(k8s).Runtime() = %q, want k8s", got) + } + if !IsRemoteExecutorType(executorType) { + t.Fatal("IsRemoteExecutorType(k8s) = false, want true") + } + if !IsContainerizedExecutorType(executorType) { + t.Fatal("IsContainerizedExecutorType(k8s) = false, want true") + } +} + +func TestKubernetesRuntimeIsAlwaysResumable(t *testing.T) { + t.Parallel() + + if !IsAlwaysResumableRuntime(agentruntime.RuntimeKubernetes) { + t.Fatal("IsAlwaysResumableRuntime(k8s) = false, want true") + } +} diff --git a/apps/backend/internal/task/service/kubernetes_capability_contract_test.go b/apps/backend/internal/task/service/kubernetes_capability_contract_test.go new file mode 100644 index 0000000000..3e98db71a8 --- /dev/null +++ b/apps/backend/internal/task/service/kubernetes_capability_contract_test.go @@ -0,0 +1,76 @@ +package service + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +// These cases document existing fail-closed behavior for Kubernetes-specific +// capability boundaries; they are parity contract coverage, not fix tests. +func TestKubernetesWorkspaceSourceCapabilities(t *testing.T) { + t.Run("folder sources are rejected before host path resolution", func(t *testing.T) { + svc, taskID := newKubernetesWorkspaceSourceTestService(t) + _, err := svc.AttachWorkspaceSources(context.Background(), AttachWorkspaceSourcesRequest{ + TaskID: taskID, + Sources: []WorkspaceSourceInput{{ + Kind: WorkspaceSourceFolder, LocalPath: filepath.Join(t.TempDir(), "missing"), DisplayName: "docs", + }}, + }) + if !errors.Is(err, ErrUnsupportedWorkspaceSource) { + t.Fatalf("error = %v, want unsupported workspace source", err) + } + }) + + t.Run("host sibling worktree branch add is rejected", func(t *testing.T) { + svc, taskID := newKubernetesWorkspaceSourceTestService(t) + _, err := svc.AddBranchToTask(context.Background(), AddBranchToTaskRequest{ + TaskID: taskID, CheckoutBranch: "feature/other", + }) + if err == nil || !strings.Contains(err.Error(), "worktree executor") { + t.Fatalf("error = %v, want worktree executor rejection", err) + } + }) +} + +func newKubernetesWorkspaceSourceTestService(t *testing.T) (*Service, string) { + t.Helper() + svc, _, repo := createTestService(t) + svc.workspaceFolders = repo + ctx := context.Background() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "ws-k8s", Name: "Kubernetes"}); err != nil { + t.Fatal(err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "wf-k8s", WorkspaceID: "ws-k8s", Name: "Workflow"}); err != nil { + t.Fatal(err) + } + if err := repo.CreateRepository(ctx, &models.Repository{ + ID: "repo-k8s", WorkspaceID: "ws-k8s", Name: "api", DefaultBranch: "main", + }); err != nil { + t.Fatal(err) + } + result, err := svc.CreateTask(ctx, &CreateTaskRequest{ + WorkspaceID: "ws-k8s", WorkflowID: "wf-k8s", WorkflowStepID: "step-k8s", Title: "Task", + Repositories: []TaskRepositoryInput{{RepositoryID: "repo-k8s", BaseBranch: "main"}}, + }) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + if err := repo.CreateTaskEnvironment(ctx, &models.TaskEnvironment{ + ID: "env-k8s", TaskID: result.Task.ID, ExecutorType: string(models.ExecutorTypeKubernetes), + Status: models.TaskEnvironmentStatusReady, CreatedAt: now, UpdatedAt: now, + Repos: []*models.TaskEnvironmentRepo{{ + RepositoryID: "repo-k8s", + Position: 0, + }}, + }); err != nil { + t.Fatal(err) + } + return svc, result.Task.ID +} diff --git a/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go b/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go new file mode 100644 index 0000000000..cdbca0c5a9 --- /dev/null +++ b/apps/backend/internal/task/service/service_kubernetes_restart_cleanup_test.go @@ -0,0 +1,226 @@ +package service + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + agentexecutor "github.com/kandev/kandev/internal/agent/executor" + "github.com/kandev/kandev/internal/agent/runtime/lifecycle" + "github.com/kandev/kandev/internal/agentctl/server/process" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/task/models" +) + +func TestCleanupTaskResourcesRecoversPersistedKubernetesRuntimeAfterRestart(t *testing.T) { + tests := []struct { + name string + stopErr error + wantDeleted bool + }{ + {name: "exact cleanup succeeds", wantDeleted: true}, + {name: "exact cleanup failure stays retryable", stopErr: errors.New("RBAC denied exact cleanup")}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + seedPersistedKubernetesCleanupFixture(t, ctx, repo) + + backend := &persistedKubernetesCleanupBackend{stopErr: test.stopErr} + registry := lifecycle.NewExecutorRegistry(svc.logger) + registry.Register(backend) + manager := lifecycle.NewManager( + nil, svc.eventBus, registry, nil, nil, nil, + lifecycle.ExecutorFallbackWarn, "", svc.logger, + ) + manager.SetExecutorRunningWriter(repo) + t.Cleanup(func() { _ = manager.Stop() }) + svc.SetExecutionStopper(&lifecycleManagerTaskStopper{manager: manager}) + svc.setCleanupDoneForTestHook(make(chan struct{}, 1)) + + quickChatRoot := t.TempDir() + svc.SetQuickChatDir(quickChatRoot) + sessionDir := filepath.Join(quickChatRoot, "session-k8s") + if err := os.MkdirAll(sessionDir, 0o755); err != nil { + t.Fatalf("create session dir: %v", err) + } + + svc.CleanupTaskResources(ctx, "task-k8s", false) + waitForCleanupDone(t, svc) + + _, rowErr := repo.GetExecutorRunningBySessionID(ctx, "session-k8s") + if test.wantDeleted { + if !backend.podDeleted || !backend.pvcDeleted { + t.Fatalf("exact Kubernetes cleanup = pod:%t pvc:%t, want both", backend.podDeleted, backend.pvcDeleted) + } + if !errors.Is(rowErr, models.ErrExecutorRunningNotFound) { + t.Fatalf("executor row remains after exact cleanup: %v", rowErr) + } + if _, err := os.Stat(sessionDir); !os.IsNotExist(err) { + t.Fatalf("session dir remains after exact cleanup: %v", err) + } + return + } + if rowErr != nil { + t.Fatalf("failed cleanup lost authoritative executor row: %v", rowErr) + } + if backend.podDeleted || backend.pvcDeleted { + t.Fatalf("failed cleanup reported deletion = pod:%t pvc:%t", backend.podDeleted, backend.pvcDeleted) + } + if _, err := os.Stat(sessionDir); err != nil { + t.Fatalf("failed cleanup removed retryable session resources: %v", err) + } + }) + } +} + +func seedPersistedKubernetesCleanupFixture( + t *testing.T, + ctx context.Context, + repo interface { + CreateWorkspace(context.Context, *models.Workspace) error + CreateWorkflow(context.Context, *models.Workflow) error + CreateTask(context.Context, *models.Task) error + CreateTaskSession(context.Context, *models.TaskSession) error + CreateExecutor(context.Context, *models.Executor) error + UpsertExecutorRunning(context.Context, *models.ExecutorRunning) error + }, +) { + t.Helper() + if err := repo.CreateWorkspace(ctx, &models.Workspace{ID: "workspace-k8s", Name: "WS"}); err != nil { + t.Fatalf("create workspace: %v", err) + } + if err := repo.CreateWorkflow(ctx, &models.Workflow{ID: "workflow-k8s", WorkspaceID: "workspace-k8s", Name: "WF"}); err != nil { + t.Fatalf("create workflow: %v", err) + } + if err := repo.CreateTask(ctx, &models.Task{ + ID: "task-k8s", WorkspaceID: "workspace-k8s", WorkflowID: "workflow-k8s", + WorkflowStepID: "step-k8s", Title: "Kubernetes", Priority: "medium", + }); err != nil { + t.Fatalf("create task: %v", err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-k8s", TaskID: "task-k8s", State: models.TaskSessionStateCancelled, + AgentExecutionID: "execution-k8s", + }); err != nil { + t.Fatalf("create task session: %v", err) + } + if err := repo.CreateExecutor(ctx, &models.Executor{ + ID: "executor-k8s", Name: "Kubernetes", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + }); err != nil { + t.Fatalf("create executor: %v", err) + } + metadata := map[string]interface{}{ + "executor_id": "executor-k8s", + lifecycle.MetadataKeyExecutorProfileID: "profile-k8s", + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/stale/kubeconfig", + lifecycle.MetadataKeyKubernetesKubeContext: "stale-context", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "10", + lifecycle.MetadataKeyKubernetesNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesPodName: "pod-k8s", + lifecycle.MetadataKeyKubernetesPodUID: "pod-uid", + lifecycle.MetadataKeyKubernetesMainContainer: "kandev-agent", + lifecycle.MetadataKeyKubernetesPlatform: "linux/amd64", + lifecycle.MetadataKeyKubernetesRuntimeWorkspaceMode: "managed_pvc", + lifecycle.MetadataKeyKubernetesPVCName: "pvc-k8s", + lifecycle.MetadataKeyKubernetesPVCUID: "pvc-uid", + lifecycle.MetadataKeyKubernetesPVCCreated: true, + lifecycle.MetadataKeyKubernetesAgentctlRemotePort: "41001", + lifecycle.MetadataKeyKubernetesAgentctlInstanceID: "resource-k8s", + lifecycle.MetadataKeyKubernetesResourceInstanceID: "resource-k8s", + lifecycle.MetadataKeyKubernetesResourceExecutorID: "executor-k8s", + lifecycle.MetadataKeyKubernetesResourceProfileID: "profile-k8s", + lifecycle.MetadataKeyKubernetesResourceTaskID: "task-k8s", + lifecycle.MetadataKeyKubernetesResourceSessionID: "session-k8s", + lifecycle.MetadataKeyKubernetesResourceEnvironmentID: "environment-k8s", + lifecycle.MetadataKeyKubernetesProfileSnapshot: `{}`, + lifecycle.MetadataKeyIsRemote: true, + } + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "session-k8s", SessionID: "session-k8s", TaskID: "task-k8s", + ExecutorID: "executor-k8s", ExecutionProfileID: "agent-profile-k8s", + AgentExecutionID: "execution-k8s", Runtime: agentruntime.RuntimeKubernetes, + Status: models.ExecutorRunningStatusRunning, Metadata: metadata, + }); err != nil { + t.Fatalf("upsert executor running: %v", err) + } +} + +type lifecycleManagerTaskStopper struct { + manager *lifecycle.Manager +} + +func (*lifecycleManagerTaskStopper) StopTask(context.Context, string, string, bool) error { return nil } +func (*lifecycleManagerTaskStopper) StopSession(context.Context, string, string, bool) error { + return errors.New("session-only stop is not expected") +} +func (s *lifecycleManagerTaskStopper) StopExecution( + ctx context.Context, + executionID string, + reason string, + force bool, +) error { + return s.manager.StopAgentWithReason(ctx, executionID, reason, force) +} +func (*lifecycleManagerTaskStopper) RegisterExecutionStopOwner(string, string, bool) {} + +type persistedKubernetesCleanupBackend struct { + stopErr error + podDeleted bool + pvcDeleted bool +} + +func (*persistedKubernetesCleanupBackend) Name() agentexecutor.Name { + return agentexecutor.NameKubernetes +} +func (*persistedKubernetesCleanupBackend) HealthCheck(context.Context) error { return nil } +func (*persistedKubernetesCleanupBackend) CreateInstance( + context.Context, + *lifecycle.ExecutorCreateRequest, +) (*lifecycle.ExecutorInstance, error) { + return nil, errors.New("create is not expected") +} +func (b *persistedKubernetesCleanupBackend) StopInstance( + _ context.Context, + instance *lifecycle.ExecutorInstance, + force bool, +) error { + if instance == nil || instance.TaskID != "task-k8s" || instance.SessionID != "session-k8s" || + instance.InstanceID != "execution-k8s" || !force || + instance.Metadata[lifecycle.MetadataKeyKubernetesPodName] != "pod-k8s" || + instance.Metadata[lifecycle.MetadataKeyKubernetesPodUID] != "pod-uid" || + instance.Metadata[lifecycle.MetadataKeyKubernetesPVCName] != "pvc-k8s" || + instance.Metadata[lifecycle.MetadataKeyKubernetesPVCUID] != "pvc-uid" || + instance.Metadata[lifecycle.MetadataKeyKubernetesAuthMode] != "in_cluster" || + instance.Metadata[lifecycle.MetadataKeyKubernetesKubeconfigPath] != "" || + instance.Metadata[lifecycle.MetadataKeyKubernetesKubeContext] != "" || + instance.Metadata[lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds] != "45" { + return errors.New("reconstructed Kubernetes cleanup instance is not exact") + } + if b.stopErr != nil { + return b.stopErr + } + b.podDeleted = true + b.pvcDeleted = true + return nil +} +func (*persistedKubernetesCleanupBackend) RecoverInstances(context.Context) ([]*lifecycle.ExecutorInstance, error) { + return nil, nil +} +func (*persistedKubernetesCleanupBackend) GetInteractiveRunner() *process.InteractiveRunner { + return nil +} +func (*persistedKubernetesCleanupBackend) RequiresCloneURL() bool { return true } +func (*persistedKubernetesCleanupBackend) ShouldApplyPreferredShell() bool { return false } +func (*persistedKubernetesCleanupBackend) IsAlwaysResumable() bool { return true } diff --git a/apps/backend/internal/task/service/service_resources.go b/apps/backend/internal/task/service/service_resources.go index a237a03112..9a161f9158 100644 --- a/apps/backend/internal/task/service/service_resources.go +++ b/apps/backend/internal/task/service/service_resources.go @@ -15,6 +15,9 @@ import ( "github.com/jmoiron/sqlx" "go.uber.org/zap" + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/auth/authn" "github.com/kandev/kandev/internal/common/securityutil" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/secrets" @@ -1492,8 +1495,43 @@ func (s *Service) ListScriptsByRepositoryIDs(ctx context.Context, repoIDs []stri // Executor operations +var ErrKubernetesAdminRequired = errors.New("administrator identity required for Kubernetes settings") + +func requireKubernetesAdmin(ctx context.Context) error { + identity, ok := authn.IdentityFromContext(ctx) + if !ok || !identity.IsAdmin() { + return ErrKubernetesAdminRequired + } + return nil +} + +func validateExecutorForType(executorType models.ExecutorType, config map[string]string) error { + if err := validateExecutorConfig(config); err != nil { + return err + } + if executorType != models.ExecutorTypeKubernetes { + return nil + } + if _, err := agentkubernetes.ParseExecutorConfig(config); err != nil { + return fmt.Errorf("%w: %w", ErrInvalidExecutorConfig, err) + } + return nil +} + +func validateKubernetesProfileConfig(config map[string]string) error { + if _, err := agentkubernetes.ParseProfileConfig(config); err != nil { + return fmt.Errorf("%w: %w", ErrInvalidExecutorConfig, err) + } + return nil +} + func (s *Service) CreateExecutor(ctx context.Context, req *CreateExecutorRequest) (*models.Executor, error) { - if err := validateExecutorConfig(req.Config); err != nil { + if req.Type == models.ExecutorTypeKubernetes { + if err := requireKubernetesAdmin(ctx); err != nil { + return nil, err + } + } + if err := validateExecutorForType(req.Type, req.Config); err != nil { return nil, err } executor := &models.Executor{ @@ -1522,9 +1560,30 @@ func (s *Service) UpdateExecutor(ctx context.Context, id string, req *UpdateExec if err != nil { return nil, err } + targetType := executor.Type + if req.Type != nil { + targetType = *req.Type + } + if executor.Type == models.ExecutorTypeKubernetes || targetType == models.ExecutorTypeKubernetes { + if err := requireKubernetesAdmin(ctx); err != nil { + return nil, err + } + } if err := validateExecutorUpdateRequest(executor, req); err != nil { return nil, err } + if req.Type != nil && *req.Type != executor.Type && + (executor.Type == models.ExecutorTypeKubernetes || *req.Type == models.ExecutorTypeKubernetes) { + retained, inventoryErr := s.hasExecutorRunningInventory(ctx, id) + if inventoryErr != nil { + s.logger.Error("failed to check retained Kubernetes inventory before executor type change", + zap.String("executor_id", id), zap.Error(inventoryErr)) + return nil, inventoryErr + } + if retained { + return nil, ErrActiveTaskSessions + } + } applyExecutorUpdates(executor, req) executor.UpdatedAt = time.Now().UTC() if err := s.executors.UpdateExecutor(ctx, executor); err != nil { @@ -1536,8 +1595,16 @@ func (s *Service) UpdateExecutor(ctx context.Context, id string, req *UpdateExec // validateExecutorUpdateRequest validates config and system executor constraints. func validateExecutorUpdateRequest(executor *models.Executor, req *UpdateExecutorRequest) error { + targetType := executor.Type + if req.Type != nil { + targetType = *req.Type + } if req.Config != nil { - if err := validateExecutorConfig(req.Config); err != nil { + if err := validateExecutorForType(targetType, req.Config); err != nil { + return err + } + } else if targetType == models.ExecutorTypeKubernetes { + if err := validateExecutorForType(targetType, executor.Config); err != nil { return err } } @@ -1583,6 +1650,11 @@ func (s *Service) DeleteExecutor(ctx context.Context, id string) error { if err != nil { return err } + if executor.Type == models.ExecutorTypeKubernetes { + if err := requireKubernetesAdmin(ctx); err != nil { + return err + } + } if executor.IsSystem { return fmt.Errorf("system executors cannot be deleted") } @@ -1594,6 +1666,17 @@ func (s *Service) DeleteExecutor(ctx context.Context, id string) error { if active { return ErrActiveTaskSessions } + if executor.Type == models.ExecutorTypeKubernetes { + retained, inventoryErr := s.hasExecutorRunningInventory(ctx, id) + if inventoryErr != nil { + s.logger.Error("failed to check retained Kubernetes inventory for executor", + zap.String("executor_id", id), zap.Error(inventoryErr)) + return inventoryErr + } + if retained { + return ErrActiveTaskSessions + } + } if err := s.executors.DeleteExecutor(ctx, id); err != nil { return err } @@ -1601,6 +1684,34 @@ func (s *Service) DeleteExecutor(ctx context.Context, id string) error { return nil } +func (s *Service) hasExecutorRunningInventory(ctx context.Context, executorID string) (bool, error) { + running, err := s.executors.ListExecutorsRunning(ctx) + if err != nil { + return false, err + } + for _, row := range running { + if row == nil { + continue + } + currentID := strings.TrimSpace(row.ExecutorID) + recordedID, recordedOK := row.Metadata[agentkubernetes.MetadataKeyResourceExecutorID].(string) + recordedID = strings.TrimSpace(recordedID) + if currentID == executorID { + return true, nil + } + if recordedID == executorID { + return true, nil + } + // Only two matching, nonempty associations can prove that a Kubernetes + // row belongs to some other executor. Anything less must block cleanup. + if row.Runtime == agentruntime.RuntimeKubernetes && + (currentID == "" || !recordedOK || recordedID == "" || currentID != recordedID) { + return true, nil + } + } + return false, nil +} + func (s *Service) ListExecutors(ctx context.Context) ([]*models.Executor, error) { return s.executors.ListExecutors(ctx) } @@ -1614,14 +1725,22 @@ func (s *Service) CreateExecutorProfile(ctx context.Context, req *CreateExecutor if req.ExecutorID == "" { return nil, fmt.Errorf("executor_id is required") } - if err := s.validateGlobalProfileEnvRefs(ctx, req.EnvVars); err != nil { - return nil, err - } // Verify executor exists executor, err := s.executors.GetExecutor(ctx, req.ExecutorID) if err != nil { return nil, fmt.Errorf("executor not found: %w", err) } + if executor.Type == models.ExecutorTypeKubernetes { + if err := requireKubernetesAdmin(ctx); err != nil { + return nil, err + } + if err := validateKubernetesProfileConfig(req.Config); err != nil { + return nil, err + } + } + if err := s.validateGlobalProfileEnvRefs(ctx, req.EnvVars); err != nil { + return nil, err + } if executor.Type == models.ExecutorTypeSprites && !hasSpritesToken(req.EnvVars) { return nil, fmt.Errorf("sprites profiles require %s env var", spritesTokenEnvKey) } @@ -1654,6 +1773,18 @@ func (s *Service) UpdateExecutorProfile(ctx context.Context, id string, req *Upd if err != nil { return nil, err } + if executor.Type == models.ExecutorTypeKubernetes { + if err := requireKubernetesAdmin(ctx); err != nil { + return nil, err + } + config := profile.Config + if req.Config != nil { + config = req.Config + } + if err := validateKubernetesProfileConfig(config); err != nil { + return nil, err + } + } if req.Name != nil { profile.Name = *req.Name } @@ -1755,6 +1886,15 @@ func (s *Service) DeleteExecutorProfile(ctx context.Context, id string) error { if err != nil { return err } + executor, err := s.executors.GetExecutor(ctx, profile.ExecutorID) + if err != nil && !errors.Is(err, models.ErrExecutorNotFound) { + return err + } + if executor != nil && executor.Type == models.ExecutorTypeKubernetes { + if err := requireKubernetesAdmin(ctx); err != nil { + return err + } + } if err := s.executors.DeleteExecutorProfile(ctx, id); err != nil { return err } diff --git a/apps/backend/internal/task/service/service_resources_kubernetes_test.go b/apps/backend/internal/task/service/service_resources_kubernetes_test.go new file mode 100644 index 0000000000..55e4bab0fc --- /dev/null +++ b/apps/backend/internal/task/service/service_resources_kubernetes_test.go @@ -0,0 +1,599 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agentruntime" + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/secrets" + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/task/repository" +) + +func TestCreateKubernetesExecutorRejectsMember(t *testing.T) { + svc, _, _ := createTestService(t) + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", + Role: authn.RoleMember, + }) + + _, err := svc.CreateExecutor(ctx, &CreateExecutorRequest{ + Name: "Cluster", + Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: validKubernetesExecutorConfig(), + }) + if err == nil { + t.Fatal("member created a Kubernetes executor, want authorization error") + } +} + +func TestCreateKubernetesExecutorValidatesTypedConfig(t *testing.T) { + svc, _, _ := createTestService(t) + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "admin-1", + Role: authn.RoleAdmin, + }) + + _, err := svc.CreateExecutor(ctx, &CreateExecutorRequest{ + Name: "Cluster", + Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: map[string]string{ + "auth_mode": "kubeconfig", + "kubeconfig_path": "relative/config", + "namespace": "kandev", + "request_timeout_seconds": "30", + }, + }) + var fieldErr *agentkubernetes.FieldError + if !errors.As(err, &fieldErr) || fieldErr.Path != "config.kubeconfig_path" { + t.Fatalf("error = %v, want config.kubeconfig_path FieldError", err) + } +} + +func TestCreateKubernetesExecutorRejectsMissingIdentity(t *testing.T) { + svc, _, _ := createTestService(t) + + _, err := svc.CreateExecutor(context.Background(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } +} + +func TestKubernetesMutationsAllowSyntheticAdmin(t *testing.T) { + svc, _, _ := createTestService(t) + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "default-user", Role: authn.RoleAdmin, Synthetic: true, + }) + + executor, err := svc.CreateExecutor(ctx, &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil || executor == nil { + t.Fatalf("CreateExecutor() = %+v, %v; want success", executor, err) + } +} + +func TestUpdateExecutorRejectsMemberTransitionToKubernetes(t *testing.T) { + svc, _, _ := createTestService(t) + executor := createTestExecutor(t, svc, "Runner", models.ExecutorTypeLocal) + kubernetesType := models.ExecutorTypeKubernetes + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", + Role: authn.RoleMember, + }) + + _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{ + Type: &kubernetesType, + Config: validKubernetesExecutorConfig(), + }) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } +} + +func TestUpdateExecutorValidatesTransitionToKubernetes(t *testing.T) { + svc, _, _ := createTestService(t) + executor := createTestExecutor(t, svc, "Runner", models.ExecutorTypeLocal) + kubernetesType := models.ExecutorTypeKubernetes + ctx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "admin-1", + Role: authn.RoleAdmin, + }) + + _, err := svc.UpdateExecutor(ctx, executor.ID, &UpdateExecutorRequest{ + Type: &kubernetesType, + Config: map[string]string{ + "auth_mode": "kubeconfig", + "kubeconfig_path": "relative/config", + "namespace": "kandev", + "request_timeout_seconds": "30", + }, + }) + var fieldErr *agentkubernetes.FieldError + if !errors.As(err, &fieldErr) || fieldErr.Path != "config.kubeconfig_path" { + t.Fatalf("error = %v, want config.kubeconfig_path FieldError", err) + } +} + +func TestUpdateKubernetesExecutorRejectsTypeChangeWhileRunningInventoryIsRetained(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{ + SessionID: "session-retained", TaskID: "task-retained", ExecutorID: executor.ID, + Runtime: agentruntime.RuntimeKubernetes, + }); err != nil { + t.Fatalf("UpsertExecutorRunning: %v", err) + } + localType := models.ExecutorTypeLocal + + _, err = svc.UpdateExecutor(kubernetesAdminContext(), executor.ID, &UpdateExecutorRequest{ + Type: &localType, + }) + + if !errors.Is(err, ErrActiveTaskSessions) { + t.Fatalf("UpdateExecutor() error = %v, want ErrActiveTaskSessions", err) + } + stored, getErr := repo.GetExecutor(context.Background(), executor.ID) + if getErr != nil { + t.Fatalf("GetExecutor: %v", getErr) + } + if stored.Type != models.ExecutorTypeKubernetes { + t.Fatalf("blocked update changed executor type to %q", stored.Type) + } +} + +func TestDeleteKubernetesExecutorRejectsMember(t *testing.T) { + svc, _, _ := createTestService(t) + adminCtx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "admin-1", + Role: authn.RoleAdmin, + }) + executor, err := svc.CreateExecutor(adminCtx, &CreateExecutorRequest{ + Name: "Cluster", + Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + memberCtx := authn.WithIdentity(context.Background(), authn.Identity{ + UserID: "member-1", + Role: authn.RoleMember, + }) + + err = svc.DeleteExecutor(memberCtx, executor.ID) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } +} + +func TestDeleteKubernetesExecutorRejectsAuthoritativeRunningInventoryAfterSessionRepoint(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + repointed := createTestExecutor(t, svc, "Local replacement", models.ExecutorTypeLocal) + seedExecutorSession(t, repo, "session-repointed", repointed.ID) + if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{ + SessionID: "session-repointed", + TaskID: "task-session-repointed", + ExecutorID: executor.ID, + Runtime: agentruntime.RuntimeKubernetes, + Metadata: map[string]interface{}{ + "kubernetes_resource_executor_id": executor.ID, + }, + }); err != nil { + t.Fatalf("UpsertExecutorRunning: %v", err) + } + + err = svc.DeleteExecutor(kubernetesAdminContext(), executor.ID) + + if !errors.Is(err, ErrActiveTaskSessions) { + t.Fatalf("DeleteExecutor() error = %v, want ErrActiveTaskSessions", err) + } + if _, getErr := repo.GetExecutor(context.Background(), executor.ID); getErr != nil { + t.Fatalf("blocked delete removed executor needed for Kubernetes cleanup: %v", getErr) + } +} + +func TestDeleteKubernetesExecutorFailsClosedForAmbiguousRunningInventory(t *testing.T) { + tests := []struct { + name string + executorID string + metadata map[string]interface{} + }{ + {name: "both associations missing"}, + { + name: "current association missing", metadata: map[string]interface{}{ + "kubernetes_resource_executor_id": "other-executor", + }, + }, + {name: "recorded association missing", executorID: "other-executor"}, + { + name: "associations conflict", executorID: "current-executor", + metadata: map[string]interface{}{ + "kubernetes_resource_executor_id": "recorded-executor", + }, + }, + { + name: "recorded association has invalid type", executorID: "other-executor", + metadata: map[string]interface{}{ + "kubernetes_resource_executor_id": 42, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{ + SessionID: "session-ambiguous", TaskID: "task-ambiguous", + ExecutorID: test.executorID, Runtime: agentruntime.RuntimeKubernetes, + Metadata: test.metadata, + }); err != nil { + t.Fatalf("UpsertExecutorRunning: %v", err) + } + + err = svc.DeleteExecutor(kubernetesAdminContext(), executor.ID) + + if !errors.Is(err, ErrActiveTaskSessions) { + t.Fatalf("DeleteExecutor() error = %v, want ErrActiveTaskSessions", err) + } + if _, getErr := repo.GetExecutor(context.Background(), executor.ID); getErr != nil { + t.Fatalf("blocked delete removed executor: %v", getErr) + } + }) + } +} + +func TestDeleteKubernetesExecutorIgnoresUnrelatedNonKubernetesInventory(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{ + SessionID: "session-docker", TaskID: "task-docker", Runtime: agentruntime.RuntimeDocker, + }); err != nil { + t.Fatalf("UpsertExecutorRunning: %v", err) + } + + if err := svc.DeleteExecutor(kubernetesAdminContext(), executor.ID); err != nil { + t.Fatalf("DeleteExecutor() error = %v, want unrelated Docker row ignored", err) + } +} + +func TestUpdateKubernetesExecutorFailsClosedForAmbiguousRunningInventory(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{ + SessionID: "session-ambiguous-update", TaskID: "task-ambiguous-update", + Runtime: agentruntime.RuntimeKubernetes, + }); err != nil { + t.Fatalf("UpsertExecutorRunning: %v", err) + } + localType := models.ExecutorTypeLocal + + _, err = svc.UpdateExecutor(kubernetesAdminContext(), executor.ID, &UpdateExecutorRequest{ + Type: &localType, + }) + + if !errors.Is(err, ErrActiveTaskSessions) { + t.Fatalf("UpdateExecutor() error = %v, want ErrActiveTaskSessions", err) + } + stored, getErr := repo.GetExecutor(context.Background(), executor.ID) + if getErr != nil { + t.Fatalf("GetExecutor: %v", getErr) + } + if stored.Type != models.ExecutorTypeKubernetes { + t.Fatalf("blocked update changed executor type to %q", stored.Type) + } +} + +func TestDeleteKubernetesExecutorIgnoresUnambiguousOtherKubernetesInventory(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + if err := repo.UpsertExecutorRunning(context.Background(), &models.ExecutorRunning{ + SessionID: "session-other-k8s", TaskID: "task-other-k8s", + ExecutorID: "other-executor", Runtime: agentruntime.RuntimeKubernetes, + Metadata: map[string]interface{}{ + "kubernetes_resource_executor_id": "other-executor", + }, + }); err != nil { + t.Fatalf("UpsertExecutorRunning: %v", err) + } + + if err := svc.DeleteExecutor(kubernetesAdminContext(), executor.ID); err != nil { + t.Fatalf("DeleteExecutor() error = %v, want fully-associated other row ignored", err) + } +} + +func TestDeleteKubernetesExecutorFailsClosedOnInventoryReadError(t *testing.T) { + svc, _, repo := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + inventoryErr := errors.New("inventory unavailable") + svc.executors = runningInventoryErrorRepository{ + ExecutorRepository: repo, + err: inventoryErr, + } + + err = svc.DeleteExecutor(kubernetesAdminContext(), executor.ID) + + if !errors.Is(err, inventoryErr) { + t.Fatalf("DeleteExecutor() error = %v, want inventory error", err) + } + if _, getErr := repo.GetExecutor(context.Background(), executor.ID); getErr != nil { + t.Fatalf("inventory error removed executor: %v", getErr) + } +} + +type runningInventoryErrorRepository struct { + repository.ExecutorRepository + err error +} + +func (r runningInventoryErrorRepository) ListExecutorsRunning(context.Context) ([]*models.ExecutorRunning, error) { + return nil, r.err +} + +func TestCreateKubernetesProfileRejectsMember(t *testing.T) { + svc, _, _ := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", + Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + + _, err = svc.CreateExecutorProfile(kubernetesMemberContext(), &CreateExecutorProfileRequest{ + ExecutorID: executor.ID, + Name: "Default", + Config: validKubernetesProfileConfig(), + }) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } +} + +func TestCreateKubernetesProfileAuthorizesBeforeSecretLookup(t *testing.T) { + svc, _, _ := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + secretStore := &recordingKubernetesSecretStore{} + svc.SetSecretStore(secretStore) + + _, err = svc.CreateExecutorProfile(kubernetesMemberContext(), &CreateExecutorProfileRequest{ + ExecutorID: executor.ID, Name: "Default", Config: validKubernetesProfileConfig(), + EnvVars: []models.ProfileEnvVar{{Key: "TOKEN", SecretID: "secret-1"}}, + }) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } + if secretStore.getCalls != 0 { + t.Fatalf("secret Get calls = %d, want 0 before authorization", secretStore.getCalls) + } +} + +func TestCreateKubernetesProfileValidatesTypedConfig(t *testing.T) { + svc, _, _ := createTestService(t) + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", + Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, + Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + config := validKubernetesProfileConfig() + config["platform"] = "windows/amd64" + + _, err = svc.CreateExecutorProfile(kubernetesAdminContext(), &CreateExecutorProfileRequest{ + ExecutorID: executor.ID, + Name: "Default", + Config: config, + }) + var fieldErr *agentkubernetes.FieldError + if !errors.As(err, &fieldErr) || fieldErr.Path != "config.platform" { + t.Fatalf("error = %v, want config.platform FieldError", err) + } +} + +func TestUpdateKubernetesProfileRejectsMember(t *testing.T) { + svc, _, _ := createTestService(t) + profile := createKubernetesProfile(t, svc) + name := "Renamed" + + _, err := svc.UpdateExecutorProfile(kubernetesMemberContext(), profile.ID, &UpdateExecutorProfileRequest{ + Name: &name, + }) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } +} + +func TestUpdateKubernetesProfileValidatesTypedConfig(t *testing.T) { + svc, _, _ := createTestService(t) + profile := createKubernetesProfile(t, svc) + config := validKubernetesProfileConfig() + config["workspace.mode"] = "host_path" + + _, err := svc.UpdateExecutorProfile(kubernetesAdminContext(), profile.ID, &UpdateExecutorProfileRequest{ + Config: config, + }) + var fieldErr *agentkubernetes.FieldError + if !errors.As(err, &fieldErr) || fieldErr.Path != "config.workspace.mode" { + t.Fatalf("error = %v, want config.workspace.mode FieldError", err) + } +} + +func TestDeleteKubernetesProfileRejectsMember(t *testing.T) { + svc, _, _ := createTestService(t) + profile := createKubernetesProfile(t, svc) + + err := svc.DeleteExecutorProfile(kubernetesMemberContext(), profile.ID) + if !errors.Is(err, ErrKubernetesAdminRequired) { + t.Fatalf("error = %v, want ErrKubernetesAdminRequired", err) + } +} + +func TestDeleteExecutorProfileSucceedsAfterExecutorWasSoftDeleted(t *testing.T) { + svc, _, _ := createTestService(t) + profile := createKubernetesProfile(t, svc) + + require.NoError(t, svc.DeleteExecutor(kubernetesAdminContext(), profile.ExecutorID)) + require.NoError(t, svc.DeleteExecutorProfile(kubernetesAdminContext(), profile.ID)) + _, err := svc.GetExecutorProfile(context.Background(), profile.ID) + require.Error(t, err) +} + +func TestKubernetesExecutorAndProfileReadsRemainAvailableToMembers(t *testing.T) { + svc, _, _ := createTestService(t) + profile := createKubernetesProfile(t, svc) + + executor, err := svc.GetExecutor(kubernetesMemberContext(), profile.ExecutorID) + if err != nil || executor == nil { + t.Fatalf("GetExecutor() = %+v, %v; want member-readable", executor, err) + } + profiles, err := svc.ListExecutorProfiles(kubernetesMemberContext(), profile.ExecutorID) + if err != nil || len(profiles) != 1 { + t.Fatalf("ListExecutorProfiles() = %+v, %v; want one member-readable profile", profiles, err) + } +} + +func validKubernetesExecutorConfig() map[string]string { + return map[string]string{ + "auth_mode": "kubeconfig", + "kubeconfig_path": "/etc/kandev/kubeconfig", + "namespace": "kandev", + "request_timeout_seconds": "30", + } +} + +func validKubernetesProfileConfig() map[string]string { + return map[string]string{ + "platform": "linux/amd64", + "main_container": "kandev-agent", + "pod_template_yaml": `apiVersion: v1 +kind: PodTemplate +template: + spec: + containers: + - name: kandev-agent + image: alpine:3.20 +`, + "workspace.mode": "empty_dir", + } +} + +func kubernetesAdminContext() context.Context { + return authn.WithIdentity(context.Background(), authn.Identity{UserID: "admin-1", Role: authn.RoleAdmin}) +} + +func kubernetesMemberContext() context.Context { + return authn.WithIdentity(context.Background(), authn.Identity{UserID: "member-1", Role: authn.RoleMember}) +} + +func createKubernetesProfile(t *testing.T, svc *Service) *models.ExecutorProfile { + t.Helper() + executor, err := svc.CreateExecutor(kubernetesAdminContext(), &CreateExecutorRequest{ + Name: "Cluster", Type: models.ExecutorTypeKubernetes, + Status: models.ExecutorStatusActive, Config: validKubernetesExecutorConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutor: %v", err) + } + profile, err := svc.CreateExecutorProfile(kubernetesAdminContext(), &CreateExecutorProfileRequest{ + ExecutorID: executor.ID, Name: "Default", Config: validKubernetesProfileConfig(), + }) + if err != nil { + t.Fatalf("CreateExecutorProfile: %v", err) + } + return profile +} + +type recordingKubernetesSecretStore struct { + getCalls int +} + +func (*recordingKubernetesSecretStore) Create(context.Context, *secrets.SecretWithValue) error { + return nil +} + +func (s *recordingKubernetesSecretStore) Get(context.Context, string) (*secrets.Secret, error) { + s.getCalls++ + return nil, errors.New("secret lookup should not run") +} + +func (*recordingKubernetesSecretStore) Reveal(context.Context, string) (string, error) { + return "", nil +} + +func (*recordingKubernetesSecretStore) Update(context.Context, string, *secrets.UpdateSecretRequest) error { + return nil +} + +func (*recordingKubernetesSecretStore) Delete(context.Context, string) error { return nil } + +func (*recordingKubernetesSecretStore) List(context.Context) ([]*secrets.SecretListItem, error) { + return nil, nil +} + +func (*recordingKubernetesSecretStore) Close() error { return nil } diff --git a/apps/backend/internal/task/service/service_tasks.go b/apps/backend/internal/task/service/service_tasks.go index b2e4b3bc7e..ada5184856 100644 --- a/apps/backend/internal/task/service/service_tasks.go +++ b/apps/backend/internal/task/service/service_tasks.go @@ -14,6 +14,7 @@ import ( "github.com/google/uuid" runtimeapi "github.com/kandev/kandev/internal/agent/runtime" + "github.com/kandev/kandev/internal/agentruntime" v1 "github.com/kandev/kandev/pkg/api/v1" "go.uber.org/zap" @@ -78,6 +79,7 @@ type pendingTaskTitleSetter interface { type taskStopTarget struct { sessionID string executionID string + runtime agentruntime.Runtime // terminal records the session state observed when the cleanup snapshot was // taken. It is retained for snapshot compatibility, but it must never // suppress a non-not-found runtime stop failure. @@ -2816,6 +2818,7 @@ func (s *Service) buildStopTargets(ctx context.Context, taskID string, activeSes target := taskStopTarget{ sessionID: running.SessionID, executionID: strings.TrimSpace(running.AgentExecutionID), + runtime: running.Runtime, terminal: isCleanableSessionState(sessionStates[running.SessionID]), } targets = append(targets, target) @@ -2883,6 +2886,7 @@ func taskStopTargetsFromRunningRows(runningRows []*models.ExecutorRunning) []tas targets = append(targets, taskStopTarget{ sessionID: strings.TrimSpace(running.SessionID), executionID: strings.TrimSpace(running.AgentExecutionID), + runtime: running.Runtime, }) } return targets @@ -2906,6 +2910,9 @@ func mergeTaskStopTargets(live, snapshot []taskStopTarget) []taskStopTarget { key := target.sessionID + "\x00" + target.executionID if index, exists := seen[key]; exists { targets[index].terminal = targets[index].terminal || target.terminal + if targets[index].runtime == "" { + targets[index].runtime = target.runtime + } return } seen[key] = len(targets) diff --git a/apps/backend/internal/task/service/service_tasks_stop_test.go b/apps/backend/internal/task/service/service_tasks_stop_test.go index a9d833ade4..6453cafea4 100644 --- a/apps/backend/internal/task/service/service_tasks_stop_test.go +++ b/apps/backend/internal/task/service/service_tasks_stop_test.go @@ -354,6 +354,30 @@ func TestStopTaskRuntimeTargets_TerminalStopFailureBlocksCleanup(t *testing.T) { } } +func TestStopTaskRuntimeTargets_KubernetesTerminalCleanupFailurePreservesRow(t *testing.T) { + svc, _, _ := createTestService(t) + executors := &stubExecutors{} + svc.executors = executors + svc.executionStopper = &stubStopper{stopExecutionErr: errors.New("exact Kubernetes cleanup failed")} + targets := []taskStopTarget{{ + sessionID: "sess-k8s", executionID: "exec-k8s", terminal: true, + runtime: agentruntime.RuntimeKubernetes, + }} + + outcome := svc.stopTaskRuntimeTargets(context.Background(), "task-k8s", targets, "delete", "stop failed") + svc.performTaskCleanup( + context.Background(), "task-k8s", nil, nil, targets, taskEnvironmentCleanup{}, + taskCleanupPreserveRows(outcome), + ) + + if _, ok := outcome.failed["sess-k8s"]; !ok { + t.Fatalf("Kubernetes cleanup failure must remain retryable: %#v", outcome.failed) + } + if len(executors.deletedSessions) != 0 { + t.Fatalf("failed Kubernetes cleanup deleted authoritative row: %v", executors.deletedSessions) + } +} + func TestStopTaskRuntimeTargets_ExactRuntimeAbsenceIsIdempotent(t *testing.T) { svc, _, _ := createTestService(t) svc.executors = &stubExecutors{} diff --git a/apps/backend/internal/task/service/service_turns.go b/apps/backend/internal/task/service/service_turns.go index 151ab6910d..b2e5e0cd8b 100644 --- a/apps/backend/internal/task/service/service_turns.go +++ b/apps/backend/internal/task/service/service_turns.go @@ -16,6 +16,7 @@ import ( "github.com/kandev/kandev/internal/agent/runtime/lifecycle" "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" "github.com/kandev/kandev/internal/steptelemetry" @@ -894,9 +895,7 @@ func (s *Service) GetWorkspaceInfoForSession(ctx context.Context, taskID, sessio zap.String("session_id", sessionID), zap.Error(err)) } else { - s.logger.Warn("failed to get executor running for session", - zap.String("session_id", sessionID), - zap.Error(err)) + return nil, fmt.Errorf("load runtime inventory for session %q: %w", sessionID, err) } } else if running != nil { info.RuntimeName = running.Runtime @@ -906,36 +905,62 @@ func (s *Service) GetWorkspaceInfoForSession(ctx context.Context, taskID, sessio ensureWorkspaceMetadata(info)[lifecycle.MetadataKeyContainerID] = running.ContainerID } } - if session.ExecutorID != "" { - exec, err := s.executors.GetExecutor(ctx, session.ExecutorID) - if err != nil { - s.logger.Warn("failed to get executor for session", - zap.String("session_id", sessionID), - zap.String("executor_id", session.ExecutorID), - zap.Error(err)) - } else if exec != nil { - info.ExecutorType = string(exec.Type) - // Project the executor record's connection config (e.g. ssh_host, - // ssh_host_fingerprint, ssh_user) into the workspace metadata as a - // fallback. The agent-launch path gets these via the orchestrator's - // executor-config merge, but the workspace-restore / terminal path - // only carries them forward from a live ExecutorRunning record. When - // no running record exists — terminal-state sessions (completed / - // failed / cancelled), post-restart, or after agentctl cleanup — the - // SSH executor would otherwise fail with "host (or host_alias) is - // required in executor config" when opening a terminal or restoring - // the workspace. Existing values (from the running record) win. - // Scoped to SSH: this fallback only makes sense for the SSH executor - // and the projected keys are SSH connection/profile keys. - if exec.Type == models.ExecutorTypeSSH { - mergeExecutorConfigMetadata(info, exec.Config) - } + executorID := session.ExecutorID + recordedKubernetes := running != nil && running.Runtime == agentruntime.RuntimeKubernetes + if recordedKubernetes { + executorID = strings.TrimSpace(running.ExecutorID) + if executorID == "" { + return nil, errors.New("restore Kubernetes workspace: recorded executor ID is missing") + } + } + if executorID != "" { + if err := s.applyWorkspaceExecutorRecord(ctx, sessionID, executorID, recordedKubernetes, info); err != nil { + return nil, err } } return info, nil } +func (s *Service) applyWorkspaceExecutorRecord( + ctx context.Context, + sessionID, executorID string, + recordedKubernetes bool, + info *lifecycle.WorkspaceInfo, +) error { + exec, err := s.executors.GetExecutor(ctx, executorID) + if err != nil { + if recordedKubernetes { + return fmt.Errorf("restore Kubernetes workspace executor %q: %w", executorID, err) + } + s.logger.Warn("failed to get executor for session", + zap.String("session_id", sessionID), + zap.String("executor_id", executorID), + zap.Error(err)) + return nil + } + if exec == nil { + if recordedKubernetes { + return fmt.Errorf("restore Kubernetes workspace executor %q: executor not found", executorID) + } + return nil + } + if recordedKubernetes && exec.Type != models.ExecutorTypeKubernetes { + return fmt.Errorf("restore Kubernetes workspace executor %q: executor is no longer Kubernetes", executorID) + } + info.ExecutorType = string(exec.Type) + // Only stable SSH connection/profile keys fill missing metadata. A retained + // Kubernetes row instead keeps resource inventory while current connection + // config authoritatively replaces every connection key. + if exec.Type == models.ExecutorTypeSSH { + mergeExecutorConfigMetadata(info, exec.Config) + } + if exec.Type == models.ExecutorTypeKubernetes { + mergeKubernetesExecutorConfigMetadata(info, exec.Config) + } + return nil +} + type workspaceWorktreeKey struct { repositoryID string branchSlug string @@ -1192,6 +1217,19 @@ func mergeExecutorConfigMetadata(info *lifecycle.WorkspaceInfo, config map[strin } } +func mergeKubernetesExecutorConfigMetadata(info *lifecycle.WorkspaceInfo, config map[string]string) { + dst := ensureWorkspaceMetadata(info) + for _, key := range []string{ + lifecycle.MetadataKeyKubernetesAuthMode, + lifecycle.MetadataKeyKubernetesKubeconfigPath, + lifecycle.MetadataKeyKubernetesKubeContext, + lifecycle.MetadataKeyKubernetesConfigNamespace, + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds, + } { + dst[key] = config[key] + } +} + // GetWorkspaceInfoForEnvironment returns workspace information for a task environment. func (s *Service) GetWorkspaceInfoForEnvironment(ctx context.Context, taskEnvironmentID string) (*lifecycle.WorkspaceInfo, error) { if taskEnvironmentID == "" { diff --git a/apps/backend/internal/task/service/service_turns_test.go b/apps/backend/internal/task/service/service_turns_test.go index 27727a23f2..850ac962fb 100644 --- a/apps/backend/internal/task/service/service_turns_test.go +++ b/apps/backend/internal/task/service/service_turns_test.go @@ -14,6 +14,7 @@ import ( "github.com/kandev/kandev/internal/agent/runtime/lifecycle" "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/agentruntime" "github.com/kandev/kandev/internal/events" "github.com/kandev/kandev/internal/events/bus" "github.com/kandev/kandev/internal/task/models" @@ -27,6 +28,18 @@ type nilTaskSessionRepo struct { repository.SessionRepository } +type failGetExecutorRunningRepository struct { + repository.ExecutorRepository + err error +} + +func (r failGetExecutorRunningRepository) GetExecutorRunningBySessionID( + context.Context, + string, +) (*models.ExecutorRunning, error) { + return nil, r.err +} + type failGetAfterPublishMetadataRepo struct { repository.TurnRepository activeMetadataUpdates int @@ -1411,6 +1424,200 @@ func TestGetWorkspaceInfoForSession_ExecutorInfo(t *testing.T) { } } +func TestGetWorkspaceInfoForSession_KubernetesRunningRecordOwnsExecutorAndCurrentConnection(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + setupTestTask(t, repo) + now := time.Now().UTC() + if err := repo.CreateExecutor(ctx, &models.Executor{ + ID: "recorded-kubernetes", Name: "Kubernetes", Type: models.ExecutorTypeKubernetes, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "in_cluster", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.CreateExecutor(ctx, &models.Executor{ + ID: "repointed-local", Name: "Local", Type: models.ExecutorTypeLocal, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-k8s", TaskID: "task-123", ExecutorID: "repointed-local", + State: models.TaskSessionStateCompleted, StartedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "running-k8s", SessionID: "session-k8s", TaskID: "task-123", + ExecutorID: "recorded-kubernetes", Runtime: agentruntime.RuntimeKubernetes, + AgentExecutionID: "recorded-execution", + Metadata: map[string]interface{}{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/old/kubeconfig", + lifecycle.MetadataKeyKubernetesKubeContext: "old-context", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "30", + lifecycle.MetadataKeyKubernetesNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesPodName: "recorded-pod", + lifecycle.MetadataKeyKubernetesPodUID: "recorded-pod-uid", + lifecycle.MetadataKeyKubernetesProfileSnapshot: "recorded-snapshot", + }, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + + info, err := svc.GetWorkspaceInfoForSession(ctx, "task-123", "session-k8s") + + if err != nil { + t.Fatalf("GetWorkspaceInfoForSession() error = %v", err) + } + if info.ExecutorType != string(models.ExecutorTypeKubernetes) || info.RuntimeName != agentruntime.RuntimeKubernetes { + t.Fatalf("executor projection = type %q runtime %q", info.ExecutorType, info.RuntimeName) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesPodUID]; got != "recorded-pod-uid" { + t.Fatalf("recorded Pod UID = %v", got) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesAuthMode]; got != "in_cluster" { + t.Fatalf("auth mode = %v, want current in_cluster", got) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds]; got != "45" { + t.Fatalf("request timeout = %v, want current 45", got) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesKubeconfigPath]; got != "" { + t.Fatalf("kubeconfig path = %v, want authoritative empty", got) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesKubeContext]; got != "" { + t.Fatalf("kube context = %v, want authoritative empty", got) + } +} + +func TestGetWorkspaceInfoForSession_KubernetesWithoutRunningRowUsesCurrentConnection(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + setupTestTask(t, repo) + now := time.Now().UTC() + if err := repo.CreateExecutor(ctx, &models.Executor{ + ID: "kubernetes-executor", Name: "Kubernetes", Type: models.ExecutorTypeKubernetes, + Config: map[string]string{ + lifecycle.MetadataKeyKubernetesAuthMode: "kubeconfig", + lifecycle.MetadataKeyKubernetesKubeconfigPath: "/etc/kandev/kubeconfig", + lifecycle.MetadataKeyKubernetesConfigNamespace: "kandev-agents", + lifecycle.MetadataKeyKubernetesRequestTimeoutSeconds: "45", + }, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-k8s", TaskID: "task-123", ExecutorID: "kubernetes-executor", + State: models.TaskSessionStateCompleted, StartedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + + info, err := svc.GetWorkspaceInfoForSession(ctx, "task-123", "session-k8s") + + if err != nil { + t.Fatalf("GetWorkspaceInfoForSession() error = %v", err) + } + if info.ExecutorType != string(models.ExecutorTypeKubernetes) { + t.Fatalf("ExecutorType = %q, want Kubernetes", info.ExecutorType) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesKubeconfigPath]; got != "/etc/kandev/kubeconfig" { + t.Fatalf("kubeconfig path = %v, want current connection metadata", got) + } + if got := info.Metadata[lifecycle.MetadataKeyKubernetesConfigNamespace]; got != "kandev-agents" { + t.Fatalf("namespace = %v, want current connection metadata", got) + } +} + +func TestGetWorkspaceInfoForSession_KubernetesRunningRecordFailsClosedWithoutExecutor(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + setupTestTask(t, repo) + now := time.Now().UTC() + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-k8s-missing", TaskID: "task-123", ExecutorID: "unrelated-executor", + State: models.TaskSessionStateCompleted, StartedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "running-k8s-missing", SessionID: "session-k8s-missing", TaskID: "task-123", + ExecutorID: "deleted-kubernetes", Runtime: agentruntime.RuntimeKubernetes, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + + _, err := svc.GetWorkspaceInfoForSession(ctx, "task-123", "session-k8s-missing") + + if err == nil || !strings.Contains(err.Error(), "executor not found") { + t.Fatalf("GetWorkspaceInfoForSession() error = %v", err) + } +} + +func TestGetWorkspaceInfoForSession_KubernetesRunningRecordFailsClosedAfterExecutorTypeChange(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + setupTestTask(t, repo) + now := time.Now().UTC() + if err := repo.CreateExecutor(ctx, &models.Executor{ + ID: "changed-kubernetes", Name: "Changed", Type: models.ExecutorTypeLocal, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-k8s-changed", TaskID: "task-123", ExecutorID: "unrelated-executor", + State: models.TaskSessionStateCompleted, StartedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + if err := repo.UpsertExecutorRunning(ctx, &models.ExecutorRunning{ + ID: "running-k8s-changed", SessionID: "session-k8s-changed", TaskID: "task-123", + ExecutorID: "changed-kubernetes", Runtime: agentruntime.RuntimeKubernetes, + CreatedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + + _, err := svc.GetWorkspaceInfoForSession(ctx, "task-123", "session-k8s-changed") + + if err == nil || !strings.Contains(err.Error(), "executor is no longer Kubernetes") { + t.Fatalf("GetWorkspaceInfoForSession() error = %v", err) + } +} + +func TestGetWorkspaceInfoForSession_FailsClosedWhenRuntimeInventoryReadFails(t *testing.T) { + svc, _, repo := createTestService(t) + ctx := context.Background() + setupTestTask(t, repo) + now := time.Now().UTC() + if err := repo.CreateTaskSession(ctx, &models.TaskSession{ + ID: "session-inventory-error", TaskID: "task-123", + State: models.TaskSessionStateCompleted, StartedAt: now, UpdatedAt: now, + }); err != nil { + t.Fatal(err) + } + svc.executors = failGetExecutorRunningRepository{ + ExecutorRepository: repo, + err: errors.New("database unavailable"), + } + + _, err := svc.GetWorkspaceInfoForSession(ctx, "task-123", "session-inventory-error") + + if err == nil || !strings.Contains(err.Error(), "load runtime inventory") { + t.Fatalf("GetWorkspaceInfoForSession() error = %v, want runtime inventory read failure", err) + } +} + func TestGetWorkspaceInfoForSession_IncludesEnvironmentReconnectMetadata(t *testing.T) { svc, _, repo := createTestService(t) ctx := context.Background() diff --git a/apps/backend/internal/task/service/service_workspace_sources.go b/apps/backend/internal/task/service/service_workspace_sources.go index 6559de5604..b9bb566f84 100644 --- a/apps/backend/internal/task/service/service_workspace_sources.go +++ b/apps/backend/internal/task/service/service_workspace_sources.go @@ -22,8 +22,9 @@ import ( type WorkspaceSourceKind string const ( - WorkspaceSourceRepository WorkspaceSourceKind = "repository" - WorkspaceSourceFolder WorkspaceSourceKind = "folder" + WorkspaceSourceRepository WorkspaceSourceKind = "repository" + WorkspaceSourceFolder WorkspaceSourceKind = "folder" + workspaceSourceRemoteProjection models.ExecutorType = "remote" ) type WorkspaceSourceInput struct { @@ -518,6 +519,9 @@ func WorkspaceSourceRuntimeEntryName(executorType string, repository *models.Rep } return name, nil } + if !supportsBranchedWorkspaceSources(executorType) { + return "", fmt.Errorf("%w: executor %q cannot materialize repository sources", ErrUnsupportedWorkspaceSource, executorType) + } branch := worktree.SanitizeBranchSlug(taskRepository.CheckoutBranch) if branch == "" { branch = worktree.SanitizeBranchSlug(taskRepository.BaseBranch) @@ -532,6 +536,13 @@ func isLocalWorkspaceExecutor(executorType string) bool { return executorType == string(models.ExecutorTypeLocal) || executorType == "local_pc" } +func supportsBranchedWorkspaceSources(executorType string) bool { + t := models.ExecutorType(executorType) + // "remote" is the backend materializer's explicit executor-agnostic + // projection mode. Unknown persisted executor types still fail closed. + return t == "" || t == workspaceSourceRemoteProjection || t == models.ExecutorTypeWorktree || models.IsRemoteExecutorType(t) +} + func (s *Service) workspaceSourceExecutorType(ctx context.Context, taskID string) (string, error) { if s.taskEnvironments == nil { return "", nil diff --git a/apps/backend/internal/task/service/workspace_source_kubernetes_test.go b/apps/backend/internal/task/service/workspace_source_kubernetes_test.go new file mode 100644 index 0000000000..b87bb96d08 --- /dev/null +++ b/apps/backend/internal/task/service/workspace_source_kubernetes_test.go @@ -0,0 +1,36 @@ +package service + +import ( + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +func TestWorkspaceSourceRuntimeEntryNameKubernetesUsesBranchIdentity(t *testing.T) { + repository := &models.Repository{ID: "repo-1", Name: "api"} + taskRepository := &models.TaskRepository{ + RepositoryID: "repo-1", BaseBranch: "main", CheckoutBranch: "feature/k8s", + } + + got, err := WorkspaceSourceRuntimeEntryName( + string(models.ExecutorTypeKubernetes), repository, taskRepository, + ) + if err != nil { + t.Fatal(err) + } + if got != "api-feature-k8s" { + t.Fatalf("runtime entry name = %q, want api-feature-k8s", got) + } +} + +func TestWorkspaceSourceRuntimeEntryNameRejectsUnknownExecutor(t *testing.T) { + _, err := WorkspaceSourceRuntimeEntryName( + "future_executor", + &models.Repository{ID: "repo-1", Name: "api"}, + &models.TaskRepository{RepositoryID: "repo-1", BaseBranch: "main"}, + ) + if !errors.Is(err, ErrUnsupportedWorkspaceSource) { + t.Fatalf("error = %v, want unsupported workspace source", err) + } +} diff --git a/apps/web/app/settings/executor/[id]/page.tsx b/apps/web/app/settings/executor/[id]/page.tsx index e92b0514eb..666a80943d 100644 --- a/apps/web/app/settings/executor/[id]/page.tsx +++ b/apps/web/app/settings/executor/[id]/page.tsx @@ -67,6 +67,7 @@ function executorDescriptionKey(type: ExecutorType): string { if (type === "local_docker") return "executors:descriptionLocalDocker"; if (type === "remote_docker") return "executors:descriptionRemoteDocker"; if (type === "sprites") return "executors:descriptionSprites"; + if (type === "k8s") return "executors:descriptionKubernetes"; return "executors:descriptionCustom"; } diff --git a/apps/web/app/settings/executors/[profileId]/page.tsx b/apps/web/app/settings/executors/[profileId]/page.tsx index 8426aac7d5..7f2de59fa6 100644 --- a/apps/web/app/settings/executors/[profileId]/page.tsx +++ b/apps/web/app/settings/executors/[profileId]/page.tsx @@ -6,7 +6,6 @@ import { useRouter } from "@/lib/routing/client-router"; import { runWithNavigationBlockerBypassed } from "@/lib/routing/navigation-guard"; import { Button } from "@kandev/ui/button"; import { Card, CardContent } from "@kandev/ui/card"; -import { IconShieldLock } from "@tabler/icons-react"; import { useAppStore } from "@/components/state-provider"; import { useSecrets } from "@/hooks/domains/settings/use-secrets"; import { @@ -26,15 +25,11 @@ import { import { EnvVarsCard, useEnvVarRows, - rowsToEnvVars, envVarsToRows, } from "@/components/settings/profile-edit/env-vars-card"; import { ProfileScriptCards } from "@/components/settings/profile-edit/profile-script-cards"; import { SSHAgentReadinessCard } from "@/components/settings/ssh-agent-readiness-card"; -import { - SSHTaskDirReclamationCard, - isSSHReclaimEnabled, -} from "@/components/settings/ssh-task-dir-reclamation-card"; +import { SSHTaskDirReclamationCard } from "@/components/settings/ssh-task-dir-reclamation-card"; import { type GitIdentityMode, type GitIdentityState, @@ -52,9 +47,17 @@ import { upsertExecutorProfile, type SaveStatus, } from "@/components/settings/profile-edit/profile-edit-page-chrome"; +import { ProfileConnectionSettingsAction } from "@/components/settings/profile-edit/profile-connection-settings-action"; +import type { ExecutorProfileSavePayload } from "@/components/settings/profile-edit/use-executor-profile-save-contributor"; +import { + useKubernetesProfileConnection, + useProfileSettingsSave, +} from "@/components/settings/profile-edit/use-profile-settings-save"; +import { + buildProfileEnvVars, + profileSaveInvalidReason, +} from "@/components/settings/profile-edit/profile-edit-form-helpers"; import { useToast } from "@/components/toast-provider"; -import { useSettingsSaveContributor } from "@/components/settings/settings-save-provider"; -import { serializeSettingsRevision } from "@/components/settings/settings-save-revision"; import { deriveSpritesSecretId, getGitIdentityBaseline, @@ -63,13 +66,27 @@ import { parseRemoteAuthSecrets, parseRemoteCredentials, } from "@/components/settings/profile-edit/executor-profile-baselines"; -import type { Executor, ExecutorProfile, ExecutorType, ProfileEnvVar } from "@/lib/types/http"; +import type { Executor, ExecutorProfile } from "@/lib/types/http"; import type { NetworkPolicyRule } from "@/lib/api/domains/settings-api"; import { executorProfileDiscoveryTarget } from "@/lib/settings-discovery/dynamic-targets"; import { buildSaveConfig } from "@/components/settings/profile-edit/serialize-executor-config"; +import { useProfileRuntimeFormState } from "@/components/settings/profile-edit/use-profile-runtime-form-state"; +import { KubernetesProfileSections } from "@/components/settings/kubernetes-profile-sections"; +import { KubernetesReadOnlyNotice } from "@/components/settings/kubernetes-read-only-notice"; +import { KubernetesProfileClusterSection } from "@/components/settings/kubernetes-profile-cluster-section"; +import { + parseKubernetesProfileConfig, + replaceKubernetesProfileConfig, +} from "@/components/settings/kubernetes-config"; +import { kubernetesExecutorInvalidReason } from "@/components/settings/kubernetes-validation"; +import { + useKubernetesAdminAccess, + useKubernetesDiagnostics, + useKubernetesSessions, +} from "@/hooks/domains/settings/use-kubernetes-settings"; const EXECUTORS_ROUTE = "/settings/executors"; -const SPRITES_TOKEN_KEY = "SPRITES_API_TOKEN"; + function useProfileFromStore(profileId: string) { const executor = useAppStore( (state) => @@ -80,23 +97,6 @@ function useProfileFromStore(profileId: string) { return executor && profile ? { executor, profile } : null; } -function useRemoteExecutorFlags(executorType: ExecutorType) { - // SSH joins the "remote" set because it runs the agent on a host whose - // filesystem doesn't share paths with the kandev backend — so the same - // remote-credentials + auth-secrets surface applies (the SSH executor - // SFTPs files into the remote user's $HOME). - const isRemote = - executorType === "local_docker" || - executorType === "remote_docker" || - executorType === "sprites" || - executorType === "ssh"; - return { - isRemote, - isDocker: executorType === "local_docker" || executorType === "remote_docker", - isSprites: executorType === "sprites", - }; -} - function useRemoteAuthState(profile: ExecutorProfile) { const [networkPolicyRules, setNetworkPolicyRules] = useState(() => parseNetworkPolicyRules(profile.config), @@ -115,6 +115,13 @@ function useRemoteAuthState(profile: ExecutorProfile) { setAgentEnvVars((prev) => ({ ...prev, [agentId]: secretId })); }, []); + const reset = useCallback(() => { + setNetworkPolicyRules(parseNetworkPolicyRules(profile.config)); + setRemoteCredentials(parseRemoteCredentials(profile.config)); + setConfigBundleIds(parseAgentConfigBundles(profile.config)); + setAgentEnvVars(parseRemoteAuthSecrets(profile.config)); + }, [profile.config]); + return { networkPolicyRules, setNetworkPolicyRules, @@ -124,6 +131,7 @@ function useRemoteAuthState(profile: ExecutorProfile) { setConfigBundleIds, agentEnvVars, handleAgentEnvVarChange, + reset, }; } @@ -172,6 +180,13 @@ function useGitIdentityState(isRemote: boolean, profile: ExecutorProfile) { .finally(() => setLoaded(true)); }, [isRemote, profile.config?.git_user_email, profile.config?.git_user_name]); + const reset = useCallback(() => { + const baseline = getGitIdentityBaseline(profile, localGitIdentity); + setGitIdentityMode(baseline.mode); + setGitUserName(baseline.userName); + setGitUserEmail(baseline.userEmail); + }, [localGitIdentity, profile]); + return { localGitIdentity, gitIdentityMode, @@ -181,6 +196,7 @@ function useGitIdentityState(isRemote: boolean, profile: ExecutorProfile) { gitUserEmail, setGitUserEmail, loaded, + reset, }; } @@ -219,14 +235,7 @@ function useProfilePersistence(executor: Executor, profile: ExecutorProfile) { const [deleting, setDeleting] = useState(false); const save = useCallback( - async (data: { - name: string; - mcp_policy?: string; - config?: Record; - prepare_script: string; - cleanup_script: string; - env_vars: ProfileEnvVar[]; - }) => { + async (data: ExecutorProfileSavePayload) => { setSaveStatus("loading"); setError(null); try { @@ -275,26 +284,22 @@ function useProfilePersistence(executor: Executor, profile: ExecutorProfile) { return { saveStatus, error, deleting, deleteDialogOpen, setDeleteDialogOpen, save, remove }; } -function useProfileFormState(executor: Executor, profile: ExecutorProfile) { +export function useProfileFormState(executor: Executor, profile: ExecutorProfile) { const { t } = useTranslation(); const [name, setName] = useState(profile.name); const [mcpPolicy, setMcpPolicy] = useState(profile.mcp_policy ?? ""); const [prepareScript, setPrepareScript] = useState(profile.prepare_script ?? ""); const [cleanupScript, setCleanupScript] = useState(profile.cleanup_script ?? ""); - const [dockerfile, setDockerfile] = useState(profile.config?.dockerfile ?? ""); - const [imageTag, setImageTag] = useState(profile.config?.image_tag ?? ""); - const [sshShell, setSshShell] = useState(profile.config?.ssh_shell ?? ""); - const [sshReclaimTaskDir, setSshReclaimTaskDir] = useState(() => - isSSHReclaimEnabled(profile.config), + const runtime = useProfileRuntimeFormState(executor, profile); + const { envVarRows, addEnvVar, removeEnvVar, updateEnvVar, resetEnvVars } = useEnvVarRows( + profile.env_vars, ); - const { envVarRows, addEnvVar, removeEnvVar, updateEnvVar } = useEnvVarRows(profile.env_vars); const [placeholders, setPlaceholders] = useState([]); const [spritesSecretId, setSpritesSecretId] = useState(() => deriveSpritesSecretId(profile.env_vars), ); - const flags = useRemoteExecutorFlags(executor.type); const remoteAuth = useRemoteAuthState(profile); - const gitIdentity = useGitIdentityState(flags.isRemote, profile); + const gitIdentity = useGitIdentityState(runtime.isRemote, profile); const mcpPolicyErrorKey = useMemo(() => validateMcpPolicy(mcpPolicy), [mcpPolicy]); useEffect(() => { @@ -303,19 +308,29 @@ function useProfileFormState(executor: Executor, profile: ExecutorProfile) { .catch(() => {}); }, []); - const buildEnvVars = useCallback((): ProfileEnvVar[] => { - const vars = rowsToEnvVars(envVarRows).filter((ev) => ev.key !== SPRITES_TOKEN_KEY); - if (flags.isSprites && spritesSecretId) { - vars.push({ key: SPRITES_TOKEN_KEY, secret_id: spritesSecretId }); - } - return vars; - }, [envVarRows, flags.isSprites, spritesSecretId]); + const buildEnvVars = useCallback( + () => buildProfileEnvVars(envVarRows, runtime.isSprites, spritesSecretId), + [envVarRows, runtime.isSprites, spritesSecretId], + ); - const prepareDesc = flags.isRemote + const prepareDesc = runtime.isRemote ? t("executors:prepareScriptDescriptionRemote", { trigger: "{{" }) : t("executors:prepareScriptDescriptionLocal"); + const reset = useCallback(() => { + setName(profile.name); + setMcpPolicy(profile.mcp_policy ?? ""); + setPrepareScript(profile.prepare_script ?? ""); + setCleanupScript(profile.cleanup_script ?? ""); + runtime.resetRuntime(); + resetEnvVars(profile.env_vars); + setSpritesSecretId(deriveSpritesSecretId(profile.env_vars)); + remoteAuth.reset(); + gitIdentity.reset(); + }, [gitIdentity, profile, remoteAuth, resetEnvVars, runtime]); + return { + ...runtime, name, setName, mcpPolicy, @@ -324,14 +339,6 @@ function useProfileFormState(executor: Executor, profile: ExecutorProfile) { setPrepareScript, cleanupScript, setCleanupScript, - dockerfile, - setDockerfile, - imageTag, - setImageTag, - sshShell, - setSshShell, - sshReclaimTaskDir, - setSshReclaimTaskDir, envVarRows, addEnvVar, removeEnvVar, @@ -355,13 +362,10 @@ function useProfileFormState(executor: Executor, profile: ExecutorProfile) { setGitUserName: gitIdentity.setGitUserName, gitUserEmail: gitIdentity.gitUserEmail, setGitUserEmail: gitIdentity.setGitUserEmail, - isRemote: flags.isRemote, - isDocker: flags.isDocker, - isSprites: flags.isSprites, - isSSH: executor.type === "ssh", mcpPolicyErrorKey, buildEnvVars, prepareDesc, + reset, }; } @@ -374,6 +378,7 @@ type ProfileEditSectionsProps = { function ExecutorSpecificSections({ executor, profile, form, secrets }: ProfileEditSectionsProps) { const gitIdentityBaseline = getGitIdentityBaseline(profile, form.localGitIdentity); + const canManageKubernetes = useKubernetesAdminAccess(); return ( <> {executor.type === "ssh" && ( @@ -409,6 +414,14 @@ function ExecutorSpecificSections({ executor, profile, form, secrets }: ProfileE onImageTagChange={form.setImageTag} /> )} + {form.isKubernetes && ( + + )} {form.isRemote && ( router.push(`/settings/executors/ssh/${executor.id}`)} - className="w-full cursor-pointer sm:w-auto" - data-testid="ssh-connection-settings-link" - > - - {t("executors:connectionSettings")} - - ) : undefined; - + const memberReadOnly = form.isKubernetes && !canManageKubernetes; + const sharedConfig = buildSaveConfig(form, profile.config); const savePayload = { name: form.name.trim(), mcp_policy: form.mcpPolicy || undefined, - config: buildSaveConfig(form, profile.config), + config: form.isKubernetes + ? replaceKubernetesProfileConfig(sharedConfig, form.kubernetesProfile) + : sharedConfig, prepare_script: form.prepareScript, cleanup_script: form.cleanupScript, env_vars: form.buildEnvVars(), }; - const saveRevision = serializeSettingsRevision(savePayload); - const [savedRevision, setSavedRevision] = useState(saveRevision); - const [baselineReady, setBaselineReady] = useState(!form.isRemote); - useEffect(() => { - if (!baselineReady && form.gitIdentityLoaded) { - setSavedRevision(saveRevision); - setBaselineReady(true); - } - }, [baselineReady, form.gitIdentityLoaded, saveRevision]); - const handleSave = async () => { - const submittedPayload = savePayload; - const submittedRevision = saveRevision; - await persistence.save(submittedPayload); - setSavedRevision(submittedRevision); - }; - let invalidReason: string | undefined; - if (!form.name.trim()) invalidReason = t("executors:profileNameIsRequired"); - else if (form.mcpPolicyErrorKey) invalidReason = t(form.mcpPolicyErrorKey); - else if (spritesTokenMissing) invalidReason = t("executors:spritesTokenIsRequired"); - useSettingsSaveContributor({ - id: `executor-profile:${profile.id}`, - revision: saveRevision, - isDirty: baselineReady && saveRevision !== savedRevision, - canSave: - baselineReady && Boolean(form.name.trim()) && !form.mcpPolicyErrorKey && !spritesTokenMissing, + const profileInvalidReason = profileSaveInvalidReason(form, canManageKubernetes, t); + const connectionInvalidReason = form.isKubernetes + ? kubernetesExecutorInvalidReason(connection.form, canManageKubernetes, t) + : undefined; + const invalidReason = connectionInvalidReason ?? profileInvalidReason; + const baselineReady = useProfileSettingsSave({ + executorId: executor.id, + profileId: profile.id, + payload: savePayload, + isKubernetes: form.isKubernetes, + isRemote: form.isRemote, + gitIdentityLoaded: form.gitIdentityLoaded, + canManageKubernetes, invalidReason, - save: handleSave, - discard: () => undefined, + saveProfile: persistence.save, + discardProfile: form.reset, + clearDiagnostics: diagnostics.clear, + connection, }); const handleDelete = (options?: { removeRelatedDockerContainers?: boolean }) => { @@ -567,16 +563,29 @@ function ProfileEditForm({ executor, profile }: { executor: Executor; profile: E }; return ( -
+
} /> + {memberReadOnly && } + {form.isKubernetes && ( + + )}
@@ -584,7 +593,10 @@ function ProfileEditForm({ executor, profile }: { executor: Executor; profile: E

{t("executors:spritesApiKeyIsRequired")}

)} {persistence.error &&

{persistence.error}

} - persistence.setDeleteDialogOpen(true)} /> + persistence.setDeleteDialogOpen(true)} + disabled={memberReadOnly} + /> ({ + replace: vi.fn(), + useKubernetesExecutorResource: vi.fn(), +})); + +vi.mock("@/lib/routing/client-router", () => ({ + useRouter: () => ({ replace, push: vi.fn() }), +})); + +vi.mock("@/hooks/domains/settings/use-kubernetes-settings", () => ({ + useKubernetesExecutorResource, +})); + +const TIMESTAMP = "2026-08-31T00:00:00Z"; + +function kubernetesExecutor(withProfile: boolean): Executor { + const id = "executor/primary"; + const profile: ExecutorProfile = { + id: "profile/primary", + executor_id: id, + executor_type: "k8s", + name: "Primary profile", + config: {}, + prepare_script: "", + cleanup_script: "", + env_vars: [], + created_at: TIMESTAMP, + updated_at: TIMESTAMP, + }; + return { + id, + name: "Primary executor", + type: "k8s", + status: "active", + is_system: false, + config: {}, + profiles: withProfile ? [profile] : [], + created_at: TIMESTAMP, + updated_at: TIMESTAMP, + }; +} + +function renderPage(record: Executor) { + render( + + + , + ); +} + +afterEach(() => { + cleanup(); + replace.mockReset(); + useKubernetesExecutorResource.mockReset(); +}); + +describe("KubernetesExecutorPage", () => { + it("redirects a configured executor to its unified profile settings page", async () => { + renderPage(kubernetesExecutor(true)); + + await waitFor(() => + expect(replace).toHaveBeenCalledWith("/settings/executors/profile%2Fprimary"), + ); + expect(useKubernetesExecutorResource).not.toHaveBeenCalled(); + }); + + it("keeps the standalone connection page for an executor without profiles", () => { + useKubernetesExecutorResource.mockReturnValue({ loading: true }); + + renderPage(kubernetesExecutor(false)); + + expect(screen.getByText("Loading executor...")).toBeTruthy(); + expect(replace).not.toHaveBeenCalled(); + expect(useKubernetesExecutorResource).toHaveBeenCalledWith("executor/primary"); + }); +}); diff --git a/apps/web/app/settings/executors/k8s/[executorId]/page.tsx b/apps/web/app/settings/executors/k8s/[executorId]/page.tsx new file mode 100644 index 0000000000..a672027d4e --- /dev/null +++ b/apps/web/app/settings/executors/k8s/[executorId]/page.tsx @@ -0,0 +1,318 @@ +"use client"; + +import { useCallback, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useRouter } from "@/lib/routing/client-router"; +import { runWithNavigationBlockerBypassed } from "@/lib/routing/navigation-guard"; +import { Badge } from "@kandev/ui/badge"; +import { Button } from "@kandev/ui/button"; +import { Card, CardContent } from "@kandev/ui/card"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@kandev/ui/dialog"; +import { Separator } from "@kandev/ui/separator"; +import { IconTrash } from "@tabler/icons-react"; +import { KubernetesConnectionCard } from "@/components/settings/kubernetes-connection-card"; +import { KubernetesDiagnosticsCard } from "@/components/settings/kubernetes-diagnostics-card"; +import { KubernetesReadOnlyNotice } from "@/components/settings/kubernetes-read-only-notice"; +import { saveWithKubernetesSessionConfirmation } from "@/components/settings/kubernetes-save-confirmation"; +import { KubernetesSessionsCard } from "@/components/settings/kubernetes-sessions-card"; +import { useAppStore } from "@/components/state-provider"; +import { + SettingsSaveCancelledError, + useSettingsSaveContributor, +} from "@/components/settings/settings-save-provider"; +import { serializeSettingsRevision } from "@/components/settings/settings-save-revision"; +import { + isKubernetesExecutorDirty, + parseKubernetesExecutorConfig, + serializeKubernetesExecutorConfig, + type KubernetesExecutorForm, +} from "@/components/settings/kubernetes-config"; +import { kubernetesExecutorInvalidReason } from "@/components/settings/kubernetes-validation"; +import { + useKubernetesAdminAccess, + useKubernetesDiagnostics, + useKubernetesExecutorResource, + useKubernetesSessionImpact, + useKubernetesSessions, +} from "@/hooks/domains/settings/use-kubernetes-settings"; +import { getExecutorIcon, getExecutorLabel } from "@/lib/executor-icons"; +import { executorProfileSettingsPath } from "@/lib/settings/executor-settings-routes"; +import { SettingsRedirect } from "@/src/settings-route-helpers"; + +const EXECUTORS_ROUTE = "/settings/executors"; +const KubernetesIcon = getExecutorIcon("k8s"); + +export default function KubernetesExecutorPage({ executorId }: { executorId: string }) { + const profileId = useAppStore( + (state) => + state.executors.items.find( + (executor) => executor.id === executorId && executor.type === "k8s", + )?.profiles?.[0]?.id ?? null, + ); + if (profileId) { + return ( + + ); + } + return ; +} + +function KubernetesExecutorRecoveryPage({ executorId }: { executorId: string }) { + const resource = useKubernetesExecutorResource(executorId); + const { t } = useTranslation(); + if (resource.loading) return ; + if (resource.error || !resource.executor) { + const message = + resource.error instanceof Error && resource.error.message + ? resource.error.message + : t("executors:executorNotFound"); + return ; + } + if (resource.executor.type !== "k8s") { + return ; + } + return ; +} + +function ExecutorMessage({ message }: { message: string }) { + const { t } = useTranslation(); + const router = useRouter(); + return ( + + +

{message}

+ + + + ); +} + +type ResourceController = ReturnType; +type KubernetesExecutorRecord = NonNullable; + +function KubernetesExecutorView({ + executor, + resource, +}: { + executor: KubernetesExecutorRecord; + resource: ResourceController; +}) { + const { t } = useTranslation(); + const router = useRouter(); + const canManage = useKubernetesAdminAccess(); + const diagnostics = useKubernetesDiagnostics(); + const sessions = useKubernetesSessions(executor.id); + const sessionImpact = useKubernetesSessionImpact(executor.id); + const initial = parseKubernetesExecutorConfig(executor.name, executor.config); + const [baseline, setBaseline] = useState(initial); + const [form, setForm] = useState(initial); + const [error, setError] = useState(null); + const [deleting, setDeleting] = useState(false); + const [deleteOpen, setDeleteOpen] = useState(false); + const config = serializeKubernetesExecutorConfig(form); + const revision = serializeSettingsRevision({ name: form.name.trim(), config }); + const invalidReason = kubernetesExecutorInvalidReason(form, canManage, t); + + const save = useCallback(async () => { + setError(null); + try { + await saveWithKubernetesSessionConfirmation({ + kind: "connection", + loadActiveSessionCount: sessionImpact.loadActiveSessionCount, + save: () => resource.update(form.name.trim(), config), + }); + setBaseline({ ...form }); + } catch (cause) { + if (!(cause instanceof SettingsSaveCancelledError)) setError(cause); + throw cause; + } + }, [config, form, resource, sessionImpact.loadActiveSessionCount]); + + const discard = useCallback(() => { + setForm({ ...baseline }); + diagnostics.clear(); + setError(null); + }, [baseline, diagnostics]); + + useSettingsSaveContributor({ + id: `kubernetes-executor:${executor.id}`, + revision, + isDirty: canManage && isKubernetesExecutorDirty(form, baseline), + canSave: !invalidReason, + invalidReason, + save, + discard, + }); + + const runTest = () => { + void diagnostics.run({ config }).catch(() => undefined); + }; + const remove = async () => { + setDeleting(true); + setError(null); + try { + await resource.remove(); + runWithNavigationBlockerBypassed(() => router.push(EXECUTORS_ROUTE)); + } catch (cause) { + setError(cause); + } finally { + setDeleting(false); + setDeleteOpen(false); + } + }; + + return ( +
+ setDeleteOpen(true)} + /> + {!canManage && } + + + + + void remove()} + deleting={deleting} + /> +
+ ); +} + +function KubernetesActionError({ error }: { error: unknown }) { + const { t } = useTranslation(); + if (!error) return null; + const message = + error instanceof Error && error.message + ? error.message + : t("executors:kubernetesExecutorActionFailed"); + return ( +

+ {message} +

+ ); +} + +function KubernetesExecutorHeader({ + name, + canManage, + onDelete, +}: { + name: string; + canManage: boolean; + onDelete: () => void; +}) { + const { t } = useTranslation(); + const router = useRouter(); + return ( + <> +
+
+
+ +

{name}

+ + {getExecutorLabel("k8s")} + +
+

+ {t("executors:kubernetesExecutorPageDescription")} +

+
+
+ + +
+
+ + + ); +} + +function DeleteExecutorDialog({ + open, + onOpenChange, + onDelete, + deleting, +}: { + open: boolean; + onOpenChange: (open: boolean) => void; + onDelete: () => void; + deleting: boolean; +}) { + const { t } = useTranslation(); + return ( + + + + {t("executors:deleteExecutor")} + + {t("executors:kubernetesDeleteExecutorDescription")} + + + + + + + + + ); +} diff --git a/apps/web/app/settings/executors/new/[type]/executor-types.ts b/apps/web/app/settings/executors/new/[type]/executor-types.ts index c4d956ad49..bee74777c6 100644 --- a/apps/web/app/settings/executors/new/[type]/executor-types.ts +++ b/apps/web/app/settings/executors/new/[type]/executor-types.ts @@ -3,7 +3,7 @@ // the 600-line lint cap and new types can be added without touching layout. // // The map keys (`local`, `worktree`, `local_docker`, `remote_docker`, -// `sprites`, `ssh`) are the persisted executor enum and the create route's +// `sprites`, `ssh`, `k8s`) are the persisted executor enum and the create route's // path segment, and `executorId` is a backend row id — none of them is copy. // Labels and descriptions travel as catalog keys and resolve at render; // `brandLabel` carries a brand/protocol name that reads the same in every @@ -52,6 +52,11 @@ export const EXECUTOR_TYPE_MAP: Record = { brandLabel: "SSH", descriptionKey: "executors:descriptionSshCreate", }, + k8s: { + executorId: "exec-k8s", + labelKey: "executors:typeKubernetes", + descriptionKey: "executors:descriptionKubernetes", + }, }; /** The label for a type card: a brand name verbatim, or the translated label. */ diff --git a/apps/web/app/settings/executors/new/[type]/kubernetes-create-page.tsx b/apps/web/app/settings/executors/new/[type]/kubernetes-create-page.tsx new file mode 100644 index 0000000000..738d76186f --- /dev/null +++ b/apps/web/app/settings/executors/new/[type]/kubernetes-create-page.tsx @@ -0,0 +1,191 @@ +"use client"; + +import { useCallback, useMemo, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { useRouter } from "@/lib/routing/client-router"; +import { runWithNavigationBlockerBypassed } from "@/lib/routing/navigation-guard"; +import { Badge } from "@kandev/ui/badge"; +import { Button } from "@kandev/ui/button"; +import { Separator } from "@kandev/ui/separator"; +import { KubernetesConnectionCard } from "@/components/settings/kubernetes-connection-card"; +import { kubernetesCreateErrorMessage } from "@/components/settings/kubernetes-create-error"; +import { KubernetesDiagnosticsCard } from "@/components/settings/kubernetes-diagnostics-card"; +import { KubernetesReadOnlyNotice } from "@/components/settings/kubernetes-read-only-notice"; +import { KubernetesWorkloadCard } from "@/components/settings/kubernetes-workload-card"; +import { KubernetesWorkspaceCard } from "@/components/settings/kubernetes-workspace-card"; +import { ProfileDetailsCard } from "@/components/settings/profile-edit/profile-details-card"; +import { useSettingsSaveContributor } from "@/components/settings/settings-save-provider"; +import { serializeSettingsRevision } from "@/components/settings/settings-save-revision"; +import { + createDefaultKubernetesExecutorForm, + createDefaultKubernetesProfileConfig, + serializeKubernetesExecutorConfig, + serializeKubernetesProfileConfig, + type KubernetesExecutorForm, + type KubernetesProfileConfigForm, +} from "@/components/settings/kubernetes-config"; +import { + getKubernetesCreateContributorState, + kubernetesProfileInvalidReason, +} from "@/components/settings/kubernetes-validation"; +import { + useKubernetesAdminAccess, + useKubernetesDiagnostics, + useKubernetesExecutorResource, +} from "@/hooks/domains/settings/use-kubernetes-settings"; +import { getExecutorIcon, getExecutorLabel } from "@/lib/executor-icons"; +import { executorProfileSettingsPath } from "@/lib/settings/executor-settings-routes"; + +const EXECUTORS_ROUTE = "/settings/executors"; +const KubernetesIcon = getExecutorIcon("k8s"); + +export function KubernetesCreatePage() { + const { t } = useTranslation(); + const router = useRouter(); + const canManage = useKubernetesAdminAccess(); + const resource = useKubernetesExecutorResource(); + const diagnostics = useKubernetesDiagnostics(); + const [executorBaseline] = useState(createDefaultKubernetesExecutorForm); + const [profileBaseline] = useState(createDefaultKubernetesProfileConfig); + const [executor, setExecutor] = useState(executorBaseline); + const [profileName, setProfileName] = useState(""); + const [profile, setProfile] = useState(profileBaseline); + const [error, setError] = useState(null); + + const payload = useMemo( + () => ({ + name: executor.name.trim(), + config: serializeKubernetesExecutorConfig(executor), + profileName: profileName.trim(), + profileConfig: serializeKubernetesProfileConfig(profile), + }), + [executor, profile, profileName], + ); + const invalidReason = createInvalidReason(executor, profileName, profile, canManage, t); + const contributorState = getKubernetesCreateContributorState(canManage, invalidReason); + const revision = serializeSettingsRevision(payload); + + const save = useCallback(async () => { + setError(null); + try { + const created = await resource.create(payload); + runWithNavigationBlockerBypassed(() => + router.push(executorProfileSettingsPath(created.executor, created.profile.id)), + ); + } catch (cause) { + setError(cause); + throw cause; + } + }, [payload, resource, router]); + + const discard = useCallback(() => { + setExecutor({ ...executorBaseline }); + setProfileName(""); + setProfile({ ...profileBaseline, accessModes: [...profileBaseline.accessModes] }); + diagnostics.clear(); + setError(null); + }, [diagnostics, executorBaseline, profileBaseline]); + + useSettingsSaveContributor({ + id: "kubernetes-executor:create", + revision, + isDirty: contributorState.isDirty, + canSave: contributorState.canSave, + invalidReason, + save, + discard, + }); + + const runTest = () => { + void diagnostics + .run({ config: payload.config, profile_config: payload.profileConfig }) + .catch(() => undefined); + }; + const errorMessage = kubernetesCreateErrorMessage(error, t); + + return ( +
+ + {!canManage && } + +
+ + + +
+ + {Boolean(error) && ( +

+ {errorMessage} +

+ )} +
+ ); +} + +function KubernetesCreateHeader() { + const { t } = useTranslation(); + const router = useRouter(); + return ( + <> +
+
+
+ +

+ {t("executors:kubernetesCreateTitle")} +

+ + {getExecutorLabel("k8s")} + +
+

+ {t("executors:kubernetesCreateDescription")} +

+
+ +
+ + + ); +} + +function createInvalidReason( + executor: KubernetesExecutorForm, + profileName: string, + profile: KubernetesProfileConfigForm, + canManage: boolean, + t: (key: string) => string, +): string | undefined { + if (!canManage) return t("executors:kubernetesAdminSaveOnly"); + if (!executor.name.trim()) return t("executors:kubernetesExecutorNameRequired"); + if (!executor.namespace.trim()) return t("executors:kubernetesNamespaceRequired"); + if (executor.authMode === "kubeconfig" && !executor.kubeconfigPath.trim()) { + return t("executors:kubernetesKubeconfigPathRequired"); + } + const timeout = Number(executor.requestTimeoutSeconds); + if (!Number.isInteger(timeout) || timeout < 1 || timeout > 300) { + return t("executors:kubernetesTimeoutInvalid"); + } + if (!profileName.trim()) return t("executors:profileNameIsRequired"); + return kubernetesProfileInvalidReason(profile, t); +} diff --git a/apps/web/app/settings/executors/new/[type]/page.tsx b/apps/web/app/settings/executors/new/[type]/page.tsx index 40a88791ca..893f7eb6fb 100644 --- a/apps/web/app/settings/executors/new/[type]/page.tsx +++ b/apps/web/app/settings/executors/new/[type]/page.tsx @@ -47,6 +47,7 @@ import type { Executor, ExecutorType, ProfileEnvVar } from "@/lib/types/http"; import { EXECUTOR_TYPE_MAP, executorTypeLabel, type ExecutorTypeInfo } from "./executor-types"; import { SSHCreatePage } from "./ssh-create-page"; +import { KubernetesCreatePage } from "./kubernetes-create-page"; const EXECUTORS_ROUTE = "/settings/executors"; const SPRITES_TOKEN_KEY = "SPRITES_API_TOKEN"; @@ -69,6 +70,10 @@ export default function CreateProfilePage({ executorType }: { executorType: stri return ; } + if (executorType === "k8s") { + return ; + } + return ; } diff --git a/apps/web/app/settings/executors/page.tsx b/apps/web/app/settings/executors/page.tsx index 36796e4b0c..84fe78a06c 100644 --- a/apps/web/app/settings/executors/page.tsx +++ b/apps/web/app/settings/executors/page.tsx @@ -20,6 +20,7 @@ import { useAppStore } from "@/components/state-provider"; import { deleteExecutorProfile } from "@/lib/api/domains/settings-api"; import { EXECUTOR_ICON_MAP, getExecutorLabel } from "@/lib/executor-icons"; import type { Executor, ExecutorProfile } from "@/lib/types/http"; +import { KubernetesReadOnlyNotice } from "@/components/settings/kubernetes-read-only-notice"; type ProfileWithExecutor = ExecutorProfile & { executor_type: string; @@ -80,6 +81,11 @@ const EXECUTOR_TYPES: readonly ExecutorTypeCard[] = [ descriptionKey: "executors:hubDescriptionSprites", }, { type: "ssh", brandLabel: "SSH", descriptionKey: "executors:hubDescriptionSsh" }, + { + type: "k8s", + labelKey: "executors:typeKubernetes", + descriptionKey: "executors:hubDescriptionKubernetes", + }, ]; function ExecutorIconBadge({ type }: { type: string }) { @@ -94,13 +100,15 @@ function ExecutorIconBadge({ type }: { type: string }) { function ProfileCard({ profile, onDelete, + canDelete, }: { profile: ProfileWithExecutor; onDelete: (id: string) => void; + canDelete: boolean; }) { // Subscribes so the `getExecutorLabel` badge below re-renders on a locale // switch; the helper resolves at call time but does not itself notify React. - useTranslation(); + const { t } = useTranslation(); const router = useRouter(); return ( { e.stopPropagation(); onDelete(profile.id); @@ -134,15 +144,29 @@ function ProfileCard({ function CreateTypeCard({ execType, onClick, + disabled = false, }: { execType: ExecutorTypeCard; onClick: () => void; + disabled?: boolean; }) { const { t } = useTranslation(); return ( { + if (disabled || (event.key !== "Enter" && event.key !== " ")) return; + event.preventDefault(); + onClick(); + }} > @@ -180,14 +204,18 @@ function DeleteProfileDialog({ - @@ -203,6 +231,8 @@ export default function ExecutorsHubPage() { const allProfiles = useAllProfiles(); const executors = useAppStore((state) => state.executors.items); const setExecutors = useAppStore((state) => state.setExecutors); + const role = useAppStore((state) => state.auth.user?.role); + const canManageKubernetes = role === undefined || role === "admin"; const [deleteProfileId, setDeleteProfileId] = useState(null); const [deleting, setDeleting] = useState(false); @@ -212,6 +242,7 @@ export default function ExecutorsHubPage() { const handleDelete = async () => { if (!profileToDelete) return; + if (profileToDelete.executor_type === "k8s" && !canManageKubernetes) return; setDeleting(true); try { await deleteExecutorProfile(profileToDelete.parent_executor_id, profileToDelete.id); @@ -235,12 +266,18 @@ export default function ExecutorsHubPage() {

{t("executors:hubDescription")}

+ {!canManageKubernetes && } {allProfiles.length > 0 && (

{t("executors:profiles")}

{allProfiles.map((profile) => ( - + ))}
@@ -253,6 +290,7 @@ export default function ExecutorsHubPage() { key={execType.type} execType={execType} onClick={() => router.push(`/settings/executors/new/${execType.type}`)} + disabled={execType.type === "k8s" && !canManageKubernetes} /> ))}
diff --git a/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.test.ts b/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.test.ts index 713d5019a0..272ad15839 100644 --- a/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.test.ts +++ b/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.test.ts @@ -14,6 +14,7 @@ import { const WORKSPACE_ID = "ws-1"; const WORKSPACES_HREF = "/settings/workspaces"; +const EXECUTORS_HREF = "/settings/executors"; const WORKSPACES = [{ id: WORKSPACE_ID, name: "Main Workspace" }]; const ORDERED_WORKSPACES = [ { id: "ws-first", name: "First" }, @@ -326,6 +327,34 @@ describe("buildExecutorsBranch", () => { "/settings/executor/exec-1/profile/exec-profile-1", ]); }); + + it("makes a configured Kubernetes executor disclosure-only and links its profile root", () => { + const [executor] = buildExecutorsBranch([ + { + id: "cluster/primary", + name: "Kubernetes", + type: "k8s", + profiles: [{ id: "profile/primary", name: "Primary" }], + }, + ]); + + expect(executor.href).toBeNull(); + expect(hrefsOf(executor.children ?? [])).toEqual(["/settings/executors/profile%2Fprimary"]); + }); + + it("keeps the Kubernetes connection route reachable for an executor without profiles", () => { + const [executor] = buildExecutorsBranch([ + { + id: "cluster/primary", + name: "Kubernetes", + type: "k8s", + profiles: [], + }, + ]); + + expect(executor.href).toBe("/settings/executors/k8s/cluster%2Fprimary"); + expect(executor.children).toEqual([]); + }); }); describe("findActiveNodePath", () => { @@ -378,6 +407,32 @@ describe("findActiveNodePath", () => { ]); }); + it("reaches a Kubernetes profile through its disclosure-only executor", () => { + const kubernetesForest = [ + buildBranchRoot( + { + href: EXECUTORS_HREF, + labelKey: "common:executors", + activePrefixes: ["/settings/executors/", "/settings/executor/"], + }, + buildExecutorsBranch([ + { + id: "cluster-primary", + name: "Kubernetes", + type: "k8s", + profiles: [{ id: "profile-primary", name: "Primary" }], + }, + ]), + ), + ]; + + expect(findActiveNodePath(kubernetesForest, `${EXECUTORS_HREF}/profile-primary`)).toEqual([ + settingsMenuRowKey(EXECUTORS_HREF), + "executor:cluster-primary", + "executor:cluster-primary:profile:profile-primary", + ]); + }); + it("stops at the row when the route has no node — the row stays active", () => { // The install catalogue is not an agent, and the flat profile spelling has // no executor to nest under. Both are the row's own pages. @@ -385,7 +440,7 @@ describe("findActiveNodePath", () => { settingsMenuRowKey("/settings/agents"), ]); expect(findActiveNodePath(forest, "/settings/executors/exec-profile-1")).toEqual([ - settingsMenuRowKey("/settings/executors"), + settingsMenuRowKey(EXECUTORS_HREF), ]); }); @@ -393,7 +448,7 @@ describe("findActiveNodePath", () => { // `/settings/executor/` is not under `/settings/executors`, but the // Executors row owns it — and so must its branch. expect(findActiveNodePath(forest, "/settings/executor/exec-1")).toEqual([ - settingsMenuRowKey("/settings/executors"), + settingsMenuRowKey(EXECUTORS_HREF), "executor:exec-1", ]); }); diff --git a/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.ts b/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.ts index edfde9694c..1990fdc160 100644 --- a/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.ts +++ b/apps/web/components/app-sidebar/sections/settings/settings-menu-branches.ts @@ -6,6 +6,10 @@ import { EXECUTORS_SETTINGS_HREF } from "@/lib/settings-discovery/catalog/execut import { WORKSPACE_INTEGRATIONS } from "@/lib/settings-discovery/catalog/integrations"; import { WORKSPACES_SETTINGS_HREF } from "@/lib/settings-discovery/catalog/workspaces"; import { INTEGRATION_ICONS } from "@/lib/settings/integration-icons"; +import { + executorConnectionSettingsPath, + executorProfileSettingsPath, +} from "@/lib/settings/executor-settings-routes"; import { WORKSPACE_SETTINGS_TABS, workspaceSettingsHref, @@ -307,25 +311,28 @@ export function buildAgentsBranch( /** * One node per executor, holding its profiles. * - * Uses the executor-scoped spellings (`/settings/executor/` and - * `/settings/executor//profile/`) rather than the flat - * `/settings/executors/` the Executors page links to. Both are live - * routes for the same profile, but only the scoped pair has an executor - * breadcrumb — so it is the one whose crumb chain matches the branch the user - * clicked through. + * Most executors use the scoped legacy spellings so their executor breadcrumb + * matches the branch. A configured Kubernetes executor only discloses its + * profile children: connection, diagnostics, sessions and workload settings + * all live on that profile page. Its standalone connection route remains + * reachable only when there is no profile to recover through. */ export function buildExecutorsBranch(executors: ReadonlyArray): SettingsMenuNode[] { return executors.map((executor) => { - const executorHref = `/settings/executor/${encodeURIComponent(executor.id)}`; + const profiles = executor.profiles ?? []; + const executorHref = + executor.type === "k8s" && profiles.length > 0 + ? null + : executorConnectionSettingsPath(executor); return { key: `executor:${executor.id}`, href: executorHref, label: { text: executor.name }, icon: getExecutorIcon(executor.type), // Same split as agents: the executor ships with kandev, the profiles do not. - children: (executor.profiles ?? []).map((profile) => ({ + children: profiles.map((profile) => ({ key: `executor:${executor.id}:profile:${profile.id}`, - href: `${executorHref}/profile/${profile.id}`, + href: executorProfileSettingsPath(executor, profile.id), label: { text: profile.name }, isUserRecord: true, })), @@ -336,7 +343,8 @@ export function buildExecutorsBranch(executors: ReadonlyArray): /** True when `pathname` is this node's page or lives underneath it. */ function nodeOwns(node: SettingsMenuNode, pathname: string): boolean { if (node.href && (pathname === node.href || pathname.startsWith(`${node.href}/`))) return true; - return (node.ownsPrefixes ?? []).some((prefix) => pathname.startsWith(prefix)); + if ((node.ownsPrefixes ?? []).some((prefix) => pathname.startsWith(prefix))) return true; + return (node.children ?? []).some((child) => nodeOwns(child, pathname)); } /** diff --git a/apps/web/components/integrations/use-change-request-task-tooltip-state.ts b/apps/web/components/integrations/use-change-request-task-tooltip-state.ts index 91a78a0687..acaa1d1953 100644 --- a/apps/web/components/integrations/use-change-request-task-tooltip-state.ts +++ b/apps/web/components/integrations/use-change-request-task-tooltip-state.ts @@ -1,38 +1 @@ -"use client"; - -import { useState, type FocusEvent, type PointerEvent } from "react"; - -/** Shared fine-pointer/keyboard disclosure behavior for task change-request summaries. */ -export function useChangeRequestTaskTooltipState(onOpen?: () => void) { - const [hovered, setHovered] = useState(false); - const [focused, setFocused] = useState(false); - const [dismissed, setDismissed] = useState(false); - - return { - open: !dismissed && (hovered || focused), - onPointerEnter(event: PointerEvent) { - if (event.pointerType !== "mouse") return; - setHovered(true); - setDismissed(false); - onOpen?.(); - }, - onPointerLeave(event: PointerEvent) { - if (event.pointerType !== "mouse") return; - setHovered(false); - if (!focused) setDismissed(false); - }, - onFocus(event: FocusEvent) { - if (!event.currentTarget.matches(":focus-visible")) return; - setFocused(true); - setDismissed(false); - onOpen?.(); - }, - onBlur() { - setFocused(false); - if (!hovered) setDismissed(false); - }, - onEscapeKeyDown() { - setDismissed(true); - }, - }; -} +export { useTaskIconTooltipState as useChangeRequestTaskTooltipState } from "@/components/task/use-task-icon-tooltip-state"; diff --git a/apps/web/components/kanban-card-content.tsx b/apps/web/components/kanban-card-content.tsx index df08392860..c265039c24 100644 --- a/apps/web/components/kanban-card-content.tsx +++ b/apps/web/components/kanban-card-content.tsx @@ -158,6 +158,7 @@ export function KanbanCardBody({ diff --git a/apps/web/components/settings/executor-copy.test.ts b/apps/web/components/settings/executor-copy.test.ts index 37c7c0eda4..d28d68f3eb 100644 --- a/apps/web/components/settings/executor-copy.test.ts +++ b/apps/web/components/settings/executor-copy.test.ts @@ -33,6 +33,7 @@ import { getExecutorDescription } from "./executor-description"; */ const SPRITES_DEV = "Sprites.dev"; const SSH = "SSH"; +const KUBERNETES = "Kubernetes"; const HUB_CARD_COPY: Array<{ type: string; label: string; description: string }> = [ { type: "local", label: "Local", description: "Run agents directly in the repository folder." }, @@ -56,6 +57,11 @@ const HUB_CARD_COPY: Array<{ type: string; label: string; description: string }> label: SSH, description: "Connect to a remote host over SSH and run agentctl there.", }, + { + type: "k8s", + label: KUBERNETES, + description: "Run agents in administrator-configured Kubernetes Pods.", + }, ]; const HUB_CARD_KEYS: Record = @@ -68,6 +74,10 @@ const HUB_CARD_KEYS: Record { @@ -110,6 +120,11 @@ describe("create-page executor type registry", () => { label: SSH, description: "Connects to a remote host over SSH and runs agentctl there.", }, + { + type: "k8s", + label: KUBERNETES, + description: "Runs agents in administrator-configured Kubernetes Pods.", + }, ]; it.each(CREATE_COPY)("renders $type unchanged", ({ type, label, description }) => { @@ -120,6 +135,7 @@ describe("create-page executor type registry", () => { it("keeps the persisted executor enum and backend ids out of the catalog", () => { expect(Object.keys(EXECUTOR_TYPE_MAP).sort()).toEqual([ + "k8s", "local", "local_docker", "remote_docker", @@ -128,6 +144,7 @@ describe("create-page executor type registry", () => { "worktree", ]); expect(EXECUTOR_TYPE_MAP.ssh.executorId).toBe("exec-ssh"); + expect(EXECUTOR_TYPE_MAP.k8s.executorId).toBe("exec-k8s"); expect(EXECUTOR_TYPE_MAP.local_docker.executorId).toBe("exec-local-docker"); }); }); @@ -144,6 +161,7 @@ describe("getExecutorDescription", () => { ["remote_docker", "Connects to a remote Docker host."], ["sprites", "Runs agents in Sprites.dev cloud sandboxes."], ["ssh", "Runs agents on a trusted Linux amd64 or macOS host over SSH."], + ["k8s", "Runs agents in administrator-configured Kubernetes Pods."], ] as const)("renders %s unchanged", (type, expected) => { expect(getExecutorDescription(type)).toBe(expected); }); @@ -168,6 +186,7 @@ describe("getExecutorLabel", () => { expect(getExecutorLabel("worktree")).toBe("Worktree"); expect(getExecutorLabel("local_docker")).toBe("Local Docker"); expect(getExecutorLabel("remote_docker")).toBe("Remote Docker"); + expect(getExecutorLabel("k8s")).toBe(KUBERNETES); }); it("echoes an unmapped wire value", () => { diff --git a/apps/web/components/settings/executor-description.ts b/apps/web/components/settings/executor-description.ts index 2392cf1a26..7523cc34b3 100644 --- a/apps/web/components/settings/executor-description.ts +++ b/apps/web/components/settings/executor-description.ts @@ -19,5 +19,6 @@ export function getExecutorDescription(type: ExecutorType): string { if (type === "remote_docker") return t("executors:descriptionRemoteDocker"); if (type === "sprites") return t("executors:descriptionSprites"); if (type === "ssh") return t("executors:descriptionSsh"); + if (type === "k8s") return t("executors:descriptionKubernetes"); return t("executors:descriptionCustom"); } diff --git a/apps/web/components/settings/executor-profiles-card.test.tsx b/apps/web/components/settings/executor-profiles-card.test.tsx new file mode 100644 index 0000000000..d9408d0aa3 --- /dev/null +++ b/apps/web/components/settings/executor-profiles-card.test.tsx @@ -0,0 +1,75 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { StateProvider } from "@/components/state-provider"; +import type { Executor, ExecutorProfile } from "@/lib/types/http"; +import { ExecutorProfilesCard } from "./executor-profiles-card"; + +const { push } = vi.hoisted(() => ({ push: vi.fn() })); + +vi.mock("@/lib/routing/client-router", () => ({ useRouter: () => ({ push }) })); + +const TIMESTAMP = "2026-08-24T00:00:00Z"; + +afterEach(() => { + cleanup(); + push.mockReset(); +}); + +function profile(executorId: string, executorType: Executor["type"]): ExecutorProfile { + return { + id: "profile/primary", + executor_id: executorId, + executor_type: executorType, + name: "Primary profile", + config: {}, + prepare_script: "", + cleanup_script: "", + env_vars: [], + created_at: TIMESTAMP, + updated_at: TIMESTAMP, + }; +} + +function executor(type: Executor["type"]): Executor { + const id = "executor/primary"; + return { + id, + name: "Primary executor", + type, + status: "active", + is_system: false, + config: {}, + profiles: [profile(id, type)], + created_at: TIMESTAMP, + updated_at: TIMESTAMP, + }; +} + +function renderCard(record: Executor) { + render( + + + , + ); +} + +describe("ExecutorProfilesCard profile navigation", () => { + it("opens a Kubernetes profile in the Kubernetes-aware editor", () => { + renderCard(executor("k8s")); + + fireEvent.click(screen.getByText("Primary profile")); + + expect(push).toHaveBeenCalledWith("/settings/executors/profile%2Fprimary"); + }); + + it("preserves the scoped legacy profile route for SSH", () => { + renderCard(executor("ssh")); + + fireEvent.click(screen.getByText("Primary profile")); + + expect(push).toHaveBeenCalledWith( + "/settings/executor/executor%2Fprimary/profile/profile%2Fprimary", + ); + }); +}); diff --git a/apps/web/components/settings/executor-profiles-card.tsx b/apps/web/components/settings/executor-profiles-card.tsx index 9ab3d417c3..12f2ebe505 100644 --- a/apps/web/components/settings/executor-profiles-card.tsx +++ b/apps/web/components/settings/executor-profiles-card.tsx @@ -14,6 +14,7 @@ import { useTranslation } from "react-i18next"; import { SettingsCardHeader } from "@/components/settings/settings-card-header"; import { SETTINGS_TYPOGRAPHY } from "@/components/settings/settings-typography"; import { settingsActionClassName } from "@/components/settings/settings-control"; +import { executorProfileSettingsPath } from "@/lib/settings/executor-settings-routes"; type ExecutorProfilesCardProps = { executorId: string; @@ -56,6 +57,7 @@ export function ExecutorProfilesCard({ executorId, profiles }: ExecutorProfilesC const [dialogOpen, setDialogOpen] = useState(false); const executors = useAppStore((state) => state.executors.items); const setExecutors = useAppStore((state) => state.setExecutors); + const executorType = executors.find((executor) => executor.id === executorId)?.type; const refreshProfiles = useCallback(async () => { try { @@ -75,9 +77,9 @@ export function ExecutorProfilesCard({ executorId, profiles }: ExecutorProfilesC const handleProfileCreated = useCallback( (profile: ExecutorProfile) => { refreshProfiles(); - router.push(`/settings/executor/${executorId}/profile/${profile.id}`); + router.push(executorProfileSettingsPath({ id: executorId, type: executorType }, profile.id)); }, - [executorId, refreshProfiles, router], + [executorId, executorType, refreshProfiles, router], ); const handleDelete = useCallback( @@ -112,7 +114,12 @@ export function ExecutorProfilesCard({ executorId, profiles }: ExecutorProfilesC key={profile.id} className="flex items-center justify-between rounded-md border px-3 py-2 hover:bg-muted/50 cursor-pointer transition-colors" onClick={() => - router.push(`/settings/executor/${executorId}/profile/${profile.id}`) + router.push( + executorProfileSettingsPath( + { id: executorId, type: executorType }, + profile.id, + ), + ) } >
diff --git a/apps/web/components/settings/kubernetes-config.test.ts b/apps/web/components/settings/kubernetes-config.test.ts new file mode 100644 index 0000000000..92bc58bcd8 --- /dev/null +++ b/apps/web/components/settings/kubernetes-config.test.ts @@ -0,0 +1,287 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_KUBERNETES_IMAGE, + createDefaultKubernetesProfileConfig, + getKubernetesProfileValidationError, + isKubernetesExecutorDirty, + isKubernetesProfileDirty, + parseKubernetesExecutorConfig, + parseKubernetesProfileConfig, + replaceKubernetesProfileConfig, + serializeKubernetesExecutorConfig, + serializeKubernetesProfileConfig, +} from "./kubernetes-config"; + +describe("Kubernetes executor settings config", () => { + it("uses the multi-architecture Kandev image as the starter workload", () => { + expect(DEFAULT_KUBERNETES_IMAGE).toBe("ghcr.io/kdlbs/kandev:latest"); + expect(createDefaultKubernetesProfileConfig().podTemplateYaml).toContain( + `image: ${DEFAULT_KUBERNETES_IMAGE}`, + ); + }); + + it("provides serialization and dirty-baseline helpers", async () => { + const config = (await import("./kubernetes-config")) as Record; + + expect( + [ + config.serializeKubernetesExecutorConfig, + config.parseKubernetesProfileConfig, + config.serializeKubernetesProfileConfig, + config.replaceKubernetesProfileConfig, + config.isKubernetesExecutorDirty, + config.isKubernetesProfileDirty, + ].every((value) => typeof value === "function"), + ).toBe(true); + }); + + it("parses a persisted kubeconfig executor", () => { + expect( + parseKubernetesExecutorConfig("Production cluster", { + auth_mode: "kubeconfig", + kubeconfig_path: " /etc/kandev/cluster.yaml ", + kube_context: "production", + namespace: "agents", + request_timeout_seconds: "45", + }), + ).toEqual({ + name: "Production cluster", + authMode: "kubeconfig", + kubeconfigPath: "/etc/kandev/cluster.yaml", + kubeContext: "production", + namespace: "agents", + requestTimeoutSeconds: "45", + }); + }); + + it("clears kubeconfig-only fields for in-cluster auth", () => { + expect( + parseKubernetesExecutorConfig("Cluster", { + auth_mode: "in_cluster", + kubeconfig_path: "/stale/config", + kube_context: "stale", + namespace: "kandev", + }), + ).toMatchObject({ + authMode: "in_cluster", + kubeconfigPath: "", + kubeContext: "", + namespace: "kandev", + requestTimeoutSeconds: "30", + }); + }); + + it("serializes only fields accepted by kubeconfig auth", () => { + expect( + serializeKubernetesExecutorConfig({ + name: " Cluster ", + authMode: "kubeconfig", + kubeconfigPath: " /etc/kandev/config ", + kubeContext: " production ", + namespace: " agents ", + requestTimeoutSeconds: " 45 ", + }), + ).toEqual({ + auth_mode: "kubeconfig", + kubeconfig_path: "/etc/kandev/config", + kube_context: "production", + namespace: "agents", + request_timeout_seconds: "45", + }); + }); + + it("omits stale kubeconfig fields from in-cluster serialization", () => { + expect( + serializeKubernetesExecutorConfig({ + name: "Cluster", + authMode: "in_cluster", + kubeconfigPath: "/stale/config", + kubeContext: "stale", + namespace: "kandev", + requestTimeoutSeconds: "30", + }), + ).toEqual({ + auth_mode: "in_cluster", + namespace: "kandev", + request_timeout_seconds: "30", + }); + }); +}); + +describe("Kubernetes profile config parsing", () => { + it("parses dotted managed-PVC profile config and defaults access mode", () => { + expect( + parseKubernetesProfileConfig({ + platform: "linux/arm64", + main_container: "worker", + pod_template_yaml: "template-yaml\n", + "workspace.mode": "managed_pvc", + "workspace.size": "20Gi", + "workspace.storage_class": "fast", + }), + ).toEqual({ + platform: "linux/arm64", + mainContainer: "worker", + podTemplateYaml: "template-yaml\n", + workspaceMode: "managed_pvc", + workspaceSize: "20Gi", + storageClass: "fast", + accessModes: ["ReadWriteOnce"], + claimName: "", + }); + }); + + it("parses legacy workspace aliases without retaining irrelevant fields", () => { + expect( + parseKubernetesProfileConfig({ + platform: "linux/amd64", + main_container: "kandev-agent", + pod_template_yaml: "template-yaml", + workspace_mode: "existing_claim", + workspace_claim_name: "shared-workspace", + workspace_size: "stale", + workspace_access_modes: '["ReadWriteMany"]', + }), + ).toMatchObject({ + workspaceMode: "existing_claim", + workspaceSize: "", + storageClass: "", + accessModes: [], + claimName: "shared-workspace", + }); + }); + + it("preserves unsupported persisted access modes and reports them as invalid", () => { + const parsed = parseKubernetesProfileConfig({ + "workspace.mode": "managed_pvc", + "workspace.access_modes": '["UnsupportedMode"]', + }); + + expect(parsed.accessModes).toEqual(["UnsupportedMode"]); + expect(getKubernetesProfileValidationError(parsed)).toBe("access_mode_invalid"); + }); +}); + +describe("Kubernetes profile config serialization", () => { + it("serializes managed PVC fields and JSON access modes", () => { + expect( + serializeKubernetesProfileConfig({ + ...createDefaultKubernetesProfileConfig(), + workspaceSize: " 25Gi ", + storageClass: " fast ", + accessModes: ["ReadWriteOnce", "ReadOnlyMany"], + claimName: "stale-claim", + }), + ).toMatchObject({ + "workspace.mode": "managed_pvc", + "workspace.size": "25Gi", + "workspace.storage_class": "fast", + "workspace.access_modes": '["ReadWriteOnce","ReadOnlyMany"]', + }); + }); + + it.each([ + { + mode: "empty_dir" as const, + expected: { "workspace.mode": "empty_dir" }, + }, + { + mode: "existing_claim" as const, + expected: { "workspace.mode": "existing_claim", "workspace.claim_name": "shared" }, + }, + ])("serializes only conditional $mode workspace fields", ({ mode, expected }) => { + const serialized = serializeKubernetesProfileConfig({ + ...createDefaultKubernetesProfileConfig(), + workspaceMode: mode, + workspaceSize: "10Gi", + storageClass: "fast", + accessModes: ["ReadWriteMany"], + claimName: " shared ", + }); + + expect( + Object.fromEntries(Object.entries(serialized).filter(([key]) => key.startsWith("workspace"))), + ).toEqual(expected); + }); + + it("replaces Kubernetes keys while preserving shared profile config", () => { + expect( + replaceKubernetesProfileConfig( + { + remote_credentials: '["git"]', + workspace_mode: "empty_dir", + workspace_size: "stale", + "workspace.claim_name": "stale", + }, + { ...createDefaultKubernetesProfileConfig(), workspaceMode: "empty_dir" }, + ), + ).toMatchObject({ + remote_credentials: '["git"]', + "workspace.mode": "empty_dir", + }); + }); +}); + +describe("Kubernetes settings baselines and validation", () => { + it("normalizes executor fields before dirty comparison", () => { + const baseline = parseKubernetesExecutorConfig("Cluster", { + auth_mode: "in_cluster", + namespace: "agents", + request_timeout_seconds: "30", + }); + + expect(isKubernetesExecutorDirty({ ...baseline, namespace: " agents " }, baseline)).toBe(false); + expect(isKubernetesExecutorDirty({ ...baseline, namespace: "other" }, baseline)).toBe(true); + }); + + it("detects raw template and workspace dirty state", () => { + const baseline = createDefaultKubernetesProfileConfig(); + + expect(isKubernetesProfileDirty({ ...baseline }, baseline)).toBe(false); + expect( + isKubernetesProfileDirty( + { ...baseline, podTemplateYaml: `${baseline.podTemplateYaml}\n` }, + baseline, + ), + ).toBe(true); + expect(isKubernetesProfileDirty({ ...baseline, workspaceMode: "empty_dir" }, baseline)).toBe( + true, + ); + }); + + it("validates required and conditional profile fields", () => { + const valid = createDefaultKubernetesProfileConfig(); + + expect(getKubernetesProfileValidationError(valid)).toBeNull(); + expect(getKubernetesProfileValidationError({ ...valid, mainContainer: " " })).toBe( + "main_container_required", + ); + expect(getKubernetesProfileValidationError({ ...valid, mainContainer: "NOT_VALID" })).toBe( + "main_container_invalid", + ); + expect(getKubernetesProfileValidationError({ ...valid, podTemplateYaml: "" })).toBe( + "pod_template_required", + ); + expect( + getKubernetesProfileValidationError({ + ...valid, + workspaceMode: "managed_pvc", + workspaceSize: " ", + }), + ).toBe("workspace_size_required"); + expect( + getKubernetesProfileValidationError({ + ...valid, + workspaceMode: "managed_pvc", + accessModes: [], + }), + ).toBe("access_mode_required"); + expect( + getKubernetesProfileValidationError({ + ...valid, + workspaceMode: "existing_claim", + claimName: " ", + }), + ).toBe("claim_name_required"); + }); +}); diff --git a/apps/web/components/settings/kubernetes-config.ts b/apps/web/components/settings/kubernetes-config.ts new file mode 100644 index 0000000000..1711b81c33 --- /dev/null +++ b/apps/web/components/settings/kubernetes-config.ts @@ -0,0 +1,267 @@ +export type KubernetesAuthMode = "kubeconfig" | "in_cluster"; +export type KubernetesPlatform = "linux/amd64" | "linux/arm64"; +export type KubernetesWorkspaceMode = "managed_pvc" | "empty_dir" | "existing_claim"; +export type KubernetesAccessMode = + | "ReadWriteOnce" + | "ReadOnlyMany" + | "ReadWriteMany" + | "ReadWriteOncePod"; + +export type KubernetesExecutorForm = { + name: string; + authMode: KubernetesAuthMode; + kubeconfigPath: string; + kubeContext: string; + namespace: string; + requestTimeoutSeconds: string; +}; + +export type KubernetesProfileConfigForm = { + platform: KubernetesPlatform; + mainContainer: string; + podTemplateYaml: string; + workspaceMode: KubernetesWorkspaceMode; + workspaceSize: string; + storageClass: string; + accessModes: string[]; + claimName: string; +}; + +export type KubernetesProfileValidationError = + | "main_container_required" + | "main_container_invalid" + | "pod_template_required" + | "pod_template_too_large" + | "workspace_size_required" + | "access_mode_required" + | "access_mode_invalid" + | "claim_name_required"; + +const MAX_POD_TEMPLATE_BYTES = 256 * 1024; +const KUBERNETES_CONTAINER_NAME = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/; + +// i18n-exempt: container image reference persisted in the Kubernetes PodTemplate. +export const DEFAULT_KUBERNETES_IMAGE = "ghcr.io/kdlbs/kandev:latest"; + +// i18n-exempt: default PodTemplate is persisted Kubernetes YAML, not user-facing copy. +export const DEFAULT_KUBERNETES_POD_TEMPLATE = `apiVersion: v1 +kind: PodTemplate +template: + spec: + containers: + - name: kandev-agent + image: ${DEFAULT_KUBERNETES_IMAGE} +`; + +export function createDefaultKubernetesExecutorForm(): KubernetesExecutorForm { + return { + name: "", + authMode: "kubeconfig", + kubeconfigPath: "", + kubeContext: "", + namespace: "default", + requestTimeoutSeconds: "30", + }; +} + +export function createDefaultKubernetesProfileConfig(): KubernetesProfileConfigForm { + return { + platform: "linux/amd64", + mainContainer: "kandev-agent", + podTemplateYaml: DEFAULT_KUBERNETES_POD_TEMPLATE, + workspaceMode: "managed_pvc", + workspaceSize: "10Gi", + storageClass: "", + accessModes: ["ReadWriteOnce"], + claimName: "", + }; +} + +export function parseKubernetesExecutorConfig( + name: string, + config?: Record, +): KubernetesExecutorForm { + const defaults = createDefaultKubernetesExecutorForm(); + const values = config ?? {}; + const authMode = values.auth_mode === "in_cluster" ? "in_cluster" : "kubeconfig"; + return { + name, + authMode, + kubeconfigPath: authMode === "kubeconfig" ? (values.kubeconfig_path ?? "").trim() : "", + kubeContext: authMode === "kubeconfig" ? (values.kube_context ?? "").trim() : "", + namespace: values.namespace?.trim() || defaults.namespace, + requestTimeoutSeconds: values.request_timeout_seconds?.trim() || defaults.requestTimeoutSeconds, + }; +} + +export function serializeKubernetesExecutorConfig( + form: KubernetesExecutorForm, +): Record { + const config: Record = { + auth_mode: form.authMode, + namespace: form.namespace.trim(), + request_timeout_seconds: form.requestTimeoutSeconds.trim(), + }; + if (form.authMode === "kubeconfig") { + config.kubeconfig_path = form.kubeconfigPath.trim(); + if (form.kubeContext.trim()) config.kube_context = form.kubeContext.trim(); + } + return config; +} + +export function parseKubernetesProfileConfig( + config?: Record, +): KubernetesProfileConfigForm { + const defaults = createDefaultKubernetesProfileConfig(); + const values = config ?? {}; + const workspaceMode = parseWorkspaceMode(configValue(values, "workspace.mode", "workspace_mode")); + const accessModes = parseAccessModes( + configValue(values, "workspace.access_modes", "workspace_access_modes"), + ); + const managedAccessModes = accessModes.length > 0 ? accessModes : defaults.accessModes; + return { + platform: values.platform === "linux/arm64" ? "linux/arm64" : "linux/amd64", + mainContainer: values.main_container?.trim() || defaults.mainContainer, + podTemplateYaml: values.pod_template_yaml ?? defaults.podTemplateYaml, + workspaceMode, + workspaceSize: + workspaceMode === "managed_pvc" + ? configValue(values, "workspace.size", "workspace_size").trim() || defaults.workspaceSize + : "", + storageClass: + workspaceMode === "managed_pvc" + ? configValue(values, "workspace.storage_class", "workspace_storage_class").trim() + : "", + accessModes: workspaceMode === "managed_pvc" ? managedAccessModes : [], + claimName: + workspaceMode === "existing_claim" + ? configValue(values, "workspace.claim_name", "workspace_claim_name").trim() + : "", + }; +} + +export function serializeKubernetesProfileConfig( + form: KubernetesProfileConfigForm, +): Record { + const config: Record = { + platform: form.platform, + main_container: form.mainContainer.trim(), + pod_template_yaml: form.podTemplateYaml, + "workspace.mode": form.workspaceMode, + }; + if (form.workspaceMode === "managed_pvc") { + config["workspace.size"] = form.workspaceSize.trim(); + if (form.storageClass.trim()) config["workspace.storage_class"] = form.storageClass.trim(); + const accessModes = form.accessModes.length > 0 ? form.accessModes : ["ReadWriteOnce"]; + config["workspace.access_modes"] = JSON.stringify(accessModes); + } + if (form.workspaceMode === "existing_claim") { + config["workspace.claim_name"] = form.claimName.trim(); + } + return config; +} + +export function replaceKubernetesProfileConfig( + baseConfig: Record, + form: KubernetesProfileConfigForm, +): Record { + const config = { ...baseConfig }; + for (const key of KUBERNETES_PROFILE_CONFIG_KEYS) delete config[key]; + return { ...config, ...serializeKubernetesProfileConfig(form) }; +} + +export function isKubernetesExecutorDirty( + form: KubernetesExecutorForm, + baseline: KubernetesExecutorForm, +): boolean { + return executorRevision(form) !== executorRevision(baseline); +} + +export function isKubernetesProfileDirty( + form: KubernetesProfileConfigForm, + baseline: KubernetesProfileConfigForm, +): boolean { + return ( + JSON.stringify(serializeKubernetesProfileConfig(form)) !== + JSON.stringify(serializeKubernetesProfileConfig(baseline)) + ); +} + +export function getKubernetesProfileValidationError( + form: KubernetesProfileConfigForm, +): KubernetesProfileValidationError | null { + if (!form.mainContainer.trim()) return "main_container_required"; + if (!KUBERNETES_CONTAINER_NAME.test(form.mainContainer.trim())) { + return "main_container_invalid"; + } + if (!form.podTemplateYaml.trim()) return "pod_template_required"; + if (new TextEncoder().encode(form.podTemplateYaml).byteLength > MAX_POD_TEMPLATE_BYTES) { + return "pod_template_too_large"; + } + if (form.workspaceMode === "managed_pvc" && !form.workspaceSize.trim()) { + return "workspace_size_required"; + } + if (form.workspaceMode === "managed_pvc" && form.accessModes.length === 0) { + return "access_mode_required"; + } + if ( + form.workspaceMode === "managed_pvc" && + form.accessModes.some((mode) => !ACCESS_MODES.has(mode as KubernetesAccessMode)) + ) { + return "access_mode_invalid"; + } + if (form.workspaceMode === "existing_claim" && !form.claimName.trim()) { + return "claim_name_required"; + } + return null; +} + +const ACCESS_MODES = new Set([ + "ReadWriteOnce", + "ReadOnlyMany", + "ReadWriteMany", + "ReadWriteOncePod", +]); + +const KUBERNETES_PROFILE_CONFIG_KEYS = [ + "platform", + "main_container", + "pod_template_yaml", + "workspace.mode", + "workspace.size", + "workspace.storage_class", + "workspace.access_modes", + "workspace.claim_name", + "workspace_mode", + "workspace_size", + "workspace_storage_class", + "workspace_access_modes", + "workspace_claim_name", +]; + +function parseWorkspaceMode(raw: string): KubernetesWorkspaceMode { + if (raw === "empty_dir" || raw === "existing_claim") return raw; + return "managed_pvc"; +} + +function parseAccessModes(raw: string): string[] { + if (!raw.trim()) return []; + try { + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed.filter((value): value is string => typeof value === "string"); + } catch { + return []; + } +} + +function configValue(config: Record, preferred: string, fallback: string): string { + return config[preferred] ?? config[fallback] ?? ""; +} + +function executorRevision(form: KubernetesExecutorForm): string { + return JSON.stringify({ + name: form.name.trim(), + config: serializeKubernetesExecutorConfig(form), + }); +} diff --git a/apps/web/components/settings/kubernetes-connection-card.tsx b/apps/web/components/settings/kubernetes-connection-card.tsx new file mode 100644 index 0000000000..a6e205f7cb --- /dev/null +++ b/apps/web/components/settings/kubernetes-connection-card.tsx @@ -0,0 +1,184 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { CardContent } from "@kandev/ui/card"; +import { Input } from "@kandev/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; +import { SettingsCard } from "./settings-card"; +import { SettingsCardHeader } from "./settings-card-header"; +import { SettingsField } from "./settings-field"; +import { settingsControlClassName, settingsCredentialClassName } from "./settings-control"; +import { isKubernetesExecutorDirty, type KubernetesExecutorForm } from "./kubernetes-config"; + +// i18n-exempt: example persisted Kubernetes namespace identifier. +const KUBERNETES_NAMESPACE_EXAMPLE = "agents"; +// i18n-exempt: example absolute filesystem path, rendered verbatim. +const KUBECONFIG_PATH_EXAMPLE = "/etc/kandev/kubeconfig"; + +type KubernetesConnectionCardProps = { + form: KubernetesExecutorForm; + baseline: KubernetesExecutorForm; + onChange: (form: KubernetesExecutorForm) => void; + readOnly?: boolean; +}; + +export function KubernetesConnectionCard({ + form, + baseline, + onChange, + readOnly = false, +}: KubernetesConnectionCardProps) { + const { t } = useTranslation(); + const update = ( + key: K, + value: KubernetesExecutorForm[K], + ) => onChange({ ...form, [key]: value }); + + return ( + + + +
+ + update("name", event.target.value)} + className={settingsControlClassName()} + /> + + + + update("namespace", event.target.value)} + placeholder={KUBERNETES_NAMESPACE_EXAMPLE} + className={settingsCredentialClassName()} + /> + + {form.authMode === "kubeconfig" && } + + update("requestTimeoutSeconds", event.target.value)} + className={settingsControlClassName()} + /> + +
+ + + ); +} + +type UpdateExecutorField = ( + key: K, + value: KubernetesExecutorForm[K], +) => void; + +function KubernetesAuthenticationField({ + form, + update, +}: { + form: KubernetesExecutorForm; + update: UpdateExecutorField; +}) { + const { t } = useTranslation(); + const helperKey = + form.authMode === "kubeconfig" + ? "executors:kubernetesKubeconfigAuthHelp" + : "executors:kubernetesInClusterAuthHelp"; + return ( + + + + ); +} + +function KubeconfigFields({ + form, + update, +}: { + form: KubernetesExecutorForm; + update: UpdateExecutorField; +}) { + const { t } = useTranslation(); + return ( + <> + + update("kubeconfigPath", event.target.value)} + placeholder={KUBECONFIG_PATH_EXAMPLE} + className={settingsCredentialClassName()} + /> + + + update("kubeContext", event.target.value)} + className={settingsCredentialClassName()} + /> + + + ); +} diff --git a/apps/web/components/settings/kubernetes-create-error.test.ts b/apps/web/components/settings/kubernetes-create-error.test.ts new file mode 100644 index 0000000000..fa99f3b0d4 --- /dev/null +++ b/apps/web/components/settings/kubernetes-create-error.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { KubernetesExecutorRollbackRejectedError } from "@/hooks/domains/settings/use-kubernetes-settings"; +import { kubernetesCreateErrorMessage } from "./kubernetes-create-error"; + +function translate(key: string, values?: Record): string { + if (key === "executors:kubernetesCreateRollbackFailed") { + return `${String(values?.profileError)} | ${String(values?.rollbackError)}`; + } + return key; +} + +describe("Kubernetes create error presentation", () => { + it("localizes a rejected compensating delete while retaining the profile error", () => { + const failure = new AggregateError([ + new Error("profile validation failed"), + new KubernetesExecutorRollbackRejectedError(), + ]); + + expect(kubernetesCreateErrorMessage(failure, translate)).toBe( + "profile validation failed | executors:kubernetesCreateRollbackRejected", + ); + }); + + it("uses a useful localized cause when rollback has no error detail", () => { + const failure = new AggregateError([new Error(), new Error()]); + + expect(kubernetesCreateErrorMessage(failure, translate)).toBe( + "executors:kubernetesCreateFailed | executors:kubernetesCreateRollbackUnknown", + ); + }); +}); diff --git a/apps/web/components/settings/kubernetes-create-error.ts b/apps/web/components/settings/kubernetes-create-error.ts new file mode 100644 index 0000000000..f157431775 --- /dev/null +++ b/apps/web/components/settings/kubernetes-create-error.ts @@ -0,0 +1,24 @@ +import { KubernetesExecutorRollbackRejectedError } from "@/hooks/domains/settings/use-kubernetes-settings"; + +type Translate = (key: string, values?: Record) => string; + +export function kubernetesCreateErrorMessage(error: unknown, t: Translate): string { + const createFallback = t("executors:kubernetesCreateFailed"); + if (!(error instanceof AggregateError)) return readableError(error, createFallback); + const errors = error.errors as unknown[]; + return t("executors:kubernetesCreateRollbackFailed", { + profileError: readableError(errors[0], createFallback), + rollbackError: rollbackErrorMessage(errors[1], t), + }); +} + +function rollbackErrorMessage(error: unknown, t: Translate): string { + if (error instanceof KubernetesExecutorRollbackRejectedError) { + return t("executors:kubernetesCreateRollbackRejected"); + } + return readableError(error, t("executors:kubernetesCreateRollbackUnknown")); +} + +function readableError(error: unknown, fallback: string): string { + return error instanceof Error && error.message ? error.message : fallback; +} diff --git a/apps/web/components/settings/kubernetes-diagnostics-card.tsx b/apps/web/components/settings/kubernetes-diagnostics-card.tsx new file mode 100644 index 0000000000..a96309b0e4 --- /dev/null +++ b/apps/web/components/settings/kubernetes-diagnostics-card.tsx @@ -0,0 +1,164 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { Badge } from "@kandev/ui/badge"; +import { Button } from "@kandev/ui/button"; +import { CardContent } from "@kandev/ui/card"; +import { IconAlertTriangle, IconCheck, IconLoader2, IconX } from "@tabler/icons-react"; +import { SettingsCard } from "./settings-card"; +import { SettingsCardHeader } from "./settings-card-header"; +import { settingsActionClassName } from "./settings-control"; +import type { + KubernetesTestResult, + KubernetesTestStep, + KubernetesWarning, +} from "@/lib/types/http-kubernetes"; + +const STEP_LABEL_KEYS: Record = { + configuration: "executors:kubernetesStepConfiguration", + discovery: "executors:kubernetesStepDiscovery", + namespace: "executors:kubernetesStepNamespace", + rbac: "executors:kubernetesStepRbac", + "admission.pvc": "executors:kubernetesStepPvcAdmission", + "storage.existing_claim": "executors:kubernetesStepExistingClaim", + "admission.pod": "executors:kubernetesStepPodAdmission", + streaming: "executors:kubernetesStepStreaming", +}; + +type KubernetesDiagnosticsCardProps = { + testing: boolean; + result: KubernetesTestResult | null; + error: unknown; + onTest: () => void; + canManage: boolean; + includesProfile?: boolean; +}; + +export function KubernetesDiagnosticsCard({ + testing, + result, + error, + onTest, + canManage, + includesProfile = false, +}: KubernetesDiagnosticsCardProps) { + const { t } = useTranslation(); + return ( + + + {testing ? : null} + {testing ? t("executors:kubernetesTesting") : t("executors:kubernetesTest")} + + } + /> + + {!canManage && ( +

{t("executors:kubernetesAdminTestOnly")}

+ )} + {Boolean(error) && ( +

+ {error instanceof Error && error.message + ? error.message + : t("executors:kubernetesTestRequestFailed")} +

+ )} + {result && } + + + ); +} + +function KubernetesDiagnosticResult({ result }: { result: KubernetesTestResult }) { + const { t } = useTranslation(); + return ( +
+
+ + {result.success + ? t("executors:kubernetesTestPassed") + : t("executors:kubernetesTestFailed")} + + {result.server_version && ( + + {result.server_version} + + )} + {result.namespace && ( + + {result.namespace} + + )} +
+
    + {result.steps.map((step, index) => ( + + ))} +
+ {result.warnings.length > 0 && } + {result.error &&

{result.error}

} +
+ ); +} + +function DiagnosticStep({ step }: { step: KubernetesTestStep }) { + const { t } = useTranslation(); + const labelKey = STEP_LABEL_KEYS[step.key]; + return ( +
  • + {step.success ? ( + + ) : ( + + )} +
    +
    + {labelKey ? t(labelKey) : step.key} + + {t("executors:kubernetesDurationMs", { duration: step.duration_ms })} + +
    +

    {step.detail}

    + {step.error &&

    {step.error}

    } +
    +
    • + ); +} + +function DiagnosticWarnings({ warnings }: { warnings: KubernetesWarning[] }) { + const { t } = useTranslation(); + return ( +
    +
    + + {t("executors:kubernetesWarnings")} +
    +
      + {warnings.map((warning, index) => ( +
    • + {warning.path} +

      {warning.message}

      +
      • + ))} +
    +
    + ); +} diff --git a/apps/web/components/settings/kubernetes-profile-cluster-section.test.tsx b/apps/web/components/settings/kubernetes-profile-cluster-section.test.tsx new file mode 100644 index 0000000000..b56696a4f1 --- /dev/null +++ b/apps/web/components/settings/kubernetes-profile-cluster-section.test.tsx @@ -0,0 +1,123 @@ +import { cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { Executor } from "@/lib/types/http"; +import { + createDefaultKubernetesProfileConfig, + parseKubernetesExecutorConfig, +} from "./kubernetes-config"; +import { KubernetesProfileClusterSection } from "./kubernetes-profile-cluster-section"; + +const mocks = vi.hoisted(() => ({ + runDiagnostics: vi.fn(async () => null), + refreshSessions: vi.fn(async () => []), + useSessions: vi.fn(), +})); + +vi.mock("@/hooks/domains/settings/use-kubernetes-settings", () => ({ + useKubernetesDiagnostics: () => ({ + testing: false, + result: null, + error: null, + run: mocks.runDiagnostics, + }), + useKubernetesSessions: (executorId: string) => { + mocks.useSessions(executorId); + return { + sessions: [], + loading: false, + error: null, + refresh: mocks.refreshSessions, + }; + }, +})); + +vi.mock("@/hooks/use-responsive-breakpoint", () => ({ + useResponsiveBreakpoint: () => ({ isMobile: false }), +})); + +const EXECUTOR = { + id: "executor/one", + name: "Remote production", + type: "k8s", + status: "active", + is_system: false, + config: { + auth_mode: "kubeconfig", + kube_context: "prod-eu", + namespace: "kandev-workloads", + request_timeout_seconds: "30", + }, + created_at: "2026-08-31T10:00:00Z", + updated_at: "2026-08-31T10:00:00Z", +} satisfies Executor; + +afterEach(() => { + cleanup(); + vi.clearAllMocks(); +}); + +describe("KubernetesProfileClusterSection", () => { + it("leads with the inline connection editor and tests both unsaved drafts", () => { + const form = { + ...createDefaultKubernetesProfileConfig(), + mainContainer: "draft-agent", + workspaceMode: "empty_dir" as const, + }; + const connection = { + ...parseKubernetesExecutorConfig(EXECUTOR.name, EXECUTOR.config), + namespace: "draft-namespace", + }; + const onConnectionChange = vi.fn(); + render( + , + ); + + expect(screen.getByRole("heading", { name: "Cluster connection" })).toBeTruthy(); + expect(screen.getByTestId("kubernetes-connection-card")).toBeTruthy(); + expect((screen.getByTestId("kubernetes-namespace") as HTMLInputElement).value).toBe( + "draft-namespace", + ); + expect(screen.getByTestId("kubernetes-diagnostics-card")).toBeTruthy(); + expect(screen.getByTestId("kubernetes-sessions-card")).toBeTruthy(); + expect(mocks.useSessions).toHaveBeenCalledWith("executor/one"); + + fireEvent.click(screen.getByRole("button", { name: "Test Kubernetes" })); + expect(mocks.runDiagnostics).toHaveBeenCalledWith({ + config: expect.objectContaining({ namespace: "draft-namespace" }), + profile_config: expect.objectContaining({ + main_container: "draft-agent", + "workspace.mode": "empty_dir", + }), + }); + expect(screen.queryByRole("button", { name: "Edit cluster connection" })).toBeNull(); + }); + + it("keeps connection access and sessions visible for members while disabling diagnostics", () => { + render( + , + ); + + const namespace = screen.getByTestId("kubernetes-namespace") as HTMLInputElement; + expect((namespace.closest("fieldset") as HTMLFieldSetElement).disabled).toBe(true); + expect(screen.queryByRole("button", { name: "View cluster connection" })).toBeNull(); + expect(screen.getByRole("button", { name: "Test Kubernetes" }).hasAttribute("disabled")).toBe( + true, + ); + expect(screen.getByTestId("kubernetes-sessions-card")).toBeTruthy(); + }); +}); diff --git a/apps/web/components/settings/kubernetes-profile-cluster-section.tsx b/apps/web/components/settings/kubernetes-profile-cluster-section.tsx new file mode 100644 index 0000000000..eab50a5f32 --- /dev/null +++ b/apps/web/components/settings/kubernetes-profile-cluster-section.tsx @@ -0,0 +1,75 @@ +"use client"; + +import { + useKubernetesDiagnostics, + useKubernetesSessions, +} from "@/hooks/domains/settings/use-kubernetes-settings"; +import type { Executor } from "@/lib/types/http"; +import { KubernetesConnectionCard } from "./kubernetes-connection-card"; +import { + serializeKubernetesExecutorConfig, + serializeKubernetesProfileConfig, + type KubernetesExecutorForm, + type KubernetesProfileConfigForm, +} from "./kubernetes-config"; +import { KubernetesDiagnosticsCard } from "./kubernetes-diagnostics-card"; +import { KubernetesSessionsCard } from "./kubernetes-sessions-card"; + +type KubernetesDiagnosticsState = ReturnType; +type KubernetesSessionsState = ReturnType; + +type KubernetesProfileClusterSectionProps = { + executor: Executor; + form: KubernetesProfileConfigForm; + connectionForm: KubernetesExecutorForm; + connectionBaseline: KubernetesExecutorForm; + onConnectionChange: (form: KubernetesExecutorForm) => void; + canManage: boolean; + diagnosticsState?: KubernetesDiagnosticsState; + sessionsState?: KubernetesSessionsState; +}; + +export function KubernetesProfileClusterSection({ + executor, + form, + connectionForm, + connectionBaseline, + onConnectionChange, + canManage, + diagnosticsState, + sessionsState, +}: KubernetesProfileClusterSectionProps) { + const localDiagnostics = useKubernetesDiagnostics(); + const localSessions = useKubernetesSessions(executor.id, sessionsState === undefined); + const diagnostics = diagnosticsState ?? localDiagnostics; + const sessions = sessionsState ?? localSessions; + + const runTest = () => { + void diagnostics + .run({ + config: serializeKubernetesExecutorConfig(connectionForm), + profile_config: serializeKubernetesProfileConfig(form), + }) + .catch(() => undefined); + }; + + return ( +
    + + + +
    + ); +} diff --git a/apps/web/components/settings/kubernetes-profile-sections.tsx b/apps/web/components/settings/kubernetes-profile-sections.tsx new file mode 100644 index 0000000000..8cb3a7e507 --- /dev/null +++ b/apps/web/components/settings/kubernetes-profile-sections.tsx @@ -0,0 +1,36 @@ +"use client"; + +import { KubernetesWorkloadCard } from "./kubernetes-workload-card"; +import { KubernetesWorkspaceCard } from "./kubernetes-workspace-card"; +import type { KubernetesProfileConfigForm } from "./kubernetes-config"; + +type KubernetesProfileSectionsProps = { + form: KubernetesProfileConfigForm; + baseline: KubernetesProfileConfigForm; + onChange: (form: KubernetesProfileConfigForm) => void; + canManage: boolean; +}; + +export function KubernetesProfileSections({ + form, + baseline, + onChange, + canManage, +}: KubernetesProfileSectionsProps) { + return ( + <> + + + + ); +} diff --git a/apps/web/components/settings/kubernetes-read-only-notice.tsx b/apps/web/components/settings/kubernetes-read-only-notice.tsx new file mode 100644 index 0000000000..d488965810 --- /dev/null +++ b/apps/web/components/settings/kubernetes-read-only-notice.tsx @@ -0,0 +1,16 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { Alert, AlertDescription, AlertTitle } from "@kandev/ui/alert"; +import { IconLock } from "@tabler/icons-react"; + +export function KubernetesReadOnlyNotice() { + const { t } = useTranslation(); + return ( + + + {t("executors:kubernetesReadOnlyTitle")} + {t("executors:kubernetesReadOnlyDescription")} + + ); +} diff --git a/apps/web/components/settings/kubernetes-save-confirmation.test.ts b/apps/web/components/settings/kubernetes-save-confirmation.test.ts new file mode 100644 index 0000000000..e1b3c92514 --- /dev/null +++ b/apps/web/components/settings/kubernetes-save-confirmation.test.ts @@ -0,0 +1,101 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { SettingsSaveCancelledError } from "./settings-save-provider"; +import { + requireKubernetesSessionSaveConfirmation, + saveWithKubernetesSessionConfirmation, +} from "./kubernetes-save-confirmation"; + +afterEach(() => vi.unstubAllGlobals()); + +describe("Kubernetes active-session save confirmation", () => { + it("does not prompt when no active sessions exist", () => { + const confirm = vi.fn(); + vi.stubGlobal("confirm", confirm); + + requireKubernetesSessionSaveConfirmation("connection", 0); + + expect(confirm).not.toHaveBeenCalled(); + }); + + it("explains that current connection settings apply on reconnect", () => { + const confirm = vi.fn(() => true); + vi.stubGlobal("confirm", confirm); + + requireKubernetesSessionSaveConfirmation("connection", 2); + + expect(confirm).toHaveBeenCalledWith(expect.stringMatching(/2 active sessions.*reconnect/i)); + }); + + it("explains that existing workload snapshots do not adopt profile edits", () => { + const confirm = vi.fn(() => true); + vi.stubGlobal("confirm", confirm); + + requireKubernetesSessionSaveConfirmation("workload", 1); + + expect(confirm).toHaveBeenCalledWith(expect.stringMatching(/recorded workload snapshot/i)); + }); + + it("explains both reconnect credentials and recorded workload snapshots once", () => { + const confirm = vi.fn(() => true); + vi.stubGlobal("confirm", confirm); + + requireKubernetesSessionSaveConfirmation("connection_and_workload", 2); + + expect(confirm).toHaveBeenCalledOnce(); + expect(confirm).toHaveBeenCalledWith( + expect.stringMatching(/connection.*reconnect.*workload snapshot/i), + ); + }); + + it("signals a silent coordinated-save cancellation when declined", () => { + vi.stubGlobal( + "confirm", + vi.fn(() => false), + ); + + expect(() => requireKubernetesSessionSaveConfirmation("connection", 1)).toThrow( + SettingsSaveCancelledError, + ); + }); + + it("refreshes session inventory before confirming and saving", async () => { + const events: string[] = []; + vi.stubGlobal( + "confirm", + vi.fn(() => { + events.push("confirm"); + return true; + }), + ); + + await saveWithKubernetesSessionConfirmation({ + kind: "connection", + loadActiveSessionCount: async () => { + events.push("refresh"); + return 1; + }, + save: async () => { + events.push("save"); + }, + }); + + expect(events).toEqual(["refresh", "confirm", "save"]); + }); + + it("does not save when the active-session confirmation is declined", async () => { + vi.stubGlobal( + "confirm", + vi.fn(() => false), + ); + const save = vi.fn(); + + await expect( + saveWithKubernetesSessionConfirmation({ + kind: "workload", + loadActiveSessionCount: async () => 2, + save, + }), + ).rejects.toBeInstanceOf(SettingsSaveCancelledError); + expect(save).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/components/settings/kubernetes-save-confirmation.ts b/apps/web/components/settings/kubernetes-save-confirmation.ts new file mode 100644 index 0000000000..061c0e097c --- /dev/null +++ b/apps/web/components/settings/kubernetes-save-confirmation.ts @@ -0,0 +1,37 @@ +"use client"; + +import { t as translate } from "@/lib/i18n"; +import { SettingsSaveCancelledError } from "./settings-save-provider"; + +export type KubernetesSessionSaveKind = "connection" | "workload" | "connection_and_workload"; + +const CONFIRMATION_KEYS: Record = { + connection: "executors:kubernetesConfirmConnectionSessions", + workload: "executors:kubernetesConfirmWorkloadSessions", + connection_and_workload: "executors:kubernetesConfirmCombinedSessions", +}; + +export function requireKubernetesSessionSaveConfirmation( + kind: KubernetesSessionSaveKind, + activeSessionCount: number, +): void { + if (activeSessionCount <= 0) return; + const key = CONFIRMATION_KEYS[kind]; + if (!window.confirm(translate(key, { count: activeSessionCount }))) { + throw new SettingsSaveCancelledError(); + } +} + +export async function saveWithKubernetesSessionConfirmation({ + kind, + loadActiveSessionCount, + save, +}: { + kind: KubernetesSessionSaveKind; + loadActiveSessionCount: () => Promise; + save: () => Promise; +}): Promise { + const activeSessionCount = await loadActiveSessionCount(); + requireKubernetesSessionSaveConfirmation(kind, activeSessionCount); + await save(); +} diff --git a/apps/web/components/settings/kubernetes-sessions-card.tsx b/apps/web/components/settings/kubernetes-sessions-card.tsx new file mode 100644 index 0000000000..c4f4995cb9 --- /dev/null +++ b/apps/web/components/settings/kubernetes-sessions-card.tsx @@ -0,0 +1,231 @@ +"use client"; + +import { useTranslation } from "react-i18next"; +import { Badge } from "@kandev/ui/badge"; +import { Button } from "@kandev/ui/button"; +import { CardContent } from "@kandev/ui/card"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@kandev/ui/table"; +import { IconLoader2 } from "@tabler/icons-react"; +import { SettingsCard } from "./settings-card"; +import { SettingsCardHeader } from "./settings-card-header"; +import { settingsActionClassName } from "./settings-control"; +import { useResponsiveBreakpoint } from "@/hooks/use-responsive-breakpoint"; +import type { useKubernetesSessions } from "@/hooks/domains/settings/use-kubernetes-settings"; +import { formatDateTime } from "@/lib/i18n/formats"; +import { i18n, t as translate } from "@/lib/i18n"; +import type { KubernetesSession } from "@/lib/types/http-kubernetes"; + +const STATUS_LABEL_KEYS: Record = { + running: "executors:kubernetesStatusRunning", + waiting: "executors:kubernetesStatusWaiting", + terminated: "executors:kubernetesStatusTerminated", + unknown: "executors:kubernetesStatusUnknown", + pending: "executors:kubernetesStatusPending", + succeeded: "executors:kubernetesStatusSucceeded", + failed: "executors:kubernetesStatusFailed", +}; + +type KubernetesSessionsState = ReturnType; + +export function KubernetesSessionsCard({ state }: { state: KubernetesSessionsState }) { + const { t } = useTranslation(); + const { isMobile } = useResponsiveBreakpoint(); + const errorMessage = + state.error instanceof Error && state.error.message + ? state.error.message + : t("executors:kubernetesSessionsFailed"); + return ( + + void state.refresh().catch(() => undefined)} + disabled={state.loading} + className={settingsActionClassName("w-full cursor-pointer md:w-auto")} + > + {state.loading ? : null} + {t("executors:refresh")} + + } + /> + + {Boolean(state.error) && ( +

    {errorMessage}

    + )} + {!state.error && !state.loading && state.sessions.length === 0 && ( +

    + {t("executors:kubernetesNoActiveSessions")} +

    + )} + {!state.error && + state.sessions.length > 0 && + (isMobile ? ( + + ) : ( + + ))} +     +     + ); +} + +function MobileSessionList({ sessions }: { sessions: KubernetesSession[] }) { + return ( +
    + {sessions.map((session) => ( +
    + + + +
    + ))} +
    + ); +} + +function DesktopSessionTable({ sessions }: { sessions: KubernetesSession[] }) { + const { t } = useTranslation(); + return ( +
    + + + + {t("executors:task")} + {t("executors:session")} + {t("executors:kubernetesPod")} + {t("executors:status")} + {t("executors:kubernetesWorkspace")} + {t("executors:kubernetesCreated")} + + + + {sessions.map((session) => ( + + {shortId(session.task_id)} + {shortId(session.session_id)} + + {session.pod_name || "-"} + + +
    + + +
    +     + {workspaceLabel(session.workspace_kind)} + + {formatCreatedAt(session.created_at)} + +     + ))} +     +
    +
    + ); +} + +function SessionIdentity({ session }: { session: KubernetesSession }) { + const { t } = useTranslation(); + return ( +
    +
    +

    {t("executors:kubernetesPod")}

    +

    {session.pod_name || "-"}

    +
    +
    + + +
    +
    + ); +} + +function SessionIdentityValue({ label, value }: { label: string; value: string }) { + return ( +
    +

    {label}

    +

    {value}

    +
    + ); +} + +function SessionStatus({ session }: { session: KubernetesSession }) { + const status = + session.container_state?.toLowerCase() === "terminated" && + session.pod_phase?.toLowerCase() === "succeeded" + ? session.pod_phase + : session.container_state || session.pod_phase || "unknown"; + const running = status.toLowerCase() === "running"; + return ( +
    + {statusLabel(status)} + {session.restarts > 0 && } +
    + ); +} + +function RestartCount({ count }: { count: number }) { + const { t } = useTranslation(); + return ( + + {t("executors:kubernetesRestarts", { count })} + + ); +} + +function SessionDetails({ session }: { session: KubernetesSession }) { + const { t } = useTranslation(); + return ( +
    +

    + {t("executors:kubernetesWorkspaceValue", { + value: workspaceLabel(session.workspace_kind), + })} +

    +

    {t("executors:kubernetesCreatedValue", { value: formatCreatedAt(session.created_at) })}

    + +
    + ); +} + +function SessionFailureReason({ session }: { session: KubernetesSession }) { + return session.failure_reason ? ( +

    {session.failure_reason}

    + ) : null; +} + +function workspaceLabel(value?: string): string { + const keys: Record = { + managed_pvc: "executors:kubernetesWorkspaceManagedPvc", + empty_dir: "executors:kubernetesWorkspaceEmptyDir", + existing_claim: "executors:kubernetesWorkspaceExistingClaim", + }; + return value && keys[value] ? translationLabel(keys[value]) : value || "-"; +} + +function statusLabel(value: string): string { + const key = STATUS_LABEL_KEYS[value.toLowerCase()]; + return key ? translationLabel(key, value) : value; +} + +function translationLabel(key: string, fallback?: string): string { + return i18n.exists(key) ? translate(key) : (fallback ?? key); +} + +function shortId(value: string): string { + return value.slice(0, 8); +} + +function formatCreatedAt(value?: string): string { + if (!value) return "-"; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? value : formatDateTime(date); +} diff --git a/apps/web/components/settings/kubernetes-settings-cards.test.tsx b/apps/web/components/settings/kubernetes-settings-cards.test.tsx new file mode 100644 index 0000000000..de9aec0041 --- /dev/null +++ b/apps/web/components/settings/kubernetes-settings-cards.test.tsx @@ -0,0 +1,163 @@ +import { cleanup, render, screen, within } from "@testing-library/react"; +import type { ComponentProps } from "react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { KubernetesConnectionCard } from "./kubernetes-connection-card"; +import { KubernetesDiagnosticsCard } from "./kubernetes-diagnostics-card"; +import { KubernetesSessionsCard } from "./kubernetes-sessions-card"; +import { KubernetesWorkspaceCard } from "./kubernetes-workspace-card"; +import { + createDefaultKubernetesExecutorForm, + createDefaultKubernetesProfileConfig, +} from "./kubernetes-config"; + +const responsive = vi.hoisted(() => ({ isMobile: false })); + +vi.mock("@/hooks/use-responsive-breakpoint", () => ({ + useResponsiveBreakpoint: () => ({ isMobile: responsive.isMobile }), +})); + +afterEach(cleanup); + +describe("Kubernetes settings cards", () => { + it("hides kubeconfig-only fields for in-cluster auth and disables member edits", () => { + const form = { ...createDefaultKubernetesExecutorForm(), authMode: "in_cluster" as const }; + + render(); + + expect(screen.queryByLabelText("Kubeconfig path")).toBeNull(); + expect( + (screen.getByLabelText("Fixed namespace").closest("fieldset") as HTMLFieldSetElement) + .disabled, + ).toBe(true); + }); + + it("shows only fields belonging to the selected workspace mode", () => { + const baseline = createDefaultKubernetesProfileConfig(); + const { rerender } = render( + , + ); + + expect(screen.getByTestId("kubernetes-empty-dir-warning")).toBeTruthy(); + expect(screen.queryByLabelText("Claim size")).toBeNull(); + rerender( + , + ); + expect(screen.getByLabelText("Existing claim name")).toBeTruthy(); + expect(screen.queryByLabelText("Claim size")).toBeNull(); + }); + + it("keeps member testing visible and disabled with an explanation", () => { + render( + , + ); + + expect(screen.getByRole("button", { name: "Test Kubernetes" }).hasAttribute("disabled")).toBe( + true, + ); + expect(screen.getByText("Only an administrator can run this Kubernetes test.")).toBeTruthy(); + }); + + it("renders causal diagnostic steps and PodTemplate warnings", () => { + render( + , + ); + + expect(screen.getByText("Required permissions")).toBeTruthy(); + expect(screen.getByText("Denied permission")).toBeTruthy(); + expect(screen.getByText("template.spec")).toBeTruthy(); + expect(screen.getByText("Privileged container")).toBeTruthy(); + }); + + it("shows complete task and session identities on mobile session cards", () => { + responsive.isMobile = true; + render(); + + const card = screen.getByTestId("kubernetes-mobile-session-list"); + expect(within(card).getByText("task-123456789")).toBeTruthy(); + expect(within(card).getByText("session-987654321")).toBeTruthy(); + expect(within(card).getByText("Task")).toBeTruthy(); + expect(within(card).getByText("Session")).toBeTruthy(); + }); +}); + +describe("Kubernetes session status cards", () => { + it("shows the sanitized failure reason beneath desktop session status", () => { + responsive.isMobile = false; + render(); + + const row = screen.getByText("task-123").closest("tr"); + expect(row).not.toBeNull(); + const statusCell = within(row as HTMLTableRowElement).getByTestId("kubernetes-session-status"); + expect(within(statusCell).getByText("pods is forbidden: RBAC denied")).toBeTruthy(); + }); + + it("labels a successfully completed Pod as succeeded instead of terminated", () => { + responsive.isMobile = false; + const state = sessionsState(); + state.sessions[0] = { + ...state.sessions[0], + pod_phase: "Succeeded", + container_state: "terminated", + failure_reason: undefined, + }; + + render(); + + expect(screen.getByText("Succeeded")).toBeTruthy(); + expect(screen.queryByText("Terminated")).toBeNull(); + }); +}); + +type KubernetesSessionsState = ComponentProps["state"]; + +function sessionsState(): KubernetesSessionsState { + return { + sessions: [ + { + task_id: "task-123456789", + session_id: "session-987654321", + pod_name: "kandev-session-1", + pod_phase: "Failed", + container_state: "terminated", + restarts: 2, + workspace_kind: "managed_pvc", + created_at: "2026-08-24T10:00:00Z", + failure_reason: "pods is forbidden: RBAC denied", + }, + ], + loading: false, + error: null, + refresh: vi.fn(async () => []), + }; +} diff --git a/apps/web/components/settings/kubernetes-validation.test.ts b/apps/web/components/settings/kubernetes-validation.test.ts new file mode 100644 index 0000000000..459994a8da --- /dev/null +++ b/apps/web/components/settings/kubernetes-validation.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { createDefaultKubernetesProfileConfig } from "./kubernetes-config"; +import { + getKubernetesCreateContributorState, + kubernetesExecutorInvalidReason, + kubernetesProfileInvalidReason, +} from "./kubernetes-validation"; + +describe("kubernetesProfileInvalidReason", () => { + const translate = (key: string) => `translated:${key}`; + + it("maps pure validation errors to localized explanations", () => { + const valid = createDefaultKubernetesProfileConfig(); + + expect(kubernetesProfileInvalidReason(valid, translate)).toBeUndefined(); + expect(kubernetesProfileInvalidReason({ ...valid, mainContainer: "" }, translate)).toBe( + "translated:executors:kubernetesMainContainerRequired", + ); + expect( + kubernetesProfileInvalidReason( + { ...valid, podTemplateYaml: "x".repeat(256 * 1024 + 1) }, + translate, + ), + ).toBe("translated:executors:kubernetesPodTemplateTooLarge"); + }); +}); + +describe("kubernetesExecutorInvalidReason", () => { + const translate = (key: string) => `translated:${key}`; + const valid = { + name: "Cluster", + authMode: "kubeconfig" as const, + kubeconfigPath: "/etc/kandev/config", + kubeContext: "", + namespace: "agents", + requestTimeoutSeconds: "30", + }; + + it.each(["1e2", "1.0", "+30", " 30 "])("rejects non-canonical timeout %s", (timeout) => { + expect( + kubernetesExecutorInvalidReason( + { ...valid, requestTimeoutSeconds: timeout }, + true, + translate, + ), + ).toBe("translated:executors:kubernetesTimeoutInvalid"); + }); + + it("rejects a relative kubeconfig path before save", () => { + expect( + kubernetesExecutorInvalidReason( + { ...valid, kubeconfigPath: "configs/cluster.yaml" }, + true, + translate, + ), + ).toBe("translated:executors:kubernetesKubeconfigPathRequired"); + }); +}); + +describe("Kubernetes create save participation", () => { + it("never marks a read-only member route dirty or saveable", () => { + expect(getKubernetesCreateContributorState(false, "Administrator required")).toEqual({ + isDirty: false, + canSave: false, + }); + }); + + it("lets an administrator participate when the form is valid", () => { + expect(getKubernetesCreateContributorState(true, undefined)).toEqual({ + isDirty: true, + canSave: true, + }); + expect(getKubernetesCreateContributorState(true, "Invalid form").canSave).toBe(false); + }); +}); diff --git a/apps/web/components/settings/kubernetes-validation.ts b/apps/web/components/settings/kubernetes-validation.ts new file mode 100644 index 0000000000..efe33d57cd --- /dev/null +++ b/apps/web/components/settings/kubernetes-validation.ts @@ -0,0 +1,54 @@ +import { + getKubernetesProfileValidationError, + type KubernetesExecutorForm, + type KubernetesProfileConfigForm, + type KubernetesProfileValidationError, +} from "./kubernetes-config"; + +export function kubernetesExecutorInvalidReason( + form: KubernetesExecutorForm, + canManage: boolean, + t: (key: string) => string, +): string | undefined { + if (!canManage) return t("executors:kubernetesAdminSaveOnly"); + if (!form.name.trim()) return t("executors:kubernetesExecutorNameRequired"); + if (!form.namespace.trim()) return t("executors:kubernetesNamespaceRequired"); + if (form.authMode === "kubeconfig" && !form.kubeconfigPath.trim().startsWith("/")) { + return t("executors:kubernetesKubeconfigPathRequired"); + } + const timeout = Number(form.requestTimeoutSeconds); + return /^(?:[1-9]|[1-9][0-9]|[12][0-9]{2}|300)$/.test(form.requestTimeoutSeconds) && + timeout >= 1 && + timeout <= 300 + ? undefined + : t("executors:kubernetesTimeoutInvalid"); +} + +const PROFILE_ERROR_KEYS: Record = { + main_container_required: "executors:kubernetesMainContainerRequired", + main_container_invalid: "executors:kubernetesMainContainerInvalid", + pod_template_required: "executors:kubernetesPodTemplateRequired", + pod_template_too_large: "executors:kubernetesPodTemplateTooLarge", + workspace_size_required: "executors:kubernetesWorkspaceSizeRequired", + access_mode_required: "executors:kubernetesAccessModeRequired", + access_mode_invalid: "executors:kubernetesAccessModeInvalid", + claim_name_required: "executors:kubernetesClaimNameRequired", +}; + +export function kubernetesProfileInvalidReason( + form: KubernetesProfileConfigForm, + t: (key: string) => string, +): string | undefined { + const error = getKubernetesProfileValidationError(form); + return error ? t(PROFILE_ERROR_KEYS[error]) : undefined; +} + +export function getKubernetesCreateContributorState( + canManage: boolean, + invalidReason?: string, +): { isDirty: boolean; canSave: boolean } { + return { + isDirty: canManage, + canSave: canManage && !invalidReason, + }; +} diff --git a/apps/web/components/settings/kubernetes-workload-card.test.tsx b/apps/web/components/settings/kubernetes-workload-card.test.tsx new file mode 100644 index 0000000000..bc9d0e8b92 --- /dev/null +++ b/apps/web/components/settings/kubernetes-workload-card.test.tsx @@ -0,0 +1,54 @@ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { createDefaultKubernetesProfileConfig } from "./kubernetes-config"; +import { KubernetesWorkloadCard } from "./kubernetes-workload-card"; + +const originalScrollHeight = Object.getOwnPropertyDescriptor( + HTMLTextAreaElement.prototype, + "scrollHeight", +); + +beforeEach(() => { + Object.defineProperty(HTMLTextAreaElement.prototype, "scrollHeight", { + configurable: true, + get() { + return this.value.includes("long-image-reference") ? 480 : 96; + }, + }); +}); + +afterEach(() => { + cleanup(); + if (originalScrollHeight) { + Object.defineProperty(HTMLTextAreaElement.prototype, "scrollHeight", originalScrollHeight); + } +}); + +describe("KubernetesWorkloadCard PodTemplate sizing", () => { + it("grows and shrinks with controlled YAML while keeping horizontal-only overflow", () => { + const baseline = createDefaultKubernetesProfileConfig(); + const shortForm = { ...baseline, podTemplateYaml: "kind: PodTemplate" }; + const { rerender } = render( + , + ); + const textarea = screen.getByTestId("kubernetes-pod-template") as HTMLTextAreaElement; + + expect(textarea.style.height).toBe("96px"); + expect(textarea.className).toContain("overflow-y-hidden"); + expect(textarea.className).toContain("overflow-x-auto"); + expect(textarea.className).toContain("resize-none"); + + rerender( + , + ); + expect(textarea.style.height).toBe("480px"); + + rerender(); + expect(textarea.style.height).toBe("96px"); + }); +}); diff --git a/apps/web/components/settings/kubernetes-workload-card.tsx b/apps/web/components/settings/kubernetes-workload-card.tsx new file mode 100644 index 0000000000..53d10f77a0 --- /dev/null +++ b/apps/web/components/settings/kubernetes-workload-card.tsx @@ -0,0 +1,128 @@ +"use client"; + +import { useLayoutEffect, useRef } from "react"; +import { useTranslation } from "react-i18next"; +import { CardContent } from "@kandev/ui/card"; +import { Input } from "@kandev/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@kandev/ui/select"; +import { Textarea } from "@kandev/ui/textarea"; +import { SettingsCard } from "./settings-card"; +import { SettingsCardHeader } from "./settings-card-header"; +import { SettingsField } from "./settings-field"; +import { settingsControlClassName, settingsCredentialClassName } from "./settings-control"; +import { + DEFAULT_KUBERNETES_IMAGE, + isKubernetesProfileDirty, + type KubernetesPlatform, + type KubernetesProfileConfigForm, +} from "./kubernetes-config"; + +// i18n-exempt: persisted Kubernetes platform identifiers shown verbatim. +const PLATFORM_OPTIONS: KubernetesPlatform[] = ["linux/amd64", "linux/arm64"]; + +type KubernetesWorkloadCardProps = { + form: KubernetesProfileConfigForm; + baseline: KubernetesProfileConfigForm; + onChange: (form: KubernetesProfileConfigForm) => void; + readOnly?: boolean; +}; + +export function KubernetesWorkloadCard({ + form, + baseline, + onChange, + readOnly = false, +}: KubernetesWorkloadCardProps) { + const { t } = useTranslation(); + const podTemplateRef = useRef(null); + const update = ( + key: K, + value: KubernetesProfileConfigForm[K], + ) => onChange({ ...form, [key]: value }); + + useLayoutEffect(() => { + const textarea = podTemplateRef.current; + if (!textarea) return; + textarea.style.height = "auto"; + textarea.style.height = `${textarea.scrollHeight}px`; + }, [form.podTemplateYaml]); + + return ( + + + +
    + + + + + update("mainContainer", event.target.value)} + className={settingsCredentialClassName()} + /> + + +