From 54e2e21f320601dc2e0702a03ec3dc499a98411c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Almeida?= <57680069+zeval@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:33:02 +0200 Subject: [PATCH] fix(kubernetes): make Docker worker lifecycle reliable (#3676) --- .../runtime/lifecycle/executor_kubernetes.go | 5 +- .../executor_kubernetes_bootstrap.go | 80 ++++++++++++++++ .../executor_kubernetes_bootstrap_test.go | 93 ++++++++++++++++++- .../executor_kubernetes_fakes_test.go | 8 +- .../lifecycle/executor_kubernetes_test.go | 4 +- .../plans/kubernetes-docker-workloads/plan.md | 39 +++++--- .../task-02-full-worker-image.md | 42 ++++----- .../task-03-docker-lifecycle.md | 37 ++++---- .../kubernetes-docker-workloads.md | 14 ++- .../kubernetes-startup-timing.md | 5 +- docs/specs/kubernetes-executor/spec.md | 5 +- k8s/worker-images/full/README.md | 28 ++++-- k8s/worker-images/full/build.sh | 2 +- k8s/worker-images/full/build_test.py | 7 ++ k8s/worker-images/full/daemon_test.py | 7 ++ k8s/worker-images/full/pod-template.yaml | 2 +- 16 files changed, 297 insertions(+), 81 deletions(-) diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go index 234a5dcb04..058ebd29aa 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go @@ -814,10 +814,7 @@ set -a . /opt/kandev/runtime.env . /run/kandev/auth.env set +a -if [ ! -f /opt/kandev/prepared ]; then - sh /opt/kandev/prepare.sh - : > /opt/kandev/prepared -fi +[ -f /opt/kandev/prepared ] || { echo 'kubernetes preparation marker is missing' >&2; exit 70; } if [ "${` + selectedCheckoutMarker + `:-}" = "1" ]; then ` + selectedCheckoutCredentialScrubCommands + ` rm -f /run/kandev/auth.env 2>/dev/null || true diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go index 43a8604ba7..fa308f6465 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go @@ -11,10 +11,14 @@ import ( "sort" "strconv" "strings" + "sync" + "unicode/utf8" corev1 "k8s.io/api/core/v1" kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes" + "github.com/kandev/kandev/internal/agent/runtime/routingerr" + "github.com/kandev/kandev/internal/common/constants" "github.com/kandev/kandev/internal/githubauth" "github.com/kandev/kandev/internal/scriptengine" ) @@ -63,6 +67,9 @@ func (r *KubernetesExecutor) bootstrapPod( if err := r.materializeKubernetesRemoteFiles(ctx, runtime, req, pod, profile.MainContainer); err != nil { return err } + if err := runKubernetesPrepareScript(ctx, runtime.streams, pod, profile.MainContainer); err != nil { + return err + } if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer, kubernetesStartPath, nil, 0o600); err != nil { return fmt.Errorf("kubernetes lifecycle: signal bootstrap: %w", err) @@ -70,6 +77,79 @@ func (r *KubernetesExecutor) bootstrapPod( return nil } +type kubernetesPrepareOutput struct { + mu sync.Mutex + data []byte + overflow bool +} + +const kubernetesPrepareCaptureBytes = 256 * 1024 + +func (w *kubernetesPrepareOutput) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.overflow { + return len(p), nil + } + if len(p) > kubernetesPrepareCaptureBytes-len(w.data) { + w.data = nil + w.overflow = true + return len(p), nil + } + w.data = append(w.data, p...) + return len(p), nil +} + +func (w *kubernetesPrepareOutput) diagnostic() string { + w.mu.Lock() + if w.overflow { + w.mu.Unlock() + return "" + } + raw := string(w.data) + w.mu.Unlock() + + diagnostic := routingerr.SanitizeFullUnbounded(raw) + if len(diagnostic) > routingerr.MaxRawExcerptBytes { + start := len(diagnostic) - routingerr.MaxRawExcerptBytes + for start < len(diagnostic) && !utf8.RuneStart(diagnostic[start]) { + start++ + } + diagnostic = diagnostic[start:] + } + return strings.TrimSpace(diagnostic) +} + +func runKubernetesPrepareScript( + ctx context.Context, + streams *kubeexecutor.StreamOperations, + pod *corev1.Pod, + container string, +) error { + prepareCtx, cancel := context.WithTimeout(preparationContext(ctx), constants.SetupScriptTimeout) + defer cancel() + output := &kubernetesPrepareOutput{} + command := `set -eu +set -a +. /opt/kandev/runtime.env +. /run/kandev/auth.env +set +a +sh /opt/kandev/prepare.sh +: > /opt/kandev/prepared` + err := streams.Exec(prepareCtx, kubeexecutor.ExecRequest{ + Namespace: pod.Namespace, Pod: pod.Name, Container: container, + Command: []string{"sh", "-c", command}, Stdout: output, Stderr: output, + }) + if err == nil { + return nil + } + diagnostic := output.diagnostic() + if diagnostic == "" { + return fmt.Errorf("kubernetes lifecycle: prepare script failed: %w", err) + } + return fmt.Errorf("kubernetes lifecycle: prepare script failed: %s: %w", diagnostic, err) +} + func kubernetesRuntimeEnvironment(req *ExecutorCreateRequest) map[string]string { values := map[string]string{ "AGENTCTL_LISTEN_HOST": kubernetesControlHost, diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go index a61180592b..f015519b8d 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go @@ -2,6 +2,7 @@ package lifecycle import ( "context" + "errors" "os" "os/exec" "path/filepath" @@ -36,6 +37,96 @@ func TestKubernetesWriteFileDoesNotOpenStdinForEmptySentinel(t *testing.T) { require.Contains(t, command, "mv -f \"$temporary\" \"$destination\"") } +func TestKubernetesBootstrapRunsPrepareBeforeSignallingEntrypoint(t *testing.T) { + execs := &recordingKubernetesExec{} + streams := kubeexecutor.NewStreamOperations(execs, nil) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "kandev-agents", Name: "agent-pod"}} + executor := NewKubernetesExecutor(nil, newTestLogger()) + req := validKubernetesCreateRequest() + + err := executor.bootstrapPod( + context.Background(), + &kubernetesRuntimeClient{streams: streams}, + req, + pod, + kubeexecutor.ProfileConfig{MainContainer: "kandev-agent"}, + "nonce", + []byte("agentctl"), + ) + + require.NoError(t, err) + prepareIndex, signalIndex := -1, -1 + for index, recorded := range execs.requests { + command := strings.Join(recorded.request.Command, " ") + if strings.Contains(command, "sh "+kubernetesPreparePath) { + prepareIndex = index + } + if strings.Contains(command, kubernetesStartPath) && recorded.request.Stdin == nil { + signalIndex = index + } + } + require.NotEqual(t, -1, prepareIndex) + require.NotEqual(t, -1, signalIndex) + require.Less(t, prepareIndex, signalIndex) + require.NotContains(t, kubernetesBootstrapCommand(), "sh "+kubernetesPreparePath) +} + +func TestKubernetesBootstrapReturnsSanitizedPrepareFailure(t *testing.T) { + execs := &recordingKubernetesExec{} + execs.execFunc = func(_ context.Context, request kubeexecutor.ExecRequest) error { + if !strings.Contains(strings.Join(request.Command, " "), "sh "+kubernetesPreparePath) { + return nil + } + _, _ = request.Stderr.Write([]byte( + "token=do-not-persist\nDocker daemon did not become ready before preparation deadline\n", + )) + return errors.New("remote command exited with status 1") + } + streams := kubeexecutor.NewStreamOperations(execs, nil) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "kandev-agents", Name: "agent-pod"}} + executor := NewKubernetesExecutor(nil, newTestLogger()) + + err := executor.bootstrapPod( + context.Background(), + &kubernetesRuntimeClient{streams: streams}, + validKubernetesCreateRequest(), + pod, + kubeexecutor.ProfileConfig{MainContainer: "kandev-agent"}, + "nonce", + []byte("agentctl"), + ) + + require.ErrorContains(t, err, "Docker daemon did not become ready before preparation deadline") + require.NotContains(t, err.Error(), "do-not-persist") + for _, recorded := range execs.requests { + command := strings.Join(recorded.request.Command, " ") + require.False(t, + strings.Contains(command, kubernetesStartPath) && recorded.request.Stdin == nil, + "failed preparation must not signal the managed entrypoint", + ) + } +} + +func TestKubernetesPrepareOutputDropsOversizeDiagnosticsBeforeRedaction(t *testing.T) { + output := &kubernetesPrepareOutput{} + secret := strings.Repeat("Zx.", 100_000) + _, err := output.Write([]byte("token=" + secret + "\nlate failure")) + + require.NoError(t, err) + require.Empty(t, output.diagnostic(), + "oversize raw output must be dropped instead of cutting away a credential marker") +} + +func TestKubernetesPrepareOutputKeepsLatestSanitizedDiagnostic(t *testing.T) { + output := &kubernetesPrepareOutput{} + _, err := output.Write([]byte(strings.Repeat("ordinary setup output\n", 400) + + "token=do-not-persist\nlate failure")) + + require.NoError(t, err) + require.Contains(t, output.diagnostic(), "late failure") + require.NotContains(t, output.diagnostic(), "do-not-persist") +} + func TestKubernetesWriteFileCommandRejectsSymlinkedAncestor(t *testing.T) { root := t.TempDir() outside := t.TempDir() @@ -153,8 +244,8 @@ func TestKubernetesBootstrapPublishesManagedCredentialHelperPath(t *testing.T) { ) require.NotContains(t, authData, "/tmp/hostile-agentctl") command := kubernetesBootstrapCommand() - require.Less(t, strings.Index(command, ". "+kubernetesAuthEnvPath), strings.Index(command, "sh "+kubernetesPreparePath)) require.Less(t, strings.Index(command, ". "+kubernetesAuthEnvPath), strings.Index(command, "exec "+kubernetesAgentctlPath)) + require.NotContains(t, command, "sh "+kubernetesPreparePath) } func TestKubernetesPrepareScriptUsesManagedAgentctlPathWithoutStartingSecondProcess(t *testing.T) { diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go index 58b94ffe03..12e5d096f6 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go @@ -351,9 +351,10 @@ type recordingKubernetesExec struct { mu sync.Mutex requests []recordedKubernetesExec err error + execFunc func(context.Context, kubeexecutor.ExecRequest) error } -func (r *recordingKubernetesExec) Exec(_ context.Context, request kubeexecutor.ExecRequest) error { +func (r *recordingKubernetesExec) Exec(ctx context.Context, request kubeexecutor.ExecRequest) error { var data []byte if request.Stdin != nil { var err error @@ -363,8 +364,11 @@ func (r *recordingKubernetesExec) Exec(_ context.Context, request kubeexecutor.E } } r.mu.Lock() - defer r.mu.Unlock() r.requests = append(r.requests, recordedKubernetesExec{request: request, stdin: data}) + r.mu.Unlock() + if r.execFunc != nil { + return r.execFunc(ctx, request) + } return r.err } diff --git a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go index 900c216ae8..8dfd147074 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go @@ -148,7 +148,7 @@ func TestKubernetesCreateInstanceProvisionsBootstrapsAndForwardsAgentctl(t *test require.NotNil(t, instance.Client) require.Len(t, resources.createdPVCs, 1) require.Len(t, resources.createdPods, 1) - require.Len(t, execs.requests, 4, "binary, runtime config, auth config, then start signal") + require.Len(t, execs.requests, 5, "binary, runtime config, auth config, prepare, then start signal") require.Equal(t, []uint16{uint16(kubeexecutor.DefaultAgentctlPort), 41001}, forwards.remotePorts()) for _, request := range forwards.requests { require.Equal(t, "127.0.0.1", request.LocalAddress) @@ -428,7 +428,7 @@ func TestKubernetesCreateInstanceRebuildsLostPodAgainstVerifiedManagedPVC(t *tes require.Equal(t, "pvc-uid", reconnected.Metadata[MetadataKeyKubernetesPVCUID]) require.Len(t, resources.createdPVCs, 1, "resume must reuse the recorded PVC") require.Len(t, resources.createdPods, 2) - require.Len(t, reconnectExecs.requests, 4, "replacement Pod needs full bootstrap materialization") + require.Len(t, reconnectExecs.requests, 5, "replacement Pod needs full bootstrap materialization") } func TestKubernetesStopInstancePreservesOrdinaryStopAndForceCleansManagedResources(t *testing.T) { diff --git a/docs/plans/kubernetes-docker-workloads/plan.md b/docs/plans/kubernetes-docker-workloads/plan.md index d75d832341..c9a7911786 100644 --- a/docs/plans/kubernetes-docker-workloads/plan.md +++ b/docs/plans/kubernetes-docker-workloads/plan.md @@ -1,6 +1,6 @@ --- created: 2026-09-13 -status: draft +status: completed requirements: - REQ-EXECUTORS-K8S-DOCKER-001 - REQ-EXECUTORS-K8S-DOCKER-002 @@ -13,7 +13,7 @@ legacy_specs: # Implementation Plan: Kubernetes Docker workloads -**Implementation authored; image and real Docker acceptance blocked.** +**Implemented and accepted in a bounded Linux/amd64 Kind environment.** ## Overview @@ -72,9 +72,13 @@ sidecar through a Pod-local Unix socket. Both mount the workspace; only the agent gets Kandev runtime/auth mounts. Daemon data is explicitly disposable. Provide explicit resources, writable caches and a finite daemon readiness gate -while preserving default clone/setup/agent-install/branch preparation. Deployers -must choose their own namespace policy, placement, limits and network MTU. -The recipe does not grant privileges or install anything by itself. +while preserving default clone/setup/agent-install/branch preparation. Run the +resolved preparation through the existing restricted Pod exec channel before +releasing the managed entrypoint, with a bounded sanitized failure diagnostic. +Use Docker's cgroupfs driver and a relative cgroup parent, then require runtime +evidence that nested scopes remain inside the Pod budget. Deployers must choose +their own namespace policy, placement, limits and network MTU. The recipe does +not grant privileges or install anything by itself. ## Tests @@ -117,21 +121,26 @@ Keep test evidence limited to generic fixtures, runtime versions and outcomes. Run sequentially in the primary session after an implementation request. - [x] [Task 01: Explicit companion workspace grants](task-01-workspace-grants.md) -- [ ] [Task 02: Full worker image and daemon recipe](task-02-full-worker-image.md) -- [ ] [Task 03: Real Kubernetes Docker lifecycle tests](task-03-docker-lifecycle.md) +- [x] [Task 02: Full worker image and daemon recipe](task-02-full-worker-image.md) +- [x] [Task 03: Real Kubernetes Docker lifecycle tests](task-03-docker-lifecycle.md) ## Verification results - Required Kubernetes and lifecycle Go packages passed after workspace-grant implementation. -- Focused template/readiness and clone/cache/retained-workspace tests passed. -- Documentation catalog, specification lint and public-doc validation passed. -- Recipe shell syntax, immutable-input check and renderer smoke passed. -- Four opt-in lifecycle cases are discoverable. They have not executed. +- The pinned Linux/amd64 image built and passed non-root source, browser and + tool verification with image config digest + `sha256:101d44047de5238b7846295a2e175c162c79328b42db9a2693d568a9ac59a0f6`. +- Template/readiness, clone/cache/retained-workspace, renderer and immutable-input + tests passed. +- All four Docker lifecycle cases passed against Kubernetes v1.36.1 in Kind: + full source/browser/Docker/Compose execution, finite daemon failure, + Stop/Resume and lost-Pod replacement, and isolated-daemon accounting/cleanup. +- The focused browser run completed in 3.3 minutes. Required Go packages, + documentation catalog, specification lint and public-doc validation passed. -The full image build/verification did not complete. Task 02 and task 03 remain -blocked pending a suitable isolated test environment. No Docker acceptance, -portable resource enforcement or production compatibility is claimed. See each -work order's Results for the exact scope of evidence and remaining work. +These results establish the documented recipe on the recorded Linux/amd64, +Kubernetes and Docker versions. They do not establish portable enforcement on +every container runtime or remove the documented privileged-DinD boundary. ## Risks diff --git a/docs/plans/kubernetes-docker-workloads/task-02-full-worker-image.md b/docs/plans/kubernetes-docker-workloads/task-02-full-worker-image.md index c5be78f7c1..45e32879d2 100644 --- a/docs/plans/kubernetes-docker-workloads/task-02-full-worker-image.md +++ b/docs/plans/kubernetes-docker-workloads/task-02-full-worker-image.md @@ -1,7 +1,7 @@ --- id: "02-full-worker-image" title: "Full worker image and daemon recipe" -status: blocked +status: completed wave: 2 depends_on: - "01-workspace-grants" @@ -112,30 +112,20 @@ a different host; resolve that before consuming significant resources. ## Results -Implementation authored; runtime acceptance remains blocked. Do not treat this -work order as accepted or promote the pull request to ready. +Completed on Linux/amd64. The immutable recipe built and its constrained +non-root verifier passed source builds/tests for Go, Rust, Node/pnpm, Python and +C, plus a real headless Chromium interaction. The accepted image config digest +was `sha256:101d44047de5238b7846295a2e175c162c79328b42db9a2693d568a9ac59a0f6`. +The same image then passed the Docker build/run/Compose smoke in task 03. -The immutable recipe, explicit daemon template, finite preparation, renderer and -source/browser/Docker smoke scripts are present. `TestFullWorkerTemplate` and -`TestFullWorkerPreparationContract` passed. The filesystem preparation test -`TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace` passed, proving -clone/cache ordering and retained checkout reuse. Shell syntax, input validation -and immutable image rendering passed. The renderer regression first failed -because the renderer was absent; `TestFullWorkerTemplateRequiresImmutableImage` -passed in the focused Go regression rerun after rebasing. +`TestFullWorkerTemplate`, `TestFullWorkerPreparationContract`, +`TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace` and +`TestFullWorkerTemplateRequiresImmutableImage` passed. Shell syntax, input +validation, immutable image rendering and the three Python process-boundary +tests passed. The latter include exact cleanup after failed verification. -A bounded `build.sh --build --verify` attempt did not complete. No image ID, -source/browser image verification or supported full runtime matrix is claimed. -The disposable build resources were removed. Complete the image build and -verification on a suitable isolated test host before running task 03. - -`python3 k8s/worker-images/full/build_test.py` passed after reproducing a failed -verification leaving an untracked container. The verifier now has an exact -owned name and explicit cleanup. This process-boundary test uses a fake Docker -command and does not replace real image/container acceptance. - -`python3 -m unittest discover -s k8s/worker-images/full -p '*_test.py'` -passes three tests. The daemon startup regression first reproduced failure on a -non-eth0 interface. Startup now reads the default-route interface (IPv4, then -IPv6) and its MTU, rejecting missing routes and invalid MTUs. This executes the -template shell with command fixtures; actual CNI compatibility remains unverified. +The daemon startup regression covers default-route discovery for IPv4 and IPv6, +rejecting missing routes and invalid MTUs. The runtime test used that startup +path with Docker 29.1.5 and verified the Pod interface MTU, Unix-only daemon, +cgroupfs driver and relative cgroup parent. Compatibility remains limited to +the recorded architecture and runtime boundary. diff --git a/docs/plans/kubernetes-docker-workloads/task-03-docker-lifecycle.md b/docs/plans/kubernetes-docker-workloads/task-03-docker-lifecycle.md index 6a920cf444..d117f83fd4 100644 --- a/docs/plans/kubernetes-docker-workloads/task-03-docker-lifecycle.md +++ b/docs/plans/kubernetes-docker-workloads/task-03-docker-lifecycle.md @@ -1,7 +1,7 @@ --- id: "03-docker-lifecycle" title: "Real Kubernetes Docker lifecycle tests" -status: blocked +status: completed wave: 3 depends_on: - "02-full-worker-image" @@ -113,21 +113,24 @@ are outside the shared workspace; tests must not conceal incompatible paths. ## Results -Implementation authored; runtime acceptance remains blocked. Do not treat this -work order as accepted or promote the pull request to ready. +All four opt-in browser cases passed on Linux/amd64 with Kind v0.32.0, +Kubernetes v1.36.1 and Docker 29.1.5. The focused run used one worker and +completed in 3.3 minutes: -Four opt-in browser cases and a full-worker fixture are authored. Playwright -`--list` discovers all four cases; discovery is not runtime evidence. Existing -managed workspace snapshot and cleanup Go tests passed in task 01. +- `full worker returns source and Docker results` passed in 44.4 seconds, + including source builds/tests, Chromium, an image build/run, HTTPS transfer, + Compose RO/RW binds and Kandev-visible workspace output. +- `unavailable daemon fails preparation` passed in 5.1 seconds with a finite, + visible preparation failure and no agent start. +- `resume and replacement retain workspace` passed in 14.4 seconds, preserving + the result through Stop/Resume and a lost-Pod replacement while discarding + daemon state. +- `isolated daemons clean up with owned Pods` passed in 20.0 seconds, proving + distinct daemon identities, nested cgroup ancestry/counter movement, one Pod + per session and exact cleanup without deleting the existing claim fixture. -No Kind cluster or nested daemon was launched for these cases. Source/browser -execution in the full image, real API defaulting, Compose binds, visible results, -Stop/Resume/replacement, daemon failure, cgroup accounting and exact real-resource -cleanup remain unverified. The draft must remain blocked until the full focused -suite executes successfully on a suitable isolated test host. No production -compatibility or deployment acceptance is claimed. - -Static validation: changed-file ESLint passed. A focused TypeScript check of the -spec and its imports found existing E2E support-file errors (duplicate members, -missing window augmentation and `node:sqlite` declarations), with no diagnostics -in the two new files. This is not a passing E2E typecheck or runtime result. +The exact implementation revision was exercised with image config digest +`sha256:101d44047de5238b7846295a2e175c162c79328b42db9a2693d568a9ac59a0f6`. +Required Kubernetes and lifecycle Go packages, changed-file ESLint and test +discovery also passed. This evidence applies to the recorded Kind/runtime +combination; other container runtimes still require their own cgroup validation. diff --git a/docs/specs/executors/system-design/kubernetes-docker-workloads.md b/docs/specs/executors/system-design/kubernetes-docker-workloads.md index 3c4dc51b11..806e3073a1 100644 --- a/docs/specs/executors/system-design/kubernetes-docker-workloads.md +++ b/docs/specs/executors/system-design/kubernetes-docker-workloads.md @@ -103,6 +103,12 @@ the managed workspace at `/workspace`; only the agent receives runtime/auth mounts. Docker data uses its own `emptyDir` at `/var/lib/docker` with an explicit size budget, separate from the durable checkout. +On cgroup v2, configure Docker's cgroupfs driver with a relative cgroup parent. +This keeps nested scopes beneath the daemon container's cgroup on compatible +runtimes instead of placing them in a host-root systemd slice. Treat measured +ancestry and accounting as runtime acceptance evidence; configuration alone is +not proof of containment. + Docker bridges run inside an already reduced-MTU Pod network. Set their MTU no higher than the actual Pod interface MTU, and test HTTPS/large transfers from nested containers. Do not change Flannel or @@ -111,8 +117,12 @@ the cluster API endpoint to fix a nested bridge problem. An administrator prepare script prefixes a bounded `docker info` wait before the existing Kubernetes preparation flow. Preserve clone/origin verification, repository setup, agent installation and Kandev's branch-selection postlude; -do not use a wait-only replacement script. A startup probe or running main -container is not a substitute for this preparation gate. +do not use a wait-only replacement script. The backend uploads and executes the +resolved script through the existing Pod exec channel within the setup budget, +captures a bounded sanitized diagnostic, and creates the preparation marker +only on success. It then signals the waiting managed entrypoint. A failed +prepare command must return without releasing agentctl; a startup probe or a +running main container is not a substitute for this preparation gate. Nested bind sources must exist in the daemon's mount namespace. `/workspace` is deliberately shared; arbitrary agent paths such as its private runtime home diff --git a/docs/specs/executors/system-design/kubernetes-startup-timing.md b/docs/specs/executors/system-design/kubernetes-startup-timing.md index 26df74178e..459f2d2b7c 100644 --- a/docs/specs/executors/system-design/kubernetes-startup-timing.md +++ b/docs/specs/executors/system-design/kubernetes-startup-timing.md @@ -32,12 +32,15 @@ and instance IDs provide correlation with lifecycle evidence. | --- | --- | --- | | `storage` | Before `launch.provisionWorkspace` | After workspace verification/provisioning and its checkpoint | | `pod_ready` | Before `launch.createRunningPod` | After admitted main-container-running observation and checkpoint | -| `bootstrap` | Before nonce generation and binary resolution | After `bootstrapPod` returns | +| `bootstrap` | Before nonce generation and binary resolution | After bootstrap uploads, remote preparation, and entrypoint signaling return | | `agentctl_connect` | Before `connectNewAgentctl` | After its health/nonce handshake and forward setup return | `pod_ready` includes scheduling, image pulls, init work, and storage binding waits that occur inside that call. It does not measure those substeps independently. `storage` can complete before a deferred-binding PVC becomes Bound. +`bootstrap` includes the bounded administrator preparation script executed +through Pod exec. A preparation failure therefore ends this stage with an error +before the managed entrypoint is signaled. The total begins at `createFresh` entry and ends at its final return. It includes `launch.complete`, bookkeeping, and failure rollback. diff --git a/docs/specs/kubernetes-executor/spec.md b/docs/specs/kubernetes-executor/spec.md index 6c2c667da3..019647cd25 100644 --- a/docs/specs/kubernetes-executor/spec.md +++ b/docs/specs/kubernetes-executor/spec.md @@ -199,7 +199,10 @@ or cluster-resource identity. 3. **Scheduling:** Kandev creates the merged Pod and waits for the main container to run. 4. **Bootstrapping:** Kandev uploads helper/config/credentials through - `pods/exec`, signals the managed entrypoint, and establishes port-forward. + `pods/exec`, executes the resolved preparation script within its setup + budget, signals the managed entrypoint only after preparation succeeds, and + establishes port-forward. Preparation failure returns a bounded sanitized + diagnostic without releasing agentctl. 5. **Running:** the existing nonce handshake succeeds and agentctl owns the agent subprocess. 6. **Preserved:** ordinary stop or backend shutdown closes local forwarding but diff --git a/k8s/worker-images/full/README.md b/k8s/worker-images/full/README.md index 65a0a53631..479098a6ea 100644 --- a/k8s/worker-images/full/README.md +++ b/k8s/worker-images/full/README.md @@ -51,12 +51,19 @@ container's non-root security context does not contain commands that use Docker. Schedule these Pods only on a worker node pool isolated from trusted workloads. The daemon reads the Pod interface MTU and applies it to default and -user-defined bridges. Verify nested HTTPS transfers for the actual CNI/runtime. +user-defined bridges. It selects Docker's cgroupfs driver with a relative +`docker` parent so nested container cgroups remain below the daemon container's +cgroup on compatible cgroup-v2 runtimes. Verify both properties on the actual +CNI and container runtime before rollout. + The preparation script waits at most 60 seconds (plus a bounded client call) for Docker before the standard clone/origin verification, repository setup and -agent installation. Kandev still adds task-branch preparation. Cache directories -are created only after repository materialization. Kandev owns `HOME`; baked -tools and browsers remain outside `/workspace` and `/run/kandev`. +agent installation. Kandev uploads and runs that script through its existing +restricted Pod exec channel, then releases the managed entrypoint only after +preparation succeeds. A failed script returns a bounded, sanitized diagnostic +and never starts agentctl. Kandev still adds task-branch preparation. Cache +directories are created only after repository materialization. Kandev owns +`HOME`; baked tools and browsers remain outside `/workspace` and `/run/kandev`. ## Exercise source and Docker execution @@ -81,11 +88,16 @@ existing claims remain operator-owned. ## Compatibility limits -Real acceptance results are recorded in the implementation work orders. A -successful image smoke alone does not establish Kubernetes Docker acceptance. +The focused lifecycle suite passed on Linux/amd64 with Kind v0.32.0, +Kubernetes v1.36.1 and Docker 29.1.5. It exercised source/browser work, Docker +build/run, Compose binds, finite readiness failure, Stop/Resume, lost-Pod +replacement, independent daemons, nested cgroup accounting and exact cleanup. +The implementation work orders record the commands, timings and accepted image +config digest. + Kind evidence applies only to the recorded architecture and runtime. Measure -nested cgroup ancestry and counters before claiming resource enforcement; -privileged DinD is not an adversarial isolation boundary. +nested cgroup ancestry and counters before claiming resource enforcement on a +different runtime; privileged DinD is not an adversarial isolation boundary. Only `/workspace` is shared with the daemon. Agent-only HOME/temp bind sources, callbacks, SSH fixtures and arbitrary Docker plugins are not promised to work. diff --git a/k8s/worker-images/full/build.sh b/k8s/worker-images/full/build.sh index 4bff1c2808..65e668c3af 100644 --- a/k8s/worker-images/full/build.sh +++ b/k8s/worker-images/full/build.sh @@ -53,6 +53,6 @@ if [[ ${2:-} == --verify ]]; then verify_started=true timeout 600 docker run --rm --name "$verify_container" --init --cpus=2 --memory=4g --memory-swap=4g --pids-limit=512 \ --user 1000:1000 --cap-drop=ALL --security-opt=no-new-privileges \ - --tmpfs /workspace:rw,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \ + --tmpfs /workspace:rw,exec,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \ -e HOME=/run/kandev/home "$id" bash -ceu 'mkdir -p "$HOME"; /opt/full-worker/smoke.sh --tools' fi diff --git a/k8s/worker-images/full/build_test.py b/k8s/worker-images/full/build_test.py index f61e1229b3..8a4198b39a 100644 --- a/k8s/worker-images/full/build_test.py +++ b/k8s/worker-images/full/build_test.py @@ -7,6 +7,13 @@ import unittest class BuildCleanupTest(unittest.TestCase): + def test_verification_workspace_allows_built_binaries_to_execute(self): + script = (Path(__file__).resolve().parent / "build.sh").read_text() + self.assertIn( + "--tmpfs /workspace:rw,exec,uid=1000,gid=1000,size=2g", + script, + ) + def test_failed_verification_reclaims_its_container(self): recipe = Path(__file__).resolve().parent with tempfile.TemporaryDirectory() as directory: diff --git a/k8s/worker-images/full/daemon_test.py b/k8s/worker-images/full/daemon_test.py index 4b2f68e1f4..d923e91a10 100644 --- a/k8s/worker-images/full/daemon_test.py +++ b/k8s/worker-images/full/daemon_test.py @@ -37,6 +37,13 @@ class DaemonRouteTest(unittest.TestCase): self.assertIn("--mtu=1400\n", result.stdout) self.assertIn("--default-network-opt=bridge=com.docker.network.driver.mtu=1400", result.stdout) + def test_nested_containers_use_a_relative_cgroup_parent(self): + result = self.run_startup(ipv4="default via 192.0.2.1 dev net1") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("--exec-opt\nnative.cgroupdriver=cgroupfs\n", result.stdout) + self.assertIn("--cgroup-parent=docker\n", result.stdout) + def test_missing_route_and_invalid_mtu_fail_before_daemon(self): for options in ({}, {"ipv4": "default dev net1", "mtu": "invalid"}, {"ipv4": "default dev net1", "mtu": "500"}): diff --git a/k8s/worker-images/full/pod-template.yaml b/k8s/worker-images/full/pod-template.yaml index 7d35dcba5d..231d3c7862 100644 --- a/k8s/worker-images/full/pod-template.yaml +++ b/k8s/worker-images/full/pod-template.yaml @@ -40,7 +40,7 @@ template: test -n "$iface" mtu=$(cat "/sys/class/net/$iface/mtu") test "$mtu" -ge 576 - exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2 + exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2 --exec-opt native.cgroupdriver=cgroupfs --cgroup-parent=docker securityContext: runAsUser: 0 runAsGroup: 0