dreampiler/kandev
1
0
Fork 0
Code Issues 43 Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix(kubernetes): make Docker worker lifecycle reliable (#3676)

Browse Source
This commit is contained in:
José Almeida
2026-09-15 06:33:02 +02:00
committed by GitHub
parent 941e7950d1
commit 54e2e21f32
16 changed files with 297 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go
+1 -4
View File
@@ -814,10 +814,7 @@ set -a
. /opt/kandev/runtime.env
. /run/kandev/auth.env
set +a
if [ ! -f /opt/kandev/prepared ]; then
sh /opt/kandev/prepare.sh
: > /opt/kandev/prepared
fi
[ -f /opt/kandev/prepared ] || { echo 'kubernetes preparation marker is missing' >&2; exit 70; }
if [ "${` + selectedCheckoutMarker + `:-}" = "1" ]; then
` + selectedCheckoutCredentialScrubCommands + `
rm -f /run/kandev/auth.env 2>/dev/null || true
apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go
+80
View File
@@ -11,10 +11,14 @@ import (
"sort"
"strconv"
"strings"
"sync"
"unicode/utf8"
corev1 "k8s.io/api/core/v1"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
"github.com/kandev/kandev/internal/common/constants"
"github.com/kandev/kandev/internal/githubauth"
"github.com/kandev/kandev/internal/scriptengine"
)
@@ -63,6 +67,9 @@ func (r *KubernetesExecutor) bootstrapPod(
if err := r.materializeKubernetesRemoteFiles(ctx, runtime, req, pod, profile.MainContainer); err != nil {
return err
}
if err := runKubernetesPrepareScript(ctx, runtime.streams, pod, profile.MainContainer); err != nil {
return err
}
if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer,
kubernetesStartPath, nil, 0o600); err != nil {
return fmt.Errorf("kubernetes lifecycle: signal bootstrap: %w", err)
@@ -70,6 +77,79 @@ func (r *KubernetesExecutor) bootstrapPod(
return nil
}
type kubernetesPrepareOutput struct {
mu sync.Mutex
data []byte
overflow bool
}
const kubernetesPrepareCaptureBytes = 256 * 1024
func (w *kubernetesPrepareOutput) Write(p []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
if w.overflow {
return len(p), nil
}
if len(p) > kubernetesPrepareCaptureBytes-len(w.data) {
w.data = nil
w.overflow = true
return len(p), nil
}
w.data = append(w.data, p...)
return len(p), nil
}
func (w *kubernetesPrepareOutput) diagnostic() string {
w.mu.Lock()
if w.overflow {
w.mu.Unlock()
return ""
}
raw := string(w.data)
w.mu.Unlock()
diagnostic := routingerr.SanitizeFullUnbounded(raw)
if len(diagnostic) > routingerr.MaxRawExcerptBytes {
start := len(diagnostic) - routingerr.MaxRawExcerptBytes
for start < len(diagnostic) && !utf8.RuneStart(diagnostic[start]) {
start++
}
diagnostic = diagnostic[start:]
}
return strings.TrimSpace(diagnostic)
}
func runKubernetesPrepareScript(
ctx context.Context,
streams *kubeexecutor.StreamOperations,
pod *corev1.Pod,
container string,
) error {
prepareCtx, cancel := context.WithTimeout(preparationContext(ctx), constants.SetupScriptTimeout)
defer cancel()
output := &kubernetesPrepareOutput{}
command := `set -eu
set -a
. /opt/kandev/runtime.env
. /run/kandev/auth.env
set +a
sh /opt/kandev/prepare.sh
: > /opt/kandev/prepared`
err := streams.Exec(prepareCtx, kubeexecutor.ExecRequest{
Namespace: pod.Namespace, Pod: pod.Name, Container: container,
Command: []string{"sh", "-c", command}, Stdout: output, Stderr: output,
})
if err == nil {
return nil
}
diagnostic := output.diagnostic()
if diagnostic == "" {
return fmt.Errorf("kubernetes lifecycle: prepare script failed: %w", err)
}
return fmt.Errorf("kubernetes lifecycle: prepare script failed: %s: %w", diagnostic, err)
}
func kubernetesRuntimeEnvironment(req *ExecutorCreateRequest) map[string]string {
values := map[string]string{
"AGENTCTL_LISTEN_HOST": kubernetesControlHost,
apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap_test.go
+92 -1
View File
@@ -2,6 +2,7 @@ package lifecycle
import (
"context"
"errors"
"os"
"os/exec"
"path/filepath"
@@ -36,6 +37,96 @@ func TestKubernetesWriteFileDoesNotOpenStdinForEmptySentinel(t *testing.T) {
require.Contains(t, command, "mv -f \"$temporary\" \"$destination\"")
}
func TestKubernetesBootstrapRunsPrepareBeforeSignallingEntrypoint(t *testing.T) {
execs := &recordingKubernetesExec{}
streams := kubeexecutor.NewStreamOperations(execs, nil)
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "kandev-agents", Name: "agent-pod"}}
executor := NewKubernetesExecutor(nil, newTestLogger())
req := validKubernetesCreateRequest()
err := executor.bootstrapPod(
context.Background(),
&kubernetesRuntimeClient{streams: streams},
req,
pod,
kubeexecutor.ProfileConfig{MainContainer: "kandev-agent"},
"nonce",
[]byte("agentctl"),
)
require.NoError(t, err)
prepareIndex, signalIndex := -1, -1
for index, recorded := range execs.requests {
command := strings.Join(recorded.request.Command, " ")
if strings.Contains(command, "sh "+kubernetesPreparePath) {
prepareIndex = index
}
if strings.Contains(command, kubernetesStartPath) && recorded.request.Stdin == nil {
signalIndex = index
}
}
require.NotEqual(t, -1, prepareIndex)
require.NotEqual(t, -1, signalIndex)
require.Less(t, prepareIndex, signalIndex)
require.NotContains(t, kubernetesBootstrapCommand(), "sh "+kubernetesPreparePath)
}
func TestKubernetesBootstrapReturnsSanitizedPrepareFailure(t *testing.T) {
execs := &recordingKubernetesExec{}
execs.execFunc = func(_ context.Context, request kubeexecutor.ExecRequest) error {
if !strings.Contains(strings.Join(request.Command, " "), "sh "+kubernetesPreparePath) {
return nil
}
_, _ = request.Stderr.Write([]byte(
"token=do-not-persist\nDocker daemon did not become ready before preparation deadline\n",
))
return errors.New("remote command exited with status 1")
}
streams := kubeexecutor.NewStreamOperations(execs, nil)
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "kandev-agents", Name: "agent-pod"}}
executor := NewKubernetesExecutor(nil, newTestLogger())
err := executor.bootstrapPod(
context.Background(),
&kubernetesRuntimeClient{streams: streams},
validKubernetesCreateRequest(),
pod,
kubeexecutor.ProfileConfig{MainContainer: "kandev-agent"},
"nonce",
[]byte("agentctl"),
)
require.ErrorContains(t, err, "Docker daemon did not become ready before preparation deadline")
require.NotContains(t, err.Error(), "do-not-persist")
for _, recorded := range execs.requests {
command := strings.Join(recorded.request.Command, " ")
require.False(t,
strings.Contains(command, kubernetesStartPath) && recorded.request.Stdin == nil,
"failed preparation must not signal the managed entrypoint",
)
}
}
func TestKubernetesPrepareOutputDropsOversizeDiagnosticsBeforeRedaction(t *testing.T) {
output := &kubernetesPrepareOutput{}
secret := strings.Repeat("Zx.", 100_000)
_, err := output.Write([]byte("token=" + secret + "\nlate failure"))
require.NoError(t, err)
require.Empty(t, output.diagnostic(),
"oversize raw output must be dropped instead of cutting away a credential marker")
}
func TestKubernetesPrepareOutputKeepsLatestSanitizedDiagnostic(t *testing.T) {
output := &kubernetesPrepareOutput{}
_, err := output.Write([]byte(strings.Repeat("ordinary setup output\n", 400) +
"token=do-not-persist\nlate failure"))
require.NoError(t, err)
require.Contains(t, output.diagnostic(), "late failure")
require.NotContains(t, output.diagnostic(), "do-not-persist")
}
func TestKubernetesWriteFileCommandRejectsSymlinkedAncestor(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
@@ -153,8 +244,8 @@ func TestKubernetesBootstrapPublishesManagedCredentialHelperPath(t *testing.T) {
)
require.NotContains(t, authData, "/tmp/hostile-agentctl")
command := kubernetesBootstrapCommand()
require.Less(t, strings.Index(command, ". "+kubernetesAuthEnvPath), strings.Index(command, "sh "+kubernetesPreparePath))
require.Less(t, strings.Index(command, ". "+kubernetesAuthEnvPath), strings.Index(command, "exec "+kubernetesAgentctlPath))
require.NotContains(t, command, "sh "+kubernetesPreparePath)
}
func TestKubernetesPrepareScriptUsesManagedAgentctlPathWithoutStartingSecondProcess(t *testing.T) {
apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_fakes_test.go
+6 -2
View File
@@ -351,9 +351,10 @@ type recordingKubernetesExec struct {
mu sync.Mutex
requests []recordedKubernetesExec
err error
execFunc func(context.Context, kubeexecutor.ExecRequest) error
}
func (r *recordingKubernetesExec) Exec(_ context.Context, request kubeexecutor.ExecRequest) error {
func (r *recordingKubernetesExec) Exec(ctx context.Context, request kubeexecutor.ExecRequest) error {
var data []byte
if request.Stdin != nil {
var err error
@@ -363,8 +364,11 @@ func (r *recordingKubernetesExec) Exec(_ context.Context, request kubeexecutor.E
}
}
r.mu.Lock()
defer r.mu.Unlock()
r.requests = append(r.requests, recordedKubernetesExec{request: request, stdin: data})
r.mu.Unlock()
if r.execFunc != nil {
return r.execFunc(ctx, request)
}
return r.err
}
apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_test.go
+2 -2
View File
@@ -148,7 +148,7 @@ func TestKubernetesCreateInstanceProvisionsBootstrapsAndForwardsAgentctl(t *test
require.NotNil(t, instance.Client)
require.Len(t, resources.createdPVCs, 1)
require.Len(t, resources.createdPods, 1)
require.Len(t, execs.requests, 4, "binary, runtime config, auth config, then start signal")
require.Len(t, execs.requests, 5, "binary, runtime config, auth config, prepare, then start signal")
require.Equal(t, []uint16{uint16(kubeexecutor.DefaultAgentctlPort), 41001}, forwards.remotePorts())
for _, request := range forwards.requests {
require.Equal(t, "127.0.0.1", request.LocalAddress)
@@ -428,7 +428,7 @@ func TestKubernetesCreateInstanceRebuildsLostPodAgainstVerifiedManagedPVC(t *tes
require.Equal(t, "pvc-uid", reconnected.Metadata[MetadataKeyKubernetesPVCUID])
require.Len(t, resources.createdPVCs, 1, "resume must reuse the recorded PVC")
require.Len(t, resources.createdPods, 2)
require.Len(t, reconnectExecs.requests, 4, "replacement Pod needs full bootstrap materialization")
require.Len(t, reconnectExecs.requests, 5, "replacement Pod needs full bootstrap materialization")
}
func TestKubernetesStopInstancePreservesOrdinaryStopAndForceCleansManagedResources(t *testing.T) {
docs/plans/kubernetes-docker-workloads/plan.md
+24 -15
View File
@@ -1,6 +1,6 @@
---
created: 2026-09-13
status: draft
status: completed
requirements:
- REQ-EXECUTORS-K8S-DOCKER-001
- REQ-EXECUTORS-K8S-DOCKER-002
@@ -13,7 +13,7 @@ legacy_specs:
# Implementation Plan: Kubernetes Docker workloads
**Implementation authored; image and real Docker acceptance blocked.**
**Implemented and accepted in a bounded Linux/amd64 Kind environment.**
## Overview
@@ -72,9 +72,13 @@ sidecar through a Pod-local Unix socket. Both mount the workspace; only the
agent gets Kandev runtime/auth mounts. Daemon data is explicitly disposable.
Provide explicit resources, writable caches and a finite daemon readiness gate
while preserving default clone/setup/agent-install/branch preparation. Deployers
must choose their own namespace policy, placement, limits and network MTU.
The recipe does not grant privileges or install anything by itself.
while preserving default clone/setup/agent-install/branch preparation. Run the
resolved preparation through the existing restricted Pod exec channel before
releasing the managed entrypoint, with a bounded sanitized failure diagnostic.
Use Docker's cgroupfs driver and a relative cgroup parent, then require runtime
evidence that nested scopes remain inside the Pod budget. Deployers must choose
their own namespace policy, placement, limits and network MTU. The recipe does
not grant privileges or install anything by itself.
## Tests
@@ -117,21 +121,26 @@ Keep test evidence limited to generic fixtures, runtime versions and outcomes.
Run sequentially in the primary session after an implementation request.
- [x] [Task 01: Explicit companion workspace grants](task-01-workspace-grants.md)
- [ ] [Task 02: Full worker image and daemon recipe](task-02-full-worker-image.md)
- [ ] [Task 03: Real Kubernetes Docker lifecycle tests](task-03-docker-lifecycle.md)
- [x] [Task 02: Full worker image and daemon recipe](task-02-full-worker-image.md)
- [x] [Task 03: Real Kubernetes Docker lifecycle tests](task-03-docker-lifecycle.md)
## Verification results
- Required Kubernetes and lifecycle Go packages passed after workspace-grant implementation.
- Focused template/readiness and clone/cache/retained-workspace tests passed.
- Documentation catalog, specification lint and public-doc validation passed.
- Recipe shell syntax, immutable-input check and renderer smoke passed.
- Four opt-in lifecycle cases are discoverable. They have not executed.
- The pinned Linux/amd64 image built and passed non-root source, browser and
tool verification with image config digest
`sha256:101d44047de5238b7846295a2e175c162c79328b42db9a2693d568a9ac59a0f6`.
- Template/readiness, clone/cache/retained-workspace, renderer and immutable-input
tests passed.
- All four Docker lifecycle cases passed against Kubernetes v1.36.1 in Kind:
full source/browser/Docker/Compose execution, finite daemon failure,
Stop/Resume and lost-Pod replacement, and isolated-daemon accounting/cleanup.
- The focused browser run completed in 3.3 minutes. Required Go packages,
documentation catalog, specification lint and public-doc validation passed.
The full image build/verification did not complete. Task 02 and task 03 remain
blocked pending a suitable isolated test environment. No Docker acceptance,
portable resource enforcement or production compatibility is claimed. See each
work order's Results for the exact scope of evidence and remaining work.
These results establish the documented recipe on the recorded Linux/amd64,
Kubernetes and Docker versions. They do not establish portable enforcement on
every container runtime or remove the documented privileged-DinD boundary.
## Risks
docs/plans/kubernetes-docker-workloads/task-02-full-worker-image.md
+16 -26
View File
@@ -1,7 +1,7 @@
---
id: "02-full-worker-image"
title: "Full worker image and daemon recipe"
status: blocked
status: completed
wave: 2
depends_on:
- "01-workspace-grants"
@@ -112,30 +112,20 @@ a different host; resolve that before consuming significant resources.
## Results
Implementation authored; runtime acceptance remains blocked. Do not treat this
work order as accepted or promote the pull request to ready.
Completed on Linux/amd64. The immutable recipe built and its constrained
non-root verifier passed source builds/tests for Go, Rust, Node/pnpm, Python and
C, plus a real headless Chromium interaction. The accepted image config digest
was `sha256:101d44047de5238b7846295a2e175c162c79328b42db9a2693d568a9ac59a0f6`.
The same image then passed the Docker build/run/Compose smoke in task 03.
The immutable recipe, explicit daemon template, finite preparation, renderer and
source/browser/Docker smoke scripts are present. `TestFullWorkerTemplate` and
`TestFullWorkerPreparationContract` passed. The filesystem preparation test
`TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace` passed, proving
clone/cache ordering and retained checkout reuse. Shell syntax, input validation
and immutable image rendering passed. The renderer regression first failed
because the renderer was absent; `TestFullWorkerTemplateRequiresImmutableImage`
passed in the focused Go regression rerun after rebasing.
`TestFullWorkerTemplate`, `TestFullWorkerPreparationContract`,
`TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace` and
`TestFullWorkerTemplateRequiresImmutableImage` passed. Shell syntax, input
validation, immutable image rendering and the three Python process-boundary
tests passed. The latter include exact cleanup after failed verification.
A bounded `build.sh --build --verify` attempt did not complete. No image ID,
source/browser image verification or supported full runtime matrix is claimed.
The disposable build resources were removed. Complete the image build and
verification on a suitable isolated test host before running task 03.
`python3 k8s/worker-images/full/build_test.py` passed after reproducing a failed
verification leaving an untracked container. The verifier now has an exact
owned name and explicit cleanup. This process-boundary test uses a fake Docker
command and does not replace real image/container acceptance.
`python3 -m unittest discover -s k8s/worker-images/full -p '*_test.py'`
passes three tests. The daemon startup regression first reproduced failure on a
non-eth0 interface. Startup now reads the default-route interface (IPv4, then
IPv6) and its MTU, rejecting missing routes and invalid MTUs. This executes the
template shell with command fixtures; actual CNI compatibility remains unverified.
The daemon startup regression covers default-route discovery for IPv4 and IPv6,
rejecting missing routes and invalid MTUs. The runtime test used that startup
path with Docker 29.1.5 and verified the Pod interface MTU, Unix-only daemon,
cgroupfs driver and relative cgroup parent. Compatibility remains limited to
the recorded architecture and runtime boundary.
docs/plans/kubernetes-docker-workloads/task-03-docker-lifecycle.md
+20 -17
View File
@@ -1,7 +1,7 @@
---
id: "03-docker-lifecycle"
title: "Real Kubernetes Docker lifecycle tests"
status: blocked
status: completed
wave: 3
depends_on:
- "02-full-worker-image"
@@ -113,21 +113,24 @@ are outside the shared workspace; tests must not conceal incompatible paths.
## Results
Implementation authored; runtime acceptance remains blocked. Do not treat this
work order as accepted or promote the pull request to ready.
All four opt-in browser cases passed on Linux/amd64 with Kind v0.32.0,
Kubernetes v1.36.1 and Docker 29.1.5. The focused run used one worker and
completed in 3.3 minutes:
Four opt-in browser cases and a full-worker fixture are authored. Playwright
`--list` discovers all four cases; discovery is not runtime evidence. Existing
managed workspace snapshot and cleanup Go tests passed in task 01.
- `full worker returns source and Docker results` passed in 44.4 seconds,
including source builds/tests, Chromium, an image build/run, HTTPS transfer,
Compose RO/RW binds and Kandev-visible workspace output.
- `unavailable daemon fails preparation` passed in 5.1 seconds with a finite,
visible preparation failure and no agent start.
- `resume and replacement retain workspace` passed in 14.4 seconds, preserving
the result through Stop/Resume and a lost-Pod replacement while discarding
daemon state.
- `isolated daemons clean up with owned Pods` passed in 20.0 seconds, proving
distinct daemon identities, nested cgroup ancestry/counter movement, one Pod
per session and exact cleanup without deleting the existing claim fixture.
No Kind cluster or nested daemon was launched for these cases. Source/browser
execution in the full image, real API defaulting, Compose binds, visible results,
Stop/Resume/replacement, daemon failure, cgroup accounting and exact real-resource
cleanup remain unverified. The draft must remain blocked until the full focused
suite executes successfully on a suitable isolated test host. No production
compatibility or deployment acceptance is claimed.
Static validation: changed-file ESLint passed. A focused TypeScript check of the
spec and its imports found existing E2E support-file errors (duplicate members,
missing window augmentation and `node:sqlite` declarations), with no diagnostics
in the two new files. This is not a passing E2E typecheck or runtime result.
The exact implementation revision was exercised with image config digest
`sha256:101d44047de5238b7846295a2e175c162c79328b42db9a2693d568a9ac59a0f6`.
Required Kubernetes and lifecycle Go packages, changed-file ESLint and test
discovery also passed. This evidence applies to the recorded Kind/runtime
combination; other container runtimes still require their own cgroup validation.
docs/specs/executors/system-design/kubernetes-docker-workloads.md
+12 -2
View File
@@ -103,6 +103,12 @@ the managed workspace at `/workspace`; only the agent receives runtime/auth
mounts. Docker data uses its own `emptyDir` at `/var/lib/docker` with an explicit
size budget, separate from the durable checkout.
On cgroup v2, configure Docker's cgroupfs driver with a relative cgroup parent.
This keeps nested scopes beneath the daemon container's cgroup on compatible
runtimes instead of placing them in a host-root systemd slice. Treat measured
ancestry and accounting as runtime acceptance evidence; configuration alone is
not proof of containment.
Docker bridges run inside an already reduced-MTU Pod network. Set their MTU no
higher than the actual Pod interface MTU,
and test HTTPS/large transfers from nested containers. Do not change Flannel or
@@ -111,8 +117,12 @@ the cluster API endpoint to fix a nested bridge problem.
An administrator prepare script prefixes a bounded `docker info` wait before
the existing Kubernetes preparation flow. Preserve clone/origin verification,
repository setup, agent installation and Kandev's branch-selection postlude;
do not use a wait-only replacement script. A startup probe or running main
container is not a substitute for this preparation gate.
do not use a wait-only replacement script. The backend uploads and executes the
resolved script through the existing Pod exec channel within the setup budget,
captures a bounded sanitized diagnostic, and creates the preparation marker
only on success. It then signals the waiting managed entrypoint. A failed
prepare command must return without releasing agentctl; a startup probe or a
running main container is not a substitute for this preparation gate.
Nested bind sources must exist in the daemon's mount namespace. `/workspace`
is deliberately shared; arbitrary agent paths such as its private runtime home
docs/specs/executors/system-design/kubernetes-startup-timing.md
+4 -1
View File
@@ -32,12 +32,15 @@ and instance IDs provide correlation with lifecycle evidence.
| --- | --- | --- |
| `storage` | Before `launch.provisionWorkspace` | After workspace verification/provisioning and its checkpoint |
| `pod_ready` | Before `launch.createRunningPod` | After admitted main-container-running observation and checkpoint |
| `bootstrap` | Before nonce generation and binary resolution | After `bootstrapPod` returns |
| `bootstrap` | Before nonce generation and binary resolution | After bootstrap uploads, remote preparation, and entrypoint signaling return |
| `agentctl_connect` | Before `connectNewAgentctl` | After its health/nonce handshake and forward setup return |
`pod_ready` includes scheduling, image pulls, init work, and storage binding waits
that occur inside that call. It does not measure those substeps independently.
`storage` can complete before a deferred-binding PVC becomes Bound.
`bootstrap` includes the bounded administrator preparation script executed
through Pod exec. A preparation failure therefore ends this stage with an error
before the managed entrypoint is signaled.
The total begins at `createFresh` entry and ends at its final return.
It includes `launch.complete`, bookkeeping, and failure rollback.
docs/specs/kubernetes-executor/spec.md
+4 -1
View File
@@ -199,7 +199,10 @@ or cluster-resource identity.
3. **Scheduling:** Kandev creates the merged Pod and waits for the main
container to run.
4. **Bootstrapping:** Kandev uploads helper/config/credentials through
`pods/exec`, signals the managed entrypoint, and establishes port-forward.
`pods/exec`, executes the resolved preparation script within its setup
budget, signals the managed entrypoint only after preparation succeeds, and
establishes port-forward. Preparation failure returns a bounded sanitized
diagnostic without releasing agentctl.
5. **Running:** the existing nonce handshake succeeds and agentctl owns the
agent subprocess.
6. **Preserved:** ordinary stop or backend shutdown closes local forwarding but
k8s/worker-images/full/README.md
+20 -8
View File
@@ -51,12 +51,19 @@ container's non-root security context does not contain commands that use Docker.
Schedule these Pods only on a worker node pool isolated from trusted workloads.
The daemon reads the Pod interface MTU and applies it to default and
user-defined bridges. Verify nested HTTPS transfers for the actual CNI/runtime.
user-defined bridges. It selects Docker's cgroupfs driver with a relative
`docker` parent so nested container cgroups remain below the daemon container's
cgroup on compatible cgroup-v2 runtimes. Verify both properties on the actual
CNI and container runtime before rollout.
The preparation script waits at most 60 seconds (plus a bounded client call)
for Docker before the standard clone/origin verification, repository setup and
agent installation. Kandev still adds task-branch preparation. Cache directories
are created only after repository materialization. Kandev owns `HOME`; baked
tools and browsers remain outside `/workspace` and `/run/kandev`.
agent installation. Kandev uploads and runs that script through its existing
restricted Pod exec channel, then releases the managed entrypoint only after
preparation succeeds. A failed script returns a bounded, sanitized diagnostic
and never starts agentctl. Kandev still adds task-branch preparation. Cache
directories are created only after repository materialization. Kandev owns
`HOME`; baked tools and browsers remain outside `/workspace` and `/run/kandev`.
## Exercise source and Docker execution
@@ -81,11 +88,16 @@ existing claims remain operator-owned.
## Compatibility limits
Real acceptance results are recorded in the implementation work orders. A
successful image smoke alone does not establish Kubernetes Docker acceptance.
The focused lifecycle suite passed on Linux/amd64 with Kind v0.32.0,
Kubernetes v1.36.1 and Docker 29.1.5. It exercised source/browser work, Docker
build/run, Compose binds, finite readiness failure, Stop/Resume, lost-Pod
replacement, independent daemons, nested cgroup accounting and exact cleanup.
The implementation work orders record the commands, timings and accepted image
config digest.
Kind evidence applies only to the recorded architecture and runtime. Measure
nested cgroup ancestry and counters before claiming resource enforcement;
privileged DinD is not an adversarial isolation boundary.
nested cgroup ancestry and counters before claiming resource enforcement on a
different runtime; privileged DinD is not an adversarial isolation boundary.
Only `/workspace` is shared with the daemon. Agent-only HOME/temp bind sources,
callbacks, SSH fixtures and arbitrary Docker plugins are not promised to work.
k8s/worker-images/full/build.sh
+1 -1
View File
@@ -53,6 +53,6 @@ if [[ ${2:-} == --verify ]]; then
verify_started=true
timeout 600 docker run --rm --name "$verify_container" --init --cpus=2 --memory=4g --memory-swap=4g --pids-limit=512 \
--user 1000:1000 --cap-drop=ALL --security-opt=no-new-privileges \
--tmpfs /workspace:rw,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \
--tmpfs /workspace:rw,exec,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \
-e HOME=/run/kandev/home "$id" bash -ceu 'mkdir -p "$HOME"; /opt/full-worker/smoke.sh --tools'
fi
k8s/worker-images/full/build_test.py
+7
View File
@@ -7,6 +7,13 @@ import unittest
class BuildCleanupTest(unittest.TestCase):
def test_verification_workspace_allows_built_binaries_to_execute(self):
script = (Path(__file__).resolve().parent / "build.sh").read_text()
self.assertIn(
"--tmpfs /workspace:rw,exec,uid=1000,gid=1000,size=2g",
script,
)
def test_failed_verification_reclaims_its_container(self):
recipe = Path(__file__).resolve().parent
with tempfile.TemporaryDirectory() as directory:
k8s/worker-images/full/daemon_test.py
+7
View File
@@ -37,6 +37,13 @@ class DaemonRouteTest(unittest.TestCase):
self.assertIn("--mtu=1400\n", result.stdout)
self.assertIn("--default-network-opt=bridge=com.docker.network.driver.mtu=1400", result.stdout)
def test_nested_containers_use_a_relative_cgroup_parent(self):
result = self.run_startup(ipv4="default via 192.0.2.1 dev net1")
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn("--exec-opt\nnative.cgroupdriver=cgroupfs\n", result.stdout)
self.assertIn("--cgroup-parent=docker\n", result.stdout)
def test_missing_route_and_invalid_mtu_fail_before_daemon(self):
for options in ({}, {"ipv4": "default dev net1", "mtu": "invalid"},
{"ipv4": "default dev net1", "mtu": "500"}):
k8s/worker-images/full/pod-template.yaml
+1 -1
View File
@@ -40,7 +40,7 @@ template:
test -n "$iface"
mtu=$(cat "/sys/class/net/$iface/mtu")
test "$mtu" -ge 576
exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2
exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2 --exec-opt native.cgroupdriver=cgroupfs --cgroup-parent=docker
securityContext:
runAsUser: 0
runAsGroup: 0