fix(kubernetes): make Docker worker lifecycle reliable (#3676)

k8s/worker-images/full/README.md

@@ -51,12 +51,19 @@ container's non-root security context does not contain commands that use Docker.
 Schedule these Pods only on a worker node pool isolated from trusted workloads.
 
 The daemon reads the Pod interface MTU and applies it to default and
-user-defined bridges. Verify nested HTTPS transfers for the actual CNI/runtime.
+user-defined bridges. It selects Docker's cgroupfs driver with a relative
+`docker` parent so nested container cgroups remain below the daemon container's
+cgroup on compatible cgroup-v2 runtimes. Verify both properties on the actual
+CNI and container runtime before rollout.
+
 The preparation script waits at most 60 seconds (plus a bounded client call)
 for Docker before the standard clone/origin verification, repository setup and
-agent installation. Kandev still adds task-branch preparation. Cache directories
-are created only after repository materialization. Kandev owns `HOME`; baked
-tools and browsers remain outside `/workspace` and `/run/kandev`.
+agent installation. Kandev uploads and runs that script through its existing
+restricted Pod exec channel, then releases the managed entrypoint only after
+preparation succeeds. A failed script returns a bounded, sanitized diagnostic
+and never starts agentctl. Kandev still adds task-branch preparation. Cache
+directories are created only after repository materialization. Kandev owns
+`HOME`; baked tools and browsers remain outside `/workspace` and `/run/kandev`.
 
 ## Exercise source and Docker execution
 
@@ -81,11 +88,16 @@ existing claims remain operator-owned.
 
 ## Compatibility limits
 
-Real acceptance results are recorded in the implementation work orders. A
-successful image smoke alone does not establish Kubernetes Docker acceptance.
+The focused lifecycle suite passed on Linux/amd64 with Kind v0.32.0,
+Kubernetes v1.36.1 and Docker 29.1.5. It exercised source/browser work, Docker
+build/run, Compose binds, finite readiness failure, Stop/Resume, lost-Pod
+replacement, independent daemons, nested cgroup accounting and exact cleanup.
+The implementation work orders record the commands, timings and accepted image
+config digest.
+
 Kind evidence applies only to the recorded architecture and runtime. Measure
-nested cgroup ancestry and counters before claiming resource enforcement;
-privileged DinD is not an adversarial isolation boundary.
+nested cgroup ancestry and counters before claiming resource enforcement on a
+different runtime; privileged DinD is not an adversarial isolation boundary.
 
 Only `/workspace` is shared with the daemon. Agent-only HOME/temp bind sources,
 callbacks, SSH fixtures and arbitrary Docker plugins are not promised to work.

k8s/worker-images/full/build.sh

@@ -53,6 +53,6 @@ if [[ ${2:-} == --verify ]]; then
   verify_started=true
   timeout 600 docker run --rm --name "$verify_container" --init --cpus=2 --memory=4g --memory-swap=4g --pids-limit=512 \
     --user 1000:1000 --cap-drop=ALL --security-opt=no-new-privileges \
-    --tmpfs /workspace:rw,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \
+    --tmpfs /workspace:rw,exec,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \
     -e HOME=/run/kandev/home "$id" bash -ceu 'mkdir -p "$HOME"; /opt/full-worker/smoke.sh --tools'
 fi

k8s/worker-images/full/build_test.py

@@ -7,6 +7,13 @@ import unittest
 
 
 class BuildCleanupTest(unittest.TestCase):
+    def test_verification_workspace_allows_built_binaries_to_execute(self):
+        script = (Path(__file__).resolve().parent / "build.sh").read_text()
+        self.assertIn(
+            "--tmpfs /workspace:rw,exec,uid=1000,gid=1000,size=2g",
+            script,
+        )
+
     def test_failed_verification_reclaims_its_container(self):
         recipe = Path(__file__).resolve().parent
         with tempfile.TemporaryDirectory() as directory:

k8s/worker-images/full/daemon_test.py

@@ -37,6 +37,13 @@ class DaemonRouteTest(unittest.TestCase):
                 self.assertIn("--mtu=1400\n", result.stdout)
                 self.assertIn("--default-network-opt=bridge=com.docker.network.driver.mtu=1400", result.stdout)
 
+    def test_nested_containers_use_a_relative_cgroup_parent(self):
+        result = self.run_startup(ipv4="default via 192.0.2.1 dev net1")
+
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertIn("--exec-opt\nnative.cgroupdriver=cgroupfs\n", result.stdout)
+        self.assertIn("--cgroup-parent=docker\n", result.stdout)
+
     def test_missing_route_and_invalid_mtu_fail_before_daemon(self):
         for options in ({}, {"ipv4": "default dev net1", "mtu": "invalid"},
                         {"ipv4": "default dev net1", "mtu": "500"}):

k8s/worker-images/full/pod-template.yaml

@@ -40,7 +40,7 @@ template:
             test -n "$iface"
             mtu=$(cat "/sys/class/net/$iface/mtu")
             test "$mtu" -ge 576
-            exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2
+            exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2 --exec-opt native.cgroupdriver=cgroupfs --cgroup-parent=docker
         securityContext:
           runAsUser: 0
           runAsGroup: 0