feat(kubernetes): add explicit Docker workload recipe (#3648)
This commit is contained in:
José Almeida
@@ -129,6 +129,7 @@ Kandev can run any agent CLI as a TUI inside a terminal, even when it does not s
|
||||
|:--------:|-------------|
|
||||
| **Local Process** | Runs the agent as a local process on the host machine |
|
||||
| **Docker** | Runs the agent in an isolated Docker container |
|
||||
| **Kubernetes** | Runs each task session in an administrator-configured Pod with managed, existing, or disposable workspace storage |
|
||||
| **SSH** | Runs the agent on a remote server over SSH |
|
||||
| **Sprites** | Runs the agent in a remote cloud environment via [sprites.dev](https://sprites.dev) |
|
||||
|
||||
@@ -236,8 +237,7 @@ graph LR
|
||||
end
|
||||
```
|
||||
|
||||
We also want to add support for this remote runtime:
|
||||
- **K8s operator** - run agents in a Kubernetes cluster, with auto-scaling and resource management.
|
||||
See the [Kubernetes guide](docs/public/k8s.md) for executor configuration and lifecycle. The executor uses ordinary Pods and does not require an operator.
|
||||
|
||||
<details>
|
||||
<summary><strong>Development</strong></summary>
|
||||
|
||||
@@ -33,6 +33,9 @@ func ValidateAdmittedPod(admitted, desired *corev1.Pod, mainContainer string) er
|
||||
admittedMain.WorkingDir != desiredMain.WorkingDir {
|
||||
return errors.New("admitted Pod mutated the main-container bootstrap")
|
||||
}
|
||||
if err := validateAdmittedWorkspaceGrants(admitted, desired, mainContainer); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := validateAdmittedContainers(admitted, desiredMain, mainContainer); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -292,6 +295,13 @@ func validateAdmittedContainers(
|
||||
) error {
|
||||
for index := range admitted.Spec.Containers {
|
||||
container := &admitted.Spec.Containers[index]
|
||||
if container.Name != mainContainer {
|
||||
checked, _, err := companionWithoutWorkspaceGrant(container)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
container = checked
|
||||
}
|
||||
if err := validateAdmittedContainer(container, desiredMain, container.Name == mainContainer); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -387,6 +397,9 @@ func validateRequiredMainContainerFields(admitted, desired *corev1.Container) er
|
||||
}
|
||||
|
||||
func validateAdmittedVolumes(admitted, desired []corev1.Volume) error {
|
||||
if err := validateWorkspaceClaimAliases(admitted, desired); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, required := range desired {
|
||||
if isReservedVolumeName(required.Name) && countVolumes(admitted, required) != 1 {
|
||||
return errors.New("admitted Pod mutated a reserved volume")
|
||||
|
||||
@@ -73,6 +73,9 @@ func ComposePod(template *corev1.PodTemplate, profile ProfileConfig, options Pod
|
||||
return nil, nil, err
|
||||
}
|
||||
pod.Spec.Volumes = append(pod.Spec.Volumes, managedVolumes(workspace)...)
|
||||
if err := validateWorkspaceClaimAliases(pod.Spec.Volumes, pod.Spec.Volumes); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return pod, collectWarnings(pod), nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFullWorkerTemplate(t *testing.T) {
|
||||
raw, err := os.ReadFile(filepath.Join("../../../../..", "k8s/worker-images/full/pod-template.yaml"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
template, err := ParsePodTemplate(string(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pod, _, err := ComposePod(template, validProfile(""), podOptions())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ValidateAdmittedPod(pod.DeepCopy(), pod, DefaultMainContainerName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(pod.Spec.Containers) != 2 {
|
||||
t.Fatal("expected main and per-Pod daemon")
|
||||
}
|
||||
main, daemon := pod.Spec.Containers[0], pod.Spec.Containers[1]
|
||||
if main.SecurityContext == nil || main.SecurityContext.RunAsUser == nil || *main.SecurityContext.RunAsUser != 1000 {
|
||||
t.Fatal("main must remain non-root")
|
||||
}
|
||||
if daemon.SecurityContext == nil || daemon.SecurityContext.Privileged == nil || !*daemon.SecurityContext.Privileged {
|
||||
t.Fatal("daemon privilege must be explicit")
|
||||
}
|
||||
if len(daemon.Command) == 0 || daemon.Command[0] != "/bin/sh" || strings.Contains(strings.Join(daemon.Args, " "), "tcp://") {
|
||||
t.Fatal("daemon must bypass inherited TCP entrypoint")
|
||||
}
|
||||
if pod.Spec.HostNetwork || pod.Spec.HostPID || pod.Spec.HostIPC {
|
||||
t.Fatal("host namespaces enabled")
|
||||
}
|
||||
for _, v := range pod.Spec.Volumes {
|
||||
if v.HostPath != nil {
|
||||
t.Fatal("hostPath enabled")
|
||||
}
|
||||
}
|
||||
if *pod.Spec.AutomountServiceAccountToken {
|
||||
t.Fatal("service-account token enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFullWorkerTemplateRequiresImmutableImage(t *testing.T) {
|
||||
script := "../../../../../k8s/worker-images/full/render-template.sh"
|
||||
for _, image := range []string{"", "FULL_WORKER_IMAGE_REQUIRED", "example/worker:latest"} {
|
||||
if output, err := exec.Command("bash", script, image).CombinedOutput(); err == nil {
|
||||
t.Fatalf("accepted mutable image: %s", output)
|
||||
}
|
||||
}
|
||||
image := "example/worker@sha256:" + strings.Repeat("a", 64)
|
||||
output, err := exec.Command("bash", script, image).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("render failed: %v %s", err, output)
|
||||
}
|
||||
if !strings.Contains(string(output), image) || strings.Contains(string(output), "FULL_WORKER_IMAGE_REQUIRED") {
|
||||
t.Fatal("image substitution failed")
|
||||
}
|
||||
}
|
||||
@@ -147,9 +147,14 @@ func validatePodSpecFields(spec *corev1.PodSpec) error {
|
||||
|
||||
func validateTemplateContainers(spec *corev1.PodSpec, mainContainer string) error {
|
||||
mainIndex := -1
|
||||
names := make(map[string]bool)
|
||||
for i := range spec.Containers {
|
||||
container := &spec.Containers[i]
|
||||
path := fmt.Sprintf(".template.spec.containers[%d]", i)
|
||||
if names[container.Name] {
|
||||
return fieldError(podTemplateFieldPrefix+path+".name", "duplicates a container name")
|
||||
}
|
||||
names[container.Name] = true
|
||||
if container.Image == "" {
|
||||
return fieldError(podTemplateFieldPrefix+path+".image", "is required")
|
||||
}
|
||||
@@ -162,6 +167,13 @@ func validateTemplateContainers(spec *corev1.PodSpec, mainContainer string) erro
|
||||
return err
|
||||
}
|
||||
}
|
||||
if container.Name != mainContainer {
|
||||
checked, _, err := companionWithoutWorkspaceGrant(container)
|
||||
if err != nil {
|
||||
return fieldError(podTemplateFieldPrefix+path+".volumeMounts", err.Error())
|
||||
}
|
||||
container = checked
|
||||
}
|
||||
if err := validateContainerReservedFields(container, path); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func TestWorkspaceGrantAdmissionRejectsClaimAlias(t *testing.T) {
|
||||
for _, mode := range []WorkspaceMode{WorkspaceModeManagedPVC, WorkspaceModeExistingClaim} {
|
||||
t.Run(string(mode), func(t *testing.T) {
|
||||
profile := validProfile("")
|
||||
profile.Workspace.Mode = mode
|
||||
profile.Workspace.Size = "1Gi"
|
||||
profile.Workspace.AccessModes = []string{"ReadWriteOnce"}
|
||||
if mode == WorkspaceModeExistingClaim {
|
||||
profile.Workspace.ClaimName = "existing-workspace"
|
||||
profile.Workspace.Size = ""
|
||||
profile.Workspace.AccessModes = nil
|
||||
}
|
||||
desired, _, err := ComposePod(workspaceGrantTemplate(t, true), profile, podOptions())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var workspaceClaim string
|
||||
for _, volume := range desired.Spec.Volumes {
|
||||
if volume.Name == WorkspaceVolumeName {
|
||||
workspaceClaim = volume.PersistentVolumeClaim.ClaimName
|
||||
}
|
||||
}
|
||||
for _, claim := range []string{workspaceClaim, "unrelated-cache"} {
|
||||
admitted := desired.DeepCopy()
|
||||
admitted.Spec.Volumes = append(admitted.Spec.Volumes, corev1.Volume{
|
||||
Name: "companion-cache",
|
||||
VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
|
||||
ClaimName: claim,
|
||||
}},
|
||||
})
|
||||
admitted.Spec.Containers[1].VolumeMounts = append(admitted.Spec.Containers[1].VolumeMounts,
|
||||
corev1.VolumeMount{Name: "companion-cache", MountPath: "/cache", ReadOnly: false})
|
||||
err := ValidateAdmittedPod(admitted, desired, DefaultMainContainerName)
|
||||
if claim == workspaceClaim && err == nil {
|
||||
t.Error("accepted a writable alias of the read-only workspace grant")
|
||||
}
|
||||
if claim != workspaceClaim && err != nil {
|
||||
t.Errorf("unrelated claim rejected: %v", err)
|
||||
}
|
||||
template := workspaceGrantTemplate(t, true)
|
||||
template.Template.Spec.Volumes = []corev1.Volume{admitted.Spec.Volumes[len(admitted.Spec.Volumes)-1]}
|
||||
template.Template.Spec.Containers[1] = admitted.Spec.Containers[1]
|
||||
_, _, err = ComposePod(template, profile, podOptions())
|
||||
if (err != nil) != (claim == workspaceClaim) {
|
||||
t.Errorf("composition for claim %q: %v", claim, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
pathpkg "path"
|
||||
"reflect"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func validateWorkspaceClaimAliases(volumes, desired []corev1.Volume) error {
|
||||
var claim string
|
||||
for _, volume := range desired {
|
||||
if volume.Name == WorkspaceVolumeName && volume.PersistentVolumeClaim != nil {
|
||||
claim = volume.PersistentVolumeClaim.ClaimName
|
||||
break
|
||||
}
|
||||
}
|
||||
if claim == "" {
|
||||
return nil
|
||||
}
|
||||
for _, volume := range volumes {
|
||||
if volume.Name != WorkspaceVolumeName && volume.PersistentVolumeClaim != nil &&
|
||||
volume.PersistentVolumeClaim.ClaimName == claim {
|
||||
return errors.New("workspace claim must only be referenced by the owned workspace volume")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// companionWithoutWorkspaceGrant validates and removes the one permitted grant
|
||||
// from a copy, leaving all other fields subject to the reserved-field checks.
|
||||
func companionWithoutWorkspaceGrant(container *corev1.Container) (*corev1.Container, *corev1.VolumeMount, error) {
|
||||
copy := container.DeepCopy()
|
||||
copy.VolumeMounts = nil
|
||||
var grant *corev1.VolumeMount
|
||||
for _, mount := range container.VolumeMounts {
|
||||
if mount.Name == WorkspaceVolumeName {
|
||||
allowed := corev1.VolumeMount{Name: WorkspaceVolumeName, MountPath: WorkspaceMountPath, ReadOnly: mount.ReadOnly}
|
||||
if grant != nil || !reflect.DeepEqual(mount, allowed) {
|
||||
return nil, nil, errors.New("workspace grant must be one direct RO/RW mount at /workspace")
|
||||
}
|
||||
grant = &allowed
|
||||
} else {
|
||||
copy.VolumeMounts = append(copy.VolumeMounts, mount)
|
||||
}
|
||||
}
|
||||
if grant != nil {
|
||||
for _, mount := range copy.VolumeMounts {
|
||||
path := pathpkg.Clean(mount.MountPath)
|
||||
if path == "/" || isPathAtOrBelow(WorkspaceMountPath, path) || isPathAtOrBelow(path, WorkspaceMountPath) {
|
||||
return nil, nil, errors.New("workspace grant overlaps another mount")
|
||||
}
|
||||
}
|
||||
}
|
||||
return copy, grant, nil
|
||||
}
|
||||
|
||||
func validateAdmittedWorkspaceGrants(admitted, desired *corev1.Pod, mainContainer string) error {
|
||||
seen := make(map[string]bool)
|
||||
for i := range admitted.Spec.Containers {
|
||||
container := &admitted.Spec.Containers[i]
|
||||
if seen[container.Name] {
|
||||
return errors.New("admitted Pod has duplicate container identities")
|
||||
}
|
||||
seen[container.Name] = true
|
||||
if container.Name == mainContainer {
|
||||
continue
|
||||
}
|
||||
_, grant, err := companionWithoutWorkspaceGrant(container)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
requested := admittedContainerByName(desired.Spec.Containers, container.Name)
|
||||
var expected *corev1.VolumeMount
|
||||
if requested != nil {
|
||||
_, expected, err = companionWithoutWorkspaceGrant(requested)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(grant, expected) {
|
||||
return fmt.Errorf("admitted Pod changed workspace grant for %q", container.Name)
|
||||
}
|
||||
}
|
||||
for i := range desired.Spec.Containers {
|
||||
container := &desired.Spec.Containers[i]
|
||||
if container.Name == mainContainer {
|
||||
continue
|
||||
}
|
||||
_, grant, err := companionWithoutWorkspaceGrant(container)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if grant != nil && !seen[container.Name] {
|
||||
return errors.New("admitted Pod removed a workspace recipient")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package kubernetes
|
||||
|
||||
import (
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func workspaceGrantTemplate(t *testing.T, readOnly bool) *corev1.PodTemplate {
|
||||
t.Helper()
|
||||
template, err := ParsePodTemplate(validPodTemplate(""))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
template.Template.Spec.Containers = append(template.Template.Spec.Containers, corev1.Container{Name: "docker-engine", Image: "example/daemon:v1", VolumeMounts: []corev1.VolumeMount{{Name: WorkspaceVolumeName, MountPath: WorkspaceMountPath, ReadOnly: readOnly}}})
|
||||
return template
|
||||
}
|
||||
|
||||
func TestWorkspaceGrantTemplateValidation(t *testing.T) {
|
||||
for _, ro := range []bool{false, true} {
|
||||
if err := ValidatePodTemplate(workspaceGrantTemplate(t, ro), DefaultMainContainerName); err != nil {
|
||||
t.Errorf("readOnly=%v: %v", ro, err)
|
||||
}
|
||||
}
|
||||
mutations := map[string]func(*corev1.PodSpec){
|
||||
"redirect": func(s *corev1.PodSpec) { s.Containers[1].VolumeMounts[0].MountPath = "/other" },
|
||||
"normalized path": func(s *corev1.PodSpec) { s.Containers[1].VolumeMounts[0].MountPath = "/workspace/" },
|
||||
"subpath": func(s *corev1.PodSpec) { s.Containers[1].VolumeMounts[0].SubPath = "repo" },
|
||||
"expression": func(s *corev1.PodSpec) { s.Containers[1].VolumeMounts[0].SubPathExpr = "$(REPO)" },
|
||||
"propagation": func(s *corev1.PodSpec) {
|
||||
v := corev1.MountPropagationNone
|
||||
s.Containers[1].VolumeMounts[0].MountPropagation = &v
|
||||
},
|
||||
"recursive": func(s *corev1.PodSpec) {
|
||||
v := corev1.RecursiveReadOnlyDisabled
|
||||
s.Containers[1].VolumeMounts[0].RecursiveReadOnly = &v
|
||||
},
|
||||
"duplicate": func(s *corev1.PodSpec) {
|
||||
s.Containers[1].VolumeMounts = append(s.Containers[1].VolumeMounts, s.Containers[1].VolumeMounts[0])
|
||||
},
|
||||
"overlap": func(s *corev1.PodSpec) {
|
||||
s.Containers[1].VolumeMounts = append(s.Containers[1].VolumeMounts, corev1.VolumeMount{Name: "extra", MountPath: "/workspace/cache"})
|
||||
},
|
||||
"ancestor": func(s *corev1.PodSpec) {
|
||||
s.Containers[1].VolumeMounts = append(s.Containers[1].VolumeMounts, corev1.VolumeMount{Name: "extra", MountPath: "/"})
|
||||
},
|
||||
"auth": func(s *corev1.PodSpec) { s.Containers[1].VolumeMounts[0].Name = AuthVolumeName },
|
||||
"runtime": func(s *corev1.PodSpec) { s.Containers[1].VolumeMounts[0].Name = RuntimeVolumeName },
|
||||
"volume definition": func(s *corev1.PodSpec) { s.Volumes = append(s.Volumes, corev1.Volume{Name: WorkspaceVolumeName}) },
|
||||
"device": func(s *corev1.PodSpec) {
|
||||
s.Containers[1].VolumeDevices = []corev1.VolumeDevice{{Name: WorkspaceVolumeName, DevicePath: "/dev/workspace"}}
|
||||
},
|
||||
"init": func(s *corev1.PodSpec) {
|
||||
s.InitContainers = []corev1.Container{s.Containers[1]}
|
||||
s.Containers = s.Containers[:1]
|
||||
},
|
||||
"ephemeral": func(s *corev1.PodSpec) {
|
||||
s.EphemeralContainers = []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon(s.Containers[1])}}
|
||||
s.Containers = s.Containers[:1]
|
||||
},
|
||||
"main": func(s *corev1.PodSpec) {
|
||||
s.Containers[0].VolumeMounts = s.Containers[1].VolumeMounts
|
||||
s.Containers = s.Containers[:1]
|
||||
},
|
||||
"duplicate recipient": func(s *corev1.PodSpec) { s.Containers = append(s.Containers, s.Containers[1]) },
|
||||
}
|
||||
for name, mutate := range mutations {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
template := workspaceGrantTemplate(t, false)
|
||||
mutate(&template.Template.Spec)
|
||||
if err := ValidatePodTemplate(template, DefaultMainContainerName); err == nil {
|
||||
t.Fatal("accepted invalid grant")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceGrantComposition(t *testing.T) {
|
||||
for _, mode := range []WorkspaceMode{WorkspaceModeEmptyDir, WorkspaceModeManagedPVC, WorkspaceModeExistingClaim} {
|
||||
for _, ro := range []bool{false, true} {
|
||||
template := workspaceGrantTemplate(t, ro)
|
||||
original := template.DeepCopy()
|
||||
profile := validProfile("")
|
||||
profile.Workspace.Mode = mode
|
||||
if mode == WorkspaceModeManagedPVC {
|
||||
profile.Workspace.Size = "1Gi"
|
||||
profile.Workspace.AccessModes = []string{"ReadWriteOnce"}
|
||||
}
|
||||
if mode == WorkspaceModeExistingClaim {
|
||||
profile.Workspace.ClaimName = "existing-workspace"
|
||||
profile.Workspace.Size = ""
|
||||
profile.Workspace.AccessModes = nil
|
||||
}
|
||||
pod, _, err := ComposePod(template, profile, podOptions())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(pod.Spec.Containers[1], original.Template.Spec.Containers[1]) || !reflect.DeepEqual(template, original) {
|
||||
t.Fatal("grant or input changed")
|
||||
}
|
||||
if err := ValidateAdmittedPod(pod.DeepCopy(), pod, DefaultMainContainerName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkspaceGrantAdmission(t *testing.T) {
|
||||
mutations := map[string]func(*corev1.Pod){
|
||||
"removed mount": func(p *corev1.Pod) { p.Spec.Containers[1].VolumeMounts = nil },
|
||||
"removed recipient": func(p *corev1.Pod) { p.Spec.Containers = p.Spec.Containers[:1] },
|
||||
"renamed": func(p *corev1.Pod) { p.Spec.Containers[1].Name = "other" },
|
||||
"mode": func(p *corev1.Pod) { p.Spec.Containers[1].VolumeMounts[0].ReadOnly = true },
|
||||
"duplicate mount": func(p *corev1.Pod) {
|
||||
p.Spec.Containers[1].VolumeMounts = append(p.Spec.Containers[1].VolumeMounts, p.Spec.Containers[1].VolumeMounts[0])
|
||||
},
|
||||
"duplicate recipient": func(p *corev1.Pod) { p.Spec.Containers = append(p.Spec.Containers, p.Spec.Containers[1]) },
|
||||
"added": func(p *corev1.Pod) {
|
||||
c := *p.Spec.Containers[1].DeepCopy()
|
||||
c.Name = "other"
|
||||
p.Spec.Containers = append(p.Spec.Containers, c)
|
||||
},
|
||||
"init transfer": func(p *corev1.Pod) {
|
||||
p.Spec.InitContainers = []corev1.Container{p.Spec.Containers[1]}
|
||||
p.Spec.Containers = p.Spec.Containers[:1]
|
||||
},
|
||||
"ephemeral transfer": func(p *corev1.Pod) {
|
||||
p.Spec.EphemeralContainers = []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon(p.Spec.Containers[1])}}
|
||||
p.Spec.Containers = p.Spec.Containers[:1]
|
||||
},
|
||||
}
|
||||
desired, _ := admittedPodFixture(t)
|
||||
desired.Spec.Containers = append(desired.Spec.Containers, workspaceGrantTemplate(t, false).Template.Spec.Containers[1])
|
||||
if err := ValidateAdmittedPod(desired.DeepCopy(), desired, DefaultMainContainerName); err != nil {
|
||||
t.Errorf("unchanged grant: %v", err)
|
||||
}
|
||||
for name, mutate := range mutations {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
admitted := desired.DeepCopy()
|
||||
mutate(admitted)
|
||||
if err := ValidateAdmittedPod(admitted, desired, DefaultMainContainerName); err == nil {
|
||||
t.Fatal("accepted mutated grant")
|
||||
}
|
||||
})
|
||||
}
|
||||
admitted := desired.DeepCopy()
|
||||
admitted.Spec.Containers = append(admitted.Spec.Containers, corev1.Container{Name: "metrics", Image: "example/metrics:v1"})
|
||||
admitted.Spec.Containers[0], admitted.Spec.Containers[1] = admitted.Spec.Containers[1], admitted.Spec.Containers[0]
|
||||
if err := ValidateAdmittedPod(admitted, desired, DefaultMainContainerName); err != nil {
|
||||
t.Errorf("compatible addition/reordering: %v", err)
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
|
||||
"github.com/stretchr/testify/require"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
func TestKubernetesWorkspaceGrantUsesLaunchSnapshot(t *testing.T) {
|
||||
t.Run("retains recorded grant", func(t *testing.T) { assertKubernetesWorkspaceGrantSnapshot(t, true) })
|
||||
t.Run("profile cannot add grant", func(t *testing.T) { assertKubernetesWorkspaceGrantSnapshot(t, false) })
|
||||
}
|
||||
|
||||
func assertKubernetesWorkspaceGrantSnapshot(t *testing.T, initialGrant bool) {
|
||||
t.Helper()
|
||||
controlPort := startKubernetesAgentctlServer(t, true, 41001)
|
||||
instancePort := startKubernetesAgentctlServer(t, false, 0)
|
||||
resources := &fakeKubernetesResources{}
|
||||
initial := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{
|
||||
uint16(kubeexecutor.DefaultAgentctlPort): controlPort,
|
||||
41001: instancePort,
|
||||
})
|
||||
initialRequest := validKubernetesCreateRequest()
|
||||
setManagedKubernetesWorkspace(initialRequest)
|
||||
raw := initialRequest.Metadata[MetadataKeyKubernetesPodTemplateYAML].(string)
|
||||
template, err := kubeexecutor.ParsePodTemplate(raw)
|
||||
require.NoError(t, err)
|
||||
grant := corev1.Container{Name: "docker-engine", Image: "example/daemon:v1", VolumeMounts: []corev1.VolumeMount{{Name: kubeexecutor.WorkspaceVolumeName, MountPath: kubeexecutor.WorkspaceMountPath, ReadOnly: true}}}
|
||||
if initialGrant {
|
||||
template.Template.Spec.Containers = append(template.Template.Spec.Containers, grant)
|
||||
}
|
||||
encoded, err := json.Marshal(template)
|
||||
require.NoError(t, err)
|
||||
initialRequest.Metadata[MetadataKeyKubernetesPodTemplateYAML] = string(encoded)
|
||||
created, err := initial.CreateInstance(context.Background(), initialRequest)
|
||||
require.NoError(t, err)
|
||||
resources.mu.Lock()
|
||||
originalVolumes := resources.pod.DeepCopy().Spec.Volumes
|
||||
originalClaimUID := resources.pvc.UID
|
||||
resources.pod = nil
|
||||
resources.nextPodUID = "replacement-pod-uid"
|
||||
resources.mu.Unlock()
|
||||
|
||||
restartControlPort := startKubernetesAgentctlServer(t, true, 41002)
|
||||
restartInstancePort := startKubernetesAgentctlServer(t, false, 0)
|
||||
restarted := newFakeKubernetesExecutor(t, resources, &recordingKubernetesExec{}, map[uint16]uint16{
|
||||
uint16(kubeexecutor.DefaultAgentctlPort): restartControlPort,
|
||||
41002: restartInstancePort,
|
||||
})
|
||||
reconnectRequest := kubernetesReconnectRequest(created)
|
||||
setManagedKubernetesWorkspace(reconnectRequest)
|
||||
reconnectRequest.Metadata[MetadataKeyKubernetesProfilePlatform] = "not-a-platform"
|
||||
if initialGrant {
|
||||
template.Template.Spec.Containers = template.Template.Spec.Containers[:1]
|
||||
} else {
|
||||
template.Template.Spec.Containers = append(template.Template.Spec.Containers, grant)
|
||||
}
|
||||
edited, err := json.Marshal(template)
|
||||
require.NoError(t, err)
|
||||
reconnectRequest.Metadata[MetadataKeyKubernetesPodTemplateYAML] = string(edited)
|
||||
|
||||
reconnected, err := restarted.CreateInstance(context.Background(), reconnectRequest)
|
||||
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "replacement-pod-uid", reconnected.Metadata[MetadataKeyKubernetesPodUID])
|
||||
require.Len(t, resources.createdPods, 2)
|
||||
require.Equal(t, "amd64", resources.createdPods[1].Spec.NodeSelector[corev1.LabelArchStable],
|
||||
"replacement must use the recorded launch platform, not the edited profile")
|
||||
require.Equal(t, "example.test/agent:latest", resources.createdPods[1].Spec.Containers[0].Image,
|
||||
"replacement must use the recorded Pod template")
|
||||
if initialGrant {
|
||||
require.Equal(t, grant, resources.createdPods[1].Spec.Containers[1])
|
||||
} else {
|
||||
require.Len(t, resources.createdPods[1].Spec.Containers, 1)
|
||||
}
|
||||
require.Equal(t, originalVolumes, resources.createdPods[1].Spec.Volumes)
|
||||
require.Equal(t, originalClaimUID, resources.pvc.UID)
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFullWorkerPreparationContract(t *testing.T) {
|
||||
raw, err := os.ReadFile("../../../../../../k8s/worker-images/full/prepare.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
script := string(raw)
|
||||
for _, part := range []string{"{{repository.clone_url}}", "{{repository.setup_script}}", "{{kandev.agents.install}}", "retained workspace repository origin", "non-empty but is not a valid checkout"} {
|
||||
if !strings.Contains(script, part) {
|
||||
t.Fatalf("missing preparation contract %s", part)
|
||||
}
|
||||
}
|
||||
if strings.Index(script, "docker info") > strings.Index(script, "git clone") {
|
||||
t.Fatal("daemon wait must precede clone")
|
||||
}
|
||||
if strings.Index(script, "create_workspace_caches\n") > strings.Index(script, "{{repository.setup_script}}") {
|
||||
t.Fatal("caches must be writable before source setup")
|
||||
}
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "docker"), []byte("#!/bin/sh\nexit 1\n"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "sh", "-c", script)
|
||||
cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), "DOCKER_READY_TIMEOUT_SECONDS=1")
|
||||
output, err := cmd.CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(output), "Docker daemon did not become ready") {
|
||||
t.Fatalf("readiness failure: %v %s", err, output)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("preparation exceeded its own readiness deadline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace(t *testing.T) {
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("git not available")
|
||||
}
|
||||
_, origin := setupPostludeRepo(t, "main")
|
||||
root := t.TempDir()
|
||||
raw, err := os.ReadFile("../../../../../../k8s/worker-images/full/prepare.sh")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bin := filepath.Join(root, "bin")
|
||||
if err := os.MkdirAll(bin, 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(bin, "docker"), []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
workspace := filepath.Join(root, "workspace")
|
||||
if err := os.MkdirAll(filepath.Join(workspace, "lost+found"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
home := filepath.Join(root, "home")
|
||||
if err := os.MkdirAll(home, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cloneTmp := filepath.Join(root, "runtime", "workspace-clone")
|
||||
if err := os.MkdirAll(filepath.Dir(cloneTmp), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
script := strings.NewReplacer(
|
||||
"{{git.identity_setup}}", ":",
|
||||
"{{github.auth_setup}}", ":",
|
||||
"{{repository.branch}}", shellQuote("main"),
|
||||
"{{repository.clone_url}}", shellQuote(origin),
|
||||
"{{workspace.path}}", shellQuote(workspace),
|
||||
"{{repository.setup_script}}", `test -d "$workspace/.cache/go-build" && test -d "$workspace/.npm-global"`,
|
||||
"{{kandev.agents.install}}", ":",
|
||||
"/opt/kandev/.workspace-clone", cloneTmp,
|
||||
).Replace(string(raw))
|
||||
run := func() {
|
||||
t.Helper()
|
||||
cmd := exec.Command("sh", "-eu", "-c", script)
|
||||
cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+bin+":"+os.Getenv("PATH"))
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
t.Fatalf("Kubernetes prepare failed: %v\n%s", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
run()
|
||||
localFile := filepath.Join(workspace, "local-untracked.txt")
|
||||
if err := os.WriteFile(localFile, []byte("preserve me"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run()
|
||||
|
||||
if data, err := os.ReadFile(localFile); err != nil || string(data) != "preserve me" {
|
||||
t.Fatalf("retained workspace data = %q, %v", data, err)
|
||||
}
|
||||
if output := runIn(t, workspace, "git", "branch", "--show-current"); strings.TrimSpace(string(output)) != "main" {
|
||||
t.Fatalf("workspace branch = %q", output)
|
||||
}
|
||||
}
|
||||
@@ -774,3 +774,25 @@ Record results as `{ "workers": 2, "wall_seconds": 0, "max_rss_kb": 0,
|
||||
"retries": 0, "backend_errors": 0 }`. Compare at least three repetitions
|
||||
with the same build, duration-aware manifest, and profile before considering a
|
||||
default change.
|
||||
|
||||
### Full worker Docker lifecycle acceptance
|
||||
|
||||
`tests/kubernetes/kubernetes-docker-workloads.spec.ts` is an explicit opt-in
|
||||
`containers` suite. First build and verify the [full worker recipe](../../../k8s/worker-images/full/README.md)
|
||||
on an isolated host with adequate disk/I/O capacity, then pass the exact printed
|
||||
local `IMAGE_ID` as `KANDEV_E2E_FULL_WORKER_IMAGE`:
|
||||
|
||||
```bash
|
||||
KANDEV_E2E_FULL_WORKER_IMAGE=sha256:<verified-local-image-id> \
|
||||
KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --host --shards 1 --project containers tests/kubernetes/kubernetes-docker-workloads.spec.ts
|
||||
```
|
||||
|
||||
The fixture creates a never-started container to add the mock transport, loads
|
||||
that exact derived image into its disposable Kind cluster, and caps the test
|
||||
node at two CPUs/eight GiB before starting Docker workloads. Existing Kind
|
||||
provisioning still requires an isolated bounded test host. No credentials are
|
||||
needed. The tests cover real source/browser/Compose results through the terminal,
|
||||
finite preparation failure, retained/replaced workspaces, independent daemons,
|
||||
Pod cgroup ancestry and exact cleanup including an untouched existing claim.
|
||||
The cgroup case fails if nested work escapes the Pod budget. Missing image input
|
||||
skips the suite; skips and test discovery do not count as execution acceptance.
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { kubernetesTest } from "./kubernetes-test-base";
|
||||
|
||||
const ROOT = path.resolve(__dirname, "../../../..");
|
||||
const RECIPE = path.join(ROOT, "k8s/worker-images/full");
|
||||
|
||||
export function fullWorkerPrepare(): string {
|
||||
return fs.readFileSync(path.join(RECIPE, "prepare.sh"), "utf8");
|
||||
}
|
||||
|
||||
export function fullWorkerTemplate(image: string, unavailable = false): string {
|
||||
let template = fs
|
||||
.readFileSync(path.join(RECIPE, "pod-template.yaml"), "utf8")
|
||||
.replace("FULL_WORKER_IMAGE_REQUIRED", image)
|
||||
.replace("imagePullPolicy: IfNotPresent", "imagePullPolicy: Never");
|
||||
if (unavailable) template = template.replace(/exec dockerd[^\n]+/, "exec sleep 3600");
|
||||
return template;
|
||||
}
|
||||
|
||||
export const test = kubernetesTest.extend<object, { fullWorkerImage: string }>({
|
||||
fullWorkerImage: [
|
||||
async ({ cluster }, use) => {
|
||||
const image = process.env.KANDEV_E2E_FULL_WORKER_IMAGE ?? "";
|
||||
if (!/^sha256:[a-f0-9]{64}$/.test(image))
|
||||
throw new Error("KANDEV_E2E_FULL_WORKER_IMAGE must be the verified local image ID");
|
||||
const node = `${cluster.name}-control-plane`;
|
||||
execFileSync("docker", ["update", "--cpus=2", "--memory=8g", "--memory-swap=8g", node], {
|
||||
timeout: 30_000,
|
||||
});
|
||||
const limits = execFileSync(
|
||||
"docker",
|
||||
["inspect", "--format", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", node],
|
||||
{ encoding: "utf8", timeout: 30_000 },
|
||||
).trim();
|
||||
if (limits !== "2000000000 8589934592")
|
||||
throw new Error("Kind workload resource limits were not applied");
|
||||
const suffix = randomUUID();
|
||||
const container = `kandev-full-harness-${suffix}`;
|
||||
const tag = `kandev-full-harness:${suffix}`;
|
||||
let created = false;
|
||||
try {
|
||||
// Copy the mock transport into a never-started container; no tool rebuild
|
||||
// or mutable image input is needed for this fixture-only layer.
|
||||
execFileSync(
|
||||
"docker",
|
||||
["create", "--name", container, "--network=none", "--memory=128m", "--cpus=1", image],
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
created = true;
|
||||
execFileSync(
|
||||
"docker",
|
||||
[
|
||||
"cp",
|
||||
path.join(ROOT, "apps/backend/bin/mock-agent-linux-amd64"),
|
||||
`${container}:/usr/local/bin/mock-agent`,
|
||||
],
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
execFileSync("docker", ["commit", container, tag], { timeout: 60_000 });
|
||||
execFileSync(cluster.kindBin, ["load", "docker-image", tag, "--name", cluster.name], {
|
||||
timeout: 300_000,
|
||||
});
|
||||
await use(tag);
|
||||
} finally {
|
||||
if (created) execFileSync("docker", ["rm", container], { timeout: 30_000 });
|
||||
const found = execFileSync("docker", ["image", "ls", "--quiet", tag], {
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
}).trim();
|
||||
if (found) execFileSync("docker", ["image", "rm", tag], { timeout: 60_000 });
|
||||
}
|
||||
},
|
||||
{ scope: "worker", timeout: 480_000 },
|
||||
],
|
||||
});
|
||||
export { expect } from "@playwright/test";
|
||||
@@ -1146,6 +1146,14 @@ export class ApiClient {
|
||||
});
|
||||
}
|
||||
|
||||
async updateExecutorProfile(
|
||||
executorId: string,
|
||||
profileId: string,
|
||||
updates: { config: Record<string, string> },
|
||||
): Promise<void> {
|
||||
await this.request("PATCH", `/api/v1/executors/${executorId}/profiles/${profileId}`, updates);
|
||||
}
|
||||
|
||||
async deleteExecutorProfile(profileId: string): Promise<void> {
|
||||
await this.request("DELETE", `/api/v1/executor-profiles/${profileId}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import path from "node:path";
|
||||
import {
|
||||
test,
|
||||
expect,
|
||||
fullWorkerTemplate,
|
||||
fullWorkerPrepare,
|
||||
} from "../../fixtures/kubernetes-docker-test-base";
|
||||
import { kubernetesProfileConfig } from "../../fixtures/kubernetes-test-base";
|
||||
import type { ApiClient } from "../../helpers/api-client";
|
||||
import type { KubernetesCluster } from "../../fixtures/kubernetes-tools";
|
||||
import {
|
||||
waitForKubernetesPod,
|
||||
waitForKubernetesPVC,
|
||||
waitForKubernetesResourceAbsent,
|
||||
waitForTaskSessionState,
|
||||
waitForFailedSession,
|
||||
} from "../../helpers/kubernetes";
|
||||
import { SessionPage } from "../../pages/session-page";
|
||||
|
||||
// A skipped opt-in test is never Docker acceptance evidence.
|
||||
test.skip(
|
||||
!process.env.KANDEV_E2E_FULL_WORKER_IMAGE,
|
||||
"Requires an explicitly built full worker image ID",
|
||||
);
|
||||
|
||||
function run(cluster: KubernetesCluster, pod: string, script: string, container = "kandev-agent") {
|
||||
return cluster
|
||||
.kubectl(["-n", cluster.namespace, "exec", pod, "-c", container, "--", "sh", "-ceu", script], {
|
||||
timeoutMs: 600_000,
|
||||
})
|
||||
.trim();
|
||||
}
|
||||
|
||||
async function archive(
|
||||
api: ApiClient,
|
||||
cluster: KubernetesCluster,
|
||||
task: { id: string },
|
||||
pod?: string,
|
||||
claim?: string,
|
||||
) {
|
||||
await api.archiveTask(task.id);
|
||||
if (pod) await waitForKubernetesResourceAbsent(cluster, "pod", pod);
|
||||
if (claim) await waitForKubernetesResourceAbsent(cluster, "persistentvolumeclaim", claim);
|
||||
}
|
||||
|
||||
test("full worker returns source and Docker results", async ({
|
||||
apiClient,
|
||||
cluster,
|
||||
seedData,
|
||||
fullWorkerImage,
|
||||
testPage,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(900_000);
|
||||
const profile = await apiClient.createExecutorProfile(seedData.executorId, {
|
||||
name: "Full worker source acceptance",
|
||||
config: kubernetesProfileConfig(cluster, {
|
||||
pod_template_yaml: fullWorkerTemplate(fullWorkerImage),
|
||||
"workspace.mode": "managed_pvc",
|
||||
"workspace.size": "2Gi",
|
||||
"workspace.access_modes": '["ReadWriteOnce"]',
|
||||
}),
|
||||
prepare_script: fullWorkerPrepare(),
|
||||
cleanup_script: "",
|
||||
env_vars: [],
|
||||
});
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
"Full worker source results",
|
||||
seedData.agentProfileId,
|
||||
{
|
||||
description: 'e2e:message("ready")',
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
executor_id: seedData.executorId,
|
||||
executor_profile_id: profile.id,
|
||||
},
|
||||
);
|
||||
let podName: string | undefined;
|
||||
let claimName: string | undefined;
|
||||
try {
|
||||
const pod = await waitForKubernetesPod(cluster, task.id, task.session_id!);
|
||||
podName = pod.metadata.name;
|
||||
const claim = await waitForKubernetesPVC(cluster, task.id, task.session_id!);
|
||||
claimName = claim.metadata.name;
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "WAITING_FOR_INPUT");
|
||||
const output = run(cluster, podName, "bash /opt/full-worker/smoke.sh --all");
|
||||
expect(output).toContain("source-browser-docker-compose-ok");
|
||||
expect(run(cluster, podName, "id -u")).toBe("1000");
|
||||
expect(
|
||||
run(
|
||||
cluster,
|
||||
podName,
|
||||
"test -S /run/docker/docker.sock; cat /workspace/full-worker-result.txt",
|
||||
),
|
||||
).toBe("source-browser-docker-compose-ok");
|
||||
expect(run(cluster, podName, "cat /proc/1/cmdline", "docker-engine")).not.toContain("tcp://");
|
||||
await testPage.goto(`/t/${task.id}`);
|
||||
const session = new SessionPage(testPage);
|
||||
await session.waitForLoad(30_000);
|
||||
await session.clickTab("Terminal");
|
||||
await session.typeInTerminal("cat /workspace/full-worker-result.txt");
|
||||
await session.expectTerminalHasText("source-browser-docker-compose-ok");
|
||||
await testInfo.attach("full-worker-source-results", {
|
||||
body: output,
|
||||
contentType: "text/plain",
|
||||
});
|
||||
} finally {
|
||||
await archive(apiClient, cluster, task, podName, claimName);
|
||||
}
|
||||
});
|
||||
|
||||
test("unavailable daemon fails preparation", async ({
|
||||
apiClient,
|
||||
cluster,
|
||||
seedData,
|
||||
fullWorkerImage,
|
||||
testPage,
|
||||
}) => {
|
||||
test.setTimeout(240_000);
|
||||
const profile = await apiClient.createExecutorProfile(seedData.executorId, {
|
||||
name: "Unavailable Docker daemon",
|
||||
config: kubernetesProfileConfig(cluster, {
|
||||
pod_template_yaml: fullWorkerTemplate(fullWorkerImage, true),
|
||||
}),
|
||||
prepare_script: "export DOCKER_READY_TIMEOUT_SECONDS=2\n" + fullWorkerPrepare(),
|
||||
cleanup_script: "",
|
||||
env_vars: [],
|
||||
});
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
"Docker readiness failure",
|
||||
seedData.agentProfileId,
|
||||
{
|
||||
description: 'e2e:message("must not start")',
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
executor_id: seedData.executorId,
|
||||
executor_profile_id: profile.id,
|
||||
},
|
||||
);
|
||||
try {
|
||||
await waitForFailedSession(apiClient, task.id, /Docker daemon did not become ready/, 90_000);
|
||||
await testPage.goto(`/t/${task.id}`);
|
||||
const session = new SessionPage(testPage);
|
||||
await session.waitForLoad(30_000);
|
||||
const error = session.activeChat().getByTestId("task-launch-error-entry");
|
||||
await expect(error).toBeVisible();
|
||||
await error.getByRole("button", { name: "Show details" }).click();
|
||||
await expect(error.getByTestId("task-launch-error-details")).toContainText(
|
||||
"Docker daemon did not become ready",
|
||||
);
|
||||
} finally {
|
||||
await archive(apiClient, cluster, task);
|
||||
}
|
||||
});
|
||||
|
||||
test("resume and replacement retain workspace", async ({
|
||||
apiClient,
|
||||
cluster,
|
||||
seedData,
|
||||
fullWorkerImage,
|
||||
}) => {
|
||||
test.setTimeout(480_000);
|
||||
const config = kubernetesProfileConfig(cluster, {
|
||||
pod_template_yaml: fullWorkerTemplate(fullWorkerImage),
|
||||
"workspace.mode": "managed_pvc",
|
||||
"workspace.size": "2Gi",
|
||||
"workspace.access_modes": '["ReadWriteOnce"]',
|
||||
});
|
||||
const profile = await apiClient.createExecutorProfile(seedData.executorId, {
|
||||
name: "Full worker retention",
|
||||
config,
|
||||
prepare_script: fullWorkerPrepare(),
|
||||
cleanup_script: "",
|
||||
env_vars: [],
|
||||
});
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
"Docker workspace retention",
|
||||
seedData.agentProfileId,
|
||||
{
|
||||
description: 'e2e:message("ready")',
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
executor_id: seedData.executorId,
|
||||
executor_profile_id: profile.id,
|
||||
},
|
||||
);
|
||||
let podName: string | undefined;
|
||||
let claimName: string | undefined;
|
||||
try {
|
||||
const pod = await waitForKubernetesPod(cluster, task.id, task.session_id!);
|
||||
podName = pod.metadata.name;
|
||||
const claim = await waitForKubernetesPVC(cluster, task.id, task.session_id!);
|
||||
claimName = claim.metadata.name;
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "WAITING_FOR_INPUT");
|
||||
const daemonID = run(cluster, podName, "docker info --format '{{.ID}}'");
|
||||
run(
|
||||
cluster,
|
||||
podName,
|
||||
"printf uncommitted-result > /workspace/retained-result.txt; docker volume create retained-daemon-state",
|
||||
);
|
||||
await apiClient.stopSession({ session_id: task.session_id! });
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "CANCELLED");
|
||||
expect((await waitForKubernetesPod(cluster, task.id, task.session_id!)).metadata.uid).toBe(
|
||||
pod.metadata.uid,
|
||||
);
|
||||
expect(run(cluster, podName, "docker info --format '{{.ID}}'")).toBe(daemonID);
|
||||
await apiClient.launchSession(
|
||||
{ task_id: task.id, intent: "resume", session_id: task.session_id! },
|
||||
90_000,
|
||||
);
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "WAITING_FOR_INPUT");
|
||||
expect(run(cluster, podName, "cat /workspace/retained-result.txt")).toBe("uncommitted-result");
|
||||
await apiClient.stopSession({ session_id: task.session_id! });
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "CANCELLED");
|
||||
await apiClient.updateExecutorProfile(seedData.executorId, profile.id, {
|
||||
config: { ...config, pod_template_yaml: cluster.podTemplate() },
|
||||
});
|
||||
cluster.kubectl(["-n", cluster.namespace, "delete", "pod", podName, "--wait=true"]);
|
||||
await apiClient.launchSession(
|
||||
{ task_id: task.id, intent: "resume", session_id: task.session_id! },
|
||||
90_000,
|
||||
);
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "WAITING_FOR_INPUT");
|
||||
const replacement = await waitForKubernetesPod(cluster, task.id, task.session_id!);
|
||||
podName = replacement.metadata.name;
|
||||
expect(replacement.metadata.uid).not.toBe(pod.metadata.uid);
|
||||
expect(replacement.spec?.containers?.map((c) => c.name)).toContain("docker-engine");
|
||||
expect((await waitForKubernetesPVC(cluster, task.id, task.session_id!)).metadata.uid).toBe(
|
||||
claim.metadata.uid,
|
||||
);
|
||||
expect(run(cluster, podName, "cat /workspace/retained-result.txt")).toBe("uncommitted-result");
|
||||
expect(run(cluster, podName, "docker info --format '{{.ID}}'")).not.toBe(daemonID);
|
||||
expect(run(cluster, podName, "docker volume ls --format '{{.Name}}'")).not.toContain(
|
||||
"retained-daemon-state",
|
||||
);
|
||||
} finally {
|
||||
await archive(apiClient, cluster, task, podName, claimName);
|
||||
}
|
||||
});
|
||||
|
||||
test("isolated daemons clean up with owned Pods", async ({
|
||||
apiClient,
|
||||
cluster,
|
||||
seedData,
|
||||
fullWorkerImage,
|
||||
}, testInfo) => {
|
||||
test.setTimeout(600_000);
|
||||
const tasks: Array<{ id: string; session_id?: string }> = [];
|
||||
const pods: string[] = [];
|
||||
const claims: Array<string | undefined> = [];
|
||||
const existingClaim = `full-existing-${randomUUID()}`;
|
||||
let existingCreated = false;
|
||||
const profile = await apiClient.createExecutorProfile(seedData.executorId, {
|
||||
name: "Independent Docker daemons",
|
||||
config: kubernetesProfileConfig(cluster, {
|
||||
pod_template_yaml: fullWorkerTemplate(fullWorkerImage),
|
||||
"workspace.mode": "managed_pvc",
|
||||
"workspace.size": "2Gi",
|
||||
"workspace.access_modes": '["ReadWriteOnce"]',
|
||||
}),
|
||||
prepare_script: fullWorkerPrepare(),
|
||||
cleanup_script: "",
|
||||
env_vars: [],
|
||||
});
|
||||
try {
|
||||
cluster.kubectl(["-n", cluster.namespace, "create", "-f", "-"], {
|
||||
input: JSON.stringify({
|
||||
apiVersion: "v1",
|
||||
kind: "PersistentVolumeClaim",
|
||||
metadata: { name: existingClaim },
|
||||
spec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: "2Gi" } } },
|
||||
}),
|
||||
});
|
||||
existingCreated = true;
|
||||
const existingProfile = await apiClient.createExecutorProfile(seedData.executorId, {
|
||||
name: "Existing Docker workspace",
|
||||
config: kubernetesProfileConfig(cluster, {
|
||||
pod_template_yaml: fullWorkerTemplate(fullWorkerImage),
|
||||
"workspace.mode": "existing_claim",
|
||||
"workspace.claim_name": existingClaim,
|
||||
}),
|
||||
prepare_script: fullWorkerPrepare(),
|
||||
cleanup_script: "",
|
||||
env_vars: [],
|
||||
});
|
||||
for (const suffix of ["one", "two"]) {
|
||||
const task = await apiClient.createTaskWithAgent(
|
||||
seedData.workspaceId,
|
||||
`Docker isolation ${suffix}`,
|
||||
seedData.agentProfileId,
|
||||
{
|
||||
description: 'e2e:message("ready")',
|
||||
workflow_id: seedData.workflowId,
|
||||
workflow_step_id: seedData.startStepId,
|
||||
executor_id: seedData.executorId,
|
||||
executor_profile_id: suffix === "one" ? profile.id : existingProfile.id,
|
||||
},
|
||||
);
|
||||
tasks.push(task);
|
||||
pods.push((await waitForKubernetesPod(cluster, task.id, task.session_id!)).metadata.name);
|
||||
claims.push(
|
||||
suffix === "one"
|
||||
? (await waitForKubernetesPVC(cluster, task.id, task.session_id!)).metadata.name
|
||||
: undefined,
|
||||
);
|
||||
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "WAITING_FOR_INPUT");
|
||||
}
|
||||
const existingUID = cluster.json<{ metadata: { uid: string } }>([
|
||||
"-n",
|
||||
cluster.namespace,
|
||||
"get",
|
||||
"pvc",
|
||||
existingClaim,
|
||||
]).metadata.uid;
|
||||
const daemonIDs = pods.map((pod) => run(cluster, pod, "docker info --format '{{.ID}}'"));
|
||||
expect(daemonIDs[0]).not.toBe(daemonIDs[1]);
|
||||
run(cluster, pods[0], "docker volume create isolated-volume");
|
||||
expect(run(cluster, pods[1], "docker volume ls --format '{{.Name}}'")).not.toContain(
|
||||
"isolated-volume",
|
||||
);
|
||||
const inventory = await apiClient.listKubernetesSessions(seedData.executorId);
|
||||
for (const task of tasks)
|
||||
expect(inventory.filter((row) => row.task_id === task.id)).toHaveLength(1);
|
||||
const accounting = fs.readFileSync(
|
||||
path.resolve(__dirname, "../../../../../k8s/worker-images/full/accounting.sh"),
|
||||
"utf8",
|
||||
);
|
||||
const nestedID = run(cluster, pods[0], accounting).split("\n").at(-1)!;
|
||||
expect(nestedID).toMatch(/^[a-f0-9]{64}$/);
|
||||
const pod = cluster.json<{ metadata: { uid: string } }>([
|
||||
"-n",
|
||||
cluster.namespace,
|
||||
"get",
|
||||
"pod",
|
||||
pods[0],
|
||||
]);
|
||||
const evidence = execFileSync(
|
||||
"docker",
|
||||
[
|
||||
"exec",
|
||||
`${cluster.name}-control-plane`,
|
||||
"sh",
|
||||
"-ceu",
|
||||
`
|
||||
nested=$(find /sys/fs/cgroup -type d \\( -name '${nestedID}' -o -name 'docker-${nestedID}.scope' \\) | head -1)
|
||||
test -n "$nested"
|
||||
pod=$(find /sys/fs/cgroup -type d -name '*pod${pod.metadata.uid.replaceAll("-", "_")}*' | head -1)
|
||||
test -n "$pod"
|
||||
case "$nested" in "$pod"/*) ;; *) echo 'Nested process escaped Pod cgroup' >&2; exit 1;; esac
|
||||
test "$(cat "$pod/memory.max")" != max
|
||||
test "$(cut -d ' ' -f1 "$pod/cpu.max")" != max
|
||||
echo pod-memory-limit; cat "$pod/memory.max"
|
||||
echo nested-memory-current; cat "$nested/memory.current"
|
||||
echo nested-cpu-stat; cat "$nested/cpu.stat"
|
||||
test "$(cat "$nested/memory.current")" -ge 16777216
|
||||
`,
|
||||
],
|
||||
{ encoding: "utf8", timeout: 30_000 },
|
||||
);
|
||||
await testInfo.attach("nested-cgroup-accounting", {
|
||||
body: `${cluster.kubernetesVersion}\n${cluster.nodeImage}\n${evidence}`,
|
||||
contentType: "text/plain",
|
||||
});
|
||||
await archive(apiClient, cluster, tasks[0], pods[0], claims[0]);
|
||||
expect(run(cluster, pods[1], "docker info --format '{{.ID}}'")).toBe(daemonIDs[1]);
|
||||
const remaining = execFileSync(
|
||||
"docker",
|
||||
[
|
||||
"exec",
|
||||
`${cluster.name}-control-plane`,
|
||||
"find",
|
||||
"/sys/fs/cgroup",
|
||||
"-type",
|
||||
"d",
|
||||
"-name",
|
||||
`*${nestedID}*`,
|
||||
],
|
||||
{ encoding: "utf8", timeout: 30_000 },
|
||||
);
|
||||
expect(remaining.trim()).toBe("");
|
||||
await archive(apiClient, cluster, tasks[1], pods[1]);
|
||||
expect(
|
||||
cluster.json<{ metadata: { uid: string } }>([
|
||||
"-n",
|
||||
cluster.namespace,
|
||||
"get",
|
||||
"pvc",
|
||||
existingClaim,
|
||||
]).metadata.uid,
|
||||
).toBe(existingUID);
|
||||
} finally {
|
||||
try {
|
||||
for (let i = 0; i < tasks.length; i++)
|
||||
await archive(apiClient, cluster, tasks[i], pods[i], claims[i]);
|
||||
} finally {
|
||||
if (existingCreated)
|
||||
cluster.kubectl(["-n", cluster.namespace, "delete", "pvc", existingClaim, "--wait=true"]);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -39,6 +39,13 @@ compatible privileged/host settings are warned about and left to administrator
|
||||
trust plus cluster admission policy; fields that collide with Kandev invariants
|
||||
are rejected.
|
||||
|
||||
An ordinary non-main container may explicitly request one RO/RW workspace
|
||||
mount at exactly `/workspace`. Kandev retains the volume definition; redirected,
|
||||
overlapping, subpath, propagation and recursive grants remain prohibited.
|
||||
Admission compares grants in both directions by named recipient. Runtime/auth
|
||||
mounts and init/ephemeral recipients remain excluded. The recorded launch
|
||||
snapshot governs replacements. See the [workspace grant design](../specs/executors/system-design/kubernetes-docker-workloads.md).
|
||||
|
||||
Managed PVCs do not receive a Pod owner reference. Kandev preserves them on
|
||||
ordinary stop, backend shutdown, and unexpected Pod loss, then explicitly
|
||||
deletes them after a terminal cleanup proves the stored UID and full ownership
|
||||
|
||||
+6
-266
@@ -1,268 +1,8 @@
|
||||
# Kubernetes Deployment Guide
|
||||
# Kubernetes guide
|
||||
|
||||
This guide covers building the Kandev Docker image and deploying it to a Kubernetes cluster.
|
||||
Use the current [Kubernetes guide](public/k8s.md) for control-plane deployment,
|
||||
executor setup, Pod templates, workspace storage and cleanup.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Docker (for building the image)
|
||||
- A container registry (Docker Hub, GHCR, ECR, etc.)
|
||||
- A Kubernetes cluster with `kubectl` configured
|
||||
- A StorageClass that supports `ReadWriteOnce` PVCs (for SQLite persistence)
|
||||
|
||||
## Building the Image
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
# Build for your current architecture
|
||||
docker build -t kandev:latest .
|
||||
|
||||
# Build for a specific architecture (e.g., for amd64 clusters)
|
||||
docker build --platform linux/amd64 -t kandev:latest .
|
||||
|
||||
# Multi-arch build (requires buildx)
|
||||
docker buildx build --platform linux/amd64,linux/arm64 -t kandev:latest .
|
||||
```
|
||||
|
||||
### Using the Pre-built Image
|
||||
|
||||
Kandev publishes images to GitHub Container Registry. Pull directly:
|
||||
|
||||
```bash
|
||||
docker pull ghcr.io/kdlbs/kandev:latest
|
||||
```
|
||||
|
||||
Or reference it in your K8s deployment:
|
||||
|
||||
```yaml
|
||||
image: ghcr.io/kdlbs/kandev:latest
|
||||
```
|
||||
|
||||
### Choosing your image: vanilla vs. universal
|
||||
|
||||
Kandev publishes two flavors: the default vanilla image (smallest, npm-installable agent CLIs only) and a `:universal` image (~1.4 GB) that adds language toolchains (Go, Rust, build-essential), linters, and Playwright Chromium system libs - useful when your agents work on Go/Rust/Python projects or drive headless browsers.
|
||||
|
||||
```yaml
|
||||
image: ghcr.io/kdlbs/kandev:universal
|
||||
```
|
||||
|
||||
See [`images.md`](images.md) for the full comparison, inclusion policy, and recipes for deriving your own image when you need something neither flavor includes.
|
||||
|
||||
## Deploying to Kubernetes
|
||||
|
||||
### Quick Start
|
||||
|
||||
```bash
|
||||
# Apply all manifests
|
||||
kubectl apply -f k8s/
|
||||
|
||||
# Check status
|
||||
kubectl get pods -l app=kandev
|
||||
kubectl logs -l app=kandev -f
|
||||
```
|
||||
|
||||
### What Gets Created
|
||||
|
||||
| Resource | File | Purpose |
|
||||
|----------|------|---------|
|
||||
| Deployment | `deployment.yaml` | Single-replica pod running backend + web |
|
||||
| Service | `service.yaml` | ClusterIP exposing port 38429 |
|
||||
| ConfigMap | `configmap.yaml` | Non-sensitive environment configuration |
|
||||
| PVC | `pvc.yaml` | 10Gi persistent volume for SQLite + worktrees |
|
||||
| Ingress | `ingress.yaml` | Example ingress with WebSocket support |
|
||||
|
||||
### Accessing the UI
|
||||
|
||||
**Port-forward** (quickest for testing):
|
||||
|
||||
```bash
|
||||
kubectl port-forward svc/kandev 38429:38429
|
||||
# Open http://localhost:38429
|
||||
```
|
||||
|
||||
**Ingress**: Edit `k8s/ingress.yaml` to set your domain, then apply. The ingress routes all traffic to the backend on port 38429; the Go backend serves API, WebSocket, and SPA traffic on that port.
|
||||
|
||||
### Custom Domain / Reverse Proxy
|
||||
|
||||
No extra configuration is needed. The frontend automatically uses `window.location.origin` to reach the API, which works with any domain, reverse proxy, or ingress setup.
|
||||
|
||||
## Installing Agent CLIs
|
||||
|
||||
The kandev image ships with `git`, `gh` (GitHub CLI), `node`, and `npm`, but **does not bundle the coding-agent CLIs** (`claude-code`, `codex`, `auggie`, etc.) — agent choice is per-user, and bundling all of them would bloat the image significantly.
|
||||
|
||||
> Looking to add tools *beyond* agent CLIs - language toolchains, build tools, internal CLIs? See [`images.md`](images.md) for the universal-image option and recipes for deriving your own image.
|
||||
|
||||
To install an agent inside the running pod, open **Settings → Agents** in the UI and click **Install** on the agent card under "Available to Install". The backend runs the agent's hard-coded install script (`npm install -g <pkg>`) and rescans on success.
|
||||
|
||||
The image sets `NPM_CONFIG_PREFIX=/data/.npm-global` so user-installed npm globals land on the PV and **survive pod restarts and image upgrades**. The same persistence applies if you `kubectl exec` and install manually:
|
||||
|
||||
```bash
|
||||
kubectl exec -it deployment/kandev -- npm install -g @anthropic-ai/claude-code
|
||||
```
|
||||
|
||||
After installing, log in with the agent's own auth (e.g. `claude login`), then click **Rescan** on the agents page.
|
||||
|
||||
### Persistent agent and selectable `gh` CLI auth
|
||||
|
||||
The image sets `HOME=/data/home` for the `kandev` user, so every CLI that writes its auth state under `$HOME` lands on the PV and survives pod restarts and image upgrades. This includes:
|
||||
|
||||
- `gh` CLI — `~/.config/gh/hosts.yml`
|
||||
- Claude Code — `~/.claude/.credentials.json`, `~/.claude.json`
|
||||
- Codex — `~/.codex/auth.json`, `~/.codex/config.toml`
|
||||
- Auggie — `~/.augment/session.json`
|
||||
- GitHub Copilot — `~/.copilot/...`
|
||||
- OpenCode, Amp — `~/.config/<tool>/...`
|
||||
|
||||
So a one-time `kubectl exec -it deployment/kandev -- gh auth login` makes that named GitHub account available for workspace selection. Agent CLI logins such as `claude login` and `codex login` also persist, so you do not need to redo them after `kubectl set image` or a `helm upgrade`.
|
||||
|
||||
> Workspace PATs are stored in Kandev's encrypted secret store. A workspace configured for GitHub CLI resolves the exact selected host/login without changing the active CLI account. Host-active `gh`, backend `GITHUB_TOKEN`/`GH_TOKEN`, and old globally named secrets are consulted only by migration-only **Legacy shared** connections.
|
||||
|
||||
Set `KANDEV_GITHUB_CREDENTIAL_BROKER_PUBLIC_BASE_URL` to the HTTPS Kandev service URL reachable from agent containers and remote executors. Set the stable secret `KANDEV_GITHUB_CREDENTIAL_BROKER_REISSUE_SIGNING_KEY` to let a running executor recover its managed lease after a backend restart; changing this key invalidates outstanding execution capabilities. Drain or quiesce active agent sessions before key rotation. This is independent of GitHub App setup and is required for managed PAT or CLI credentials outside local/worktree execution. Ingress must pass `GET` and `POST` on `/api/v1/git/credentials/resolve` and `POST` on `/api/v1/git/credentials/reissue`; the older `/api/v1/github/credentials/*` paths remain compatibility aliases. Executors require the non-secret `GET` readiness response before startup, credential redemption uses the lease-authenticated `POST`, and reissue uses an execution-scoped capability.
|
||||
|
||||
## Configuration
|
||||
|
||||
Kandev reads configuration via `KANDEV_`-prefixed environment variables (Viper). Set these in `k8s/configmap.yaml` or as environment variables in the deployment.
|
||||
|
||||
### Core Settings
|
||||
|
||||
See [`configuration.md`](configuration.md) for the full reference (every backend knob and its YAML form). The tables below cover what's most commonly set in K8s manifests.
|
||||
|
||||
| Env Var | Required | Default | Description |
|
||||
|---------|----------|---------|-------------|
|
||||
| `KANDEV_SERVER_PORT` | No | `38429` | Server port (API + WebSocket + Web UI) |
|
||||
| `KANDEV_HOME_DIR` | No | `/data` | Kandev home directory - contains `data/` (DB), `tasks/`, `worktrees/`, `repos/`, `sessions/`, and `lsp-servers/` |
|
||||
| `KANDEV_DATABASE_DRIVER` | No | `sqlite` | Database driver (`sqlite` or `postgres`) |
|
||||
| `KANDEV_DATABASE_PATH` | No | `$KANDEV_HOME_DIR/data/kandev.db` | SQLite database file path (override) |
|
||||
| `KANDEV_DOCKER_ENABLED` | No | `false` | Enable Docker runtime for agents (requires DinD) |
|
||||
| `KANDEV_LOG_LEVEL` | No | `info` | Log level: `debug`, `info`, `warn`, `error` |
|
||||
| `KANDEV_LOGGING_FORMAT` | No | auto | Log format: `json` (auto-detected in K8s) or `text` |
|
||||
| `KANDEV_LOGGING_OUTPUTPATH` | No | `stdout` | Log destination: `stdout`, `stderr`, or a file path (rotated when a file) |
|
||||
| `KANDEV_LOGGING_MAXSIZEMB` | No | `100` | Rotate the log file when it exceeds this size (MB). File output only. |
|
||||
| `KANDEV_LOGGING_MAXBACKUPS` | No | `5` | Max rotated files to retain (`0` = unlimited). File output only. |
|
||||
| `KANDEV_LOGGING_MAXAGEDAYS` | No | `30` | Max age of rotated files in days (`0` = unlimited). File output only. |
|
||||
| `KANDEV_LOGGING_COMPRESS` | No | `true` | Gzip rotated files. File output only. |
|
||||
|
||||
> **Logging in K8s:** prefer the default `stdout` so kubelet collects logs. If you set `KANDEV_LOGGING_OUTPUTPATH` to a file, the active log is created with mode `0600` (owner read/write only); any sidecar reading it must run as the same user.
|
||||
|
||||
> **Upgrading from a pre-`KANDEV_HOME_DIR` deployment?** The SQLite DB path moved from `/data/kandev.db` to `/data/data/kandev.db`, and `KANDEV_DATA_DIR` is gone. Point `KANDEV_HOME_DIR` at the same volume mount (`/data`) instead. (`KANDEV_WORKTREE_BASEPATH` still works as an explicit override if you want to keep worktrees outside the home dir.) With no explicit database path, the backend checks the legacy `kandev.db` and, when valid, installs a validated snapshot at the current path while retaining the legacy database and its `-wal`/`-shm` files. Look for the `SQLite database selected` diagnostic with outcome `legacy_adopted`. If both candidates need operator review, startup stops and names both paths; preserve them and select the intended file with `KANDEV_DATABASE_PATH`. To keep the old path explicitly, set `KANDEV_DATABASE_PATH=/data/kandev.db` in the ConfigMap.
|
||||
|
||||
### PostgreSQL Settings (when `KANDEV_DATABASE_DRIVER=postgres`)
|
||||
|
||||
| Env Var | Required | Default | Description |
|
||||
|---------|----------|---------|-------------|
|
||||
| `KANDEV_DATABASE_HOST` | No | `localhost` | PostgreSQL host |
|
||||
| `KANDEV_DATABASE_PORT` | No | `5432` | PostgreSQL port |
|
||||
| `KANDEV_DATABASE_USER` | Yes | `kandev` | Database user |
|
||||
| `KANDEV_DATABASE_PASSWORD` | Usually | (empty) | Database password - required unless your Postgres allows passwordless auth |
|
||||
| `KANDEV_DATABASE_DBNAME` | Yes | `kandev` | Database name |
|
||||
| `KANDEV_DATABASE_SSLMODE` | No | `disable` | SSL mode (`disable`, `require`, `verify-ca`, `verify-full`) |
|
||||
|
||||
## Database: SQLite vs PostgreSQL
|
||||
|
||||
### SQLite (default)
|
||||
|
||||
- Zero-config, works out of the box
|
||||
- Database stored at `/data/data/kandev.db` on the PV (derived from `KANDEV_HOME_DIR=/data`)
|
||||
- **Single replica only** (SQLite is single-writer)
|
||||
- Deployment strategy is `Recreate` to prevent concurrent writes
|
||||
- Good for small teams / personal use
|
||||
|
||||
### PostgreSQL (recommended for production)
|
||||
|
||||
- Supports multiple replicas for horizontal scaling
|
||||
- Change deployment strategy to `RollingUpdate`
|
||||
- Set via environment variables:
|
||||
|
||||
```yaml
|
||||
# In configmap.yaml or a Secret
|
||||
KANDEV_DATABASE_DRIVER: postgres
|
||||
KANDEV_DATABASE_HOST: postgres.default.svc.cluster.local
|
||||
KANDEV_DATABASE_PORT: "5432"
|
||||
KANDEV_DATABASE_USER: kandev
|
||||
KANDEV_DATABASE_PASSWORD: <from-secret>
|
||||
KANDEV_DATABASE_DBNAME: kandev
|
||||
```
|
||||
|
||||
When using Postgres, the PVC is still needed for worktree storage but the database itself is external.
|
||||
|
||||
## Persistent Storage
|
||||
|
||||
The PVC at `/data` stores:
|
||||
|
||||
- **SQLite database** (`/data/data/kandev.db`, `/data/data/kandev.db-wal`, `/data/data/kandev.db-shm`)
|
||||
- **Git worktrees** (`/data/worktrees/`), **tasks** (`/data/tasks/`), **repos** (`/data/repos/`), **sessions** (`/data/sessions/`), and **LSP servers** (`/data/lsp-servers/`)
|
||||
- **User home** (`/data/home/`) — `$HOME` for the in-pod `kandev` user; holds `gh` CLI auth and agent CLI auth state (see [Persistent agent and `gh` CLI auth](#persistent-agent-and-gh-cli-auth) above)
|
||||
- **npm globals** (`/data/.npm-global/`) — agent CLIs installed via `npm install -g`
|
||||
|
||||
The PVC uses `ReadWriteOnce` access mode. If your cluster requires a specific StorageClass, add it to `k8s/pvc.yaml`:
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
storageClassName: your-storage-class
|
||||
```
|
||||
|
||||
## Health Checks
|
||||
|
||||
The deployment's liveness probe calls `/health`; the readiness probe calls `/ready`:
|
||||
|
||||
- **Liveness probe**: Restarts the pod if the backend becomes unresponsive (30s interval, 3 failures). `/health` returns 200 as soon as the TCP listener accepts connections, before startup finishes.
|
||||
- **Readiness probe**: Removes the pod from service during startup or issues (10s interval, 3 failures). `/ready` returns 503 until the real router is wired in and the pod can serve real traffic, then 200.
|
||||
|
||||
The CLI launcher performs the same two-stage check: it waits for `/health` (the backend process is alive) and then for `/ready` (startup finished, the backend can serve real requests) before starting the web server.
|
||||
|
||||
## Scaling
|
||||
|
||||
**Single replica (SQLite)**: The default configuration uses `replicas: 1` with `Recreate` strategy. This ensures only one instance writes to SQLite at a time.
|
||||
|
||||
**Multiple replicas (PostgreSQL)**: Switch to Postgres, change the deployment strategy to `RollingUpdate`, and increase replicas:
|
||||
|
||||
```yaml
|
||||
spec:
|
||||
replicas: 3
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
```
|
||||
|
||||
## Upgrading
|
||||
|
||||
```bash
|
||||
# Build and push new image
|
||||
docker build -t your-registry.com/kandev:v1.1.0 .
|
||||
docker push your-registry.com/kandev:v1.1.0
|
||||
|
||||
# Update deployment
|
||||
kubectl set image deployment/kandev kandev=your-registry.com/kandev:v1.1.0
|
||||
|
||||
# Or edit the deployment directly
|
||||
kubectl edit deployment kandev
|
||||
```
|
||||
|
||||
SQLite migrations run automatically on startup — no manual migration step needed.
|
||||
|
||||
> **Upgrading across the `HOME=/data/home` change:** if you used `kubectl exec` to `gh auth login` or log in to agent CLIs on a pre-`HOME=/data/home` image, that state lived in the ephemeral `/home/kandev` and is not carried over. Log in once on the new pod and it will persist for all subsequent upgrades. If you want to keep the old state, copy it onto the PV before upgrading:
|
||||
>
|
||||
> ```bash
|
||||
> kubectl exec deployment/kandev -- sh -c 'cp -a /home/kandev/. /data/home/ 2>/dev/null || true'
|
||||
> ```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
```bash
|
||||
# Check pod status
|
||||
kubectl get pods -l app=kandev
|
||||
|
||||
# View logs
|
||||
kubectl logs -l app=kandev -f
|
||||
|
||||
# Shell into the pod (if needed)
|
||||
kubectl exec -it deployment/kandev -- /bin/bash
|
||||
|
||||
# Check PVC status
|
||||
kubectl get pvc kandev-data
|
||||
|
||||
# Describe pod for events
|
||||
kubectl describe pod -l app=kandev
|
||||
```
|
||||
To run task Pods from an existing Kandev instance, configure its Kubernetes
|
||||
executor. Do not bulk-apply the repository's Kubernetes directory: it contains
|
||||
separate control-plane deployment examples as well as task-worker inputs.
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
---
|
||||
created: 2026-09-13
|
||||
status: draft
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-DOCKER-001
|
||||
- REQ-EXECUTORS-K8S-DOCKER-002
|
||||
- REQ-EXECUTORS-K8S-DOCKER-003
|
||||
system_design:
|
||||
- ../../specs/executors/system-design/kubernetes-docker-workloads.md
|
||||
legacy_specs:
|
||||
- ../../specs/kubernetes-executor/spec.md
|
||||
---
|
||||
|
||||
# Implementation Plan: Kubernetes Docker workloads
|
||||
|
||||
**Implementation authored; image and real Docker acceptance blocked.**
|
||||
|
||||
## Overview
|
||||
|
||||
Allow an administrator-selected ordinary sidecar to mount the same managed
|
||||
workspace as an agent. Supply an opt-in full worker recipe with a per-Pod
|
||||
Docker daemon, then verify source builds, Docker/Compose bind mounts and
|
||||
workspace lifecycle through the existing Kubernetes executor.
|
||||
|
||||
The executor already supports kubeconfig authentication, Pod templates,
|
||||
scheduling/resources, managed PVCs, credential injection and exact cleanup.
|
||||
It currently rejects the reserved workspace mount in every sidecar. This plan
|
||||
adds explicit companion access without changing storage ownership or exposing
|
||||
the private runtime/auth volumes in companion specs.
|
||||
|
||||
The [requirements](../../specs/executors/requirements/kubernetes-docker-workloads.md)
|
||||
and [system design](../../specs/executors/system-design/kubernetes-docker-workloads.md)
|
||||
define the contract. No operator, Helm release or new control plane is needed.
|
||||
|
||||
## Scope
|
||||
|
||||
### In scope
|
||||
|
||||
- Explicit ordinary companion workspace grants with strict template and
|
||||
both-direction admitted-Pod validation.
|
||||
- Snapshot replacement and identity-based cleanup regressions.
|
||||
- An opt-in full worker image, explicit per-Pod daemon configuration, finite
|
||||
preparation and actual source/browser/container-build smoke.
|
||||
- Isolated Kubernetes integration tests and accurate public documentation.
|
||||
|
||||
### Out of scope
|
||||
|
||||
- Installation-specific topology, hostnames, addresses, credentials, rollout
|
||||
plans, operating logs or production acceptance records. Keep these outside
|
||||
this repository and its PR titles, descriptions, comments and artifacts.
|
||||
- Live deployment, service migration, new cluster-wide permissions, portable
|
||||
storage, automatic capacity management or a Docker sandbox guarantee.
|
||||
- A new UI, API/profile field or implicit change to existing worker presets.
|
||||
|
||||
## Technical approach
|
||||
|
||||
In `apps/backend/internal/agent/kubernetes/{template,compose,admission}.go`,
|
||||
permit one explicit ordinary sidecar mount of `kandev-workspace` at exactly
|
||||
`/workspace`, with RO/RW access. Kandev retains the volume definition and main
|
||||
mount. Reject redirected/overlapping grants, subpaths and runtime/auth access;
|
||||
init/ephemeral rules stay unchanged. Compare desired and admitted grants by
|
||||
container identity in both directions, preserving only proven neutral defaults.
|
||||
|
||||
The launch snapshot already stores the raw template. Replacement must reuse
|
||||
its grant and recorded claim identity despite later profile edits. Existing
|
||||
nonce, UID, ownership and existing-claim cleanup contracts remain intact.
|
||||
|
||||
Add a separate `k8s/worker-images/full/` recipe with immutable inputs and
|
||||
repository-compatible language tools, linters and Playwright Chromium. The
|
||||
non-root main agent connects to a separately configured privileged Docker
|
||||
sidecar through a Pod-local Unix socket. Both mount the workspace; only the
|
||||
agent gets Kandev runtime/auth mounts. Daemon data is explicitly disposable.
|
||||
|
||||
Provide explicit resources, writable caches and a finite daemon readiness gate
|
||||
while preserving default clone/setup/agent-install/branch preparation. Deployers
|
||||
must choose their own namespace policy, placement, limits and network MTU.
|
||||
The recipe does not grant privileges or install anything by itself.
|
||||
|
||||
## Tests
|
||||
|
||||
All new file/method names below are implementation outputs.
|
||||
|
||||
| Acceptance criteria | Evidence |
|
||||
| --- | --- |
|
||||
| 001.1, 001.2 | New `workspace_grant_test.go`: `TestWorkspaceGrantTemplateValidation`, `TestWorkspaceGrantComposition`; RO/RW grants, all workspace modes, unchanged templates and rejected reserved access. |
|
||||
| 001.3 | `TestWorkspaceGrantAdmission`; added/removed/transferred/duplicated/changed grants fail, unrelated additions pass; task 03 proves API defaulting. |
|
||||
| 001.4 | New lifecycle `executor_kubernetes_workspace_grant_test.go`: `TestKubernetesWorkspaceGrantUsesLaunchSnapshot`; profile edits cannot change replacement grants or storage. |
|
||||
| 002.1-002.4 | Task 02 pinned image/tool/browser checks, template contract and finite daemon preparation; task 03 actual Unix-only daemon operation. |
|
||||
| 003.1 | Real source, browser, image build/run and Compose bind assertions with Kandev-visible results in task 03. |
|
||||
| 003.2, 003.3 | Stop/Resume, snapshot replacement, exact managed cleanup, untouched existing claim and nested compute teardown. |
|
||||
| 003.4 | Tested runtime/image inventory, independent daemons, nested resource accounting and two-containers/one-Pod inventory in task 03. |
|
||||
|
||||
Short IDs in the table use `AC-EXECUTORS-K8S-DOCKER-`.
|
||||
|
||||
```bash
|
||||
(cd apps/backend && GOMAXPROCS=2 go test -p 1 ./internal/agent/kubernetes ./internal/agent/runtime/lifecycle -count=1)
|
||||
```
|
||||
|
||||
## E2E tests
|
||||
|
||||
Task 03 adds `apps/web/e2e/tests/kubernetes/kubernetes-docker-workloads.spec.ts`
|
||||
to the existing `containers` project. Reuse the real Kind/API/Docker fixtures
|
||||
with a mock agent and no account credentials. Verify visible workspace results,
|
||||
daemon failure, retained/replaced workspace, independent daemons and cleanup.
|
||||
Existing UI is exercised without rendered layout changes.
|
||||
|
||||
```bash
|
||||
(cd apps && pnpm install --frozen-lockfile)
|
||||
(cd apps/web && KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --host --shards 1 --project containers tests/kubernetes/kubernetes-docker-workloads.spec.ts)
|
||||
```
|
||||
|
||||
Select a bounded build/test host before these implementation-time commands.
|
||||
Keep test evidence limited to generic fixtures, runtime versions and outcomes.
|
||||
|
||||
## Work orders
|
||||
|
||||
Run sequentially in the primary session after an implementation request.
|
||||
|
||||
- [x] [Task 01: Explicit companion workspace grants](task-01-workspace-grants.md)
|
||||
- [ ] [Task 02: Full worker image and daemon recipe](task-02-full-worker-image.md)
|
||||
- [ ] [Task 03: Real Kubernetes Docker lifecycle tests](task-03-docker-lifecycle.md)
|
||||
|
||||
## Verification results
|
||||
|
||||
- Required Kubernetes and lifecycle Go packages passed after workspace-grant implementation.
|
||||
- Focused template/readiness and clone/cache/retained-workspace tests passed.
|
||||
- Documentation catalog, specification lint and public-doc validation passed.
|
||||
- Recipe shell syntax, immutable-input check and renderer smoke passed.
|
||||
- Four opt-in lifecycle cases are discoverable. They have not executed.
|
||||
|
||||
The full image build/verification did not complete. Task 02 and task 03 remain
|
||||
blocked pending a suitable isolated test environment. No Docker acceptance,
|
||||
portable resource enforcement or production compatibility is claimed. See each
|
||||
work order's Results for the exact scope of evidence and remaining work.
|
||||
|
||||
## Risks
|
||||
|
||||
- Privileged DinD weakens isolation; verify nested cgroup accounting before
|
||||
claiming resource enforcement. Kind results do not prove all runtimes.
|
||||
- Arbitrary agent-only HOME/temp binds and callbacks are not covered by sharing
|
||||
`/workspace`; document tested compatibility without claiming complete parity.
|
||||
- Image input versions, browser OS dependencies and writable paths must be
|
||||
verified in the built image, not inferred from installed client commands.
|
||||
- An operator must upgrade to the verified backend before using the new grant.
|
||||
Old binaries reject it; production upgrade/recovery is outside this package.
|
||||
@@ -0,0 +1,130 @@
|
||||
---
|
||||
id: "01-workspace-grants"
|
||||
title: "Explicit companion workspace grants"
|
||||
status: completed
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-DOCKER-001
|
||||
- REQ-EXECUTORS-K8S-DOCKER-003
|
||||
acceptance_criteria:
|
||||
- AC-EXECUTORS-K8S-DOCKER-001.1
|
||||
- AC-EXECUTORS-K8S-DOCKER-001.2
|
||||
- AC-EXECUTORS-K8S-DOCKER-001.3
|
||||
- AC-EXECUTORS-K8S-DOCKER-001.4
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.2
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.3
|
||||
system_design:
|
||||
- ../../specs/executors/system-design/kubernetes-docker-workloads.md
|
||||
---
|
||||
|
||||
# Task 01: Explicit companion workspace grants
|
||||
|
||||
## Summary
|
||||
|
||||
Allow an explicitly named ordinary sidecar to share Kandev's workspace through
|
||||
the existing raw template. Keep runtime/auth mounts private in the composed
|
||||
spec, verify admitted grants against the requested template, and preserve
|
||||
snapshot and exact-resource lifecycle behavior.
|
||||
|
||||
## In scope
|
||||
|
||||
- Use TDD for the template, composition and both-direction admission checks
|
||||
described in the system design; retain the main-container contract.
|
||||
- Cover RO/RW, all three storage modes, unchanged profiles, extra sidecars,
|
||||
malicious admission and lost-Pod replacement after a profile edit.
|
||||
- Keep UID, nonce, ownership and existing-claim cleanup regressions passing.
|
||||
- Reconcile the foundation, ownership ADR and public executor template contract.
|
||||
Clarify README executor support and replace obsolete `docs/k8s.md` bulk-deploy
|
||||
advice with the current guide. Do not imply the full recipe is shipped yet.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Image builds, live cluster changes, new API/UI/profile fields, automatic mounts,
|
||||
privilege grants and broad refactors of unrelated executor lifecycle code.
|
||||
|
||||
## Acceptance
|
||||
|
||||
1. An explicit ordinary sidecar mount survives composition; ambiguous/redirected
|
||||
grants and access to reserved runtime/auth material fail before API writes.
|
||||
2. Admission cannot add, remove, transfer or alter a requested grant; known
|
||||
neutral defaults and unrelated compatible additions remain supported.
|
||||
3. Snapshot replacement preserves the original grant and claim identity.
|
||||
Existing managed/existing-claim cleanup and nonce checks remain unchanged.
|
||||
|
||||
## Verification
|
||||
|
||||
From the repository root, after the red/green implementation cycle:
|
||||
|
||||
```bash
|
||||
(cd apps/backend && GOMAXPROCS=2 go test -p 1 ./internal/agent/kubernetes ./internal/agent/runtime/lifecycle -count=1)
|
||||
python3 scripts/list-docs.py validate
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Add `workspace_grant_test.go` with `TestWorkspaceGrantTemplateValidation`,
|
||||
`TestWorkspaceGrantComposition`, `TestWorkspaceGrantAdmission`; add
|
||||
`executor_kubernetes_workspace_grant_test.go` with
|
||||
`TestKubernetesWorkspaceGrantUsesLaunchSnapshot`. Use assertions on resulting
|
||||
mounts/files/inventory and rejection outcomes, not internal call counts.
|
||||
Task 03 owns API-server defaulting and actual Docker bind evidence.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/backend/internal/agent/kubernetes/{template,compose,admission}.go`
|
||||
- `apps/backend/internal/agent/kubernetes/workspace_grant_test.go` (new)
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_workspace_grant_test.go` (new)
|
||||
- `apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_reconnect.go`, only if tests expose a required adjustment
|
||||
- `docs/public/k8s.md`, `docs/k8s.md`, `README.md`
|
||||
- `docs/specs/kubernetes-executor/spec.md`
|
||||
- `docs/decisions/2026-08-24-kubernetes-executor-resource-ownership.md`
|
||||
|
||||
## Dependencies
|
||||
|
||||
None. Start only after the user requests implementation of this package.
|
||||
|
||||
## Risks
|
||||
|
||||
Comparing only admitted grants misses a removed recipient; comparing container
|
||||
positions permits transfer. Overly broad default normalization could hide a
|
||||
changed access mode. Avoid changing existing main-container validation.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/executors/requirements/kubernetes-docker-workloads.md), requirement 001.
|
||||
- [System design](../../specs/executors/system-design/kubernetes-docker-workloads.md), Workspace grant and admission; Lifecycle and evidence.
|
||||
- Existing `template_validation_test.go`, `compose_test.go`, `admission_test.go`,
|
||||
`executor_kubernetes_test.go` and `executor_kubernetes_restart_test.go` patterns.
|
||||
|
||||
## Results
|
||||
|
||||
Implemented explicit RO/RW companion grants with exact mount shape, duplicate
|
||||
identity rejection and admission checks in both directions. Composition retains
|
||||
the requested grant without changing the managed volume or main container.
|
||||
Snapshot replacement tests cover retained grants and prevent profile edits from
|
||||
adding grants to recorded workloads. Existing ownership and cleanup checks pass.
|
||||
|
||||
Validation:
|
||||
|
||||
- Red: `TestWorkspaceGrant*` rejected valid grants and exposed accepted grant/recipient removal.
|
||||
- Green: `GOMAXPROCS=2 go test -p 1 ./internal/agent/kubernetes ./internal/agent/runtime/lifecycle -count=1` passed.
|
||||
- `python3 scripts/list-docs.py validate`: 267 decisions and 870 specifications.
|
||||
- `python3 scripts/lint-spec-files.py --all`: passed.
|
||||
- `node --test scripts/validate-public-docs.test.mjs`: passed.
|
||||
- `git diff --check`: passed.
|
||||
|
||||
Real API defaulting and Docker lifecycle evidence remain assigned to task 03.
|
||||
No neutral mount default normalization has been introduced.
|
||||
|
||||
Review regression: a writable PVC alias at `/cache` initially bypassed a read-only
|
||||
workspace grant in both managed-PVC and existing-claim modes. Composition and
|
||||
admission now reject those aliases while allowing unrelated PVC mounts.
|
||||
`TestWorkspaceGrantAdmissionRejectsClaimAlias` failed before the fix; the full
|
||||
Kubernetes package passed afterward with `GOMAXPROCS=2 go test -p 1
|
||||
./internal/agent/kubernetes -count=1 -timeout=90s` (0.281s test execution).
|
||||
@@ -0,0 +1,141 @@
|
||||
---
|
||||
id: "02-full-worker-image"
|
||||
title: "Full worker image and daemon recipe"
|
||||
status: blocked
|
||||
wave: 2
|
||||
depends_on:
|
||||
- "01-workspace-grants"
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-DOCKER-002
|
||||
- REQ-EXECUTORS-K8S-DOCKER-003
|
||||
acceptance_criteria:
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.1
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.2
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.3
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.4
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.1
|
||||
system_design:
|
||||
- ../../specs/executors/system-design/kubernetes-docker-workloads.md
|
||||
---
|
||||
|
||||
# Task 02: Full worker image and daemon recipe
|
||||
|
||||
## Summary
|
||||
|
||||
Produce a pinned full worker and an opt-in per-Pod Docker template, retaining
|
||||
the current bootstrap contract. Provide a reproducible build and smoke entry
|
||||
point that task 03 can execute against real Kubernetes.
|
||||
|
||||
## In scope
|
||||
|
||||
- New `k8s/worker-images/full/` recipe, pins, README, build and smoke scripts.
|
||||
Preserve the existing minimal/node-pnpm/python recipes.
|
||||
- Confirm a bounded build host before building. Inspect actual headroom and
|
||||
builder resource controls; do not start unbounded builds or introduce new
|
||||
host daemons, registry services or publication by assumption.
|
||||
- Resolve immutable amd64 universal/DinD image references and added tool inputs.
|
||||
Match repository Go, pnpm, golangci-lint v2 and lockfile Playwright browser/OS
|
||||
dependencies. Include Rust, Python/venv, compilers, Docker CLI/Buildx/Compose.
|
||||
- Non-root writable tool/cache paths; no credential layers. Baked browser/tools
|
||||
live outside `/workspace` and the runtime HOME. Create caches after clone.
|
||||
- Profile example with main plus privileged `docker-engine`, the explicit
|
||||
workspace grant, Pod-local Unix socket, Docker data `emptyDir`, explicit
|
||||
resources and daemon MTU. Bypass inherited TCP-listener entrypoint defaults.
|
||||
- Finite daemon readiness before repository setup, preserving default clone,
|
||||
origin validation, selected agent installation and task-branch postlude.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Applying the template, installing cluster policy, production service updates,
|
||||
real credentials, image publication or claiming full Docker-executor parity.
|
||||
|
||||
## Acceptance
|
||||
|
||||
1. A reproducible image builds from recorded immutable inputs and passes small
|
||||
Go, Node/pnpm, Python and headless Chromium tests as UID/GID 1000.
|
||||
2. The documented template passes Kandev validation after task 01 and includes
|
||||
explicit daemon privilege, limits, workspace/socket/data mounts and startup
|
||||
handling without implicitly changing any existing profile.
|
||||
3. The smoke includes actual Docker build/run/Compose and filesystem assertions,
|
||||
finite timeouts and targeted teardown. Task 03 must run those tests; tool
|
||||
version output alone does not complete container-execution acceptance.
|
||||
|
||||
## Verification
|
||||
|
||||
The following scripts are outputs of this work order. Their interfaces must
|
||||
fail if inputs/pins are missing. Run from the repository root on the selected
|
||||
bounded build host; `--check` validates inputs without building.
|
||||
|
||||
```bash
|
||||
bash -n k8s/worker-images/full/build.sh k8s/worker-images/full/smoke.sh
|
||||
bash k8s/worker-images/full/build.sh --check
|
||||
bash k8s/worker-images/full/build.sh --build --verify
|
||||
(cd apps/backend && GOMAXPROCS=2 go test -p 1 ./internal/agent/kubernetes -count=1)
|
||||
git diff --check -- k8s docs/public/k8s.md docs/plans/kubernetes-docker-workloads
|
||||
```
|
||||
|
||||
`--verify` must exercise actual source/browser tools in a constrained non-root
|
||||
container and record image ID/digest, versions, platform and build limits.
|
||||
It does not silently launch privileged Docker on a production cluster. Task 03 loads
|
||||
the exact produced image and exercises `smoke.sh --all` inside the task Pod.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `k8s/worker-images/full/{Dockerfile,pins.env,README.md,build.sh,smoke.sh}` (new)
|
||||
- `k8s/worker-images/full/{pod-template.yaml,prepare.sh}` (new)
|
||||
- `docs/public/k8s.md` (opt-in recipe link and explicit runtime constraints)
|
||||
- A focused recipe contract test alongside existing Kubernetes preset tests.
|
||||
|
||||
## Dependencies
|
||||
|
||||
Task 01. No worker import or production Pod can use the new grant until the
|
||||
operator deliberately upgrades the backend to the verified implementation.
|
||||
|
||||
## Risks
|
||||
|
||||
Image size, browser native dependencies, Rust paths under an overridden HOME,
|
||||
non-root socket permissions, cache creation before clone and daemon entrypoint
|
||||
defaults can each break an otherwise valid template. A bounded build may need
|
||||
a different host; resolve that before consuming significant resources.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- The system design's Worker and daemon configuration section.
|
||||
- [System design](../../specs/executors/system-design/kubernetes-docker-workloads.md), Worker and daemon configuration.
|
||||
- `Dockerfile.universal`, `mise.toml`, `apps/pnpm-lock.yaml`,
|
||||
`k8s/worker-images/README.md`, `k8s/presets/node-pnpm.yaml`.
|
||||
|
||||
## Results
|
||||
|
||||
Implementation authored; runtime acceptance remains blocked. Do not treat this
|
||||
work order as accepted or promote the pull request to ready.
|
||||
|
||||
The immutable recipe, explicit daemon template, finite preparation, renderer and
|
||||
source/browser/Docker smoke scripts are present. `TestFullWorkerTemplate` and
|
||||
`TestFullWorkerPreparationContract` passed. The filesystem preparation test
|
||||
`TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace` passed, proving
|
||||
clone/cache ordering and retained checkout reuse. Shell syntax, input validation
|
||||
and immutable image rendering passed. The renderer regression first failed
|
||||
because the renderer was absent; `TestFullWorkerTemplateRequiresImmutableImage`
|
||||
passed in the focused Go regression rerun after rebasing.
|
||||
|
||||
A bounded `build.sh --build --verify` attempt did not complete. No image ID,
|
||||
source/browser image verification or supported full runtime matrix is claimed.
|
||||
The disposable build resources were removed. Complete the image build and
|
||||
verification on a suitable isolated test host before running task 03.
|
||||
|
||||
`python3 k8s/worker-images/full/build_test.py` passed after reproducing a failed
|
||||
verification leaving an untracked container. The verifier now has an exact
|
||||
owned name and explicit cleanup. This process-boundary test uses a fake Docker
|
||||
command and does not replace real image/container acceptance.
|
||||
|
||||
`python3 -m unittest discover -s k8s/worker-images/full -p '*_test.py'`
|
||||
passes three tests. The daemon startup regression first reproduced failure on a
|
||||
non-eth0 interface. Startup now reads the default-route interface (IPv4, then
|
||||
IPv6) and its MTU, rejecting missing routes and invalid MTUs. This executes the
|
||||
template shell with command fixtures; actual CNI compatibility remains unverified.
|
||||
@@ -0,0 +1,133 @@
|
||||
---
|
||||
id: "03-docker-lifecycle"
|
||||
title: "Real Kubernetes Docker lifecycle tests"
|
||||
status: blocked
|
||||
wave: 3
|
||||
depends_on:
|
||||
- "02-full-worker-image"
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-DOCKER-001
|
||||
- REQ-EXECUTORS-K8S-DOCKER-002
|
||||
- REQ-EXECUTORS-K8S-DOCKER-003
|
||||
acceptance_criteria:
|
||||
- AC-EXECUTORS-K8S-DOCKER-001.3
|
||||
- AC-EXECUTORS-K8S-DOCKER-001.4
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.2
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.3
|
||||
- AC-EXECUTORS-K8S-DOCKER-002.4
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.1
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.2
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.3
|
||||
- AC-EXECUTORS-K8S-DOCKER-003.4
|
||||
system_design:
|
||||
- ../../specs/executors/system-design/kubernetes-docker-workloads.md
|
||||
---
|
||||
|
||||
# Task 03: Real Kubernetes Docker lifecycle tests
|
||||
|
||||
## Summary
|
||||
|
||||
Exercise the new template and full worker through the existing Kind-backed
|
||||
Kandev fixture. Verify the actual agent workspace, Docker filesystem behavior,
|
||||
visible results and resource lifecycle before staging a production rollout.
|
||||
|
||||
## In scope
|
||||
|
||||
- New `kubernetes-docker-workloads.spec.ts` in the `containers` project; reuse
|
||||
existing fixture ownership, pinned Kubernetes tools and mock-agent transport.
|
||||
- Load exact task 02 image inputs into the disposable Kind node. Use only a
|
||||
bounded test host selected in task 02, never the production cluster/context.
|
||||
- Real API-defaulted mount grants, daemon readiness failure, actual source and
|
||||
Docker smoke, Compose bind writes visible in the Kandev terminal/diff.
|
||||
- Ordinary Stop retains Pod/PVC/daemon; Resume retains result and uncommitted
|
||||
edit. Lost-Pod replacement uses the original snapshot and PVC after a profile
|
||||
edit, while Docker caches/test containers are disposable.
|
||||
- Two test Pods have independent daemons; API inventory counts one Pod each.
|
||||
Record nested process cgroup ancestry/counters and bounded CPU/memory load.
|
||||
- Terminal cleanup removes exact owned resources and nested compute; an
|
||||
operator-owned existing claim is untouched. Persist sanitized evidence.
|
||||
- Update E2E documentation and any duration/shard catalog required by the
|
||||
existing runner, without widening worker concurrency.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Production credentials, new production control plane, deployment,
|
||||
full-suite execution, UI layout changes or full Docker plugin compatibility.
|
||||
|
||||
## Acceptance
|
||||
|
||||
1. Source, Chromium, Docker build/run and Compose tests execute, with actual
|
||||
workspace read/write assertions and Kandev-visible results.
|
||||
2. Daemon failure is finite and visible; Stop/Resume/replacement and destructive
|
||||
cleanup obey the selected persistence model with no foreign resource deletion.
|
||||
3. Recorded evidence proves daemon separation and nested resource accounting
|
||||
for the tested runtime; limitations remain explicit for untested runtimes.
|
||||
|
||||
## Verification
|
||||
|
||||
Run from the repository root. These are new tests; first add failing cases,
|
||||
then make the selected flow pass. Do not run against an active production instance.
|
||||
|
||||
```bash
|
||||
(cd apps && pnpm install --frozen-lockfile)
|
||||
(cd apps/web && KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --host --shards 1 --project containers tests/kubernetes/kubernetes-docker-workloads.spec.ts)
|
||||
(cd apps/backend && GOMAXPROCS=2 go test -p 1 ./internal/agent/kubernetes ./internal/agent/runtime/lifecycle -count=1)
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Name the browser cases `full worker returns source and Docker results`,
|
||||
`unavailable daemon fails preparation`, `resume and replacement retain workspace`,
|
||||
and `isolated daemons clean up with owned Pods`. Keep each independently
|
||||
reclaimable on assertion failure. The first case runs the new
|
||||
`k8s/worker-images/full/smoke.sh --all` inside the agent container.
|
||||
Attach exact fixture/runtime/image versions and command results to Results.
|
||||
|
||||
## Files likely touched
|
||||
|
||||
- `apps/web/e2e/tests/kubernetes/kubernetes-docker-workloads.spec.ts` (new)
|
||||
- `apps/web/e2e/fixtures/kubernetes-worker-images.ts`
|
||||
- `apps/web/e2e/fixtures/kubernetes-test-base.ts`, only needed shared fixture hooks
|
||||
- `apps/web/e2e/README.md` and applicable runner catalog
|
||||
- `k8s/worker-images/full/` for defects proven by these cases
|
||||
|
||||
## Dependencies
|
||||
|
||||
Tasks 01 and 02.
|
||||
|
||||
## Risks
|
||||
|
||||
Privileged nested Docker can behave differently under Kind's outer container
|
||||
and other runtimes. Kind evidence does not establish production compatibility. Agent-only temp/HOME binds
|
||||
are outside the shared workspace; tests must not conceal incompatible paths.
|
||||
|
||||
## Parallelism
|
||||
|
||||
`sequential`
|
||||
|
||||
## Inputs
|
||||
|
||||
- [System design](../../specs/executors/system-design/kubernetes-docker-workloads.md), Lifecycle and evidence.
|
||||
- Existing `kubernetes-executor.spec.ts`, `kubernetes-worker-presets.spec.ts`,
|
||||
`kubernetes-test-base.ts` and the E2E skill/README.
|
||||
|
||||
## Results
|
||||
|
||||
Implementation authored; runtime acceptance remains blocked. Do not treat this
|
||||
work order as accepted or promote the pull request to ready.
|
||||
|
||||
Four opt-in browser cases and a full-worker fixture are authored. Playwright
|
||||
`--list` discovers all four cases; discovery is not runtime evidence. Existing
|
||||
managed workspace snapshot and cleanup Go tests passed in task 01.
|
||||
|
||||
No Kind cluster or nested daemon was launched for these cases. Source/browser
|
||||
execution in the full image, real API defaulting, Compose binds, visible results,
|
||||
Stop/Resume/replacement, daemon failure, cgroup accounting and exact real-resource
|
||||
cleanup remain unverified. The draft must remain blocked until the full focused
|
||||
suite executes successfully on a suitable isolated test host. No production
|
||||
compatibility or deployment acceptance is claimed.
|
||||
|
||||
Static validation: changed-file ESLint passed. A focused TypeScript check of the
|
||||
spec and its imports found existing E2E support-file errors (duplicate members,
|
||||
missing window augmentation and `node:sqlite` declarations), with no diagnostics
|
||||
in the two new files. This is not a passing E2E typecheck or runtime result.
|
||||
+38
-1
@@ -215,7 +215,25 @@ Kandev owns generated resource names, namespace, `restartPolicy: Always`, Linux
|
||||
| `kandev-auth` | `/run/kandev` | Memory-backed auth environment and runtime home |
|
||||
| `kandev-workspace` | `/workspace` | Selected `emptyDir`, managed PVC, or existing claim |
|
||||
|
||||
The template cannot set the main container's command, args, or working directory; use reserved volume names or mount paths; use container port 8765 or the `kandev-agentctl` port name; or define `HOME`, `AGENTCTL_*`, or `KANDEV_*` environment keys. It also cannot set Pod name/namespace/UID, owner references, finalizers, restart policy, node name, OS, or Linux OS/architecture selectors. Kandev defaults `automountServiceAccountToken: false` only when the template leaves it unset.
|
||||
The template cannot set the main container's command, args, or working directory; use reserved volume names or mount paths except the explicit companion workspace grant below; use container port 8765 or the `kandev-agentctl` port name; or define `HOME`, `AGENTCTL_*`, or `KANDEV_*` environment keys. It also cannot set Pod name/namespace/UID, owner references, finalizers, restart policy, node name, OS, or Linux OS/architecture selectors. Kandev defaults `automountServiceAccountToken: false` only when the template leaves it unset.
|
||||
|
||||
An ordinary non-main container may explicitly mount `kandev-workspace` once at
|
||||
exactly `/workspace`, with `readOnly: true` or `false`. Kandev still defines the
|
||||
volume from the profile's storage policy. Do not define that volume in the
|
||||
template. Subpaths, subpath expressions, propagation, recursive mount options,
|
||||
redirected paths, devices and overlapping mounts are rejected. Init and ephemeral
|
||||
containers cannot receive this grant. Runtime and auth mounts remain private to
|
||||
the main container.
|
||||
|
||||
In PVC modes, a second volume cannot reference the workspace claim under another
|
||||
name. This is checked during composition and admission, including aliases mounted
|
||||
outside `/workspace`. Unrelated PVC mounts remain allowed.
|
||||
|
||||
Admission must preserve each grant on the same named container exactly once;
|
||||
added, removed, transferred or changed grants fail before bootstrap. Replacement
|
||||
Pods use the recorded launch template and claim identity, even after profile edits.
|
||||
A writable companion can change repository contents. This grant does not provide
|
||||
a security boundary against a privileged container.
|
||||
|
||||
Every Kandev-created Pod and managed PVC has this complete custom identity:
|
||||
|
||||
@@ -296,6 +314,25 @@ artifacts, no service-account token automount, dropped capabilities, and no
|
||||
privilege escalation. Review resource requests, Pod Security admission, storage
|
||||
ownership, registry access, and repository tool requirements before use.
|
||||
|
||||
### Opt in to a full worker with Docker
|
||||
|
||||
The [full worker recipe](../../k8s/worker-images/full/README.md) adds pinned
|
||||
source/browser tooling and an explicit per-Pod privileged Docker daemon. It
|
||||
requires the companion workspace-grant implementation. Existing presets do not
|
||||
change. The main agent remains non-root and connects over a Pod-local Unix
|
||||
socket; Docker data is disposable. Use the supplied complete prepare script
|
||||
so daemon readiness precedes clone/setup/agent installation and caches are
|
||||
created only after clone.
|
||||
|
||||
The agent controls a privileged root daemon through the socket; its non-root
|
||||
container security context does not contain Docker commands. Schedule these
|
||||
Pods only on a worker node pool isolated from trusted workloads.
|
||||
|
||||
Ordinary Stop retains companion compute. Only `/workspace` is shared for nested
|
||||
bind mounts; agent-only HOME/temp paths and full Docker-executor parity are not
|
||||
covered. Review the recipe's recorded test evidence and runtime limitations
|
||||
before use. No production deployment or universal cgroup compatibility is implied.
|
||||
|
||||
### Choose workspace storage
|
||||
|
||||
| Mode | Profile fields | Lifecycle |
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
status: draft
|
||||
system: executors
|
||||
created: 2026-09-13
|
||||
owners:
|
||||
- kandev
|
||||
---
|
||||
|
||||
# Kubernetes Docker Workloads Requirements
|
||||
|
||||
## Overview
|
||||
|
||||
Administrators need task Pods whose agents can build source, build container
|
||||
images and run Docker/Compose tests against the task checkout. The executor
|
||||
system owns the Pod-template, workspace and companion-container boundaries.
|
||||
This capability extends the [Kubernetes foundation](../../kubernetes-executor/spec.md)
|
||||
without changing existing profiles or baseline worker presets.
|
||||
|
||||
## Requirements
|
||||
|
||||
### REQ-EXECUTORS-K8S-DOCKER-001: Explicit companion workspace access
|
||||
|
||||
**Intent:** A selected companion can consume the same task checkout while
|
||||
Kandev retains ownership of workspace storage and private runtime material.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-EXECUTORS-K8S-DOCKER-001.1:** An administrator shall be able to explicitly
|
||||
grant an ordinary non-main container access to the task workspace at the same
|
||||
path as the main container, with either read-only or read-write access.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-001.2:** Profiles without such a grant shall retain
|
||||
their current mount behavior. The template shall not redefine workspace
|
||||
storage, redirect a grant, or grant companion access to Kandev runtime/auth
|
||||
volumes; init and ephemeral containers shall remain ineligible.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-001.3:** Admission that adds, removes, duplicates,
|
||||
transfers or changes a workspace grant shall fail validation before agent
|
||||
bootstrap. Unrelated compatible admission defaults shall remain accepted.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-001.4:** A replacement Pod shall use the recorded
|
||||
workload's grants and storage identity. Editing a profile shall not change a
|
||||
retained session's replacement workload or authorize deletion of foreign data.
|
||||
|
||||
### REQ-EXECUTORS-K8S-DOCKER-002: Explicit full-toolchain Docker configuration
|
||||
|
||||
**Intent:** Operators can prepare a full worker without mistaking installed
|
||||
client commands for functioning container execution.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-EXECUTORS-K8S-DOCKER-002.1:** A documented opt-in worker recipe shall
|
||||
identify immutable image inputs, Linux architecture, installed language and
|
||||
browser tools, runtime user, writable caches and Docker client/daemon versions.
|
||||
It shall contain no task credentials or private repository contents.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-002.2:** The example shall give each task Pod its own
|
||||
Docker daemon and local client connection, with no shared host Docker socket,
|
||||
public Docker endpoint or implicit cluster credential in the workload.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-002.3:** Daemon privileges, resources and storage
|
||||
shall be explicit operator configuration. The agent shall run non-root;
|
||||
configuration shall not silently enable privileged workloads in existing
|
||||
profiles or claim compatibility with restricted admission.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-002.4:** Docker-dependent preparation shall wait for
|
||||
daemon readiness with a finite timeout and report failure if it cannot become
|
||||
ready, rather than reporting a Docker-capable environment based on CLI presence.
|
||||
|
||||
### REQ-EXECUTORS-K8S-DOCKER-003: Verified execution and lifecycle
|
||||
|
||||
**Intent:** A Docker-capable task remains inspectable, resumable and reclaimable
|
||||
through the existing Kandev lifecycle.
|
||||
|
||||
#### Acceptance criteria
|
||||
|
||||
- **AC-EXECUTORS-K8S-DOCKER-003.1:** Verification shall exercise actual source
|
||||
tests, image build/run and a Compose service that reads and writes the same
|
||||
task-workspace files seen by the agent. CLI version checks alone shall not
|
||||
establish compatibility.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-003.2:** Ordinary Stop/Resume shall preserve the Pod
|
||||
and managed workspace; documentation shall disclose retained companion
|
||||
compute. Replacement shall recover workspace results while permitting loss
|
||||
of explicitly disposable Docker caches and test containers.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-003.3:** Terminal cleanup shall remove exactly the
|
||||
owned Pod and managed workspace, leave existing claims untouched, and leave
|
||||
no nested compute belonging to the removed Pod. Agent results and diagnostics
|
||||
shall be checked through Kandev before destructive cleanup.
|
||||
- **AC-EXECUTORS-K8S-DOCKER-003.4:** Published evidence shall distinguish tested
|
||||
architecture/runtime versions from untested combinations and verify resource
|
||||
accounting for the agent, daemon and nested test containers. One task with a
|
||||
companion shall remain one Kubernetes Pod for workload inventory purposes.
|
||||
|
||||
## Out of scope
|
||||
|
||||
- New control-plane deployments, operators, CRDs, Helm charts or release channels.
|
||||
- A Docker security sandbox guarantee, automatic permission grants or a new UI.
|
||||
- Cross-node migration of local PVCs, automatic suspend-to-zero or a capacity queue.
|
||||
- Support for arbitrary host-path mounts, shared Docker daemons or all Docker plugins.
|
||||
- Changing the security contract of the existing minimal, Node/pnpm or Python presets.
|
||||
|
||||
## Delivery
|
||||
|
||||
See the [system design](../system-design/kubernetes-docker-workloads.md) and
|
||||
[implementation plan](../../../plans/kubernetes-docker-workloads/plan.md).
|
||||
@@ -0,0 +1,185 @@
|
||||
---
|
||||
status: draft
|
||||
system: executors
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-DOCKER-001
|
||||
- REQ-EXECUTORS-K8S-DOCKER-002
|
||||
- REQ-EXECUTORS-K8S-DOCKER-003
|
||||
---
|
||||
|
||||
# Kubernetes Docker Workloads System Design
|
||||
|
||||
## Purpose and boundaries
|
||||
|
||||
The executor owns workspace composition, workload snapshots and exact resource
|
||||
cleanup. The [requirements](../requirements/kubernetes-docker-workloads.md)
|
||||
introduce explicit companion workspace access plus an opt-in Docker workload
|
||||
recipe. Operators supply deployment-specific placement, credentials and capacity.
|
||||
The [implementation plan](../../../plans/kubernetes-docker-workloads/plan.md)
|
||||
contains only the reusable executor change, recipe and integration evidence.
|
||||
|
||||
The existing [resource-ownership decision](../../../decisions/2026-08-24-kubernetes-executor-resource-ownership.md)
|
||||
continues to govern names, UIDs, nonces, storage identity and cleanup. The new
|
||||
mount exception must be reconciled with the foundation and public documentation
|
||||
when implemented. Existing release behavior still rejects the exception until
|
||||
that change ships. This design contains its rationale; no independent ADR is
|
||||
needed for a parallel copy of the same contract.
|
||||
|
||||
## Requirement mapping
|
||||
|
||||
| Requirement | Design section |
|
||||
| --- | --- |
|
||||
| REQ-EXECUTORS-K8S-DOCKER-001 | Workspace grant and admission |
|
||||
| REQ-EXECUTORS-K8S-DOCKER-002 | Worker and daemon configuration |
|
||||
| REQ-EXECUTORS-K8S-DOCKER-003 | Lifecycle and evidence |
|
||||
|
||||
## Workspace grant and admission
|
||||
|
||||
Use existing `pod_template_yaml`; add no profile/API field, migration or UI.
|
||||
An ordinary container other than `ProfileConfig.MainContainer` may explicitly
|
||||
reference `kandev-workspace` at exactly `/workspace` once. `readOnly` selects
|
||||
access mode. Reject subpaths, subpath expressions, mount propagation, recursive
|
||||
mount options, alternate paths/devices and additional overlapping mounts. The
|
||||
template still cannot define the volume itself. Main, init and ephemeral
|
||||
container rules remain unchanged, including all runtime/auth mount prohibitions.
|
||||
|
||||
`ValidatePodTemplate` and `validateContainerReservedFields` in
|
||||
`internal/agent/kubernetes/template.go` need container-role context for this
|
||||
narrow exception. `ComposePod` preserves the requested sidecar reference and
|
||||
continues to inject/own the actual workspace volume and the main-container
|
||||
mount. All three workspace modes use the same grant rule.
|
||||
|
||||
`ValidateAdmittedPod` in `internal/agent/kubernetes/admission.go` must carry the
|
||||
desired Pod's ordinary-container identities into companion mount validation.
|
||||
Compare grants by container name and full permitted mount shape, not by array
|
||||
position, volume name alone or the main container's mount. Verify both directions:
|
||||
every admitted grant must have been requested, and every requested grant must
|
||||
remain exactly once on the same named ordinary container. Reject added/removed
|
||||
grants, missing/renamed recipients, duplicate grants, changed access modes and
|
||||
attempts to move a grant into init or ephemeral containers.
|
||||
|
||||
Normalize only documented neutral API defaults before comparison; real API
|
||||
integration must prove any such normalization. Continue accepting unrelated
|
||||
labels and compatible additions that carry no reserved access. Do not weaken
|
||||
existing main-container, control-port, env, volume, nonce or metadata validation.
|
||||
Existing Kandev-owned volumes retain exact-source validation.
|
||||
|
||||
The grant is an administrative data-access decision, not a security barrier
|
||||
against a privileged companion. A writable companion can alter repository
|
||||
contents. Runtime/auth mounts are not explicitly shared with it.
|
||||
|
||||
## Worker and daemon configuration
|
||||
|
||||
Add a separate recipe under `k8s/worker-images/full/`; preserve the existing
|
||||
minimal/node-pnpm/python recipe and its checks. Derive from an immutable
|
||||
universal digest and pin all added inputs in the new recipe's `pins.env`.
|
||||
Resolve actual registry digests during implementation; source examples may use
|
||||
clearly marked substitutions, but deployable configuration must reject them.
|
||||
|
||||
Use `mise.toml`, the Go module and pnpm lockfile for repository-compatible tools.
|
||||
The inspected universal defaults need golangci-lint v2 instead of v1.62.0 and
|
||||
Playwright libraries/browser matching the resolved current package. Include
|
||||
Go, Rust, Node, pnpm, Python/venv, native compilers, Docker CLI, Buildx and Compose.
|
||||
Baked tools/browser binaries live outside mounted workspace/auth paths.
|
||||
|
||||
Set writable npm, pnpm, Python and Go caches under `/workspace`, but create
|
||||
them after initial repository clone: the default Kubernetes prepare script
|
||||
correctly refuses a fresh nonempty non-repository workspace. Kandev owns HOME
|
||||
and injects it as `/run/kandev/home`; do not freeze host-home paths into config.
|
||||
Agent CLIs continue through Kandev's selected-agent installation path.
|
||||
|
||||
The advanced template has ordinary containers `kandev-agent` and `docker-engine`.
|
||||
The agent uses UID/GID 1000, dropped capabilities, no escalation and default
|
||||
seccomp. Pod fsGroup is 1000. Put non-root constraints on the main container;
|
||||
a Pod-wide `runAsNonRoot` would incorrectly conflict with the chosen daemon.
|
||||
The daemon is an explicitly selected pinned DinD image, privileged and root,
|
||||
without hostPath, host namespaces or any service-account token.
|
||||
|
||||
Use a shared Pod-scoped `emptyDir` mounted at `/run/docker` for the Unix socket;
|
||||
run the daemon with an explicit socket group compatible with GID 1000. The
|
||||
client's `DOCKER_HOST` targets that socket. Do not expose a TCP daemon listener,
|
||||
even through the sidecar's inherited entrypoint defaults. Both containers mount
|
||||
the managed workspace at `/workspace`; only the agent receives runtime/auth
|
||||
mounts. Docker data uses its own `emptyDir` at `/var/lib/docker` with an explicit
|
||||
size budget, separate from the durable checkout.
|
||||
|
||||
Docker bridges run inside an already reduced-MTU Pod network. Set their MTU no
|
||||
higher than the actual Pod interface MTU,
|
||||
and test HTTPS/large transfers from nested containers. Do not change Flannel or
|
||||
the cluster API endpoint to fix a nested bridge problem.
|
||||
|
||||
An administrator prepare script prefixes a bounded `docker info` wait before
|
||||
the existing Kubernetes preparation flow. Preserve clone/origin verification,
|
||||
repository setup, agent installation and Kandev's branch-selection postlude;
|
||||
do not use a wait-only replacement script. A startup probe or running main
|
||||
container is not a substitute for this preparation gate.
|
||||
|
||||
Nested bind sources must exist in the daemon's mount namespace. `/workspace`
|
||||
is deliberately shared; arbitrary agent paths such as its private runtime home
|
||||
are not. Repository tests requiring callbacks, port forwarding, temporary
|
||||
directories or SSH fixtures must either use supported shared workspace paths
|
||||
or be reported as incompatible. Do not describe a basic Compose smoke as full
|
||||
Kandev Docker-executor parity.
|
||||
|
||||
## Lifecycle and evidence
|
||||
|
||||
`executors_running` remains authoritative. The raw template already persists
|
||||
in the workload snapshot, so grants need no new persistence schema. Replacement
|
||||
through `executor_kubernetes_reconnect.go` composes the recorded template with
|
||||
the recorded PVC. Tests must prove that edited profiles cannot add grants to
|
||||
old snapshots and that existing identity-based cleanup remains intact.
|
||||
|
||||
Ordinary Stop retains the daemon and nested containers because it retains the
|
||||
Pod. Source tests clean their own disposable Compose project. Terminal Pod
|
||||
deletion must remove the nested processes; managed-PVC deletion remains Kandev's
|
||||
existing UID/nonce-checked operation. Docker cache loss on replacement is expected.
|
||||
Never mount a retained Docker cache across separate sessions.
|
||||
|
||||
Provide a bounded smoke script that tests source tools, browser launch, image
|
||||
build/run, workspace bind mounts and Compose cleanup. Use real filesystem
|
||||
assertions and emitted results, not method-call counts. Automated Kubernetes
|
||||
coverage uses the existing Kind fixture and mock agent, then verifies visible
|
||||
terminal/results and lifecycle through the existing UI. This changes no rendered
|
||||
UI layout; desktop/mobile settings retain their current shared raw-template flow.
|
||||
|
||||
Deployment acceptance additionally uses an operator-selected agent credential,
|
||||
tests each selected worker, and verifies nested cgroup accounting, cross-session
|
||||
daemon isolation, results, retained compute and exact final cleanup. Sidecar resource
|
||||
fields alone are not proof of nested resource enforcement. Refuse acceptance if
|
||||
nested processes escape the measured Pod budget; record the result before any
|
||||
design revision. Privileged operation does not promise adversarial isolation.
|
||||
|
||||
## Deployment and compatibility
|
||||
|
||||
This feature is opt-in through a privileged administrator-authored template,
|
||||
not a rollout to existing users. Only a separately reviewed namespace policy
|
||||
may permit the chosen companion privilege. Operators still configure placement,
|
||||
tolerations, quotas, network access, workload identity and image distribution.
|
||||
The backend uses a restricted kubeconfig; it does not install cluster RBAC or
|
||||
copy cluster credentials into task Pods.
|
||||
|
||||
The existing service must run a binary containing the workspace-grant change
|
||||
before it can accept the advanced template. Building the branch alone does not
|
||||
update the active service. Stage a compatible build and backup, then schedule
|
||||
the existing service update explicitly around active sessions; no second control
|
||||
plane is a production fallback. Development/Kind fixtures remain disposable.
|
||||
|
||||
Public docs must distinguish experimental evidence from universal runtime support
|
||||
and remove the conflicting bulk-apply guidance that could deploy another control
|
||||
plane. The ordinary executor's existing behavior and image presets remain valid.
|
||||
|
||||
## Alternatives
|
||||
|
||||
Running a daemon in the main container would require changing the agent's
|
||||
privilege/process model. A shared host socket would merge session trust and
|
||||
daemon lifecycle. An existing claim duplicated under another mount path would
|
||||
couple profiles to fixed storage and bypass the intended per-session ownership.
|
||||
An explicit sidecar grant is the smallest durable change that preserves the
|
||||
existing managed-workspace model. Rootless BuildKit alone does not run general
|
||||
Docker/Compose workloads; a different runtime would require separate evidence.
|
||||
|
||||
## Delivery
|
||||
|
||||
See [implementation work orders](../../../plans/kubernetes-docker-workloads/plan.md).
|
||||
All named new recipe, smoke and test files are implementation outputs, not
|
||||
claims that they already exist.
|
||||
@@ -31,8 +31,11 @@ first-class Pod with an administrator-reviewed workload template.
|
||||
- The template can define images, sidecars, init containers, resource and
|
||||
security settings, scheduling policy, image-pull secrets, and workload
|
||||
service accounts. Kandev retains exclusive control of the fields needed for
|
||||
runtime identity, bootstrap, transport, workspace access, recovery, and
|
||||
cleanup.
|
||||
runtime identity, bootstrap, transport, workspace storage, recovery, and
|
||||
cleanup. Ordinary non-main containers may explicitly request one RO/RW
|
||||
`kandev-workspace` mount at exactly `/workspace`; admission must preserve
|
||||
its recipient and shape. Runtime/auth mounts and init/ephemeral containers
|
||||
remain excluded. See [workspace grants](../executors/system-design/kubernetes-docker-workloads.md).
|
||||
- Profiles support a Kandev-managed per-session PVC, a Pod-scoped `emptyDir`,
|
||||
or an existing namespaced PVC. Kandev never deletes an existing PVC.
|
||||
- Ordinary Stop and backend shutdown preserve the Pod and workspace for
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
ARG DOCKER_IMAGE
|
||||
ARG BASE_IMAGE
|
||||
FROM ${DOCKER_IMAGE} AS docker_tools
|
||||
FROM ${BASE_IMAGE}
|
||||
USER root
|
||||
ARG GOLANGCI_VERSION
|
||||
ARG GOLANGCI_SHA256
|
||||
ARG PLAYWRIGHT_VERSION
|
||||
ARG PLAYWRIGHT_SHA256
|
||||
ARG CHROMIUM_VERSION
|
||||
ARG CHROMIUM_REVISION
|
||||
ARG CHROMIUM_SHA256
|
||||
COPY --from=docker_tools /usr/local/bin/docker /usr/local/bin/docker
|
||||
COPY --from=docker_tools /usr/local/libexec/docker/cli-plugins/ /usr/local/libexec/docker/cli-plugins/
|
||||
RUN set -eu; \
|
||||
curl -fsSL --max-time 180 "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_VERSION}/golangci-lint-${GOLANGCI_VERSION}-linux-amd64.tar.gz" -o /tmp/lint.tgz; \
|
||||
echo "${GOLANGCI_SHA256} /tmp/lint.tgz" | sha256sum -c -; \
|
||||
tar -xzf /tmp/lint.tgz -C /tmp; \
|
||||
cp "/tmp/golangci-lint-${GOLANGCI_VERSION}-linux-amd64/golangci-lint" /usr/local/bin/golangci-lint; \
|
||||
rm -rf /tmp/lint.tgz "/tmp/golangci-lint-${GOLANGCI_VERSION}-linux-amd64"
|
||||
RUN set -eu; \
|
||||
mkdir -p /opt/full-worker/playwright-core /opt/full-worker/browsers; \
|
||||
curl -fsSL --max-time 180 "https://registry.npmjs.org/playwright-core/-/playwright-core-${PLAYWRIGHT_VERSION}.tgz" -o /tmp/playwright.tgz; \
|
||||
echo "${PLAYWRIGHT_SHA256} /tmp/playwright.tgz" | sha256sum -c -; \
|
||||
tar -xzf /tmp/playwright.tgz --strip-components=1 -C /opt/full-worker/playwright-core; \
|
||||
curl -fsSL --max-time 180 "https://cdn.playwright.dev/builds/cft/${CHROMIUM_VERSION}/linux64/chrome-headless-shell-linux64.zip" -o /tmp/chromium.zip; \
|
||||
echo "${CHROMIUM_SHA256} /tmp/chromium.zip" | sha256sum -c -; \
|
||||
python3 -m zipfile -e /tmp/chromium.zip "/opt/full-worker/browsers/chromium_headless_shell-${CHROMIUM_REVISION}"; \
|
||||
chmod -R a+rX /opt/full-worker; \
|
||||
chmod a+rx "/opt/full-worker/browsers/chromium_headless_shell-${CHROMIUM_REVISION}/chrome-headless-shell-linux64/chrome-headless-shell"; \
|
||||
rm /tmp/playwright.tgz /tmp/chromium.zip
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/opt/full-worker/browsers \
|
||||
npm_config_prefix=/workspace/.npm-global \
|
||||
npm_config_cache=/workspace/.cache/npm \
|
||||
PNPM_HOME=/workspace/.pnpm \
|
||||
npm_config_store_dir=/workspace/.pnpm-store \
|
||||
PIP_CACHE_DIR=/workspace/.cache/pip \
|
||||
GOCACHE=/workspace/.cache/go-build \
|
||||
GOMODCACHE=/workspace/.cache/go-mod \
|
||||
CARGO_HOME=/workspace/.cache/cargo \
|
||||
RUSTUP_HOME=/usr/local/rustup \
|
||||
GOMAXPROCS=2 \
|
||||
GOFLAGS=-p=1 \
|
||||
CARGO_BUILD_JOBS=2
|
||||
ENV PATH=/workspace/.npm-global/bin:/workspace/.pnpm:/usr/local/cargo/bin:/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin
|
||||
COPY smoke.sh /opt/full-worker/smoke.sh
|
||||
RUN chmod 0755 /opt/full-worker/smoke.sh
|
||||
USER 1000:1000
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT []
|
||||
CMD ["sleep", "infinity"]
|
||||
@@ -0,0 +1,94 @@
|
||||
# Full Kubernetes worker with per-Pod Docker
|
||||
|
||||
This opt-in Linux/amd64 recipe extends the pinned universal image with
|
||||
Go-compatible golangci-lint v2, Playwright Chromium and Docker CLI/Buildx/Compose.
|
||||
The existing minimal, Node/pnpm and Python presets are unchanged.
|
||||
|
||||
## Build and verify
|
||||
|
||||
```bash
|
||||
bash k8s/worker-images/full/build.sh --check
|
||||
bash k8s/worker-images/full/build.sh --build --verify
|
||||
```
|
||||
|
||||
The script requires Docker Buildx, at least 20 GiB free disk and capacity for a
|
||||
2-CPU, 4-GiB disposable builder. It validates actual Docker resource controls,
|
||||
uses one BuildKit step at a time and removes its exact builder on exit. It
|
||||
prints the local image tag, image ID, platform and limits. It never publishes
|
||||
an image. The output image remains available for testing; remove that exact
|
||||
tag when finished. Source/browser verification is bounded to ten minutes and
|
||||
runs as UID/GID 1000 with dropped capabilities and no host mounts.
|
||||
|
||||
`pins.env` records immutable base, daemon and builder manifests, plus checksums
|
||||
for every added download. Rust, Go, Node, pnpm, Python/venv and native compilers
|
||||
are inherited from the immutable universal image. The smoke prints the actual
|
||||
versions. The browser revision matches Playwright 1.61.1 in the repository
|
||||
lockfile. Only Chromium headless shell is baked, outside the managed mounts.
|
||||
|
||||
## Configure an executor profile
|
||||
|
||||
1. Distribute your verified image through your own registry and replace
|
||||
`FULL_WORKER_IMAGE_REQUIRED` using `bash k8s/worker-images/full/render-template.sh
|
||||
registry/image@sha256:<digest>`. The renderer rejects placeholders and mutable
|
||||
tags. The unrendered example is documentation only and must not be deployed.
|
||||
2. Select `linux/amd64`, main container `kandev-agent`, and your workspace mode
|
||||
in an existing Kubernetes executor profile. Managed PVCs preserve results.
|
||||
3. Paste the template into the profile's raw Pod template and `prepare.sh` into
|
||||
its prepare script. Preserve the placeholders: Kandev resolves repository,
|
||||
identity, authentication, setup and selected-agent installation at launch.
|
||||
4. Review namespace privilege policy, image access, resources and storage before
|
||||
starting a disposable task. No cluster policy is installed by this recipe.
|
||||
|
||||
The main agent stays non-root. The explicitly privileged root daemon listens
|
||||
only on a Pod-local Unix socket, accessible through GID 1000. Both containers
|
||||
share `/workspace`; the daemon receives neither Kandev runtime nor auth mounts.
|
||||
Its Docker data is a separate disposable 12-GiB `emptyDir`. Pod-wide non-root
|
||||
policy is deliberately absent because it would conflict with this daemon.
|
||||
No host namespaces, hostPath or service-account token are requested.
|
||||
|
||||
The agent can control the privileged daemon through that socket. The main
|
||||
container's non-root security context does not contain commands that use Docker.
|
||||
Schedule these Pods only on a worker node pool isolated from trusted workloads.
|
||||
|
||||
The daemon reads the Pod interface MTU and applies it to default and
|
||||
user-defined bridges. Verify nested HTTPS transfers for the actual CNI/runtime.
|
||||
The preparation script waits at most 60 seconds (plus a bounded client call)
|
||||
for Docker before the standard clone/origin verification, repository setup and
|
||||
agent installation. Kandev still adds task-branch preparation. Cache directories
|
||||
are created only after repository materialization. Kandev owns `HOME`; baked
|
||||
tools and browsers remain outside `/workspace` and `/run/kandev`.
|
||||
|
||||
## Exercise source and Docker execution
|
||||
|
||||
Inside the agent container:
|
||||
|
||||
```bash
|
||||
bash /opt/full-worker/smoke.sh --all
|
||||
cat /workspace/full-worker-result.txt
|
||||
```
|
||||
|
||||
The smoke executes Go tests/lint/build, Rust tests, Node/pnpm tests, Python/venv
|
||||
tests, a C compilation and a real Chromium page interaction/screenshot. Docker
|
||||
acceptance additionally builds/runs a scratch image, performs a large HTTPS
|
||||
transfer and runs Compose with RO input and RW output workspace binds. It
|
||||
checks files and removes only its own Compose project and image.
|
||||
|
||||
Ordinary Stop retains the Pod, daemon and nested containers. Clean up test
|
||||
Compose projects explicitly. Resume retains workspace results; lost-Pod
|
||||
replacement uses the recorded template/PVC and loses disposable daemon state.
|
||||
Terminal cleanup deletes only resources proven to belong to the session;
|
||||
existing claims remain operator-owned.
|
||||
|
||||
## Compatibility limits
|
||||
|
||||
Real acceptance results are recorded in the implementation work orders. A
|
||||
successful image smoke alone does not establish Kubernetes Docker acceptance.
|
||||
Kind evidence applies only to the recorded architecture and runtime. Measure
|
||||
nested cgroup ancestry and counters before claiming resource enforcement;
|
||||
privileged DinD is not an adversarial isolation boundary.
|
||||
|
||||
Only `/workspace` is shared with the daemon. Agent-only HOME/temp bind sources,
|
||||
callbacks, SSH fixtures and arbitrary Docker plugins are not promised to work.
|
||||
This recipe does not establish full Kandev Docker-executor parity, provision a
|
||||
control plane or update an existing service. The backend must contain the
|
||||
explicit workspace-grant implementation before accepting this template.
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Test-only bounded nested process. The caller must delete its owned Pod even
|
||||
# when a cgroup assertion fails; the process also exits after 90 seconds.
|
||||
set -eu
|
||||
work=$(mktemp -d /workspace/full-accounting.XXXXXX)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
cd "$work"
|
||||
cat > main.go <<'GO'
|
||||
package main
|
||||
import ("runtime";"time")
|
||||
func main(){
|
||||
data:=make([]byte,32*1024*1024)
|
||||
for i:=range data {data[i]=1}
|
||||
until:=time.Now().Add(90*time.Second)
|
||||
for time.Now().Before(until) {for i:=range data {data[i]++};runtime.KeepAlive(data)}
|
||||
}
|
||||
GO
|
||||
CGO_ENABLED=0 GOMAXPROCS=2 timeout 120 go build -p 1 -o accounting main.go
|
||||
printf 'FROM scratch\nCOPY accounting /accounting\nENTRYPOINT ["/accounting"]\n' > Dockerfile
|
||||
timeout 120 docker build --network=none -t full-worker-accounting . >&2
|
||||
# Its private daemon/image is discarded with the test Pod.
|
||||
timeout 10 docker run -d --name full-worker-accounting --memory=128m --cpus=0.25 full-worker-accounting
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
source "$here/pins.env"
|
||||
mode=${1:---check}
|
||||
[[ "$mode" == --check || "$mode" == --build ]] || { echo 'Usage: build.sh --check|--build [--verify]' >&2; exit 2; }
|
||||
[[ $# -le 2 && (${2:-} == '' || ${2:-} == --verify) ]] || exit 2
|
||||
for key in BASE_IMAGE DOCKER_IMAGE BUILDKIT_IMAGE; do
|
||||
[[ ${!key} =~ @sha256:[a-f0-9]{64}$ ]] || { echo "Missing immutable $key" >&2; exit 1; }
|
||||
done
|
||||
for key in GOLANGCI_SHA256 PLAYWRIGHT_SHA256 CHROMIUM_SHA256; do
|
||||
[[ ${!key} =~ ^[a-f0-9]{64}$ ]] || { echo "Missing checksum $key" >&2; exit 1; }
|
||||
done
|
||||
for key in GOLANGCI_VERSION PLAYWRIGHT_VERSION CHROMIUM_VERSION CHROMIUM_REVISION PNPM_VERSION; do
|
||||
[[ ${!key} =~ ^[0-9]+(\.[0-9]+)*$ ]] || { echo "Missing version $key" >&2; exit 1; }
|
||||
done
|
||||
[[ "$PLATFORM" == linux/amd64 ]]
|
||||
grep -Fq "$DOCKER_IMAGE" "$here/pod-template.yaml"
|
||||
for file in Dockerfile smoke.sh prepare.sh pod-template.yaml; do test -s "$here/$file"; done
|
||||
if [[ "$mode" == --check ]]; then echo 'Full worker immutable inputs and recipe files valid'; exit; fi
|
||||
# The dedicated builder enforces limits; shell variables alone do not bound builds.
|
||||
free_kb=$(df -Pk "$here" | awk 'END {print $4}')
|
||||
(( free_kb >= 20*1024*1024 )) || { echo 'Build requires at least 20 GiB free disk' >&2; exit 1; }
|
||||
scratch=$(mktemp -d)
|
||||
builder="kandev-full-$(basename "$scratch" | tr '[:upper:].' '[:lower:]-')"
|
||||
tag=${FULL_WORKER_TAG:-"kandev-full-worker:$builder"}
|
||||
if docker image inspect "$tag" >/dev/null 2>&1; then echo 'Refusing to overwrite an existing image tag' >&2; exit 1; fi
|
||||
container="buildx_buildkit_${builder}0"
|
||||
verify_started=false
|
||||
verify_container="${builder}-verify"
|
||||
cleanup() {
|
||||
if [[ "$verify_started" == true ]]; then
|
||||
docker rm --force "$verify_container" >/dev/null 2>&1 || true
|
||||
fi
|
||||
docker buildx rm "$builder" >/dev/null 2>&1 || true
|
||||
rm -rf "$scratch"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
printf '[worker.oci]\n max-parallelism = 1\n gc = true\n reservedSpace = "1GB"\n maxUsedSpace = "12GB"\n' > "$scratch/buildkitd.toml"
|
||||
docker buildx create --name "$builder" --driver docker-container \
|
||||
--driver-opt "image=$BUILDKIT_IMAGE,memory=4g,memory-swap=4g,cpu-period=100000,cpu-quota=200000" \
|
||||
--buildkitd-config "$scratch/buildkitd.toml" >/dev/null
|
||||
timeout 120 docker buildx inspect "$builder" --bootstrap
|
||||
limits=$(docker inspect --format '{{.HostConfig.Memory}} {{.HostConfig.MemorySwap}} {{.HostConfig.CpuPeriod}} {{.HostConfig.CpuQuota}}' "$container")
|
||||
[[ "$limits" == '4294967296 4294967296 100000 200000' ]] || { echo "Builder limits not enforced: $limits" >&2; exit 1; }
|
||||
args=()
|
||||
for key in BASE_IMAGE DOCKER_IMAGE GOLANGCI_VERSION GOLANGCI_SHA256 PLAYWRIGHT_VERSION PLAYWRIGHT_SHA256 CHROMIUM_VERSION CHROMIUM_REVISION CHROMIUM_SHA256; do args+=(--build-arg "$key=${!key}"); done
|
||||
timeout 1800 docker buildx build --builder "$builder" --platform "$PLATFORM" --load --progress plain \
|
||||
"${args[@]}" -t "$tag" "$here"
|
||||
id=$(docker image inspect --format '{{.Id}}' "$tag")
|
||||
printf 'FULL_WORKER_IMAGE=%s\nIMAGE_ID=%s\nPLATFORM=%s\nBUILD_LIMITS=%s\n' "$tag" "$id" "$PLATFORM" "$limits"
|
||||
if [[ ${2:-} == --verify ]]; then
|
||||
verify_started=true
|
||||
timeout 600 docker run --rm --name "$verify_container" --init --cpus=2 --memory=4g --memory-swap=4g --pids-limit=512 \
|
||||
--user 1000:1000 --cap-drop=ALL --security-opt=no-new-privileges \
|
||||
--tmpfs /workspace:rw,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \
|
||||
-e HOME=/run/kandev/home "$id" bash -ceu 'mkdir -p "$HOME"; /opt/full-worker/smoke.sh --tools'
|
||||
fi
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Fast process-boundary checks; these do not establish Docker acceptance."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
class BuildCleanupTest(unittest.TestCase):
|
||||
def test_failed_verification_reclaims_its_container(self):
|
||||
recipe = Path(__file__).resolve().parent
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
docker = root / "docker"
|
||||
docker.write_text('''#!/bin/sh
|
||||
set -eu
|
||||
case "$1 $2" in
|
||||
'image inspect')
|
||||
case "$*" in *--format*) printf 'sha256:fixture-image\\n';; *) exit 1;; esac;;
|
||||
'image rm') :;;
|
||||
'buildx create'|'buildx inspect'|'buildx build'|'buildx rm') :;;
|
||||
'inspect --format') printf '4294967296 4294967296 100000 200000\\n';;
|
||||
'run --rm')
|
||||
name=unnamed
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = --name ]; then shift; name=$1; fi
|
||||
shift
|
||||
done
|
||||
printf '%s' "$name" > "$CONTAINER_STATE"
|
||||
exit 1;;
|
||||
'rm --force')
|
||||
test "$(cat "$CONTAINER_STATE")" = "$3"
|
||||
rm "$CONTAINER_STATE";;
|
||||
*) echo "Unexpected Docker operation: $*" >&2; exit 2;;
|
||||
esac
|
||||
''')
|
||||
docker.chmod(0o755)
|
||||
df = root / "df"
|
||||
df.write_text("#!/bin/sh\nprintf 'Filesystem 1024-blocks Used Available Capacity Mounted\\nfixture 99999999 0 99999999 0%% /\\n'\n")
|
||||
df.chmod(0o755)
|
||||
state = root / "container-state"
|
||||
result = subprocess.run(
|
||||
["bash", str(recipe / "build.sh"), "--build", "--verify"],
|
||||
env={**os.environ, "PATH": f"{root}:{os.environ['PATH']}",
|
||||
"CONTAINER_STATE": str(state)},
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
self.assertFalse(state.exists(), "failed verifier left its container alive")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Exercise daemon startup shell without starting Docker or proving CNI compatibility."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
|
||||
class DaemonRouteTest(unittest.TestCase):
|
||||
def run_startup(self, ipv4="", ipv6="", mtu="1400"):
|
||||
template = Path(__file__).with_name("pod-template.yaml").read_text()
|
||||
script = textwrap.dedent(template.split(" - |\n", 1)[1].split(" securityContext:", 1)[0])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
commands = {
|
||||
"ip": 'case "$*" in "-4 route show default") printf "%s\\n" "$IPV4";; "-6 route show default") printf "%s\\n" "$IPV6";; *) exit 1;; esac',
|
||||
"cat": 'test "$1" = /sys/class/net/net1/mtu || exit 1; printf "%s\\n" "$MTU"',
|
||||
"dockerd": 'printf "%s\\n" "$@"',
|
||||
}
|
||||
for name, body in commands.items():
|
||||
command = root / name
|
||||
command.write_text("#!/bin/sh\n" + body + "\n")
|
||||
command.chmod(0o755)
|
||||
return subprocess.run(
|
||||
["sh", "-ceu", script], capture_output=True, text=True, timeout=5,
|
||||
env={**os.environ, "PATH": f"{root}:{os.environ['PATH']}",
|
||||
"IPV4": ipv4, "IPV6": ipv6, "MTU": mtu},
|
||||
)
|
||||
|
||||
def test_non_eth0_default_routes(self):
|
||||
for routes in ({"ipv4": "default via 192.0.2.1 dev net1"},
|
||||
{"ipv6": "default via 2001:db8::1 dev net1"}):
|
||||
with self.subTest(routes=routes):
|
||||
result = self.run_startup(**routes)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("--mtu=1400\n", result.stdout)
|
||||
self.assertIn("--default-network-opt=bridge=com.docker.network.driver.mtu=1400", result.stdout)
|
||||
|
||||
def test_missing_route_and_invalid_mtu_fail_before_daemon(self):
|
||||
for options in ({}, {"ipv4": "default dev net1", "mtu": "invalid"},
|
||||
{"ipv4": "default dev net1", "mtu": "500"}):
|
||||
with self.subTest(options=options):
|
||||
result = self.run_startup(**options)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn("--host=", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,13 @@
|
||||
# Immutable linux/amd64 inputs. Existing worker presets are independent.
|
||||
PLATFORM=linux/amd64
|
||||
BASE_IMAGE=ghcr.io/kdlbs/kandev:0.93.0-universal@sha256:a485ac5bfb82f025cb60a57a31583803162ea031165ace97ac33ab2206158136
|
||||
DOCKER_IMAGE=docker.io/library/docker:29.1.5-dind@sha256:0ff14e6a992f2fdb7bef46123cd7fe61a58df0494e8df8bb6f9a5dac7486e1cb
|
||||
BUILDKIT_IMAGE=docker.io/moby/buildkit:v0.26.3@sha256:e5e990fee60e7aea97363281995ca9833e354361c2efefae6ca546cb75b1ccfc
|
||||
GOLANGCI_VERSION=2.9.0
|
||||
GOLANGCI_SHA256=493aaaca2eba6c8bcef847d92716bbd91bbac4b22cdbb0ab5b6a581b32946091
|
||||
PLAYWRIGHT_VERSION=1.61.1
|
||||
PLAYWRIGHT_SHA256=3740bba3a3e93b4c7bd7c0f6e3c0e5f8cdce498bef13511e4d534cc99a8ce198
|
||||
CHROMIUM_VERSION=149.0.7827.55
|
||||
CHROMIUM_REVISION=1228
|
||||
CHROMIUM_SHA256=410c9407d5de3fea80d9398666be06f2aa09154a3fa7b327dc254e336bb4c4b7
|
||||
PNPM_VERSION=9.15.9
|
||||
@@ -0,0 +1,59 @@
|
||||
apiVersion: v1
|
||||
kind: PodTemplate
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: kandev-agent
|
||||
# Replace with the immutable digest of your verified full worker build.
|
||||
image: FULL_WORKER_IMAGE_REQUIRED
|
||||
imagePullPolicy: IfNotPresent
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {cpu: '2', memory: 4Gi}
|
||||
env:
|
||||
# This socket controls the privileged daemon. The main container's
|
||||
# security context does not contain Docker commands. Use a worker
|
||||
# node pool isolated from trusted workloads.
|
||||
- {name: DOCKER_HOST, value: 'unix:///run/docker/docker.sock'}
|
||||
volumeMounts:
|
||||
- {name: docker-socket, mountPath: /run/docker}
|
||||
- name: docker-engine
|
||||
image: docker.io/library/docker:29.1.5-dind@sha256:0ff14e6a992f2fdb7bef46123cd7fe61a58df0494e8df8bb6f9a5dac7486e1cb
|
||||
# Bypass the image entrypoint, which otherwise enables TCP listeners.
|
||||
command: [/bin/sh]
|
||||
args:
|
||||
- -ceu
|
||||
- |
|
||||
iface=$({ ip -4 route show default; ip -6 route show default; } | awk '$1 == "default" {for (i=1; i<NF; i++) if ($i == "dev") {print $(i+1); exit}}')
|
||||
test -n "$iface"
|
||||
mtu=$(cat "/sys/class/net/$iface/mtu")
|
||||
test "$mtu" -ge 576
|
||||
exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
privileged: true
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {cpu: '2', memory: 3Gi}
|
||||
volumeMounts:
|
||||
- {name: kandev-workspace, mountPath: /workspace, readOnly: false}
|
||||
- {name: docker-socket, mountPath: /run/docker}
|
||||
- {name: docker-data, mountPath: /var/lib/docker}
|
||||
volumes:
|
||||
- name: docker-socket
|
||||
emptyDir: {medium: Memory, sizeLimit: 16Mi}
|
||||
- name: docker-data
|
||||
emptyDir: {sizeLimit: 12Gi}
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/bin/sh
|
||||
# Prepare a Kubernetes Pod workspace. The workspace may be a retained PVC
|
||||
# mounted into a replacement Pod, so repository materialization is idempotent.
|
||||
|
||||
set -eu
|
||||
|
||||
# Bound each client call as well as the overall readiness window.
|
||||
ready_timeout=${DOCKER_READY_TIMEOUT_SECONDS:-60}
|
||||
case "$ready_timeout" in ''|*[!0-9]*) echo 'Invalid Docker readiness timeout' >&2; exit 1;; esac
|
||||
[ "$ready_timeout" -ge 1 ] && [ "$ready_timeout" -le 120 ]
|
||||
deadline=$(( $(date +%s) + ready_timeout ))
|
||||
until timeout 3 docker info >/dev/null 2>&1; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo 'Docker daemon did not become ready before preparation deadline' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
create_workspace_caches() {
|
||||
mkdir -p "$workspace/.cache/npm" "$workspace/.cache/pip" "$workspace/.cache/go-build" \
|
||||
"$workspace/.cache/go-mod" "$workspace/.npm-global" "$workspace/.pnpm" \
|
||||
"$workspace/.pnpm-store" "$workspace/.cache/cargo"
|
||||
}
|
||||
|
||||
workspace={{workspace.path}}
|
||||
repository_url={{repository.clone_url}}
|
||||
repository_branch={{repository.branch}}
|
||||
clone_tmp=/opt/kandev/.workspace-clone
|
||||
|
||||
# ---- Git identity and HTTPS authentication ----
|
||||
{{git.identity_setup}}
|
||||
git config --global --add safe.directory '*'
|
||||
git config --global url."https://github.com/".insteadOf "git@github.com:"
|
||||
git config --global url."https://github.com/".insteadOf "ssh://git@github.com/"
|
||||
{{github.auth_setup}}
|
||||
|
||||
mkdir -p "$workspace"
|
||||
if [ -n "$repository_url" ]; then
|
||||
if git -C "$workspace" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
workspace_root=$(git -C "$workspace" rev-parse --show-toplevel)
|
||||
if [ "$workspace_root" != "$workspace" ]; then
|
||||
echo 'kandev: retained workspace repository root does not match the mount root' >&2
|
||||
exit 1
|
||||
fi
|
||||
workspace_origin=$(git -C "$workspace" remote get-url origin 2>/dev/null || true)
|
||||
expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
|
||||
retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
|
||||
if [ -z "$workspace_origin" ] || [ "$retained_origin" != "$expected_origin" ]; then
|
||||
echo 'kandev: retained workspace repository origin does not match the configured repository' >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# A fresh filesystem-backed PVC may contain only lost+found. Preserve it,
|
||||
# clone on the runtime emptyDir, then copy the checkout into the mount root.
|
||||
if find "$workspace" -mindepth 1 -maxdepth 1 ! -name lost+found -print -quit | grep -q .; then
|
||||
echo 'kandev: retained workspace is non-empty but is not a valid checkout' >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -rf "$clone_tmp"
|
||||
trap 'rm -rf "$clone_tmp"' 0 1 2 15
|
||||
git clone --depth=1 --branch "$repository_branch" "$repository_url" "$clone_tmp"
|
||||
cp -a "$clone_tmp"/. "$workspace"/
|
||||
rm -rf "$clone_tmp"
|
||||
trap - 0 1 2 15
|
||||
fi
|
||||
|
||||
cd "$workspace"
|
||||
|
||||
# Strip embedded token from remote URL to avoid persisting credentials.
|
||||
git remote set-url origin "$(git remote get-url origin | sed 's|https://[^@]*@github.com/|https://github.com/|')" 2>/dev/null || true
|
||||
|
||||
create_workspace_caches
|
||||
|
||||
# ---- Repository setup (if configured) ----
|
||||
{{repository.setup_script}}
|
||||
else
|
||||
cd "$workspace"
|
||||
create_workspace_caches
|
||||
fi
|
||||
|
||||
# ---- Pre-install agent CLI(s) ----
|
||||
{{kandev.agents.install}}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
image=${1:-}
|
||||
[[ $# == 1 && "$image" =~ ^[a-z0-9][a-z0-9./:_-]*@sha256:[a-f0-9]{64}$ ]] || {
|
||||
echo 'Usage: render-template.sh registry/image@sha256:<64 lowercase hex digits>' >&2
|
||||
exit 2
|
||||
}
|
||||
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
sed "s|FULL_WORKER_IMAGE_REQUIRED|$image|g" "$here/pod-template.yaml"
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mode=${1:---tools}
|
||||
[[ "$mode" == --tools || "$mode" == --all ]] || { echo 'Usage: smoke.sh --tools|--all' >&2; exit 2; }
|
||||
[[ $(id -u) == 1000 ]] || { echo 'Run smoke as UID 1000' >&2; exit 1; }
|
||||
work=$(mktemp -d /workspace/full-worker-smoke.XXXXXX)
|
||||
project="full-worker-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
|
||||
image="$project:smoke"
|
||||
cleanup() {
|
||||
if [[ "$mode" == --all ]]; then
|
||||
timeout 30 docker compose -p "$project" -f "$work/compose.yaml" down --volumes --remove-orphans >/dev/null 2>&1 || true
|
||||
timeout 15 docker image rm "$image" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
cd "$work"
|
||||
export GOMAXPROCS=2 GOFLAGS=-p=1 CARGO_BUILD_JOBS=2
|
||||
printf 'module fullworker\n\ngo 1.26.0\n' > go.mod
|
||||
cat > main.go <<'GO'
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
func validInput(data []byte) bool { return string(data)=="workspace-input\n" }
|
||||
|
||||
func main() {
|
||||
if len(os.Args)>1 && os.Args[1]=="fetch" {
|
||||
response,err:=http.Get("https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz")
|
||||
if err!=nil {panic(err)}
|
||||
defer func(){ _ = response.Body.Close() }()
|
||||
if response.StatusCode!=200 {panic(response.Status)}
|
||||
n,err:=io.Copy(io.Discard,response.Body)
|
||||
if err!=nil || n<100000 {panic("incomplete HTTPS transfer")}
|
||||
fmt.Println("https-transfer-ok")
|
||||
return
|
||||
}
|
||||
data,err:=os.ReadFile("/input/source.txt");if err!=nil {panic(err)}
|
||||
if !validInput(data) {panic("wrong bind source")}
|
||||
if err:=os.WriteFile("/input/forbidden",data,0600);err==nil {panic("read-only bind accepted write")}
|
||||
if err:=os.WriteFile("/output/result.txt",data,0600);err!=nil {panic(err)}
|
||||
fmt.Println("compose-bind-ok")
|
||||
}
|
||||
GO
|
||||
cat > main_test.go <<'GO'
|
||||
package main
|
||||
import "testing"
|
||||
func TestSource(t *testing.T) {for _,test:=range []struct{input string;valid bool}{{"workspace-input\n",true},{"workspace-input",false},{"",false}} {if validInput([]byte(test.input))!=test.valid {t.Fatalf("input %q",test.input)}}}
|
||||
GO
|
||||
timeout 120 go test ./...
|
||||
printf 'version: "2"\n' > .golangci.yml
|
||||
timeout 120 golangci-lint run ./...
|
||||
CGO_ENABLED=0 timeout 120 go build -o bind-smoke .
|
||||
printf '{"name":"full-worker-smoke","version":"1.0.0","scripts":{"test":"node --test source.test.cjs"}}\n' > package.json
|
||||
printf "const {test}=require('node:test');const assert=require('node:assert/strict');test('source',()=>assert.equal(6*7,42));\n" > source.test.cjs
|
||||
timeout 30 pnpm test
|
||||
python3 -m venv .venv
|
||||
printf 'import unittest\nclass Source(unittest.TestCase):\n def test_source(self): self.assertEqual(6*7,42)\n' > test_source.py
|
||||
timeout 30 .venv/bin/python -m unittest test_source.py
|
||||
printf '#include <stdio.h>\nint main(void){puts("native-ok");return 0;}\n' > native.c
|
||||
cc -Wall -Werror native.c -o native
|
||||
test "$(./native)" = native-ok
|
||||
mkdir rust
|
||||
printf '[package]\nname="full-worker-smoke"\nversion="0.1.0"\nedition="2021"\n' > rust/Cargo.toml
|
||||
mkdir rust/src
|
||||
printf '#[test] fn source(){assert_eq!(6*7,42);}\n' > rust/src/lib.rs
|
||||
timeout 120 cargo test --offline --manifest-path rust/Cargo.toml
|
||||
node <<'JS'
|
||||
const {chromium}=require('/opt/full-worker/playwright-core');
|
||||
(async()=>{
|
||||
const browser=await chromium.launch({headless:true,timeout:30000,args:['--disable-dev-shm-usage']});
|
||||
try {const page=await browser.newPage();await page.setContent(`<button onclick="this.textContent='clicked'">run</button>`);await page.getByRole('button').click();if(await page.getByRole('button').textContent()!=='clicked')throw Error('browser interaction failed');await page.screenshot({path:'browser.png'});}
|
||||
finally {await browser.close();}
|
||||
})().catch(e=>{console.error(e);process.exitCode=1;});
|
||||
JS
|
||||
test -s browser.png
|
||||
printf 'source-browser-ok\n'
|
||||
if [[ "$mode" == --all ]]; then
|
||||
timeout 10 docker info >/dev/null
|
||||
cat > Dockerfile <<'DOCKER'
|
||||
FROM scratch
|
||||
COPY bind-smoke /bind-smoke
|
||||
COPY ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/bind-smoke"]
|
||||
DOCKER
|
||||
cp /etc/ssl/certs/ca-certificates.crt .
|
||||
mkdir input output
|
||||
printf 'workspace-input\n' > input/source.txt
|
||||
timeout 120 docker build --network=none -t "$image" .
|
||||
timeout 60 docker run --rm --memory=128m --cpus=0.5 "$image" fetch
|
||||
cat > compose.yaml <<COMPOSE
|
||||
services:
|
||||
bind:
|
||||
image: $image
|
||||
user: "1000:1000"
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
volumes:
|
||||
- type: bind
|
||||
source: $work/input
|
||||
target: /input
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: $work/output
|
||||
target: /output
|
||||
COMPOSE
|
||||
timeout 60 docker compose -p "$project" -f compose.yaml run --rm bind
|
||||
cmp input/source.txt output/result.txt
|
||||
test ! -e input/forbidden
|
||||
timeout 30 docker compose -p "$project" -f compose.yaml down --volumes --remove-orphans
|
||||
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")"
|
||||
printf 'source-browser-docker-compose-ok\n' > /workspace/full-worker-result.txt
|
||||
else
|
||||
printf 'source-browser-ok\n' > /workspace/full-worker-result.txt
|
||||
fi
|
||||
go version
|
||||
rustc --version
|
||||
node --version
|
||||
pnpm --version
|
||||
python3 --version
|
||||
golangci-lint version
|
||||
docker --version
|
||||
docker buildx version
|
||||
docker compose version
|
||||
cat /workspace/full-worker-result.txt
|
||||
Reference in New Issue
Block a user