feat(kubernetes): add explicit Docker workload recipe (#3648)
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
ARG DOCKER_IMAGE
|
||||
ARG BASE_IMAGE
|
||||
FROM ${DOCKER_IMAGE} AS docker_tools
|
||||
FROM ${BASE_IMAGE}
|
||||
USER root
|
||||
ARG GOLANGCI_VERSION
|
||||
ARG GOLANGCI_SHA256
|
||||
ARG PLAYWRIGHT_VERSION
|
||||
ARG PLAYWRIGHT_SHA256
|
||||
ARG CHROMIUM_VERSION
|
||||
ARG CHROMIUM_REVISION
|
||||
ARG CHROMIUM_SHA256
|
||||
COPY --from=docker_tools /usr/local/bin/docker /usr/local/bin/docker
|
||||
COPY --from=docker_tools /usr/local/libexec/docker/cli-plugins/ /usr/local/libexec/docker/cli-plugins/
|
||||
RUN set -eu; \
|
||||
curl -fsSL --max-time 180 "https://github.com/golangci/golangci-lint/releases/download/v${GOLANGCI_VERSION}/golangci-lint-${GOLANGCI_VERSION}-linux-amd64.tar.gz" -o /tmp/lint.tgz; \
|
||||
echo "${GOLANGCI_SHA256} /tmp/lint.tgz" | sha256sum -c -; \
|
||||
tar -xzf /tmp/lint.tgz -C /tmp; \
|
||||
cp "/tmp/golangci-lint-${GOLANGCI_VERSION}-linux-amd64/golangci-lint" /usr/local/bin/golangci-lint; \
|
||||
rm -rf /tmp/lint.tgz "/tmp/golangci-lint-${GOLANGCI_VERSION}-linux-amd64"
|
||||
RUN set -eu; \
|
||||
mkdir -p /opt/full-worker/playwright-core /opt/full-worker/browsers; \
|
||||
curl -fsSL --max-time 180 "https://registry.npmjs.org/playwright-core/-/playwright-core-${PLAYWRIGHT_VERSION}.tgz" -o /tmp/playwright.tgz; \
|
||||
echo "${PLAYWRIGHT_SHA256} /tmp/playwright.tgz" | sha256sum -c -; \
|
||||
tar -xzf /tmp/playwright.tgz --strip-components=1 -C /opt/full-worker/playwright-core; \
|
||||
curl -fsSL --max-time 180 "https://cdn.playwright.dev/builds/cft/${CHROMIUM_VERSION}/linux64/chrome-headless-shell-linux64.zip" -o /tmp/chromium.zip; \
|
||||
echo "${CHROMIUM_SHA256} /tmp/chromium.zip" | sha256sum -c -; \
|
||||
python3 -m zipfile -e /tmp/chromium.zip "/opt/full-worker/browsers/chromium_headless_shell-${CHROMIUM_REVISION}"; \
|
||||
chmod -R a+rX /opt/full-worker; \
|
||||
chmod a+rx "/opt/full-worker/browsers/chromium_headless_shell-${CHROMIUM_REVISION}/chrome-headless-shell-linux64/chrome-headless-shell"; \
|
||||
rm /tmp/playwright.tgz /tmp/chromium.zip
|
||||
ENV PLAYWRIGHT_BROWSERS_PATH=/opt/full-worker/browsers \
|
||||
npm_config_prefix=/workspace/.npm-global \
|
||||
npm_config_cache=/workspace/.cache/npm \
|
||||
PNPM_HOME=/workspace/.pnpm \
|
||||
npm_config_store_dir=/workspace/.pnpm-store \
|
||||
PIP_CACHE_DIR=/workspace/.cache/pip \
|
||||
GOCACHE=/workspace/.cache/go-build \
|
||||
GOMODCACHE=/workspace/.cache/go-mod \
|
||||
CARGO_HOME=/workspace/.cache/cargo \
|
||||
RUSTUP_HOME=/usr/local/rustup \
|
||||
GOMAXPROCS=2 \
|
||||
GOFLAGS=-p=1 \
|
||||
CARGO_BUILD_JOBS=2
|
||||
ENV PATH=/workspace/.npm-global/bin:/workspace/.pnpm:/usr/local/cargo/bin:/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin
|
||||
COPY smoke.sh /opt/full-worker/smoke.sh
|
||||
RUN chmod 0755 /opt/full-worker/smoke.sh
|
||||
USER 1000:1000
|
||||
WORKDIR /workspace
|
||||
ENTRYPOINT []
|
||||
CMD ["sleep", "infinity"]
|
||||
@@ -0,0 +1,94 @@
|
||||
# Full Kubernetes worker with per-Pod Docker
|
||||
|
||||
This opt-in Linux/amd64 recipe extends the pinned universal image with
|
||||
Go-compatible golangci-lint v2, Playwright Chromium and Docker CLI/Buildx/Compose.
|
||||
The existing minimal, Node/pnpm and Python presets are unchanged.
|
||||
|
||||
## Build and verify
|
||||
|
||||
```bash
|
||||
bash k8s/worker-images/full/build.sh --check
|
||||
bash k8s/worker-images/full/build.sh --build --verify
|
||||
```
|
||||
|
||||
The script requires Docker Buildx, at least 20 GiB free disk and capacity for a
|
||||
2-CPU, 4-GiB disposable builder. It validates actual Docker resource controls,
|
||||
uses one BuildKit step at a time and removes its exact builder on exit. It
|
||||
prints the local image tag, image ID, platform and limits. It never publishes
|
||||
an image. The output image remains available for testing; remove that exact
|
||||
tag when finished. Source/browser verification is bounded to ten minutes and
|
||||
runs as UID/GID 1000 with dropped capabilities and no host mounts.
|
||||
|
||||
`pins.env` records immutable base, daemon and builder manifests, plus checksums
|
||||
for every added download. Rust, Go, Node, pnpm, Python/venv and native compilers
|
||||
are inherited from the immutable universal image. The smoke prints the actual
|
||||
versions. The browser revision matches Playwright 1.61.1 in the repository
|
||||
lockfile. Only Chromium headless shell is baked, outside the managed mounts.
|
||||
|
||||
## Configure an executor profile
|
||||
|
||||
1. Distribute your verified image through your own registry and replace
|
||||
`FULL_WORKER_IMAGE_REQUIRED` using `bash k8s/worker-images/full/render-template.sh
|
||||
registry/image@sha256:<digest>`. The renderer rejects placeholders and mutable
|
||||
tags. The unrendered example is documentation only and must not be deployed.
|
||||
2. Select `linux/amd64`, main container `kandev-agent`, and your workspace mode
|
||||
in an existing Kubernetes executor profile. Managed PVCs preserve results.
|
||||
3. Paste the template into the profile's raw Pod template and `prepare.sh` into
|
||||
its prepare script. Preserve the placeholders: Kandev resolves repository,
|
||||
identity, authentication, setup and selected-agent installation at launch.
|
||||
4. Review namespace privilege policy, image access, resources and storage before
|
||||
starting a disposable task. No cluster policy is installed by this recipe.
|
||||
|
||||
The main agent stays non-root. The explicitly privileged root daemon listens
|
||||
only on a Pod-local Unix socket, accessible through GID 1000. Both containers
|
||||
share `/workspace`; the daemon receives neither Kandev runtime nor auth mounts.
|
||||
Its Docker data is a separate disposable 12-GiB `emptyDir`. Pod-wide non-root
|
||||
policy is deliberately absent because it would conflict with this daemon.
|
||||
No host namespaces, hostPath or service-account token are requested.
|
||||
|
||||
The agent can control the privileged daemon through that socket. The main
|
||||
container's non-root security context does not contain commands that use Docker.
|
||||
Schedule these Pods only on a worker node pool isolated from trusted workloads.
|
||||
|
||||
The daemon reads the Pod interface MTU and applies it to default and
|
||||
user-defined bridges. Verify nested HTTPS transfers for the actual CNI/runtime.
|
||||
The preparation script waits at most 60 seconds (plus a bounded client call)
|
||||
for Docker before the standard clone/origin verification, repository setup and
|
||||
agent installation. Kandev still adds task-branch preparation. Cache directories
|
||||
are created only after repository materialization. Kandev owns `HOME`; baked
|
||||
tools and browsers remain outside `/workspace` and `/run/kandev`.
|
||||
|
||||
## Exercise source and Docker execution
|
||||
|
||||
Inside the agent container:
|
||||
|
||||
```bash
|
||||
bash /opt/full-worker/smoke.sh --all
|
||||
cat /workspace/full-worker-result.txt
|
||||
```
|
||||
|
||||
The smoke executes Go tests/lint/build, Rust tests, Node/pnpm tests, Python/venv
|
||||
tests, a C compilation and a real Chromium page interaction/screenshot. Docker
|
||||
acceptance additionally builds/runs a scratch image, performs a large HTTPS
|
||||
transfer and runs Compose with RO input and RW output workspace binds. It
|
||||
checks files and removes only its own Compose project and image.
|
||||
|
||||
Ordinary Stop retains the Pod, daemon and nested containers. Clean up test
|
||||
Compose projects explicitly. Resume retains workspace results; lost-Pod
|
||||
replacement uses the recorded template/PVC and loses disposable daemon state.
|
||||
Terminal cleanup deletes only resources proven to belong to the session;
|
||||
existing claims remain operator-owned.
|
||||
|
||||
## Compatibility limits
|
||||
|
||||
Real acceptance results are recorded in the implementation work orders. A
|
||||
successful image smoke alone does not establish Kubernetes Docker acceptance.
|
||||
Kind evidence applies only to the recorded architecture and runtime. Measure
|
||||
nested cgroup ancestry and counters before claiming resource enforcement;
|
||||
privileged DinD is not an adversarial isolation boundary.
|
||||
|
||||
Only `/workspace` is shared with the daemon. Agent-only HOME/temp bind sources,
|
||||
callbacks, SSH fixtures and arbitrary Docker plugins are not promised to work.
|
||||
This recipe does not establish full Kandev Docker-executor parity, provision a
|
||||
control plane or update an existing service. The backend must contain the
|
||||
explicit workspace-grant implementation before accepting this template.
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Test-only bounded nested process. The caller must delete its owned Pod even
|
||||
# when a cgroup assertion fails; the process also exits after 90 seconds.
|
||||
set -eu
|
||||
work=$(mktemp -d /workspace/full-accounting.XXXXXX)
|
||||
trap 'rm -rf "$work"' EXIT
|
||||
cd "$work"
|
||||
cat > main.go <<'GO'
|
||||
package main
|
||||
import ("runtime";"time")
|
||||
func main(){
|
||||
data:=make([]byte,32*1024*1024)
|
||||
for i:=range data {data[i]=1}
|
||||
until:=time.Now().Add(90*time.Second)
|
||||
for time.Now().Before(until) {for i:=range data {data[i]++};runtime.KeepAlive(data)}
|
||||
}
|
||||
GO
|
||||
CGO_ENABLED=0 GOMAXPROCS=2 timeout 120 go build -p 1 -o accounting main.go
|
||||
printf 'FROM scratch\nCOPY accounting /accounting\nENTRYPOINT ["/accounting"]\n' > Dockerfile
|
||||
timeout 120 docker build --network=none -t full-worker-accounting . >&2
|
||||
# Its private daemon/image is discarded with the test Pod.
|
||||
timeout 10 docker run -d --name full-worker-accounting --memory=128m --cpus=0.25 full-worker-accounting
|
||||
@@ -0,0 +1,58 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
source "$here/pins.env"
|
||||
mode=${1:---check}
|
||||
[[ "$mode" == --check || "$mode" == --build ]] || { echo 'Usage: build.sh --check|--build [--verify]' >&2; exit 2; }
|
||||
[[ $# -le 2 && (${2:-} == '' || ${2:-} == --verify) ]] || exit 2
|
||||
for key in BASE_IMAGE DOCKER_IMAGE BUILDKIT_IMAGE; do
|
||||
[[ ${!key} =~ @sha256:[a-f0-9]{64}$ ]] || { echo "Missing immutable $key" >&2; exit 1; }
|
||||
done
|
||||
for key in GOLANGCI_SHA256 PLAYWRIGHT_SHA256 CHROMIUM_SHA256; do
|
||||
[[ ${!key} =~ ^[a-f0-9]{64}$ ]] || { echo "Missing checksum $key" >&2; exit 1; }
|
||||
done
|
||||
for key in GOLANGCI_VERSION PLAYWRIGHT_VERSION CHROMIUM_VERSION CHROMIUM_REVISION PNPM_VERSION; do
|
||||
[[ ${!key} =~ ^[0-9]+(\.[0-9]+)*$ ]] || { echo "Missing version $key" >&2; exit 1; }
|
||||
done
|
||||
[[ "$PLATFORM" == linux/amd64 ]]
|
||||
grep -Fq "$DOCKER_IMAGE" "$here/pod-template.yaml"
|
||||
for file in Dockerfile smoke.sh prepare.sh pod-template.yaml; do test -s "$here/$file"; done
|
||||
if [[ "$mode" == --check ]]; then echo 'Full worker immutable inputs and recipe files valid'; exit; fi
|
||||
# The dedicated builder enforces limits; shell variables alone do not bound builds.
|
||||
free_kb=$(df -Pk "$here" | awk 'END {print $4}')
|
||||
(( free_kb >= 20*1024*1024 )) || { echo 'Build requires at least 20 GiB free disk' >&2; exit 1; }
|
||||
scratch=$(mktemp -d)
|
||||
builder="kandev-full-$(basename "$scratch" | tr '[:upper:].' '[:lower:]-')"
|
||||
tag=${FULL_WORKER_TAG:-"kandev-full-worker:$builder"}
|
||||
if docker image inspect "$tag" >/dev/null 2>&1; then echo 'Refusing to overwrite an existing image tag' >&2; exit 1; fi
|
||||
container="buildx_buildkit_${builder}0"
|
||||
verify_started=false
|
||||
verify_container="${builder}-verify"
|
||||
cleanup() {
|
||||
if [[ "$verify_started" == true ]]; then
|
||||
docker rm --force "$verify_container" >/dev/null 2>&1 || true
|
||||
fi
|
||||
docker buildx rm "$builder" >/dev/null 2>&1 || true
|
||||
rm -rf "$scratch"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
printf '[worker.oci]\n max-parallelism = 1\n gc = true\n reservedSpace = "1GB"\n maxUsedSpace = "12GB"\n' > "$scratch/buildkitd.toml"
|
||||
docker buildx create --name "$builder" --driver docker-container \
|
||||
--driver-opt "image=$BUILDKIT_IMAGE,memory=4g,memory-swap=4g,cpu-period=100000,cpu-quota=200000" \
|
||||
--buildkitd-config "$scratch/buildkitd.toml" >/dev/null
|
||||
timeout 120 docker buildx inspect "$builder" --bootstrap
|
||||
limits=$(docker inspect --format '{{.HostConfig.Memory}} {{.HostConfig.MemorySwap}} {{.HostConfig.CpuPeriod}} {{.HostConfig.CpuQuota}}' "$container")
|
||||
[[ "$limits" == '4294967296 4294967296 100000 200000' ]] || { echo "Builder limits not enforced: $limits" >&2; exit 1; }
|
||||
args=()
|
||||
for key in BASE_IMAGE DOCKER_IMAGE GOLANGCI_VERSION GOLANGCI_SHA256 PLAYWRIGHT_VERSION PLAYWRIGHT_SHA256 CHROMIUM_VERSION CHROMIUM_REVISION CHROMIUM_SHA256; do args+=(--build-arg "$key=${!key}"); done
|
||||
timeout 1800 docker buildx build --builder "$builder" --platform "$PLATFORM" --load --progress plain \
|
||||
"${args[@]}" -t "$tag" "$here"
|
||||
id=$(docker image inspect --format '{{.Id}}' "$tag")
|
||||
printf 'FULL_WORKER_IMAGE=%s\nIMAGE_ID=%s\nPLATFORM=%s\nBUILD_LIMITS=%s\n' "$tag" "$id" "$PLATFORM" "$limits"
|
||||
if [[ ${2:-} == --verify ]]; then
|
||||
verify_started=true
|
||||
timeout 600 docker run --rm --name "$verify_container" --init --cpus=2 --memory=4g --memory-swap=4g --pids-limit=512 \
|
||||
--user 1000:1000 --cap-drop=ALL --security-opt=no-new-privileges \
|
||||
--tmpfs /workspace:rw,uid=1000,gid=1000,size=2g --tmpfs /run/kandev:rw,uid=1000,gid=1000,size=16m \
|
||||
-e HOME=/run/kandev/home "$id" bash -ceu 'mkdir -p "$HOME"; /opt/full-worker/smoke.sh --tools'
|
||||
fi
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Fast process-boundary checks; these do not establish Docker acceptance."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
class BuildCleanupTest(unittest.TestCase):
|
||||
def test_failed_verification_reclaims_its_container(self):
|
||||
recipe = Path(__file__).resolve().parent
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
docker = root / "docker"
|
||||
docker.write_text('''#!/bin/sh
|
||||
set -eu
|
||||
case "$1 $2" in
|
||||
'image inspect')
|
||||
case "$*" in *--format*) printf 'sha256:fixture-image\\n';; *) exit 1;; esac;;
|
||||
'image rm') :;;
|
||||
'buildx create'|'buildx inspect'|'buildx build'|'buildx rm') :;;
|
||||
'inspect --format') printf '4294967296 4294967296 100000 200000\\n';;
|
||||
'run --rm')
|
||||
name=unnamed
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = --name ]; then shift; name=$1; fi
|
||||
shift
|
||||
done
|
||||
printf '%s' "$name" > "$CONTAINER_STATE"
|
||||
exit 1;;
|
||||
'rm --force')
|
||||
test "$(cat "$CONTAINER_STATE")" = "$3"
|
||||
rm "$CONTAINER_STATE";;
|
||||
*) echo "Unexpected Docker operation: $*" >&2; exit 2;;
|
||||
esac
|
||||
''')
|
||||
docker.chmod(0o755)
|
||||
df = root / "df"
|
||||
df.write_text("#!/bin/sh\nprintf 'Filesystem 1024-blocks Used Available Capacity Mounted\\nfixture 99999999 0 99999999 0%% /\\n'\n")
|
||||
df.chmod(0o755)
|
||||
state = root / "container-state"
|
||||
result = subprocess.run(
|
||||
["bash", str(recipe / "build.sh"), "--build", "--verify"],
|
||||
env={**os.environ, "PATH": f"{root}:{os.environ['PATH']}",
|
||||
"CONTAINER_STATE": str(state)},
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0, result.stdout)
|
||||
self.assertFalse(state.exists(), "failed verifier left its container alive")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Exercise daemon startup shell without starting Docker or proving CNI compatibility."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
|
||||
class DaemonRouteTest(unittest.TestCase):
|
||||
def run_startup(self, ipv4="", ipv6="", mtu="1400"):
|
||||
template = Path(__file__).with_name("pod-template.yaml").read_text()
|
||||
script = textwrap.dedent(template.split(" - |\n", 1)[1].split(" securityContext:", 1)[0])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
commands = {
|
||||
"ip": 'case "$*" in "-4 route show default") printf "%s\\n" "$IPV4";; "-6 route show default") printf "%s\\n" "$IPV6";; *) exit 1;; esac',
|
||||
"cat": 'test "$1" = /sys/class/net/net1/mtu || exit 1; printf "%s\\n" "$MTU"',
|
||||
"dockerd": 'printf "%s\\n" "$@"',
|
||||
}
|
||||
for name, body in commands.items():
|
||||
command = root / name
|
||||
command.write_text("#!/bin/sh\n" + body + "\n")
|
||||
command.chmod(0o755)
|
||||
return subprocess.run(
|
||||
["sh", "-ceu", script], capture_output=True, text=True, timeout=5,
|
||||
env={**os.environ, "PATH": f"{root}:{os.environ['PATH']}",
|
||||
"IPV4": ipv4, "IPV6": ipv6, "MTU": mtu},
|
||||
)
|
||||
|
||||
def test_non_eth0_default_routes(self):
|
||||
for routes in ({"ipv4": "default via 192.0.2.1 dev net1"},
|
||||
{"ipv6": "default via 2001:db8::1 dev net1"}):
|
||||
with self.subTest(routes=routes):
|
||||
result = self.run_startup(**routes)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertIn("--mtu=1400\n", result.stdout)
|
||||
self.assertIn("--default-network-opt=bridge=com.docker.network.driver.mtu=1400", result.stdout)
|
||||
|
||||
def test_missing_route_and_invalid_mtu_fail_before_daemon(self):
|
||||
for options in ({}, {"ipv4": "default dev net1", "mtu": "invalid"},
|
||||
{"ipv4": "default dev net1", "mtu": "500"}):
|
||||
with self.subTest(options=options):
|
||||
result = self.run_startup(**options)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertNotIn("--host=", result.stdout)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,13 @@
|
||||
# Immutable linux/amd64 inputs. Existing worker presets are independent.
|
||||
PLATFORM=linux/amd64
|
||||
BASE_IMAGE=ghcr.io/kdlbs/kandev:0.93.0-universal@sha256:a485ac5bfb82f025cb60a57a31583803162ea031165ace97ac33ab2206158136
|
||||
DOCKER_IMAGE=docker.io/library/docker:29.1.5-dind@sha256:0ff14e6a992f2fdb7bef46123cd7fe61a58df0494e8df8bb6f9a5dac7486e1cb
|
||||
BUILDKIT_IMAGE=docker.io/moby/buildkit:v0.26.3@sha256:e5e990fee60e7aea97363281995ca9833e354361c2efefae6ca546cb75b1ccfc
|
||||
GOLANGCI_VERSION=2.9.0
|
||||
GOLANGCI_SHA256=493aaaca2eba6c8bcef847d92716bbd91bbac4b22cdbb0ab5b6a581b32946091
|
||||
PLAYWRIGHT_VERSION=1.61.1
|
||||
PLAYWRIGHT_SHA256=3740bba3a3e93b4c7bd7c0f6e3c0e5f8cdce498bef13511e4d534cc99a8ce198
|
||||
CHROMIUM_VERSION=149.0.7827.55
|
||||
CHROMIUM_REVISION=1228
|
||||
CHROMIUM_SHA256=410c9407d5de3fea80d9398666be06f2aa09154a3fa7b327dc254e336bb4c4b7
|
||||
PNPM_VERSION=9.15.9
|
||||
@@ -0,0 +1,59 @@
|
||||
apiVersion: v1
|
||||
kind: PodTemplate
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
securityContext:
|
||||
fsGroup: 1000
|
||||
containers:
|
||||
- name: kandev-agent
|
||||
# Replace with the immutable digest of your verified full worker build.
|
||||
image: FULL_WORKER_IMAGE_REQUIRED
|
||||
imagePullPolicy: IfNotPresent
|
||||
securityContext:
|
||||
runAsUser: 1000
|
||||
runAsGroup: 1000
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {cpu: '2', memory: 4Gi}
|
||||
env:
|
||||
# This socket controls the privileged daemon. The main container's
|
||||
# security context does not contain Docker commands. Use a worker
|
||||
# node pool isolated from trusted workloads.
|
||||
- {name: DOCKER_HOST, value: 'unix:///run/docker/docker.sock'}
|
||||
volumeMounts:
|
||||
- {name: docker-socket, mountPath: /run/docker}
|
||||
- name: docker-engine
|
||||
image: docker.io/library/docker:29.1.5-dind@sha256:0ff14e6a992f2fdb7bef46123cd7fe61a58df0494e8df8bb6f9a5dac7486e1cb
|
||||
# Bypass the image entrypoint, which otherwise enables TCP listeners.
|
||||
command: [/bin/sh]
|
||||
args:
|
||||
- -ceu
|
||||
- |
|
||||
iface=$({ ip -4 route show default; ip -6 route show default; } | awk '$1 == "default" {for (i=1; i<NF; i++) if ($i == "dev") {print $(i+1); exit}}')
|
||||
test -n "$iface"
|
||||
mtu=$(cat "/sys/class/net/$iface/mtu")
|
||||
test "$mtu" -ge 576
|
||||
exec dockerd --host=unix:///run/docker/docker.sock --group=1000 --mtu="$mtu" --default-network-opt=bridge=com.docker.network.driver.mtu="$mtu" --storage-driver=overlay2
|
||||
securityContext:
|
||||
runAsUser: 0
|
||||
runAsGroup: 0
|
||||
privileged: true
|
||||
resources:
|
||||
requests: {cpu: 250m, memory: 512Mi}
|
||||
limits: {cpu: '2', memory: 3Gi}
|
||||
volumeMounts:
|
||||
- {name: kandev-workspace, mountPath: /workspace, readOnly: false}
|
||||
- {name: docker-socket, mountPath: /run/docker}
|
||||
- {name: docker-data, mountPath: /var/lib/docker}
|
||||
volumes:
|
||||
- name: docker-socket
|
||||
emptyDir: {medium: Memory, sizeLimit: 16Mi}
|
||||
- name: docker-data
|
||||
emptyDir: {sizeLimit: 12Gi}
|
||||
@@ -0,0 +1,83 @@
|
||||
#!/bin/sh
|
||||
# Prepare a Kubernetes Pod workspace. The workspace may be a retained PVC
|
||||
# mounted into a replacement Pod, so repository materialization is idempotent.
|
||||
|
||||
set -eu
|
||||
|
||||
# Bound each client call as well as the overall readiness window.
|
||||
ready_timeout=${DOCKER_READY_TIMEOUT_SECONDS:-60}
|
||||
case "$ready_timeout" in ''|*[!0-9]*) echo 'Invalid Docker readiness timeout' >&2; exit 1;; esac
|
||||
[ "$ready_timeout" -ge 1 ] && [ "$ready_timeout" -le 120 ]
|
||||
deadline=$(( $(date +%s) + ready_timeout ))
|
||||
until timeout 3 docker info >/dev/null 2>&1; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo 'Docker daemon did not become ready before preparation deadline' >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
create_workspace_caches() {
|
||||
mkdir -p "$workspace/.cache/npm" "$workspace/.cache/pip" "$workspace/.cache/go-build" \
|
||||
"$workspace/.cache/go-mod" "$workspace/.npm-global" "$workspace/.pnpm" \
|
||||
"$workspace/.pnpm-store" "$workspace/.cache/cargo"
|
||||
}
|
||||
|
||||
workspace={{workspace.path}}
|
||||
repository_url={{repository.clone_url}}
|
||||
repository_branch={{repository.branch}}
|
||||
clone_tmp=/opt/kandev/.workspace-clone
|
||||
|
||||
# ---- Git identity and HTTPS authentication ----
|
||||
{{git.identity_setup}}
|
||||
git config --global --add safe.directory '*'
|
||||
git config --global url."https://github.com/".insteadOf "git@github.com:"
|
||||
git config --global url."https://github.com/".insteadOf "ssh://git@github.com/"
|
||||
{{github.auth_setup}}
|
||||
|
||||
mkdir -p "$workspace"
|
||||
if [ -n "$repository_url" ]; then
|
||||
if git -C "$workspace" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
workspace_root=$(git -C "$workspace" rev-parse --show-toplevel)
|
||||
if [ "$workspace_root" != "$workspace" ]; then
|
||||
echo 'kandev: retained workspace repository root does not match the mount root' >&2
|
||||
exit 1
|
||||
fi
|
||||
workspace_origin=$(git -C "$workspace" remote get-url origin 2>/dev/null || true)
|
||||
expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
|
||||
retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
|
||||
if [ -z "$workspace_origin" ] || [ "$retained_origin" != "$expected_origin" ]; then
|
||||
echo 'kandev: retained workspace repository origin does not match the configured repository' >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# A fresh filesystem-backed PVC may contain only lost+found. Preserve it,
|
||||
# clone on the runtime emptyDir, then copy the checkout into the mount root.
|
||||
if find "$workspace" -mindepth 1 -maxdepth 1 ! -name lost+found -print -quit | grep -q .; then
|
||||
echo 'kandev: retained workspace is non-empty but is not a valid checkout' >&2
|
||||
exit 1
|
||||
fi
|
||||
rm -rf "$clone_tmp"
|
||||
trap 'rm -rf "$clone_tmp"' 0 1 2 15
|
||||
git clone --depth=1 --branch "$repository_branch" "$repository_url" "$clone_tmp"
|
||||
cp -a "$clone_tmp"/. "$workspace"/
|
||||
rm -rf "$clone_tmp"
|
||||
trap - 0 1 2 15
|
||||
fi
|
||||
|
||||
cd "$workspace"
|
||||
|
||||
# Strip embedded token from remote URL to avoid persisting credentials.
|
||||
git remote set-url origin "$(git remote get-url origin | sed 's|https://[^@]*@github.com/|https://github.com/|')" 2>/dev/null || true
|
||||
|
||||
create_workspace_caches
|
||||
|
||||
# ---- Repository setup (if configured) ----
|
||||
{{repository.setup_script}}
|
||||
else
|
||||
cd "$workspace"
|
||||
create_workspace_caches
|
||||
fi
|
||||
|
||||
# ---- Pre-install agent CLI(s) ----
|
||||
{{kandev.agents.install}}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
image=${1:-}
|
||||
[[ $# == 1 && "$image" =~ ^[a-z0-9][a-z0-9./:_-]*@sha256:[a-f0-9]{64}$ ]] || {
|
||||
echo 'Usage: render-template.sh registry/image@sha256:<64 lowercase hex digits>' >&2
|
||||
exit 2
|
||||
}
|
||||
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
sed "s|FULL_WORKER_IMAGE_REQUIRED|$image|g" "$here/pod-template.yaml"
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mode=${1:---tools}
|
||||
[[ "$mode" == --tools || "$mode" == --all ]] || { echo 'Usage: smoke.sh --tools|--all' >&2; exit 2; }
|
||||
[[ $(id -u) == 1000 ]] || { echo 'Run smoke as UID 1000' >&2; exit 1; }
|
||||
work=$(mktemp -d /workspace/full-worker-smoke.XXXXXX)
|
||||
project="full-worker-$(basename "$work" | tr '[:upper:].' '[:lower:]-')"
|
||||
image="$project:smoke"
|
||||
cleanup() {
|
||||
if [[ "$mode" == --all ]]; then
|
||||
timeout 30 docker compose -p "$project" -f "$work/compose.yaml" down --volumes --remove-orphans >/dev/null 2>&1 || true
|
||||
timeout 15 docker image rm "$image" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf "$work"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
cd "$work"
|
||||
export GOMAXPROCS=2 GOFLAGS=-p=1 CARGO_BUILD_JOBS=2
|
||||
printf 'module fullworker\n\ngo 1.26.0\n' > go.mod
|
||||
cat > main.go <<'GO'
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
)
|
||||
|
||||
func validInput(data []byte) bool { return string(data)=="workspace-input\n" }
|
||||
|
||||
func main() {
|
||||
if len(os.Args)>1 && os.Args[1]=="fetch" {
|
||||
response,err:=http.Get("https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz")
|
||||
if err!=nil {panic(err)}
|
||||
defer func(){ _ = response.Body.Close() }()
|
||||
if response.StatusCode!=200 {panic(response.Status)}
|
||||
n,err:=io.Copy(io.Discard,response.Body)
|
||||
if err!=nil || n<100000 {panic("incomplete HTTPS transfer")}
|
||||
fmt.Println("https-transfer-ok")
|
||||
return
|
||||
}
|
||||
data,err:=os.ReadFile("/input/source.txt");if err!=nil {panic(err)}
|
||||
if !validInput(data) {panic("wrong bind source")}
|
||||
if err:=os.WriteFile("/input/forbidden",data,0600);err==nil {panic("read-only bind accepted write")}
|
||||
if err:=os.WriteFile("/output/result.txt",data,0600);err!=nil {panic(err)}
|
||||
fmt.Println("compose-bind-ok")
|
||||
}
|
||||
GO
|
||||
cat > main_test.go <<'GO'
|
||||
package main
|
||||
import "testing"
|
||||
func TestSource(t *testing.T) {for _,test:=range []struct{input string;valid bool}{{"workspace-input\n",true},{"workspace-input",false},{"",false}} {if validInput([]byte(test.input))!=test.valid {t.Fatalf("input %q",test.input)}}}
|
||||
GO
|
||||
timeout 120 go test ./...
|
||||
printf 'version: "2"\n' > .golangci.yml
|
||||
timeout 120 golangci-lint run ./...
|
||||
CGO_ENABLED=0 timeout 120 go build -o bind-smoke .
|
||||
printf '{"name":"full-worker-smoke","version":"1.0.0","scripts":{"test":"node --test source.test.cjs"}}\n' > package.json
|
||||
printf "const {test}=require('node:test');const assert=require('node:assert/strict');test('source',()=>assert.equal(6*7,42));\n" > source.test.cjs
|
||||
timeout 30 pnpm test
|
||||
python3 -m venv .venv
|
||||
printf 'import unittest\nclass Source(unittest.TestCase):\n def test_source(self): self.assertEqual(6*7,42)\n' > test_source.py
|
||||
timeout 30 .venv/bin/python -m unittest test_source.py
|
||||
printf '#include <stdio.h>\nint main(void){puts("native-ok");return 0;}\n' > native.c
|
||||
cc -Wall -Werror native.c -o native
|
||||
test "$(./native)" = native-ok
|
||||
mkdir rust
|
||||
printf '[package]\nname="full-worker-smoke"\nversion="0.1.0"\nedition="2021"\n' > rust/Cargo.toml
|
||||
mkdir rust/src
|
||||
printf '#[test] fn source(){assert_eq!(6*7,42);}\n' > rust/src/lib.rs
|
||||
timeout 120 cargo test --offline --manifest-path rust/Cargo.toml
|
||||
node <<'JS'
|
||||
const {chromium}=require('/opt/full-worker/playwright-core');
|
||||
(async()=>{
|
||||
const browser=await chromium.launch({headless:true,timeout:30000,args:['--disable-dev-shm-usage']});
|
||||
try {const page=await browser.newPage();await page.setContent(`<button onclick="this.textContent='clicked'">run</button>`);await page.getByRole('button').click();if(await page.getByRole('button').textContent()!=='clicked')throw Error('browser interaction failed');await page.screenshot({path:'browser.png'});}
|
||||
finally {await browser.close();}
|
||||
})().catch(e=>{console.error(e);process.exitCode=1;});
|
||||
JS
|
||||
test -s browser.png
|
||||
printf 'source-browser-ok\n'
|
||||
if [[ "$mode" == --all ]]; then
|
||||
timeout 10 docker info >/dev/null
|
||||
cat > Dockerfile <<'DOCKER'
|
||||
FROM scratch
|
||||
COPY bind-smoke /bind-smoke
|
||||
COPY ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
|
||||
ENTRYPOINT ["/bind-smoke"]
|
||||
DOCKER
|
||||
cp /etc/ssl/certs/ca-certificates.crt .
|
||||
mkdir input output
|
||||
printf 'workspace-input\n' > input/source.txt
|
||||
timeout 120 docker build --network=none -t "$image" .
|
||||
timeout 60 docker run --rm --memory=128m --cpus=0.5 "$image" fetch
|
||||
cat > compose.yaml <<COMPOSE
|
||||
services:
|
||||
bind:
|
||||
image: $image
|
||||
user: "1000:1000"
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
volumes:
|
||||
- type: bind
|
||||
source: $work/input
|
||||
target: /input
|
||||
read_only: true
|
||||
- type: bind
|
||||
source: $work/output
|
||||
target: /output
|
||||
COMPOSE
|
||||
timeout 60 docker compose -p "$project" -f compose.yaml run --rm bind
|
||||
cmp input/source.txt output/result.txt
|
||||
test ! -e input/forbidden
|
||||
timeout 30 docker compose -p "$project" -f compose.yaml down --volumes --remove-orphans
|
||||
test -z "$(docker ps -aq --filter "label=com.docker.compose.project=$project")"
|
||||
printf 'source-browser-docker-compose-ok\n' > /workspace/full-worker-result.txt
|
||||
else
|
||||
printf 'source-browser-ok\n' > /workspace/full-worker-result.txt
|
||||
fi
|
||||
go version
|
||||
rustc --version
|
||||
node --version
|
||||
pnpm --version
|
||||
python3 --version
|
||||
golangci-lint version
|
||||
docker --version
|
||||
docker buildx version
|
||||
docker compose version
|
||||
cat /workspace/full-worker-result.txt
|
||||
Reference in New Issue
Block a user