dreampiler/kandev
1
0
Fork 0
Code Issues 43 Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix(kubernetes): make retained PVC preparation robust (#3697)

Browse Source
This commit is contained in:
José Almeida
2026-09-15 16:16:08 +02:00
committed by GitHub
parent 6c5ab15980
commit d986a14326
7 changed files with 233 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/internal/agent/runtime/lifecycle/default_scripts.go
+53 -3
View File
@@ -3,6 +3,7 @@ package lifecycle
import (
"crypto/sha256"
"encoding/hex"
"strings"
)
const executorTypeSSH = "ssh"
@@ -12,6 +13,13 @@ const executorTypeSSH = "ssh"
// user-authored prepare scripts must remain unchanged.
const legacySpritesPrepareScriptSHA256 = "e656f9e51496e1bb1e5cee205058bbeaa7fe1ab37073f600e6c94310ead7be4d"
// These hashes identify scripts supplied by Kandev. Only exact matches receive
// managed fixes; customized scripts remain byte-for-byte unchanged.
const (
legacyKubernetesDefaultPrepareScriptSHA256 = "5fe05aedccc24565f338fc6d03f6814024aaf15655643b9ab6317986dfac0ce2"
legacyKubernetesFullPrepareScriptSHA256 = "540eece886af9577c6e9e652a845e373692c222453d62144cd6f03a4ab8bd734"
)
// DefaultPrepareScript returns the default prepare script for a given executor type string.
func DefaultPrepareScript(executorType string) string {
switch executorType {
@@ -37,6 +45,41 @@ func isLegacySpritesPrepareScript(script string) bool {
return hex.EncodeToString(digest[:]) == legacySpritesPrepareScriptSHA256
}
func upgradeLegacyKubernetesPrepareScript(script string) string {
digest := sha256.Sum256([]byte(script))
switch hex.EncodeToString(digest[:]) {
case legacyKubernetesDefaultPrepareScriptSHA256:
return defaultKubernetesPrepareScript
case legacyKubernetesFullPrepareScriptSHA256:
upgraded := strings.Replace(script,
"clone_tmp=/opt/kandev/.workspace-clone\n",
"clone_tmp=/opt/kandev/.workspace-clone\n\n"+kubernetesRepositoryOriginNormalizer,
1,
)
upgraded = strings.Replace(upgraded, legacyKubernetesRepositoryOriginComparison,
currentKubernetesRepositoryOriginComparison, 1)
return strings.Replace(upgraded,
`cp -a "$clone_tmp"/. "$workspace"/`,
`cp -R "$clone_tmp"/. "$workspace"/`, 1)
default:
return script
}
}
const kubernetesRepositoryOriginNormalizer = `normalize_repository_origin() {
printf '%s\n' "$1" | sed \
-e 's|^https://[^/@]*@github.com/|https://github.com/|' \
-e 's|^git@github.com:|https://github.com/|' \
-e 's|^ssh://git@github.com/|https://github.com/|'
}
`
const legacyKubernetesRepositoryOriginComparison = ` expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')`
const currentKubernetesRepositoryOriginComparison = ` expected_origin=$(normalize_repository_origin "$repository_url")
retained_origin=$(normalize_repository_origin "$workspace_origin")`
// KandevBranchCheckoutPostlude returns a kandev-managed shell snippet that
// guarantees the session's feature branch is checked out inside the
// workspace, no matter what the user's stored prepare_script does.
@@ -166,6 +209,13 @@ repository_url={{repository.clone_url}}
repository_branch={{repository.branch}}
clone_tmp=/opt/kandev/.workspace-clone
normalize_repository_origin() {
printf '%s\n' "$1" | sed \
-e 's|^https://[^/@]*@github.com/|https://github.com/|' \
-e 's|^git@github.com:|https://github.com/|' \
-e 's|^ssh://git@github.com/|https://github.com/|'
}
# ---- Git identity and HTTPS authentication ----
{{git.identity_setup}}
git config --global --add safe.directory '*'
@@ -182,8 +232,8 @@ if [ -n "$repository_url" ]; then
exit 1
fi
workspace_origin=$(git -C "$workspace" remote get-url origin 2>/dev/null || true)
expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
expected_origin=$(normalize_repository_origin "$repository_url")
retained_origin=$(normalize_repository_origin "$workspace_origin")
if [ -z "$workspace_origin" ] || [ "$retained_origin" != "$expected_origin" ]; then
echo 'kandev: retained workspace repository origin does not match the configured repository' >&2
exit 1
@@ -198,7 +248,7 @@ if [ -n "$repository_url" ]; then
rm -rf "$clone_tmp"
trap 'rm -rf "$clone_tmp"' 0 1 2 15
git clone --depth=1 --branch "$repository_branch" "$repository_url" "$clone_tmp"
cp -a "$clone_tmp"/. "$workspace"/
cp -R "$clone_tmp"/. "$workspace"/
rm -rf "$clone_tmp"
trap - 0 1 2 15
fi
apps/backend/internal/agent/runtime/lifecycle/default_scripts_kubernetes_test.go
+6 -1
View File
@@ -40,6 +40,11 @@ func TestDefaultPrepareScriptKubernetesReusesRetainedPVCWorkspace(t *testing.T)
if err := os.MkdirAll(home, 0o700); err != nil {
t.Fatal(err)
}
bin := filepath.Join(root, "bin")
if err := os.MkdirAll(bin, 0o755); err != nil {
t.Fatal(err)
}
installMetadataRestrictedCopy(t, bin)
cloneTmp := filepath.Join(root, "runtime", "workspace-clone")
if err := os.MkdirAll(filepath.Dir(cloneTmp), 0o700); err != nil {
t.Fatal(err)
@@ -58,7 +63,7 @@ func TestDefaultPrepareScriptKubernetesReusesRetainedPVCWorkspace(t *testing.T)
run := func() {
t.Helper()
cmd := exec.Command("sh", "-eu", "-c", script)
cmd.Env = append(os.Environ(), "HOME="+home)
cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+bin+":"+os.Getenv("PATH"))
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("Kubernetes prepare failed: %v\n%s", err, output)
}
apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_bootstrap.go
+2
View File
@@ -304,6 +304,8 @@ func kubernetesPrepareScript(req *ExecutorCreateRequest) (string, error) {
script := getMetadataString(req.Metadata, MetadataKeySetupScript)
if script == "" {
script = DefaultPrepareScript("k8s")
} else {
script = upgradeLegacyKubernetesPrepareScript(script)
}
if script == "" {
return ":\n", nil
apps/backend/internal/agent/runtime/lifecycle/full_worker_prepare_test.go
+1
View File
@@ -58,6 +58,7 @@ func TestFullWorkerPreparationClonesBeforeCachesAndRetainsWorkspace(t *testing.T
if err := os.MkdirAll(bin, 0755); err != nil {
t.Fatal(err)
}
installMetadataRestrictedCopy(t, bin)
if err := os.WriteFile(filepath.Join(bin, "docker"), []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil {
t.Fatal(err)
}
apps/backend/internal/agent/runtime/lifecycle/kubernetes_prepare_copy_test.go
+147
View File
@@ -0,0 +1,147 @@
package lifecycle
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
)
func installMetadataRestrictedCopy(t *testing.T, bin string) {
t.Helper()
realCopy, err := exec.LookPath("cp")
if err != nil {
t.Fatal(err)
}
wrapper := `#!/bin/sh
for arg in "$@"; do
case "$arg" in
-a|-p|--archive|--preserve|--preserve=*)
echo 'cp: preserving metadata: Operation not permitted' >&2
exit 1
;;
esac
done
exec ` + shellQuote(realCopy) + ` "$@"
`
if err := os.WriteFile(filepath.Join(bin, "cp"), []byte(wrapper), 0o755); err != nil {
t.Fatal(err)
}
}
func TestKubernetesPreparationAcceptsEquivalentGitHubOrigins(t *testing.T) {
full, err := os.ReadFile("../../../../../../k8s/worker-images/full/prepare.sh")
if err != nil {
t.Fatal(err)
}
for name, script := range map[string]string{
"default": DefaultPrepareScript("k8s"),
"full": string(full),
} {
t.Run(name, func(t *testing.T) {
runKubernetesPreparationWithEquivalentGitHubOrigins(t, script, name == "full")
})
}
}
func TestKubernetesPrepareScriptUpgradesPersistedManagedScripts(t *testing.T) {
full, err := os.ReadFile("../../../../../../k8s/worker-images/full/prepare.sh")
if err != nil {
t.Fatal(err)
}
legacyDefault := persistedKubernetesPrepareScriptBeforePVCFix(t, DefaultPrepareScript("k8s"))
for name, current := range map[string]string{
"default": DefaultPrepareScript("k8s"),
"full": string(full),
} {
t.Run(name, func(t *testing.T) {
legacy := persistedKubernetesPrepareScriptBeforePVCFix(t, current)
if upgraded := upgradeLegacyKubernetesPrepareScript(legacy); upgraded != current {
t.Fatalf("managed script upgrade did not produce the current script")
}
req := validKubernetesCreateRequest()
req.Metadata[MetadataKeySetupScript] = legacy
resolved, err := kubernetesPrepareScript(req)
requireNoError(t, err)
if !strings.Contains(resolved, "normalize_repository_origin()") ||
!strings.Contains(resolved, `cp -R "$clone_tmp"/. "$workspace"/`) {
t.Fatalf("persisted managed script was not upgraded:\n%s", resolved)
}
if strings.Contains(resolved, `cp -a "$clone_tmp"/. "$workspace"/`) {
t.Fatalf("persisted managed script retained archive copy:\n%s", resolved)
}
})
}
const customization = "# user-managed customization"
req := validKubernetesCreateRequest()
req.Metadata[MetadataKeySetupScript] = legacyDefault + "\n" + customization
resolved, err := kubernetesPrepareScript(req)
requireNoError(t, err)
if !strings.Contains(resolved, customization) ||
!strings.Contains(resolved, `cp -a "$clone_tmp"/. "$workspace"/`) ||
strings.Contains(resolved, "normalize_repository_origin()") {
t.Fatalf("custom prepare script was modified:\n%s", resolved)
}
}
func persistedKubernetesPrepareScriptBeforePVCFix(t *testing.T, current string) string {
t.Helper()
legacy := strings.Replace(current, `
normalize_repository_origin() {
printf '%s\n' "$1" | sed \
-e 's|^https://[^/@]*@github.com/|https://github.com/|' \
-e 's|^git@github.com:|https://github.com/|' \
-e 's|^ssh://git@github.com/|https://github.com/|'
}
`, "", 1)
legacy = strings.Replace(legacy, ` expected_origin=$(normalize_repository_origin "$repository_url")
retained_origin=$(normalize_repository_origin "$workspace_origin")`, ` expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')`, 1)
legacy = strings.Replace(legacy,
`cp -R "$clone_tmp"/. "$workspace"/`,
`cp -a "$clone_tmp"/. "$workspace"/`, 1)
if legacy == current {
t.Fatal("legacy fixture did not reverse the managed PVC preparation changes")
}
return legacy
}
func runKubernetesPreparationWithEquivalentGitHubOrigins(t *testing.T, script string, needsDocker bool) {
t.Helper()
workspace, _ := setupPostludeRepo(t, "main")
runIn(t, workspace, "git", "remote", "set-url", "origin", "https://github.com/acme/repo.git")
marker := filepath.Join(workspace, "retained.txt")
requireNoError(t, os.WriteFile(marker, []byte("preserve me"), 0o600))
root := t.TempDir()
home := filepath.Join(root, "home")
bin := filepath.Join(root, "bin")
requireNoError(t, os.MkdirAll(home, 0o700))
requireNoError(t, os.MkdirAll(bin, 0o755))
if needsDocker {
requireNoError(t, os.WriteFile(filepath.Join(bin, "docker"), []byte("#!/bin/sh\nexit 0\n"), 0o755))
}
rendered := strings.NewReplacer(
"{{git.identity_setup}}", ":",
"{{github.auth_setup}}", ":",
"{{repository.branch}}", shellQuote("main"),
"{{repository.clone_url}}", shellQuote("git@github.com:acme/repo.git"),
"{{workspace.path}}", shellQuote(workspace),
"{{repository.setup_script}}", ":",
"{{kandev.agents.install}}", ":",
"/opt/kandev/.workspace-clone", filepath.Join(root, "runtime", "workspace-clone"),
).Replace(script)
cmd := exec.Command("sh", "-eu", "-c", rendered)
cmd.Env = append(os.Environ(), "HOME="+home, "PATH="+bin+":"+os.Getenv("PATH"))
if output, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("Kubernetes prepare rejected equivalent GitHub origins: %v\n%s", err, output)
}
data, err := os.ReadFile(marker)
if err != nil || string(data) != "preserve me" {
t.Fatalf("retained workspace data = %q, %v", data, err)
}
}
docs/plans/kubernetes-docker-workloads/task-03-docker-lifecycle.md
+14
View File
@@ -134,3 +134,17 @@ The exact implementation revision was exercised with image config digest
Required Kubernetes and lifecycle Go packages, changed-file ESLint and test
discovery also passed. This evidence applies to the recorded Kind/runtime
combination; other container runtimes still require their own cgroup validation.
A later Kubernetes acceptance run exposed two retained-workspace portability
gaps. A non-root workload could copy a checkout onto a group-writable,
root-owned PVC and then fail when `cp -a` restored metadata on the mount root.
The preparation scripts now copy checkout contents recursively without
restoring source ownership or timestamps. The same run also showed that a
credential-free GitHub HTTPS origin and the equivalent GitHub SSH origin were
compared literally during lost-Pod replacement. Preparation now normalizes
those GitHub URL forms before comparison while continuing to reject a different
repository. Runtime resolution upgrades exact persisted copies of the two
Kandev-supplied scripts; any customized script remains byte-for-byte unchanged.
Focused lifecycle regressions exercise both the built-in Kubernetes script and
the full worker script, preserve retained files, and reproduce the restricted
copy boundary with the real `cp` implementation.
k8s/worker-images/full/prepare.sh
+10 -3
View File
@@ -28,6 +28,13 @@ repository_url={{repository.clone_url}}
repository_branch={{repository.branch}}
clone_tmp=/opt/kandev/.workspace-clone
normalize_repository_origin() {
printf '%s\n' "$1" | sed \
-e 's|^https://[^/@]*@github.com/|https://github.com/|' \
-e 's|^git@github.com:|https://github.com/|' \
-e 's|^ssh://git@github.com/|https://github.com/|'
}
# ---- Git identity and HTTPS authentication ----
{{git.identity_setup}}
git config --global --add safe.directory '*'
@@ -44,8 +51,8 @@ if [ -n "$repository_url" ]; then
exit 1
fi
workspace_origin=$(git -C "$workspace" remote get-url origin 2>/dev/null || true)
expected_origin=$(printf '%s\n' "$repository_url" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
retained_origin=$(printf '%s\n' "$workspace_origin" | sed 's|^https://[^/@]*@github.com/|https://github.com/|')
expected_origin=$(normalize_repository_origin "$repository_url")
retained_origin=$(normalize_repository_origin "$workspace_origin")
if [ -z "$workspace_origin" ] || [ "$retained_origin" != "$expected_origin" ]; then
echo 'kandev: retained workspace repository origin does not match the configured repository' >&2
exit 1
@@ -60,7 +67,7 @@ if [ -n "$repository_url" ]; then
rm -rf "$clone_tmp"
trap 'rm -rf "$clone_tmp"' 0 1 2 15
git clone --depth=1 --branch "$repository_branch" "$repository_url" "$clone_tmp"
cp -a "$clone_tmp"/. "$workspace"/
cp -R "$clone_tmp"/. "$workspace"/
rm -rf "$clone_tmp"
trap - 0 1 2 15
fi