fix(k8s): restore executor profile environment on resume (#3835)

This commit is contained in:
José Almeida
2026-09-21 11:18:08 +01:00
committed by GitHub
parent db8131436f
commit 3ebd7381bb
9 changed files with 417 additions and 2 deletions
@@ -1530,7 +1530,11 @@ func (e *Executor) applyRecordedKubernetesExecutorConfigToResumeRequest(
ExecutorID: executorID, ExecutorType: string(current.Type), ExecutorCfg: current.Config,
Metadata: metadata, Resumable: current.Resumable, RuntimeName: string(current.Type),
}
if err := e.restoreKubernetesProfileEnvironment(ctx, &config, running.Metadata); err != nil {
return executorConfig{}, err
}
session.ExecutorID = executorID
session.ExecutorProfileID, _ = running.Metadata[lifecycle.MetadataKeyExecutorProfileID].(string)
req.ExecutorType = config.ExecutorType
req.ExecutorConfig = config.ExecutorCfg
req.Metadata = metadata
@@ -0,0 +1,36 @@
package executor
import (
"context"
"errors"
"fmt"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/task/repository/repoerrors"
)
// Reload environment sources independently of the immutable workload snapshot.
// Secret references remain unresolved until lifecycle's launch checkpoint.
func (e *Executor) restoreKubernetesProfileEnvironment(
ctx context.Context, config *executorConfig, recorded map[string]interface{},
) error {
profileID, _ := recorded["executor_profile_id"].(string)
if profileID == "" {
return nil
}
profile, err := e.repo.GetExecutorProfile(ctx, profileID)
if errors.Is(err, repoerrors.ErrExecutorProfileNotFound) {
return nil
}
if err != nil {
return fmt.Errorf("resume Kubernetes executor: load recorded profile environment: %w", err)
}
if profile == nil {
return nil
}
if profile.ExecutorID != config.ExecutorID {
return errors.New("resume Kubernetes executor: recorded profile belongs to a different executor")
}
config.ProfileEnvVars = append([]models.ProfileEnvVar(nil), profile.EnvVars...)
return nil
}
@@ -0,0 +1,109 @@
package executor
import (
"context"
"errors"
"reflect"
"testing"
runtimeenv "github.com/kandev/kandev/internal/agent/runtime/environment"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
"github.com/kandev/kandev/internal/agentruntime"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/task/repository/repoerrors"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
func TestKubernetesResumeRestoresRecordedProfileEnvironment(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
session := repo.sessions["sess-1"]
session.ExecutorProfileID = "untrusted-profile"
repo.executors["recorded-kubernetes-executor"] = &models.Executor{ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, Config: map[string]string{"auth_mode": "kubeconfig", "kubeconfig_path": "/tmp/test-config", "namespace": "kandev-agents"}}
repo.executorsRunning[session.ID] = &models.ExecutorRunning{
ID: session.ID, SessionID: session.ID, TaskID: session.TaskID,
ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, Metadata: recordedKubernetesResumeMetadata(),
}
repo.executorProfiles["recorded-profile"] = &models.ExecutorProfile{
ID: "recorded-profile", ExecutorID: "recorded-kubernetes-executor",
Config: map[string]string{"pod_template_yaml": "edited template must not replace snapshot"},
EnvVars: []models.ProfileEnvVar{{Key: "PYTHONPATH", Value: "/artifacts/runtime"}, {Key: "INTEL_DATABASE_URL", SecretID: "database-secret"}},
}
repo.executorProfiles["untrusted-profile"] = &models.ExecutorProfile{
ID: "untrusted-profile", ExecutorID: "recorded-kubernetes-executor", EnvVars: []models.ProfileEnvVar{{Key: "INJECTED", Value: "bad"}},
}
exec := newTestExecutor(t, &mockAgentManager{}, repo)
req, _, _, _, _, err := exec.buildResumeRequest(context.Background(), &v1.Task{ID: session.TaskID, WorkspaceID: "workspace-1"}, session, true)
if err != nil {
t.Fatal(err)
}
var got []runtimeenv.Definition
for _, def := range req.EnvironmentDefinitions {
if def.Origin == runtimeenv.OriginExecutorProfile {
got = append(got, def)
}
}
want := []runtimeenv.Definition{
{Key: "PYTHONPATH", Literal: "/artifacts/runtime", Origin: runtimeenv.OriginExecutorProfile},
{Key: "INTEL_DATABASE_URL", SecretID: "database-secret", Origin: runtimeenv.OriginExecutorProfile},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("profile environment = %#v, want %#v", got, want)
}
if req.Metadata[lifecycle.MetadataKeyKubernetesProfileSnapshot] != recordedKubernetesResumeMetadata()[lifecycle.MetadataKeyKubernetesProfileSnapshot] {
t.Fatal("recorded workload snapshot changed")
}
}
func TestKubernetesResumeProfileEnvironmentFailures(t *testing.T) {
for _, tc := range []struct {
name string
profile *models.ExecutorProfile
lookupErr error
wantError bool
}{
{name: "deleted profile", lookupErr: repoerrors.ErrExecutorProfileNotFound},
{name: "absent profile"},
{name: "database failure", lookupErr: errors.New("database unavailable"), wantError: true},
{name: "different executor", profile: &models.ExecutorProfile{ExecutorID: "other"}, wantError: true},
} {
t.Run(tc.name, func(t *testing.T) {
repo := &resumeProfileRepository{mockRepository: newMockRepository(), profile: tc.profile, err: tc.lookupErr}
exec := newTestExecutor(t, &mockAgentManager{}, repo.mockRepository)
exec.repo = repo
config := executorConfig{ExecutorID: "recorded"}
err := exec.restoreKubernetesProfileEnvironment(context.Background(), &config, map[string]interface{}{lifecycle.MetadataKeyExecutorProfileID: "profile"})
if (err != nil) != tc.wantError {
t.Fatalf("error = %v, want error %v", err, tc.wantError)
}
if len(config.ProfileEnvVars) != 0 {
t.Fatal("loaded environment from unavailable or foreign profile")
}
})
}
}
func TestKubernetesResumeWithoutRecordedProfileSkipsLookup(t *testing.T) {
for _, recorded := range []map[string]interface{}{nil, {}, {lifecycle.MetadataKeyExecutorProfileID: ""}} {
repo := &resumeProfileRepository{mockRepository: newMockRepository(), err: errors.New("unexpected profile lookup")}
exec := newTestExecutor(t, &mockAgentManager{}, repo.mockRepository)
exec.repo = repo
config := executorConfig{ExecutorID: "recorded"}
if err := exec.restoreKubernetesProfileEnvironment(context.Background(), &config, recorded); err != nil {
t.Fatalf("resume without a recorded profile: %v", err)
}
if len(config.ProfileEnvVars) != 0 {
t.Fatal("loaded environment without a recorded profile")
}
}
}
type resumeProfileRepository struct {
*mockRepository
profile *models.ExecutorProfile
err error
}
func (r *resumeProfileRepository) GetExecutorProfile(context.Context, string) (*models.ExecutorProfile, error) {
return r.profile, r.err
}
@@ -0,0 +1,69 @@
package executor
import (
"context"
"testing"
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
"github.com/kandev/kandev/internal/agentruntime"
"github.com/kandev/kandev/internal/task/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
func TestKubernetesResumePreservesProfileForExistingWorkspaceStart(t *testing.T) {
ctx := context.Background()
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
session := cloneMockTaskSession(repo.sessions["sess-1"])
session.ExecutorProfileID = "untrusted-profile"
repo.sessions[session.ID] = cloneMockTaskSession(session)
repo.executors["recorded-kubernetes-executor"] = &models.Executor{ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, Config: map[string]string{"auth_mode": "kubeconfig", "kubeconfig_path": "/tmp/test-config", "namespace": "kandev-agents"}}
repo.executorsRunning[session.ID] = &models.ExecutorRunning{
SessionID: session.ID, TaskID: session.TaskID, ExecutorID: "recorded-kubernetes-executor",
Runtime: agentruntime.RuntimeKubernetes, Metadata: recordedKubernetesResumeMetadata(),
}
repo.executorProfiles["recorded-profile"] = &models.ExecutorProfile{
ID: "recorded-profile", ExecutorID: "recorded-kubernetes-executor",
EnvVars: []models.ProfileEnvVar{{Key: "PYTHONPATH", Value: "/runtime"}, {Key: "DATABASE_URL", SecretID: "database"}},
}
repo.executorProfiles["untrusted-profile"] = &models.ExecutorProfile{
ID: "untrusted-profile", EnvVars: []models.ProfileEnvVar{{Key: "PYTHONPATH", Value: "/wrong"}, {Key: "INJECTED", Value: "bad"}},
}
profileManager := &lifecycle.Manager{}
profileManager.SetExecutorProfileReader(&resumeEnvironmentProfileReader{repo})
profileManager.SetSecretStore(&mockSecretStore{secrets: map[string]string{"database": "synthetic-database"}})
var delivered map[string]string
manager := &mockAgentManager{
executorProfileEnvFunc: profileManager.ExecutorProfileEnvForSession,
setExecutionEnvFunc: func(_ context.Context, _ string, env map[string]string) error {
delivered = env
return nil
},
}
exec := newTestExecutor(t, manager, repo)
task := &v1.Task{ID: session.TaskID, WorkspaceID: "workspace-1"}
if _, _, _, _, _, err := exec.buildResumeRequest(ctx, task, session, true); err != nil {
t.Fatal(err)
}
if err := exec.persistSessionFullRowIfCurrentState(ctx, session, session.State); err != nil {
t.Fatal(err)
}
reloaded, err := repo.GetTaskSession(ctx, session.ID)
if err != nil {
t.Fatal(err)
}
request := &LaunchAgentRequest{ExecutorType: string(models.ExecutorTypeKubernetes)}
if err := exec.configureExistingWorkspace(ctx, task, reloaded, "execution", "", request); err != nil {
t.Fatal(err)
}
want := map[string]string{"PYTHONPATH": "/runtime", "DATABASE_URL": "synthetic-database"}
if delivered["PYTHONPATH"] != want["PYTHONPATH"] || delivered["DATABASE_URL"] != want["DATABASE_URL"] || delivered["INJECTED"] != "" {
t.Fatalf("next subprocess environment = %#v, want %#v", delivered, want)
}
}
type resumeEnvironmentProfileReader struct{ *mockRepository }
func (*resumeEnvironmentProfileReader) HasActiveTaskResourceCleanupJob(context.Context, string) (bool, error) {
return false, nil
}
@@ -0,0 +1,56 @@
---
created: 2026-09-20
status: done
requirements:
- REQ-EXECUTORS-K8S-FAILURE-RECOVERY-001
system_design:
- ../../specs/executors/system-design/kubernetes-failure-recovery.md
legacy_specs: []
---
# Kubernetes Resume Profile Environment Repair
## Overview
Record the already implemented resume-environment regression fix and its
verification. The executor system owns both retained-runtime recovery and
profile environment construction. The user authorized PR publication and
required fixups; merging and deployment remain separate actions.
## Scope and technical approach
Restore environment definitions from the runtime inventory's recorded executor
profile before launch-environment construction. Preserve workload/storage
snapshots and the existing secret-resolution checkpoint. Missing profiles allow
retained recovery; lookup errors and foreign ownership fail closed.
This repair does not change provider mode selection, resource cleanup, schemas,
or rendered UI. The existing recovery specifications retain their draft status.
## Tests and acceptance
- Criteria .6/.8: `TestKubernetesResumeRestoresRecordedProfileEnvironment`.
- Subsequent-start profile identity: `TestKubernetesResumePreservesProfileForExistingWorkspaceStart`.
- Criterion .7: `TestKubernetesResumeProfileEnvironmentFailures`.
- Full executor package and focused Kubernetes recovery race tests.
- Isolated synthetic acceptance: same-Pod Stop/Resume, lost-Pod replacement with
retained workspace, missing provider rollout fallback, and idempotent replay
of previous submissions without manual environment overrides. Independent
database baseline and submission hashes remained unchanged. Temporary test
resources were removed. No browser layout change requires new screenshots.
## Work orders
- [x] [Restore recorded profile environment](task-01-restore-profile-env.md)
## Verification results
Implementation and acceptance passed before PR publication. PR follow-up adds
these linked records after the documentation coverage check identified the gap.
See the work order for commands and results; remote CI/review is tracked in the
PR rather than treated as complete here.
## Risks
Current profile environment edits affect future resumed processes. A deleted
profile cannot supply its former definitions; this is documented publicly.
@@ -0,0 +1,99 @@
---
id: "01-restore-profile-env"
title: "Restore recorded Kubernetes profile environment"
status: done
wave: 1
depends_on: []
plan: "plan.md"
requirements:
- REQ-EXECUTORS-K8S-FAILURE-RECOVERY-001
acceptance_criteria:
- AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.6
- AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.7
- AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.8
system_design:
- ../../specs/executors/system-design/kubernetes-failure-recovery.md
---
# Restore Recorded Kubernetes Profile Environment
## Summary and scope
Restore only environment source definitions during recorded Kubernetes resume.
Cover authoritative profile selection, retained snapshot preservation, deferred
secret resolution, deleted profiles, lookup failure, and foreign ownership.
Update the public Kubernetes guide and linked recovery contract.
## Out of scope
Provider modes, Pod/storage mutation, deployment, and unrelated cleanup.
## Acceptance
1. Recorded profile sources reach the resume launch request without substituting
the mutable session profile or rewriting workload metadata.
2. Missing profiles and actual lookup/ownership failures have distinct outcomes.
3. Literal values and unresolved secret references retain their source identity.
## Verification
Run from the repository root with Go 1.26 and golangci-lint on PATH:
```bash
(cd apps/backend && go test ./internal/orchestrator/executor -count=1)
(cd apps/backend && go test -race ./internal/orchestrator/executor -run 'TestKubernetesResume' -count=1)
python3 scripts/list-docs.py validate
python3 scripts/lint-spec-files.py --all
git diff --check
```
## Files
- `apps/backend/internal/orchestrator/executor/executor_resume.go`
- `apps/backend/internal/orchestrator/executor/executor_resume_kubernetes_env.go`
- `apps/backend/internal/orchestrator/executor/executor_resume_kubernetes_env_test.go`
- `docs/public/k8s.md`
- Linked recovery requirements and system design.
## Dependencies and parallelism
None; sequential.
## Inputs
- [Requirements](../../specs/executors/requirements/kubernetes-failure-recovery.md)
- [Design](../../specs/executors/system-design/kubernetes-failure-recovery.md#resume-environment-sources)
- Existing recorded Kubernetes resume fixtures and repository error contracts.
## Results
The regression first failed with missing profile definitions. Executor package
and targeted recovery race tests, changed-code Go lint, architecture and commit
hooks passed before publication. Public documentation validation checked 47
pages and passed all 62 validator tests. Synthetic end-to-end acceptance is
summarized in the plan without private infrastructure details.
Documentation catalog validation, full specification lint, all 36 spec-linter
tests, diff checks, and the PR documentation coverage evaluator passed. The
focused `TestKubernetesResume` race command passed again during PR follow-up.
Full executor package tests also passed on a detached synthetic merge with the
current base. Remote CI/review remains a PR delivery gate.
## Review regression coverage
Added `TestKubernetesResumeWithoutRecordedProfileSkipsLookup` for nil, empty,
and explicitly empty profile metadata. A temporary Go source overlay removing
the empty-ID guard made the test fail with the sentinel lookup error, proving
the test detects unintended repository access. This test does not require a production change.
A second review found that the resumed session could retain a conflicting
profile selection for subsequent subprocess starts. The regression
`TestKubernetesResumePreservesProfileForExistingWorkspaceStart` first reproduced
wrong-profile injection after request construction and guarded session
persistence. It uses the real lifecycle profile/secret resolver and existing
workspace environment delivery. Resume now restores the recorded profile ID
alongside the executor ID before persistence.
Post-review validation passed: `go test -race ./internal/orchestrator/executor
-count=1`, full changed-code Go lint against the current PR base (0 issues),
documentation catalog validation, specification lint, and diff checks.
+4 -2
View File
@@ -193,7 +193,7 @@ Run **Test Kubernetes** before Save and after changing cluster, namespace, templ
No diagnostic PVC is retained. The streaming probe is a real temporary Pod, so quota, scheduling, image pulls, admission, and Pod billing can apply. A cleanup failure makes the test fail; inspect `kandev-stream-probe-*` in the configured namespace before retrying. Kandev returns sanitized errors and template-policy warnings, but do not place secrets in a template or error-producing field.
After saving, each Kubernetes profile opens with the shared **Cluster connection** editor, **Test Kubernetes**, and executor-wide **Active sessions** before workload, workspace, credential, and script settings. The test combines the current unsaved connection and profile values. Administrators edit both resources through the page's one Save/Reset flow; members see the same hierarchy read-only. A connection edit is shared by every profile on that executor and can change how active sessions reconnect, while a profile edit affects only new workload snapshots. If both are dirty, Kandev confirms active-session impact once and saves the connection before the profile. The raw PodTemplate field grows and shrinks with its YAML; long unwrapped lines scroll inside the field without widening the settings page.
After saving, each Kubernetes profile opens with the shared **Cluster connection** editor, **Test Kubernetes**, and executor-wide **Active sessions** before workload, workspace, credential, and script settings. The test combines the current unsaved connection and profile values. Administrators edit both resources through the page's one Save/Reset flow; members see the same hierarchy read-only. A connection edit is shared by every profile on that executor and can change how active sessions reconnect, while edits to profile workload settings affect only new workload snapshots. If both are dirty, Kandev confirms active-session impact once and saves the connection before the profile. The raw PodTemplate field grows and shrinks with its YAML; long unwrapped lines scroll inside the field without widening the settings page.
Configured executor rows expand to their profile pages, and task settings links open the exact selected profile. The standalone `/settings/executors/k8s/<executor-id>` connection editor is only a recovery surface for an executor that has no profiles; a bookmarked URL for a configured executor redirects to its first profile.
@@ -350,7 +350,9 @@ Kandev's `executors_running` inventory is authoritative. It records exact Pod an
Current connection configuration and recorded workload configuration serve different purposes:
- Saved kubeconfig/in-cluster mode, kubeconfig path/context, and timeout are used for later status, reconnect, replacement, and cleanup. Changing credentials or context can restore or break access to an existing session. Retained sessions continue to target their recorded namespace; changing the saved namespace affects new sessions only.
- Saved profile edits affect new sessions. An existing Pod is never live-mutated, and a missing-Pod replacement uses the recorded workload snapshot rather than the profile's current image, template, platform, main container, or storage fields.
- Saved workload profile edits affect new sessions. An existing Pod is never live-mutated, and a missing-Pod replacement uses the recorded workload snapshot rather than the profile's current image, template, platform, main container, or storage fields.
On resume, Kandev reloads environment definitions from the recorded executor profile and resolves secret references at the launch checkpoint. Changes to that profile's environment can therefore affect a resumed process; they do not rewrite the Pod or storage snapshot. If the profile has been deleted, its environment definitions are unavailable. Profile lookup failures and executor ownership mismatches block resume.
Ordinary Stop and backend shutdown close local clients and forwards but preserve the Pod and workspace. Agent or main-container restart keeps the Pod volumes; Kandev performs a new nonce handshake and local port-forward. If a Pod disappears, managed or existing PVC storage can support a replacement Pod after identity checks; `emptyDir` cannot.
@@ -40,6 +40,19 @@ The executor system owns resource retention; task state remains task-owned.
silently substitute credentials, adopt another workload, or recreate an empty
workspace as if recovery succeeded.
- **AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.6:** On Kubernetes Resume, the
resumed process shall receive current environment definitions from the profile
recorded for its retained runtime, even if the session's selected profile has
changed. Subsequent process starts in that resumed session shall retain that
profile identity. Environment edits shall not change retained Pod or storage
settings.
- **AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.7:** A deleted recorded profile shall
permit recovery without its unavailable environment definitions. Other profile
lookup failures or a profile owned by another executor shall block Resume.
- **AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.8:** Profile secret references shall
remain unresolved until the existing launch checkpoint; recovery shall retain
the existing environment precedence and secret-resolution rules.
## Exclusions
Restoring already-deleted data, changing provider error classification, changing
@@ -68,6 +68,26 @@ Shutdown rejects new workers, cancels active work, and drains within the existin
shutdown bound. Workflow recovery checks cancellation after cleanup and receives
the worker context, so a stop finishing during shutdown cannot start Resume.
## Resume environment sources
Criteria .6-.8 use the authoritative `executors_running` profile identity.
`applyRecordedKubernetesExecutorConfigToResumeRequest` restores current executor
connection settings and immutable recorded workload metadata, then calls
`restoreKubernetesProfileEnvironment` to load only `ProfileEnvVars` from that
recorded profile. The mutable session profile selection and current profile
workload configuration cannot replace the recorded runtime snapshot. Resume also
restores the session's `ExecutorProfileID` before the existing guarded full-row
persistence, so `configureExistingWorkspace` and the lifecycle
`ExecutorProfileEnvForSession` reader use the same profile on later starts.
The environment definitions flow through `resolveLaunchEnvironment` to the
existing lifecycle launch checkpoint. Literal and secret-reference definitions
remain distinct; this helper does not resolve secrets or change precedence.
Current profile environment edits therefore apply to resumed processes.
`ErrExecutorProfileNotFound` or an absent profile returns no definitions;
other repository failures and executor ownership mismatches return an error.
No missing profile is replaced with a different profile.
## Validation
Map criteria .1-.3 to orchestrator failure-handler, teardown ownership, and
@@ -80,6 +100,13 @@ a backend restart; archive at the end to prove cleanup remains available.
No rendered interface changes are required. The existing Resume control is the
user-visible entry point. The separate error-UI task owns presentation changes.
`TestKubernetesResumeRestoresRecordedProfileEnvironment` covers .6 and .8,
including a conflicting session profile, unresolved secret references, and an
unchanged workload snapshot. `TestKubernetesResumeProfileEnvironmentFailures`
covers .7 with deleted, absent, lookup-error, and foreign-executor cases.
## Implementation plan
[Kubernetes recoverable failure cleanup](../../../plans/kubernetes-recoverable-failure-cleanup/plan.md)
[Resume profile environment repair](../../../plans/kubernetes-resume-profile-env/plan.md)