fix(k8s): restore executor profile environment on resume (#3835)
This commit is contained in:
@@ -1530,7 +1530,11 @@ func (e *Executor) applyRecordedKubernetesExecutorConfigToResumeRequest(
|
||||
ExecutorID: executorID, ExecutorType: string(current.Type), ExecutorCfg: current.Config,
|
||||
Metadata: metadata, Resumable: current.Resumable, RuntimeName: string(current.Type),
|
||||
}
|
||||
if err := e.restoreKubernetesProfileEnvironment(ctx, &config, running.Metadata); err != nil {
|
||||
return executorConfig{}, err
|
||||
}
|
||||
session.ExecutorID = executorID
|
||||
session.ExecutorProfileID, _ = running.Metadata[lifecycle.MetadataKeyExecutorProfileID].(string)
|
||||
req.ExecutorType = config.ExecutorType
|
||||
req.ExecutorConfig = config.ExecutorCfg
|
||||
req.Metadata = metadata
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"github.com/kandev/kandev/internal/task/repository/repoerrors"
|
||||
)
|
||||
|
||||
// Reload environment sources independently of the immutable workload snapshot.
|
||||
// Secret references remain unresolved until lifecycle's launch checkpoint.
|
||||
func (e *Executor) restoreKubernetesProfileEnvironment(
|
||||
ctx context.Context, config *executorConfig, recorded map[string]interface{},
|
||||
) error {
|
||||
profileID, _ := recorded["executor_profile_id"].(string)
|
||||
if profileID == "" {
|
||||
return nil
|
||||
}
|
||||
profile, err := e.repo.GetExecutorProfile(ctx, profileID)
|
||||
if errors.Is(err, repoerrors.ErrExecutorProfileNotFound) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume Kubernetes executor: load recorded profile environment: %w", err)
|
||||
}
|
||||
if profile == nil {
|
||||
return nil
|
||||
}
|
||||
if profile.ExecutorID != config.ExecutorID {
|
||||
return errors.New("resume Kubernetes executor: recorded profile belongs to a different executor")
|
||||
}
|
||||
config.ProfileEnvVars = append([]models.ProfileEnvVar(nil), profile.EnvVars...)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
runtimeenv "github.com/kandev/kandev/internal/agent/runtime/environment"
|
||||
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
|
||||
"github.com/kandev/kandev/internal/agentruntime"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
"github.com/kandev/kandev/internal/task/repository/repoerrors"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
)
|
||||
|
||||
func TestKubernetesResumeRestoresRecordedProfileEnvironment(t *testing.T) {
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
session := repo.sessions["sess-1"]
|
||||
session.ExecutorProfileID = "untrusted-profile"
|
||||
repo.executors["recorded-kubernetes-executor"] = &models.Executor{ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, Config: map[string]string{"auth_mode": "kubeconfig", "kubeconfig_path": "/tmp/test-config", "namespace": "kandev-agents"}}
|
||||
repo.executorsRunning[session.ID] = &models.ExecutorRunning{
|
||||
ID: session.ID, SessionID: session.ID, TaskID: session.TaskID,
|
||||
ExecutorID: "recorded-kubernetes-executor", Runtime: agentruntime.RuntimeKubernetes, Metadata: recordedKubernetesResumeMetadata(),
|
||||
}
|
||||
repo.executorProfiles["recorded-profile"] = &models.ExecutorProfile{
|
||||
ID: "recorded-profile", ExecutorID: "recorded-kubernetes-executor",
|
||||
Config: map[string]string{"pod_template_yaml": "edited template must not replace snapshot"},
|
||||
EnvVars: []models.ProfileEnvVar{{Key: "PYTHONPATH", Value: "/artifacts/runtime"}, {Key: "INTEL_DATABASE_URL", SecretID: "database-secret"}},
|
||||
}
|
||||
repo.executorProfiles["untrusted-profile"] = &models.ExecutorProfile{
|
||||
ID: "untrusted-profile", ExecutorID: "recorded-kubernetes-executor", EnvVars: []models.ProfileEnvVar{{Key: "INJECTED", Value: "bad"}},
|
||||
}
|
||||
exec := newTestExecutor(t, &mockAgentManager{}, repo)
|
||||
req, _, _, _, _, err := exec.buildResumeRequest(context.Background(), &v1.Task{ID: session.TaskID, WorkspaceID: "workspace-1"}, session, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []runtimeenv.Definition
|
||||
for _, def := range req.EnvironmentDefinitions {
|
||||
if def.Origin == runtimeenv.OriginExecutorProfile {
|
||||
got = append(got, def)
|
||||
}
|
||||
}
|
||||
want := []runtimeenv.Definition{
|
||||
{Key: "PYTHONPATH", Literal: "/artifacts/runtime", Origin: runtimeenv.OriginExecutorProfile},
|
||||
{Key: "INTEL_DATABASE_URL", SecretID: "database-secret", Origin: runtimeenv.OriginExecutorProfile},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("profile environment = %#v, want %#v", got, want)
|
||||
}
|
||||
if req.Metadata[lifecycle.MetadataKeyKubernetesProfileSnapshot] != recordedKubernetesResumeMetadata()[lifecycle.MetadataKeyKubernetesProfileSnapshot] {
|
||||
t.Fatal("recorded workload snapshot changed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesResumeProfileEnvironmentFailures(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
profile *models.ExecutorProfile
|
||||
lookupErr error
|
||||
wantError bool
|
||||
}{
|
||||
{name: "deleted profile", lookupErr: repoerrors.ErrExecutorProfileNotFound},
|
||||
{name: "absent profile"},
|
||||
{name: "database failure", lookupErr: errors.New("database unavailable"), wantError: true},
|
||||
{name: "different executor", profile: &models.ExecutorProfile{ExecutorID: "other"}, wantError: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
repo := &resumeProfileRepository{mockRepository: newMockRepository(), profile: tc.profile, err: tc.lookupErr}
|
||||
exec := newTestExecutor(t, &mockAgentManager{}, repo.mockRepository)
|
||||
exec.repo = repo
|
||||
config := executorConfig{ExecutorID: "recorded"}
|
||||
err := exec.restoreKubernetesProfileEnvironment(context.Background(), &config, map[string]interface{}{lifecycle.MetadataKeyExecutorProfileID: "profile"})
|
||||
if (err != nil) != tc.wantError {
|
||||
t.Fatalf("error = %v, want error %v", err, tc.wantError)
|
||||
}
|
||||
if len(config.ProfileEnvVars) != 0 {
|
||||
t.Fatal("loaded environment from unavailable or foreign profile")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestKubernetesResumeWithoutRecordedProfileSkipsLookup(t *testing.T) {
|
||||
for _, recorded := range []map[string]interface{}{nil, {}, {lifecycle.MetadataKeyExecutorProfileID: ""}} {
|
||||
repo := &resumeProfileRepository{mockRepository: newMockRepository(), err: errors.New("unexpected profile lookup")}
|
||||
exec := newTestExecutor(t, &mockAgentManager{}, repo.mockRepository)
|
||||
exec.repo = repo
|
||||
config := executorConfig{ExecutorID: "recorded"}
|
||||
if err := exec.restoreKubernetesProfileEnvironment(context.Background(), &config, recorded); err != nil {
|
||||
t.Fatalf("resume without a recorded profile: %v", err)
|
||||
}
|
||||
if len(config.ProfileEnvVars) != 0 {
|
||||
t.Fatal("loaded environment without a recorded profile")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type resumeProfileRepository struct {
|
||||
*mockRepository
|
||||
profile *models.ExecutorProfile
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *resumeProfileRepository) GetExecutorProfile(context.Context, string) (*models.ExecutorProfile, error) {
|
||||
return r.profile, r.err
|
||||
}
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package executor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/kandev/kandev/internal/agent/runtime/lifecycle"
|
||||
"github.com/kandev/kandev/internal/agentruntime"
|
||||
"github.com/kandev/kandev/internal/task/models"
|
||||
v1 "github.com/kandev/kandev/pkg/api/v1"
|
||||
)
|
||||
|
||||
func TestKubernetesResumePreservesProfileForExistingWorkspaceStart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
repo := newMockRepository()
|
||||
setupLiveResumeTestFixture(repo)
|
||||
session := cloneMockTaskSession(repo.sessions["sess-1"])
|
||||
session.ExecutorProfileID = "untrusted-profile"
|
||||
repo.sessions[session.ID] = cloneMockTaskSession(session)
|
||||
repo.executors["recorded-kubernetes-executor"] = &models.Executor{ID: "recorded-kubernetes-executor", Type: models.ExecutorTypeKubernetes, Config: map[string]string{"auth_mode": "kubeconfig", "kubeconfig_path": "/tmp/test-config", "namespace": "kandev-agents"}}
|
||||
repo.executorsRunning[session.ID] = &models.ExecutorRunning{
|
||||
SessionID: session.ID, TaskID: session.TaskID, ExecutorID: "recorded-kubernetes-executor",
|
||||
Runtime: agentruntime.RuntimeKubernetes, Metadata: recordedKubernetesResumeMetadata(),
|
||||
}
|
||||
repo.executorProfiles["recorded-profile"] = &models.ExecutorProfile{
|
||||
ID: "recorded-profile", ExecutorID: "recorded-kubernetes-executor",
|
||||
EnvVars: []models.ProfileEnvVar{{Key: "PYTHONPATH", Value: "/runtime"}, {Key: "DATABASE_URL", SecretID: "database"}},
|
||||
}
|
||||
repo.executorProfiles["untrusted-profile"] = &models.ExecutorProfile{
|
||||
ID: "untrusted-profile", EnvVars: []models.ProfileEnvVar{{Key: "PYTHONPATH", Value: "/wrong"}, {Key: "INJECTED", Value: "bad"}},
|
||||
}
|
||||
profileManager := &lifecycle.Manager{}
|
||||
profileManager.SetExecutorProfileReader(&resumeEnvironmentProfileReader{repo})
|
||||
profileManager.SetSecretStore(&mockSecretStore{secrets: map[string]string{"database": "synthetic-database"}})
|
||||
var delivered map[string]string
|
||||
manager := &mockAgentManager{
|
||||
executorProfileEnvFunc: profileManager.ExecutorProfileEnvForSession,
|
||||
setExecutionEnvFunc: func(_ context.Context, _ string, env map[string]string) error {
|
||||
delivered = env
|
||||
return nil
|
||||
},
|
||||
}
|
||||
exec := newTestExecutor(t, manager, repo)
|
||||
task := &v1.Task{ID: session.TaskID, WorkspaceID: "workspace-1"}
|
||||
if _, _, _, _, _, err := exec.buildResumeRequest(ctx, task, session, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := exec.persistSessionFullRowIfCurrentState(ctx, session, session.State); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reloaded, err := repo.GetTaskSession(ctx, session.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
request := &LaunchAgentRequest{ExecutorType: string(models.ExecutorTypeKubernetes)}
|
||||
if err := exec.configureExistingWorkspace(ctx, task, reloaded, "execution", "", request); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := map[string]string{"PYTHONPATH": "/runtime", "DATABASE_URL": "synthetic-database"}
|
||||
if delivered["PYTHONPATH"] != want["PYTHONPATH"] || delivered["DATABASE_URL"] != want["DATABASE_URL"] || delivered["INJECTED"] != "" {
|
||||
t.Fatalf("next subprocess environment = %#v, want %#v", delivered, want)
|
||||
}
|
||||
}
|
||||
|
||||
type resumeEnvironmentProfileReader struct{ *mockRepository }
|
||||
|
||||
func (*resumeEnvironmentProfileReader) HasActiveTaskResourceCleanupJob(context.Context, string) (bool, error) {
|
||||
return false, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
---
|
||||
created: 2026-09-20
|
||||
status: done
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-FAILURE-RECOVERY-001
|
||||
system_design:
|
||||
- ../../specs/executors/system-design/kubernetes-failure-recovery.md
|
||||
legacy_specs: []
|
||||
---
|
||||
|
||||
# Kubernetes Resume Profile Environment Repair
|
||||
|
||||
## Overview
|
||||
|
||||
Record the already implemented resume-environment regression fix and its
|
||||
verification. The executor system owns both retained-runtime recovery and
|
||||
profile environment construction. The user authorized PR publication and
|
||||
required fixups; merging and deployment remain separate actions.
|
||||
|
||||
## Scope and technical approach
|
||||
|
||||
Restore environment definitions from the runtime inventory's recorded executor
|
||||
profile before launch-environment construction. Preserve workload/storage
|
||||
snapshots and the existing secret-resolution checkpoint. Missing profiles allow
|
||||
retained recovery; lookup errors and foreign ownership fail closed.
|
||||
|
||||
This repair does not change provider mode selection, resource cleanup, schemas,
|
||||
or rendered UI. The existing recovery specifications retain their draft status.
|
||||
|
||||
## Tests and acceptance
|
||||
|
||||
- Criteria .6/.8: `TestKubernetesResumeRestoresRecordedProfileEnvironment`.
|
||||
- Subsequent-start profile identity: `TestKubernetesResumePreservesProfileForExistingWorkspaceStart`.
|
||||
- Criterion .7: `TestKubernetesResumeProfileEnvironmentFailures`.
|
||||
- Full executor package and focused Kubernetes recovery race tests.
|
||||
- Isolated synthetic acceptance: same-Pod Stop/Resume, lost-Pod replacement with
|
||||
retained workspace, missing provider rollout fallback, and idempotent replay
|
||||
of previous submissions without manual environment overrides. Independent
|
||||
database baseline and submission hashes remained unchanged. Temporary test
|
||||
resources were removed. No browser layout change requires new screenshots.
|
||||
|
||||
## Work orders
|
||||
|
||||
- [x] [Restore recorded profile environment](task-01-restore-profile-env.md)
|
||||
|
||||
## Verification results
|
||||
|
||||
Implementation and acceptance passed before PR publication. PR follow-up adds
|
||||
these linked records after the documentation coverage check identified the gap.
|
||||
See the work order for commands and results; remote CI/review is tracked in the
|
||||
PR rather than treated as complete here.
|
||||
|
||||
## Risks
|
||||
|
||||
Current profile environment edits affect future resumed processes. A deleted
|
||||
profile cannot supply its former definitions; this is documented publicly.
|
||||
@@ -0,0 +1,99 @@
|
||||
---
|
||||
id: "01-restore-profile-env"
|
||||
title: "Restore recorded Kubernetes profile environment"
|
||||
status: done
|
||||
wave: 1
|
||||
depends_on: []
|
||||
plan: "plan.md"
|
||||
requirements:
|
||||
- REQ-EXECUTORS-K8S-FAILURE-RECOVERY-001
|
||||
acceptance_criteria:
|
||||
- AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.6
|
||||
- AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.7
|
||||
- AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.8
|
||||
system_design:
|
||||
- ../../specs/executors/system-design/kubernetes-failure-recovery.md
|
||||
---
|
||||
|
||||
# Restore Recorded Kubernetes Profile Environment
|
||||
|
||||
## Summary and scope
|
||||
|
||||
Restore only environment source definitions during recorded Kubernetes resume.
|
||||
Cover authoritative profile selection, retained snapshot preservation, deferred
|
||||
secret resolution, deleted profiles, lookup failure, and foreign ownership.
|
||||
Update the public Kubernetes guide and linked recovery contract.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Provider modes, Pod/storage mutation, deployment, and unrelated cleanup.
|
||||
|
||||
## Acceptance
|
||||
|
||||
1. Recorded profile sources reach the resume launch request without substituting
|
||||
the mutable session profile or rewriting workload metadata.
|
||||
2. Missing profiles and actual lookup/ownership failures have distinct outcomes.
|
||||
3. Literal values and unresolved secret references retain their source identity.
|
||||
|
||||
## Verification
|
||||
|
||||
Run from the repository root with Go 1.26 and golangci-lint on PATH:
|
||||
|
||||
```bash
|
||||
(cd apps/backend && go test ./internal/orchestrator/executor -count=1)
|
||||
(cd apps/backend && go test -race ./internal/orchestrator/executor -run 'TestKubernetesResume' -count=1)
|
||||
python3 scripts/list-docs.py validate
|
||||
python3 scripts/lint-spec-files.py --all
|
||||
git diff --check
|
||||
```
|
||||
|
||||
## Files
|
||||
|
||||
- `apps/backend/internal/orchestrator/executor/executor_resume.go`
|
||||
- `apps/backend/internal/orchestrator/executor/executor_resume_kubernetes_env.go`
|
||||
- `apps/backend/internal/orchestrator/executor/executor_resume_kubernetes_env_test.go`
|
||||
- `docs/public/k8s.md`
|
||||
- Linked recovery requirements and system design.
|
||||
|
||||
## Dependencies and parallelism
|
||||
|
||||
None; sequential.
|
||||
|
||||
## Inputs
|
||||
|
||||
- [Requirements](../../specs/executors/requirements/kubernetes-failure-recovery.md)
|
||||
- [Design](../../specs/executors/system-design/kubernetes-failure-recovery.md#resume-environment-sources)
|
||||
- Existing recorded Kubernetes resume fixtures and repository error contracts.
|
||||
|
||||
## Results
|
||||
|
||||
The regression first failed with missing profile definitions. Executor package
|
||||
and targeted recovery race tests, changed-code Go lint, architecture and commit
|
||||
hooks passed before publication. Public documentation validation checked 47
|
||||
pages and passed all 62 validator tests. Synthetic end-to-end acceptance is
|
||||
summarized in the plan without private infrastructure details.
|
||||
|
||||
Documentation catalog validation, full specification lint, all 36 spec-linter
|
||||
tests, diff checks, and the PR documentation coverage evaluator passed. The
|
||||
focused `TestKubernetesResume` race command passed again during PR follow-up.
|
||||
Full executor package tests also passed on a detached synthetic merge with the
|
||||
current base. Remote CI/review remains a PR delivery gate.
|
||||
|
||||
## Review regression coverage
|
||||
|
||||
Added `TestKubernetesResumeWithoutRecordedProfileSkipsLookup` for nil, empty,
|
||||
and explicitly empty profile metadata. A temporary Go source overlay removing
|
||||
the empty-ID guard made the test fail with the sentinel lookup error, proving
|
||||
the test detects unintended repository access. This test does not require a production change.
|
||||
|
||||
A second review found that the resumed session could retain a conflicting
|
||||
profile selection for subsequent subprocess starts. The regression
|
||||
`TestKubernetesResumePreservesProfileForExistingWorkspaceStart` first reproduced
|
||||
wrong-profile injection after request construction and guarded session
|
||||
persistence. It uses the real lifecycle profile/secret resolver and existing
|
||||
workspace environment delivery. Resume now restores the recorded profile ID
|
||||
alongside the executor ID before persistence.
|
||||
|
||||
Post-review validation passed: `go test -race ./internal/orchestrator/executor
|
||||
-count=1`, full changed-code Go lint against the current PR base (0 issues),
|
||||
documentation catalog validation, specification lint, and diff checks.
|
||||
+4
-2
@@ -193,7 +193,7 @@ Run **Test Kubernetes** before Save and after changing cluster, namespace, templ
|
||||
|
||||
No diagnostic PVC is retained. The streaming probe is a real temporary Pod, so quota, scheduling, image pulls, admission, and Pod billing can apply. A cleanup failure makes the test fail; inspect `kandev-stream-probe-*` in the configured namespace before retrying. Kandev returns sanitized errors and template-policy warnings, but do not place secrets in a template or error-producing field.
|
||||
|
||||
After saving, each Kubernetes profile opens with the shared **Cluster connection** editor, **Test Kubernetes**, and executor-wide **Active sessions** before workload, workspace, credential, and script settings. The test combines the current unsaved connection and profile values. Administrators edit both resources through the page's one Save/Reset flow; members see the same hierarchy read-only. A connection edit is shared by every profile on that executor and can change how active sessions reconnect, while a profile edit affects only new workload snapshots. If both are dirty, Kandev confirms active-session impact once and saves the connection before the profile. The raw PodTemplate field grows and shrinks with its YAML; long unwrapped lines scroll inside the field without widening the settings page.
|
||||
After saving, each Kubernetes profile opens with the shared **Cluster connection** editor, **Test Kubernetes**, and executor-wide **Active sessions** before workload, workspace, credential, and script settings. The test combines the current unsaved connection and profile values. Administrators edit both resources through the page's one Save/Reset flow; members see the same hierarchy read-only. A connection edit is shared by every profile on that executor and can change how active sessions reconnect, while edits to profile workload settings affect only new workload snapshots. If both are dirty, Kandev confirms active-session impact once and saves the connection before the profile. The raw PodTemplate field grows and shrinks with its YAML; long unwrapped lines scroll inside the field without widening the settings page.
|
||||
|
||||
Configured executor rows expand to their profile pages, and task settings links open the exact selected profile. The standalone `/settings/executors/k8s/<executor-id>` connection editor is only a recovery surface for an executor that has no profiles; a bookmarked URL for a configured executor redirects to its first profile.
|
||||
|
||||
@@ -350,7 +350,9 @@ Kandev's `executors_running` inventory is authoritative. It records exact Pod an
|
||||
Current connection configuration and recorded workload configuration serve different purposes:
|
||||
|
||||
- Saved kubeconfig/in-cluster mode, kubeconfig path/context, and timeout are used for later status, reconnect, replacement, and cleanup. Changing credentials or context can restore or break access to an existing session. Retained sessions continue to target their recorded namespace; changing the saved namespace affects new sessions only.
|
||||
- Saved profile edits affect new sessions. An existing Pod is never live-mutated, and a missing-Pod replacement uses the recorded workload snapshot rather than the profile's current image, template, platform, main container, or storage fields.
|
||||
- Saved workload profile edits affect new sessions. An existing Pod is never live-mutated, and a missing-Pod replacement uses the recorded workload snapshot rather than the profile's current image, template, platform, main container, or storage fields.
|
||||
|
||||
On resume, Kandev reloads environment definitions from the recorded executor profile and resolves secret references at the launch checkpoint. Changes to that profile's environment can therefore affect a resumed process; they do not rewrite the Pod or storage snapshot. If the profile has been deleted, its environment definitions are unavailable. Profile lookup failures and executor ownership mismatches block resume.
|
||||
|
||||
Ordinary Stop and backend shutdown close local clients and forwards but preserve the Pod and workspace. Agent or main-container restart keeps the Pod volumes; Kandev performs a new nonce handshake and local port-forward. If a Pod disappears, managed or existing PVC storage can support a replacement Pod after identity checks; `emptyDir` cannot.
|
||||
|
||||
|
||||
@@ -40,6 +40,19 @@ The executor system owns resource retention; task state remains task-owned.
|
||||
silently substitute credentials, adopt another workload, or recreate an empty
|
||||
workspace as if recovery succeeded.
|
||||
|
||||
- **AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.6:** On Kubernetes Resume, the
|
||||
resumed process shall receive current environment definitions from the profile
|
||||
recorded for its retained runtime, even if the session's selected profile has
|
||||
changed. Subsequent process starts in that resumed session shall retain that
|
||||
profile identity. Environment edits shall not change retained Pod or storage
|
||||
settings.
|
||||
- **AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.7:** A deleted recorded profile shall
|
||||
permit recovery without its unavailable environment definitions. Other profile
|
||||
lookup failures or a profile owned by another executor shall block Resume.
|
||||
- **AC-EXECUTORS-K8S-FAILURE-RECOVERY-001.8:** Profile secret references shall
|
||||
remain unresolved until the existing launch checkpoint; recovery shall retain
|
||||
the existing environment precedence and secret-resolution rules.
|
||||
|
||||
## Exclusions
|
||||
|
||||
Restoring already-deleted data, changing provider error classification, changing
|
||||
|
||||
@@ -68,6 +68,26 @@ Shutdown rejects new workers, cancels active work, and drains within the existin
|
||||
shutdown bound. Workflow recovery checks cancellation after cleanup and receives
|
||||
the worker context, so a stop finishing during shutdown cannot start Resume.
|
||||
|
||||
## Resume environment sources
|
||||
|
||||
Criteria .6-.8 use the authoritative `executors_running` profile identity.
|
||||
`applyRecordedKubernetesExecutorConfigToResumeRequest` restores current executor
|
||||
connection settings and immutable recorded workload metadata, then calls
|
||||
`restoreKubernetesProfileEnvironment` to load only `ProfileEnvVars` from that
|
||||
recorded profile. The mutable session profile selection and current profile
|
||||
workload configuration cannot replace the recorded runtime snapshot. Resume also
|
||||
restores the session's `ExecutorProfileID` before the existing guarded full-row
|
||||
persistence, so `configureExistingWorkspace` and the lifecycle
|
||||
`ExecutorProfileEnvForSession` reader use the same profile on later starts.
|
||||
|
||||
The environment definitions flow through `resolveLaunchEnvironment` to the
|
||||
existing lifecycle launch checkpoint. Literal and secret-reference definitions
|
||||
remain distinct; this helper does not resolve secrets or change precedence.
|
||||
Current profile environment edits therefore apply to resumed processes.
|
||||
`ErrExecutorProfileNotFound` or an absent profile returns no definitions;
|
||||
other repository failures and executor ownership mismatches return an error.
|
||||
No missing profile is replaced with a different profile.
|
||||
|
||||
## Validation
|
||||
|
||||
Map criteria .1-.3 to orchestrator failure-handler, teardown ownership, and
|
||||
@@ -80,6 +100,13 @@ a backend restart; archive at the end to prove cleanup remains available.
|
||||
No rendered interface changes are required. The existing Resume control is the
|
||||
user-visible entry point. The separate error-UI task owns presentation changes.
|
||||
|
||||
`TestKubernetesResumeRestoresRecordedProfileEnvironment` covers .6 and .8,
|
||||
including a conflicting session profile, unresolved secret references, and an
|
||||
unchanged workload snapshot. `TestKubernetesResumeProfileEnvironmentFailures`
|
||||
covers .7 with deleted, absent, lookup-error, and foreign-executor cases.
|
||||
|
||||
## Implementation plan
|
||||
|
||||
[Kubernetes recoverable failure cleanup](../../../plans/kubernetes-recoverable-failure-cleanup/plan.md)
|
||||
|
||||
[Resume profile environment repair](../../../plans/kubernetes-resume-profile-env/plan.md)
|
||||
|
||||
Reference in New Issue
Block a user