fix(k8s): share one pod across task sessions (#3865)

This commit is contained in:
José Almeida
2026-09-24 07:37:17 +01:00
committed by GitHub
parent 9885fd40ba
commit 25271b6e7f
65 changed files with 3627 additions and 220 deletions
+2 -2
View File
@@ -72,7 +72,7 @@ apps/backend/
│ │ └── secretadapter/ # Upsert-style adapter over secrets.SecretStore
│ ├── i18n/ # Localization for backend-rendered browser/share artifacts
│ ├── jira/ # Jira/Atlassian Cloud integration (config, REST client, poller)
│ ├── kubernetes/ # Kubernetes diagnostics and exact recorded-session status API
│ ├── kubernetes/ # Task-owned Pod/PVC diagnostics; session stop preserves compute. See docs/specs/executors/system-design/kubernetes-task-pod.md for ownership, credentials, and recovery invariants.
│ ├── linear/ # Linear integration (config, GraphQL client, poller)
│ ├── lsp/ # LSP server
│ ├── mcp/ # MCP protocol support
@@ -175,7 +175,7 @@ Standalone agentctl is launched in its own process group so terminal Ctrl+C is h
- `k8s` - Namespaced Kubernetes Pod with optional PVC workspace
- `remote_docker`, `remote_vps` - Planned
**Kubernetes lifecycle:** `executors_running` is the authoritative resource inventory. Persist the exact Pod/PVC names, UIDs, full `kandev.ai/*` identity, workload snapshot, and internal runtime-secret references before reporting a launch as durable. Ordinary stop and backend shutdown preserve resources; terminal cleanup deletes the Pod and only a Kandev-created PVC after exact identity checks and confirmed absence. Reconnect uses the current executor connection config but the recorded workload/resource snapshot, and any ambiguity fails closed. Keep agentctl reachable only through a process-local loopback port-forward; never add a Service or place resolved credentials in a Pod spec.
**Kubernetes lifecycle:** `task_environment_kubernetes` owns shared physical Pod/PVC inventory; `executors_running` records individual sessions and legacy session-owned pods. Persist the exact Pod/PVC names, UIDs, full `kandev.ai/*` identity, workload snapshot, and internal runtime-secret references before reporting a launch as durable. Session stop deletes only its agentctl instance, and backend shutdown preserves resources; task cleanup deletes the Pod and only a Kandev-created PVC after exact identity checks and confirmed absence. Reconnect uses the current executor connection config but the recorded workload/resource snapshot, and any ambiguity fails closed. Keep agentctl reachable only through a process-local loopback port-forward; never add a Service or place resolved credentials in a Pod spec.
**Remote SSH executor platforms:** Treat supported remote OS/arch values as an end-to-end contract. Platform probe/normalization, lifecycle support checks, agentctl helper resolution, platform default shell, SSH readiness endpoints, frontend response types, and tests must stay aligned. Preserve raw unsupported platform details in user-facing errors, but use normalized values for supported-platform matching. Keep shell defaults platform-aware: Darwin defaults to `zsh`, Linux defaults to `bash`, unless an explicit shell is saved.
@@ -12,6 +12,9 @@ import (
// Runtime inventory metadata keys are shared by lifecycle persistence and
// higher-level fail-closed inventory checks.
const (
MetadataKeyTaskOwned = "kubernetes_task_owned"
MetadataKeyOwnershipVersion = "kubernetes_resource_ownership_version"
TaskOwnershipVersion = "task-v1"
MetadataKeyResourceExecutorID = "kubernetes_resource_executor_id"
MetadataKeyResourceProfileID = "kubernetes_resource_profile_id"
MetadataKeyResourceInstanceID = "kubernetes_resource_instance_id"
@@ -21,6 +24,7 @@ const (
)
type ResourceIdentity struct {
TaskOwned bool
ExecutorID string
ProfileID string
InstanceID string
@@ -98,6 +102,10 @@ func OwnershipLabels(identity ResourceIdentity) (map[string]string, error) {
"app.kubernetes.io/managed-by": "kandev",
"app.kubernetes.io/instance": identity.InstanceID,
}
if identity.TaskOwned {
labels["kandev.ai/ownership-version"] = TaskOwnershipVersion
labels["app.kubernetes.io/component"] = "task-environment"
}
for _, item := range values {
if item.value == "" || len(validation.IsValidLabelValue(item.value)) > 0 {
return nil, fieldError(item.path, "must be a non-empty Kubernetes label value")
@@ -344,6 +344,9 @@ func (c *ControlClient) DeleteInstance(ctx context.Context, instanceID string) e
return nil
}
// ErrInstanceNotFound identifies an absent agentctl instance.
var ErrInstanceNotFound = errors.New("instance not found")
// GetInstance gets information about a specific instance.
func (c *ControlClient) GetInstance(ctx context.Context, instanceID string) (*InstanceInfo, error) {
req, err := http.NewRequestWithContext(ctx, "GET", c.baseURL+"/api/v1/instances/"+instanceID, nil)
@@ -358,7 +361,7 @@ func (c *ControlClient) GetInstance(ctx context.Context, instanceID string) (*In
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
return nil, fmt.Errorf("instance %q not found", instanceID)
return nil, fmt.Errorf("%w: %q", ErrInstanceNotFound, instanceID)
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to get instance: status %d", resp.StatusCode)
@@ -587,7 +587,7 @@ func TestGetInstance_FailureModes(t *testing.T) {
body string
wantErr string
}{
{"not found", http.StatusNotFound, `{}`, `instance "inst-3" not found`},
{"not found", http.StatusNotFound, `{}`, `instance not found: "inst-3"`},
{"other status", http.StatusInternalServerError, `{}`, "failed to get instance: status 500"},
{"malformed body", http.StatusOK, `{"id":`, "failed to decode response"},
}
@@ -600,6 +600,9 @@ func TestGetInstance_FailureModes(t *testing.T) {
if err == nil || !strings.Contains(err.Error(), tc.wantErr) {
t.Fatalf("error = %v, want %q", err, tc.wantErr)
}
if errors.Is(err, ErrInstanceNotFound) != (tc.status == http.StatusNotFound) {
t.Errorf("not-found classification for status %d: %v", tc.status, err)
}
if info != nil {
t.Errorf("info = %+v, want nil on failure", info)
}
@@ -343,34 +343,36 @@ var persistentMetadataKeys = map[string]bool{
MetadataKeySSHReclaimTaskDir: true,
// Kubernetes connection and exact resource inventory.
MetadataKeyKubernetesAuthMode: true,
MetadataKeyKubernetesKubeconfigPath: true,
MetadataKeyKubernetesKubeContext: true,
MetadataKeyKubernetesConfigNamespace: true,
MetadataKeyKubernetesRequestTimeoutSeconds: true,
MetadataKeyKubernetesNamespace: true,
MetadataKeyKubernetesPodName: true,
MetadataKeyKubernetesPodUID: true,
MetadataKeyKubernetesMainContainer: true,
MetadataKeyKubernetesPlatform: true,
MetadataKeyKubernetesRuntimeWorkspaceMode: true,
MetadataKeyKubernetesPVCName: true,
MetadataKeyKubernetesPVCUID: true,
MetadataKeyKubernetesPVCCreated: true,
MetadataKeyKubernetesAgentctlRemotePort: true,
MetadataKeyKubernetesAgentctlInstanceID: true,
MetadataKeyKubernetesContainerRestartCount: true,
MetadataKeyKubernetesResourceInstanceID: true,
MetadataKeyKubernetesResourceExecutorID: true,
MetadataKeyKubernetesResourceProfileID: true,
MetadataKeyKubernetesResourceTaskID: true,
MetadataKeyKubernetesResourceSessionID: true,
MetadataKeyKubernetesResourceEnvironmentID: true,
MetadataKeyKubernetesExecutorConfigHash: true,
MetadataKeyKubernetesProfileConfigHash: true,
MetadataKeyKubernetesTemplateHash: true,
MetadataKeyKubernetesProfileSnapshot: true,
MetadataKeyKubernetesInventoryState: true,
MetadataKeyKubernetesAuthMode: true,
MetadataKeyKubernetesKubeconfigPath: true,
MetadataKeyKubernetesKubeContext: true,
MetadataKeyKubernetesConfigNamespace: true,
MetadataKeyKubernetesRequestTimeoutSeconds: true,
MetadataKeyKubernetesNamespace: true,
MetadataKeyKubernetesPodName: true,
MetadataKeyKubernetesPodUID: true,
MetadataKeyKubernetesMainContainer: true,
MetadataKeyKubernetesPlatform: true,
MetadataKeyKubernetesRuntimeWorkspaceMode: true,
MetadataKeyKubernetesPVCName: true,
MetadataKeyKubernetesPVCUID: true,
MetadataKeyKubernetesPVCCreated: true,
MetadataKeyKubernetesAgentctlRemotePort: true,
MetadataKeyKubernetesAgentctlInstanceID: true,
MetadataKeyKubernetesContainerRestartCount: true,
metadataKubernetesTaskOwned: true,
agentkubernetes.MetadataKeyOwnershipVersion: true,
MetadataKeyKubernetesResourceInstanceID: true,
MetadataKeyKubernetesResourceExecutorID: true,
MetadataKeyKubernetesResourceProfileID: true,
MetadataKeyKubernetesResourceTaskID: true,
MetadataKeyKubernetesResourceSessionID: true,
MetadataKeyKubernetesResourceEnvironmentID: true,
MetadataKeyKubernetesExecutorConfigHash: true,
MetadataKeyKubernetesProfileConfigHash: true,
MetadataKeyKubernetesTemplateHash: true,
MetadataKeyKubernetesProfileSnapshot: true,
MetadataKeyKubernetesInventoryState: true,
// Executor type marker
MetadataKeyIsRemote: true,
@@ -417,42 +419,44 @@ var sessionScopedMetadataKeys = map[string]bool{
// must not be inherited by a sibling session sharing the environment.
MetadataKeyOfficeAgentProfileID: true,
MetadataKeySSHRemoteSessionDir: true,
MetadataKeySSHRemoteAgentctlPort: true,
MetadataKeySSHRemoteAgentctlPID: true,
MetadataKeySSHAgentctlInstanceID: true,
MetadataKeySSHLocalForwardPort: true,
MetadataKeySSHRemoteAgentctlURL: true,
MetadataKeySSHRuntimeAPILocalURL: true,
MetadataKeySSHRuntimeAPIRemotePort: true,
MetadataKeyKubernetesAuthMode: true,
MetadataKeyKubernetesKubeconfigPath: true,
MetadataKeyKubernetesKubeContext: true,
MetadataKeyKubernetesConfigNamespace: true,
MetadataKeyKubernetesRequestTimeoutSeconds: true,
MetadataKeyKubernetesNamespace: true,
MetadataKeyKubernetesPodName: true,
MetadataKeyKubernetesPodUID: true,
MetadataKeyKubernetesMainContainer: true,
MetadataKeyKubernetesPlatform: true,
MetadataKeyKubernetesRuntimeWorkspaceMode: true,
MetadataKeyKubernetesPVCName: true,
MetadataKeyKubernetesPVCUID: true,
MetadataKeyKubernetesPVCCreated: true,
MetadataKeyKubernetesAgentctlRemotePort: true,
MetadataKeyKubernetesAgentctlInstanceID: true,
MetadataKeyKubernetesContainerRestartCount: true,
MetadataKeyKubernetesResourceInstanceID: true,
MetadataKeyKubernetesResourceExecutorID: true,
MetadataKeyKubernetesResourceProfileID: true,
MetadataKeyKubernetesResourceTaskID: true,
MetadataKeyKubernetesResourceSessionID: true,
MetadataKeyKubernetesResourceEnvironmentID: true,
MetadataKeyKubernetesExecutorConfigHash: true,
MetadataKeyKubernetesProfileConfigHash: true,
MetadataKeyKubernetesTemplateHash: true,
MetadataKeyKubernetesProfileSnapshot: true,
MetadataKeyKubernetesInventoryState: true,
MetadataKeySSHRemoteSessionDir: true,
MetadataKeySSHRemoteAgentctlPort: true,
MetadataKeySSHRemoteAgentctlPID: true,
MetadataKeySSHAgentctlInstanceID: true,
MetadataKeySSHLocalForwardPort: true,
MetadataKeySSHRemoteAgentctlURL: true,
MetadataKeySSHRuntimeAPILocalURL: true,
MetadataKeySSHRuntimeAPIRemotePort: true,
MetadataKeyKubernetesAuthMode: true,
MetadataKeyKubernetesKubeconfigPath: true,
MetadataKeyKubernetesKubeContext: true,
MetadataKeyKubernetesConfigNamespace: true,
MetadataKeyKubernetesRequestTimeoutSeconds: true,
MetadataKeyKubernetesNamespace: true,
MetadataKeyKubernetesPodName: true,
MetadataKeyKubernetesPodUID: true,
MetadataKeyKubernetesMainContainer: true,
MetadataKeyKubernetesPlatform: true,
MetadataKeyKubernetesRuntimeWorkspaceMode: true,
MetadataKeyKubernetesPVCName: true,
MetadataKeyKubernetesPVCUID: true,
MetadataKeyKubernetesPVCCreated: true,
MetadataKeyKubernetesAgentctlRemotePort: true,
MetadataKeyKubernetesAgentctlInstanceID: true,
MetadataKeyKubernetesContainerRestartCount: true,
metadataKubernetesTaskOwned: true,
agentkubernetes.MetadataKeyOwnershipVersion: true,
MetadataKeyKubernetesResourceInstanceID: true,
MetadataKeyKubernetesResourceExecutorID: true,
MetadataKeyKubernetesResourceProfileID: true,
MetadataKeyKubernetesResourceTaskID: true,
MetadataKeyKubernetesResourceSessionID: true,
MetadataKeyKubernetesResourceEnvironmentID: true,
MetadataKeyKubernetesExecutorConfigHash: true,
MetadataKeyKubernetesProfileConfigHash: true,
MetadataKeyKubernetesTemplateHash: true,
MetadataKeyKubernetesProfileSnapshot: true,
MetadataKeyKubernetesInventoryState: true,
}
// ShouldPersistMetadataKey returns true if the given metadata key should
@@ -39,6 +39,8 @@ func TestShouldPersistMetadataKey(t *testing.T) {
func TestKubernetesRuntimeMetadataKeysPersistWithoutLocalForward(t *testing.T) {
persistent := []string{
"kubernetes_task_owned",
"kubernetes_resource_ownership_version",
"auth_mode",
"kubeconfig_path",
"kube_context",
@@ -20,6 +20,7 @@ import (
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/agentctl/server/process"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
)
@@ -46,6 +47,8 @@ var errKubernetesLifecycleRequestIncomplete = errors.New("kubernetes lifecycle r
// KubernetesExecutor owns Kubernetes Pod/PVC lifecycle and process-local
// forwards. Cluster clients are initialized lazily from the selected executor.
type KubernetesExecutor struct {
environmentStore KubernetesEnvironmentStore
secretStore secrets.SecretStore
agentctlResolver *AgentctlResolver
logger *logger.Logger
clientFactory kubernetesRuntimeClientFactory
@@ -53,9 +56,10 @@ type KubernetesExecutor struct {
healthRetryDelay time.Duration
launchTimingClock func() time.Time
mu sync.Mutex
sessions map[string]*kubernetesSession
locks map[string]*kubernetesInstanceLock
mu sync.Mutex
sessions map[string]*kubernetesSession
locks map[string]*kubernetesInstanceLock
pendingControlTokens map[string]string
}
type kubernetesInstanceLock struct {
@@ -107,6 +111,13 @@ func (r *KubernetesExecutor) Name() executor.Name { return executor.NameKubernet
func (r *KubernetesExecutor) HealthCheck(context.Context) error { return nil }
func (r *KubernetesExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateRequest) (*ExecutorInstance, error) {
if req != nil && r.environmentStore != nil && req.TaskEnvironmentID != "" {
return r.createTaskInstance(ctx, req)
}
return r.createSessionInstance(ctx, req)
}
func (r *KubernetesExecutor) createSessionInstance(ctx context.Context, req *ExecutorCreateRequest) (*ExecutorInstance, error) {
if req == nil || req.InstanceID == "" {
return nil, errKubernetesLifecycleRequestIncomplete
}
@@ -236,7 +247,7 @@ func newKubernetesFreshLaunch(
if err != nil {
return nil, err
}
podName, pvcName := kubernetesResourceNames(req.InstanceID)
podName, pvcName := kubernetesResourceNames(identity.InstanceID)
return &kubernetesFreshLaunch{
executor: executor, runtime: runtime, req: req, executorConfig: executorConfig,
profile: profile, identity: identity, podName: podName, pvcName: pvcName,
@@ -307,6 +318,7 @@ func (l *kubernetesFreshLaunch) complete(
l.executorConfig, l.profile, l.identity, runningPod, l.workspace.claim,
l.workspace.createdClaim != nil, remotePort, KubernetesInventoryStateReady,
)
metadata[MetadataKeyKubernetesAgentctlInstanceID] = l.req.InstanceID
// A fresh lifecycle-managed launch already has provisional Pod/PVC inventory.
// Manager persists the final ready row only after both required runtime secret
// references are durable, avoiding a restart-visible ready row that cannot
@@ -639,6 +651,11 @@ func kubernetesIdentity(req *ExecutorCreateRequest) (kubeexecutor.ResourceIdenti
ExecutorID: getMetadataString(req.Metadata, "executor_id"), ProfileID: getMetadataString(req.Metadata, MetadataKeyExecutorProfileID),
InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, EnvironmentID: req.TaskEnvironmentID,
}
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
identity.TaskOwned = true
identity.InstanceID = req.TaskEnvironmentID
}
if _, err := kubeexecutor.OwnershipLabels(identity); err != nil {
return kubeexecutor.ResourceIdentity{}, err
}
@@ -681,6 +698,11 @@ func kubernetesRuntimeMetadata(
"executor_id": identity.ExecutorID,
MetadataKeyExecutorProfileID: identity.ProfileID,
}
if identity.TaskOwned {
metadata[metadataKubernetesTaskOwned] = true
metadata[kubeexecutor.MetadataKeyOwnershipVersion] = kubeexecutor.TaskOwnershipVersion
}
if pod != nil {
metadata[MetadataKeyKubernetesNamespace] = pod.Namespace
metadata[MetadataKeyKubernetesPodName] = pod.Name
@@ -56,6 +56,12 @@ func (r *KubernetesExecutor) bootstrapPod(
}
authEnv := kubernetesAuthEnvironment(req.Env, nonce)
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
authEnv["HOME"] = kubernetesSessionHome(req)
if err := r.writeKubernetesSessionAuth(ctx, runtime, req, pod, profile.MainContainer); err != nil {
return err
}
}
authData, err := kubernetesSerializeEnvironment(authEnv)
if err != nil {
return err
@@ -70,6 +76,15 @@ func (r *KubernetesExecutor) bootstrapPod(
if err := runKubernetesPrepareScript(ctx, runtime.streams, pod, profile.MainContainer); err != nil {
return err
}
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
controlEnv, err := kubernetesSerializeEnvironment(kubernetesAuthEnvironment(nil, nonce))
if err != nil {
return err
}
if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer, kubernetesAuthEnvPath, controlEnv, 0o600); err != nil {
return err
}
}
if err := kubernetesWriteFile(ctx, runtime.streams, pod, profile.MainContainer,
kubernetesStartPath, nil, 0o600); err != nil {
return fmt.Errorf("kubernetes lifecycle: signal bootstrap: %w", err)
@@ -163,6 +178,12 @@ func kubernetesRuntimeEnvironment(req *ExecutorCreateRequest) map[string]string
kubernetesEnvAgentProfile: req.OfficeAgentProfileID,
kubernetesEnvExecutionProfile: req.AgentProfileID,
}
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
values[kubernetesEnvInstanceID] = req.TaskEnvironmentID
delete(values, kubernetesEnvSessionID)
delete(values, kubernetesEnvAgentProfile)
delete(values, kubernetesEnvExecutionProfile)
}
if selectedCheckoutIsPullRequest(req.Metadata) {
values[selectedCheckoutMarker] = "1"
}
@@ -22,6 +22,12 @@ func (r *KubernetesExecutor) StopInstance(ctx context.Context, instance *Executo
if instance == nil {
return nil
}
if err := r.resolveTaskKubernetesStopOwnership(ctx, instance); err != nil {
return err
}
if getMetadataBool(instance.Metadata, metadataKubernetesTaskOwned) {
return r.stopSharedKubernetesInstance(ctx, instance)
}
unlock := r.lockInstance(instance.InstanceID)
defer unlock()
@@ -283,6 +289,7 @@ func kubernetesCleanupInventory(
req := &ExecutorCreateRequest{
TaskID: instance.TaskID,
SessionID: instance.SessionID,
TaskEnvironmentID: getMetadataString(instance.Metadata, MetadataKeyKubernetesResourceEnvironmentID),
PreviousExecutionID: getMetadataString(metadata, MetadataKeyKubernetesResourceInstanceID),
Metadata: metadata,
}
@@ -0,0 +1,51 @@
package lifecycle
import (
"context"
"errors"
"github.com/kandev/kandev/internal/secrets"
)
// Keep the single-use handshake result until storage acknowledges it. All callers
// hold the environment's control lock, so a sibling retries the same credential.
func (r *KubernetesExecutor) persistSharedKubernetesControlToken(ctx context.Context, id, token string) error {
r.mu.Lock()
if r.pendingControlTokens == nil {
r.pendingControlTokens = make(map[string]string)
}
r.pendingControlTokens[id] = token
r.mu.Unlock()
persistCtx, cancel := kubernetesDurableContext(ctx)
defer cancel()
if err := r.secretStore.Update(persistCtx, id, &secrets.UpdateSecretRequest{Value: &token}); err != nil {
return errors.Join(err, r.saveKubernetesControlRecovery(persistCtx, id, token))
}
if err := r.secretStore.Delete(persistCtx, kubernetesControlRecoverySecretID(id)); err != nil && !errors.Is(err, secrets.ErrNotFound) {
return err
}
r.mu.Lock()
delete(r.pendingControlTokens, id)
r.mu.Unlock()
return nil
}
func kubernetesControlRecoverySecretID(id string) string {
if id == "" {
return ""
}
return id + ":recovery"
}
// A failed canonical update must not discard the credential after nonce
// consumption. The deterministic encrypted recovery record survives restart.
func (r *KubernetesExecutor) saveKubernetesControlRecovery(ctx context.Context, id, token string) error {
recoveryID := kubernetesControlRecoverySecretID(id)
_, err := r.secretStore.Get(ctx, recoveryID)
if errors.Is(err, secrets.ErrNotFound) {
return r.secretStore.Create(ctx, &secrets.SecretWithValue{Secret: secrets.Secret{ID: recoveryID, Name: recoveryID}, Value: token})
}
if err != nil {
return err
}
return r.secretStore.Update(ctx, recoveryID, &secrets.UpdateSecretRequest{Value: &token})
}
@@ -0,0 +1,275 @@
package lifecycle
import (
"context"
"errors"
"fmt"
"strings"
"github.com/google/uuid"
"github.com/kandev/kandev/internal/agent/executor"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
)
const metadataKubernetesTaskOwned = kubeexecutor.MetadataKeyTaskOwned
// KubernetesEnvironmentStore separates physical inventory from session execution rows.
type KubernetesEnvironmentStore interface {
GetExecutor(context.Context, string) (*models.Executor, error)
GetTaskEnvironment(context.Context, string) (*models.TaskEnvironment, error)
GetKubernetesEnvironment(context.Context, string) (*models.KubernetesEnvironment, error)
ClaimKubernetesEnvironment(context.Context, string, string, int64, string) (*models.KubernetesEnvironment, error)
ClaimKubernetesEnvironmentCleanup(context.Context, string, string, int64, string) (*models.KubernetesEnvironment, error)
SaveKubernetesEnvironment(context.Context, *models.KubernetesEnvironment, bool) error
DeleteKubernetesEnvironment(context.Context, *models.KubernetesEnvironment) error
ListExecutorsRunningByTaskID(context.Context, string) ([]*models.ExecutorRunning, error)
}
func (m *Manager) wireKubernetesEnvironmentStore() {
if m.executorRegistry == nil {
return
}
backend, err := m.executorRegistry.GetBackend(executor.NameKubernetes)
if err != nil {
return
}
runtime, ok := backend.(*KubernetesExecutor)
if !ok {
return
}
runtime.environmentStore, _ = m.runningWriter.(KubernetesEnvironmentStore)
runtime.secretStore = m.secretStore
}
func (r *KubernetesExecutor) createTaskInstance(ctx context.Context, original *ExecutorCreateRequest) (_ *ExecutorInstance, returnedErr error) {
unlock := r.lockInstance("environment:" + original.TaskEnvironmentID)
defer unlock()
legacy, err := r.isUnadoptedKubernetesResume(ctx, original)
if err != nil {
return nil, err
}
if legacy {
return r.createSessionInstance(ctx, original)
}
record, err := r.claimTaskKubernetesEnvironment(ctx, original)
if err != nil {
return nil, err
}
defer func() {
if record.OperationID != "" {
persistCtx, cancel := kubernetesDurableContext(ctx)
defer cancel()
returnedErr = errors.Join(returnedErr, r.environmentStore.SaveKubernetesEnvironment(persistCtx, record, true))
}
}()
req, err := r.taskKubernetesCreateRequest(ctx, original, record)
if err != nil {
return nil, err
}
instance, err := r.createSessionInstance(ctx, req)
if err != nil {
return nil, err
}
defer func() {
if returnedErr != nil {
rollbackCtx, cancel := kubernetesDurableContext(ctx)
defer cancel()
returnedErr = errors.Join(returnedErr, r.rollbackTaskKubernetesAttachment(rollbackCtx, instance))
}
}()
instance.Metadata[metadataKubernetesTaskOwned] = true
instance.Metadata[MetadataKeyCleanupScript] = getMetadataString(req.Metadata, MetadataKeyCleanupScript)
record.Metadata = cloneKubernetesMetadata(instance.Metadata)
if err := r.persistKubernetesEnvironmentSecrets(ctx, record, instance); err != nil {
return nil, err
}
if err := r.environmentStore.SaveKubernetesEnvironment(ctx, record, true); err != nil {
return nil, err
}
return instance, nil
}
func (r *KubernetesExecutor) adoptKubernetesEnvironment(ctx context.Context, req *ExecutorCreateRequest, record *models.KubernetesEnvironment) error {
rows, err := r.environmentStore.ListExecutorsRunningByTaskID(ctx, req.TaskID)
if err != nil {
return err
}
var candidate *models.ExecutorRunning
for _, row := range rows {
if row == nil || row.Runtime != executor.NameKubernetes {
continue
}
if getMetadataString(row.Metadata, MetadataKeyKubernetesResourceEnvironmentID) != req.TaskEnvironmentID {
return models.ErrWorkspaceReuseUnsafe
}
if getMetadataString(row.Metadata, MetadataKeyKubernetesPodUID) == "" {
return models.ErrWorkspaceReuseUnsafe
}
if candidate != nil && !kubernetesLegacyInventoriesAgree(candidate.Metadata, row.Metadata) {
return models.ErrWorkspaceReuseUnsafe
}
candidate = row
}
if candidate == nil {
return nil
}
record.Metadata = cloneKubernetesMetadata(candidate.Metadata)
record.ControlSecretID = getMetadataString(record.Metadata, MetadataKeyAuthTokenSecret)
record.BootstrapSecretID = getMetadataString(record.Metadata, MetadataKeyBootstrapNonceSecret)
return nil
}
func (r *KubernetesExecutor) attachKubernetesEnvironmentRequest(ctx context.Context, req *ExecutorCreateRequest, record *models.KubernetesEnvironment) error {
if getMetadataString(req.Metadata, "executor_id") != getMetadataString(record.Metadata, MetadataKeyKubernetesResourceExecutorID) || getMetadataString(req.Metadata, MetadataKeyExecutorProfileID) != getMetadataString(record.Metadata, MetadataKeyKubernetesResourceProfileID) {
return models.ErrWorkspaceReuseUnsafe
}
if record.ControlSecretID == "" || record.BootstrapSecretID == "" {
return models.ErrWorkspaceReuseUnsafe
}
token, err := r.secretStore.Reveal(ctx, record.ControlSecretID)
if err != nil {
return err
}
nonce, err := r.secretStore.Reveal(ctx, record.BootstrapSecretID)
if err != nil {
return err
}
if strings.TrimSpace(token) == "" || strings.TrimSpace(nonce) == "" {
return models.ErrWorkspaceReuseUnsafe
}
remoteID := req.InstanceID
if req.PreviousExecutionID != "" {
if previousID := getMetadataString(req.Metadata, MetadataKeyKubernetesAgentctlInstanceID); previousID != "" {
remoteID = previousID
}
}
for key, value := range record.Metadata {
req.Metadata[key] = value
}
req.Metadata[metadataKubernetesTaskOwned] = true
// A resumed session retains its own instance independently of sibling inventory.
req.Metadata[MetadataKeyKubernetesAgentctlInstanceID] = remoteID
req.AuthToken = token
req.BootstrapNonce = nonce
return nil
}
func (r *KubernetesExecutor) persistKubernetesEnvironmentSecrets(ctx context.Context, record *models.KubernetesEnvironment, instance *ExecutorInstance) error {
refs := []struct {
target *string
name, value, key string
}{
{&record.ControlSecretID, "agentctl-auth", instance.AuthToken, MetadataKeyAuthTokenSecret},
{&record.BootstrapSecretID, "agentctl-bootstrap", instance.BootstrapNonce, MetadataKeyBootstrapNonceSecret},
}
for _, item := range refs {
if item.value == "" {
return errors.New("kubernetes environment control credential is empty")
}
*item.target = kubernetesRuntimeSecretID(record.EnvironmentID, item.name)
name := item.name + "-" + record.EnvironmentID
if _, err := r.secretStore.Get(ctx, *item.target); errors.Is(err, secrets.ErrNotFound) {
if err := r.secretStore.Create(ctx, &secrets.SecretWithValue{Secret: secrets.Secret{ID: *item.target, Name: name}, Value: item.value}); err != nil {
return err
}
} else if err != nil {
return err
}
instance.Metadata[item.key] = *item.target
record.Metadata[item.key] = *item.target
}
return nil
}
func kubernetesLegacyInventoriesAgree(first, second map[string]interface{}) bool {
keys := []string{
MetadataKeyKubernetesPodUID, MetadataKeyKubernetesPodName, MetadataKeyKubernetesNamespace,
MetadataKeyKubernetesPVCUID, MetadataKeyKubernetesPVCName, MetadataKeyKubernetesResourceExecutorID,
MetadataKeyKubernetesResourceProfileID, MetadataKeyKubernetesResourceInstanceID,
MetadataKeyKubernetesResourceTaskID, MetadataKeyKubernetesResourceSessionID,
MetadataKeyKubernetesResourceEnvironmentID, MetadataKeyKubernetesProfileSnapshot,
MetadataKeyKubernetesInventoryState, MetadataKeyAuthTokenSecret, MetadataKeyBootstrapNonceSecret,
}
for _, key := range keys {
if getMetadataString(first, key) != getMetadataString(second, key) {
return false
}
}
return getMetadataBool(first, MetadataKeyKubernetesPVCCreated) == getMetadataBool(second, MetadataKeyKubernetesPVCCreated)
}
func (r *KubernetesExecutor) claimTaskKubernetesEnvironment(ctx context.Context, original *ExecutorCreateRequest) (*models.KubernetesEnvironment, error) {
env, err := r.environmentStore.GetTaskEnvironment(ctx, original.TaskEnvironmentID)
if err != nil {
return nil, fmt.Errorf("load Kubernetes task environment: %w", err)
}
if env == nil || env.TaskID != original.TaskID || env.ExecutorType != "k8s" {
return nil, models.ErrWorkspaceReuseUnsafe
}
if original.WorkspaceReuseRequired && env.Status != models.TaskEnvironmentStatusReady {
if env.Status == models.TaskEnvironmentStatusCreating {
return nil, models.ErrWorkspacePreparing
}
return nil, models.ErrWorkspaceReuseUnsafe
}
if r.secretStore == nil {
return nil, errors.New("kubernetes environment secret store is unavailable")
}
record, err := r.environmentStore.ClaimKubernetesEnvironment(ctx, env.ID, env.TaskID, env.OwnershipGeneration, uuid.NewString())
if err != nil {
return nil, fmt.Errorf("%w: claim Kubernetes runtime: %w", models.ErrWorkspaceReuseUnsafe, err)
}
return record, nil
}
func (r *KubernetesExecutor) taskKubernetesCreateRequest(ctx context.Context, original *ExecutorCreateRequest, record *models.KubernetesEnvironment) (*ExecutorCreateRequest, error) {
req := cloneKubernetesCreateRequest(original)
req.Metadata[metadataKubernetesTaskOwned] = true
req.Env = kubernetesSessionEnvironment(req)
if len(record.Metadata) == 0 {
if err := r.adoptKubernetesEnvironment(ctx, req, record); err != nil {
return nil, err
}
}
if len(record.Metadata) > 0 {
if err := r.attachKubernetesEnvironmentRequest(ctx, req, record); err != nil {
return nil, err
}
} else if req.WorkspaceReuseRequired {
return nil, models.ErrWorkspaceReuseUnsafe
}
previousCheckpoint := req.CheckpointRuntimeInventory
req.CheckpointRuntimeInventory = func(checkpointCtx context.Context, metadata map[string]interface{}) error {
metadata[metadataKubernetesTaskOwned] = true
metadata[MetadataKeyCleanupScript] = getMetadataString(req.Metadata, MetadataKeyCleanupScript)
record.Metadata = cloneKubernetesMetadata(metadata)
if err := r.environmentStore.SaveKubernetesEnvironment(checkpointCtx, record, false); err != nil {
return err
}
if previousCheckpoint != nil {
return previousCheckpoint(checkpointCtx, metadata)
}
return nil
}
return req, nil
}
func (r *KubernetesExecutor) isUnadoptedKubernetesResume(ctx context.Context, req *ExecutorCreateRequest) (bool, error) {
if req.PreviousExecutionID == "" || getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) || getMetadataString(req.Metadata, MetadataKeyKubernetesPodUID) == "" {
return false, nil
}
_, err := r.environmentStore.GetKubernetesEnvironment(ctx, req.TaskEnvironmentID)
if !errors.Is(err, models.ErrKubernetesEnvironmentNotFound) {
return false, err
}
env, err := r.environmentStore.GetTaskEnvironment(ctx, req.TaskEnvironmentID)
if err != nil {
return false, err
}
if env == nil || env.TaskID != req.TaskID || env.ExecutorType != "k8s" {
return false, models.ErrWorkspaceReuseUnsafe
}
return true, nil
}
@@ -68,7 +68,7 @@ func (r *KubernetesExecutor) materializeKubernetesCredentials(
if req.AgentConfig == nil {
return nil
}
home := kubernetesAuthHomePath
home := kubernetesSessionHome(req)
catalog := remoteauth.BuildCatalogForHost([]agents.Agent{req.AgentConfig}, runtime.GOOS, mustUserHome())
selectedMethods, err := kubernetesSelectedCredentialMethods(req.Metadata, catalog)
if err != nil {
@@ -87,7 +87,7 @@ func (r *KubernetesExecutor) materializeKubernetesCredentials(
continue
}
command := "set -eu; export HOME=" + shellQuote(home) +
"; set -a; . " + shellQuote(kubernetesAuthEnvPath) + "; set +a; " + method.SetupScript
"; set -a; . " + shellQuote(kubernetesSessionAuthPath(req)) + "; set +a; " + method.SetupScript
if execErr := runtimeClient.streams.Exec(ctx, kubeexecutor.ExecRequest{
Namespace: pod.Namespace, Pod: pod.Name, Container: container,
Command: []string{"sh", "-c", command},
@@ -92,7 +92,7 @@ func kubernetesRecordedInventory(
if err != nil {
return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, err
}
if recorded.agentctlInstanceID != identity.InstanceID {
if !getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) && recorded.agentctlInstanceID != identity.InstanceID {
return kubernetesRecordedState{}, kubeexecutor.ResourceIdentity{}, errors.New(
"kubernetes lifecycle: recorded agentctl instance does not match resource identity",
)
@@ -136,6 +136,7 @@ func kubernetesRecordedResourceIdentity(
validateConnectionIdentity bool,
) (kubeexecutor.ResourceIdentity, error) {
identity := kubeexecutor.ResourceIdentity{
TaskOwned: getMetadataString(req.Metadata, kubeexecutor.MetadataKeyOwnershipVersion) == kubeexecutor.TaskOwnershipVersion,
ExecutorID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceExecutorID),
ProfileID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceProfileID),
InstanceID: getMetadataString(req.Metadata, MetadataKeyKubernetesResourceInstanceID),
@@ -153,7 +154,11 @@ func kubernetesRecordedResourceIdentity(
if err := validateKubernetesRecordedIdentityField("task", req.TaskID, identity.TaskID); err != nil {
return kubeexecutor.ResourceIdentity{}, err
}
if err := validateKubernetesRecordedIdentityField("session", req.SessionID, identity.SessionID); err != nil {
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
if err := validateKubernetesRecordedIdentityField("environment", req.TaskEnvironmentID, identity.EnvironmentID); err != nil {
return kubeexecutor.ResourceIdentity{}, err
}
} else if err := validateKubernetesRecordedIdentityField("session", req.SessionID, identity.SessionID); err != nil {
return kubeexecutor.ResourceIdentity{}, err
}
if validateConnectionIdentity {
@@ -184,7 +189,9 @@ func kubernetesRequestWithRecordedIdentity(
) *ExecutorCreateRequest {
recorded := *req
recorded.TaskID = identity.TaskID
recorded.SessionID = identity.SessionID
if !getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
recorded.SessionID = identity.SessionID
}
recorded.TaskEnvironmentID = identity.EnvironmentID
return &recorded
}
@@ -257,6 +264,9 @@ func (r *KubernetesExecutor) reconnectKubernetesAgentctl(
pod *corev1.Pod,
recorded kubernetesRecordedState,
) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) {
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
return r.connectSharedKubernetesAgentctl(ctx, runtime, req, pod, recorded.agentctlInstanceID)
}
if req.AuthToken == "" {
return nil, nil, "", 0, errors.New("kubernetes lifecycle: recorded agentctl auth token is unavailable")
}
@@ -297,6 +307,9 @@ func (r *KubernetesExecutor) connectRestartedKubernetesAgentctl(
pod *corev1.Pod,
remoteInstanceID string,
) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) {
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
return r.connectSharedKubernetesAgentctl(ctx, runtime, req, pod, remoteInstanceID)
}
controlForward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod)
if err != nil {
return nil, nil, "", 0, err
@@ -503,7 +516,7 @@ func ValidateKubernetesResumeMetadata(
if _, err := kubeexecutor.ParseExecutorConfig(executorConfigValues); err != nil {
return fmt.Errorf("kubernetes lifecycle: validate current executor config: %w", err)
}
req := &ExecutorCreateRequest{TaskID: taskID, SessionID: sessionID, Metadata: metadata}
req := &ExecutorCreateRequest{TaskID: taskID, SessionID: sessionID, TaskEnvironmentID: getMetadataString(metadata, MetadataKeyKubernetesResourceEnvironmentID), Metadata: metadata}
recorded, _, err := kubernetesRecordedInventory(req, true)
if err != nil {
return err
@@ -67,7 +67,7 @@ func (r *KubernetesExecutor) RefreshRemoteInstance(
Instance: &ExecutorInstance{
InstanceID: instance.InstanceID,
TaskID: inspection.identity.TaskID,
SessionID: inspection.identity.SessionID,
SessionID: req.SessionID,
RuntimeName: r.Name(), Client: client, WorkspacePath: kubernetesWorkspacePath,
Metadata: metadata, AuthToken: token, BootstrapNonce: instance.BootstrapNonce,
},
@@ -180,6 +180,9 @@ func kubernetesActiveRefreshRequest(
req.InstanceID = instance.InstanceID
req.TaskID = identity.TaskID
req.SessionID = identity.SessionID
if getMetadataBool(instance.Metadata, metadataKubernetesTaskOwned) {
req.SessionID = instance.SessionID
}
req.TaskEnvironmentID = identity.EnvironmentID
req.Metadata = cloneKubernetesMetadata(instance.Metadata)
req.AuthToken = instance.AuthToken
@@ -194,6 +197,9 @@ func (r *KubernetesExecutor) connectKubernetesRefresh(
req *ExecutorCreateRequest,
inspection kubernetesRefreshInspection,
) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) {
if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
return r.connectSharedKubernetesAgentctl(ctx, freshRuntime, req, inspection.pod, inspection.recorded.agentctlInstanceID)
}
if inspection.restarted {
return r.connectRestartedKubernetesAgentctl(
ctx, freshRuntime, req, inspection.pod, inspection.recorded.agentctlInstanceID,
@@ -0,0 +1,62 @@
package lifecycle
import (
"context"
"crypto/sha256"
"encoding/hex"
"path"
"github.com/kandev/kandev/internal/githubauth"
corev1 "k8s.io/api/core/v1"
)
func kubernetesSessionHome(req *ExecutorCreateRequest) string {
if !getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
return kubernetesAuthHomePath
}
sum := sha256.Sum256([]byte(req.SessionID))
return path.Join("/run/kandev/sessions", hex.EncodeToString(sum[:16]), "home")
}
func kubernetesSessionAuthPath(req *ExecutorCreateRequest) string {
if !getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
return kubernetesAuthEnvPath
}
return path.Join(kubernetesSessionHome(req), "auth.env")
}
func kubernetesSessionEnvironment(req *ExecutorCreateRequest) map[string]string {
env := make(map[string]string, len(req.Env)+8)
for key, value := range req.Env {
if !isKubernetesOwnedControlEnvironmentKey(key) {
env[key] = value
}
}
env["HOME"] = kubernetesSessionHome(req)
env[kubernetesEnvSessionID] = req.SessionID
env[kubernetesEnvTaskID] = req.TaskID
env[kubernetesEnvInstanceID] = req.InstanceID
env[kubernetesEnvEnvironmentID] = req.TaskEnvironmentID
env[kubernetesEnvAgentProfile] = req.OfficeAgentProfileID
env[kubernetesEnvExecutionProfile] = req.AgentProfileID
if hasManagedGitCredentialBrokerEnv(req.Env) {
env[githubauth.CredentialHelperPathEnv] = kubernetesAgentctlPath
}
return env
}
func (r *KubernetesExecutor) writeKubernetesSessionAuth(ctx context.Context, runtime *kubernetesRuntimeClient, req *ExecutorCreateRequest, pod *corev1.Pod, container string) error {
data, err := kubernetesSerializeEnvironment(kubernetesSessionEnvironment(req))
if err != nil {
return err
}
return kubernetesWriteFile(ctx, runtime.streams, pod, container, kubernetesSessionAuthPath(req), data, 0o600)
}
func (r *KubernetesExecutor) prepareSharedKubernetesCredentials(ctx context.Context, runtime *kubernetesRuntimeClient, req *ExecutorCreateRequest, pod *corev1.Pod, container string) error {
if err := r.writeKubernetesSessionAuth(ctx, runtime, req, pod, container); err != nil {
return err
}
uploader := kubernetesPodFileUploader{streams: runtime.streams, pod: pod, container: container}
return r.materializeKubernetesCredentials(ctx, uploader, runtime, req, pod, container)
}
@@ -0,0 +1,174 @@
package lifecycle
import (
"context"
"errors"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/task/models"
apierrors "k8s.io/apimachinery/pkg/api/errors"
)
func (r *KubernetesExecutor) stopSharedKubernetesInstance(ctx context.Context, instance *ExecutorInstance) error {
unlock := r.lockInstance(instance.InstanceID)
defer unlock()
ctx, cancel := kubernetesDurableContext(ctx)
defer cancel()
session := r.closeSharedKubernetesSessions(instance)
if r.environmentStore == nil || r.secretStore == nil {
return errors.New("kubernetes task environment store is unavailable")
}
environmentID := getMetadataString(instance.Metadata, MetadataKeyKubernetesResourceEnvironmentID)
record, err := r.environmentStore.GetKubernetesEnvironment(ctx, environmentID)
if errors.Is(err, models.ErrKubernetesEnvironmentNotFound) {
return nil
}
if err != nil {
return err
}
if record.TaskID != instance.TaskID {
return models.ErrWorkspaceReuseUnsafe
}
instance, err = r.taskKubernetesStopInstance(ctx, instance, record)
if err != nil {
return err
}
runtime, err := r.kubernetesCleanupRuntime(instance, session)
if err != nil {
return err
}
recorded, identity, err := kubernetesCleanupInventory(instance)
if err != nil {
return err
}
pod, err := runtime.resources.GetPod(ctx, recorded.namespace, recorded.podName)
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
if err := verifyRecordedPod(pod, recorded.namespace, recorded.podName, recorded.podUID, identity); err != nil {
return err
}
token, err := r.secretStore.Reveal(ctx, record.ControlSecretID)
if err != nil {
return err
}
forward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod)
if err != nil {
return err
}
defer func() { _ = forward.Close() }()
control.SetAuthToken(token)
return control.DeleteInstance(ctx, getMetadataString(instance.Metadata, MetadataKeyKubernetesAgentctlInstanceID))
}
// Rollback owns only the newly attached agent, even if credential persistence failed.
func (r *KubernetesExecutor) rollbackTaskKubernetesAttachment(ctx context.Context, instance *ExecutorInstance) error {
session := r.closeKubernetesSession(instance.InstanceID)
runtime, err := r.kubernetesCleanupRuntime(instance, session)
if err != nil {
return err
}
recorded, identity, err := kubernetesCleanupInventory(instance)
if err != nil {
return err
}
pod, err := runtime.resources.GetPod(ctx, recorded.namespace, recorded.podName)
if apierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
if err := verifyRecordedPod(pod, recorded.namespace, recorded.podName, recorded.podUID, identity); err != nil {
return err
}
forward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod)
if err != nil {
return err
}
defer func() { _ = forward.Close() }()
control.SetAuthToken(instance.AuthToken)
return control.DeleteInstance(ctx, getMetadataString(instance.Metadata, MetadataKeyKubernetesAgentctlInstanceID))
}
func (r *KubernetesExecutor) resolveTaskKubernetesStopOwnership(ctx context.Context, instance *ExecutorInstance) error {
if getMetadataBool(instance.Metadata, metadataKubernetesTaskOwned) || r.environmentStore == nil {
return nil
}
envID := getMetadataString(instance.Metadata, MetadataKeyKubernetesResourceEnvironmentID)
if envID == "" {
return nil
}
record, err := r.environmentStore.GetKubernetesEnvironment(ctx, envID)
if errors.Is(err, models.ErrKubernetesEnvironmentNotFound) {
return nil
}
if err != nil {
return err
}
if record.TaskID != instance.TaskID || getMetadataString(record.Metadata, MetadataKeyKubernetesPodUID) != getMetadataString(instance.Metadata, MetadataKeyKubernetesPodUID) {
return models.ErrWorkspaceReuseUnsafe
}
instance.Metadata[metadataKubernetesTaskOwned] = true
return nil
}
func (r *KubernetesExecutor) taskKubernetesStopInstance(ctx context.Context, instance *ExecutorInstance, record *models.KubernetesEnvironment) (*ExecutorInstance, error) {
current, err := r.environmentStore.GetExecutor(ctx, getMetadataString(record.Metadata, MetadataKeyKubernetesResourceExecutorID))
if err != nil {
return nil, err
}
if current == nil || current.Type != models.ExecutorTypeKubernetes {
return nil, models.ErrWorkspaceReuseUnsafe
}
config, err := kubeexecutor.ParseExecutorConfig(current.Config)
if err != nil {
return nil, err
}
remoteID := getMetadataString(instance.Metadata, MetadataKeyKubernetesAgentctlInstanceID)
copyInstance := *instance
copyInstance.Metadata = overlayCurrentKubernetesConnectionMetadata(record.Metadata, current.Config, config)
copyInstance.Metadata[MetadataKeyKubernetesAgentctlInstanceID] = remoteID
return &copyInstance, nil
}
// A resumed execution can have a new local ID while agentctl retains the remote
// session ID. Release every local connection for that exact remote session.
func (r *KubernetesExecutor) closeSharedKubernetesSessions(instance *ExecutorInstance) *kubernetesSession {
r.mu.Lock()
var closing []*kubernetesSession
for id, session := range r.sessions {
if id == instance.InstanceID || matchesSharedKubernetesSession(session, instance) {
delete(r.sessions, id)
closing = append(closing, session)
}
}
r.mu.Unlock()
for _, session := range closing {
_ = closeKubernetesSessionResources(session)
}
if len(closing) == 0 {
return nil
}
// Every matching connection uses the same pod and executor configuration;
// any one supplies the fallback runtime client after its forward is closed.
return closing[0]
}
func matchesSharedKubernetesSession(session *kubernetesSession, instance *ExecutorInstance) bool {
if session == nil || session.request == nil {
return false
}
req := session.request
remoteID := getMetadataString(req.Metadata, MetadataKeyKubernetesAgentctlInstanceID)
if remoteID == "" {
remoteID = req.InstanceID
}
return req.TaskID == instance.TaskID && req.SessionID == instance.SessionID &&
req.TaskEnvironmentID == getMetadataString(instance.Metadata, MetadataKeyKubernetesResourceEnvironmentID) &&
remoteID == getMetadataString(instance.Metadata, MetadataKeyKubernetesAgentctlInstanceID)
}
@@ -0,0 +1,105 @@
package lifecycle
import (
"context"
"errors"
"fmt"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/secrets"
corev1 "k8s.io/api/core/v1"
)
func (r *KubernetesExecutor) connectSharedKubernetesAgentctl(ctx context.Context, runtime *kubernetesRuntimeClient, req *ExecutorCreateRequest, pod *corev1.Pod, remoteID string) (_ *agentctl.Client, _ kubeexecutor.PortForwardSession, _ string, _ int, returnedErr error) {
if err := r.prepareSharedKubernetesCredentials(ctx, runtime, req, pod, getMetadataString(req.Metadata, MetadataKeyKubernetesMainContainer)); err != nil {
return nil, nil, "", 0, err
}
forward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod)
if err != nil {
return nil, nil, "", 0, err
}
defer func() { _ = forward.Close() }()
unlock := r.lockInstance("control:" + req.TaskEnvironmentID)
defer unlock()
token, err := r.sharedKubernetesControlToken(ctx, req)
if err != nil {
return nil, nil, "", 0, err
}
secretID := getMetadataString(req.Metadata, MetadataKeyAuthTokenSecret)
control.SetAuthToken(token)
response, created, err := getOrCreateSharedKubernetesInstance(ctx, control, req, remoteID)
defer func() {
if returnedErr != nil && created {
rollbackCtx, cancel := kubernetesDurableContext(ctx)
defer cancel()
returnedErr = errors.Join(returnedErr, control.DeleteInstance(rollbackCtx, remoteID))
}
}()
if isAgentctlAuthError(err) && req.BootstrapNonce != "" {
token, err = control.Handshake(ctx, req.BootstrapNonce)
if err == nil && r.secretStore != nil && secretID != "" {
err = r.persistSharedKubernetesControlToken(ctx, secretID, token)
}
if err == nil {
response, created, err = getOrCreateSharedKubernetesInstance(ctx, control, req, remoteID)
}
}
if err != nil {
return nil, nil, "", 0, err
}
instanceForward, err := startKubernetesForward(ctx, runtime.streams, pod, uint16(response.Port))
if err != nil {
return nil, nil, "", 0, err
}
client := newKubernetesAgentctlClient(r.logger, req, remoteID, token, instanceForward.LocalPort())
if err := client.Health(ctx); err != nil {
client.Close()
_ = instanceForward.Close()
return nil, nil, "", 0, err
}
return client, instanceForward, token, response.Port, nil
}
func getOrCreateSharedKubernetesInstance(ctx context.Context, control *agentctl.ControlClient, req *ExecutorCreateRequest, remoteID string) (*agentctl.CreateInstanceResponse, bool, error) {
request := buildReconnectCreateInstanceRequest(req, remoteID)
info, err := control.GetInstance(ctx, remoteID)
if err == nil {
response := &agentctl.CreateInstanceResponse{ID: info.ID, Port: info.Port}
if err := validateKubernetesAgentctlInstanceResponse(response, request); err != nil {
return nil, false, err
}
if info.WorkspacePath != request.WorkspacePath {
return nil, false, fmt.Errorf("kubernetes instance workspace mismatch")
}
return response, false, nil
}
if !errors.Is(err, agentctl.ErrInstanceNotFound) {
return nil, false, err
}
response, createErr := createOrReconcileKubernetesAgentctlInstance(ctx, control, request)
return response, true, createErr
}
func (r *KubernetesExecutor) sharedKubernetesControlToken(ctx context.Context, req *ExecutorCreateRequest) (string, error) {
id := getMetadataString(req.Metadata, MetadataKeyAuthTokenSecret)
if r.secretStore == nil || id == "" {
return req.AuthToken, nil
}
r.mu.Lock()
pending := r.pendingControlTokens[id]
r.mu.Unlock()
if pending == "" {
var err error
pending, err = r.secretStore.Reveal(ctx, kubernetesControlRecoverySecretID(id))
if err != nil && !errors.Is(err, secrets.ErrNotFound) {
return "", err
}
}
if pending != "" {
if err := r.persistSharedKubernetesControlToken(ctx, id, pending); err != nil {
return "", err
}
}
return r.secretStore.Reveal(ctx, id)
}
@@ -0,0 +1,162 @@
package lifecycle
import (
"context"
"errors"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"testing"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
)
type taskPodFailedSecrets struct{ secrets.SecretStore }
func (s taskPodFailedSecrets) Get(context.Context, string) (*secrets.Secret, error) {
return nil, errors.New("injected secret storage failure")
}
func TestKubernetesTaskPodSecretFailureRollsBackAttachment(t *testing.T) {
f := newTaskPodFixture(t)
f.launch(t, 1)
f.runtime.secretStore = taskPodFailedSecrets{f.secretStore}
_, err := f.runtime.CreateInstance(context.Background(), taskPodRequest(2))
require.ErrorContains(t, err, "injected secret storage failure")
_, active, _ := f.control.snapshot()
require.Len(t, active, 1, "failed attachment must not leave a remote agent")
require.Contains(t, active, "instance-1")
require.Empty(t, f.resources.deletedPods)
require.Empty(t, f.resources.deletedPVCs)
f.runtime.mu.Lock()
_, exists := f.runtime.sessions["instance-2"]
f.runtime.mu.Unlock()
require.False(t, exists, "failed attachment must close its forward")
}
func TestKubernetesTaskPodPinsCleanupScript(t *testing.T) {
f := newTaskPodFixture(t)
req := taskPodRequest(1)
req.Metadata[MetadataKeyCleanupScript] = "echo clean-task"
_, err := f.runtime.CreateInstance(context.Background(), req)
require.NoError(t, err)
record, err := f.repo.GetKubernetesEnvironment(context.Background(), req.TaskEnvironmentID)
require.NoError(t, err)
require.Equal(t, "echo clean-task", getMetadataString(record.Metadata, MetadataKeyCleanupScript))
}
func TestKubernetesTaskPodRejectsDifferentProfile(t *testing.T) {
f := newTaskPodFixture(t)
f.launch(t, 1)
req := taskPodRequest(2)
req.Metadata[MetadataKeyExecutorProfileID] = "different-profile"
_, err := f.runtime.CreateInstance(context.Background(), req)
require.ErrorIs(t, err, models.ErrWorkspaceReuseUnsafe)
require.Len(t, f.resources.createdPods, 1)
}
type taskPodLegacyStore struct {
KubernetesEnvironmentStore
rows []*models.ExecutorRunning
}
func (s taskPodLegacyStore) ListExecutorsRunningByTaskID(context.Context, string) ([]*models.ExecutorRunning, error) {
return s.rows, nil
}
func TestKubernetesTaskPodLegacyInventoryMustAgree(t *testing.T) {
for _, key := range []string{MetadataKeyKubernetesPodUID, MetadataKeyKubernetesNamespace, MetadataKeyKubernetesResourceExecutorID, MetadataKeyKubernetesProfileSnapshot} {
t.Run(key, func(t *testing.T) {
f := newTaskPodFixture(t)
first := f.launch(t, 1)
second := cloneKubernetesMetadata(first.Metadata)
second[key] = "conflicting"
f.runtime.environmentStore = taskPodLegacyStore{KubernetesEnvironmentStore: f.repo, rows: []*models.ExecutorRunning{
{Runtime: "k8s", Metadata: first.Metadata}, {Runtime: "k8s", Metadata: second},
}}
record := &models.KubernetesEnvironment{}
err := f.runtime.adoptKubernetesEnvironment(context.Background(), taskPodRequest(2), record)
require.ErrorIs(t, err, models.ErrWorkspaceReuseUnsafe)
})
}
}
func TestKubernetesTaskPodLegacySessionCannotDeleteAdoptedPod(t *testing.T) {
f := newTaskPodFixture(t)
first := f.launch(t, 1)
f.launch(t, 2)
delete(first.Metadata, metadataKubernetesTaskOwned)
first.StopReason = StopReasonTaskDeleted
require.NoError(t, f.runtime.StopInstance(context.Background(), first, true))
require.Empty(t, f.resources.deletedPods)
require.Empty(t, f.resources.deletedPVCs)
_, active, _ := f.control.snapshot()
require.Contains(t, active, "instance-2")
}
type taskPodFailedReleaseStore struct{ KubernetesEnvironmentStore }
func (s taskPodFailedReleaseStore) SaveKubernetesEnvironment(ctx context.Context, record *models.KubernetesEnvironment, release bool) error {
if release {
return errors.New("injected release failure")
}
return s.KubernetesEnvironmentStore.SaveKubernetesEnvironment(ctx, record, release)
}
func TestKubernetesTaskPodReleaseFailureRollsBackAttachment(t *testing.T) {
f := newTaskPodFixture(t)
f.launch(t, 1)
f.runtime.environmentStore = taskPodFailedReleaseStore{f.repo}
_, err := f.runtime.CreateInstance(context.Background(), taskPodRequest(2))
require.ErrorContains(t, err, "injected release failure")
_, active, _ := f.control.snapshot()
require.Len(t, active, 1)
require.Contains(t, active, "instance-1")
}
func TestKubernetesTaskPodConnectionFailureRollsBackOnlyNewInstance(t *testing.T) {
for _, resume := range []bool{false, true} {
t.Run(map[bool]string{false: "new attachment", true: "existing instance"}[resume], func(t *testing.T) {
f := newTaskPodFixture(t)
first := f.launch(t, 1)
attaching := newFakeKubernetesExecutor(t, f.resources, f.execs, map[uint16]uint16{uint16(kubeexecutor.DefaultAgentctlPort): f.port})
attaching.environmentStore = f.repo
attaching.secretStore = f.secretStore
t.Cleanup(func() { require.NoError(t, attaching.Close()) })
req := taskPodRequest(2)
if resume {
req.SessionID = first.SessionID
req.PreviousExecutionID = first.InstanceID
req.Metadata = cloneKubernetesMetadata(first.Metadata)
}
_, err := attaching.CreateInstance(context.Background(), req)
require.Error(t, err)
_, active, _ := f.control.snapshot()
require.Len(t, active, 1)
require.Contains(t, active, "instance-1")
})
}
}
func TestKubernetesTaskPodLegacyResumeDoesNotRequireConsolidation(t *testing.T) {
f := newTaskPodFixture(t)
ctx := context.Background()
req := taskPodRequest(1)
first, err := f.runtime.createSessionInstance(ctx, req)
require.NoError(t, err)
other := cloneKubernetesMetadata(first.Metadata)
other[MetadataKeyKubernetesPodUID] = "different-legacy-pod"
f.runtime.environmentStore = taskPodLegacyStore{KubernetesEnvironmentStore: f.repo, rows: []*models.ExecutorRunning{{Runtime: "k8s", Metadata: first.Metadata}, {Runtime: "k8s", Metadata: other}}}
resumed := taskPodRequest(1)
resumed.InstanceID = "resumed-legacy"
resumed.PreviousExecutionID = first.InstanceID
resumed.AuthToken = first.AuthToken
resumed.BootstrapNonce = first.BootstrapNonce
for key, value := range first.Metadata {
resumed.Metadata[key] = value
}
instance, err := f.runtime.CreateInstance(ctx, resumed)
require.NoError(t, err, "an existing legacy session can resume its exact pod without consolidating other pods")
require.Equal(t, first.Metadata[MetadataKeyKubernetesPodUID], instance.Metadata[MetadataKeyKubernetesPodUID])
require.Len(t, f.resources.createdPods, 1)
}
@@ -0,0 +1,172 @@
package lifecycle
import (
"context"
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"github.com/jmoiron/sqlx"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite"
"github.com/stretchr/testify/require"
)
type taskPodControl struct {
mu sync.Mutex
active map[string]agentctl.CreateInstanceRequest
created []agentctl.CreateInstanceRequest
handshakes int
token string
failID string
beforeDelete func()
}
func (c *taskPodControl) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodDelete && c.beforeDelete != nil {
c.beforeDelete()
}
c.mu.Lock()
defer c.mu.Unlock()
if r.URL.Path == "/health" {
w.WriteHeader(http.StatusOK)
return
}
if r.URL.Path == "/auth/handshake" {
c.handshakes++
if c.handshakes > 1 {
http.Error(w, "nonce already used", http.StatusUnauthorized)
return
}
_ = json.NewEncoder(w).Encode(map[string]string{"token": c.token})
return
}
if r.Header.Get("Authorization") != "Bearer "+c.token {
w.WriteHeader(http.StatusUnauthorized)
_ = json.NewEncoder(w).Encode(map[string]string{"error": "unauthorized"})
return
}
if r.URL.Path == "/api/v1/instances" {
var request agentctl.CreateInstanceRequest
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
http.Error(w, err.Error(), 400)
return
}
if request.ID == c.failID {
w.WriteHeader(http.StatusBadRequest)
_ = json.NewEncoder(w).Encode(map[string]string{"error": "injected create failure"})
return
}
c.created = append(c.created, request)
c.active[request.ID] = request
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(agentctl.CreateInstanceResponse{ID: request.ID, Port: 41001})
return
}
id := strings.TrimPrefix(r.URL.Path, "/api/v1/instances/")
request, exists := c.active[id]
if !exists {
http.NotFound(w, r)
return
}
if r.Method == http.MethodDelete {
delete(c.active, id)
w.WriteHeader(http.StatusNoContent)
return
}
_ = json.NewEncoder(w).Encode(agentctl.InstanceInfo{ID: id, Port: 41001, WorkspacePath: request.WorkspacePath})
}
func (c *taskPodControl) snapshot() ([]agentctl.CreateInstanceRequest, map[string]agentctl.CreateInstanceRequest, int) {
c.mu.Lock()
defer c.mu.Unlock()
active := make(map[string]agentctl.CreateInstanceRequest, len(c.active))
for k, v := range c.active {
active[k] = v
}
return append([]agentctl.CreateInstanceRequest(nil), c.created...), active, c.handshakes
}
func (c *taskPodControl) reboot() {
c.mu.Lock()
defer c.mu.Unlock()
c.active = map[string]agentctl.CreateInstanceRequest{}
c.handshakes = 0
c.token += "-restarted"
}
type taskPodFixture struct {
runtime *KubernetesExecutor
manager *Manager
repo *tasksqlite.Repository
database *sqlx.DB
resources *fakeKubernetesResources
execs *recordingKubernetesExec
control *taskPodControl
secretStore secrets.SecretStore
port uint16
}
func newTaskPodFixture(t *testing.T) *taskPodFixture {
t.Helper()
repo, database := runOwnerTestRepository(t)
ctx := context.Background()
require.NoError(t, repo.CreateTask(ctx, &models.Task{ID: "task-1", Title: "shared pod"}))
require.NoError(t, repo.CreateTaskEnvironment(ctx, &models.TaskEnvironment{ID: "environment-1", TaskID: "task-1", ExecutorType: "k8s", ExecutorID: "executor-1", ExecutorProfileID: "profile-1", Status: models.TaskEnvironmentStatusCreating, OwnershipGeneration: 1}))
require.NoError(t, repo.CreateExecutor(ctx, &models.Executor{ID: "executor-1", Name: "cluster", Type: models.ExecutorTypeKubernetes, Config: map[string]string{"auth_mode": "in_cluster", "namespace": "kandev-agents", "request_timeout_seconds": "30"}}))
control := &taskPodControl{active: map[string]agentctl.CreateInstanceRequest{}, token: "shared-control-token"}
server := httptest.NewServer(control)
t.Cleanup(server.Close)
_, raw, err := net.SplitHostPort(server.Listener.Addr().String())
require.NoError(t, err)
port, err := strconv.Atoi(raw)
require.NoError(t, err)
f := &taskPodFixture{repo: repo, database: database, resources: &fakeKubernetesResources{}, execs: &recordingKubernetesExec{}, control: control, secretStore: newInMemorySecretStore(), port: uint16(port)}
f.restartBackend(t)
return f
}
func (f *taskPodFixture) restartBackend(t *testing.T) {
t.Helper()
if f.runtime != nil {
require.NoError(t, f.runtime.Close())
}
f.runtime = newFakeKubernetesExecutor(t, f.resources, f.execs, map[uint16]uint16{uint16(kubeexecutor.DefaultAgentctlPort): f.port, 41001: f.port})
current := f.runtime
t.Cleanup(func() { require.NoError(t, current.Close()) })
f.manager = newTestManager(t)
f.manager.executorRegistry = NewExecutorRegistry(f.manager.logger)
f.manager.executorRegistry.Register(f.runtime)
f.manager.SetExecutorRunningWriter(f.repo)
f.manager.SetSecretStore(f.secretStore)
}
func taskPodRequest(number int) *ExecutorCreateRequest {
req := validKubernetesCreateRequest()
setManagedKubernetesWorkspace(req)
req.InstanceID = "instance-" + strconv.Itoa(number)
req.SessionID = "session-" + strconv.Itoa(number)
req.AgentProfileID = "agent-profile-" + strconv.Itoa(number)
req.Env = map[string]string{"SESSION_CREDENTIAL": "credential-" + strconv.Itoa(number)}
req.WorkspaceReuseRequired = number > 1
return req
}
func (f *taskPodFixture) launch(t *testing.T, number int) *ExecutorInstance {
t.Helper()
instance, err := f.runtime.CreateInstance(context.Background(), taskPodRequest(number))
require.NoError(t, err)
if number == 1 {
_, err = f.database.Exec(`UPDATE task_environments SET status = 'ready' WHERE id = 'environment-1'`)
require.NoError(t, err)
}
return instance
}
@@ -0,0 +1,231 @@
package lifecycle
import (
"context"
"github.com/kandev/kandev/internal/secrets"
"testing"
"time"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
)
func TestKubernetesTaskPodBackendRestartKeepsSessionInstances(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
f.launch(t, 2)
f.restartBackend(t)
req := taskPodRequest(1)
req.InstanceID = "resumed-execution"
req.PreviousExecutionID = a.InstanceID
req.Metadata = cloneKubernetesMetadata(a.Metadata)
resumed, err := f.runtime.CreateInstance(context.Background(), req)
require.NoError(t, err)
require.Equal(t, a.Metadata[MetadataKeyKubernetesPodUID], resumed.Metadata[MetadataKeyKubernetesPodUID])
require.Equal(t, "instance-1", resumed.Metadata[MetadataKeyKubernetesAgentctlInstanceID])
_, active, _ := f.control.snapshot()
require.Len(t, active, 2, "resume must not leave a second agent process for the same session")
require.Len(t, f.resources.createdPods, 1)
}
func TestKubernetesTaskPodRefreshKeepsSiblingSessionIdentity(t *testing.T) {
req := taskPodRequest(2)
req.Metadata[metadataKubernetesTaskOwned] = true
instance := &ExecutorInstance{InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, Metadata: req.Metadata}
identity := kubeexecutor.ResourceIdentity{TaskID: req.TaskID, SessionID: "session-1", EnvironmentID: req.TaskEnvironmentID}
refreshed := kubernetesActiveRefreshRequest(req, instance, identity)
require.Equal(t, "session-2", refreshed.SessionID)
}
func TestKubernetesTaskPodContainerRestartSharesRotatedControlToken(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
b := f.launch(t, 2)
f.control.reboot()
f.resources.mu.Lock()
f.resources.pod.Status.ContainerStatuses[0].RestartCount = 1
f.resources.mu.Unlock()
ctx := context.Background()
for _, instance := range []*ExecutorInstance{a, b} {
refreshed, err := f.runtime.RefreshRemoteInstance(ctx, kubernetesRefreshInstance(instance, taskPodRequest(1).Metadata))
require.NoError(t, err)
require.NotNil(t, refreshed)
require.Equal(t, instance.SessionID, refreshed.Instance.SessionID)
require.NoError(t, refreshed.Commit(nil))
}
_, active, handshakes := f.control.snapshot()
require.Len(t, active, 2)
require.Equal(t, 1, handshakes)
}
func TestKubernetesTaskPodRejectsPreparingEnvironment(t *testing.T) {
f := newTaskPodFixture(t)
_, err := f.runtime.CreateInstance(context.Background(), taskPodRequest(2))
require.ErrorIs(t, err, models.ErrWorkspacePreparing)
require.Empty(t, f.resources.createdPods)
_, err = f.repo.GetKubernetesEnvironment(context.Background(), "environment-1")
require.ErrorIs(t, err, models.ErrKubernetesEnvironmentNotFound)
}
func TestKubernetesTaskPodConcurrentAttachAndFailure(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
results := make(chan error, 2)
for _, n := range []int{2, 3} {
go func(number int) {
_, err := f.runtime.CreateInstance(context.Background(), taskPodRequest(number))
results <- err
}(n)
}
require.NoError(t, <-results)
require.NoError(t, <-results)
require.Len(t, f.resources.createdPods, 1)
require.Len(t, f.resources.createdPVCs, 1)
f.control.mu.Lock()
f.control.failID = "instance-4"
f.control.mu.Unlock()
_, err := f.runtime.CreateInstance(context.Background(), taskPodRequest(4))
require.Error(t, err)
_, active, _ := f.control.snapshot()
require.Len(t, active, 3)
require.Contains(t, active, a.InstanceID)
require.Empty(t, f.resources.deletedPods)
}
func TestKubernetesTaskPodResumeValidationAllowsSibling(t *testing.T) {
f := newTaskPodFixture(t)
f.launch(t, 1)
b := f.launch(t, 2)
require.NoError(t, ValidateKubernetesResumeMetadata(b.Metadata, b.TaskID, b.SessionID, map[string]string{"auth_mode": "in_cluster", "namespace": "kandev-agents"}))
}
func TestKubernetesTaskPodMissingPodReplacementSharedAcrossResumes(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
b := f.launch(t, 2)
f.restartBackend(t)
f.control.reboot()
f.resources.mu.Lock()
f.resources.pod = nil
f.resources.nextPodUID = "replacement-uid"
f.resources.mu.Unlock()
for _, instance := range []*ExecutorInstance{a, b} {
req := taskPodRequest(2)
req.SessionID = instance.SessionID
req.InstanceID = instance.InstanceID + "-resumed"
req.PreviousExecutionID = instance.InstanceID
req.Metadata = cloneKubernetesMetadata(instance.Metadata)
resumed, err := f.runtime.CreateInstance(context.Background(), req)
require.NoError(t, err)
require.Equal(t, "replacement-uid", resumed.Metadata[MetadataKeyKubernetesPodUID])
}
require.Len(t, f.resources.createdPods, 2)
require.Len(t, f.resources.createdPVCs, 1)
_, active, handshakes := f.control.snapshot()
require.Len(t, active, 2)
require.Equal(t, 1, handshakes)
require.NoError(t, f.runtime.StopInstance(context.Background(), a, true))
_, active, _ = f.control.snapshot()
require.NotContains(t, active, "instance-1")
require.Contains(t, active, "instance-2")
}
func TestKubernetesTaskPodStaleSessionCannotOverwriteRotatedToken(t *testing.T) {
for _, layer := range []string{"lifecycle", "manager"} {
t.Run(layer, func(t *testing.T) {
f := newTaskPodFixture(t)
instance := f.launch(t, 1)
ctx := context.Background()
id := getMetadataString(instance.Metadata, MetadataKeyAuthTokenSecret)
rotated := "newer-sibling-token"
require.NoError(t, f.secretStore.Update(ctx, id, &secrets.UpdateSecretRequest{Value: &rotated}))
if layer == "lifecycle" {
record, err := f.repo.GetKubernetesEnvironment(ctx, "environment-1")
require.NoError(t, err)
require.NoError(t, f.runtime.persistKubernetesEnvironmentSecrets(ctx, record, instance))
} else {
execution := &AgentExecution{ID: instance.InstanceID, metadata: cloneKubernetesMetadata(instance.Metadata)}
_, err := f.manager.persistRequiredKubernetesRuntimeSecrets(ctx, instance, execution)
require.NoError(t, err)
}
got, err := f.secretStore.Reveal(ctx, id)
require.NoError(t, err)
require.Equal(t, rotated, got)
})
}
}
func TestKubernetesTaskPodRetriesRotatedTokenPersistence(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
f.launch(t, 2)
id := getMetadataString(a.Metadata, MetadataKeyAuthTokenSecret)
store := &failingUpdateSecretStore{inMemorySecretStore: f.secretStore.(*inMemorySecretStore), failUpdateFor: id}
f.runtime.secretStore = store
f.control.reboot()
f.resources.mu.Lock()
f.resources.pod.Status.ContainerStatuses[0].RestartCount = 1
f.resources.mu.Unlock()
ctx := context.Background()
_, err := f.runtime.RefreshRemoteInstance(ctx, kubernetesRefreshInstance(a, taskPodRequest(1).Metadata))
require.ErrorContains(t, err, "injected secret update failure")
store.failUpdateFor = ""
f.restartBackend(t)
req := taskPodRequest(1)
req.PreviousExecutionID = a.InstanceID
req.Metadata = cloneKubernetesMetadata(a.Metadata)
_, err = f.runtime.CreateInstance(ctx, req)
require.NoError(t, err)
_, _, handshakes := f.control.snapshot()
require.Equal(t, 1, handshakes, "storage retry must preserve the single-use handshake result")
f.restartBackend(t)
f.launch(t, 3)
require.Len(t, f.resources.createdPods, 1)
}
// Reviewer-requested contract coverage: attaching a different session may overlap
// deletion of the stopped session's remote agent without deleting shared storage.
func TestKubernetesTaskPodConcurrentStopAndAttach(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
f.launch(t, 2)
entered, release := make(chan struct{}), make(chan struct{})
f.control.beforeDelete = func() { close(entered); <-release }
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stopped := make(chan error, 1)
go func() { stopped <- f.runtime.StopInstance(ctx, a, true) }()
select {
case <-entered:
case <-ctx.Done():
close(release)
t.Fatal("stop did not reach remote deletion")
}
_, attachErr := f.runtime.CreateInstance(ctx, taskPodRequest(3))
close(release)
require.NoError(t, attachErr)
require.NoError(t, <-stopped)
_, active, _ := f.control.snapshot()
require.NotContains(t, active, "instance-1")
require.Contains(t, active, "instance-2")
require.Contains(t, active, "instance-3")
require.Len(t, f.resources.createdPods, 1)
require.Empty(t, f.resources.deletedPods)
require.Empty(t, f.resources.deletedPVCs)
}
func TestKubernetesTaskPodStopOutlivesCanceledCaller(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
f.launch(t, 2)
f.resources.rejectCanceledGetContexts = true
f.secretStore.(*inMemorySecretStore).rejectCanceledCalls = true
ctx, cancel := context.WithCancel(context.Background())
cancel()
require.NoError(t, f.runtime.StopInstance(ctx, a, true))
_, active, _ := f.control.snapshot()
require.NotContains(t, active, "instance-1")
require.Contains(t, active, "instance-2")
require.Empty(t, f.resources.deletedPods)
}
@@ -0,0 +1,146 @@
package lifecycle
import (
"context"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"path"
"strings"
"testing"
)
func TestKubernetesTaskPodAdditionalSession(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
b := f.launch(t, 2)
require.Len(t, f.resources.createdPods, 1)
require.Len(t, f.resources.createdPVCs, 1)
require.Equal(t, a.Metadata[MetadataKeyKubernetesPodUID], b.Metadata[MetadataKeyKubernetesPodUID])
instances, active, handshakes := f.control.snapshot()
require.Len(t, instances, 2)
require.Len(t, active, 2)
require.Equal(t, 1, handshakes)
require.NotEqual(t, instances[0].ID, instances[1].ID)
require.Equal(t, "session-2", instances[1].SessionID)
require.NotEmpty(t, instances[0].Env["HOME"])
require.NotEqual(t, instances[0].Env["HOME"], instances[1].Env["HOME"])
require.Equal(t, "agent-profile-1", instances[0].Env[kubernetesEnvExecutionProfile])
require.Equal(t, "agent-profile-2", instances[1].Env[kubernetesEnvExecutionProfile])
require.Equal(t, "credential-1", instances[0].Env["SESSION_CREDENTIAL"])
require.Equal(t, "credential-2", instances[1].Env["SESSION_CREDENTIAL"])
require.Equal(t, "session-2", instances[1].Env[kubernetesEnvSessionID])
}
func TestKubernetesTaskPodStopPreservesSibling(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
b := f.launch(t, 2)
ctx := context.Background()
b.StopReason = StopReasonTaskDeleted
require.NoError(t, f.runtime.StopInstance(ctx, b, true))
require.Empty(t, f.resources.deletedPods)
require.Empty(t, f.resources.deletedPVCs)
require.NoError(t, a.Client.Health(ctx))
require.NoError(t, f.manager.deleteKubernetesRuntimeSecrets(ctx, b.Metadata))
_, err := f.secretStore.Reveal(ctx, getMetadataString(a.Metadata, MetadataKeyAuthTokenSecret))
require.NoError(t, err)
_, active, _ := f.control.snapshot()
require.Contains(t, active, "instance-1")
require.NotContains(t, active, "instance-2")
}
func TestKubernetesTaskPodTerminalCleanup(t *testing.T) {
f := newTaskPodFixture(t)
a := f.launch(t, 1)
b := f.launch(t, 2)
ctx := context.Background()
env, err := f.repo.GetTaskEnvironment(ctx, "environment-1")
require.NoError(t, err)
require.Error(t, f.manager.DestroyKubernetesEnvironment(ctx, env), "a live sibling blocks physical teardown")
require.Empty(t, f.resources.deletedPods)
require.NoError(t, f.runtime.StopInstance(ctx, a, true))
require.NoError(t, f.runtime.StopInstance(ctx, b, true))
require.NoError(t, f.manager.DestroyKubernetesEnvironment(ctx, env))
require.Len(t, f.resources.deletedPods, 1)
require.Len(t, f.resources.deletedPVCs, 1)
_, err = f.repo.GetKubernetesEnvironment(ctx, env.ID)
require.ErrorIs(t, err, models.ErrKubernetesEnvironmentNotFound)
require.NoError(t, f.manager.DestroyKubernetesEnvironment(ctx, env))
require.NoError(t, f.runtime.StopInstance(ctx, a, true), "session stop remains idempotent after task cleanup")
require.Len(t, f.resources.deletedPods, 1)
}
func TestKubernetesTaskPodControlEnvironmentIsTaskScoped(t *testing.T) {
req := validKubernetesCreateRequest()
req.Metadata[metadataKubernetesTaskOwned] = true
env := kubernetesRuntimeEnvironment(req)
require.Empty(t, env[kubernetesEnvSessionID])
require.Empty(t, env[kubernetesEnvExecutionProfile])
require.Equal(t, req.TaskEnvironmentID, env[kubernetesEnvInstanceID])
}
func TestKubernetesTaskPodBootstrapDropsAgentCredentials(t *testing.T) {
req := validKubernetesCreateRequest()
req.Metadata[metadataKubernetesTaskOwned] = true
req.Env = map[string]string{"SESSION_CREDENTIAL": "must-not-be-inherited"}
execs := &recordingKubernetesExec{}
runtime := newFakeKubernetesExecutor(t, &fakeKubernetesResources{}, execs, nil)
pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "kandev-agents", Name: "task-pod"}}
require.NoError(t, runtime.bootstrapPod(context.Background(), &kubernetesRuntimeClient{streams: kubeexecutor.NewStreamOperations(execs, nil)}, req, pod, kubeexecutor.ProfileConfig{MainContainer: "kandev-agent"}, "bootstrap-nonce", []byte("agentctl")))
var controlEnv string
for _, call := range execs.requests {
if strings.Contains(strings.Join(call.request.Command, " "), kubernetesAuthEnvPath) && len(call.stdin) > 0 {
controlEnv = string(call.stdin)
}
}
require.NotEmpty(t, controlEnv)
require.NotContains(t, controlEnv, "must-not-be-inherited")
require.Contains(t, controlEnv, "bootstrap-nonce")
}
func TestKubernetesTaskPodResourceIdentityIsEnvironmentScoped(t *testing.T) {
req := validKubernetesCreateRequest()
req.Metadata[metadataKubernetesTaskOwned] = true
identity, err := kubernetesIdentity(req)
require.NoError(t, err)
require.Equal(t, req.TaskEnvironmentID, identity.InstanceID)
labels, err := kubeexecutor.OwnershipLabels(identity)
require.NoError(t, err)
require.Equal(t, "task-v1", labels["kandev.ai/ownership-version"])
}
func TestKubernetesTaskPodCleanupAfterTaskDeletion(t *testing.T) {
f := newTaskPodFixture(t)
instance := f.launch(t, 1)
ctx := context.Background()
env, err := f.repo.GetTaskEnvironment(ctx, "environment-1")
require.NoError(t, err)
require.NoError(t, f.repo.DeleteTask(ctx, env.TaskID))
require.NoError(t, f.runtime.StopInstance(ctx, instance, true))
require.NoError(t, f.manager.DestroyKubernetesEnvironment(ctx, env))
require.Len(t, f.resources.deletedPods, 1)
require.Len(t, f.resources.deletedPVCs, 1)
}
func TestKubernetesTaskPodAuthUploadCreatesSessionHome(t *testing.T) {
req := validKubernetesCreateRequest()
req.Metadata[metadataKubernetesTaskOwned] = true
// The hardened uploader creates destination parents before preparation or
// agent startup uses HOME, including sessions with no credential files.
require.Equal(t, kubernetesSessionHome(req), path.Dir(kubernetesSessionAuthPath(req)))
}
func TestKubernetesTaskPodStopAfterExecutionIdentityChanges(t *testing.T) {
f := newTaskPodFixture(t)
instance := f.launch(t, 1)
rebound := *instance
rebound.InstanceID = "resumed-local-execution"
require.NoError(t, f.runtime.StopInstance(context.Background(), &rebound, true))
env, err := f.repo.GetTaskEnvironment(context.Background(), "environment-1")
require.NoError(t, err)
require.NoError(t, f.manager.DestroyKubernetesEnvironment(context.Background(), env), "stopping the same remote session must release its original local connection")
require.Len(t, f.resources.deletedPods, 1)
}
@@ -836,6 +836,7 @@ func (m *Manager) SetPreparerRegistry(registry *PreparerRegistry) {
// SetSecretStore sets the secret store for encrypting runtime auth tokens.
func (m *Manager) SetSecretStore(store secrets.SecretStore) {
m.secretStore = store
m.wireKubernetesEnvironmentStore()
}
// SetAgentProfileReader wires the reader the launch-prep SkillDeployer uses
@@ -1474,6 +1474,19 @@ func (m *Manager) persistRuntimeSecretResult(
if instance == nil || execution == nil || m.secretStore == nil {
return false, errors.New("runtime secret persistence is unavailable")
}
if getMetadataBool(instance.Metadata, metadataKubernetesTaskOwned) {
// The shared control connection owns token rotation. A session snapshot
// must not overwrite a newer token published by a sibling refresh.
secretID := getMetadataString(instance.Metadata, metadataKey)
if secretID == "" {
return false, errors.New("kubernetes environment secret reference is missing")
}
if _, err := m.secretStore.Reveal(ctx, secretID); err != nil {
return false, err
}
execution.setMetadataValue(metadataKey, secretID)
return false, nil
}
secretID := execution.metadataString(metadataKey)
if secretID == "" {
resourceInstanceID := execution.metadataString(MetadataKeyKubernetesResourceInstanceID)
@@ -1535,6 +1548,9 @@ func (m *Manager) deleteCreatedRuntimeSecrets(
}
func (m *Manager) deleteKubernetesRuntimeSecrets(ctx context.Context, metadata map[string]interface{}) error {
if getMetadataBool(metadata, metadataKubernetesTaskOwned) {
return nil
}
secretIDs := kubernetesRuntimeSecretIDs(metadata)
if secretIDs[0] == "" && secretIDs[1] == "" {
return nil
@@ -0,0 +1,136 @@
package lifecycle
import (
"context"
"errors"
"fmt"
"github.com/google/uuid"
"github.com/kandev/kandev/internal/agent/executor"
kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
)
// DestroyKubernetesEnvironment is called only by task resource teardown after
// session termination and the task cleanup barrier have excluded new launches.
func (m *Manager) DestroyKubernetesEnvironment(ctx context.Context, env *models.TaskEnvironment) error {
if env == nil {
return nil
}
if m.executorRegistry == nil {
return errors.New("kubernetes runtime registry is unavailable")
}
backend, err := m.executorRegistry.GetBackend(executor.NameKubernetes)
if err != nil {
return err
}
runtime, ok := backend.(*KubernetesExecutor)
if !ok {
return errors.New("kubernetes runtime is unavailable")
}
return runtime.destroyTaskEnvironment(ctx, env)
}
func (r *KubernetesExecutor) destroyTaskEnvironment(ctx context.Context, env *models.TaskEnvironment) (returnedErr error) {
if r.environmentStore == nil {
return errors.New("kubernetes environment inventory is unavailable")
}
unlock := r.lockInstance("environment:" + env.ID)
defer unlock()
if _, err := r.environmentStore.GetKubernetesEnvironment(ctx, env.ID); errors.Is(err, models.ErrKubernetesEnvironmentNotFound) {
return nil
} else if err != nil {
return err
}
if err := r.ensureKubernetesEnvironmentStopped(ctx, env); err != nil {
return err
}
record, err := r.environmentStore.ClaimKubernetesEnvironmentCleanup(ctx, env.ID, env.TaskID, env.OwnershipGeneration, uuid.NewString())
if err != nil {
return err
}
defer func() {
if returnedErr != nil {
persistCtx, cancel := kubernetesDurableContext(ctx)
defer cancel()
returnedErr = errors.Join(returnedErr, r.environmentStore.SaveKubernetesEnvironment(persistCtx, record, true))
}
}()
if err := r.deleteTaskKubernetesResources(ctx, record); err != nil {
return err
}
for _, id := range []string{record.ControlSecretID, record.BootstrapSecretID, kubernetesControlRecoverySecretID(record.ControlSecretID)} {
if id == "" {
continue
}
if r.secretStore == nil {
return errors.New("kubernetes environment secret store is unavailable")
}
if err := r.secretStore.Delete(ctx, id); err != nil && !errors.Is(err, secrets.ErrNotFound) {
return err
}
}
return r.environmentStore.DeleteKubernetesEnvironment(ctx, record)
}
func (r *KubernetesExecutor) ensureKubernetesEnvironmentStopped(ctx context.Context, env *models.TaskEnvironment) error {
r.mu.Lock()
for _, session := range r.sessions {
if session.request != nil && session.request.TaskEnvironmentID == env.ID {
r.mu.Unlock()
return models.ErrWorkspaceReuseUnsafe
}
}
r.mu.Unlock()
rows, err := r.environmentStore.ListExecutorsRunningByTaskID(ctx, env.TaskID)
if err != nil {
return err
}
for _, row := range rows {
if row == nil || getMetadataString(row.Metadata, MetadataKeyKubernetesResourceEnvironmentID) != env.ID {
continue
}
switch row.Status {
case models.ExecutorRunningStatusStopped, models.ExecutorRunningStatusFailed, models.ExecutorRunningStatusComplete:
default:
return models.ErrWorkspaceReuseUnsafe
}
}
return nil
}
func (r *KubernetesExecutor) deleteTaskKubernetesResources(ctx context.Context, record *models.KubernetesEnvironment) error {
if len(record.Metadata) == 0 {
return nil
}
current, err := r.environmentStore.GetExecutor(ctx, getMetadataString(record.Metadata, MetadataKeyKubernetesResourceExecutorID))
if err != nil {
return err
}
if current == nil || current.Type != models.ExecutorTypeKubernetes {
return models.ErrWorkspaceReuseUnsafe
}
config, err := kubeexecutor.ParseExecutorConfig(current.Config)
if err != nil {
return err
}
metadata := overlayCurrentKubernetesConnectionMetadata(record.Metadata, current.Config, config)
instance := &ExecutorInstance{TaskID: record.TaskID, SessionID: getMetadataString(metadata, MetadataKeyKubernetesResourceSessionID), Metadata: metadata, StopReason: StopReasonTaskDeleted}
runtime, err := r.kubernetesCleanupRuntime(instance, nil)
if err != nil {
return err
}
recorded, identity, err := kubernetesCleanupInventory(instance)
if err != nil {
return err
}
if err := verifyKubernetesCleanupTargets(ctx, runtime.resources, recorded, identity); err != nil {
return err
}
r.runKubernetesTerminalCleanupScript(ctx, runtime.streams, instance, recorded)
if err := deleteKubernetesResources(ctx, runtime.resources, recorded, identity); err != nil {
return fmt.Errorf("delete task Kubernetes resources: %w", err)
}
return nil
}
@@ -115,7 +115,7 @@ func resolvePersistedKubernetesCleanupMetadata(
if err != nil {
return nil, err
}
req := &ExecutorCreateRequest{TaskID: row.TaskID, SessionID: row.SessionID, Metadata: metadata}
req := &ExecutorCreateRequest{TaskID: row.TaskID, SessionID: row.SessionID, TaskEnvironmentID: getMetadataString(metadata, MetadataKeyKubernetesResourceEnvironmentID), Metadata: metadata}
if _, _, err := kubernetesRecordedCleanupInventory(req, currentConfig, true); err != nil {
return nil, fmt.Errorf("validate persisted Kubernetes runtime inventory: %w", err)
}
@@ -53,6 +53,7 @@ type ExecutorRunningWriter interface {
// Optional only for tests that don't exercise the persistence path.
func (m *Manager) SetExecutorRunningWriter(w ExecutorRunningWriter) {
m.runningWriter = w
m.wireKubernetesEnvironmentStore()
}
// buildRunningFromExecution maps an in-memory execution into the persistence
@@ -1,6 +1,7 @@
package api
import (
"errors"
"net/http/httptest"
"testing"
"time"
@@ -261,10 +262,9 @@ func TestNonRenewingControlOperationsDoNotRenewOwnership(t *testing.T) {
}},
{"instance operation", func(t *testing.T) {
_, err := client.GetInstance(t.Context(), "no-such-instance")
// The not-found wording is the handler's 404, distinct from the
// "failed to get instance" a transport error produces, so this
// asserts the request was served rather than merely refused.
if err == nil || err.Error() != `instance "no-such-instance" not found` {
// Only the handler's 404 maps to this sentinel; a transport failure
// must not make the ownership assertion pass vacuously.
if !errors.Is(err, agentctl.ErrInstanceNotFound) {
t.Fatalf("GetInstance error = %v, want the handler's not-found response", err)
}
}},
@@ -0,0 +1,15 @@
package backendapp
import (
"context"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
"testing"
)
func TestKubernetesEnvironmentDestroyerWiring(t *testing.T) {
_, ok := interface{}(&environmentDestroyerAdapter{}).(interface {
DestroyKubernetesEnvironment(context.Context, *models.TaskEnvironment) error
})
require.True(t, ok, "task cleanup adapter must expose shared Kubernetes teardown")
}
@@ -83,6 +83,9 @@ func provideRepositories(ctx context.Context, cfg *config.Config, log *logger.Lo
if err := recordRequiredStore(ctx, tracker, "schema-meta", nil); err != nil {
return nil, nil, nil, err
}
if taskRepoErr == nil {
taskRepoErr = taskRepoImpl.RecoverInterruptedKubernetesOperations(ctx)
}
if err := recordRequiredStore(ctx, tracker, "task", taskRepoErr); err != nil {
return nil, nil, nil, fmt.Errorf("task store: %w", err)
}
@@ -109,6 +109,10 @@ type environmentDestroyerAdapter struct {
worktrees *worktree.Manager
}
func (a *environmentDestroyerAdapter) DestroyKubernetesEnvironment(ctx context.Context, env *models.TaskEnvironment) error {
return a.lifecycle.DestroyKubernetesEnvironment(ctx, env)
}
func (a *environmentDestroyerAdapter) DestroyContainer(ctx context.Context, containerID string) error {
return a.lifecycle.DestroyContainer(ctx, containerID)
}
+12 -6
View File
@@ -8,7 +8,6 @@ import (
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeclient "k8s.io/client-go/kubernetes"
agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes"
@@ -100,11 +99,12 @@ func (h *Handler) listSessions(
return nil, err
}
rows := make([]SessionRow, 0, len(runs))
cache := newSessionStatusCache(client)
for _, run := range runs {
if !filter.matches(run) {
continue
}
row, visible, rowErr := h.sessionRow(ctx, client, executorID, run)
row, visible, rowErr := h.sessionRow(ctx, cache, executorID, run)
if rowErr != nil {
return nil, rowErr
}
@@ -112,7 +112,7 @@ func (h *Handler) listSessions(
rows = append(rows, row)
}
}
return rows, nil
return h.appendRetainedTaskPods(ctx, client, executorID, filter, rows)
}
func (f SessionFilter) matches(run *models.ExecutorRunning) bool {
@@ -162,7 +162,7 @@ func (h *Handler) sessionStatusSource(
func (h *Handler) sessionRow(
ctx context.Context,
client kubeclient.Interface,
cache *sessionStatusCache,
executorID string,
run *models.ExecutorRunning,
) (SessionRow, bool, error) {
@@ -188,14 +188,19 @@ func (h *Handler) sessionRow(
}
return SessionRow{}, false, err
}
run, inventoryErr := h.canonicalTaskPodInventory(ctx, run, session, cache)
row := newInventorySessionRow(run)
row.SessionState = projectedTaskSessionState(session.State)
if inventoryErr != nil {
row.FailureReason = "Kubernetes task inventory is unavailable"
return row, true, nil
}
if inventoryFailure := validateSessionInventory(run, executorID, row); inventoryFailure != "" {
row.FailureReason = inventoryFailure
return row, true, nil
}
namespace := metadataString(run.Metadata, metadataNamespace)
pod, err := client.CoreV1().Pods(namespace).Get(ctx, row.PodName, metav1.GetOptions{})
pod, err := cache.pod(ctx, namespace, row.PodName)
if err != nil {
row.FailureReason = podLookupFailure(err)
if apierrors.IsNotFound(err) {
@@ -233,7 +238,7 @@ func validateSessionInventory(
identity, validIdentity := recordedResourceIdentity(run.Metadata)
if !validIdentity || run.ID != run.SessionID || run.ExecutorID != executorID ||
identity.ExecutorID != executorID || identity.TaskID != run.TaskID ||
identity.SessionID != run.SessionID || metadataString(run.Metadata, metadataNamespace) == "" ||
(!isTaskPodInventory(run.Metadata) && identity.SessionID != run.SessionID) || metadataString(run.Metadata, metadataNamespace) == "" ||
row.PodName == "" || metadataString(run.Metadata, metadataPodUID) == "" ||
metadataString(run.Metadata, metadataMainContainer) == "" || row.WorkspaceKind == "" {
return "Kubernetes runtime inventory is incomplete"
@@ -274,6 +279,7 @@ func matchesSessionIdentity(pod *corev1.Pod, run *models.ExecutorRunning) bool {
func recordedResourceIdentity(metadata map[string]interface{}) (agentkubernetes.ResourceIdentity, bool) {
identity := agentkubernetes.ResourceIdentity{
TaskOwned: metadataString(metadata, agentkubernetes.MetadataKeyOwnershipVersion) == agentkubernetes.TaskOwnershipVersion,
ExecutorID: metadataString(metadata, metadataResourceExecutor),
ProfileID: metadataString(metadata, metadataResourceProfile),
InstanceID: metadataString(metadata, metadataResourceInstance),
@@ -0,0 +1,39 @@
package kubernetes
import (
"context"
"github.com/kandev/kandev/internal/task/models"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeclient "k8s.io/client-go/kubernetes"
)
type environmentStatusResult struct {
record *models.KubernetesEnvironment
err error
}
type podStatusResult struct {
pod *corev1.Pod
err error
}
// A request projects one observed physical state onto every authorized session.
// Identity and task authorization remain checked separately for every row.
type sessionStatusCache struct {
client kubeclient.Interface
environments map[string]environmentStatusResult
pods map[string]podStatusResult
}
func newSessionStatusCache(client kubeclient.Interface) *sessionStatusCache {
return &sessionStatusCache{client: client, environments: make(map[string]environmentStatusResult), pods: make(map[string]podStatusResult)}
}
func (c *sessionStatusCache) pod(ctx context.Context, namespace, name string) (*corev1.Pod, error) {
key := namespace + "/" + name
result, exists := c.pods[key]
if !exists {
result.pod, result.err = c.client.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{})
c.pods[key] = result
}
return result.pod, result.err
}
@@ -66,7 +66,7 @@ func TestKubernetesSessionRetentionProjectionTable(t *testing.T) {
clientset := kubernetesfake.NewSimpleClientset(pod)
handler := retentionProjectionHandler(repo, clientset)
row, visible, err := handler.sessionRow(context.Background(), clientset, "executor-1", run)
row, visible, err := handler.sessionRow(context.Background(), newSessionStatusCache(clientset), "executor-1", run)
require.NoError(t, err)
require.True(t, visible)
@@ -120,7 +120,7 @@ func TestKubernetesRetentionProjectionOmitsUnverifiedRequests(t *testing.T) {
}
handler := retentionProjectionHandler(repo, clientset)
row, visible, err := handler.sessionRow(context.Background(), clientset, "executor-1", run)
row, visible, err := handler.sessionRow(context.Background(), newSessionStatusCache(clientset), "executor-1", run)
require.NoError(t, err)
require.True(t, visible)
@@ -153,7 +153,7 @@ func TestKubernetesMainContainerRequestsPreserveExplicitZero(t *testing.T) {
clientset := kubernetesfake.NewSimpleClientset(pod)
handler := retentionProjectionHandler(repo, clientset)
row, visible, err := handler.sessionRow(context.Background(), clientset, "executor-1", run)
row, visible, err := handler.sessionRow(context.Background(), newSessionStatusCache(clientset), "executor-1", run)
require.NoError(t, err)
require.True(t, visible)
@@ -0,0 +1,104 @@
package kubernetes
import (
"context"
"errors"
"github.com/kandev/kandev/internal/task/repository/repoerrors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeclient "k8s.io/client-go/kubernetes"
agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/task/models"
)
func isTaskPodInventory(metadata map[string]interface{}) bool {
shared, _ := metadata[agentkubernetes.MetadataKeyTaskOwned].(bool)
return shared
}
// Resolve physical inventory only after authorizing the session's owning task.
func (h *Handler) canonicalTaskPodInventory(ctx context.Context, run *models.ExecutorRunning, session *models.TaskSession, cache *sessionStatusCache) (*models.ExecutorRunning, error) {
if !isTaskPodInventory(run.Metadata) {
return run, nil
}
store, ok := h.repo.(interface {
GetKubernetesEnvironment(context.Context, string) (*models.KubernetesEnvironment, error)
})
if !ok {
return run, errors.New("kubernetes environment store unavailable")
}
cached, exists := cache.environments[session.TaskEnvironmentID]
if !exists {
cached.record, cached.err = store.GetKubernetesEnvironment(ctx, session.TaskEnvironmentID)
cache.environments[session.TaskEnvironmentID] = cached
}
record, err := cached.record, cached.err
if err != nil {
return run, err
}
if record == nil || record.TaskID != run.TaskID || record.EnvironmentID != session.TaskEnvironmentID || metadataString(record.Metadata, metadataResourceEnv) != record.EnvironmentID {
return run, models.ErrWorkspaceReuseUnsafe
}
copyRun := *run
copyRun.Metadata = record.Metadata
return &copyRun, nil
}
// Retained physical resources stay visible when their last session is removed.
func (h *Handler) appendRetainedTaskPods(ctx context.Context, client kubeclient.Interface, executorID string, filter SessionFilter, rows []SessionRow) ([]SessionRow, error) {
if filter.SessionID != "" {
return rows, nil
}
store, ok := h.repo.(interface {
ListKubernetesEnvironments(context.Context) ([]*models.KubernetesEnvironment, error)
})
if !ok {
return rows, nil
}
records, err := store.ListKubernetesEnvironments(ctx)
if err != nil {
return nil, err
}
visibleTasks := make(map[string]bool, len(rows))
for _, row := range rows {
visibleTasks[row.TaskID] = true
}
for _, record := range records {
if record == nil || visibleTasks[record.TaskID] || (filter.TaskID != "" && filter.TaskID != record.TaskID) || metadataString(record.Metadata, metadataResourceExecutor) != executorID {
continue
}
if err := h.access.AuthorizeTaskAccess(ctx, record.TaskID); err != nil {
if errors.Is(err, repoerrors.ErrTaskNotFound) {
continue
}
return nil, err
}
row := retainedTaskPodRow(ctx, client, executorID, record)
rows = append(rows, row)
}
return rows, nil
}
func retainedTaskPodRow(ctx context.Context, client kubeclient.Interface, executorID string, record *models.KubernetesEnvironment) SessionRow {
run := &models.ExecutorRunning{TaskID: record.TaskID, ExecutorID: executorID, Metadata: record.Metadata}
row := newInventorySessionRow(run)
if !isTaskPodInventory(record.Metadata) || metadataString(record.Metadata, metadataResourceEnv) != record.EnvironmentID {
row.FailureReason = "Kubernetes task inventory is incomplete"
return row
}
if failure := validateSessionInventory(run, executorID, row); failure != "" {
row.FailureReason = failure
return row
}
pod, err := client.CoreV1().Pods(metadataString(record.Metadata, metadataNamespace)).Get(ctx, row.PodName, metav1.GetOptions{})
if err != nil {
row.FailureReason = podLookupFailure(err)
return row
}
if !matchesSessionIdentity(pod, run) {
row.FailureReason = "Pod identity does not match runtime inventory"
return row
}
populatePodStatus(&row, pod, metadataString(record.Metadata, metadataMainContainer), models.TaskSessionStateIdle)
return row
}
@@ -0,0 +1,89 @@
package kubernetes
import (
"context"
"testing"
"time"
agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
kubernetesfake "k8s.io/client-go/kubernetes/fake"
)
type taskPodStatusRepository struct {
*fakeResourceRepository
environment *models.KubernetesEnvironment
reads int
}
func (r *taskPodStatusRepository) GetKubernetesEnvironment(context.Context, string) (*models.KubernetesEnvironment, error) {
r.reads++
return r.environment, nil
}
func TestSharedTaskPodStatusUsesCanonicalInventory(t *testing.T) {
run := kubernetesRunningRow("environment-1", "session-2", "task-1", "pod-1", "old-uid", time.Now())
run.Metadata[agentkubernetes.MetadataKeyTaskOwned] = true
run.Metadata[agentkubernetes.MetadataKeyOwnershipVersion] = agentkubernetes.TaskOwnershipVersion
run.Metadata[metadataResourceSession] = "session-1"
metadata := make(map[string]interface{}, len(run.Metadata))
for k, v := range run.Metadata {
metadata[k] = v
}
metadata[metadataPodUID] = "new-uid"
session := kubernetesTaskSession("session-2", "task-1", "executor-1", "profile-1")
repo := &taskPodStatusRepository{fakeResourceRepository: &fakeResourceRepository{
executor: &models.Executor{ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig()},
runs: []*models.ExecutorRunning{run}, sessions: map[string]*models.TaskSession{session.ID: session},
}, environment: &models.KubernetesEnvironment{EnvironmentID: "environment-1", TaskID: "task-1", Metadata: metadata}}
pod := kubernetesOwnedPodForIdentity("pod-1", "new-uid", agentkubernetes.ResourceIdentity{ExecutorID: "executor-1", ProfileID: "profile-1", InstanceID: "environment-1", TaskID: "task-1", SessionID: "session-1", EnvironmentID: "environment-1", TaskOwned: true})
clientset := kubernetesfake.NewSimpleClientset(pod)
handler := NewHandler(repo, &fakeAccessChecker{}, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) {
return &agentkubernetes.Client{Clientset: clientset}, nil
})
sibling := *run
sibling.ID, sibling.SessionID = "session-3", "session-3"
repo.runs = append(repo.runs, &sibling)
repo.sessions["session-3"] = kubernetesTaskSession("session-3", "task-1", "executor-1", "profile-1")
rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{})
require.NoError(t, err)
require.Len(t, rows, 2)
require.Equal(t, 1, repo.reads)
require.Len(t, clientset.Actions(), 1, "probe shared Pod once per request")
require.Empty(t, rows[0].FailureReason)
require.Equal(t, "session-2", rows[0].SessionID)
require.Equal(t, "Running", rows[0].PodPhase)
repo.environment.TaskID = "foreign-task"
rows, err = handler.listSessions(context.Background(), "executor-1", SessionFilter{})
require.NoError(t, err)
require.NotEmpty(t, rows[0].FailureReason)
}
func (r *taskPodStatusRepository) ListKubernetesEnvironments(context.Context) ([]*models.KubernetesEnvironment, error) {
return []*models.KubernetesEnvironment{r.environment}, nil
}
func TestSharedTaskPodStatusRetainedWithoutSessions(t *testing.T) {
run := kubernetesRunningRow("environment-1", "session-1", "task-1", "pod-1", "pod-uid", time.Now())
run.Metadata[agentkubernetes.MetadataKeyTaskOwned] = true
run.Metadata[agentkubernetes.MetadataKeyOwnershipVersion] = agentkubernetes.TaskOwnershipVersion
repo := &taskPodStatusRepository{fakeResourceRepository: &fakeResourceRepository{
executor: &models.Executor{ID: "executor-1", Type: models.ExecutorTypeKubernetes, Config: validHandlerExecutorConfig()},
}, environment: &models.KubernetesEnvironment{EnvironmentID: "environment-1", TaskID: "task-1", Metadata: run.Metadata}}
identity, _ := recordedResourceIdentity(run.Metadata)
pod := kubernetesOwnedPodForIdentity("pod-1", "pod-uid", identity)
clientset := kubernetesfake.NewSimpleClientset(pod)
access := &fakeAccessChecker{}
handler := NewHandler(repo, access, func(agentkubernetes.ExecutorConfig) (*agentkubernetes.Client, error) {
return &agentkubernetes.Client{Clientset: clientset}, nil
})
rows, err := handler.listSessions(context.Background(), "executor-1", SessionFilter{})
require.NoError(t, err)
require.Len(t, rows, 1, "retained task pod must remain visible after deleting every session")
require.Empty(t, rows[0].SessionID)
require.Equal(t, "task-1", rows[0].TaskID)
require.Equal(t, "retained", rows[0].RetentionState)
require.Equal(t, "Running", rows[0].PodPhase)
require.NotEmpty(t, access.calls)
}
@@ -211,8 +211,7 @@ func TestLaunchPreparedSession_ObservesWorkingSiblingOnAgentStart(t *testing.T)
}
// TestResumeSession_ObservesWorkingSiblingOnAgentStart pins the wiring
// half of AC-004.1 for the resume seam. Ordering is enforced by
// production code structure, not a synchronized assertion here.
// half of AC-004.1 for resume admission and asynchronous process startup.
func TestResumeSession_ObservesWorkingSiblingOnAgentStart(t *testing.T) {
repo := newMockRepository()
setupLiveResumeTestFixture(repo)
@@ -226,25 +225,42 @@ func TestResumeSession_ObservesWorkingSiblingOnAgentStart(t *testing.T) {
if err != nil {
t.Fatalf("NewFromZap: %v", err)
}
observedAtStart := make(chan int, 1)
finished := make(chan struct{})
agentMgr := &mockAgentManager{
launchAgentFunc: func(ctx context.Context, req *LaunchAgentRequest) (*LaunchAgentResponse, error) {
return &LaunchAgentResponse{AgentExecutionID: "exec-new"}, nil
},
startAgentProcessFunc: func(context.Context, string) error {
observedAtStart <- logs.FilterLevelExact(zapcore.WarnLevel).Len()
return nil
},
}
exec := NewExecutor(agentMgr, repo, log, ExecutorConfig{ShellPrefs: &mockShellPrefs{}})
exec.SetCapabilities(&mockCapabilities{})
exec.SetOnAgentProcessStarted(func(context.Context, string, string, string) { close(finished) })
before := counterValue(sessionCoresidencyAdmittedTotalVar, sessionCoresidencySiteResume)
if _, err := exec.ResumeSession(context.Background(), repo.sessions["sess-1"], true); err != nil {
t.Fatalf("ResumeSession: %v", err)
}
if after := counterValue(sessionCoresidencyAdmittedTotalVar, sessionCoresidencySiteResume); after != before+1 {
t.Fatalf("admitted[resume] counter = %d, want %d", after, before+1)
select {
case <-finished:
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for resumed agent startup")
}
if observed := <-observedAtStart; observed != 2 {
t.Fatalf("observations before process start = %d, want launch admission and process startup", observed)
}
// Other tests can still emit asynchronous observations into the global counter.
if after := counterValue(sessionCoresidencyAdmittedTotalVar, sessionCoresidencySiteResume); after < before+2 {
t.Fatalf("admitted[resume] counter = %d, want at least %d", after, before+2)
}
warnings := logs.FilterLevelExact(zapcore.WarnLevel).All()
if len(warnings) != 1 {
t.Fatalf("warning entries = %d, want 1; all=%v", len(warnings), logs.All())
if len(warnings) != 2 {
t.Fatalf("warning entries = %d, want 2; all=%v", len(warnings), logs.All())
}
fields := warnings[0].ContextMap()
if fields["site"] != sessionCoresidencySiteResume {
@@ -63,7 +63,7 @@ type Descriptor struct {
// of the database contract and must be initialized before readiness.
var catalog = []Descriptor{
{ID: storeIDSchemaMeta, OwnerPackage: "internal/persistence", RequiredTables: []string{"kandev_meta"}, Sweep: startup.StepStoresRepositories},
{ID: storeIDTask, OwnerPackage: "internal/task/repository/sqlite", RequiredTables: []string{"workspaces", "tasks", "task_workflow_session_bindings"}, DependsOn: []string{storeIDSchemaMeta}, Capabilities: []Capability{CapabilityBoolean, CapabilityTimestamp, CapabilityConflict, CapabilityTransaction}, Sweep: startup.StepStoresRepositories},
{ID: storeIDTask, OwnerPackage: "internal/task/repository/sqlite", RequiredTables: []string{"workspaces", "tasks", "task_workflow_session_bindings", "task_environment_kubernetes"}, DependsOn: []string{storeIDSchemaMeta}, Capabilities: []Capability{CapabilityBoolean, CapabilityTimestamp, CapabilityConflict, CapabilityTransaction}, Sweep: startup.StepStoresRepositories},
{ID: storeIDWorkflow, OwnerPackage: "internal/workflow/repository", RequiredTables: []string{"workflow_templates", "workflow_steps"}, DependsOn: []string{storeIDTask}, Capabilities: []Capability{CapabilityTimestamp, CapabilityConflict, CapabilityTransaction}, Sweep: startup.StepStoresRepositories},
{ID: "analytics", OwnerPackage: "internal/analytics/repository", RequiredTables: []string{"tasks"}, DependsOn: []string{storeIDTask, storeIDWorkflow}, Capabilities: []Capability{CapabilityTimestamp}, Sweep: startup.StepStoresRepositories},
{ID: storeIDAgentSettings, OwnerPackage: "internal/agent/settings/store", RequiredTables: []string{"agents", "agent_profiles"}, DependsOn: []string{storeIDSchemaMeta}, Capabilities: []Capability{CapabilityBoolean, CapabilityTimestamp, CapabilityConflict}, Sweep: startup.StepStoresRepositories},
@@ -0,0 +1,22 @@
package models
import "errors"
var (
ErrKubernetesEnvironmentNotFound = errors.New("kubernetes environment inventory not found")
ErrKubernetesEnvironmentConflict = errors.New("kubernetes environment ownership or operation changed")
)
// KubernetesEnvironment owns physical resource inventory independently of a session.
// Metadata contains the admitted resource identity and workload snapshot. Secrets
// are encrypted store references and must never be serialized in public responses.
type KubernetesEnvironment struct {
EnvironmentID string
TaskID string
OwnershipGeneration int64
Revision int64
OperationID string
Metadata map[string]interface{}
ControlSecretID string
BootstrapSecretID string
}
@@ -482,6 +482,10 @@ func (r *Repository) runMigrations(ctx context.Context) error {
return fmt.Errorf("required task migration: %w", err)
}
if _, err := r.db.ExecContext(ctx, kubernetesEnvironmentSchemaDDL); err != nil {
return fmt.Errorf("create Kubernetes environment inventory: %w", err)
}
return nil
}
@@ -0,0 +1,220 @@
package sqlite
import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"github.com/jmoiron/sqlx"
"github.com/kandev/kandev/internal/db/dialect"
"github.com/kandev/kandev/internal/task/models"
)
const kubernetesEnvironmentSchemaDDL = `CREATE TABLE IF NOT EXISTS task_environment_kubernetes (
environment_id TEXT PRIMARY KEY,
task_id TEXT NOT NULL UNIQUE,
ownership_generation BIGINT NOT NULL,
revision BIGINT NOT NULL DEFAULT 1,
operation_id TEXT NOT NULL DEFAULT '',
metadata TEXT NOT NULL DEFAULT '{}',
control_secret_id TEXT NOT NULL DEFAULT '',
bootstrap_secret_id TEXT NOT NULL DEFAULT ''
)`
// ClaimKubernetesEnvironment serializes physical resource operations while leaving
// the database transaction closed during remote Kubernetes API calls.
func (r *Repository) ClaimKubernetesEnvironment(ctx context.Context, environmentID, taskID string, generation int64, operationID string) (*models.KubernetesEnvironment, error) {
if environmentID == "" || taskID == "" || generation < 1 || operationID == "" {
return nil, models.ErrKubernetesEnvironmentConflict
}
tx, err := r.db.BeginTxx(ctx, nil)
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback() }()
if err := r.validateKubernetesEnvironmentOwner(ctx, tx, environmentID, taskID, generation); err != nil {
return nil, err
}
_, err = tx.ExecContext(ctx, r.db.Rebind(`INSERT INTO task_environment_kubernetes (environment_id, task_id, ownership_generation)
VALUES (?, ?, ?) ON CONFLICT DO NOTHING`), environmentID, taskID, generation)
if err != nil {
return nil, err
}
record, err := scanKubernetesEnvironment(tx.QueryRowContext(ctx, r.db.Rebind(kubernetesEnvironmentSelect+` WHERE environment_id = ?`), environmentID))
if err != nil {
return nil, err
}
if record.TaskID != taskID || record.OwnershipGeneration != generation || (record.OperationID != "" && record.OperationID != operationID) {
return nil, models.ErrKubernetesEnvironmentConflict
}
if record.OperationID == "" {
result, err := tx.ExecContext(ctx, r.db.Rebind(`UPDATE task_environment_kubernetes SET operation_id = ?, revision = revision + 1 WHERE environment_id = ? AND revision = ? AND operation_id = ''`), operationID, environmentID, record.Revision)
if err := requireKubernetesEnvironmentWrite(result, err); err != nil {
return nil, err
}
record.OperationID = operationID
record.Revision++
}
return record, tx.Commit()
}
const kubernetesEnvironmentSelect = `SELECT environment_id, task_id, ownership_generation, revision, operation_id, metadata, control_secret_id, bootstrap_secret_id FROM task_environment_kubernetes`
type kubernetesInventoryScanner interface{ Scan(...interface{}) error }
func scanKubernetesEnvironment(row kubernetesInventoryScanner) (*models.KubernetesEnvironment, error) {
record := &models.KubernetesEnvironment{}
var raw string
err := row.Scan(&record.EnvironmentID, &record.TaskID, &record.OwnershipGeneration, &record.Revision, &record.OperationID, &raw, &record.ControlSecretID, &record.BootstrapSecretID)
if errors.Is(err, sql.ErrNoRows) {
return nil, models.ErrKubernetesEnvironmentNotFound
}
if err != nil {
return nil, err
}
if err := json.Unmarshal([]byte(raw), &record.Metadata); err != nil {
return nil, fmt.Errorf("decode Kubernetes environment inventory: %w", err)
}
return record, nil
}
func (r *Repository) GetKubernetesEnvironment(ctx context.Context, environmentID string) (*models.KubernetesEnvironment, error) {
return scanKubernetesEnvironment(r.db.QueryRowContext(ctx, r.db.Rebind(kubernetesEnvironmentSelect+` WHERE environment_id = ?`), environmentID))
}
// SaveKubernetesEnvironment checkpoints only the current operation and revision.
// Releasing its operation allows the next attachment, recovery, or cleanup.
func (r *Repository) SaveKubernetesEnvironment(ctx context.Context, record *models.KubernetesEnvironment, release bool) error {
if record == nil || record.OperationID == "" {
return models.ErrKubernetesEnvironmentConflict
}
raw, err := json.Marshal(record.Metadata)
if err != nil {
return err
}
operation := record.OperationID
if release {
operation = ""
}
result, err := r.db.ExecContext(ctx, r.db.Rebind(`UPDATE task_environment_kubernetes SET metadata = ?, control_secret_id = ?, bootstrap_secret_id = ?, operation_id = ?, revision = revision + 1
WHERE environment_id = ? AND task_id = ? AND ownership_generation = ? AND revision = ? AND operation_id = ? AND (`+kubernetesEnvironmentWritable+`)`), string(raw), record.ControlSecretID, record.BootstrapSecretID, operation, record.EnvironmentID, record.TaskID, record.OwnershipGeneration, record.Revision, record.OperationID)
if err := requireKubernetesEnvironmentWrite(result, err); err != nil {
return err
}
record.Revision++
record.OperationID = operation
return nil
}
func requireKubernetesEnvironmentWrite(result sql.Result, err error) error {
if err != nil {
return err
}
n, err := result.RowsAffected()
if err != nil {
return err
}
if n != 1 {
return models.ErrKubernetesEnvironmentConflict
}
return nil
}
// DeleteKubernetesEnvironment releases inventory after exact remote cleanup.
func (r *Repository) DeleteKubernetesEnvironment(ctx context.Context, record *models.KubernetesEnvironment) error {
if record == nil || record.OperationID == "" {
return models.ErrKubernetesEnvironmentConflict
}
result, err := r.db.ExecContext(ctx, r.db.Rebind(`DELETE FROM task_environment_kubernetes
WHERE environment_id = ? AND task_id = ? AND ownership_generation = ? AND revision = ? AND operation_id = ?
AND (`+kubernetesEnvironmentWritable+`)`),
record.EnvironmentID, record.TaskID, record.OwnershipGeneration, record.Revision, record.OperationID)
return requireKubernetesEnvironmentWrite(result, err)
}
// RecoverInterruptedKubernetesOperations fences claims from a terminated backend.
// Call only at startup under exclusive runtime-state ownership, before launching
// workers. Inventory remains intact for exact-resource reconciliation or cleanup.
func (r *Repository) RecoverInterruptedKubernetesOperations(ctx context.Context) error {
_, err := r.db.ExecContext(ctx, `UPDATE task_environment_kubernetes SET operation_id = '', revision = revision + 1 WHERE operation_id <> ''`)
return err
}
func (r *Repository) ListKubernetesEnvironments(ctx context.Context) ([]*models.KubernetesEnvironment, error) {
rows, err := r.db.QueryContext(ctx, kubernetesEnvironmentSelect+` ORDER BY environment_id`)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
records := make([]*models.KubernetesEnvironment, 0)
for rows.Next() {
record, err := scanKubernetesEnvironment(rows)
if err != nil {
return nil, err
}
records = append(records, record)
}
return records, rows.Err()
}
func (r *Repository) validateKubernetesEnvironmentOwner(ctx context.Context, tx *sqlx.Tx, environmentID, taskID string, generation int64) error {
if dialect.IsPostgres(r.db.DriverName()) {
if _, err := tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, "kubernetes-environment:"+taskID); err != nil {
return err
}
}
if err := r.lockTaskRowInTx(ctx, tx, taskID); err != nil {
return err
}
var owner string
var currentGeneration int64
if err := tx.QueryRowContext(ctx, r.db.Rebind(`SELECT task_id, ownership_generation FROM task_environments WHERE id = ?`), environmentID).Scan(&owner, &currentGeneration); err != nil {
return err
}
if owner != taskID || generation != currentGeneration {
return models.ErrKubernetesEnvironmentConflict
}
return nil
}
// Physical inventory survives task/environment cascades until durable cleanup
// finishes. Only cleanup claims may operate after both owner rows are gone.
const kubernetesEnvironmentWritable = `EXISTS (
SELECT 1 FROM task_environments e WHERE e.id = task_environment_kubernetes.environment_id
AND e.task_id = task_environment_kubernetes.task_id
AND e.ownership_generation = task_environment_kubernetes.ownership_generation
) OR (operation_id LIKE 'cleanup:%'
AND NOT EXISTS (SELECT 1 FROM task_environments e WHERE e.id = task_environment_kubernetes.environment_id)
AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.id = task_environment_kubernetes.task_id))`
func (r *Repository) ClaimKubernetesEnvironmentCleanup(ctx context.Context, environmentID, taskID string, generation int64, operationID string) (*models.KubernetesEnvironment, error) {
if operationID == "" {
return nil, models.ErrKubernetesEnvironmentConflict
}
operationID = "cleanup:" + operationID
// An existing record is mandatory: deletion never creates resource authority.
result, err := r.db.ExecContext(ctx, r.db.Rebind(`UPDATE task_environment_kubernetes
SET operation_id = ?, revision = revision + 1
WHERE environment_id = ? AND task_id = ? AND ownership_generation = ? AND operation_id = ''
AND (EXISTS (SELECT 1 FROM task_environments e WHERE e.id = task_environment_kubernetes.environment_id
AND e.task_id = task_environment_kubernetes.task_id AND e.ownership_generation = task_environment_kubernetes.ownership_generation)
OR (NOT EXISTS (SELECT 1 FROM task_environments e WHERE e.id = task_environment_kubernetes.environment_id)
AND NOT EXISTS (SELECT 1 FROM tasks t WHERE t.id = task_environment_kubernetes.task_id)))`), operationID, environmentID, taskID, generation)
if err := requireKubernetesEnvironmentWrite(result, err); err != nil {
return nil, err
}
return r.GetKubernetesEnvironment(ctx, environmentID)
}
func (r *Repository) ensureKubernetesInventoryAbsent(ctx context.Context, tx *sqlx.Tx, environmentID string) error {
var count int
if err := tx.QueryRowContext(ctx, r.db.Rebind(`SELECT COUNT(*) FROM task_environment_kubernetes WHERE environment_id = ?`), environmentID).Scan(&count); err != nil {
return err
}
if count != 0 {
return models.ErrKubernetesEnvironmentConflict
}
return nil
}
@@ -0,0 +1,179 @@
package sqlite
import (
"context"
"testing"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
)
func TestKubernetesEnvironmentInventorySurvivesSessionRemoval(t *testing.T) {
repo := newRepoForEntityTests(t)
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-kube", "env-kube")
// The physical resource record must not depend on a session's lifetime.
_, err := repo.db.ExecContext(ctx, `INSERT INTO task_environment_kubernetes
(environment_id, task_id, ownership_generation, metadata, control_secret_id, bootstrap_secret_id)
VALUES ('env-kube', 'task-kube', 1, '{"pod_uid":"exact-uid"}', 'control-secret', 'bootstrap-secret')`)
require.NoError(t, err)
session := &models.TaskSession{ID: "session-kube", TaskID: "task-kube", TaskEnvironmentID: "env-kube"}
require.NoError(t, repo.CreateTaskSession(ctx, session))
require.NoError(t, repo.DeleteTaskSession(ctx, session))
require.Error(t, repo.DeleteTaskEnvironment(ctx, "env-kube"), "remote inventory must be cleaned before its environment can be deleted")
var metadata string
require.NoError(t, repo.db.QueryRowContext(ctx, `SELECT metadata FROM task_environment_kubernetes WHERE environment_id = 'env-kube'`).Scan(&metadata))
require.JSONEq(t, `{"pod_uid":"exact-uid"}`, metadata)
}
func TestKubernetesEnvironmentClaimExclusion(t *testing.T) {
repo := newRepoForEntityTests(t)
store, ok := interface{}(repo).(interface {
ClaimKubernetesEnvironment(context.Context, string, string, int64, string) (*models.KubernetesEnvironment, error)
SaveKubernetesEnvironment(context.Context, *models.KubernetesEnvironment, bool) error
GetKubernetesEnvironment(context.Context, string) (*models.KubernetesEnvironment, error)
})
require.True(t, ok, "repository must support durable Kubernetes inventory claims")
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-claims", "env-claims")
claim, err := store.ClaimKubernetesEnvironment(ctx, "env-claims", "task-claims", 1, "create")
require.NoError(t, err)
_, err = store.ClaimKubernetesEnvironment(ctx, "env-claims", "task-claims", 1, "delete")
require.Error(t, err, "a competing operation must not acquire the resource")
claim.Metadata = map[string]interface{}{"pod_uid": "original"}
require.NoError(t, store.SaveKubernetesEnvironment(ctx, claim, true))
replacement, err := store.ClaimKubernetesEnvironment(ctx, "env-claims", "task-claims", 1, "delete")
require.NoError(t, err)
require.Error(t, store.SaveKubernetesEnvironment(ctx, claim, true), "stale operation must not overwrite a replacement")
got, err := store.GetKubernetesEnvironment(ctx, "env-claims")
require.NoError(t, err)
require.Equal(t, replacement.OperationID, got.OperationID)
require.Equal(t, "original", got.Metadata["pod_uid"])
_, err = store.ClaimKubernetesEnvironment(ctx, "env-claims", "foreign-task", 1, "foreign")
require.Error(t, err)
_, err = store.ClaimKubernetesEnvironment(ctx, "env-claims", "task-claims", 2, "stale-generation")
require.Error(t, err)
}
func TestKubernetesEnvironmentRestartReplay(t *testing.T) {
repo := newRepoForEntityTests(t)
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-replay-kube", "env-replay-kube")
claim, err := repo.ClaimKubernetesEnvironment(ctx, "env-replay-kube", "task-replay-kube", 1, "operation")
require.NoError(t, err)
claim.Metadata = map[string]interface{}{"pod_uid": "retained"}
claim.ControlSecretID = "encrypted-control-reference"
claim.BootstrapSecretID = "encrypted-bootstrap-reference"
require.NoError(t, repo.SaveKubernetesEnvironment(ctx, claim, false))
require.NoError(t, repo.runMigrations(ctx))
recovered, err := repo.ClaimKubernetesEnvironment(ctx, claim.EnvironmentID, claim.TaskID, 1, "operation")
require.NoError(t, err)
require.Equal(t, claim, recovered)
}
func TestPostgresKubernetesEnvironmentClaimExclusion(t *testing.T) {
first, second, _ := newTaskPostgresRepoPair(t)
ctx := context.Background()
seedRecoveryClaimEnvironment(t, first, "task-pg-kube", "env-pg-kube")
claim, err := first.ClaimKubernetesEnvironment(ctx, "env-pg-kube", "task-pg-kube", 1, "create")
require.NoError(t, err)
_, err = second.ClaimKubernetesEnvironment(ctx, "env-pg-kube", "task-pg-kube", 1, "delete")
require.ErrorIs(t, err, models.ErrKubernetesEnvironmentConflict)
require.NoError(t, first.SaveKubernetesEnvironment(ctx, claim, true))
_, err = second.ClaimKubernetesEnvironment(ctx, "env-pg-kube", "task-pg-kube", 1, "delete")
require.NoError(t, err)
}
func TestKubernetesEnvironmentRejectsChangedOwnerCheckpoint(t *testing.T) {
repo := newRepoForEntityTests(t)
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-owner-kube", "env-owner-kube")
claim, err := repo.ClaimKubernetesEnvironment(ctx, "env-owner-kube", "task-owner-kube", 1, "create")
require.NoError(t, err)
_, err = repo.db.ExecContext(ctx, `UPDATE task_environments SET ownership_generation = 2 WHERE id = 'env-owner-kube'`)
require.NoError(t, err)
require.ErrorIs(t, repo.SaveKubernetesEnvironment(ctx, claim, true), models.ErrKubernetesEnvironmentConflict)
}
func TestKubernetesEnvironmentDeletionRequiresCurrentClaim(t *testing.T) {
repo := newRepoForEntityTests(t)
deleter, ok := interface{}(repo).(interface {
DeleteKubernetesEnvironment(context.Context, *models.KubernetesEnvironment) error
})
require.True(t, ok, "resource cleanup requires a conditional inventory release")
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-delete-kube", "env-delete-kube")
claim, err := repo.ClaimKubernetesEnvironment(ctx, "env-delete-kube", "task-delete-kube", 1, "delete")
require.NoError(t, err)
stale := *claim
stale.Revision--
require.ErrorIs(t, deleter.DeleteKubernetesEnvironment(ctx, &stale), models.ErrKubernetesEnvironmentConflict)
require.NoError(t, deleter.DeleteKubernetesEnvironment(ctx, claim))
require.NoError(t, repo.DeleteTaskEnvironment(ctx, "env-delete-kube"))
}
func TestKubernetesEnvironmentInterruptedOperationRecovery(t *testing.T) {
repo := newRepoForEntityTests(t)
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-interrupted", "env-interrupted")
record, err := repo.ClaimKubernetesEnvironment(ctx, "env-interrupted", "task-interrupted", 1, "old-process")
require.NoError(t, err)
record.Metadata = map[string]interface{}{"pod_uid": "retained"}
require.NoError(t, repo.SaveKubernetesEnvironment(ctx, record, false))
recovery, ok := interface{}(repo).(interface{ RecoverInterruptedKubernetesOperations(context.Context) error })
require.True(t, ok, "startup under exclusive runtime ownership must fence interrupted operations")
require.NoError(t, recovery.RecoverInterruptedKubernetesOperations(ctx))
require.ErrorIs(t, repo.SaveKubernetesEnvironment(ctx, record, true), models.ErrKubernetesEnvironmentConflict)
resumed, err := repo.ClaimKubernetesEnvironment(ctx, record.EnvironmentID, record.TaskID, 1, "new-process")
require.NoError(t, err)
require.Equal(t, "retained", resumed.Metadata["pod_uid"])
}
func TestKubernetesEnvironmentInventoryListsWithoutSessions(t *testing.T) {
repo := newRepoForEntityTests(t)
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-retained", "env-retained")
_, err := repo.ClaimKubernetesEnvironment(ctx, "env-retained", "task-retained", 1, "create")
require.NoError(t, err)
lister, ok := interface{}(repo).(interface {
ListKubernetesEnvironments(context.Context) ([]*models.KubernetesEnvironment, error)
})
require.True(t, ok, "retained pods must be discoverable without a session row")
records, err := lister.ListKubernetesEnvironments(ctx)
require.NoError(t, err)
require.Len(t, records, 1)
require.Equal(t, "env-retained", records[0].EnvironmentID)
}
func TestKubernetesEnvironmentInventorySurvivesTaskDeletion(t *testing.T) {
testKubernetesEnvironmentInventorySurvivesTaskDeletion(t, newRepoForEntityTests(t))
}
func TestPostgresKubernetesEnvironmentInventorySurvivesTaskDeletion(t *testing.T) {
repo, _, _ := newTaskPostgresRepoPair(t)
testKubernetesEnvironmentInventorySurvivesTaskDeletion(t, repo)
}
func testKubernetesEnvironmentInventorySurvivesTaskDeletion(t *testing.T, repo *Repository) {
t.Helper()
ctx := context.Background()
seedRecoveryClaimEnvironment(t, repo, "task-deleted-kube", "env-deleted-kube")
record, err := repo.ClaimKubernetesEnvironment(ctx, "env-deleted-kube", "task-deleted-kube", 1, "create")
require.NoError(t, err)
record.Metadata = map[string]interface{}{"pod_uid": "retained"}
require.NoError(t, repo.SaveKubernetesEnvironment(ctx, record, true))
require.NoError(t, repo.DeleteTask(ctx, record.TaskID))
retained, err := repo.GetKubernetesEnvironment(ctx, record.EnvironmentID)
require.NoError(t, err)
require.Equal(t, record.Metadata, retained.Metadata)
_, err = repo.ClaimKubernetesEnvironment(ctx, record.EnvironmentID, record.TaskID, 1, "launch")
require.Error(t, err, "deleted owners cannot launch")
_, err = repo.ClaimKubernetesEnvironmentCleanup(ctx, record.EnvironmentID, record.TaskID, 2, "wrong-generation")
require.Error(t, err)
cleanup, err := repo.ClaimKubernetesEnvironmentCleanup(ctx, record.EnvironmentID, record.TaskID, 1, "delete")
require.NoError(t, err)
require.NoError(t, repo.SaveKubernetesEnvironment(ctx, cleanup, true))
cleanup, err = repo.ClaimKubernetesEnvironmentCleanup(ctx, record.EnvironmentID, record.TaskID, 1, "retry")
require.NoError(t, err)
require.NoError(t, repo.DeleteKubernetesEnvironment(ctx, cleanup))
}
@@ -878,6 +878,9 @@ func (r *Repository) DeleteTaskEnvironment(ctx context.Context, id string) error
if err := recoveryclaim.EnsureAvailableTx(ctx, r.db, tx, id); err != nil {
return err
}
if err := r.ensureKubernetesInventoryAbsent(ctx, tx, id); err != nil {
return err
}
result, err := tx.ExecContext(ctx, r.db.Rebind(`DELETE FROM task_environments WHERE id = ?`), id)
if err != nil {
return err
@@ -919,6 +922,9 @@ func (r *Repository) DeleteTaskEnvironmentsByTask(ctx context.Context, taskID st
return err
}
for _, environmentID := range environmentIDs {
if err := r.ensureKubernetesInventoryAbsent(ctx, tx, environmentID); err != nil {
return err
}
if err := recoveryclaim.EnsureAvailableTx(ctx, r.db, tx, environmentID); err != nil {
return err
}
@@ -0,0 +1,63 @@
package service
import (
"context"
"errors"
"fmt"
agentkubernetes "github.com/kandev/kandev/internal/agent/kubernetes"
"strings"
"github.com/kandev/kandev/internal/task/models"
)
func (s *Service) teardownKubernetesEnvironment(ctx context.Context, env *models.TaskEnvironment) error {
if env.ExecutorType != string(models.ExecutorTypeKubernetes) {
return nil
}
inventory, ok := s.taskEnvironments.(interface {
GetKubernetesEnvironment(context.Context, string) (*models.KubernetesEnvironment, error)
})
if !ok {
return nil
}
record, err := inventory.GetKubernetesEnvironment(ctx, env.ID)
if errors.Is(err, models.ErrKubernetesEnvironmentNotFound) {
return nil
}
if err != nil {
return err
}
if record == nil || record.TaskID != env.TaskID || record.EnvironmentID != env.ID {
return models.ErrWorkspaceReuseUnsafe
}
destroyer, ok := s.envDestroyer.(interface {
DestroyKubernetesEnvironment(context.Context, *models.TaskEnvironment) error
})
if !ok {
return fmt.Errorf("kubernetes environment destroyer is unavailable")
}
return destroyer.DestroyKubernetesEnvironment(ctx, env)
}
func (s *Service) hasKubernetesEnvironmentInventory(ctx context.Context, executorID string) (bool, error) {
store, ok := s.taskEnvironments.(interface {
ListKubernetesEnvironments(context.Context) ([]*models.KubernetesEnvironment, error)
})
if !ok {
return false, nil
}
records, err := store.ListKubernetesEnvironments(ctx)
if err != nil {
return false, err
}
for _, record := range records {
if record == nil {
continue
}
id, _ := record.Metadata[agentkubernetes.MetadataKeyResourceExecutorID].(string)
if strings.TrimSpace(id) == "" || id == executorID {
return true, nil
}
}
return false, nil
}
@@ -0,0 +1,62 @@
package service
import (
"context"
"errors"
"testing"
"github.com/kandev/kandev/internal/task/models"
"github.com/stretchr/testify/require"
)
type kubernetesInventoryEnvRepo struct {
stubEnvRepo
inventory *models.KubernetesEnvironment
inventoryErr error
}
func (r *kubernetesInventoryEnvRepo) GetKubernetesEnvironment(context.Context, string) (*models.KubernetesEnvironment, error) {
return r.inventory, r.inventoryErr
}
type kubernetesEnvironmentDestroyer struct {
stubDestroyer
deleted string
err error
}
func (d *kubernetesEnvironmentDestroyer) DestroyKubernetesEnvironment(_ context.Context, env *models.TaskEnvironment) error {
d.deleted = env.ID
return d.err
}
func TestKubernetesEnvironmentCleanupUsesTaskOwner(t *testing.T) {
for _, fail := range []bool{false, true} {
t.Run(map[bool]string{false: "successful teardown", true: "failed teardown retains inventory"}[fail], func(t *testing.T) {
env := &models.TaskEnvironment{ID: "env-kube", TaskID: "task-kube", ExecutorType: "k8s", OwnershipGeneration: 1}
destroyer := &kubernetesEnvironmentDestroyer{}
if fail {
destroyer.err = errors.New("cluster unavailable")
}
service := &Service{taskEnvironments: &kubernetesInventoryEnvRepo{inventory: &models.KubernetesEnvironment{EnvironmentID: env.ID, TaskID: env.TaskID}}, envDestroyer: destroyer}
err := service.teardownEnvironmentRuntimeResources(context.Background(), env)
require.Equal(t, env.ID, destroyer.deleted)
require.ErrorIs(t, err, destroyer.err)
})
}
}
func (r *kubernetesInventoryEnvRepo) ListKubernetesEnvironments(context.Context) ([]*models.KubernetesEnvironment, error) {
return []*models.KubernetesEnvironment{r.inventory}, r.inventoryErr
}
func TestKubernetesRetainedEnvironmentBlocksExecutorDeletion(t *testing.T) {
service := &Service{taskEnvironments: &kubernetesInventoryEnvRepo{inventory: &models.KubernetesEnvironment{Metadata: map[string]interface{}{"kubernetes_resource_executor_id": "executor-kube"}}}}
checker, ok := interface{}(service).(interface {
hasKubernetesEnvironmentInventory(context.Context, string) (bool, error)
})
require.True(t, ok, "executor mutation guard must include retained environments")
retained, err := checker.hasKubernetesEnvironmentInventory(context.Background(), "executor-kube")
require.NoError(t, err)
require.True(t, retained)
}
@@ -1955,6 +1955,9 @@ func (s *Service) DeleteExecutor(ctx context.Context, id string) error {
}
func (s *Service) hasExecutorRunningInventory(ctx context.Context, executorID string) (bool, error) {
if retained, err := s.hasKubernetesEnvironmentInventory(ctx, executorID); err != nil || retained {
return retained, err
}
running, err := s.executors.ListExecutorsRunning(ctx)
if err != nil {
return false, err
@@ -394,6 +394,10 @@ func (s *Service) teardownEnvironmentResourcesWithWorktrees(
if cause := context.Cause(ctx); cause != nil {
return cause
}
if err := s.teardownKubernetesEnvironment(ctx, env); err != nil {
return err
}
worktreeIDs := environmentWorktreeIDs(env)
if !environmentHasResources(env, includeWorktrees, worktreeIDs) {
return nil
@@ -28,7 +28,7 @@ export function MobileSessionList({ sessions }: { sessions: KubernetesSession[]
<div className="space-y-3" data-testid="kubernetes-mobile-session-list">
{sessions.map((session) => (
<AppLink
key={session.session_id}
key={`${session.task_id}:${session.session_id}`}
href={taskHref(session.task_id)}
aria-label={translateTaskLink(session, t)}
data-testid="kubernetes-session-task-link"
@@ -62,9 +62,10 @@ export function DesktopSessionTable({ sessions }: { sessions: KubernetesSession[
</TableHeader>
<TableBody>
{sessions.map((session) => {
const expanded = expandedSessionId === session.session_id;
const rowKey = `${session.task_id}:${session.session_id}`;
const expanded = expandedSessionId === rowKey;
return (
<Fragment key={session.session_id}>
<Fragment key={`${session.task_id}:${session.session_id}`}>
<TableRow data-testid="kubernetes-session-row">
<TableCell className="whitespace-nowrap py-2 font-mono text-xs">
<AppLink
@@ -99,7 +100,7 @@ export function DesktopSessionTable({ sessions }: { sessions: KubernetesSession[
: "executors:kubernetesShowSessionDetails",
)}
className="cursor-pointer"
onClick={() => setExpandedSessionId(expanded ? null : session.session_id)}
onClick={() => setExpandedSessionId(expanded ? null : rowKey)}
>
<IconChevronDown
aria-hidden="true"
@@ -1,4 +1,4 @@
import { cleanup, render, screen, within } from "@testing-library/react";
import { cleanup, fireEvent, render, screen, within } from "@testing-library/react";
import type { ComponentProps } from "react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { KubernetesSessionsCard } from "./kubernetes-sessions-card";
@@ -9,9 +9,40 @@ vi.mock("@/hooks/use-responsive-breakpoint", () => ({
useResponsiveBreakpoint: () => ({ isMobile: responsive.isMobile }),
}));
afterEach(cleanup);
afterEach(() => {
cleanup();
responsive.isMobile = false;
});
describe("KubernetesSessionsCard", () => {
it("keeps retained pods without sessions distinct on both layouts", () => {
responsive.isMobile = false;
const state = sessionsState();
state.sessions = ["task-first", "task-second"].map((task) => ({
...state.sessions[0],
task_id: task,
session_id: "",
pod_name: task,
}));
const rendered = render(<KubernetesSessionsCard state={state} />);
expect(
screen
.getAllByTestId("kubernetes-session-task-link")
.map((link) => link.getAttribute("href")),
).toEqual(["/t/task-first", "/t/task-second"]);
fireEvent.click(screen.getAllByRole("button", { name: "Show session details" })[0]);
expect(screen.getAllByTestId("kubernetes-session-details-row")).toHaveLength(1);
state.sessions = state.sessions.slice(1);
rendered.rerender(<KubernetesSessionsCard state={state} />);
expect(screen.getAllByTestId("kubernetes-session-row")).toHaveLength(1);
responsive.isMobile = true;
rendered.rerender(<KubernetesSessionsCard state={state} />);
expect(screen.getByTestId("kubernetes-session-task-link").getAttribute("href")).toBe(
"/t/task-second",
);
expect(screen.getByTestId("kubernetes-mobile-session-list").textContent).toContain("Retained");
});
it("shows retained state and main-container requests on both layouts", () => {
const state = sessionsState();
const rendered = render(<KubernetesSessionsCard state={state} />);
@@ -0,0 +1,136 @@
import {
test,
expect,
kubernetesExecutorConfig,
kubernetesProfileConfig,
} from "../../fixtures/kubernetes-test-base";
import type { KubernetesPod } from "../../fixtures/kubernetes-tools";
import {
execInKubernetesPod,
waitForKubernetesPod,
waitForKubernetesPVC,
waitForKubernetesResourceAbsent,
waitForTaskSessionState,
} from "../../helpers/kubernetes";
import {
waitForAgentMessage,
waitForLatestSessionDone,
waitForSessionDone,
} from "../../helpers/session";
test("sessions share one task pod through stop and backend restart", async ({
apiClient,
seedData,
cluster,
backend,
}) => {
test.setTimeout(360_000);
await apiClient.updateExecutor(seedData.executorId, {
config: kubernetesExecutorConfig(cluster, { request_timeout_seconds: "120" }),
});
const profile = await apiClient.createExecutorProfile(seedData.executorId, {
name: "Shared task pod",
prepare_script: "",
cleanup_script: "",
env_vars: [],
config: kubernetesProfileConfig(cluster, {
"workspace.mode": "managed_pvc",
"workspace.size": "1Gi",
"workspace.access_modes": JSON.stringify(["ReadWriteOnce"]),
}),
});
const task = await apiClient.createTaskWithAgent(
seedData.workspaceId,
"Shared Kubernetes task",
seedData.agentProfileId,
{
description: "/e2e:simple-message",
workflow_id: seedData.workflowId,
workflow_step_id: seedData.startStepId,
executor_id: seedData.executorId,
executor_profile_id: profile.id,
},
);
await waitForLatestSessionDone(apiClient, task.id, 1, "First session ready", 180_000);
expect(task.session_id).toBeTruthy();
const pod = await waitForKubernetesPod(cluster, task.id, task.session_id!);
const claim = await waitForKubernetesPVC(cluster, task.id, task.session_id!);
execInKubernetesPod(cluster, pod.metadata.name, [
"sh",
"-c",
"printf shared > /workspace/shared-marker",
]);
const launchAdditional = () =>
apiClient.launchSession(
{
task_id: task.id,
agent_profile_id: seedData.agentProfileId,
executor_id: seedData.executorId,
executor_profile_id: profile.id,
prompt: "/e2e:simple-message",
},
90_000,
);
const [second, third] = await Promise.all([launchAdditional(), launchAdditional()]);
await waitForSessionDone(apiClient, task.id, second.session_id, "Second session ready");
await waitForSessionDone(apiClient, task.id, third.session_id, "Concurrent sibling ready");
const taskPods = () =>
cluster.json<{ items: KubernetesPod[] }>([
"-n",
cluster.namespace,
"get",
"pods",
"-l",
`kandev.ai/task-id=${task.id}`,
]).items;
expect(taskPods()).toHaveLength(1);
expect(taskPods()[0].metadata.uid).toBe(pod.metadata.uid);
const rows = (await apiClient.listKubernetesSessions(seedData.executorId)).filter(
(row) => row.task_id === task.id,
);
expect(rows).toHaveLength(3);
await test.info().attach("shared-task-session-status", {
body: JSON.stringify(rows, null, 2),
contentType: "application/json",
});
const instances = execInKubernetesPod(cluster, pod.metadata.name, [
"sh",
"-c",
"find /run/kandev/sessions -name auth.env | wc -l",
]);
expect(Number(instances.trim())).toBe(3);
await apiClient.stopSession({ session_id: task.session_id!, force: true });
await waitForTaskSessionState(apiClient, task.id, task.session_id!, "CANCELLED");
expect(taskPods()).toHaveLength(1);
await apiClient.addUserMessage(
task.id,
second.session_id,
'e2e:message("Sibling survived stop")',
);
await waitForAgentMessage(apiClient, second.session_id, "Sibling survived stop");
await waitForSessionDone(apiClient, task.id, second.session_id, "Sibling survives stop");
await backend.restart();
await apiClient.addUserMessage(
task.id,
second.session_id,
'e2e:message("Sibling survived restart")',
);
await waitForAgentMessage(apiClient, second.session_id, "Sibling survived restart");
await waitForSessionDone(
apiClient,
task.id,
second.session_id,
"Sibling survives backend restart",
);
expect(taskPods()).toHaveLength(1);
expect(taskPods()[0].metadata.uid).toBe(pod.metadata.uid);
expect(execInKubernetesPod(cluster, pod.metadata.name, ["cat", "/workspace/shared-marker"])).toBe(
"shared",
);
await apiClient.archiveTask(task.id);
await waitForKubernetesResourceAbsent(cluster, "pod", pod.metadata.name);
await waitForKubernetesResourceAbsent(cluster, "persistentvolumeclaim", claim.metadata.name);
expect(rows.map((row) => ({ pod: row.pod_name, failure: row.failure_reason ?? "" }))).toEqual(
rows.map(() => ({ pod: pod.metadata.name, failure: "" })),
);
});
+12 -56
View File
@@ -386,24 +386,24 @@
"sshProbeHint": "「{{button}}」をクリックして、リモートにインストールされているエージェントを確認します。",
"sshProxyJumpHint": "単一の踏み台ホップのみ対応しています。連続したジャンプはサポートされていません。",
"sshProxyJumpLabel": "ProxyJump(任意)",
"sshReachabilityAgeLastProbe": "最終プローブ: {{age}}",
"sshReachabilityAgeLastSuccess": "最終成功: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}} 回連続で失敗",
"sshReachabilityConsecutiveFailures_other": "{{count}} 回連続で失敗",
"sshReachabilityDescription": "このホストに対する直近の SSH プローブ結果です。バックグラウンドで自動的に確認されます。",
"sshReachabilityHost": "プローブ対象ホスト: {{host}}",
"sshReachabilityLoadFailed": "到達性の状態を読み込めませんでした。状態は不明です。",
"sshReachabilityAgeLastProbe": "最後のプローブ: {{age}}",
"sshReachabilityAgeLastSuccess": "最後の成功: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}} 回連続して失敗",
"sshReachabilityConsecutiveFailures_other": "{{count}} 回連続して失敗",
"sshReachabilityDescription": "このホストの最新の SSH プローブ結果。バックグラウンドで自動的に確認されます。",
"sshReachabilityHost": "プローブしたホスト: {{host}}",
"sshReachabilityLoadFailed": "到達性ステータスを読み込めませんでした。不明です。",
"sshReachabilityNeverProbed": "まだプローブされていません。",
"sshReachabilityNoSuccess": "成功したプローブの記録がありません。",
"sshReachabilityNoSuccess": "成功したプローブの記録はありません。",
"sshReachabilityProbeNow": "今すぐプローブ",
"sshReachabilityProbingOff": "定期プローブは無効です。現在の状態ではなく、最後に記録された結果を表示しています。",
"sshReachabilityProbingOff": "定期プローブはオフです。現在の状態ではなく、最後に記録された結果を表示します。",
"sshReachabilityReasonAuth": "認証に失敗しました",
"sshReachabilityReasonConfig": "接続設定が無効です",
"sshReachabilityReasonHostKey": "ホストキーが固定されたフィンガープリントと一致しません",
"sshReachabilityReasonHostKey": "ホストキーが固定フィンガープリントと一致しません",
"sshReachabilityReasonNetwork": "ネットワークエラーが発生しました",
"sshReachabilityReasonTimeout": "接続試行がタイムアウトしました",
"sshReachabilityReasonUnknown": "不明なエラーが発生しました",
"sshReachabilityStale": "古い結果: 過去 3 回の確認間隔内にプローブがありません",
"sshReachabilityStale": "古い情報: 過去 3 回のチェック間隔内にプローブされていません",
"sshReachabilityStateReachable": "到達可能",
"sshReachabilityStateUnknown": "不明",
"sshReachabilityStateUnreachable": "到達不能",
@@ -421,28 +421,6 @@
"sshTrustThisHost": "<0>このホストを信頼します。</0> 上記のフィンガープリントが自分のサーバーと一致することを確認しました。今後、異なるフィンガープリントを報告する接続は拒否されます。",
"sshTrusted": "信頼済み",
"sshUnverified": "未確認",
"sshReachabilityAgeLastProbe": "前回の確認: {{age}}",
"sshReachabilityAgeLastSuccess": "前回の成功: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}}回連続で失敗",
"sshReachabilityConsecutiveFailures_other": "{{count}}回連続で失敗",
"sshReachabilityDescription": "このホストに対する最新のSSHプローブ結果です。バックグラウンドで自動的に確認します。",
"sshReachabilityHost": "確認対象ホスト: {{host}}",
"sshReachabilityLoadFailed": "到達性ステータスを読み込めませんでした。不明です。",
"sshReachabilityNeverProbed": "まだ確認されていません。",
"sshReachabilityNoSuccess": "成功した確認記録はありません。",
"sshReachabilityProbeNow": "今すぐ確認",
"sshReachabilityProbingOff": "定期確認は無効です。現在の状態ではなく、最後に記録された結果を表示します。",
"sshReachabilityReasonAuth": "認証に失敗しました",
"sshReachabilityReasonConfig": "接続設定が無効です",
"sshReachabilityReasonHostKey": "ホストキーが固定されたフィンガープリントと一致しません",
"sshReachabilityReasonNetwork": "ネットワークエラーが発生しました",
"sshReachabilityReasonTimeout": "接続がタイムアウトしました",
"sshReachabilityReasonUnknown": "不明なエラーが発生しました",
"sshReachabilityStale": "古い情報: 直近3回の確認間隔内にプローブされていません",
"sshReachabilityStateReachable": "到達可能",
"sshReachabilityStateUnknown": "不明",
"sshReachabilityStateUnreachable": "到達不能",
"sshReachabilityTitle": "到達性",
"status": "ステータス",
"stop": "停止",
"success": "成功",
@@ -472,27 +450,5 @@
"allowUserNamespacesTitle": "ユーザー名前空間サポート",
"allowUserNamespacesDescription": "コンテナ内で権限なしユーザーの名前空間作成と mount 操作を許可。bwrap みたいなサンドボックスツールが使える。",
"allowUserNamespacesNote": "この設定が効くのは新規作成コンテナだけ。変更を反映するには、既存タスク環境をリセットする必要あり。",
"allowUserNamespacesWarning": "有効にすると、Docker のデフォルト AppArmor 制限をオプトイン済みコンテナに対して無効化。user-namespace サンドボックスが必要な信頼済み agent runtime だけで有効にすること。",
"sshReachabilityAgeLastProbe": "最後のプローブ: {{age}}",
"sshReachabilityAgeLastSuccess": "最後の成功: {{age}}",
"sshReachabilityConsecutiveFailures_one": "{{count}} 回連続して失敗",
"sshReachabilityConsecutiveFailures_other": "{{count}} 回連続して失敗",
"sshReachabilityDescription": "このホストの最新の SSH プローブ結果。バックグラウンドで自動的に確認されます。",
"sshReachabilityHost": "プローブしたホスト: {{host}}",
"sshReachabilityLoadFailed": "到達性ステータスを読み込めませんでした。不明です。",
"sshReachabilityNeverProbed": "まだプローブされていません。",
"sshReachabilityNoSuccess": "成功したプローブの記録はありません。",
"sshReachabilityProbeNow": "今すぐプローブ",
"sshReachabilityProbingOff": "定期プローブはオフです。現在の状態ではなく、最後に記録された結果を表示します。",
"sshReachabilityReasonAuth": "認証に失敗しました",
"sshReachabilityReasonConfig": "接続設定が無効です",
"sshReachabilityReasonHostKey": "ホストキーが固定フィンガープリントと一致しません",
"sshReachabilityReasonNetwork": "ネットワークエラーが発生しました",
"sshReachabilityReasonTimeout": "接続試行がタイムアウトしました",
"sshReachabilityReasonUnknown": "不明なエラーが発生しました",
"sshReachabilityStale": "古い情報: 過去 3 回のチェック間隔内にプローブされていません",
"sshReachabilityStateReachable": "到達可能",
"sshReachabilityStateUnknown": "不明",
"sshReachabilityStateUnreachable": "到達不能",
"sshReachabilityTitle": "到達性"
"allowUserNamespacesWarning": "有効にすると、Docker のデフォルト AppArmor 制限をオプトイン済みコンテナに対して無効化。user-namespace サンドボックスが必要な信頼済み agent runtime だけで有効にすること。"
}
+6 -26
View File
@@ -2280,27 +2280,17 @@
"launchErrorGeneric": "タスク開始できなかった。",
"launchErrorSessionPreview": "タスク launch に対応必要。",
"launchNeedsAttention": "launch に対応必要",
"launchWarningTitle": "SSHホストに到達できない可能性があります",
"launchWarningHost": "プローブ対象ホスト: {{host}}",
"launchWarningLastSuccess": "前回の成功: {{age}}",
"launchWarningTitle": "SSH ホストに到達できない可能性があります",
"launchWarningHost": "プローブしたホスト: {{host}}",
"launchWarningLastSuccess": "最後の成功: {{age}}",
"launchWarningNeverSucceeded": "このホストで成功したプローブはまだ記録されていません。",
"launchWarningReasonConfig": "最後の試行は接続設定が無効なため失敗しました。",
"launchWarningReasonConfig": "最後の試行は接続設定が無効だったため失敗しました。",
"launchWarningReasonTimeout": "最後の試行はタイムアウトしました。",
"launchWarningReasonHostKey": "最後の試行はホストキーが固定されたフィンガープリントと一致しなかったため失敗しました。",
"launchWarningReasonHostKey": "最後の試行はホストキーが固定フィンガープリントと一致しなかったため失敗しました。",
"launchWarningReasonAuth": "最後の試行は認証に失敗しました。",
"launchWarningReasonNetwork": "最後の試行はネットワークエラーで失敗しました。",
"launchWarningReasonUnknown": "最後の試行は不明な理由で失敗しました。",
"launchErrorNoChanges": "task か branch 設定、どちらも変更なし。",
"launchWarningTitle": "SSH ホストに到達できない可能性があります",
"launchWarningHost": "プローブ対象ホスト: {{host}}",
"launchWarningLastSuccess": "最終成功: {{age}}",
"launchWarningNeverSucceeded": "このホストで成功したプローブは一度も記録されていません。",
"launchWarningReasonConfig": "前回の試行は、接続設定が無効なため失敗しました。",
"launchWarningReasonTimeout": "前回の試行はタイムアウトしました。",
"launchWarningReasonHostKey": "前回の試行は、ホストキーが固定されたフィンガープリントと一致しなかったため失敗しました。",
"launchWarningReasonAuth": "前回の試行は認証に失敗しました。",
"launchWarningReasonNetwork": "前回の試行はネットワークエラーで失敗しました。",
"launchWarningReasonUnknown": "前回の試行は不明な理由で失敗しました。",
"launchPromptPreview": "プロンプトプレビューを起動",
"retryWithDefaultBranch": "default branch で再試行",
"pickBaseBranch": "base branch を選ぶ",
@@ -2384,15 +2374,5 @@
"checkoutOptions.credentialsUnsupported": "高度なチェックアウトオプションには、Kandev 管理の Git 認証情報を使って。",
"toolPayloadRemovedAt": "ツール詳細は {{date}} に削除済み。",
"toolPayloadRemoved": "ツール詳細は削除済み。",
"openTaskSwitcherFor": "タスク切替を開く: {{title}}",
"launchWarningTitle": "SSH ホストに到達できない可能性があります",
"launchWarningHost": "プローブしたホスト: {{host}}",
"launchWarningLastSuccess": "最後の成功: {{age}}",
"launchWarningNeverSucceeded": "このホストで成功したプローブはまだ記録されていません。",
"launchWarningReasonConfig": "最後の試行は接続設定が無効だったため失敗しました。",
"launchWarningReasonTimeout": "最後の試行はタイムアウトしました。",
"launchWarningReasonHostKey": "最後の試行はホストキーが固定フィンガープリントと一致しなかったため失敗しました。",
"launchWarningReasonAuth": "最後の試行は認証に失敗しました。",
"launchWarningReasonNetwork": "最後の試行はネットワークエラーで失敗しました。",
"launchWarningReasonUnknown": "最後の試行は不明な理由で失敗しました。"
"openTaskSwitcherFor": "タスク切替を開く: {{title}}"
}
@@ -4,6 +4,11 @@
**Date:** 2026-08-24
**Area:** backend, frontend, infra, security
The session-scoped ownership choice is superseded for shared task pods
by [task-owned compute](2026-09-21-kubernetes-task-pod-ownership.md).
Its exact-resource safety invariants remain in force; existing legacy resources
retain their recorded identity until safely adopted.
## Context
The Kubernetes executor must combine an administrator-authored Pod template
@@ -0,0 +1,41 @@
# ADR-2026-09-21-kubernetes-task-pod-ownership: Task-owned Kubernetes compute
**Status:** accepted
**Date:** 2026-09-21
**Area:** backend, infra
## Context
The user requires one Kubernetes pod per task with multiple agent sessions.
The foundation intentionally allocated one pod per lifecycle/session instance.
Simply changing resource names would leave session cleanup and authentication
owning shared resources, and would race concurrent creation and recovery.
## Decision
Make the canonical task environment the durable owner of Kubernetes compute.
Sessions own independent agentctl instances inside that pod. Persist physical
resource inventory and encrypted control credentials independently of session
execution rows; task cleanup owns physical deletion.
This decision supersedes only the session-scoped ownership choice in
[the original ownership ADR](2026-08-24-kubernetes-executor-resource-ownership.md)
for newly shared task resources. Exact UID, full ownership validation, admission,
create-nonce continuity, launch snapshots, and external-PVC protection remain.
Legacy ambiguous multi-pod tasks are never automatically consolidated.
Implementation follows the [design](../specs/executors/system-design/kubernetes-task-pod.md); validation evidence is recorded in the [plan](../plans/kubernetes-task-pod/plan.md).
## Consequences
Additional sessions share workspace data and compute without duplicate pods.
Pod failure affects all attached sessions. Durable environment inventory and
creation/deletion exclusion are required even when no sessions remain.
Compatibility must preserve legacy independent workspaces and recorded identities.
## Alternatives Considered
- Keep per-session pods: violates the requested compute and workspace contract.
- Change names to task IDs only: leaves races, per-session cleanup, and control
authentication unresolved.
- Reuse an arbitrary sibling execution: loses ownership when that session is
deleted and cannot safely choose among legacy independent workspaces.
@@ -0,0 +1,41 @@
# ADR-2026-09-23-kubernetes-task-trust-boundary: Shared task credential trust
**Status:** accepted
**Date:** 2026-09-23
**Area:** backend, infra
## Context
[Task-owned Kubernetes compute](2026-09-21-kubernetes-task-pod-ownership.md)
places independent agent sessions in one container under one OS user. Separate
HOME paths and private file permissions prevent accidental configuration mixing,
but sibling processes can read each other's credentials. A security review
required an explicit boundary before supporting different credential bindings.
The user selected shared task trust on 2026-09-23.
## Decision
Treat every session in a task as mutually trusted. Allow different agent profiles
and credentials in the shared pod. Per-session environment and auth preparation
must preserve each session's launch settings, without claiming security isolation
from other sessions. Operators own agent trust and provider-side credential
revocation. Document this boundary in the public Kubernetes executor guide.
## Consequences
A compromised agent can expose sibling credentials. Operators must stop the task's
agents and revoke or rotate exposed credentials with their providers. Stopping or
deleting one session cannot undo exposure. Agents requiring separate trust must
use separate tasks with appropriate executor isolation policies.
Regression tests verify that different profiles and credentials share one pod
while retaining distinct launch configuration. They do not claim same-UID access
is prevented. The [requirements](../specs/executors/requirements/kubernetes-task-pod.md)
and [design](../specs/executors/system-design/kubernetes-task-pod.md) define this contract.
## Alternatives Considered
- Reject differing credential bindings: limits multi-profile tasks and does not
isolate other same-UID secrets. The user chose to permit differing bindings.
- Separate OS identities or a credential broker: changes the executor security
architecture and operational contract; not selected for shared task trust.
+152
View File
@@ -0,0 +1,152 @@
---
created: 2026-09-21
status: done
requirements:
- REQ-EXECUTORS-KUBERNETES-TASK-POD-001
system_design:
- ../../specs/executors/system-design/kubernetes-task-pod.md
legacy_specs:
- ../../specs/kubernetes-executor/spec.md
---
# Implementation Plan: Kubernetes task pod
## Overview
Replace per-session pod allocation with one task-owned pod containing independent
agentctl instances. Deliver additive durable inventory first, integrate lifecycle
ownership second, and prove the full flow with Kind plus operator documentation.
All three work orders are complete.
## Scope
### In scope
Task-owned Pod/PVC inventory and control credentials, safe session attachment,
concurrent creation/recovery, session-only termination, task-owned cleanup,
status projection, and non-destructive legacy compatibility.
### Out of scope
Cross-task pod pooling, automatic merging of existing pods/workspaces, new UI
controls, and changing workspace inheritance or executor-transition semantics.
## Confirmed evidence
Source trace, not a reproduction on the user's cluster:
`KubernetesExecutor.CreateInstance` locks by execution ID and chooses
`createFresh` without recorded pod metadata. `newKubernetesFreshLaunch` calls
`kubernetesResourceNames(req.InstanceID)`, so different sessions create different
pods. The foundation explicitly requires one pod per session and excludes
concurrent sharing. `StopInstance` also owns terminal pod deletion. The requested
behavior therefore changes physical ownership, not just a reuse lookup.
Smallest reproduction: create session A and then session B in the same ready
Kubernetes task environment with distinct execution IDs; compare Pod UIDs and
managed PVCs. The regression must assert one pod, one claim, two agentctl
instances, and unchanged workspace content.
## Technical approach
Follow the [design](../../specs/executors/system-design/kubernetes-task-pod.md)
and [requirements](../../specs/executors/requirements/kubernetes-task-pod.md).
Add `task_environment_kubernetes` with durable operation/generation exclusion and
secret references through the existing task repository and lifecycle adapters.
Keep `executors_running` per-session. Integrate create, attach, reconnect,
checkpoint, refresh, stop, cleanup, and session-status resolution together so no
intermediate shared-resource path retains destructive session cleanup.
## Tests
Proposed regression files/methods (names are implementation targets):
| Criteria | Evidence |
| --- | --- |
| AC-EXECUTORS-KUBERNETES-TASK-POD-001.1, .2 | `executor_kubernetes_task_pod_test.go`: `TestKubernetesTaskPodAdditionalSession`, `TestKubernetesTaskPodConcurrentAttach`; independent env/auth and unchanged files |
| AC-EXECUTORS-KUBERNETES-TASK-POD-001.2, .4, .5 | `environment_kubernetes_test.go`: `TestKubernetesEnvironmentClaimExclusion`, `TestKubernetesEnvironmentRestartReplay`; PostgreSQL multi-connection counterparts |
| AC-EXECUTORS-KUBERNETES-TASK-POD-001.3, .5 | `executor_kubernetes_task_pod_test.go`: `TestKubernetesTaskPodStopPreservesSibling`, `TestKubernetesTaskPodTerminalCleanup`; live + failed mixed rows |
| AC-EXECUTORS-KUBERNETES-TASK-POD-001.4, .6 | `executor_kubernetes_task_pod_test.go`: `TestKubernetesTaskPodConcurrentRecovery`, `TestKubernetesTaskPodLegacyAmbiguity`; foreign UID, partial failure, retained zero-session inventory |
| AC-EXECUTORS-KUBERNETES-TASK-POD-001.7 | `sessions_task_pod_test.go`: `TestSessionsSharedTaskPod`; lifecycle profile pinning and instance environment tests |
## E2E tests
Add `apps/web/e2e/tests/kubernetes/kubernetes-task-pod.spec.ts` in the
`containers` project using the existing disposable Kind fixture. Cover criteria
.1 through .5 and .7 with sequential and concurrent second-session creation,
shared files, stop-one, backend restart/resume, and final archive cleanup.
Criterion .6 uses exact fake-client/repository tests, with a Kind ownership
mismatch scenario if the existing fixture can inject it safely.
No rendering change is planned; desktop/mobile consume the same status contract.
## Work orders
- [x] [Task 01: Durable environment inventory](task-01-durable-inventory.md)
- [x] [Task 02: Shared pod lifecycle](task-02-shared-lifecycle.md)
- [x] [Task 03: Kind evidence and operator docs](task-03-kind-docs.md)
## Verification results
SQLite and PostgreSQL inventory regressions pass with race detection. The targeted
Kubernetes lifecycle, task service, status, orchestrator executor, and backend
wiring packages pass with race detection. Eleven desktop/mobile component tests
pass. Public-doc validators, spec catalog/linter, and SQL guard pass.
The Kind regression passed (1 test, 6.9 minutes), proving three concurrent/shared
sessions in one Pod UID and one managed PVC, distinct per-session auth files,
shared untracked data, force-stop isolation, a real response after backend
restart, healthy status rows, and final archive deletion of the Pod/PVC.
Live runs exposed and fixed task-delete inventory retention, private HOME creation,
and missing persistent ownership markers. The passing run also exposed a cleanup
retry after local execution identity changed. A follow-up failing-then-passing
regression verifies exact remote-session connection cleanup after that rebind;
the final lifecycle race suite and lint pass. Kind was not repeated after this
narrow connection-cleanup fix.
Cold-host setup required temporary image-load/fixture deadlines. Those edits were
restored; the test itself uses a supported 120-second executor request timeout for
cold PVC provisioning. All disposable Kind clusters were removed by teardown.
Mobile state-only change: use a task/session composite row identity so multiple
retained task Pods without sessions stay distinct. Existing desktop table and
mobile linked cards keep their composition and touch behavior. Targeted component
coverage exercises both layouts; no new viewport interaction is introduced.
## PR remediation
Review fixes add typed agentctl not-found errors, request-scoped canonical
inventory/Pod caching, required-table upgrade probes, and concurrent stop/attach
coverage. Credential persistence recovery retains the issued token until its
canonical encrypted record is saved, using an encrypted recovery secret across
backend restart. Cleanup detaches cancellation with a one-minute budget. Updated backend guidance distinguishes
physical environment ownership from per-session inventory.
CI remediation also incorporates the current base and fills 32 missing Japanese
SSH strings. The base's preview-feedback subscription requires an updated exact
subscription count; no WebSocket production behavior changes for that fix.
Post-fix verification: targeted lifecycle/status race regressions, the full
agentctl race suite, SQLite store conformance and previous-stable upgrades pass.
The WebSocket subscription regression passes 20 race-enabled runs. Full i18n
checks and three current component tests pass; harness/spec checks pass.
Remote CI/review completion remains pending. The user selected shared task trust on 2026-09-23: differing profiles and credentials
are allowed, and same-UID sibling credential access is accepted. The trust ADR,
requirements, design and public guide record operator trust and revocation duties.
## Risks
Control authentication and bootstrap files currently assume one session; sharing
must not replace sibling credentials. Legacy tasks can contain genuinely different
workspaces. Durable claims must recover after crashes without duplicate API creates.
Public docs reflect the implemented and verified behavior. Persistence coverage requires a disposable
PostgreSQL database; Kind coverage requires Docker and the pinned worker image.
## 2026-09-23 fixup validation
Merged main and resolved Japanese catalog conflicts while preserving all keys and
removing duplicate keys. Both CI assertion failures reproduced locally: ownership
coverage expected obsolete error text, and resume observations were asynchronous.
Typed error assertions and synchronized resume coverage now pass 20 race-enabled
runs, as does the shared-pod suite with differing agent profiles and credentials.
Full orchestrator executor and agentctl API race suites pass. Translation checks,
specification validation and all 199 harness files pass. Postgres CI failed while
downloading protobuf from the Go proxy; the next head will rerun that job.
@@ -0,0 +1,106 @@
---
id: "01-durable-inventory"
title: "Durable environment inventory"
status: done
wave: 1
depends_on: []
plan: "plan.md"
requirements:
- REQ-EXECUTORS-KUBERNETES-TASK-POD-001
acceptance_criteria:
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.2
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.4
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.5
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.6
system_design:
- ../../specs/executors/system-design/kubernetes-task-pod.md
---
# Task 01: Durable environment inventory
## Summary
Add the durable environment-level Kubernetes inventory and conditional operation
claims as a compatible expansion. Existing lifecycle behavior remains unchanged
until Task 02 wires the new record into allocation and cleanup.
## In scope
- Typed runtime record, secret references, schema initialization and replayable migration.
- Transactional create/recover/delete claims and generation checks on both dialects.
- Legacy inventory classification is integrated and tested in Task 02 at the lifecycle consumer.
## Out of scope
Cross-task pooling, automated legacy consolidation, and UI redesign.
## Acceptance
- Concurrent claims elect one owner and cannot overlap deletion with attachment.
- Restart replay preserves exact inventory and secret references; migration deletes no remote resources or legacy rows.
- Changed ownership generations and stale cleanup operations cannot overwrite or release inventory.
## Verification
Use TDD for changed logic: run the named regression before and after implementation.
Commands run from repository root. Read scoped guidance and applicable TDD/E2E
skills before implementation.
```bash
(cd apps/backend && go test -tags fts5 ./internal/task/repository/sqlite -run 'KubernetesEnvironment' -count=1)
(cd apps/backend && go test -race -tags fts5 ./internal/task/repository/sqlite -run 'KubernetesEnvironment' -count=1)
(cd apps/backend && go run ./cmd/sqlguard ./internal)
(cd apps/backend && go test -race ./internal/persistence/storeconformance -count=1)
```
## Files likely touched
- `apps/backend/internal/task/models/models.go`
- `apps/backend/internal/task/repository/sqlite/environment.go`
- New `apps/backend/internal/task/repository/sqlite/environment_kubernetes.go` and `_test.go`
- `apps/backend/internal/task/repository/sqlite/base_migrations.go` and schema/replay fixtures
- Task repository interface and lifecycle/backendapp adapters needed by the typed record
## Dependencies
None.
## Risks
Claims require real PostgreSQL multi-connection evidence using `KANDEV_TEST_POSTGRES_DSN`; an unset variable is skipped coverage, not a pass.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/executors/requirements/kubernetes-task-pod.md).
- [System design](../../specs/executors/system-design/kubernetes-task-pod.md).
- [Plan](plan.md) regression matrix.
- Existing `executor_kubernetes_fakes_test.go`, checkpoint/restart tests, and
task environment repository tests as patterns.
## Results
Passed targeted KubernetesEnvironment tests with SQLite and disposable PostgreSQL,
including the race detector. Passed `go run ./cmd/sqlguard ./internal` and
`go test -race ./internal/persistence/storeconformance -count=1`.
Initial inventory regression failed with missing table; claim regression failed
with missing repository capability; stale-generation checkpoint failed because
it accepted an outdated owner. Each passed after its implementation.
Checks used `GOCACHE=/tmp/kandev-kube-pod-gocache-87d89c84` because unrelated
shared-cache removal interrupted the first broad compiler runs.
PostgreSQL used an isolated Docker container on loopback port 36019.
Live validation exposed the task-delete cascade preceding asynchronous cleanup.
The regression failed with a foreign-key error and now passes on SQLite and
PostgreSQL: physical inventory survives task deletion, normal launch claims
reject deleted owners, and cleanup claims retain exact generation/revision
fencing. Direct environment deletion remains guarded until remote cleanup.
## PR remediation
PR remediation adds task_environment_kubernetes to the required-table probe; SQLite conformance and previous-stable upgrade tests pass with race detection.
@@ -0,0 +1,116 @@
---
id: "02-shared-lifecycle"
title: "Shared pod lifecycle"
status: done
wave: 2
depends_on:
- "01-durable-inventory"
plan: "plan.md"
requirements:
- REQ-EXECUTORS-KUBERNETES-TASK-POD-001
acceptance_criteria:
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.1
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.2
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.3
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.4
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.5
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.6
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.7
system_design:
- ../../specs/executors/system-design/kubernetes-task-pod.md
---
# Task 02: Shared pod lifecycle
## Summary
Wire task-owned inventory into the complete Kubernetes lifecycle. Start additional
sessions as independent agentctl instances in the canonical pod and remove all
session-level authority to destroy shared compute.
## In scope
- Fresh create, attach-only admission, per-instance configuration, secret resolution, and pinned workload snapshot.
- Generation-guarded reconnect/replacement, checkpoints, partial-failure rollback, and restart refresh.
- Session-only termination, final task cleanup, zero-session retention, legacy compatibility, and authorized status projection.
## Out of scope
Cross-task pooling, automated legacy consolidation, and UI redesign.
## Acceptance
- The two-session regression fails before the change and passes with one Pod/PVC and distinct agentctl instances; concurrent attach never creates another pod.
- Stop/delete/force/failure of one session leaves a live sibling usable; task cleanup excludes new attachment and deletes only exact owned resources.
- Restart, legacy ambiguity, foreign identity, profile edits, independent runtime env, and status projection pass targeted regressions.
## Verification
Use TDD for changed logic: run the named regression before and after implementation.
Commands run from repository root. Read scoped guidance and applicable TDD/E2E
skills before implementation.
```bash
(cd apps/backend && go test -race -tags fts5 ./internal/agent/kubernetes ./internal/agent/runtime/lifecycle ./internal/task/service ./internal/kubernetes ./internal/orchestrator/executor ./internal/backendapp -run 'Kubernetes|SharedTaskPod' -count=1)
```
## Files likely touched
- `apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go`
- Kubernetes lifecycle bootstrap, identity, reconnect, cleanup, refresh, and checkpoint files
- New `apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes_task_pod_test.go`
- `apps/backend/internal/agent/kubernetes/compose.go` and ownership/admission tests
- `apps/backend/internal/task/service/service_resources.go` and Kubernetes resource cleanup tests
- `apps/backend/internal/kubernetes/sessions.go`, `sessions_retention.go`, and new `sessions_task_pod_test.go`
- `apps/backend/internal/orchestrator/executor/` and `apps/backend/internal/backendapp/` launch/cleanup adapters
## Dependencies
01-durable-inventory.
## Risks
Do not copy sibling execution metadata wholesale, rerun pod-wide bootstrap on attach, or treat force-stop as task deletion authority.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/executors/requirements/kubernetes-task-pod.md).
- [System design](../../specs/executors/system-design/kubernetes-task-pod.md).
- [Plan](plan.md) regression matrix.
- Existing `executor_kubernetes_fakes_test.go`, checkpoint/restart tests, and
task environment repository tests as patterns.
## Results
Targeted Kubernetes package tests passed with `-race -tags fts5`. Added regressions
cover independent agent environments, concurrent attachment, session-only stop,
exact task cleanup, backend/container/Pod recovery, stale-token protection,
secret/inventory/connection failure rollback, legacy conflicts and adopted-resource
protection, retained zero-session status, and executor deletion guards.
Startup fences interrupted claims under the existing exclusive runtime-state lock.
Incomplete bootstrap inventory remains available for exact cleanup and blocks
unsafe attachment. Desktop and mobile retained-row component tests passed.
Kind validation additionally exposed two integration gaps that unit fixtures did
not cover: creating private HOME before credential-less session preparation, and
preserving shared ownership/version markers through execution metadata filtering.
Both have failing-before/passing-after regressions. The final targeted lifecycle,
orchestrator executor, and status packages pass with race detection; scoped
lifecycle lint reports zero issues. The live test proves three sessions in one
Pod UID and a sibling continuing after force-stopping the creator; restart and
final cleanup subsequently passed in work order 03.
The passing Kind run exposed a transient cleanup retry after execution identity
rebinding. The final regression reproduces stopping the same remote session with
a new local execution ID and verifies immediate task cleanup. Exact task/session/
environment/remote-instance matching closes its original local connection while
leaving siblings intact. Final lifecycle race tests pass (8.512s); lint is clean.
## PR remediation
PR remediation adds typed not-found errors, encrypted credential recovery across backend restart, concurrent stop/attach coverage, and bounded cancellation-detached cleanup. Targeted race coverage passes; the user accepted shared task credential trust on 2026-09-23. Different profiles and credentials remain supported; the trust ADR and public guide describe sibling access and operator revocation responsibilities.
@@ -0,0 +1,112 @@
---
id: "03-kind-docs"
title: "Kind evidence and operator docs"
status: done
wave: 3
depends_on:
- "02-shared-lifecycle"
plan: "plan.md"
requirements:
- REQ-EXECUTORS-KUBERNETES-TASK-POD-001
acceptance_criteria:
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.1
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.2
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.3
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.4
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.5
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.6
- AC-EXECUTORS-KUBERNETES-TASK-POD-001.7
system_design:
- ../../specs/executors/system-design/kubernetes-task-pod.md
---
# Task 03: Kind evidence and operator docs
## Summary
Prove the user-visible second-session flow on disposable Kind and update the
operator lifecycle documentation after the implementation is verified.
## In scope
- Two sessions, one exact pod and managed PVC, shared untracked file, and distinct agent instances.
- Stop-one/sibling-live, backend restart/resume, concurrent second launch, and final archive cleanup.
- Foundation/ADR compatibility notes, public Kubernetes guide, backend engineering ownership guidance, and implementation result/status updates.
## Out of scope
Cross-task pooling, automated legacy consolidation, and UI redesign.
## Acceptance
- Kind evidence proves the shared lifecycle and unchanged workspace without operating on the user's cluster.
- Public docs explain task-owned resources, independent sessions, and the non-destructive legacy limitation.
- Each earlier work order records actual command results and the design matches the implemented boundaries.
## Verification
Use TDD for changed logic: run the named regression before and after implementation.
Commands run from repository root. Read scoped guidance and applicable TDD/E2E
skills before implementation.
```bash
(cd apps && pnpm install --frozen-lockfile)
(cd apps/web && KANDEV_E2E_CONTAINERS=1 pnpm e2e:run --host --project containers tests/kubernetes/kubernetes-task-pod.spec.ts)
node --test scripts/validate-public-docs.test.mjs
node scripts/validate-public-docs.mjs
python3 scripts/list-docs.py validate
python3 scripts/lint-spec-files.py --all
git diff --check
```
## Files likely touched
- New `apps/web/e2e/tests/kubernetes/kubernetes-task-pod.spec.ts`
- `apps/web/e2e/fixtures/kubernetes-test-base.ts` only if a bounded fixture extension is needed
- `docs/public/k8s.md` (reference/how-to lifecycle wording) and `docs/k8s.md` if duplicated
- `docs/specs/kubernetes-executor/spec.md`
- `docs/specs/executors/README.md` and the task-pod requirement/design pair
- `apps/backend/AGENTS.md`
- This plan and its three work orders
## Dependencies
02-shared-lifecycle.
## Risks
Requires Docker/Kind; use existing guarded host-mode runner. Do not claim skipped E2E as verified or publish draft behavior as already available.
## Parallelism
`sequential`
## Inputs
- [Requirements](../../specs/executors/requirements/kubernetes-task-pod.md).
- [System design](../../specs/executors/system-design/kubernetes-task-pod.md).
- [Plan](plan.md) regression matrix.
- Existing `executor_kubernetes_fakes_test.go`, checkpoint/restart tests, and
task environment repository tests as patterns.
## Results
Passed the disposable Kind scenario: 1 test in 6.9 minutes. Three sessions shared
one Pod UID and managed PVC; independent auth files and shared untracked data were
verified. Force-stopping the creator preserved a sibling, a new response arrived
after backend restart, status rows were healthy, and archive removed Pod/PVC.
The passing run exposed a transient cleanup retry when a resumed execution used a
new local ID. A follow-up lifecycle regression reproduced the stale connection,
and the exact-session cleanup fix passes the final Kubernetes race suite and
lint. The Kind scenario was not repeated after this narrow fix.
Eleven desktop/mobile component tests, frontend lint, 62 public-doc validator
tests, public-doc validation (47 pages), specification catalog/linter, and
`git diff --check` passed. Temporary Kind setup timeout edits were restored.
Operator docs, foundation compatibility notes, ADR, backend guidance, requirements,
and design now describe the implemented task-owned lifecycle.
## PR remediation
PR remediation caches canonical inventory and Pod probes per status request and corrects ownership guidance. Targeted status race coverage and three current frontend component tests pass. Kind evidence above remains historical; this remediation did not repeat Kind.
+6 -6
View File
@@ -21,7 +21,7 @@ An executor determines where Kandev creates a task environment and runs `agentct
| Worktree | Supported; normal default | Dedicated Git worktree on the Kandev host | Parallel coding on a trusted machine |
| Local | Supported | The selected checkout, or an explicit folder for a repository-free task | One controlled task must work in that exact folder |
| Local Docker | Supported when the global Docker runtime is enabled and its daemon is reachable | `/workspace` in a new Docker container | You need a repeatable container boundary |
| Kubernetes | Dependency-bound on cluster access, namespaced RBAC, admission, storage, and streaming support | `/workspace` in one Pod per task session | You need sessions scheduled inside an administrator-managed cluster boundary |
| Kubernetes | Dependency-bound on cluster access, namespaced RBAC, admission, storage, and streaming support | `/workspace` in one Pod per task | You need sessions scheduled inside an administrator-managed cluster boundary |
| Sprites.dev | Supported, provider-dependent | `/workspace` in a provider sandbox | You need remote compute and accept provider lifecycle/billing |
| SSH | Supported for repository sources on a trusted host | A task folder on a trusted SSH host | You need a remote host with SSH, SFTP, forwarding, and clone credentials |
| Remote Docker | **Not implemented** | None | Do not select or create this type |
@@ -317,7 +317,7 @@ docker ps -a --filter label=kandev.managed=true
> **Cluster authority:** A Kubernetes profile is an administrator-authored Pod template. It can request powerful workload settings, and every injected task credential is available to the main container. Use admission policy, a dedicated namespace, a narrowly scoped API identity, and a separate workload service account.
Kubernetes maps one task session to one namespaced Pod and reaches the injected `agentctl` through a process-local `127.0.0.1` port-forward. The backend can authenticate from an absolute kubeconfig path on the Kandev host or from its own in-cluster service account. It never falls back to a local executor when cluster configuration, admission, exec, or port-forward fails.
Kubernetes maps one task to one namespaced Pod, with an independent agentctl instance for each session, and reaches the injected `agentctl` through a process-local `127.0.0.1` port-forward. The backend can authenticate from an absolute kubeconfig path on the Kandev host or from its own in-cluster service account. It never falls back to a local executor when cluster configuration, admission, exec, or port-forward fails.
The current experimental matrix validates API and `agentctl` connectivity on Kubernetes 1.34.8 and 1.36.1 and runs the full lifecycle suite on 1.36.1. Other server versions have not yet been validated.
@@ -331,7 +331,7 @@ Each executor fixes one namespace and connection configuration. Each profile sup
| Workspace mode | Persistence and ownership |
|---|---|
| Managed PVC | Kandev creates one claim for the session, preserves it across ordinary stop, backend restart, and Pod replacement, then deletes it only during terminal or forced cleanup after exact identity checks. |
| Managed PVC | Kandev creates one claim for the task, preserves it across ordinary stop, backend restart, and Pod replacement, then deletes it only during task archive/delete cleanup after exact identity checks. |
| `emptyDir` | Fast Pod-scoped storage. It survives a main-container restart but is lost with the Pod, so a missing Pod cannot be recovered. No PVC permission is required. |
| Existing claim | Kandev verifies and mounts the named claim in the executor namespace. It never creates or deletes that claim. Concurrent-access safety depends on the claim and application. |
@@ -345,9 +345,9 @@ an immutable registry digest. The current Kind evidence covers Linux `amd64`
only. These files are profile inputs, not standalone Pod manifests; Kandev
continues to own bootstrap, credentials, runtime mounts, and the workspace.
Ordinary Stop, agent restart, main-container restart, and backend restart preserve the Pod and workspace. Resume verifies the recorded name, UID, and complete ownership-label identity, creates a new local port-forward, and reconnects. Every managed create also carries a fresh 256-bit request nonce so an ambiguous API response cannot make Kandev adopt or delete a copied-label object. Archive/delete terminal cleanup or an explicit force cleanup deletes only the exact recorded Pod and, for managed storage, the exact Kandev-created PVC. A same-name object with another UID, ownership identity, or create nonce is left untouched and cleanup fails closed.
Stop, including force-stop, removes only the selected agentctl instance. Other sessions and the task workspace remain available. Agent, main-container, and backend restarts preserve the Pod and workspace. Resume verifies the recorded name, UID, and complete ownership-label identity, creates a new local port-forward, and reconnects. Every managed create also carries a fresh 256-bit request nonce so an ambiguous API response cannot make Kandev adopt or delete a copied-label object. Task archive/delete cleanup deletes only the exact recorded Pod and, for managed storage, the exact Kandev-created PVC. A same-name object with another UID, ownership identity, or create nonce is left untouched and cleanup fails closed.
Saved executor connection settings are different from the recorded workload snapshot. Current kubeconfig/in-cluster credentials, context, and timeout are used to reach an existing session; changing them can restore or break reconnect and cleanup. Existing sessions continue to target their recorded namespace even if the saved namespace changes, and the saved namespace affects new sessions only. Current Pod template, image, platform, main container, and storage settings also affect new sessions only. If Kandev must replace a missing Pod, it uses the recorded namespace and workload snapshot rather than the edited profile.
Saved executor connection settings are different from the recorded workload snapshot. Current kubeconfig/in-cluster credentials, context, and timeout are used to reach an existing session; changing them can restore or break reconnect and cleanup. Existing sessions continue to target their recorded namespace even if the saved namespace changes, and the saved namespace affects new task Pods only. Current Pod template, image, platform, main container, and storage settings also affect new task Pods only; additional sessions reuse the task's recorded workload. If Kandev must replace a missing Pod, it uses the recorded namespace and workload snapshot rather than the edited profile.
An executor cannot be deleted or changed into or out of Kubernetes while runtime inventory still references it. Clear the sessions through normal terminal cleanup first; deleting a profile does not mutate or destroy a retained workload.
@@ -358,7 +358,7 @@ memory values are requests from the verified Pod spec, not actual usage or
cost. Stop preserves a resumable Kandev-managed workspace; Archive or Delete
can remove it. Kandev does not delete an operator-owned existing claim.
See [Kubernetes](k8s.md#configure-the-kubernetes-executor) for kubeconfig and in-cluster setup, the opt-in namespaced RBAC manifest, exact ownership labels, diagnostics, template rules, and recovery guidance.
All sessions in a task share one credential trust boundary. Same-UID agents can read sibling credentials even when their profiles differ. Operators must trust every attached agent and revoke or rotate exposed credentials after compromise. See [Kubernetes](k8s.md#configure-the-kubernetes-executor) for the trust boundary, kubeconfig and in-cluster setup, the opt-in namespaced RBAC manifest, exact ownership labels, diagnostics, template rules, and recovery guidance.
## Sprites.dev
+1 -1
View File
@@ -105,7 +105,7 @@ Released builds can lag this page. Check **Settings > System > About** or `kande
| Profile permissions, CLI flags, literal environment values, and encrypted secret references | Supported | Secret references resolve at launch; a missing/deleted reference omits that variable. Auto-approve and permission-skipping settings enlarge the process trust boundary. |
| Local and Worktree executors | Supported | Both are seeded. They execute on the Kandev host; a worktree isolates Git checkouts, not processes, network, credentials, or the rest of the filesystem. |
| Local Docker and Sprites executors | Dependency-bound | Require the relevant daemon or provider, compatible host/image, token, platform, and credential-copy configuration. |
| Kubernetes executor | Dependency-bound | One session runs in one Pod in an administrator-configured namespace on Linux `amd64` or `arm64`. Launch and recovery require kubeconfig or in-cluster authentication, namespaced Pod RBAC, exec and port-forward streaming, compatible admission and image policy, and storage permissions for the selected workspace mode. Ordinary stops preserve resources; terminal or forced cleanup is UID-, full-identity-, and create-nonce-bound. The experimental matrix validates API and `agentctl` connectivity on Kubernetes 1.34.8 and 1.36.1, with full lifecycle coverage on 1.36.1; other server versions are unvalidated. |
| Kubernetes executor | Dependency-bound | Each task uses one Pod shared by its independent sessions in an administrator-configured namespace on Linux `amd64` or `arm64`. Launch and recovery require kubeconfig or in-cluster authentication, namespaced Pod RBAC, exec and port-forward streaming, compatible admission and image policy, and storage permissions for the selected workspace mode. Session stops, including force-stop, preserve task resources; task archive/delete cleanup is UID-, full-identity-, and create-nonce-bound. The experimental matrix validates API and `agentctl` connectivity on Kubernetes 1.34.8 and 1.36.1, with full lifecycle coverage on 1.36.1; other server versions are unvalidated. |
| SSH executor | Limited | Remote sessions support trusted Linux or macOS hosts on amd64 or arm64, with public-key authentication, SFTP, and TCP forwarding. The runtime materializes the primary repository at the remote task-workspace root, reuses matching checkouts without resetting local work, runs the resolved prepare hook before `agentctl`, and runs cleanup hooks only for terminal archive/delete stops. Failed preparation blocks launch. Task directories are kept by default; a profile can opt in to **Delete a task's directory when the task is archived or deleted**, which reclaims the remote checkout only on a terminal archive, delete, or cascade delete, and only after the checkout is confirmed to hold no uncommitted changes, unpushed commits, stashes, or ignored files outside Kandev-owned runtime data, and to be shared with no other task. An ordinary stop, a backend shutdown, and a directory that fails any of those checks leave the checkout in place. Password authentication and hostile-user isolation are not provided. |
| Remote Docker executor | In progress | The type and legacy/direct settings fields are registered, but runtime create/stop return `not yet implemented`, and the current **Settings > Executors** hub does not offer it. |
| GitHub integration | Dependency-bound | Workspace repository scope, issue/PR browsing, actions, watches, presets, reviews, checks, and Gists depend on GitHub auth, scopes, rate limits, and repository permissions. Supported Local Docker, Kubernetes, Sprites, and SSH launch paths may receive the globally stored token as `GITHUB_TOKEN`/`GH_TOKEN`; workspace repository scope does not constrain its privileges. |
+21 -16
View File
@@ -17,13 +17,13 @@ Kandev ships example Kubernetes YAML in `k8s/`. The control-plane files are a si
To run task sessions as Pods, follow [Configure the Kubernetes executor](#configure-the-kubernetes-executor) after the control plane is reachable. That is a separate cluster-authority decision.
![Kubernetes placement showing one persistent Kandev control-plane Pod, its PVC and Service, optional task-session Pods, and external repositories and providers.](../screenshots/k8s.svg)
![Kubernetes placement showing one persistent Kandev control-plane Pod, its PVC and Service, optional task Pods, and external repositories and providers.](../screenshots/k8s.svg)
[Open full-size SVG diagram][k8s-diagram]
[k8s-diagram]: ../../docs/screenshots/k8s.svg
The control-plane Pod owns the persistent home and API identity. Task-session Pods are a separate, opt-in workload path with their own service-account permissions.
The control-plane Pod owns the persistent home and API identity. Task Pods are a separate, opt-in workload path with their own service-account permissions.
## Architecture and limitations
@@ -36,7 +36,7 @@ The example creates:
| `k8s/deployment.yaml` | `Deployment/kandev` | one replica, `Recreate`, example resources and probes |
| `k8s/service.yaml` | `Service/kandev` | `ClusterIP`, TCP 38429 |
| `k8s/ingress.yaml` | `Ingress/kandev` | ingress-nginx-oriented example for `kandev.example.com` |
| `k8s/executor-rbac.yaml` | `ServiceAccount`, `Role`, and `RoleBinding` | Opt-in access for task-session Pods in `kandev-agents`; not part of the deployment quick path |
| `k8s/executor-rbac.yaml` | `ServiceAccount`, `Role`, and `RoleBinding` | Opt-in access for task Pods in `kandev-agents`; not part of the deployment quick path |
One pod serves the SPA, API, WebSocket, external MCP endpoint, and `/health` on port 38429. With SQLite, the same PVC holds the database, workspaces, CLI installs, and authentication files.
@@ -92,7 +92,9 @@ Open `http://localhost:38429`.
## Configure the Kubernetes executor
The Kubernetes executor is independent of where the Kandev control plane runs. A host process can use a kubeconfig, while a Kandev Pod can use either a mounted kubeconfig or its in-cluster service account. One executor connects to one API server context and one namespace; every selected task session gets one Pod in that namespace.
The Kubernetes executor is independent of where the Kandev control plane runs. A host process can use a kubeconfig, while a Kandev Pod can use either a mounted kubeconfig or its in-cluster service account. One executor connects to one API server context and one namespace; each task gets one Pod in that namespace. Additional sessions on the same task share that Pod and workspace, with independent agentctl instances and session configuration.
**A task is one credential trust boundary.** All agents in its Pod run as the same OS user and can read sibling credentials, including credentials from different agent profiles. Separate session HOME directories and configuration prevent accidental mixing; they do not isolate secrets. Attach only mutually trusted agents and credentials. If any sibling is compromised, stop the task's agents and revoke or rotate exposed credentials with their providers. Stopping a session alone does not revoke credentials or undo exposure. Use separate tasks with appropriately isolated executor policies for agents that must not share trust.
Only administrators can create, edit, delete, or test Kubernetes executors and profiles. Members can list and use administrator-configured profiles, resume their authorized sessions, and view the sanitized status rows available to them. When Kandev authentication is disabled, requests use the synthetic single-user administrator identity.
@@ -241,12 +243,13 @@ Every Kandev-created Pod and managed PVC has this complete custom identity:
|---|---|
| `kandev.ai/executor-id` | Executor record ID |
| `kandev.ai/profile-id` | Executor-profile record ID |
| `kandev.ai/instance-id` | Runtime instance ID |
| `kandev.ai/instance-id` | Stable task-environment resource instance ID |
| `kandev.ai/task-id` | Task ID |
| `kandev.ai/session-id` | Task-session ID |
| `kandev.ai/session-id` | Original creating session ID, retained as an immutable provenance label |
| `kandev.ai/environment-id` | Task-environment ID |
| `kandev.ai/ownership-version` | `task-v1` for new task-owned resources |
It also owns `app.kubernetes.io/name=kandev-agent`, `app.kubernetes.io/component=agent-session`, `app.kubernetes.io/managed-by=kandev`, and `app.kubernetes.io/instance=<runtime-instance-id>`. All `kandev.ai/*` label and annotation keys are reserved, including keys not listed above. Kandev rejects an admission response that changes or adds one of those reserved identities.
It also owns `app.kubernetes.io/name=kandev-agent`, `app.kubernetes.io/component=task-environment`, `app.kubernetes.io/managed-by=kandev`, and `app.kubernetes.io/instance=<task-environment-resource-instance-id>`. All `kandev.ai/*` label and annotation keys are reserved, including keys not listed above. Kandev rejects an admission response that changes or adds one of those reserved identities.
Every Kandev-created Pod and managed PVC also receives a fresh 256-bit `kandev.ai/create-nonce` annotation for that individual create request. If a create response is lost or ambiguous, Kandev adopts, checkpoints, bootstraps, or deletes the resulting object only when the API object preserves that exact nonce and the complete recorded identity.
@@ -337,7 +340,7 @@ before use. No production deployment or universal cgroup compatibility is implie
| Mode | Profile fields | Lifecycle |
|---|---|---|
| `managed_pvc` | Positive Kubernetes quantity, optional StorageClass, and one or more access modes. Defaults are `10Gi` and `ReadWriteOnce`. | One filesystem PVC per session, without a Pod owner reference. It survives ordinary stop, backend restart, main-container restart, and Pod replacement. Terminal/forced cleanup deletes it only after exact identity verification. |
| `managed_pvc` | Positive Kubernetes quantity, optional StorageClass, and one or more access modes. Defaults are `10Gi` and `ReadWriteOnce`. | One filesystem PVC per task, without a Pod owner reference. It survives ordinary stop, backend restart, main-container restart, and Pod replacement. Task archive/delete cleanup deletes it only after exact identity verification. |
| `empty_dir` | No PVC fields. | Pod-scoped storage. It survives a container restart but disappears with the Pod. A missing Pod makes the workspace unrecoverable. |
| `existing_claim` | Exact claim name in the executor namespace. | Kandev gets and mounts the claim but never creates, relabels, or deletes it. Data retention, access modes, concurrent mounts, and backup remain the operator's responsibility. |
@@ -345,16 +348,16 @@ A managed claim may remain Pending because of StorageClass, topology, quota, acc
### Recovery, snapshots, and cleanup
Kandev's `executors_running` inventory is authoritative. It records exact Pod and PVC namespace/name/UID, full resource identity, workspace ownership, platform, main container, remote port, hashes, and the validated workload launch snapshot. The snapshot contains Pod template, platform, main container, and storage configuration only. It excludes Kandev-resolved credentials, resolved profile environment values, injected files, and scripts; literal values written by an administrator into the Pod template remain in the snapshot.
Kandev's `task_environment_kubernetes` inventory is authoritative for task-owned physical resources; `executors_running` records per-session executions and legacy resources. The physical inventory records exact Pod and PVC namespace/name/UID, full resource identity, workspace ownership, platform, main container, remote port, hashes, and the validated workload launch snapshot. The snapshot contains Pod template, platform, main container, and storage configuration only. It excludes Kandev-resolved credentials, resolved profile environment values, injected files, and scripts; literal values written by an administrator into the Pod template remain in the snapshot.
Current connection configuration and recorded workload configuration serve different purposes:
- Saved kubeconfig/in-cluster mode, kubeconfig path/context, and timeout are used for later status, reconnect, replacement, and cleanup. Changing credentials or context can restore or break access to an existing session. Retained sessions continue to target their recorded namespace; changing the saved namespace affects new sessions only.
- Saved workload profile edits affect new sessions. An existing Pod is never live-mutated, and a missing-Pod replacement uses the recorded workload snapshot rather than the profile's current image, template, platform, main container, or storage fields.
- Saved kubeconfig/in-cluster mode, kubeconfig path/context, and timeout are used for later status, reconnect, replacement, and cleanup. Changing credentials or context can restore or break access to an existing session. Retained sessions continue to target their recorded namespace; changing the saved namespace affects new task Pods only.
- Saved workload profile edits affect newly provisioned task Pods. Additional sessions reuse the existing task workload snapshot. An existing Pod is never live-mutated, and a missing-Pod replacement uses the recorded workload snapshot rather than the profile's current image, template, platform, main container, or storage fields.
On resume, Kandev reloads environment definitions from the recorded executor profile and resolves secret references at the launch checkpoint. Changes to that profile's environment can therefore affect a resumed process; they do not rewrite the Pod or storage snapshot. If the profile has been deleted, its environment definitions are unavailable. Profile lookup failures and executor ownership mismatches block resume.
Ordinary Stop and backend shutdown close local clients and forwards but preserve the Pod and workspace. Agent or main-container restart keeps the Pod volumes; Kandev performs a new nonce handshake and local port-forward. If a Pod disappears, managed or existing PVC storage can support a replacement Pod after identity checks; `emptyDir` cannot.
Stop, including force-stop, terminates only the selected session’s agentctl instance and closes its local clients and forwards. Sibling sessions, the Pod, and the workspace remain available. Backend shutdown closes local connections and preserves the remote resources. Agent or main-container restart keeps the Pod volumes; Kandev performs a new nonce handshake and local port-forward. If a Pod disappears, managed or existing PVC storage can support a replacement Pod after identity checks; `emptyDir` cannot.
A recoverable agent error also preserves the established Pod, workspace, and
recovery credentials. Use **Resume** to continue that session. If its retained
@@ -362,17 +365,19 @@ credentials or storage are missing, recovery fails rather than silently creating
an empty replacement workspace. A software update cannot restore resources that
were already deleted.
Archive/delete terminal cleanup and explicit force cleanup are destructive. Before deletion, Kandev verifies the recorded namespace, name, UID, and complete standard plus `kandev.ai/*` ownership-label set. It then deletes the exact Pod with UID/resource-version preconditions and deletes a PVC only when inventory proves Kandev created that managed claim. Existing claims are never deleted. A missing object is idempotent; an inventory read error, same-name replacement, UID mismatch, missing label, changed label, extra `kandev.ai/*` label, or mismatched create nonce fails closed without deleting the ambiguous object.
Task archive/delete cleanup is destructive after all attached sessions are settled. Deleting or stopping an individual session does not delete the task Pod or its managed claim. Before deletion, Kandev verifies the recorded namespace, name, UID, and complete standard plus `kandev.ai/*` ownership-label set. It then deletes the exact Pod with UID/resource-version preconditions and deletes a PVC only when inventory proves Kandev created that managed claim. Existing claims are never deleted. A missing object is idempotent; an inventory read error, same-name replacement, UID mismatch, missing label, changed label, extra `kandev.ai/*` label, or mismatched create nonce fails closed without deleting the ambiguous object.
Kandev blocks deleting an executor, or changing an executor into or out of Kubernetes, while runtime inventory still refers to it. Finish normal session cleanup first. Profile deletion does not rewrite or destroy a retained workload because recovery owns the recorded profile ID and snapshot.
Kandev blocks deleting an executor, or changing an executor into or out of Kubernetes, while runtime inventory still refers to it. Archive or delete the owning tasks to finish resource cleanup first. Removing the last session alone retains the task Pod. Profile deletion does not rewrite or destroy a retained workload because recovery owns the recorded profile ID and snapshot.
Do not manually delete by name or label alone. If manual incident cleanup is unavoidable, compare the Kandev runtime inventory with namespace, name, UID, and all ownership labels first, preserve required workspace data, and record the out-of-band action.
Older installations may have several session Pods for one task. Kandev does not merge or delete those Pods automatically. Additional attachment requires an unambiguous recorded Pod and claim with matching task ownership and recoverable credentials; conflicting inventory blocks attachment. Adopted legacy resources keep their original labels.
### Read retained resource status
The **Active sessions** card shows the sanitized session inventory for the
selected executor. It includes sessions that Kandev can resume after ordinary
Stop, not only sessions whose agent process is currently running. The card
Stop, not only sessions whose agent process is currently running. A task whose last session has been removed remains listed with its retained Pod and no session ID. Multiple sessions on a task show the same current Pod identity. The card
shows separate session state, Pod state, retention state, workspace mode, and
main-container CPU and memory requests.
@@ -511,7 +516,7 @@ Use the SSL mode and trust material required by your database. PostgreSQL moves
> **Docker boundary:** Do not add only a Docker socket mount. The runtime also needs helper, credential-session, and local-clone bind sources at matching daemon-host paths. Privileged Docker-in-Docker has a separate security and persistence model and no supplied manifest.
The [Kubernetes executor](#configure-the-kubernetes-executor) creates separate task-session Pods through the Kubernetes API and does not require a Docker socket. The Local and Worktree behavior below instead runs agent processes inside the control-plane Pod itself.
The [Kubernetes executor](#configure-the-kubernetes-executor) creates separate task Pods through the Kubernetes API and does not require a Docker socket. The Local and Worktree behavior below instead runs agent processes inside the control-plane Pod itself.
<details>
<summary>Agent execution, resources, and probes</summary>
+5 -3
View File
@@ -75,9 +75,11 @@ canonical requirement and system-design documents. Use the catalog command to
find current sources.
The [Kubernetes executor foundation](../kubernetes-executor/spec.md) remains
the current lifecycle contract. The three Kubernetes pairs own additive
presets, launch diagnostics, and retained-compute visibility. They do not
replace the foundation's resource ownership, recovery, or cleanup rules.
the shipped lifecycle contract. Kubernetes capability pairs extend it. The
[task-pod requirements](requirements/kubernetes-task-pod.md) and
[design](system-design/kubernetes-task-pod.md) define task-owned compute and
replace session ownership. Other foundation
security, admission, and recovery guarantees remain applicable.
## Related systems
@@ -0,0 +1,74 @@
---
status: active
system: executors
created: 2026-09-21
owners:
- kandev
---
# Kubernetes task pod requirements
## Overview
A task's additional Kubernetes sessions share its pod and workspace while
retaining independent agent conversations. Executors owns this contract because
it governs remote compute ownership. The task system continues to own canonical
workspace selection and session identity.
This contract replaces the session-per-pod rule and concurrent-sharing
exclusion in the [foundation](../../kubernetes-executor/spec.md).
It implements the user's explicit one-pod-per-task expectation.
## Requirements
### REQ-EXECUTORS-KUBERNETES-TASK-POD-001: Task-owned Kubernetes compute
**Intent:** Starting another agent session must not allocate another task pod.
#### Acceptance criteria
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.1:** When an additional session starts in a task with a ready
Kubernetes environment, it shall use the same pod UID and workspace, with an
independent agent instance and conversation. Session attachment shall preserve
tracked and untracked files without rerunning workspace preparation.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.2:** When sessions start concurrently, they shall not allocate
duplicate task pods or managed workspace claims. An environment still being
prepared shall return the existing recoverable preparation error.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.3:** Stopping, deleting, or failing one session shall not interrupt
a sibling's agent, pod, or workspace. A task with no remaining sessions shall
retain its environment until task-level cleanup authorizes removal.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.4:** Backend restart and session resume shall recover the shared
recorded pod. Missing-pod recovery shall create at most one replacement for
the task, preserve managed workspace data, and retain separate session state.
Loss of ephemeral workspace data shall follow the existing explicit recovery
failure contract, never silently become a fresh workspace.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.5:** Final task resource cleanup shall remove only the exactly
recorded owned pod and managed claim, after preventing concurrent attachment.
Existing administrator-owned claims shall never be deleted.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.6:** Incompatible executor selection, ambiguous legacy pod inventory,
or ownership mismatch shall fail attachment with a recoverable error without
creating another pod, adopting foreign resources, or deleting existing data.
Existing legacy sessions shall remain resumable under their recorded identity;
independent legacy workspaces shall not be merged automatically.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.7:** Each authorized session's Kubernetes status shall identify its
actual shared pod on desktop and mobile. Profile edits shall not mutate the
retained task's workload; additional sessions shall use its recorded workload
configuration while keeping independent agent-profile runtime settings.
- **AC-EXECUTORS-KUBERNETES-TASK-POD-001.8:** The task shall be one credential trust boundary.
Sessions may use different agent profiles and credentials in the same pod.
Per-session settings shall prevent accidental credential inheritance or overwrite,
but shall not promise protection from sibling processes running as the same OS user.
Operators shall trust every agent attached to a task and manage provider-side
credential revocation after suspected exposure.
## Related contracts
- [Additional-session workspace reuse](../../tasks/requirements/additional-session-workspace-reuse.md).
- [Kubernetes retained compute](kubernetes-retained-compute.md).
## Out of scope
Cross-task pod pooling, changing workspace inheritance/group semantics,
automatically consolidating legacy pods, concurrent-edit conflict resolution,
and redesigning executor UI controls.
@@ -0,0 +1,183 @@
---
status: current
system: executors
requirements:
- REQ-EXECUTORS-KUBERNETES-TASK-POD-001
---
# Kubernetes task pod system design
## Boundary and evidence
The executor system owns physical Kubernetes compute; task environments own
canonical workspace identity. This design implements
[task pod requirements](../requirements/kubernetes-task-pod.md).
Current `KubernetesExecutor.CreateInstance` locks by `req.InstanceID`, takes the
reconnect branch only with recorded pod metadata, and otherwise calls
`createFresh`. `newKubernetesFreshLaunch` derives pod/PVC names from that instance
ID. `kubernetesIdentity` includes the session ID. Consequently distinct session
executions allocate distinct pods even when their task/environment IDs match.
`StopInstance` can delete those resources on force/terminal cleanup.
`connectNewAgentctl` already uses the agentctl control server's `CreateInstance`
API. Reuse that multi-instance boundary instead of launching another agentctl
server or copying a sibling's execution identity.
## Requirement mapping
| Requirement | Implementation boundary |
| --- | --- |
| `REQ-EXECUTORS-KUBERNETES-TASK-POD-001` | Durable ownership, attach/recovery, cleanup, and projection below |
## Durable ownership
Add a typed Kubernetes runtime record associated with the canonical
`TaskEnvironment`, separate from `executors_running`. The latter remains the
per-session execution inventory; it must not be the sole physical resource owner.
Use a dedicated `task_environment_kubernetes` table with environment ID as its
primary key and task ID constrained to one canonical Kubernetes runtime record.
Do not overload Docker `ContainerID` or its control-secret fields.
Record executor ID, pinned workload profile/snapshot, environment ownership
generation, immutable resource identity, namespace, Pod/PVC names and UIDs,
managed-claim ownership, admission checkpoints, create nonce continuity, container
restart observation, and encrypted control-token/bootstrap-nonce secret references.
No plaintext credential enters this record or a public DTO. Environment deletion
must not cascade this inventory before remote cleanup succeeds. The inventory
has no cascading environment foreign key: task deletion removes task/environment
rows synchronously and durable cleanup runs afterward. Direct environment
deletion is guarded while inventory exists. A separate cleanup-only claim can
use the retained owner/generation only when both original owner rows are absent;
normal launch claims still require a live matching environment.
Keep resource ownership separate from session identity. New shared resources use
an environment-derived stable resource instance identity and a versioned ownership
label contract; their authorization must not depend on the current attaching
session. Preserve exact legacy ownership labels for adopted legacy resources.
Persist the ownership version and validate its complete expected label set plus
UIDs. Never relax exact UID checks or create-nonce reconciliation.
Repository operations claim creation/recovery/deletion with conditional writes
against generation and operation identity. Creation and deletion claims are
mutually exclusive with attachment. PostgreSQL uses the repository's advisory
transaction lock convention; SQLite uses its existing write serialization.
Do not hold a database transaction during Kubernetes network calls. Persist each
checkpoint before allowing another operation to advance; after restart reconcile
the recorded claim and exact create result before retrying allocation. Startup
fences interrupted operation revisions under the backend’s existing exclusive
runtime-state ownership lock before any worker starts. Incomplete bootstrap
inventory without recoverable control credentials blocks attachment and remains
available for exact task cleanup.
## Initial launch and additional attachment
The canonical environment materializer creates and bootstraps the pod once,
persists its control credentials and resource inventory, prepares repositories,
and publishes ready only after workspace inventory is complete.
An additional launch validates task authorization, canonical environment,
executor identity, generation and complete repository inventory. Read only the
canonical runtime record; a sibling `PreviousExecutionID` is never an attachment
handle. Reject a different executor/profile selection that requires changing the
retained workload. Editing the same profile does not replace its pinned snapshot.
Verify the recorded pod, connect to its control server using environment-scoped
credentials, and create a new agentctl instance with the new execution/session
IDs and canonical workspace. Retain per-instance port-forward/client lifetime.
Resolve each agent profile's environment/auth/MCP configuration for that instance;
attachment must not rewrite pod-wide runtime/auth files or restart the control
server. Audit `buildReconnectCreateInstanceRequest` and bootstrap-file consumers
so a new instance does not accidentally inherit another session's agent identity
or credentials.
Attach-only launch skips cloning, repository setup, bootstrap, and pod-level
prepare scripts. Preserve current task workspace inheritance admission; this
change does not grant cross-task compute sharing when a parent/group environment
cannot safely satisfy the task-owned pod boundary. Return typed unsupported reuse
rather than silently provisioning a separate workspace.
A consumed bootstrap handshake retains its issued token until encrypted canonical
storage acknowledges it. If that update fails, an environment-derived encrypted
recovery secret preserves the token across backend restart; attachment retries
finish the canonical save and remove the recovery secret before proceeding.
Request cancellation does not cancel the bounded credential persistence attempt.
Task teardown also removes the recovery secret.
## Credential trust boundary
The task is one trust boundary, as recorded in the
[credential trust ADR](../../../decisions/2026-09-23-kubernetes-task-trust-boundary.md).
Different agent profiles and credential bindings are allowed within its pod.
Per-session HOME, environment and auth files separate configuration; all agents
run as the same OS user and can read sibling files and process credentials.
File permissions and hashed session paths do not provide security isolation.
Operators must attach only mutually trusted agents and credentials. After a
suspected compromise, stop the task's agents and revoke or rotate exposed
credentials with their providers; stopping one session cannot undo exposure.
Use separate tasks with appropriately isolated executor policies when agents
must not share trust. Regression coverage verifies distinct profiles and
credentials use one pod without overwriting each other's launch settings.
## Session termination and task cleanup
Session stop/delete terminates only that agentctl instance and its local
connections. Force is not authority to delete the task's resources. Failed
attachment rolls back only resources allocated by that attachment.
Task archive/delete resource cleanup claims the environment for deletion, blocks
new attachment, settles all attached executions through the existing task cleanup
orchestrator, and verifies that no execution still needs the resource. One stopped
or failed row is insufficient when any sibling is live. Run cleanup scripts from task teardown rather than individual session stops,
then perform exact-UID/ownership-checked Pod and managed-PVC deletion.
Retain inventory and credentials for retry on partial deletion failure. External
claims are never deleted. Zero-session environments remain discoverable for
retention/status and terminal cleanup.
## Recovery and compatibility
Backend restart reconstructs environment control connections independently of
session connections. A single generation claim owns missing-pod replacement from
the pinned launch snapshot and existing managed PVC. Other resumes wait or return
a recoverable preparing result; they cannot provision replacements independently.
A pod/container restart invalidates all affected per-instance clients; refresh
projects the same physical generation while each session reconnects independently.
Migration is additive and non-destructive. At first use, adopt legacy inventory
only when all relevant retained rows agree on one exact pod/PVC identity, matching
canonical task/environment ownership, and available encrypted control credentials.
Preserve legacy resource labels and record the ownership version. Multiple pods,
conflicting snapshots, missing identity, or unreadable inventory block additional
attachment. Existing recorded legacy sessions retain their current resume/cleanup
path; do not rewrite their files, move claims, or delete a supposed duplicate.
An explicitly selected executor transition must use the existing task environment
transition contract and cannot attach to stale Kubernetes state. This work does
not add automatic executor migration.
## Status and diagnostics
`internal/kubernetes/sessions.go` keeps authorized session rows but resolves shared
physical identity through their canonical runtime record. Probe each exact pod
once per request and project its result to authorized rows; session counts remain
session counts. Retained environment visibility must survive removal of its last
session. Never expose control secrets. Existing desktop/mobile components consume
the same response shape, so this package introduces no layout or control changes.
Log task/environment and execution identifiers for attach/create/recover/delete
operations and bounded failure categories, without credential payloads.
## Verification
Use fake Kubernetes clients through real lifecycle entry points for two-session
attachment, concurrent launch/recovery, stop-one/sibling-live, forced session
cleanup, failed attachment rollback, task cleanup, retained zero-session state,
profile snapshot pinning, per-instance env isolation, and foreign UID rejection.
Repository tests cover claim exclusion, replay, and mixed legacy inventories.
Kind E2E starts two real sessions, observes one Pod/PVC and distinct instances,
checks an untracked file from both, stops one, restarts the backend, resumes, then
archives the task and checks cleanup. No production cluster reproduction is needed.
## Related decisions
[Task-owned Kubernetes compute](../../../decisions/2026-09-21-kubernetes-task-pod-ownership.md).
+12 -6
View File
@@ -7,6 +7,12 @@ owner: Kandev
# Kubernetes Executor
## Ownership transition
This file records the Kubernetes foundation. The
[task-pod contract](../executors/requirements/kubernetes-task-pod.md) and
[design](../executors/system-design/kubernetes-task-pod.md) own the task-shared Pod lifecycle; the remaining foundation stays in force.
## Why
Teams that already operate Kubernetes need Kandev sessions to run inside their
@@ -25,7 +31,7 @@ first-class Pod with an administrator-reviewed workload template.
- Administrators can create Kubernetes profiles from one strict
`core/v1 PodTemplate` YAML document. The profile chooses the main container,
Linux architecture, and workspace storage policy.
- One task session maps to one Pod. Kandev starts its existing `agentctl`
- One task maps to one Pod shared by its independent sessions. Kandev starts its existing `agentctl`
control process in the designated container and reaches it through a
loopback-only Kubernetes port-forward.
- The template can define images, sidecars, init containers, resource and
@@ -36,11 +42,11 @@ first-class Pod with an administrator-reviewed workload template.
`kandev-workspace` mount at exactly `/workspace`; admission must preserve
its recipient and shape. Runtime/auth mounts and init/ephemeral containers
remain excluded. See [workspace grants](../executors/system-design/kubernetes-docker-workloads.md).
- Profiles support a Kandev-managed per-session PVC, a Pod-scoped `emptyDir`,
- Profiles support a Kandev-managed per-task PVC, a Pod-scoped `emptyDir`,
or an existing namespaced PVC. Kandev never deletes an existing PVC.
- Ordinary Stop and backend shutdown preserve the Pod and workspace for
resume. Archive, delete, and explicit force cleanup remove only resources
proven to belong to the recorded session.
resume. Session stop/delete removes only its agentctl instance. Task archive/delete
removes only resources proven to belong to the canonical task environment.
- Members can discover and use administrator-configured Kubernetes profiles.
They cannot create, edit, delete, or test Kubernetes executor/profile
configuration.
@@ -193,7 +199,7 @@ or cluster-resource identity.
agent subprocess.
6. **Preserved:** ordinary stop or backend shutdown closes local forwarding but
retains Pod/workspace for resume.
7. **Cleaning:** terminal/forced cleanup validates exact identities, deletes
7. **Cleaning:** task archive/delete cleanup validates exact identities, deletes
the Pod, then deletes only a Kandev-created PVC.
8. **Cleaned:** resources are absent and runtime inventory can be removed.
@@ -356,7 +362,7 @@ identity check or inventory lookup stops cleanup without deleting anything.
sessions, autoscaling, and pooled/multi-cluster scheduling.
- Cross-namespace profiles inside one executor.
- Windows Pods and automatic node-architecture discovery.
- Sharing one Pod or one managed PVC among concurrent sessions.
- Sharing one Pod or one managed PVC among different tasks.
- Live mutation of running Pods after profile edits.
- Raw kubeconfig storage in executor JSON or the current user-scoped secret
store.