kandev/apps/backend/internal/agent/runtime/lifecycle/executor_kubernetes.go

package lifecycle

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	corev1 "k8s.io/api/core/v1"

	"github.com/kandev/kandev/internal/agent/executor"
	kubeexecutor "github.com/kandev/kandev/internal/agent/kubernetes"
	agentctl "github.com/kandev/kandev/internal/agent/runtime/agentctl"
	"github.com/kandev/kandev/internal/agentctl/server/process"
	"github.com/kandev/kandev/internal/common/logger"
	"github.com/kandev/kandev/internal/secrets"
	"github.com/kandev/kandev/internal/task/models"
)

const (
	kubernetesWorkspacePath       = "/workspace"
	kubernetesAgentctlPath        = "/opt/kandev/agentctl"
	kubernetesRuntimeEnvPath      = "/opt/kandev/runtime.env"
	kubernetesAuthEnvPath         = "/run/kandev/auth.env"
	kubernetesAuthHomePath        = "/run/kandev/home"
	kubernetesPreparePath         = "/opt/kandev/prepare.sh"
	kubernetesStartPath           = "/opt/kandev/start"
	kubernetesControlHost         = "127.0.0.1"
	kubernetesEnvInstanceID       = "KANDEV_INSTANCE_ID"
	kubernetesEnvSessionID        = "KANDEV_SESSION_ID"
	kubernetesEnvTaskID           = "KANDEV_TASK_ID"
	kubernetesEnvEnvironmentID    = "KANDEV_TASK_ENVIRONMENT_ID"
	kubernetesEnvAgentProfile     = "KANDEV_AGENT_PROFILE_ID"
	kubernetesEnvExecutionProfile = "KANDEV_EXECUTION_PROFILE_ID"
	kubernetesRepositoryCloneURL  = "repository.clone_url"
)

var errKubernetesLifecycleRequestIncomplete = errors.New("kubernetes lifecycle request is incomplete")

// KubernetesExecutor owns Kubernetes Pod/PVC lifecycle and process-local
// forwards. Cluster clients are initialized lazily from the selected executor.
type KubernetesExecutor struct {
	environmentStore  KubernetesEnvironmentStore
	secretStore       secrets.SecretStore
	agentctlResolver  *AgentctlResolver
	logger            *logger.Logger
	clientFactory     kubernetesRuntimeClientFactory
	resolveBinary     kubernetesAgentctlBinaryResolver
	healthRetryDelay  time.Duration
	launchTimingClock func() time.Time

	mu                   sync.Mutex
	sessions             map[string]*kubernetesSession
	locks                map[string]*kubernetesInstanceLock
	pendingControlTokens map[string]string
}

type kubernetesInstanceLock struct {
	mutex sync.Mutex
	refs  int
}

type kubernetesSession struct {
	runtime      *kubernetesRuntimeClient
	forward      kubeexecutor.PortForwardSession
	client       *agentctl.Client
	request      *ExecutorCreateRequest
	restartCount int32
}

const (
	KubernetesInventoryStatePVCCreated  = "pvc_created"
	KubernetesInventoryStatePVCAdmitted = "pvc_admitted"
	KubernetesInventoryStatePodCreated  = "pod_created"
	KubernetesInventoryStatePodAdmitted = "pod_admitted"
	KubernetesInventoryStateReady       = "ready"
)

func NewKubernetesExecutor(agentctlResolver *AgentctlResolver, log *logger.Logger) *KubernetesExecutor {
	runtime := &KubernetesExecutor{
		agentctlResolver:  agentctlResolver,
		logger:            log,
		clientFactory:     newKubernetesRuntimeClient,
		healthRetryDelay:  agentctlHealthRetryDelay,
		launchTimingClock: time.Now,
		sessions:          make(map[string]*kubernetesSession),
		locks:             make(map[string]*kubernetesInstanceLock),
	}
	if agentctlResolver != nil {
		runtime.resolveBinary = func(platform kubeexecutor.Platform) ([]byte, error) {
			arch := strings.TrimPrefix(string(platform), "linux/")
			path, err := agentctlResolver.ResolveRemoteBinary(SSHRemotePlatform{GOOS: "linux", GOARCH: arch})
			if err != nil {
				return nil, err
			}
			return os.ReadFile(path)
		}
	}
	return runtime
}

func (r *KubernetesExecutor) Name() executor.Name { return executor.NameKubernetes }

func (r *KubernetesExecutor) HealthCheck(context.Context) error { return nil }

func (r *KubernetesExecutor) CreateInstance(ctx context.Context, req *ExecutorCreateRequest) (*ExecutorInstance, error) {
	if req != nil && r.environmentStore != nil && req.TaskEnvironmentID != "" {
		return r.createTaskInstance(ctx, req)
	}
	return r.createSessionInstance(ctx, req)
}

func (r *KubernetesExecutor) createSessionInstance(ctx context.Context, req *ExecutorCreateRequest) (*ExecutorInstance, error) {
	if req == nil || req.InstanceID == "" {
		return nil, errKubernetesLifecycleRequestIncomplete
	}
	unlock := r.lockInstance(req.InstanceID)
	defer unlock()

	executorConfig, err := kubernetesExecutorConfigFromMetadata(req.Metadata)
	if err != nil {
		return nil, err
	}
	reconnect := getMetadataString(req.Metadata, MetadataKeyKubernetesPodName) != ""
	var profile kubeexecutor.ProfileConfig
	if !reconnect {
		profile, err = kubernetesProfileConfigFromMetadata(req.Metadata)
		if err != nil {
			return nil, err
		}
	}
	if r.clientFactory == nil {
		return nil, fmt.Errorf("kubernetes lifecycle: client factory is not configured")
	}
	runtime, err := r.clientFactory(executorConfig)
	if err != nil {
		return nil, fmt.Errorf("kubernetes lifecycle: create client: %w", err)
	}
	if runtime == nil || runtime.resources == nil || runtime.streams == nil {
		return nil, fmt.Errorf("kubernetes lifecycle: client is incomplete")
	}

	if reconnect {
		return r.reconnect(ctx, runtime, req, executorConfig)
	}
	if r.resolveBinary == nil {
		return nil, fmt.Errorf("kubernetes lifecycle: agentctl resolver is not configured")
	}
	return r.createFresh(ctx, runtime, req, executorConfig, profile)
}

func (r *KubernetesExecutor) createFresh(
	ctx context.Context,
	runtime *kubernetesRuntimeClient,
	req *ExecutorCreateRequest,
	executorConfig kubeexecutor.ExecutorConfig,
	profile kubeexecutor.ProfileConfig,
) (_ *ExecutorInstance, returnedErr error) {
	timing := newKubernetesLaunchTimingWithClock(r.logger, req, r.launchTimingClock)
	defer func() { timing.complete(returnedErr) }()

	launch, err := newKubernetesFreshLaunch(r, runtime, req, executorConfig, profile)
	if err != nil {
		timing.failureSite = "initialize"
		return nil, err
	}
	defer func() {
		returnedErr = launch.rollbackAfterFailure(ctx, returnedErr)
	}()
	if err := timing.runStage("storage", func() error {
		return launch.provisionWorkspace(ctx)
	}); err != nil {
		return nil, err
	}
	var runningPod *corev1.Pod
	if err := timing.runStage("pod_ready", func() error {
		var stageErr error
		runningPod, stageErr = launch.createRunningPod(ctx)
		return stageErr
	}); err != nil {
		return nil, err
	}
	var nonce string
	var binary []byte
	if err := timing.runStage("bootstrap", func() error {
		var stageErr error
		nonce, stageErr = generateBootstrapNonce()
		if stageErr != nil {
			return stageErr
		}
		binary, stageErr = r.resolveBinary(profile.Platform)
		if stageErr != nil {
			return fmt.Errorf("kubernetes lifecycle: resolve agentctl for %s: %w", profile.Platform, stageErr)
		}
		return r.bootstrapPod(ctx, runtime, req, runningPod, profile, nonce, binary)
	}); err != nil {
		return nil, err
	}
	var client *agentctl.Client
	var finalForward kubeexecutor.PortForwardSession
	var token string
	var remotePort int
	if err := timing.runStage("agentctl_connect", func() error {
		var stageErr error
		client, finalForward, token, remotePort, stageErr = r.connectNewAgentctl(
			ctx, runtime, req, runningPod, nonce,
		)
		return stageErr
	}); err != nil {
		return nil, err
	}
	instance, err := launch.complete(ctx, runningPod, client, finalForward, token, nonce, remotePort)
	if err != nil {
		timing.failureSite = "finalize"
	}
	return instance, err
}

type kubernetesFreshLaunch struct {
	executor       *KubernetesExecutor
	runtime        *kubernetesRuntimeClient
	req            *ExecutorCreateRequest
	executorConfig kubeexecutor.ExecutorConfig
	profile        kubeexecutor.ProfileConfig
	identity       kubeexecutor.ResourceIdentity
	podName        string
	pvcName        string
	workspace      kubernetesWorkspaceProvision
	createdPod     *corev1.Pod
}

func newKubernetesFreshLaunch(
	executor *KubernetesExecutor,
	runtime *kubernetesRuntimeClient,
	req *ExecutorCreateRequest,
	executorConfig kubeexecutor.ExecutorConfig,
	profile kubeexecutor.ProfileConfig,
) (*kubernetesFreshLaunch, error) {
	identity, err := kubernetesIdentity(req)
	if err != nil {
		return nil, err
	}
	podName, pvcName := kubernetesResourceNames(identity.InstanceID)
	return &kubernetesFreshLaunch{
		executor: executor, runtime: runtime, req: req, executorConfig: executorConfig,
		profile: profile, identity: identity, podName: podName, pvcName: pvcName,
	}, nil
}

func (l *kubernetesFreshLaunch) checkpoint(
	ctx context.Context,
	state string,
	pod *corev1.Pod,
	pvc *corev1.PersistentVolumeClaim,
	pvcCreated bool,
	remotePort int,
) error {
	metadata := kubernetesRuntimeMetadataAtState(
		l.executorConfig, l.profile, l.identity, pod, pvc, pvcCreated, remotePort, state,
	)
	return checkpointKubernetesRuntimeInventory(ctx, l.req, metadata)
}

func (l *kubernetesFreshLaunch) provisionWorkspace(ctx context.Context) error {
	workspace, err := provisionKubernetesWorkspace(
		ctx, l.runtime.resources, l.executorConfig, l.profile, l.identity, l.pvcName,
		func(created *corev1.PersistentVolumeClaim) error {
			return l.checkpoint(ctx, KubernetesInventoryStatePVCCreated, nil, created, true, 0)
		},
	)
	l.workspace = workspace
	if err != nil {
		return err
	}
	if workspace.createdClaim == nil {
		return nil
	}
	return l.checkpoint(ctx, KubernetesInventoryStatePVCAdmitted, nil, workspace.claim, true, 0)
}

func (l *kubernetesFreshLaunch) createRunningPod(ctx context.Context) (*corev1.Pod, error) {
	desiredPod, err := composeKubernetesLifecyclePod(
		l.profile, l.identity, l.podName, l.executorConfig.Namespace, l.workspace.claimName,
	)
	if err != nil {
		return nil, err
	}
	podWaitCtx, cancel := kubernetesRequestContext(ctx, l.executorConfig.RequestTimeoutSeconds)
	defer cancel()
	created, running, err := createAndWaitKubernetesPod(
		podWaitCtx, l.runtime.resources, desiredPod, l.identity, l.profile.MainContainer, "Pod",
		func(state string, pod *corev1.Pod) error {
			return l.checkpoint(
				ctx, state, pod, l.workspace.claim, l.workspace.createdClaim != nil, 0,
			)
		},
	)
	l.createdPod = created
	return running, err
}

func (l *kubernetesFreshLaunch) complete(
	ctx context.Context,
	runningPod *corev1.Pod,
	client *agentctl.Client,
	forward kubeexecutor.PortForwardSession,
	token, nonce string,
	remotePort int,
) (*ExecutorInstance, error) {
	metadata := kubernetesRuntimeMetadataAtState(
		l.executorConfig, l.profile, l.identity, runningPod, l.workspace.claim,
		l.workspace.createdClaim != nil, remotePort, KubernetesInventoryStateReady,
	)
	metadata[MetadataKeyKubernetesAgentctlInstanceID] = l.req.InstanceID
	// A fresh lifecycle-managed launch already has provisional Pod/PVC inventory.
	// Manager persists the final ready row only after both required runtime secret
	// references are durable, avoiding a restart-visible ready row that cannot
	// reconnect. Standalone callers without a release callback keep the direct
	// ready checkpoint behavior.
	if l.req.ReleaseRuntimeInventory == nil {
		if err := checkpointKubernetesRuntimeInventory(ctx, l.req, metadata); err != nil {
			client.Close()
			_ = forward.Close()
			return nil, err
		}
	}
	l.executor.replaceSession(l.req.InstanceID, &kubernetesSession{
		runtime: l.runtime, forward: forward, client: client,
		request:      cloneKubernetesCreateRequest(l.req),
		restartCount: kubernetesMainContainerRestartCount(runningPod, l.profile.MainContainer),
	})
	return &ExecutorInstance{
		InstanceID: l.req.InstanceID, TaskID: l.req.TaskID, SessionID: l.req.SessionID,
		RuntimeName: l.executor.Name(), Client: client, WorkspacePath: kubernetesWorkspacePath,
		Metadata: metadata, AuthToken: token, BootstrapNonce: nonce,
		ReleaseRuntimeInventory: l.req.ReleaseRuntimeInventory,
	}, nil
}

func (l *kubernetesFreshLaunch) rollbackAfterFailure(ctx context.Context, launchErr error) error {
	if launchErr == nil {
		return nil
	}
	rollbackCtx, cancel := kubernetesDetachedRequestContext(ctx, l.executorConfig.RequestTimeoutSeconds)
	defer cancel()
	var rollbackErrors []error
	if l.createdPod != nil {
		if err := rollbackCreatedKubernetesPod(rollbackCtx, l.runtime.resources, l.createdPod); err != nil {
			rollbackErrors = append(rollbackErrors, err)
		}
	}
	if l.workspace.createdClaim != nil {
		if err := rollbackCreatedKubernetesPVC(rollbackCtx, l.runtime.resources, l.workspace.createdClaim); err != nil {
			rollbackErrors = append(rollbackErrors, err)
		}
	}
	if len(rollbackErrors) == 0 && l.req.ReleaseRuntimeInventory != nil {
		if err := l.req.ReleaseRuntimeInventory(rollbackCtx); err != nil {
			rollbackErrors = append(rollbackErrors, fmt.Errorf(
				"kubernetes lifecycle: release rolled-back runtime inventory: %w", err,
			))
		}
	}
	return errors.Join(launchErr, errors.Join(rollbackErrors...))
}

func kubernetesDetachedRequestContext(ctx context.Context, requestTimeoutSeconds int) (context.Context, context.CancelFunc) {
	if requestTimeoutSeconds <= 0 {
		requestTimeoutSeconds = kubeexecutor.DefaultRequestTimeoutSeconds
	}
	return context.WithTimeout(
		context.WithoutCancel(ctx), time.Duration(requestTimeoutSeconds)*time.Second,
	)
}

func kubernetesAmbiguousCreateContext(ctx context.Context) (context.Context, context.CancelFunc) {
	return kubernetesDetachedRequestContext(ctx, kubeexecutor.DefaultRequestTimeoutSeconds)
}

func kubernetesRequestContext(ctx context.Context, requestTimeoutSeconds int) (context.Context, context.CancelFunc) {
	if requestTimeoutSeconds <= 0 {
		requestTimeoutSeconds = kubeexecutor.DefaultRequestTimeoutSeconds
	}
	return context.WithTimeout(ctx, time.Duration(requestTimeoutSeconds)*time.Second)
}

func (r *KubernetesExecutor) connectNewAgentctl(
	ctx context.Context,
	runtime *kubernetesRuntimeClient,
	req *ExecutorCreateRequest,
	pod *corev1.Pod,
	nonce string,
) (*agentctl.Client, kubeexecutor.PortForwardSession, string, int, error) {
	controlForward, control, err := r.connectHealthyKubernetesControl(ctx, runtime, pod)
	if err != nil {
		return nil, nil, "", 0, err
	}
	defer func() { _ = controlForward.Close() }()
	token, err := control.Handshake(ctx, nonce)
	if err != nil {
		return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: nonce handshake: %w", err)
	}
	createRequest := buildReconnectCreateInstanceRequest(req, req.InstanceID)
	createResponse, err := createOrReconcileKubernetesAgentctlInstance(ctx, control, createRequest)
	if err != nil {
		return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: create agentctl instance: %w", err)
	}
	if createResponse.Port < 1 || createResponse.Port > 65535 {
		return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: invalid agentctl instance port %d", createResponse.Port)
	}
	forward, err := startKubernetesForward(ctx, runtime.streams, pod, uint16(createResponse.Port))
	if err != nil {
		return nil, nil, "", 0, err
	}
	client := agentctl.NewClient(kubernetesControlHost, int(forward.LocalPort()), r.logger,
		agentctl.WithExecutionID(req.InstanceID), agentctl.WithSessionID(req.SessionID), agentctl.WithAuthToken(token))
	if err := client.Health(ctx); err != nil {
		client.Close()
		_ = forward.Close()
		return nil, nil, "", 0, fmt.Errorf("kubernetes lifecycle: instance health: %w", err)
	}
	return client, forward, token, createResponse.Port, nil
}

type kubernetesAgentctlInstanceControl interface {
	CreateInstance(context.Context, *agentctl.CreateInstanceRequest) (*agentctl.CreateInstanceResponse, error)
	GetInstance(context.Context, string) (*agentctl.InstanceInfo, error)
}

func createOrReconcileKubernetesAgentctlInstance(
	ctx context.Context,
	control kubernetesAgentctlInstanceControl,
	request *agentctl.CreateInstanceRequest,
) (*agentctl.CreateInstanceResponse, error) {
	response, createErr := control.CreateInstance(ctx, request)
	if createErr == nil {
		if err := validateKubernetesAgentctlInstanceResponse(response, request); err != nil {
			return nil, err
		}
		return response, nil
	}
	if !kubeexecutor.IsAmbiguousCreateError(createErr) {
		return nil, createErr
	}
	reconcileCtx, cancel := kubernetesAmbiguousCreateContext(ctx)
	defer cancel()
	info, reconcileErr := control.GetInstance(reconcileCtx, request.ID)
	if reconcileErr != nil {
		return nil, errors.Join(createErr, fmt.Errorf("reconcile ambiguous agentctl create: %w", reconcileErr))
	}
	response = &agentctl.CreateInstanceResponse{ID: info.ID, Port: info.Port}
	if err := validateKubernetesAgentctlInstanceResponse(response, request); err != nil {
		return nil, errors.Join(createErr, fmt.Errorf("reconcile ambiguous agentctl create: %w", err))
	}
	if info.WorkspacePath != request.WorkspacePath {
		return nil, errors.Join(createErr, errors.New(
			"reconcile ambiguous agentctl create: existing instance workspace does not match request",
		))
	}
	return response, nil
}

func validateKubernetesAgentctlInstanceResponse(
	response *agentctl.CreateInstanceResponse,
	request *agentctl.CreateInstanceRequest,
) error {
	if response == nil || request == nil || response.ID != request.ID {
		return errors.New("kubernetes lifecycle: agentctl instance identity does not match request")
	}
	if response.Port < 1 || response.Port > 65535 {
		return fmt.Errorf("kubernetes lifecycle: invalid agentctl instance port %d", response.Port)
	}
	return nil
}

func (r *KubernetesExecutor) connectHealthyKubernetesControl(
	ctx context.Context,
	runtime *kubernetesRuntimeClient,
	pod *corev1.Pod,
) (kubeexecutor.PortForwardSession, *agentctl.ControlClient, error) {
	var lastForwardErr error
	for {
		forward, err := startKubernetesForward(
			ctx, runtime.streams, pod, uint16(kubeexecutor.DefaultAgentctlPort),
		)
		if err != nil {
			return nil, nil, errors.Join(lastForwardErr, err)
		}
		control := agentctl.NewControlClient(kubernetesControlHost, int(forward.LocalPort()), r.logger)
		stopped, healthErr := r.waitForKubernetesAgentctlHealthUntilForwardStops(ctx, control, forward.Done())
		if !stopped {
			if healthErr != nil {
				_ = forward.Close()
				return nil, nil, fmt.Errorf("kubernetes lifecycle: control health: %w", errors.Join(lastForwardErr, healthErr))
			}
			return forward, control, nil
		}
		_ = forward.Close()
		lastForwardErr = healthErr
		if err := ctx.Err(); err != nil {
			return nil, nil, fmt.Errorf("kubernetes lifecycle: control health: %w", errors.Join(err, lastForwardErr))
		}
		if err := r.waitForKubernetesControlForwardRetry(ctx); err != nil {
			return nil, nil, fmt.Errorf("kubernetes lifecycle: control health: %w", errors.Join(err, lastForwardErr))
		}
	}
}

func (r *KubernetesExecutor) waitForKubernetesControlForwardRetry(ctx context.Context) error {
	delay := r.healthRetryDelay
	if delay <= 0 {
		delay = agentctlHealthRetryDelay
	}
	timer := time.NewTimer(delay)
	defer timer.Stop()
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-timer.C:
		return nil
	}
}

func (r *KubernetesExecutor) waitForKubernetesAgentctlHealth(ctx context.Context, ctl healthChecker) error {
	_, err := r.waitForKubernetesAgentctlHealthUntilForwardStops(ctx, ctl, nil)
	return err
}

func (r *KubernetesExecutor) waitForKubernetesAgentctlHealthUntilForwardStops(
	ctx context.Context,
	ctl healthChecker,
	forwardDone <-chan error,
) (bool, error) {
	delay := r.healthRetryDelay
	if delay <= 0 {
		delay = agentctlHealthRetryDelay
	}
	var lastErr error
	for {
		if err := ctl.Health(ctx); err == nil {
			if forwardDone != nil {
				select {
				case forwardErr := <-forwardDone:
					return true, kubernetesControlForwardStoppedError(forwardErr)
				default:
				}
			}
			return false, nil
		} else {
			lastErr = err
		}
		if ctx.Err() != nil {
			return false, fmt.Errorf("agentctl did not become healthy: %w", errors.Join(ctx.Err(), lastErr))
		}
		timer := time.NewTimer(delay)
		select {
		case forwardErr := <-forwardDone:
			timer.Stop()
			return true, kubernetesControlForwardStoppedError(forwardErr)
		case <-ctx.Done():
			timer.Stop()
			return false, fmt.Errorf("agentctl did not become healthy: %w", errors.Join(ctx.Err(), lastErr))
		case <-timer.C:
		}
	}
}

func kubernetesControlForwardStoppedError(err error) error {
	if err == nil {
		err = errors.New("port forward stopped")
	}
	return fmt.Errorf("control port forward stopped before agentctl became healthy: %w", err)
}

func startKubernetesForward(
	ctx context.Context,
	streams *kubeexecutor.StreamOperations,
	pod *corev1.Pod,
	remotePort uint16,
) (kubeexecutor.PortForwardSession, error) {
	forward, err := streams.Forward(context.WithoutCancel(ctx), kubeexecutor.PortForwardRequest{
		Namespace: pod.Namespace, Pod: pod.Name, LocalAddress: kubernetesControlHost,
		LocalPort: 0, RemotePort: remotePort,
	})
	if err != nil {
		return nil, fmt.Errorf("kubernetes lifecycle: forward port %d: %w", remotePort, err)
	}
	select {
	case <-forward.Ready():
		if forward.LocalPort() == 0 {
			_ = forward.Close()
			return nil, fmt.Errorf("kubernetes lifecycle: port forward returned zero local port")
		}
		return forward, nil
	case doneErr := <-forward.Done():
		_ = forward.Close()
		if doneErr == nil {
			doneErr = errors.New("port forward stopped before becoming ready")
		}
		return nil, fmt.Errorf("kubernetes lifecycle: port forward startup: %w", doneErr)
	case <-ctx.Done():
		_ = forward.Close()
		return nil, ctx.Err()
	}
}

func kubernetesProfileConfigFromMetadata(metadata map[string]interface{}) (kubeexecutor.ProfileConfig, error) {
	values := map[string]string{
		MetadataKeyKubernetesProfilePlatform:       getMetadataString(metadata, MetadataKeyKubernetesProfilePlatform),
		MetadataKeyKubernetesProfileMainContainer:  getMetadataString(metadata, MetadataKeyKubernetesProfileMainContainer),
		MetadataKeyKubernetesPodTemplateYAML:       getMetadataString(metadata, MetadataKeyKubernetesPodTemplateYAML),
		MetadataKeyKubernetesWorkspaceMode:         getMetadataString(metadata, MetadataKeyKubernetesWorkspaceMode),
		MetadataKeyKubernetesWorkspaceSize:         getMetadataString(metadata, MetadataKeyKubernetesWorkspaceSize),
		MetadataKeyKubernetesWorkspaceStorageClass: getMetadataString(metadata, MetadataKeyKubernetesWorkspaceStorageClass),
		MetadataKeyKubernetesWorkspaceAccessModes:  getMetadataString(metadata, MetadataKeyKubernetesWorkspaceAccessModes),
		MetadataKeyKubernetesWorkspaceClaimName:    getMetadataString(metadata, MetadataKeyKubernetesWorkspaceClaimName),
	}
	return kubeexecutor.ParseProfileConfig(values)
}

func (r *KubernetesExecutor) RecoverInstances(context.Context, []*models.ExecutorRunning) ([]*ExecutorInstance, error) {
	return nil, nil
}

func (r *KubernetesExecutor) GetInteractiveRunner() *process.InteractiveRunner { return nil }

func (r *KubernetesExecutor) RequiresCloneURL() bool          { return true }
func (r *KubernetesExecutor) ShouldApplyPreferredShell() bool { return false }
func (r *KubernetesExecutor) IsAlwaysResumable() bool         { return true }

func kubernetesExecutorConfigFromMetadata(metadata map[string]interface{}) (kubeexecutor.ExecutorConfig, error) {
	values := map[string]string{
		MetadataKeyKubernetesAuthMode:              getMetadataString(metadata, MetadataKeyKubernetesAuthMode),
		MetadataKeyKubernetesKubeconfigPath:        getMetadataString(metadata, MetadataKeyKubernetesKubeconfigPath),
		MetadataKeyKubernetesKubeContext:           getMetadataString(metadata, MetadataKeyKubernetesKubeContext),
		MetadataKeyKubernetesConfigNamespace:       getMetadataString(metadata, MetadataKeyKubernetesConfigNamespace),
		MetadataKeyKubernetesRequestTimeoutSeconds: getMetadataString(metadata, MetadataKeyKubernetesRequestTimeoutSeconds),
	}
	return kubeexecutor.ParseExecutorConfig(values)
}

func kubernetesIdentity(req *ExecutorCreateRequest) (kubeexecutor.ResourceIdentity, error) {
	identity := kubeexecutor.ResourceIdentity{
		ExecutorID: getMetadataString(req.Metadata, "executor_id"), ProfileID: getMetadataString(req.Metadata, MetadataKeyExecutorProfileID),
		InstanceID: req.InstanceID, TaskID: req.TaskID, SessionID: req.SessionID, EnvironmentID: req.TaskEnvironmentID,
	}
	if getMetadataBool(req.Metadata, metadataKubernetesTaskOwned) {
		identity.TaskOwned = true
		identity.InstanceID = req.TaskEnvironmentID
	}

	if _, err := kubeexecutor.OwnershipLabels(identity); err != nil {
		return kubeexecutor.ResourceIdentity{}, err
	}
	return identity, nil
}

func kubernetesResourceNames(instanceID string) (string, string) {
	sum := sha256.Sum256([]byte(instanceID))
	suffix := hex.EncodeToString(sum[:8])
	return "kandev-" + suffix, "kandev-" + suffix + "-workspace"
}

func kubernetesRuntimeMetadata(
	executorConfig kubeexecutor.ExecutorConfig,
	profile kubeexecutor.ProfileConfig,
	identity kubeexecutor.ResourceIdentity,
	pod *corev1.Pod,
	pvc *corev1.PersistentVolumeClaim,
	pvcCreated bool,
	remotePort int,
) map[string]interface{} {
	metadata := map[string]interface{}{
		MetadataKeyKubernetesNamespace:             executorConfig.Namespace,
		MetadataKeyKubernetesMainContainer:         profile.MainContainer,
		MetadataKeyKubernetesPlatform:              string(profile.Platform),
		MetadataKeyKubernetesRuntimeWorkspaceMode:  string(profile.Workspace.Mode),
		MetadataKeyKubernetesPVCCreated:            pvcCreated,
		MetadataKeyKubernetesAgentctlInstanceID:    identity.InstanceID,
		MetadataKeyKubernetesResourceInstanceID:    identity.InstanceID,
		MetadataKeyKubernetesResourceExecutorID:    identity.ExecutorID,
		MetadataKeyKubernetesResourceProfileID:     identity.ProfileID,
		MetadataKeyKubernetesResourceTaskID:        identity.TaskID,
		MetadataKeyKubernetesResourceSessionID:     identity.SessionID,
		MetadataKeyKubernetesResourceEnvironmentID: identity.EnvironmentID,
		MetadataKeyKubernetesExecutorConfigHash:    kubernetesConfigHash(executorConfig),
		MetadataKeyKubernetesProfileConfigHash:     kubernetesConfigHash(profile),
		MetadataKeyKubernetesTemplateHash:          kubernetesStringHash(profile.PodTemplateYAML),
		MetadataKeyKubernetesProfileSnapshot:       kubernetesProfileSnapshot(profile),
		MetadataKeyIsRemote:                        true,
		"executor_id":                              identity.ExecutorID,
		MetadataKeyExecutorProfileID:               identity.ProfileID,
	}
	if identity.TaskOwned {
		metadata[metadataKubernetesTaskOwned] = true
		metadata[kubeexecutor.MetadataKeyOwnershipVersion] = kubeexecutor.TaskOwnershipVersion
	}

	if pod != nil {
		metadata[MetadataKeyKubernetesNamespace] = pod.Namespace
		metadata[MetadataKeyKubernetesPodName] = pod.Name
		metadata[MetadataKeyKubernetesPodUID] = string(pod.UID)
		metadata[MetadataKeyKubernetesContainerRestartCount] = strconv.FormatInt(
			int64(kubernetesMainContainerRestartCount(pod, profile.MainContainer)), 10,
		)
	}
	if pvc != nil {
		metadata[MetadataKeyKubernetesPVCName] = pvc.Name
		metadata[MetadataKeyKubernetesPVCUID] = string(pvc.UID)
	}
	if remotePort > 0 {
		metadata[MetadataKeyKubernetesAgentctlRemotePort] = strconv.Itoa(remotePort)
	}
	return metadata
}

func kubernetesRuntimeMetadataAtState(
	executorConfig kubeexecutor.ExecutorConfig,
	profile kubeexecutor.ProfileConfig,
	identity kubeexecutor.ResourceIdentity,
	pod *corev1.Pod,
	pvc *corev1.PersistentVolumeClaim,
	pvcCreated bool,
	remotePort int,
	state string,
) map[string]interface{} {
	metadata := kubernetesRuntimeMetadata(
		executorConfig, profile, identity, pod, pvc, pvcCreated, remotePort,
	)
	metadata[MetadataKeyKubernetesInventoryState] = state
	return metadata
}

func checkpointKubernetesRuntimeInventory(
	ctx context.Context,
	req *ExecutorCreateRequest,
	metadata map[string]interface{},
) error {
	if req == nil || req.CheckpointRuntimeInventory == nil {
		return nil
	}
	if err := req.CheckpointRuntimeInventory(ctx, metadata); err != nil {
		return fmt.Errorf("kubernetes lifecycle: checkpoint runtime inventory: %w", err)
	}
	return nil
}

func kubernetesProfileSnapshot(profile kubeexecutor.ProfileConfig) string {
	data, _ := json.Marshal(profile)
	return string(data)
}

func kubernetesConfigHash(value interface{}) string {
	data, _ := json.Marshal(value)
	return kubernetesStringHash(string(data))
}

func kubernetesStringHash(value string) string {
	sum := sha256.Sum256([]byte(value))
	return hex.EncodeToString(sum[:])
}

func (r *KubernetesExecutor) lockInstance(instanceID string) func() {
	r.mu.Lock()
	lock := r.locks[instanceID]
	if lock == nil {
		lock = &kubernetesInstanceLock{}
		r.locks[instanceID] = lock
	}
	lock.refs++
	r.mu.Unlock()
	lock.mutex.Lock()
	return func() {
		lock.mutex.Unlock()
		r.mu.Lock()
		lock.refs--
		if lock.refs == 0 && r.locks[instanceID] == lock {
			delete(r.locks, instanceID)
		}
		r.mu.Unlock()
	}
}

func (r *KubernetesExecutor) replaceSession(instanceID string, session *kubernetesSession) {
	r.mu.Lock()
	old := r.sessions[instanceID]
	r.sessions[instanceID] = session
	r.mu.Unlock()
	if old != nil {
		if old.client != nil {
			old.client.Close()
		}
		if old.forward != nil {
			_ = old.forward.Close()
		}
	}
}

func kubernetesMainContainerRestartCount(pod *corev1.Pod, mainContainer string) int32 {
	if pod == nil {
		return 0
	}
	for index := range pod.Status.ContainerStatuses {
		if pod.Status.ContainerStatuses[index].Name == mainContainer {
			return pod.Status.ContainerStatuses[index].RestartCount
		}
	}
	return 0
}

func cloneKubernetesCreateRequest(req *ExecutorCreateRequest) *ExecutorCreateRequest {
	if req == nil {
		return nil
	}
	cloned := *req
	cloned.Env = cloneStringMap(req.Env)
	cloned.Metadata = cloneKubernetesMetadata(req.Metadata)
	cloned.WorkspaceSourceRoots = append([]string(nil), req.WorkspaceSourceRoots...)
	cloned.ApprovedSecretEnvKeys = append([]string(nil), req.ApprovedSecretEnvKeys...)
	cloned.McpServers = append([]McpServerConfig(nil), req.McpServers...)
	cloned.McpProviders = append([]string(nil), req.McpProviders...)
	return &cloned
}

func kubernetesBootstrapCommand() string {
	return `set -eu
while [ ! -f /opt/kandev/start ]; do sleep 1; done
set -a
. /opt/kandev/runtime.env
. /run/kandev/auth.env
set +a
[ -f /opt/kandev/prepared ] || { echo 'kubernetes preparation marker is missing' >&2; exit 70; }
if [ "${` + selectedCheckoutMarker + `:-}" = "1" ]; then
  ` + selectedCheckoutCredentialScrubCommands + `
  rm -f /run/kandev/auth.env 2>/dev/null || true
fi
exec /opt/kandev/agentctl`
}