From f890ff54320bba67afbe2a6e7ad9a7311dc16a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Almeida?= Date: Sat, 2 May 2026 20:33:04 +0100 Subject: [PATCH] fix(docker): connect agentctl through published ports --- apps/backend/internal/agent/docker/client.go | 115 ++++++++++++++---- .../internal/agent/lifecycle/container.go | 102 ++++++++++++---- .../agent/lifecycle/container_config_test.go | 44 +++++++ .../agent/lifecycle/executor_docker.go | 46 ++++++- 4 files changed, 255 insertions(+), 52 deletions(-) diff --git a/apps/backend/internal/agent/docker/client.go b/apps/backend/internal/agent/docker/client.go index 2fee14b2a1..56b5206f88 100644 --- a/apps/backend/internal/agent/docker/client.go +++ b/apps/backend/internal/agent/docker/client.go @@ -17,6 +17,7 @@ import ( "github.com/docker/docker/api/types/image" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" + "github.com/docker/go-connections/nat" "github.com/kandev/kandev/internal/common/config" "github.com/kandev/kandev/internal/common/logger" "go.uber.org/zap" @@ -24,18 +25,26 @@ import ( // ContainerConfig holds configuration for creating a container. type ContainerConfig struct { - Name string - Image string - Entrypoint []string // Overrides the image ENTRYPOINT (nil = use image default) - Cmd []string - Env []string // Environment variables - WorkingDir string - Mounts []MountConfig - NetworkMode string - Memory int64 // Memory limit in bytes - CPUQuota int64 // CPU quota - Labels map[string]string - AutoRemove bool + Name string + Image string + Entrypoint []string // Overrides the image ENTRYPOINT (nil = use image default) + Cmd []string + Env []string // Environment variables + WorkingDir string + Mounts []MountConfig + NetworkMode string + Memory int64 // Memory limit in bytes + CPUQuota int64 // CPU quota + Labels map[string]string + AutoRemove bool + PortBindings []PortBindingConfig +} + +// PortBindingConfig describes a container port to publish on the Docker host. +type PortBindingConfig struct { + ContainerPort int + HostIP string + HostPort string } // MountConfig holds mount configuration. @@ -195,20 +204,23 @@ func (c *Client) CreateContainer(ctx context.Context, cfg ContainerConfig) (stri } // Container configuration + exposedPorts, portBindings := buildDockerPortBindings(cfg.PortBindings) containerCfg := &container.Config{ - Image: cfg.Image, - Entrypoint: cfg.Entrypoint, - Cmd: cfg.Cmd, - Env: cfg.Env, - WorkingDir: cfg.WorkingDir, - Labels: cfg.Labels, + Image: cfg.Image, + Entrypoint: cfg.Entrypoint, + Cmd: cfg.Cmd, + Env: cfg.Env, + WorkingDir: cfg.WorkingDir, + Labels: cfg.Labels, + ExposedPorts: exposedPorts, } // Host configuration hostCfg := &container.HostConfig{ - Mounts: mounts, - NetworkMode: container.NetworkMode(cfg.NetworkMode), - AutoRemove: cfg.AutoRemove, + Mounts: mounts, + NetworkMode: container.NetworkMode(cfg.NetworkMode), + AutoRemove: cfg.AutoRemove, + PortBindings: portBindings, Resources: container.Resources{ Memory: cfg.Memory, CPUQuota: cfg.CPUQuota, @@ -228,6 +240,23 @@ func (c *Client) CreateContainer(ctx context.Context, cfg ContainerConfig) (stri return resp.ID, nil } +func buildDockerPortBindings(bindings []PortBindingConfig) (nat.PortSet, nat.PortMap) { + if len(bindings) == 0 { + return nil, nil + } + exposedPorts := nat.PortSet{} + portBindings := nat.PortMap{} + for _, binding := range bindings { + containerPort := nat.Port(fmt.Sprintf("%d/tcp", binding.ContainerPort)) + exposedPorts[containerPort] = struct{}{} + portBindings[containerPort] = append(portBindings[containerPort], nat.PortBinding{ + HostIP: binding.HostIP, + HostPort: binding.HostPort, + }) + } + return exposedPorts, portBindings +} + // StartContainer starts a container. func (c *Client) StartContainer(ctx context.Context, containerID string) error { c.logger.Info("Starting container", zap.String("container_id", containerID)) @@ -376,6 +405,41 @@ func (c *Client) GetContainerIP(ctx context.Context, containerID string) (string return "", fmt.Errorf("no IP address found for container %s", containerID) } +// GetContainerHostPort returns the Docker host endpoint for a published TCP port. +func (c *Client) GetContainerHostPort(ctx context.Context, containerID string, containerPort int) (string, int, error) { + c.logger.Debug("Getting container host port", + zap.String("container_id", containerID), + zap.Int("container_port", containerPort)) + + inspect, err := c.cli.ContainerInspect(ctx, containerID) + if err != nil { + return "", 0, fmt.Errorf("failed to inspect container %s: %w", containerID, err) + } + if inspect.NetworkSettings == nil || inspect.NetworkSettings.Ports == nil { + return "", 0, fmt.Errorf("container %s has no published ports", containerID) + } + + bindings := inspect.NetworkSettings.Ports[nat.Port(fmt.Sprintf("%d/tcp", containerPort))] + if len(bindings) == 0 { + return "", 0, fmt.Errorf("container %s port %d/tcp is not published", containerID, containerPort) + } + hostPort, err := nat.ParsePort(bindings[0].HostPort) + if err != nil { + return "", 0, fmt.Errorf("invalid host port for container %s port %d/tcp: %w", containerID, containerPort, err) + } + hostIP := normalizeDockerHostIP(bindings[0].HostIP) + return hostIP, hostPort, nil +} + +func normalizeDockerHostIP(hostIP string) string { + switch hostIP { + case "", "0.0.0.0", "::": + return "127.0.0.1" + default: + return hostIP + } +} + // GetContainerLabels returns the labels of a container func (c *Client) GetContainerLabels(ctx context.Context, containerID string) (map[string]string, error) { c.logger.Debug("Getting container labels", zap.String("container_id", containerID)) @@ -531,6 +595,7 @@ func (c *Client) CreateContainerInteractive(ctx context.Context, cfg ContainerCo // Container configuration with stdin attached. Mirror CreateContainer's // handling of ContainerConfig fields — including Entrypoint — so the same // config struct produces consistent container behavior on both paths. + exposedPorts, portBindings := buildDockerPortBindings(cfg.PortBindings) containerCfg := &container.Config{ Image: cfg.Image, Entrypoint: cfg.Entrypoint, @@ -543,14 +608,16 @@ func (c *Client) CreateContainerInteractive(ctx context.Context, cfg ContainerCo AttachStdin: true, AttachStdout: true, AttachStderr: true, + ExposedPorts: exposedPorts, Tty: false, // Important: no TTY for JSON-RPC } // Host configuration hostCfg := &container.HostConfig{ - Mounts: mounts, - NetworkMode: container.NetworkMode(cfg.NetworkMode), - AutoRemove: cfg.AutoRemove, + Mounts: mounts, + NetworkMode: container.NetworkMode(cfg.NetworkMode), + AutoRemove: cfg.AutoRemove, + PortBindings: portBindings, Resources: container.Resources{ Memory: cfg.Memory, CPUQuota: cfg.CPUQuota, diff --git a/apps/backend/internal/agent/lifecycle/container.go b/apps/backend/internal/agent/lifecycle/container.go index 0550a06f2b..f67b54cfe9 100644 --- a/apps/backend/internal/agent/lifecycle/container.go +++ b/apps/backend/internal/agent/lifecycle/container.go @@ -19,6 +19,11 @@ import ( "github.com/kandev/kandev/internal/common/logger" ) +const ( + dockerAgentctlInstancePortBase = 41001 + dockerAgentctlInstancePortMax = 41100 +) + // ContainerConfig holds configuration for launching a Docker container type ContainerConfig struct { AgentConfig agents.Agent @@ -86,24 +91,24 @@ func (cm *ContainerManager) LaunchContainer(ctx context.Context, config Containe } config.BootstrapNonce = nonce - containerID, containerIP, err := cm.createAndStartContainer(ctx, config) + containerID, containerIP, controlHost, controlPort, err := cm.createAndStartContainer(ctx, config) if err != nil { return nil, err } // Create ControlClient (no auth token yet — handshake hasn't happened) - ctl := agentctl.NewControlClient(containerIP, AgentCtlPort, cm.logger) + ctl := agentctl.NewControlClient(controlHost, controlPort, cm.logger) // Wait for agentctl to be healthy if err := cm.waitForHealth(ctx, ctl); err != nil { - _ = cm.dockerClient.RemoveContainer(ctx, containerID, true) + cm.removeContainerBestEffort(containerID) return nil, fmt.Errorf("agentctl health check failed: %w", err) } // Perform handshake: nonce → token authToken, err := ctl.Handshake(ctx, nonce) if err != nil { - _ = cm.dockerClient.RemoveContainer(ctx, containerID, true) + cm.removeContainerBestEffort(containerID) return nil, fmt.Errorf("agentctl handshake failed: %w", err) } @@ -128,20 +133,20 @@ func (cm *ContainerManager) LaunchContainer(ctx context.Context, config Containe // createAndStartContainer builds, creates, and starts a Docker container. func (cm *ContainerManager) createAndStartContainer( ctx context.Context, config ContainerConfig, -) (string, string, error) { +) (string, string, string, int, error) { containerCfg, err := cm.buildContainerConfig(config) if err != nil { - return "", "", fmt.Errorf("failed to build container config: %w", err) + return "", "", "", 0, fmt.Errorf("failed to build container config: %w", err) } containerID, err := cm.dockerClient.CreateContainer(ctx, containerCfg) if err != nil { - return "", "", fmt.Errorf("failed to create container: %w", err) + return "", "", "", 0, fmt.Errorf("failed to create container: %w", err) } if err := cm.dockerClient.StartContainer(ctx, containerID); err != nil { - _ = cm.dockerClient.RemoveContainer(ctx, containerID, true) - return "", "", fmt.Errorf("failed to start container: %w", err) + cm.removeContainerBestEffort(containerID) + return "", "", "", 0, fmt.Errorf("failed to start container: %w", err) } containerIP, err := cm.dockerClient.GetContainerIP(ctx, containerID) @@ -151,7 +156,8 @@ func (cm *ContainerManager) createAndStartContainer( containerIP = "127.0.0.1" } - return containerID, containerIP, nil + controlHost, controlPort := cm.resolveContainerEndpoint(ctx, containerID, AgentCtlPort, containerIP) + return containerID, containerIP, controlHost, controlPort, nil } // createInstanceAndClient creates an agent instance in the container and returns the client. @@ -189,13 +195,15 @@ func (cm *ContainerManager) createInstanceAndClient( resp, err := ctl.CreateInstance(ctx, createReq) if err != nil { - _ = cm.dockerClient.RemoveContainer(ctx, containerID, true) + cm.removeContainerBestEffort(containerID) return nil, fmt.Errorf("failed to create instance in container: %w", err) } + instanceHost, instancePort := cm.resolveContainerEndpoint(ctx, containerID, resp.Port, containerIP) + // ControlClient already has the auth token set via Handshake — // read it back for the per-instance Client. - client := agentctl.NewClient(containerIP, resp.Port, cm.logger, + client := agentctl.NewClient(instanceHost, instancePort, cm.logger, agentctl.WithExecutionID(config.InstanceID), agentctl.WithSessionID(config.SessionID), agentctl.WithAuthToken(ctl.AuthToken())) @@ -203,6 +211,32 @@ func (cm *ContainerManager) createInstanceAndClient( return client, nil } +func (cm *ContainerManager) resolveContainerEndpoint(ctx context.Context, containerID string, containerPort int, fallbackHost string) (string, int) { + host, port, err := cm.dockerClient.GetContainerHostPort(ctx, containerID, containerPort) + if err == nil { + return host, port + } + cm.logger.Warn("failed to resolve published Docker port, falling back to container IP", + zap.String("container_id", containerID), + zap.Int("container_port", containerPort), + zap.String("fallback_host", fallbackHost), + zap.Error(err)) + return fallbackHost, containerPort +} + +func (cm *ContainerManager) removeContainerBestEffort(containerID string) { + if containerID == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := cm.dockerClient.RemoveContainer(ctx, containerID, true); err != nil { + cm.logger.Warn("failed to remove container after launch failure", + zap.String("container_id", containerID), + zap.Error(err)) + } +} + // waitForHealth waits for agentctl to be healthy with retries. // The budget covers the time it takes for the container's bootstrap to run the // prepare script (git clone, optional network installs) and then exec agentctl. @@ -371,16 +405,17 @@ exec /usr/local/bin/agentctl`, } containerCfg := docker.ContainerConfig{ - Name: containerName, - Image: imageName, - Entrypoint: bootstrap, - Cmd: nil, - Env: env, - WorkingDir: cm.expandMountSource(rt.WorkingDir, containerWorkspacePath), - Mounts: mounts, - NetworkMode: cm.networkName, - Memory: memoryBytes, - CPUQuota: cpuQuota, + Name: containerName, + Image: imageName, + Entrypoint: bootstrap, + Cmd: nil, + Env: env, + WorkingDir: cm.expandMountSource(rt.WorkingDir, containerWorkspacePath), + Mounts: mounts, + PortBindings: dockerAgentctlPortBindings(), + NetworkMode: cm.networkName, + Memory: memoryBytes, + CPUQuota: cpuQuota, Labels: map[string]string{ "kandev.managed": "true", "kandev.instance_id": config.InstanceID, @@ -397,6 +432,23 @@ exec /usr/local/bin/agentctl`, return containerCfg, nil } +func dockerAgentctlPortBindings() []docker.PortBindingConfig { + bindings := make([]docker.PortBindingConfig, 0, 1+dockerAgentctlInstancePortMax-dockerAgentctlInstancePortBase+1) + bindings = append(bindings, newDockerPortBinding(AgentCtlPort)) + for port := dockerAgentctlInstancePortBase; port <= dockerAgentctlInstancePortMax; port++ { + bindings = append(bindings, newDockerPortBinding(port)) + } + return bindings +} + +func newDockerPortBinding(containerPort int) docker.PortBindingConfig { + return docker.PortBindingConfig{ + ContainerPort: containerPort, + HostIP: "127.0.0.1", + HostPort: "0", + } +} + // expandMounts expands mount templates with actual paths func (cm *ContainerManager) expandMounts(templates []agents.MountTemplate, workspacePath string, ag agents.Agent) []docker.MountConfig { mounts := make([]docker.MountConfig, 0, len(templates)+1) // +1 for potential session dir @@ -508,6 +560,12 @@ func (cm *ContainerManager) buildEnvVars(config ContainerConfig) []string { env = append(env, fmt.Sprintf("KANDEV_AGENT_PROFILE_ID=%s", config.ProfileInfo.ProfileID)) } + env = append(env, + fmt.Sprintf("AGENTCTL_PORT=%d", AgentCtlPort), + fmt.Sprintf("AGENTCTL_INSTANCE_PORT_BASE=%d", dockerAgentctlInstancePortBase), + fmt.Sprintf("AGENTCTL_INSTANCE_PORT_MAX=%d", dockerAgentctlInstancePortMax), + ) + // Inject bootstrap nonce for agentctl handshake (NOT the auth token) if config.BootstrapNonce != "" { env = append(env, "AGENTCTL_BOOTSTRAP_NONCE="+config.BootstrapNonce) diff --git a/apps/backend/internal/agent/lifecycle/container_config_test.go b/apps/backend/internal/agent/lifecycle/container_config_test.go index 0173a40985..f102d04602 100644 --- a/apps/backend/internal/agent/lifecycle/container_config_test.go +++ b/apps/backend/internal/agent/lifecycle/container_config_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/kandev/kandev/internal/agent/agents" + "github.com/kandev/kandev/internal/agent/docker" "github.com/kandev/kandev/internal/common/logger" ) @@ -126,3 +127,46 @@ func TestBuildContainerConfig_ImageTagOverrideWins(t *testing.T) { t.Errorf("Image = %q, want kandev/agent:custom (profile override must win over rt.Image)", got.Image) } } + +func TestBuildContainerConfig_PublishesAgentctlPorts(t *testing.T) { + cm := newCMTest(t) + cfg := ContainerConfig{ + AgentConfig: newConfigStubAgent(), + InstanceID: "0123456789abcdef", + TaskID: "task-1", + } + + got, err := cm.buildContainerConfig(cfg) + if err != nil { + t.Fatalf("buildContainerConfig: %v", err) + } + + if len(got.PortBindings) == 0 { + t.Fatal("expected agentctl ports to be published") + } + assertHasPortBinding(t, got.PortBindings, AgentCtlPort) + assertHasPortBinding(t, got.PortBindings, dockerAgentctlInstancePortBase) + assertHasPortBinding(t, got.PortBindings, dockerAgentctlInstancePortMax) + assertEnvContains(t, got.Env, "AGENTCTL_INSTANCE_PORT_BASE=41001") + assertEnvContains(t, got.Env, "AGENTCTL_INSTANCE_PORT_MAX=41100") +} + +func assertHasPortBinding(t *testing.T, bindings []docker.PortBindingConfig, port int) { + t.Helper() + for _, binding := range bindings { + if binding.ContainerPort == port && binding.HostIP == "127.0.0.1" && binding.HostPort == "0" { + return + } + } + t.Fatalf("missing published port binding for %d/tcp: %#v", port, bindings) +} + +func assertEnvContains(t *testing.T, env []string, want string) { + t.Helper() + for _, item := range env { + if item == want { + return + } + } + t.Fatalf("missing env %q in %#v", want, env) +} diff --git a/apps/backend/internal/agent/lifecycle/executor_docker.go b/apps/backend/internal/agent/lifecycle/executor_docker.go index a961551f8c..ca8bfff292 100644 --- a/apps/backend/internal/agent/lifecycle/executor_docker.go +++ b/apps/backend/internal/agent/lifecycle/executor_docker.go @@ -216,20 +216,23 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient return nil, fmt.Errorf("failed to get IP for container %s: %w", containerName, err) } + controlHost, controlPort := resolveDockerEndpoint(ctx, dockerClient, containerName, AgentCtlPort, containerIP, r.logger) + // Check if agentctl is healthy - ctl := agentctl.NewControlClient(containerIP, AgentCtlPort, r.logger) + ctl := agentctl.NewControlClient(controlHost, controlPort, r.logger) if err := ctl.Health(ctx); err != nil { return nil, fmt.Errorf("agentctl not healthy in container %s: %w", containerName, err) } // Check if the previous instance is still alive - instancePort, reusingProcess, err := r.findExistingInstance(ctx, ctl, containerIP, prevID) + instancePort, reusingProcess, err := r.findExistingInstance(ctx, dockerClient, ctl, containerName, containerIP, prevID) if err != nil { return nil, fmt.Errorf("failed to find instance in container %s: %w", containerName, err) } + instanceHost, resolvedInstancePort := resolveDockerEndpoint(ctx, dockerClient, containerName, instancePort, containerIP, r.logger) // Create client pointing to the instance port - client := agentctl.NewClient(containerIP, instancePort, r.logger, + client := agentctl.NewClient(instanceHost, resolvedInstancePort, r.logger, agentctl.WithExecutionID(req.InstanceID), agentctl.WithSessionID(req.SessionID)) @@ -248,7 +251,10 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient zap.String("container_name", containerName), zap.String("container_id", info.ID), zap.String("container_ip", containerIP), - zap.Int("instance_port", instancePort), + zap.String("control_host", controlHost), + zap.Int("control_port", controlPort), + zap.String("instance_host", instanceHost), + zap.Int("instance_port", resolvedInstancePort), zap.Bool("reusing_process", reusingProcess)) return &ExecutorInstance{ @@ -266,12 +272,20 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient // findExistingInstance checks if a previous instance is still running in the container. // Returns the instance port and whether the agent subprocess is also running. -func (r *DockerExecutor) findExistingInstance(ctx context.Context, ctl *agentctl.ControlClient, containerIP, prevExecutionID string) (int, bool, error) { +func (r *DockerExecutor) findExistingInstance( + ctx context.Context, + dockerClient *docker.Client, + ctl *agentctl.ControlClient, + containerID string, + containerIP string, + prevExecutionID string, +) (int, bool, error) { // Try to get the existing instance by its ID instance, err := ctl.GetInstance(ctx, prevExecutionID) if err == nil && instance != nil && instance.Port > 0 { // Instance exists, check if agent subprocess is running - client := agentctl.NewClient(containerIP, instance.Port, r.logger) + instanceHost, instancePort := resolveDockerEndpoint(ctx, dockerClient, containerID, instance.Port, containerIP, r.logger) + client := agentctl.NewClient(instanceHost, instancePort, r.logger) status, statusErr := client.GetStatus(ctx) processRunning := statusErr == nil && status != nil && status.IsAgentRunning() return instance.Port, processRunning, nil @@ -290,6 +304,26 @@ func (r *DockerExecutor) findExistingInstance(ctx context.Context, ctl *agentctl return resp.Port, false, nil } +func resolveDockerEndpoint( + ctx context.Context, + dockerClient *docker.Client, + containerID string, + containerPort int, + fallbackHost string, + log *logger.Logger, +) (string, int) { + host, port, err := dockerClient.GetContainerHostPort(ctx, containerID, containerPort) + if err == nil { + return host, port + } + log.Warn("failed to resolve published Docker port, falling back to container IP", + zap.String("container_id", containerID), + zap.Int("container_port", containerPort), + zap.String("fallback_host", fallbackHost), + zap.Error(err)) + return fallbackHost, containerPort +} + func (r *DockerExecutor) StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error { if instance.ContainerID == "" { return nil // No container to stop