fix(docker): connect agentctl through published ports

This commit is contained in:
José Almeida
2026-05-02 20:33:04 +01:00
parent 1f5a47c1b9
commit f890ff5432
4 changed files with 255 additions and 52 deletions
+91 -24
View File
@@ -17,6 +17,7 @@ import (
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/client"
"github.com/docker/go-connections/nat"
"github.com/kandev/kandev/internal/common/config"
"github.com/kandev/kandev/internal/common/logger"
"go.uber.org/zap"
@@ -24,18 +25,26 @@ import (
// ContainerConfig holds configuration for creating a container.
type ContainerConfig struct {
Name string
Image string
Entrypoint []string // Overrides the image ENTRYPOINT (nil = use image default)
Cmd []string
Env []string // Environment variables
WorkingDir string
Mounts []MountConfig
NetworkMode string
Memory int64 // Memory limit in bytes
CPUQuota int64 // CPU quota
Labels map[string]string
AutoRemove bool
Name string
Image string
Entrypoint []string // Overrides the image ENTRYPOINT (nil = use image default)
Cmd []string
Env []string // Environment variables
WorkingDir string
Mounts []MountConfig
NetworkMode string
Memory int64 // Memory limit in bytes
CPUQuota int64 // CPU quota
Labels map[string]string
AutoRemove bool
PortBindings []PortBindingConfig
}
// PortBindingConfig describes a container port to publish on the Docker host.
type PortBindingConfig struct {
ContainerPort int
HostIP string
HostPort string
}
// MountConfig holds mount configuration.
@@ -195,20 +204,23 @@ func (c *Client) CreateContainer(ctx context.Context, cfg ContainerConfig) (stri
}
// Container configuration
exposedPorts, portBindings := buildDockerPortBindings(cfg.PortBindings)
containerCfg := &container.Config{
Image: cfg.Image,
Entrypoint: cfg.Entrypoint,
Cmd: cfg.Cmd,
Env: cfg.Env,
WorkingDir: cfg.WorkingDir,
Labels: cfg.Labels,
Image: cfg.Image,
Entrypoint: cfg.Entrypoint,
Cmd: cfg.Cmd,
Env: cfg.Env,
WorkingDir: cfg.WorkingDir,
Labels: cfg.Labels,
ExposedPorts: exposedPorts,
}
// Host configuration
hostCfg := &container.HostConfig{
Mounts: mounts,
NetworkMode: container.NetworkMode(cfg.NetworkMode),
AutoRemove: cfg.AutoRemove,
Mounts: mounts,
NetworkMode: container.NetworkMode(cfg.NetworkMode),
AutoRemove: cfg.AutoRemove,
PortBindings: portBindings,
Resources: container.Resources{
Memory: cfg.Memory,
CPUQuota: cfg.CPUQuota,
@@ -228,6 +240,23 @@ func (c *Client) CreateContainer(ctx context.Context, cfg ContainerConfig) (stri
return resp.ID, nil
}
func buildDockerPortBindings(bindings []PortBindingConfig) (nat.PortSet, nat.PortMap) {
if len(bindings) == 0 {
return nil, nil
}
exposedPorts := nat.PortSet{}
portBindings := nat.PortMap{}
for _, binding := range bindings {
containerPort := nat.Port(fmt.Sprintf("%d/tcp", binding.ContainerPort))
exposedPorts[containerPort] = struct{}{}
portBindings[containerPort] = append(portBindings[containerPort], nat.PortBinding{
HostIP: binding.HostIP,
HostPort: binding.HostPort,
})
}
return exposedPorts, portBindings
}
// StartContainer starts a container.
func (c *Client) StartContainer(ctx context.Context, containerID string) error {
c.logger.Info("Starting container", zap.String("container_id", containerID))
@@ -376,6 +405,41 @@ func (c *Client) GetContainerIP(ctx context.Context, containerID string) (string
return "", fmt.Errorf("no IP address found for container %s", containerID)
}
// GetContainerHostPort returns the Docker host endpoint for a published TCP port.
func (c *Client) GetContainerHostPort(ctx context.Context, containerID string, containerPort int) (string, int, error) {
c.logger.Debug("Getting container host port",
zap.String("container_id", containerID),
zap.Int("container_port", containerPort))
inspect, err := c.cli.ContainerInspect(ctx, containerID)
if err != nil {
return "", 0, fmt.Errorf("failed to inspect container %s: %w", containerID, err)
}
if inspect.NetworkSettings == nil || inspect.NetworkSettings.Ports == nil {
return "", 0, fmt.Errorf("container %s has no published ports", containerID)
}
bindings := inspect.NetworkSettings.Ports[nat.Port(fmt.Sprintf("%d/tcp", containerPort))]
if len(bindings) == 0 {
return "", 0, fmt.Errorf("container %s port %d/tcp is not published", containerID, containerPort)
}
hostPort, err := nat.ParsePort(bindings[0].HostPort)
if err != nil {
return "", 0, fmt.Errorf("invalid host port for container %s port %d/tcp: %w", containerID, containerPort, err)
}
hostIP := normalizeDockerHostIP(bindings[0].HostIP)
return hostIP, hostPort, nil
}
func normalizeDockerHostIP(hostIP string) string {
switch hostIP {
case "", "0.0.0.0", "::":
return "127.0.0.1"
default:
return hostIP
}
}
// GetContainerLabels returns the labels of a container
func (c *Client) GetContainerLabels(ctx context.Context, containerID string) (map[string]string, error) {
c.logger.Debug("Getting container labels", zap.String("container_id", containerID))
@@ -531,6 +595,7 @@ func (c *Client) CreateContainerInteractive(ctx context.Context, cfg ContainerCo
// Container configuration with stdin attached. Mirror CreateContainer's
// handling of ContainerConfig fields — including Entrypoint — so the same
// config struct produces consistent container behavior on both paths.
exposedPorts, portBindings := buildDockerPortBindings(cfg.PortBindings)
containerCfg := &container.Config{
Image: cfg.Image,
Entrypoint: cfg.Entrypoint,
@@ -543,14 +608,16 @@ func (c *Client) CreateContainerInteractive(ctx context.Context, cfg ContainerCo
AttachStdin: true,
AttachStdout: true,
AttachStderr: true,
ExposedPorts: exposedPorts,
Tty: false, // Important: no TTY for JSON-RPC
}
// Host configuration
hostCfg := &container.HostConfig{
Mounts: mounts,
NetworkMode: container.NetworkMode(cfg.NetworkMode),
AutoRemove: cfg.AutoRemove,
Mounts: mounts,
NetworkMode: container.NetworkMode(cfg.NetworkMode),
AutoRemove: cfg.AutoRemove,
PortBindings: portBindings,
Resources: container.Resources{
Memory: cfg.Memory,
CPUQuota: cfg.CPUQuota,
@@ -19,6 +19,11 @@ import (
"github.com/kandev/kandev/internal/common/logger"
)
const (
dockerAgentctlInstancePortBase = 41001
dockerAgentctlInstancePortMax = 41100
)
// ContainerConfig holds configuration for launching a Docker container
type ContainerConfig struct {
AgentConfig agents.Agent
@@ -86,24 +91,24 @@ func (cm *ContainerManager) LaunchContainer(ctx context.Context, config Containe
}
config.BootstrapNonce = nonce
containerID, containerIP, err := cm.createAndStartContainer(ctx, config)
containerID, containerIP, controlHost, controlPort, err := cm.createAndStartContainer(ctx, config)
if err != nil {
return nil, err
}
// Create ControlClient (no auth token yet — handshake hasn't happened)
ctl := agentctl.NewControlClient(containerIP, AgentCtlPort, cm.logger)
ctl := agentctl.NewControlClient(controlHost, controlPort, cm.logger)
// Wait for agentctl to be healthy
if err := cm.waitForHealth(ctx, ctl); err != nil {
_ = cm.dockerClient.RemoveContainer(ctx, containerID, true)
cm.removeContainerBestEffort(containerID)
return nil, fmt.Errorf("agentctl health check failed: %w", err)
}
// Perform handshake: nonce → token
authToken, err := ctl.Handshake(ctx, nonce)
if err != nil {
_ = cm.dockerClient.RemoveContainer(ctx, containerID, true)
cm.removeContainerBestEffort(containerID)
return nil, fmt.Errorf("agentctl handshake failed: %w", err)
}
@@ -128,20 +133,20 @@ func (cm *ContainerManager) LaunchContainer(ctx context.Context, config Containe
// createAndStartContainer builds, creates, and starts a Docker container.
func (cm *ContainerManager) createAndStartContainer(
ctx context.Context, config ContainerConfig,
) (string, string, error) {
) (string, string, string, int, error) {
containerCfg, err := cm.buildContainerConfig(config)
if err != nil {
return "", "", fmt.Errorf("failed to build container config: %w", err)
return "", "", "", 0, fmt.Errorf("failed to build container config: %w", err)
}
containerID, err := cm.dockerClient.CreateContainer(ctx, containerCfg)
if err != nil {
return "", "", fmt.Errorf("failed to create container: %w", err)
return "", "", "", 0, fmt.Errorf("failed to create container: %w", err)
}
if err := cm.dockerClient.StartContainer(ctx, containerID); err != nil {
_ = cm.dockerClient.RemoveContainer(ctx, containerID, true)
return "", "", fmt.Errorf("failed to start container: %w", err)
cm.removeContainerBestEffort(containerID)
return "", "", "", 0, fmt.Errorf("failed to start container: %w", err)
}
containerIP, err := cm.dockerClient.GetContainerIP(ctx, containerID)
@@ -151,7 +156,8 @@ func (cm *ContainerManager) createAndStartContainer(
containerIP = "127.0.0.1"
}
return containerID, containerIP, nil
controlHost, controlPort := cm.resolveContainerEndpoint(ctx, containerID, AgentCtlPort, containerIP)
return containerID, containerIP, controlHost, controlPort, nil
}
// createInstanceAndClient creates an agent instance in the container and returns the client.
@@ -189,13 +195,15 @@ func (cm *ContainerManager) createInstanceAndClient(
resp, err := ctl.CreateInstance(ctx, createReq)
if err != nil {
_ = cm.dockerClient.RemoveContainer(ctx, containerID, true)
cm.removeContainerBestEffort(containerID)
return nil, fmt.Errorf("failed to create instance in container: %w", err)
}
instanceHost, instancePort := cm.resolveContainerEndpoint(ctx, containerID, resp.Port, containerIP)
// ControlClient already has the auth token set via Handshake —
// read it back for the per-instance Client.
client := agentctl.NewClient(containerIP, resp.Port, cm.logger,
client := agentctl.NewClient(instanceHost, instancePort, cm.logger,
agentctl.WithExecutionID(config.InstanceID),
agentctl.WithSessionID(config.SessionID),
agentctl.WithAuthToken(ctl.AuthToken()))
@@ -203,6 +211,32 @@ func (cm *ContainerManager) createInstanceAndClient(
return client, nil
}
func (cm *ContainerManager) resolveContainerEndpoint(ctx context.Context, containerID string, containerPort int, fallbackHost string) (string, int) {
host, port, err := cm.dockerClient.GetContainerHostPort(ctx, containerID, containerPort)
if err == nil {
return host, port
}
cm.logger.Warn("failed to resolve published Docker port, falling back to container IP",
zap.String("container_id", containerID),
zap.Int("container_port", containerPort),
zap.String("fallback_host", fallbackHost),
zap.Error(err))
return fallbackHost, containerPort
}
func (cm *ContainerManager) removeContainerBestEffort(containerID string) {
if containerID == "" {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
if err := cm.dockerClient.RemoveContainer(ctx, containerID, true); err != nil {
cm.logger.Warn("failed to remove container after launch failure",
zap.String("container_id", containerID),
zap.Error(err))
}
}
// waitForHealth waits for agentctl to be healthy with retries.
// The budget covers the time it takes for the container's bootstrap to run the
// prepare script (git clone, optional network installs) and then exec agentctl.
@@ -371,16 +405,17 @@ exec /usr/local/bin/agentctl`,
}
containerCfg := docker.ContainerConfig{
Name: containerName,
Image: imageName,
Entrypoint: bootstrap,
Cmd: nil,
Env: env,
WorkingDir: cm.expandMountSource(rt.WorkingDir, containerWorkspacePath),
Mounts: mounts,
NetworkMode: cm.networkName,
Memory: memoryBytes,
CPUQuota: cpuQuota,
Name: containerName,
Image: imageName,
Entrypoint: bootstrap,
Cmd: nil,
Env: env,
WorkingDir: cm.expandMountSource(rt.WorkingDir, containerWorkspacePath),
Mounts: mounts,
PortBindings: dockerAgentctlPortBindings(),
NetworkMode: cm.networkName,
Memory: memoryBytes,
CPUQuota: cpuQuota,
Labels: map[string]string{
"kandev.managed": "true",
"kandev.instance_id": config.InstanceID,
@@ -397,6 +432,23 @@ exec /usr/local/bin/agentctl`,
return containerCfg, nil
}
func dockerAgentctlPortBindings() []docker.PortBindingConfig {
bindings := make([]docker.PortBindingConfig, 0, 1+dockerAgentctlInstancePortMax-dockerAgentctlInstancePortBase+1)
bindings = append(bindings, newDockerPortBinding(AgentCtlPort))
for port := dockerAgentctlInstancePortBase; port <= dockerAgentctlInstancePortMax; port++ {
bindings = append(bindings, newDockerPortBinding(port))
}
return bindings
}
func newDockerPortBinding(containerPort int) docker.PortBindingConfig {
return docker.PortBindingConfig{
ContainerPort: containerPort,
HostIP: "127.0.0.1",
HostPort: "0",
}
}
// expandMounts expands mount templates with actual paths
func (cm *ContainerManager) expandMounts(templates []agents.MountTemplate, workspacePath string, ag agents.Agent) []docker.MountConfig {
mounts := make([]docker.MountConfig, 0, len(templates)+1) // +1 for potential session dir
@@ -508,6 +560,12 @@ func (cm *ContainerManager) buildEnvVars(config ContainerConfig) []string {
env = append(env, fmt.Sprintf("KANDEV_AGENT_PROFILE_ID=%s", config.ProfileInfo.ProfileID))
}
env = append(env,
fmt.Sprintf("AGENTCTL_PORT=%d", AgentCtlPort),
fmt.Sprintf("AGENTCTL_INSTANCE_PORT_BASE=%d", dockerAgentctlInstancePortBase),
fmt.Sprintf("AGENTCTL_INSTANCE_PORT_MAX=%d", dockerAgentctlInstancePortMax),
)
// Inject bootstrap nonce for agentctl handshake (NOT the auth token)
if config.BootstrapNonce != "" {
env = append(env, "AGENTCTL_BOOTSTRAP_NONCE="+config.BootstrapNonce)
@@ -5,6 +5,7 @@ import (
"testing"
"github.com/kandev/kandev/internal/agent/agents"
"github.com/kandev/kandev/internal/agent/docker"
"github.com/kandev/kandev/internal/common/logger"
)
@@ -126,3 +127,46 @@ func TestBuildContainerConfig_ImageTagOverrideWins(t *testing.T) {
t.Errorf("Image = %q, want kandev/agent:custom (profile override must win over rt.Image)", got.Image)
}
}
func TestBuildContainerConfig_PublishesAgentctlPorts(t *testing.T) {
cm := newCMTest(t)
cfg := ContainerConfig{
AgentConfig: newConfigStubAgent(),
InstanceID: "0123456789abcdef",
TaskID: "task-1",
}
got, err := cm.buildContainerConfig(cfg)
if err != nil {
t.Fatalf("buildContainerConfig: %v", err)
}
if len(got.PortBindings) == 0 {
t.Fatal("expected agentctl ports to be published")
}
assertHasPortBinding(t, got.PortBindings, AgentCtlPort)
assertHasPortBinding(t, got.PortBindings, dockerAgentctlInstancePortBase)
assertHasPortBinding(t, got.PortBindings, dockerAgentctlInstancePortMax)
assertEnvContains(t, got.Env, "AGENTCTL_INSTANCE_PORT_BASE=41001")
assertEnvContains(t, got.Env, "AGENTCTL_INSTANCE_PORT_MAX=41100")
}
func assertHasPortBinding(t *testing.T, bindings []docker.PortBindingConfig, port int) {
t.Helper()
for _, binding := range bindings {
if binding.ContainerPort == port && binding.HostIP == "127.0.0.1" && binding.HostPort == "0" {
return
}
}
t.Fatalf("missing published port binding for %d/tcp: %#v", port, bindings)
}
func assertEnvContains(t *testing.T, env []string, want string) {
t.Helper()
for _, item := range env {
if item == want {
return
}
}
t.Fatalf("missing env %q in %#v", want, env)
}
@@ -216,20 +216,23 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient
return nil, fmt.Errorf("failed to get IP for container %s: %w", containerName, err)
}
controlHost, controlPort := resolveDockerEndpoint(ctx, dockerClient, containerName, AgentCtlPort, containerIP, r.logger)
// Check if agentctl is healthy
ctl := agentctl.NewControlClient(containerIP, AgentCtlPort, r.logger)
ctl := agentctl.NewControlClient(controlHost, controlPort, r.logger)
if err := ctl.Health(ctx); err != nil {
return nil, fmt.Errorf("agentctl not healthy in container %s: %w", containerName, err)
}
// Check if the previous instance is still alive
instancePort, reusingProcess, err := r.findExistingInstance(ctx, ctl, containerIP, prevID)
instancePort, reusingProcess, err := r.findExistingInstance(ctx, dockerClient, ctl, containerName, containerIP, prevID)
if err != nil {
return nil, fmt.Errorf("failed to find instance in container %s: %w", containerName, err)
}
instanceHost, resolvedInstancePort := resolveDockerEndpoint(ctx, dockerClient, containerName, instancePort, containerIP, r.logger)
// Create client pointing to the instance port
client := agentctl.NewClient(containerIP, instancePort, r.logger,
client := agentctl.NewClient(instanceHost, resolvedInstancePort, r.logger,
agentctl.WithExecutionID(req.InstanceID),
agentctl.WithSessionID(req.SessionID))
@@ -248,7 +251,10 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient
zap.String("container_name", containerName),
zap.String("container_id", info.ID),
zap.String("container_ip", containerIP),
zap.Int("instance_port", instancePort),
zap.String("control_host", controlHost),
zap.Int("control_port", controlPort),
zap.String("instance_host", instanceHost),
zap.Int("instance_port", resolvedInstancePort),
zap.Bool("reusing_process", reusingProcess))
return &ExecutorInstance{
@@ -266,12 +272,20 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient
// findExistingInstance checks if a previous instance is still running in the container.
// Returns the instance port and whether the agent subprocess is also running.
func (r *DockerExecutor) findExistingInstance(ctx context.Context, ctl *agentctl.ControlClient, containerIP, prevExecutionID string) (int, bool, error) {
func (r *DockerExecutor) findExistingInstance(
ctx context.Context,
dockerClient *docker.Client,
ctl *agentctl.ControlClient,
containerID string,
containerIP string,
prevExecutionID string,
) (int, bool, error) {
// Try to get the existing instance by its ID
instance, err := ctl.GetInstance(ctx, prevExecutionID)
if err == nil && instance != nil && instance.Port > 0 {
// Instance exists, check if agent subprocess is running
client := agentctl.NewClient(containerIP, instance.Port, r.logger)
instanceHost, instancePort := resolveDockerEndpoint(ctx, dockerClient, containerID, instance.Port, containerIP, r.logger)
client := agentctl.NewClient(instanceHost, instancePort, r.logger)
status, statusErr := client.GetStatus(ctx)
processRunning := statusErr == nil && status != nil && status.IsAgentRunning()
return instance.Port, processRunning, nil
@@ -290,6 +304,26 @@ func (r *DockerExecutor) findExistingInstance(ctx context.Context, ctl *agentctl
return resp.Port, false, nil
}
func resolveDockerEndpoint(
ctx context.Context,
dockerClient *docker.Client,
containerID string,
containerPort int,
fallbackHost string,
log *logger.Logger,
) (string, int) {
host, port, err := dockerClient.GetContainerHostPort(ctx, containerID, containerPort)
if err == nil {
return host, port
}
log.Warn("failed to resolve published Docker port, falling back to container IP",
zap.String("container_id", containerID),
zap.Int("container_port", containerPort),
zap.String("fallback_host", fallbackHost),
zap.Error(err))
return fallbackHost, containerPort
}
func (r *DockerExecutor) StopInstance(ctx context.Context, instance *ExecutorInstance, force bool) error {
if instance.ContainerID == "" {
return nil // No container to stop