refactor(docker): split reconnectToContainer + cover reconnect helpers with unit tests
Address recurring review feedback on the docker reconnect path: - reconnectToContainer was 98 lines (over the 80-line repo limit). Split into ensureContainerRunning (inspect + start + re-inspect) and bringupAgentctl (health-check, find-or-create instance, transparent re-handshake). The orchestration body now reads top-down at 49 lines. - resolveDockerEndpoint and waitForAgentctlHealth grew real branches in this PR (host-port lookup with container-IP fallback; 240×500ms retry loop) but had no direct tests. Introduce narrow hostPortLookup / healthChecker interfaces, satisfied by *docker.Client and *agentctl.ControlClient, so both functions are now driven by stubs that exercise happy-path, fallback, retry-until-healthy, exhaustion, and context-cancellation branches. - Add direct unit tests for the docker package's pure helpers (normalizeDockerHostIP, buildDockerPortBindings) and for dockerAgentctlPortBindings — a regression in the published port range or host IP would silently break container reconnect.
This commit is contained in:
@@ -227,87 +227,33 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
prevID := req.PreviousExecutionID
|
||||
|
||||
info, err := dockerClient.GetContainerInfo(ctx, containerRef)
|
||||
info, containerIP, err := r.ensureContainerRunning(ctx, dockerClient, containerRef)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to inspect container %s: %w", containerRef, err)
|
||||
}
|
||||
if shouldStartExistingDockerContainer(info.State) {
|
||||
r.logger.Info("starting stopped docker container for reconnect",
|
||||
zap.String("container_id", info.ID),
|
||||
zap.String("state", info.State))
|
||||
if err := dockerClient.StartContainer(ctx, info.ID); err != nil {
|
||||
return nil, fmt.Errorf("failed to start container %s: %w", info.ID, err)
|
||||
}
|
||||
info, err = dockerClient.GetContainerInfo(ctx, info.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to inspect started container %s: %w", containerRef, err)
|
||||
}
|
||||
}
|
||||
if info.State != containerStateRunning {
|
||||
return nil, fmt.Errorf("container %s is %s, not %s", containerRef, info.State, containerStateRunning)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get the container IP
|
||||
containerIP, err := dockerClient.GetContainerIP(ctx, info.ID)
|
||||
conn, err := r.bringupAgentctl(ctx, dockerClient, info.ID, containerIP, req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get IP for container %s: %w", info.ID, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
controlHost, controlPort := resolveDockerEndpoint(ctx, dockerClient, info.ID, AgentCtlPort, containerIP, r.logger)
|
||||
|
||||
// Check if agentctl is healthy
|
||||
ctl := agentctl.NewControlClient(controlHost, controlPort, r.logger,
|
||||
agentctl.WithControlAuthToken(req.AuthToken))
|
||||
if err := r.waitForAgentctlHealth(ctx, ctl); err != nil {
|
||||
return nil, fmt.Errorf("agentctl not healthy in container %s: %w", info.ID, err)
|
||||
}
|
||||
|
||||
// Check if the previous instance is still alive
|
||||
authToken := req.AuthToken
|
||||
instanceID := reconnectInstanceID(req, prevID)
|
||||
instancePort, reusingProcess, err := r.findExistingInstance(ctx, dockerClient, ctl, req, info.ID, containerIP, instanceID, authToken)
|
||||
if err != nil && req.BootstrapNonce != "" && isAgentctlAuthError(err) {
|
||||
var handshakeErr error
|
||||
authToken, handshakeErr = ctl.Handshake(ctx, req.BootstrapNonce)
|
||||
if handshakeErr != nil {
|
||||
return nil, fmt.Errorf("agentctl auth failed and re-handshake failed in container %s: %w", info.ID, handshakeErr)
|
||||
}
|
||||
instancePort, reusingProcess, err = r.findExistingInstance(ctx, dockerClient, ctl, req, info.ID, containerIP, instanceID, authToken)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to find instance in container %s: %w", info.ID, err)
|
||||
}
|
||||
instanceHost, resolvedInstancePort := resolveDockerEndpoint(ctx, dockerClient, info.ID, instancePort, containerIP, r.logger)
|
||||
|
||||
// Create client pointing to the instance port
|
||||
client := agentctl.NewClient(instanceHost, resolvedInstancePort, r.logger,
|
||||
client := agentctl.NewClient(conn.instanceHost, conn.instancePort, r.logger,
|
||||
agentctl.WithExecutionID(req.InstanceID),
|
||||
agentctl.WithSessionID(req.SessionID),
|
||||
agentctl.WithAuthToken(authToken))
|
||||
|
||||
metadata := map[string]interface{}{
|
||||
MetadataKeyIsRemote: true,
|
||||
MetadataKeyContainerID: info.ID,
|
||||
"reuse_existing_process": reusingProcess,
|
||||
}
|
||||
|
||||
refreshedAuthToken := ""
|
||||
if authToken != "" && authToken != req.AuthToken {
|
||||
refreshedAuthToken = authToken
|
||||
}
|
||||
agentctl.WithAuthToken(conn.authToken))
|
||||
|
||||
r.logger.Info("reconnected to existing docker container",
|
||||
zap.String("container_ref", containerRef),
|
||||
zap.String("container_id", info.ID),
|
||||
zap.String("container_ip", containerIP),
|
||||
zap.String("control_host", controlHost),
|
||||
zap.Int("control_port", controlPort),
|
||||
zap.String("instance_host", instanceHost),
|
||||
zap.Int("instance_port", resolvedInstancePort),
|
||||
zap.Bool("reusing_process", reusingProcess))
|
||||
zap.String("instance_host", conn.instanceHost),
|
||||
zap.Int("instance_port", conn.instancePort),
|
||||
zap.Bool("reusing_process", conn.reusingProcess))
|
||||
|
||||
refreshedAuthToken := ""
|
||||
if conn.authToken != "" && conn.authToken != req.AuthToken {
|
||||
refreshedAuthToken = conn.authToken
|
||||
}
|
||||
return &ExecutorInstance{
|
||||
InstanceID: req.InstanceID,
|
||||
TaskID: req.TaskID,
|
||||
@@ -317,8 +263,88 @@ func (r *DockerExecutor) reconnectToContainer(ctx context.Context, dockerClient
|
||||
ContainerID: info.ID,
|
||||
ContainerIP: containerIP,
|
||||
WorkspacePath: dockerWorkspacePath,
|
||||
Metadata: metadata,
|
||||
AuthToken: refreshedAuthToken,
|
||||
Metadata: map[string]interface{}{
|
||||
MetadataKeyIsRemote: true,
|
||||
MetadataKeyContainerID: info.ID,
|
||||
"reuse_existing_process": conn.reusingProcess,
|
||||
},
|
||||
AuthToken: refreshedAuthToken,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ensureContainerRunning inspects containerRef, starts it if it's stopped,
|
||||
// re-inspects, and returns the live container info and IP. Errors out if the
|
||||
// container can't be brought to a running state — the caller can't reconnect
|
||||
// to a container that isn't alive.
|
||||
func (r *DockerExecutor) ensureContainerRunning(ctx context.Context, dockerClient *docker.Client, containerRef string) (*docker.ContainerInfo, string, error) {
|
||||
info, err := dockerClient.GetContainerInfo(ctx, containerRef)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to inspect container %s: %w", containerRef, err)
|
||||
}
|
||||
if shouldStartExistingDockerContainer(info.State) {
|
||||
r.logger.Info("starting stopped docker container for reconnect",
|
||||
zap.String("container_id", info.ID),
|
||||
zap.String("state", info.State))
|
||||
if err := dockerClient.StartContainer(ctx, info.ID); err != nil {
|
||||
return nil, "", fmt.Errorf("failed to start container %s: %w", info.ID, err)
|
||||
}
|
||||
info, err = dockerClient.GetContainerInfo(ctx, info.ID)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to inspect started container %s: %w", containerRef, err)
|
||||
}
|
||||
}
|
||||
if info.State != containerStateRunning {
|
||||
return nil, "", fmt.Errorf("container %s is %s, not %s", containerRef, info.State, containerStateRunning)
|
||||
}
|
||||
containerIP, err := dockerClient.GetContainerIP(ctx, info.ID)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("failed to get IP for container %s: %w", info.ID, err)
|
||||
}
|
||||
return info, containerIP, nil
|
||||
}
|
||||
|
||||
// reconnectAgentctlConn captures the resolved endpoint state needed by the
|
||||
// caller after agentctl has been brought up: the instance-level host:port for
|
||||
// the user-facing client, the auth token in effect (which may have been
|
||||
// refreshed via re-handshake), and whether the agent subprocess is reusable.
|
||||
type reconnectAgentctlConn struct {
|
||||
instanceHost string
|
||||
instancePort int
|
||||
authToken string
|
||||
reusingProcess bool
|
||||
}
|
||||
|
||||
// bringupAgentctl health-checks agentctl on the container, finds (or creates)
|
||||
// the agent instance, transparently re-handshakes on a 401, and returns the
|
||||
// resolved instance endpoint for the user-facing client.
|
||||
func (r *DockerExecutor) bringupAgentctl(ctx context.Context, dockerClient *docker.Client, containerID, containerIP string, req *ExecutorCreateRequest) (reconnectAgentctlConn, error) {
|
||||
controlHost, controlPort := resolveDockerEndpoint(ctx, dockerClient, containerID, AgentCtlPort, containerIP, r.logger)
|
||||
ctl := agentctl.NewControlClient(controlHost, controlPort, r.logger,
|
||||
agentctl.WithControlAuthToken(req.AuthToken))
|
||||
if err := r.waitForAgentctlHealth(ctx, ctl); err != nil {
|
||||
return reconnectAgentctlConn{}, fmt.Errorf("agentctl not healthy in container %s: %w", containerID, err)
|
||||
}
|
||||
|
||||
authToken := req.AuthToken
|
||||
instanceID := reconnectInstanceID(req, req.PreviousExecutionID)
|
||||
instancePort, reusingProcess, err := r.findExistingInstance(ctx, dockerClient, ctl, req, containerID, containerIP, instanceID, authToken)
|
||||
if err != nil && req.BootstrapNonce != "" && isAgentctlAuthError(err) {
|
||||
var handshakeErr error
|
||||
authToken, handshakeErr = ctl.Handshake(ctx, req.BootstrapNonce)
|
||||
if handshakeErr != nil {
|
||||
return reconnectAgentctlConn{}, fmt.Errorf("agentctl auth failed and re-handshake failed in container %s: %w", containerID, handshakeErr)
|
||||
}
|
||||
instancePort, reusingProcess, err = r.findExistingInstance(ctx, dockerClient, ctl, req, containerID, containerIP, instanceID, authToken)
|
||||
}
|
||||
if err != nil {
|
||||
return reconnectAgentctlConn{}, fmt.Errorf("failed to find instance in container %s: %w", containerID, err)
|
||||
}
|
||||
instanceHost, resolvedInstancePort := resolveDockerEndpoint(ctx, dockerClient, containerID, instancePort, containerIP, r.logger)
|
||||
return reconnectAgentctlConn{
|
||||
instanceHost: instanceHost,
|
||||
instancePort: resolvedInstancePort,
|
||||
authToken: authToken,
|
||||
reusingProcess: reusingProcess,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -424,10 +450,26 @@ func buildReconnectCreateInstanceRequest(req *ExecutorCreateRequest, instanceID
|
||||
}
|
||||
}
|
||||
|
||||
func (r *DockerExecutor) waitForAgentctlHealth(ctx context.Context, ctl *agentctl.ControlClient) error {
|
||||
const maxRetries = 240
|
||||
const retryDelay = 500 * time.Millisecond
|
||||
// healthChecker is the narrow interface waitForAgentctlHealth needs from the
|
||||
// agentctl control client. *agentctl.ControlClient satisfies it; tests can
|
||||
// stub it without spinning up an HTTP server.
|
||||
type healthChecker interface {
|
||||
Health(ctx context.Context) error
|
||||
}
|
||||
|
||||
const (
|
||||
agentctlHealthMaxRetries = 240
|
||||
agentctlHealthRetryDelay = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
func (r *DockerExecutor) waitForAgentctlHealth(ctx context.Context, ctl healthChecker) error {
|
||||
return waitForAgentctlHealthWith(ctx, ctl, agentctlHealthMaxRetries, agentctlHealthRetryDelay)
|
||||
}
|
||||
|
||||
// waitForAgentctlHealthWith is the parameterised body — exposed for tests so
|
||||
// they can drive the retry loop without a 2-minute wait. The production path
|
||||
// always calls it with the package constants.
|
||||
func waitForAgentctlHealthWith(ctx context.Context, ctl healthChecker, maxRetries int, retryDelay time.Duration) error {
|
||||
var lastErr error
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
if err := ctl.Health(ctx); err == nil {
|
||||
@@ -449,9 +491,16 @@ func (r *DockerExecutor) waitForAgentctlHealth(ctx context.Context, ctl *agentct
|
||||
time.Duration(maxRetries)*retryDelay)
|
||||
}
|
||||
|
||||
// hostPortLookup is the narrow interface resolveDockerEndpoint needs from the
|
||||
// docker client. *docker.Client satisfies it; tests can stub it without
|
||||
// having to spin up a docker daemon.
|
||||
type hostPortLookup interface {
|
||||
GetContainerHostPort(ctx context.Context, containerID string, containerPort int) (string, int, error)
|
||||
}
|
||||
|
||||
func resolveDockerEndpoint(
|
||||
ctx context.Context,
|
||||
dockerClient *docker.Client,
|
||||
dockerClient hostPortLookup,
|
||||
containerID string,
|
||||
containerPort int,
|
||||
fallbackHost string,
|
||||
|
||||
Reference in New Issue
Block a user