dreampiler/kandev
1
0
Fork 0
Code Issues 43 Pull Requests Actions 6 Packages Projects Releases Wiki Activity

feat: Implement image paste functionality for Claude Code (#188)

Browse Source
This commit is contained in:
João Salavisa
2026-02-05 22:34:30 +00:00
committed by GitHub
parent 06f7fe9795
commit 93e3759c5f
26 changed files with 503 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/cmd/kandev/adapters.go
+6 -4
View File
@@ -167,8 +167,9 @@ func (a *lifecycleAdapter) ListAgentTypes(ctx context.Context) ([]*v1.AgentType,
}
// PromptAgent sends a follow-up prompt to a running agent
func (a *lifecycleAdapter) PromptAgent(ctx context.Context, agentInstanceID string, prompt string) (*executor.PromptResult, error) {
result, err := a.mgr.PromptAgent(ctx, agentInstanceID, prompt)
// Attachments (images) are passed to the agent if provided
func (a *lifecycleAdapter) PromptAgent(ctx context.Context, agentInstanceID string, prompt string, attachments []v1.MessageAttachment) (*executor.PromptResult, error) {
result, err := a.mgr.PromptAgent(ctx, agentInstanceID, prompt, attachments)
if err != nil {
return nil, err
}
@@ -219,8 +220,9 @@ type orchestratorWrapper struct {
}
// PromptTask forwards directly to the orchestrator service.
func (w *orchestratorWrapper) PromptTask(ctx context.Context, taskID, taskSessionID, prompt, model string, planMode bool) (*orchestrator.PromptResult, error) {
return w.svc.PromptTask(ctx, taskID, taskSessionID, prompt, model, planMode)
// Attachments (images) are passed through to the agent.
func (w *orchestratorWrapper) PromptTask(ctx context.Context, taskID, taskSessionID, prompt, model string, planMode bool, attachments []v1.MessageAttachment) (*orchestrator.PromptResult, error) {
return w.svc.PromptTask(ctx, taskID, taskSessionID, prompt, model, planMode, attachments)
}
// ResumeTaskSession forwards to the orchestrator service, discarding the TaskExecution result.
apps/backend/internal/agent/lifecycle/manager.go
+3 -2
View File
@@ -1682,12 +1682,13 @@ func (m *Manager) updateExecutionError(executionID, errorMsg string) {
}
// PromptAgent sends a follow-up prompt to a running agent
func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt string) (*PromptResult, error) {
// Attachments (images) are passed to the agent if provided
func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt string, attachments []v1.MessageAttachment) (*PromptResult, error) {
execution, exists := m.executionStore.Get(executionID)
if !exists {
return nil, fmt.Errorf("execution %q not found", executionID)
}
return m.sessionManager.SendPrompt(ctx, execution, prompt, true, m.MarkReady)
return m.sessionManager.SendPrompt(ctx, execution, prompt, true, attachments, m.MarkReady)
}
// CancelAgent interrupts the current agent turn without terminating the process,
apps/backend/internal/agent/lifecycle/session.go
+7 -3
View File
@@ -334,7 +334,8 @@ func (sm *SessionManager) InitializeAndPrompt(
promptCtx, cancel := appctx.Detached(ctx, sm.stopCh, constants.PromptTimeout)
defer cancel()
_, err := sm.SendPrompt(promptCtx, execution, effectivePrompt, false, markReady)
// Initial prompt has no attachments
_, err := sm.SendPrompt(promptCtx, execution, effectivePrompt, false, nil, markReady)
if err != nil {
sm.logger.Error("initial prompt failed",
zap.String("execution_id", execution.ID),
@@ -369,12 +370,14 @@ func (sm *SessionManager) InitializeAndPrompt(
// SendPrompt sends a prompt to an agent execution and waits for completion.
// For initial prompts, pass validateStatus=false. For follow-up prompts, pass validateStatus=true.
// Attachments (images) are passed to the agent if provided.
// Returns the prompt result containing the stop reason and agent message.
func (sm *SessionManager) SendPrompt(
ctx context.Context,
execution *AgentExecution,
prompt string,
validateStatus bool,
attachments []v1.MessageAttachment,
markReady func(executionID string) error,
) (*PromptResult, error) {
if execution.agentctl == nil {
@@ -436,7 +439,8 @@ func (sm *SessionManager) SendPrompt(
sm.logger.Info("sending prompt to agent",
zap.String("execution_id", execution.ID),
zap.Int("prompt_length", len(effectivePrompt)))
zap.Int("prompt_length", len(effectivePrompt)),
zap.Int("attachments_count", len(attachments)))
// Store user prompt to session history for context injection (store original, not with injected context)
if sm.historyManager != nil && execution.SessionID != "" {
@@ -446,7 +450,7 @@ func (sm *SessionManager) SendPrompt(
}
// Prompt is synchronous - blocks until agent completes
resp, err := execution.agentctl.Prompt(ctx, effectivePrompt)
resp, err := execution.agentctl.Prompt(ctx, effectivePrompt, attachments)
if err != nil {
sm.logger.Error("ACP prompt failed",
zap.String("execution_id", execution.ID),
apps/backend/internal/agentctl/client/agent.go
+6 -3
View File
@@ -13,6 +13,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/constants"
v1 "github.com/kandev/kandev/pkg/api/v1"
ws "github.com/kandev/kandev/pkg/websocket"
"go.uber.org/zap"
)
@@ -201,10 +202,12 @@ type PromptResponse struct {
// Prompt sends a prompt to the agent and returns when the agent completes
// The StopReason indicates why the agent stopped (e.g., "end_turn", "needs_input")
func (c *Client) Prompt(ctx context.Context, text string) (*PromptResponse, error) {
// Attachments (images) are passed to the agent if provided
func (c *Client) Prompt(ctx context.Context, text string, attachments []v1.MessageAttachment) (*PromptResponse, error) {
reqBody := struct {
Text string `json:"text"`
}{Text: text}
Text string `json:"text"`
Attachments []v1.MessageAttachment `json:"attachments,omitempty"`
}{Text: text, Attachments: attachments}
body, err := json.Marshal(reqBody)
if err != nil {
apps/backend/internal/agentctl/server/adapter/adapter.go
+3 -1
View File
@@ -16,6 +16,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared"
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// Re-export permission types from the shared types package for convenience.
@@ -118,7 +119,8 @@ type AgentAdapter interface {
// Prompt sends a prompt to the agent.
// The agent's responses are streamed via the Updates channel.
Prompt(ctx context.Context, message string) error
// Attachments (images) are passed to the agent if provided.
Prompt(ctx context.Context, message string, attachments []v1.MessageAttachment) error
// Cancel cancels the current operation.
Cancel(ctx context.Context) error
apps/backend/internal/agentctl/server/adapter/amp_adapter.go
+3 -1
View File
@@ -13,6 +13,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
v1 "github.com/kandev/kandev/pkg/api/v1"
"github.com/kandev/kandev/pkg/amp"
"go.uber.org/zap"
)
@@ -183,7 +184,8 @@ func (a *AmpAdapter) LoadSession(ctx context.Context, sessionID string) error {
}
// Prompt sends a prompt to Amp and waits for completion.
func (a *AmpAdapter) Prompt(ctx context.Context, message string) error {
// Note: attachments are not yet supported in Amp protocol - they are ignored.
func (a *AmpAdapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error {
a.mu.Lock()
sessionID := a.sessionID
operationID := uuid.New().String()
apps/backend/internal/agentctl/server/adapter/copilot_adapter.go
+3 -1
View File
@@ -17,6 +17,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
v1 "github.com/kandev/kandev/pkg/api/v1"
"github.com/kandev/kandev/pkg/copilot"
"go.uber.org/zap"
)
@@ -328,7 +329,8 @@ func (a *CopilotAdapter) LoadSession(ctx context.Context, sessionID string) erro
}
// Prompt sends a prompt to Copilot and waits for completion.
func (a *CopilotAdapter) Prompt(ctx context.Context, message string) error {
// Note: attachments are not yet supported in Copilot protocol - they are ignored.
func (a *CopilotAdapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error {
if a.client == nil {
return fmt.Errorf("adapter not initialized")
}
apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go
+18 -3
View File
@@ -16,6 +16,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
v1 "github.com/kandev/kandev/pkg/api/v1"
"go.uber.org/zap"
)
@@ -288,8 +289,9 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error {
// Prompt sends a prompt to the agent.
// If pending context is set (from SetPendingContext), it will be prepended to the message.
// Attachments (images) are converted to ACP ImageBlocks and included in the prompt.
// When the prompt completes, a complete event is emitted via the updates channel.
func (a *Adapter) Prompt(ctx context.Context, message string) error {
func (a *Adapter) Prompt(ctx context.Context, message string, attachments []v1.MessageAttachment) error {
a.mu.Lock()
conn := a.acpConn
sessionID := a.sessionID
@@ -310,11 +312,24 @@ func (a *Adapter) Prompt(ctx context.Context, message string) error {
zap.Int("context_length", len(pendingContext)))
}
a.logger.Info("sending prompt", zap.String("session_id", sessionID))
// Build content blocks: text first, then images
contentBlocks := []acp.ContentBlock{acp.TextBlock(finalMessage)}
// Add image attachments as ImageBlocks
for _, att := range attachments {
if att.Type == "image" {
contentBlocks = append(contentBlocks, acp.ImageBlock(att.Data, att.MimeType))
}
}
a.logger.Info("sending prompt",
zap.String("session_id", sessionID),
zap.Int("content_blocks", len(contentBlocks)),
zap.Int("image_attachments", len(attachments)))
_, err := conn.Prompt(ctx, acp.PromptRequest{
SessionId: acp.SessionId(sessionID),
Prompt: []acp.ContentBlock{acp.TextBlock(finalMessage)},
Prompt: contentBlocks,
})
if err != nil {
return err
apps/backend/internal/agentctl/server/adapter/transport/codex/adapter.go
+3 -1
View File
@@ -14,6 +14,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
v1 "github.com/kandev/kandev/pkg/api/v1"
"github.com/kandev/kandev/pkg/codex"
"go.uber.org/zap"
)
@@ -382,7 +383,8 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error {
// Prompt sends a prompt to the agent, starting a new turn.
// This method blocks until the turn completes (turn/completed notification received).
func (a *Adapter) Prompt(ctx context.Context, message string) error {
// Note: attachments are not yet supported in Codex protocol - they are ignored.
func (a *Adapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error {
a.mu.Lock()
client := a.client
threadID := a.threadID
apps/backend/internal/agentctl/server/adapter/transport/opencode/adapter.go
+3 -1
View File
@@ -19,6 +19,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
v1 "github.com/kandev/kandev/pkg/api/v1"
"github.com/kandev/kandev/pkg/opencode"
"go.uber.org/zap"
)
@@ -350,7 +351,8 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error {
}
// Prompt sends a prompt to OpenCode and waits for completion.
func (a *Adapter) Prompt(ctx context.Context, message string) error {
// Note: attachments are not yet supported in OpenCode protocol - they are ignored.
func (a *Adapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error {
a.mu.Lock()
client := a.client
sessionID := a.sessionID
apps/backend/internal/agentctl/server/adapter/transport/streamjson/adapter.go
+105 -3
View File
@@ -4,9 +4,12 @@ package streamjson
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
@@ -16,6 +19,7 @@ import (
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
v1 "github.com/kandev/kandev/pkg/api/v1"
"github.com/kandev/kandev/pkg/claudecode"
"go.uber.org/zap"
)
@@ -277,12 +281,16 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error {
}
// Prompt sends a prompt and waits for completion.
func (a *Adapter) Prompt(ctx context.Context, message string) error {
// Since stream-json protocol doesn't support multimodal content blocks, image attachments
// are saved to temp files in the workspace and the prompt is modified to reference them.
// Claude Code can then read these files with its Read tool.
func (a *Adapter) Prompt(ctx context.Context, message string, attachments []v1.MessageAttachment) error {
a.mu.Lock()
client := a.client
sessionID := a.sessionID
operationID := uuid.New().String()
a.operationID = operationID
workDir := a.cfg.WorkDir
// Create channel to wait for result
a.resultCh = make(chan resultComplete, 1)
a.mu.Unlock()
@@ -291,12 +299,24 @@ func (a *Adapter) Prompt(ctx context.Context, message string) error {
return fmt.Errorf("adapter not initialized")
}
// Handle image attachments by saving to temp files
finalMessage := message
if len(attachments) > 0 {
imagePaths, err := a.saveImageAttachments(workDir, attachments)
if err != nil {
a.logger.Warn("failed to save image attachments", zap.Error(err))
} else if len(imagePaths) > 0 {
finalMessage = a.buildMessageWithImages(message, imagePaths)
}
}
a.logger.Info("sending prompt",
zap.String("session_id", sessionID),
zap.String("operation_id", operationID))
zap.String("operation_id", operationID),
zap.Int("attachments", len(attachments)))
// Send user message
if err := client.SendUserMessage(message); err != nil {
if err := client.SendUserMessage(finalMessage); err != nil {
a.mu.Lock()
a.resultCh = nil
a.mu.Unlock()
@@ -328,6 +348,88 @@ func (a *Adapter) Prompt(ctx context.Context, message string) error {
}
}
// saveImageAttachments saves image attachments to temp files in the workspace.
// Returns the relative paths to the saved files.
func (a *Adapter) saveImageAttachments(workDir string, attachments []v1.MessageAttachment) ([]string, error) {
var imagePaths []string
// Create temp directory for images
tempDir := filepath.Join(workDir, ".kandev", "temp", "images")
if err := os.MkdirAll(tempDir, 0755); err != nil {
return nil, fmt.Errorf("failed to create temp directory: %w", err)
}
for _, att := range attachments {
if att.Type != "image" {
continue
}
// Decode base64 data
imageData, err := base64.StdEncoding.DecodeString(att.Data)
if err != nil {
a.logger.Warn("failed to decode image attachment", zap.Error(err))
continue
}
// Determine file extension from mime type
ext := ".png" // default
switch att.MimeType {
case "image/jpeg":
ext = ".jpg"
case "image/gif":
ext = ".gif"
case "image/webp":
ext = ".webp"
case "image/png":
ext = ".png"
}
// Generate unique filename
filename := fmt.Sprintf("image-%s%s", uuid.New().String()[:8], ext)
filePath := filepath.Join(tempDir, filename)
// Write image to file
if err := os.WriteFile(filePath, imageData, 0644); err != nil {
a.logger.Warn("failed to write image file", zap.Error(err), zap.String("path", filePath))
continue
}
// Store relative path from workspace root
relPath, err := filepath.Rel(workDir, filePath)
if err != nil {
relPath = filePath // Fall back to absolute path
}
imagePaths = append(imagePaths, relPath)
a.logger.Info("saved image attachment",
zap.String("path", relPath),
zap.Int("size", len(imageData)))
}
return imagePaths, nil
}
// buildMessageWithImages prepends image file references to the user's message.
func (a *Adapter) buildMessageWithImages(message string, imagePaths []string) string {
var sb strings.Builder
// Add image references at the beginning
if len(imagePaths) == 1 {
sb.WriteString(fmt.Sprintf("I've attached an image file that you should read and analyze: %s\n\n", imagePaths[0]))
} else {
sb.WriteString("I've attached image files that you should read and analyze:\n")
for _, path := range imagePaths {
sb.WriteString(fmt.Sprintf("- %s\n", path))
}
sb.WriteString("\n")
}
// Add the original message
sb.WriteString(message)
return sb.String()
}
// Cancel interrupts the current operation.
func (a *Adapter) Cancel(ctx context.Context) error {
a.mu.RLock()
apps/backend/internal/agentctl/server/api/agent.go
+5 -3
View File
@@ -12,6 +12,7 @@ import (
"github.com/gorilla/websocket"
"github.com/kandev/kandev/internal/agentctl/types"
"github.com/kandev/kandev/internal/common/constants"
v1 "github.com/kandev/kandev/pkg/api/v1"
ws "github.com/kandev/kandev/pkg/websocket"
"go.uber.org/zap"
)
@@ -216,7 +217,8 @@ func (s *Server) handleAgentLoadSession(c *gin.Context) {
// PromptRequest is a request to send a prompt to the agent
type PromptRequest struct {
Text string `json:"text"` // Simple text prompt
Text string `json:"text"` // Simple text prompt
Attachments []v1.MessageAttachment `json:"attachments,omitempty"` // Optional image attachments
}
// PromptResponse is the response to a prompt call
@@ -258,7 +260,7 @@ func (s *Server) handleAgentPrompt(c *gin.Context) {
ctx, cancel := context.WithTimeout(c.Request.Context(), constants.PromptTimeout)
defer cancel()
err := adapter.Prompt(ctx, req.Text)
err := adapter.Prompt(ctx, req.Text, req.Attachments)
if err != nil {
s.logger.Error("prompt failed", zap.Error(err))
c.JSON(http.StatusInternalServerError, PromptResponse{
@@ -268,7 +270,7 @@ func (s *Server) handleAgentPrompt(c *gin.Context) {
return
}
s.logger.Info("prompt completed")
s.logger.Info("prompt completed", zap.Int("attachments", len(req.Attachments)))
c.JSON(http.StatusOK, PromptResponse{
Success: true,
apps/backend/internal/gateway/websocket/client.go
+2 -1
View File
@@ -24,7 +24,8 @@ const (
pingPeriod = (pongWait * 9) / 10
// Maximum message size allowed from peer
maxMessageSize = 512 * 1024 // 512KB
// Increased to support image attachments (base64 encoded images are ~33% larger)
maxMessageSize = 32 * 1024 * 1024 // 32MB
)
// Client represents a single WebSocket connection
apps/backend/internal/integration/orchestrator_test.go
+2 -1
View File
@@ -299,7 +299,8 @@ func (s *SimulatedAgentManagerClient) StopAgent(ctx context.Context, agentExecut
}
// PromptAgent sends a follow-up prompt to a running agent
func (s *SimulatedAgentManagerClient) PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*executor.PromptResult, error) {
// Note: attachments parameter is accepted but not used in simulation
func (s *SimulatedAgentManagerClient) PromptAgent(ctx context.Context, agentExecutionID string, prompt string, _ []v1.MessageAttachment) (*executor.PromptResult, error) {
s.mu.Lock()
execution, exists := s.instances[agentExecutionID]
s.mu.Unlock()
apps/backend/internal/orchestrator/executor/executor.go
+7 -4
View File
@@ -44,7 +44,8 @@ type AgentManagerClient interface {
// PromptAgent sends a prompt to a running agent
// Returns PromptResult indicating if the agent needs input
PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*PromptResult, error)
// Attachments (images) are passed to the agent if provided
PromptAgent(ctx context.Context, agentExecutionID string, prompt string, attachments []v1.MessageAttachment) (*PromptResult, error)
// CancelAgent interrupts the current agent turn without terminating the process.
CancelAgent(ctx context.Context, sessionID string) error
@@ -877,7 +878,8 @@ func (e *Executor) StopByTaskID(ctx context.Context, taskID string, reason strin
// Prompt sends a follow-up prompt to a running agent for a task
// Returns PromptResult indicating if the agent needs input
func (e *Executor) Prompt(ctx context.Context, taskID, sessionID string, prompt string) (*PromptResult, error) {
// Attachments (images) are passed to the agent if provided
func (e *Executor) Prompt(ctx context.Context, taskID, sessionID string, prompt string, attachments []v1.MessageAttachment) (*PromptResult, error) {
session, err := e.repo.GetTaskSession(ctx, sessionID)
if err != nil {
return nil, ErrExecutionNotFound
@@ -893,9 +895,10 @@ func (e *Executor) Prompt(ctx context.Context, taskID, sessionID string, prompt
zap.String("task_id", taskID),
zap.String("session_id", sessionID),
zap.String("agent_execution_id", session.AgentExecutionID),
zap.Int("prompt_length", len(prompt)))
zap.Int("prompt_length", len(prompt)),
zap.Int("attachments_count", len(attachments)))
return e.agentManager.PromptAgent(ctx, session.AgentExecutionID, prompt)
return e.agentManager.PromptAgent(ctx, session.AgentExecutionID, prompt, attachments)
}
// SwitchModel stops the current agent, restarts it with a new model, and sends the prompt.
apps/backend/internal/orchestrator/executor/executor_test.go
+1 -1
View File
@@ -39,7 +39,7 @@ func (m *mockAgentManager) StopAgent(ctx context.Context, agentExecutionID strin
return nil
}
func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*PromptResult, error) {
func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string, _ []v1.MessageAttachment) (*PromptResult, error) {
return nil, nil
}
apps/backend/internal/orchestrator/handlers/handlers.go
+1 -1
View File
@@ -243,7 +243,7 @@ func (h *Handlers) wsPromptTask(ctx context.Context, msg *ws.Message) (*ws.Messa
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "prompt is required", nil)
}
result, err := h.service.PromptTask(ctx, req.TaskID, req.TaskSessionID, req.Prompt, req.Model, false)
result, err := h.service.PromptTask(ctx, req.TaskID, req.TaskSessionID, req.Prompt, req.Model, false, nil)
if err != nil {
h.logger.Error("failed to send prompt", zap.String("task_id", req.TaskID), zap.Error(err))
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to send prompt: "+err.Error(), nil)
apps/backend/internal/orchestrator/scheduler/scheduler_test.go
+1 -1
View File
@@ -51,7 +51,7 @@ func (m *mockAgentManager) StopAgent(ctx context.Context, agentExecutionID strin
return nil
}
func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*executor.PromptResult, error) {
func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string, _ []v1.MessageAttachment) (*executor.PromptResult, error) {
return &executor.PromptResult{StopReason: "end_turn"}, nil
}
apps/backend/internal/orchestrator/task_operations.go
+7 -4
View File
@@ -390,7 +390,8 @@ func (s *Service) StartSessionForWorkflowStep(ctx context.Context, taskID, sessi
// Send the prompt using PromptTask (with planMode from step)
// Note: PromptTask internally uses context.WithoutCancel for the executor call
_, err = s.PromptTask(ctx, taskID, sessionID, effectivePrompt, "", step.PlanMode)
// No attachments for workflow-initiated prompts
_, err = s.PromptTask(ctx, taskID, sessionID, effectivePrompt, "", step.PlanMode, nil)
if err != nil {
return fmt.Errorf("failed to prompt session: %w", err)
}
@@ -527,13 +528,15 @@ func (s *Service) StopSession(ctx context.Context, sessionID string, reason stri
// PromptTask sends a follow-up prompt to a running agent for a task session.
// If planMode is true, a plan mode prefix is prepended to the prompt.
func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool) (*PromptResult, error) {
// Attachments (images) are passed through to the agent if provided.
func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment) (*PromptResult, error) {
s.logger.Debug("PromptTask called",
zap.String("task_id", taskID),
zap.String("session_id", sessionID),
zap.Int("prompt_length", len(prompt)),
zap.String("requested_model", model),
zap.Bool("plan_mode", planMode))
zap.Bool("plan_mode", planMode),
zap.Int("attachments_count", len(attachments)))
if sessionID == "" {
return nil, fmt.Errorf("session_id is required")
}
@@ -610,7 +613,7 @@ func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prom
// Prompts can take a long time (minutes) while the WS request may timeout in 15 seconds.
// We still want to log and respond, but the prompt should continue regardless.
promptCtx := context.WithoutCancel(ctx)
result, err := s.executor.Prompt(promptCtx, taskID, sessionID, effectivePrompt)
result, err := s.executor.Prompt(promptCtx, taskID, sessionID, effectivePrompt, attachments)
if err != nil {
s.logger.Error("prompt failed",
zap.String("task_id", taskID),
apps/backend/internal/task/handlers/message_handlers.go
+23 -11
View File
@@ -23,7 +23,7 @@ import (
// OrchestratorService defines the interface for orchestrator operations
type OrchestratorService interface {
PromptTask(ctx context.Context, taskID, sessionID, prompt, model string, planMode bool) (*orchestrator.PromptResult, error)
PromptTask(ctx context.Context, taskID, sessionID, prompt, model string, planMode bool, attachments []v1.MessageAttachment) (*orchestrator.PromptResult, error)
ResumeTaskSession(ctx context.Context, taskID, taskSessionID string) error
}
@@ -115,12 +115,13 @@ func (h *MessageHandlers) httpListMessages(c *gin.Context) {
// WS handlers
type wsAddMessageRequest struct {
TaskID string `json:"task_id"`
TaskSessionID string `json:"session_id"`
Content string `json:"content"`
AuthorID string `json:"author_id,omitempty"`
Model string `json:"model,omitempty"`
PlanMode bool `json:"plan_mode,omitempty"`
TaskID string `json:"task_id"`
TaskSessionID string `json:"session_id"`
Content string `json:"content"`
AuthorID string `json:"author_id,omitempty"`
Model string `json:"model,omitempty"`
PlanMode bool `json:"plan_mode,omitempty"`
Attachments []v1.MessageAttachment `json:"attachments,omitempty"`
}
func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*ws.Message, error) {
@@ -134,8 +135,9 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w
if req.TaskID == "" {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id is required", nil)
}
if req.Content == "" {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "content is required", nil)
// Content can be empty if there are attachments (image-only messages)
if req.Content == "" && len(req.Attachments) == 0 {
return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "content or attachments are required", nil)
}
// Check if session is currently processing a prompt (RUNNING state)
@@ -171,12 +173,21 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w
}
}
// Build metadata with attachments if present
var metadata map[string]interface{}
if len(req.Attachments) > 0 {
metadata = map[string]interface{}{
"attachments": req.Attachments,
}
}
message, err := h.messageController.CreateMessage(ctx, dto.CreateMessageRequest{
TaskSessionID: req.TaskSessionID,
TaskID: req.TaskID,
Content: req.Content,
AuthorType: "user",
AuthorID: req.AuthorID,
Metadata: metadata,
})
if err != nil {
h.logger.Error("failed to create message", zap.Error(err))
@@ -198,9 +209,10 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w
content := req.Content
model := req.Model
planMode := req.PlanMode
attachments := req.Attachments
go func() {
promptCtx := context.WithoutCancel(ctx)
_, err := h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode)
_, err := h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode, attachments)
if err != nil {
if errors.Is(err, executor.ErrExecutionNotFound) {
if resumeErr := h.orchestrator.ResumeTaskSession(promptCtx, taskID, sessionID); resumeErr != nil {
@@ -211,7 +223,7 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w
} else {
for attempt := 0; attempt < 3; attempt++ {
time.Sleep(500 * time.Millisecond)
_, err = h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode)
_, err = h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode, attachments)
if err == nil {
break
}
apps/backend/pkg/api/v1/task.go
+7
View File
@@ -119,6 +119,13 @@ type TaskEvent struct {
CreatedAt time.Time `json:"created_at"`
}
// MessageAttachment represents an attachment (image, file, etc.) to a message
type MessageAttachment struct {
Type string `json:"type"` // "image"
Data string `json:"data"` // Base64-encoded data
MimeType string `json:"mime_type"` // MIME type (e.g., "image/png")
}
// Message represents a message in a task session (user or agent)
type Message struct {
ID string `json:"id"`
apps/web/components/task/chat/chat-input-container.tsx
+111 -4
View File
@@ -21,11 +21,28 @@ import { SlashCommandMenu } from './slash-command-menu';
import { ClarificationInputOverlay } from './clarification-input-overlay';
import { CommentBlocksContainer } from './comment-block';
import { ChatInputFocusHint } from './chat-input-focus-hint';
import {
ImageAttachmentPreview,
processImageFile,
MAX_IMAGES,
MAX_TOTAL_SIZE,
type ImageAttachment,
} from './image-attachment-preview';
import { useInlineMention } from '@/hooks/use-inline-mention';
import { useInlineSlash } from '@/hooks/use-inline-slash';
import type { Message } from '@/lib/types/http';
import type { DiffComment } from '@/lib/diff/types';
// Re-export ImageAttachment type for consumers
export type { ImageAttachment } from './image-attachment-preview';
// Type for message attachments sent to backend
export type MessageAttachment = {
type: 'image';
data: string; // Base64 data
mime_type: string; // MIME type
};
// Memoized toolbar to prevent re-renders on input value changes
type ChatInputToolbarProps = {
planModeEnabled: boolean;
@@ -146,7 +163,7 @@ export type ChatInputContainerHandle = {
};
type ChatInputContainerProps = {
onSubmit: (message: string, reviewComments?: DiffComment[]) => void;
onSubmit: (message: string, reviewComments?: DiffComment[], attachments?: MessageAttachment[]) => void;
sessionId: string | null;
taskId: string | null;
taskTitle?: string;
@@ -207,6 +224,7 @@ export const ChatInputContainer = forwardRef<ChatInputContainerHandle, ChatInput
// Keep input state local to avoid re-rendering parent on every keystroke
const [value, setValue] = useState('');
const [isInputFocused, setIsInputFocused] = useState(false);
const [attachments, setAttachments] = useState<ImageAttachment[]>([]);
const inputRef = useRef<RichTextInputHandle>(null);
// Expose imperative handle for parent
@@ -237,6 +255,75 @@ export const ChatInputContainer = forwardRef<ChatInputContainerHandle, ChatInput
valueRef.current = value;
}, [value]);
// Ref for attachments to keep handleSubmit stable
const attachmentsRef = useRef(attachments);
useEffect(() => {
attachmentsRef.current = attachments;
}, [attachments]);
// Handle image paste
const handlePaste = useCallback(async (e: ClipboardEvent) => {
const items = e.clipboardData?.items;
if (!items) return;
// Check for image items
const imageItems: DataTransferItem[] = [];
for (const item of items) {
if (item.type.startsWith('image/')) {
imageItems.push(item);
}
}
if (imageItems.length === 0) return;
// Prevent default paste behavior for images
e.preventDefault();
// Check if we've hit the max images limit
if (attachments.length >= MAX_IMAGES) {
console.warn(`Maximum ${MAX_IMAGES} images allowed`);
return;
}
// Calculate current total size
const currentTotalSize = attachments.reduce((sum, att) => sum + att.size, 0);
// Process each image
for (const item of imageItems) {
if (attachments.length + imageItems.indexOf(item) >= MAX_IMAGES) break;
const file = item.getAsFile();
if (!file) continue;
// Check total size limit
if (currentTotalSize + file.size > MAX_TOTAL_SIZE) {
console.warn('Total attachment size limit exceeded');
break;
}
const attachment = await processImageFile(file);
if (attachment) {
setAttachments(prev => [...prev, attachment]);
}
}
}, [attachments]);
// Attach paste listener to the container
useEffect(() => {
const textarea = inputRef.current?.getTextareaElement();
if (!textarea) return;
textarea.addEventListener('paste', handlePaste);
return () => {
textarea.removeEventListener('paste', handlePaste);
};
}, [handlePaste]);
// Remove attachment handler
const handleRemoveAttachment = useCallback((id: string) => {
setAttachments(prev => prev.filter(att => att.id !== id));
}, []);
// Mention menu hook
const mention = useInlineMention(inputRef, value, setValue, sessionId);
@@ -305,6 +392,7 @@ export const ChatInputContainer = forwardRef<ChatInputContainerHandle, ChatInput
const trimmed = valueRef.current.trim();
const comments = pendingCommentsRef.current;
const currentAttachments = attachmentsRef.current;
// Collect all pending comments
const allComments: DiffComment[] = [];
@@ -314,11 +402,23 @@ export const ChatInputContainer = forwardRef<ChatInputContainerHandle, ChatInput
}
}
// Allow submission if there's text OR comments
if (!trimmed && allComments.length === 0) return;
// Allow submission if there's text OR comments OR attachments
if (!trimmed && allComments.length === 0 && currentAttachments.length === 0) return;
onSubmit(trimmed, allComments.length > 0 ? allComments : undefined);
// Convert attachments to MessageAttachment format for backend
const messageAttachments: MessageAttachment[] = currentAttachments.map(att => ({
type: 'image' as const,
data: att.data,
mime_type: att.mimeType,
}));
onSubmit(
trimmed,
allComments.length > 0 ? allComments : undefined,
messageAttachments.length > 0 ? messageAttachments : undefined
);
setValue('');
setAttachments([]); // Clear attachments after submit
}, [onSubmit, isAgentBusy, isSending]);
// Disable input when agent is busy (RUNNING state), starting, or sending a message
@@ -383,6 +483,13 @@ export const ChatInputContainer = forwardRef<ChatInputContainerHandle, ChatInput
</div>
)}
{/* Image attachments preview */}
<ImageAttachmentPreview
attachments={attachments}
onRemove={handleRemoveAttachment}
disabled={isDisabled}
/>
{/* Clarification inline (replaces input area when pending) */}
{hasPendingClarification ? (
<div style={{ overflow: 'auto' }}>
apps/web/components/task/chat/image-attachment-preview.tsx
+131
View File
@@ -0,0 +1,131 @@
'use client';
import { memo } from 'react';
import { IconX } from '@tabler/icons-react';
import { cn } from '@/lib/utils';
export type ImageAttachment = {
id: string;
data: string; // Base64-encoded image data (without data: prefix)
mimeType: string; // "image/png", "image/jpeg", "image/gif", "image/webp"
preview: string; // Data URL for preview display
size: number; // File size in bytes
};
// Supported image types
export const SUPPORTED_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp'];
// Size limits
export const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB per image
export const MAX_TOTAL_SIZE = 20 * 1024 * 1024; // 20MB total
export const MAX_IMAGES = 10;
function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B';
const k = 1024;
const sizes = ['B', 'KB', 'MB', 'GB'];
const i = Math.floor(Math.log(bytes) / Math.log(k));
return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`;
}
type ImageAttachmentPreviewProps = {
attachments: ImageAttachment[];
onRemove: (id: string) => void;
disabled?: boolean;
};
export const ImageAttachmentPreview = memo(function ImageAttachmentPreview({
attachments,
onRemove,
disabled = false,
}: ImageAttachmentPreviewProps) {
if (attachments.length === 0) return null;
return (
<div className="flex gap-1.5 px-3 pt-2 flex-wrap">
{attachments.map((attachment) => (
<div
key={attachment.id}
className={cn(
'relative group rounded-md overflow-hidden border border-border bg-muted/30',
disabled && 'opacity-50'
)}
>
{/* Image thumbnail */}
<img
src={attachment.preview}
alt="Attachment preview"
className="h-10 w-10 object-cover"
/>
{/* Remove button - inside the image bounds */}
{!disabled && (
<button
type="button"
onClick={() => onRemove(attachment.id)}
className={cn(
'absolute top-0.5 right-0.5 p-0.5 rounded-full',
'bg-black/70 text-white',
'opacity-0 group-hover:opacity-100 transition-opacity',
'hover:bg-black/90 cursor-pointer',
'focus:outline-none focus:ring-1 focus:ring-white/50'
)}
aria-label="Remove image"
>
<IconX className="h-2.5 w-2.5" />
</button>
)}
</div>
))}
</div>
);
});
/**
* Process a pasted or dropped file into an ImageAttachment
* Returns null if the file is invalid (wrong type, too large, etc.)
*/
export function processImageFile(file: File): Promise<ImageAttachment | null> {
return new Promise((resolve) => {
// Validate type
if (!SUPPORTED_IMAGE_TYPES.includes(file.type)) {
console.warn(`Unsupported image type: ${file.type}`);
resolve(null);
return;
}
// Validate size
if (file.size > MAX_IMAGE_SIZE) {
console.warn(`Image too large: ${formatBytes(file.size)} (max: ${formatBytes(MAX_IMAGE_SIZE)})`);
resolve(null);
return;
}
const reader = new FileReader();
reader.onload = (event) => {
const dataUrl = event.target?.result as string;
if (!dataUrl) {
resolve(null);
return;
}
// Extract base64 data (remove "data:image/png;base64," prefix)
const base64 = dataUrl.split(',')[1];
const mimeType = dataUrl.split(';')[0].split(':')[1];
resolve({
id: crypto.randomUUID(),
data: base64,
mimeType,
preview: dataUrl,
size: file.size,
});
};
reader.onerror = () => {
console.error('Failed to read image file');
resolve(null);
};
reader.readAsDataURL(file);
});
}
apps/web/components/task/chat/messages/chat-message.tsx
+34 -3
View File
@@ -113,13 +113,44 @@ export const ChatMessage = memo(function ChatMessage({ comment, label, className
// User message: right-aligned bubble with markdown support
if (isUser) {
// Extract image attachments from metadata
const metadata = comment.metadata as { attachments?: Array<{ type: string; data: string; mime_type: string }> } | undefined;
const imageAttachments = (metadata?.attachments || []).filter(att => att.type === 'image');
const hasContent = comment.content && comment.content.trim() !== '';
const hasAttachments = imageAttachments.length > 0;
return (
<div className="flex justify-end w-full overflow-hidden">
<div className="max-w-[85%] sm:max-w-[75%] md:max-w-2xl overflow-hidden">
<div className="rounded-2xl bg-primary/30 px-4 py-2.5 text-xs overflow-hidden">
<p className="whitespace-pre-wrap break-words overflow-wrap-anywhere">
{comment.content ? renderContentWithFileRefs(comment.content) : '(empty)'}
</p>
{/* Display image attachments */}
{hasAttachments && (
<div className={cn('flex flex-wrap gap-2', hasContent && 'mb-2')}>
{imageAttachments.map((att, index) => (
<img
key={index}
src={`data:${att.mime_type};base64,${att.data}`}
alt={`Attachment ${index + 1}`}
className="max-h-48 max-w-full rounded-lg object-contain cursor-pointer hover:opacity-90 transition-opacity"
onClick={() => {
// Open image in new tab for full view
const win = window.open();
if (win) {
win.document.write(`<img src="data:${att.mime_type};base64,${att.data}" style="max-width:100%;height:auto;" />`);
}
}}
/>
))}
</div>
)}
{/* Display text content */}
{hasContent ? (
<p className="whitespace-pre-wrap break-words overflow-wrap-anywhere">
{renderContentWithFileRefs(comment.content)}
</p>
) : !hasAttachments ? (
<p className="whitespace-pre-wrap break-words overflow-wrap-anywhere">(empty)</p>
) : null}
</div>
</div>
</div>
apps/web/components/task/task-chat-panel.tsx
+3 -3
View File
@@ -14,7 +14,7 @@ import { useSessionModel } from '@/hooks/domains/session/use-session-model';
import { useMessageHandler } from '@/hooks/use-message-handler';
import { TodoSummary } from '@/components/task/chat/todo-summary';
import { VirtualizedMessageList } from '@/components/task/chat/virtualized-message-list';
import { ChatInputContainer, type ChatInputContainerHandle } from '@/components/task/chat/chat-input-container';
import { ChatInputContainer, type ChatInputContainerHandle, type MessageAttachment } from '@/components/task/chat/chat-input-container';
import { formatReviewCommentsAsMarkdown } from '@/components/task/chat/messages/review-comments-attachment';
import {
useDiffCommentsStore,
@@ -167,7 +167,7 @@ export const TaskChatPanel = memo(function TaskChatPanel({
}
}, [resolvedSessionId]);
const handleSubmit = useCallback(async (message: string, reviewComments?: DiffComment[]) => {
const handleSubmit = useCallback(async (message: string, reviewComments?: DiffComment[], attachments?: MessageAttachment[]) => {
if (isSending) return;
setIsSending(true);
try {
@@ -181,7 +181,7 @@ export const TaskChatPanel = memo(function TaskChatPanel({
if (onSend) {
await onSend(finalMessage);
} else {
await handleSendMessage(finalMessage);
await handleSendMessage(finalMessage, attachments);
}
// Mark comments as sent and clear pending
apps/web/hooks/use-message-handler.ts
+8 -2
View File
@@ -1,5 +1,6 @@
import { useCallback } from 'react';
import { getWebSocketClient } from '@/lib/ws/connection';
import type { MessageAttachment } from '@/components/task/chat/chat-input-container';
export function useMessageHandler(
resolvedSessionId: string | null,
@@ -9,7 +10,7 @@ export function useMessageHandler(
planMode: boolean = false
) {
const handleSendMessage = useCallback(
async (message: string) => {
async (message: string, attachments?: MessageAttachment[]) => {
if (!taskId || !resolvedSessionId) {
console.error('No active task session. Start an agent before sending a message.');
return;
@@ -20,6 +21,10 @@ export function useMessageHandler(
// Include active model in the request if it differs from the session model
const modelToSend = activeModel && activeModel !== sessionModel ? activeModel : undefined;
// Use longer timeout when sending attachments (images can be large)
const hasAttachments = attachments && attachments.length > 0;
const timeoutMs = hasAttachments ? 30000 : 10000;
await client.request(
'message.add',
{
@@ -28,8 +33,9 @@ export function useMessageHandler(
content: message,
...(modelToSend && { model: modelToSend }),
...(planMode && { plan_mode: true }),
...(hasAttachments && { attachments }),
},
10000
timeoutMs
);
},
[resolvedSessionId, taskId, activeModel, sessionModel, planMode]