diff --git a/apps/backend/cmd/kandev/adapters.go b/apps/backend/cmd/kandev/adapters.go index f8889d2513..92a52f0d12 100644 --- a/apps/backend/cmd/kandev/adapters.go +++ b/apps/backend/cmd/kandev/adapters.go @@ -167,8 +167,9 @@ func (a *lifecycleAdapter) ListAgentTypes(ctx context.Context) ([]*v1.AgentType, } // PromptAgent sends a follow-up prompt to a running agent -func (a *lifecycleAdapter) PromptAgent(ctx context.Context, agentInstanceID string, prompt string) (*executor.PromptResult, error) { - result, err := a.mgr.PromptAgent(ctx, agentInstanceID, prompt) +// Attachments (images) are passed to the agent if provided +func (a *lifecycleAdapter) PromptAgent(ctx context.Context, agentInstanceID string, prompt string, attachments []v1.MessageAttachment) (*executor.PromptResult, error) { + result, err := a.mgr.PromptAgent(ctx, agentInstanceID, prompt, attachments) if err != nil { return nil, err } @@ -219,8 +220,9 @@ type orchestratorWrapper struct { } // PromptTask forwards directly to the orchestrator service. -func (w *orchestratorWrapper) PromptTask(ctx context.Context, taskID, taskSessionID, prompt, model string, planMode bool) (*orchestrator.PromptResult, error) { - return w.svc.PromptTask(ctx, taskID, taskSessionID, prompt, model, planMode) +// Attachments (images) are passed through to the agent. +func (w *orchestratorWrapper) PromptTask(ctx context.Context, taskID, taskSessionID, prompt, model string, planMode bool, attachments []v1.MessageAttachment) (*orchestrator.PromptResult, error) { + return w.svc.PromptTask(ctx, taskID, taskSessionID, prompt, model, planMode, attachments) } // ResumeTaskSession forwards to the orchestrator service, discarding the TaskExecution result. diff --git a/apps/backend/internal/agent/lifecycle/manager.go b/apps/backend/internal/agent/lifecycle/manager.go index 43fad3ce16..beac83ceb9 100644 --- a/apps/backend/internal/agent/lifecycle/manager.go +++ b/apps/backend/internal/agent/lifecycle/manager.go @@ -1682,12 +1682,13 @@ func (m *Manager) updateExecutionError(executionID, errorMsg string) { } // PromptAgent sends a follow-up prompt to a running agent -func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt string) (*PromptResult, error) { +// Attachments (images) are passed to the agent if provided +func (m *Manager) PromptAgent(ctx context.Context, executionID string, prompt string, attachments []v1.MessageAttachment) (*PromptResult, error) { execution, exists := m.executionStore.Get(executionID) if !exists { return nil, fmt.Errorf("execution %q not found", executionID) } - return m.sessionManager.SendPrompt(ctx, execution, prompt, true, m.MarkReady) + return m.sessionManager.SendPrompt(ctx, execution, prompt, true, attachments, m.MarkReady) } // CancelAgent interrupts the current agent turn without terminating the process, diff --git a/apps/backend/internal/agent/lifecycle/session.go b/apps/backend/internal/agent/lifecycle/session.go index 3f56fc3b65..353c452324 100644 --- a/apps/backend/internal/agent/lifecycle/session.go +++ b/apps/backend/internal/agent/lifecycle/session.go @@ -334,7 +334,8 @@ func (sm *SessionManager) InitializeAndPrompt( promptCtx, cancel := appctx.Detached(ctx, sm.stopCh, constants.PromptTimeout) defer cancel() - _, err := sm.SendPrompt(promptCtx, execution, effectivePrompt, false, markReady) + // Initial prompt has no attachments + _, err := sm.SendPrompt(promptCtx, execution, effectivePrompt, false, nil, markReady) if err != nil { sm.logger.Error("initial prompt failed", zap.String("execution_id", execution.ID), @@ -369,12 +370,14 @@ func (sm *SessionManager) InitializeAndPrompt( // SendPrompt sends a prompt to an agent execution and waits for completion. // For initial prompts, pass validateStatus=false. For follow-up prompts, pass validateStatus=true. +// Attachments (images) are passed to the agent if provided. // Returns the prompt result containing the stop reason and agent message. func (sm *SessionManager) SendPrompt( ctx context.Context, execution *AgentExecution, prompt string, validateStatus bool, + attachments []v1.MessageAttachment, markReady func(executionID string) error, ) (*PromptResult, error) { if execution.agentctl == nil { @@ -436,7 +439,8 @@ func (sm *SessionManager) SendPrompt( sm.logger.Info("sending prompt to agent", zap.String("execution_id", execution.ID), - zap.Int("prompt_length", len(effectivePrompt))) + zap.Int("prompt_length", len(effectivePrompt)), + zap.Int("attachments_count", len(attachments))) // Store user prompt to session history for context injection (store original, not with injected context) if sm.historyManager != nil && execution.SessionID != "" { @@ -446,7 +450,7 @@ func (sm *SessionManager) SendPrompt( } // Prompt is synchronous - blocks until agent completes - resp, err := execution.agentctl.Prompt(ctx, effectivePrompt) + resp, err := execution.agentctl.Prompt(ctx, effectivePrompt, attachments) if err != nil { sm.logger.Error("ACP prompt failed", zap.String("execution_id", execution.ID), diff --git a/apps/backend/internal/agentctl/client/agent.go b/apps/backend/internal/agentctl/client/agent.go index 667affab64..c20b40a80a 100644 --- a/apps/backend/internal/agentctl/client/agent.go +++ b/apps/backend/internal/agentctl/client/agent.go @@ -13,6 +13,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/constants" + v1 "github.com/kandev/kandev/pkg/api/v1" ws "github.com/kandev/kandev/pkg/websocket" "go.uber.org/zap" ) @@ -201,10 +202,12 @@ type PromptResponse struct { // Prompt sends a prompt to the agent and returns when the agent completes // The StopReason indicates why the agent stopped (e.g., "end_turn", "needs_input") -func (c *Client) Prompt(ctx context.Context, text string) (*PromptResponse, error) { +// Attachments (images) are passed to the agent if provided +func (c *Client) Prompt(ctx context.Context, text string, attachments []v1.MessageAttachment) (*PromptResponse, error) { reqBody := struct { - Text string `json:"text"` - }{Text: text} + Text string `json:"text"` + Attachments []v1.MessageAttachment `json:"attachments,omitempty"` + }{Text: text, Attachments: attachments} body, err := json.Marshal(reqBody) if err != nil { diff --git a/apps/backend/internal/agentctl/server/adapter/adapter.go b/apps/backend/internal/agentctl/server/adapter/adapter.go index ffb155e123..4e12de18ab 100644 --- a/apps/backend/internal/agentctl/server/adapter/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/adapter.go @@ -16,6 +16,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared" "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" + v1 "github.com/kandev/kandev/pkg/api/v1" ) // Re-export permission types from the shared types package for convenience. @@ -118,7 +119,8 @@ type AgentAdapter interface { // Prompt sends a prompt to the agent. // The agent's responses are streamed via the Updates channel. - Prompt(ctx context.Context, message string) error + // Attachments (images) are passed to the agent if provided. + Prompt(ctx context.Context, message string, attachments []v1.MessageAttachment) error // Cancel cancels the current operation. Cancel(ctx context.Context) error diff --git a/apps/backend/internal/agentctl/server/adapter/amp_adapter.go b/apps/backend/internal/agentctl/server/adapter/amp_adapter.go index e776bf709c..5f09aa73af 100644 --- a/apps/backend/internal/agentctl/server/adapter/amp_adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/amp_adapter.go @@ -13,6 +13,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" + v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/kandev/kandev/pkg/amp" "go.uber.org/zap" ) @@ -183,7 +184,8 @@ func (a *AmpAdapter) LoadSession(ctx context.Context, sessionID string) error { } // Prompt sends a prompt to Amp and waits for completion. -func (a *AmpAdapter) Prompt(ctx context.Context, message string) error { +// Note: attachments are not yet supported in Amp protocol - they are ignored. +func (a *AmpAdapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error { a.mu.Lock() sessionID := a.sessionID operationID := uuid.New().String() diff --git a/apps/backend/internal/agentctl/server/adapter/copilot_adapter.go b/apps/backend/internal/agentctl/server/adapter/copilot_adapter.go index b4fbcbb14a..8c7f78d3dd 100644 --- a/apps/backend/internal/agentctl/server/adapter/copilot_adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/copilot_adapter.go @@ -17,6 +17,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" + v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/kandev/kandev/pkg/copilot" "go.uber.org/zap" ) @@ -328,7 +329,8 @@ func (a *CopilotAdapter) LoadSession(ctx context.Context, sessionID string) erro } // Prompt sends a prompt to Copilot and waits for completion. -func (a *CopilotAdapter) Prompt(ctx context.Context, message string) error { +// Note: attachments are not yet supported in Copilot protocol - they are ignored. +func (a *CopilotAdapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error { if a.client == nil { return fmt.Errorf("adapter not initialized") } diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go index a8b785797a..ea229fc160 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go @@ -16,6 +16,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" + v1 "github.com/kandev/kandev/pkg/api/v1" "go.uber.org/zap" ) @@ -288,8 +289,9 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error { // Prompt sends a prompt to the agent. // If pending context is set (from SetPendingContext), it will be prepended to the message. +// Attachments (images) are converted to ACP ImageBlocks and included in the prompt. // When the prompt completes, a complete event is emitted via the updates channel. -func (a *Adapter) Prompt(ctx context.Context, message string) error { +func (a *Adapter) Prompt(ctx context.Context, message string, attachments []v1.MessageAttachment) error { a.mu.Lock() conn := a.acpConn sessionID := a.sessionID @@ -310,11 +312,24 @@ func (a *Adapter) Prompt(ctx context.Context, message string) error { zap.Int("context_length", len(pendingContext))) } - a.logger.Info("sending prompt", zap.String("session_id", sessionID)) + // Build content blocks: text first, then images + contentBlocks := []acp.ContentBlock{acp.TextBlock(finalMessage)} + + // Add image attachments as ImageBlocks + for _, att := range attachments { + if att.Type == "image" { + contentBlocks = append(contentBlocks, acp.ImageBlock(att.Data, att.MimeType)) + } + } + + a.logger.Info("sending prompt", + zap.String("session_id", sessionID), + zap.Int("content_blocks", len(contentBlocks)), + zap.Int("image_attachments", len(attachments))) _, err := conn.Prompt(ctx, acp.PromptRequest{ SessionId: acp.SessionId(sessionID), - Prompt: []acp.ContentBlock{acp.TextBlock(finalMessage)}, + Prompt: contentBlocks, }) if err != nil { return err diff --git a/apps/backend/internal/agentctl/server/adapter/transport/codex/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/codex/adapter.go index 3d21efc6df..12360efd9f 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/codex/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/codex/adapter.go @@ -14,6 +14,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" + v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/kandev/kandev/pkg/codex" "go.uber.org/zap" ) @@ -382,7 +383,8 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error { // Prompt sends a prompt to the agent, starting a new turn. // This method blocks until the turn completes (turn/completed notification received). -func (a *Adapter) Prompt(ctx context.Context, message string) error { +// Note: attachments are not yet supported in Codex protocol - they are ignored. +func (a *Adapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error { a.mu.Lock() client := a.client threadID := a.threadID diff --git a/apps/backend/internal/agentctl/server/adapter/transport/opencode/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/opencode/adapter.go index e734a13820..5077d2f590 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/opencode/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/opencode/adapter.go @@ -19,6 +19,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" + v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/kandev/kandev/pkg/opencode" "go.uber.org/zap" ) @@ -350,7 +351,8 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error { } // Prompt sends a prompt to OpenCode and waits for completion. -func (a *Adapter) Prompt(ctx context.Context, message string) error { +// Note: attachments are not yet supported in OpenCode protocol - they are ignored. +func (a *Adapter) Prompt(ctx context.Context, message string, _ []v1.MessageAttachment) error { a.mu.Lock() client := a.client sessionID := a.sessionID diff --git a/apps/backend/internal/agentctl/server/adapter/transport/streamjson/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/streamjson/adapter.go index 4b8c4bccc8..1834783676 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/streamjson/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/streamjson/adapter.go @@ -4,9 +4,12 @@ package streamjson import ( "context" + "encoding/base64" "encoding/json" "fmt" "io" + "os" + "path/filepath" "strings" "sync" "time" @@ -16,6 +19,7 @@ import ( "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/agentctl/types/streams" "github.com/kandev/kandev/internal/common/logger" + v1 "github.com/kandev/kandev/pkg/api/v1" "github.com/kandev/kandev/pkg/claudecode" "go.uber.org/zap" ) @@ -277,12 +281,16 @@ func (a *Adapter) LoadSession(ctx context.Context, sessionID string) error { } // Prompt sends a prompt and waits for completion. -func (a *Adapter) Prompt(ctx context.Context, message string) error { +// Since stream-json protocol doesn't support multimodal content blocks, image attachments +// are saved to temp files in the workspace and the prompt is modified to reference them. +// Claude Code can then read these files with its Read tool. +func (a *Adapter) Prompt(ctx context.Context, message string, attachments []v1.MessageAttachment) error { a.mu.Lock() client := a.client sessionID := a.sessionID operationID := uuid.New().String() a.operationID = operationID + workDir := a.cfg.WorkDir // Create channel to wait for result a.resultCh = make(chan resultComplete, 1) a.mu.Unlock() @@ -291,12 +299,24 @@ func (a *Adapter) Prompt(ctx context.Context, message string) error { return fmt.Errorf("adapter not initialized") } + // Handle image attachments by saving to temp files + finalMessage := message + if len(attachments) > 0 { + imagePaths, err := a.saveImageAttachments(workDir, attachments) + if err != nil { + a.logger.Warn("failed to save image attachments", zap.Error(err)) + } else if len(imagePaths) > 0 { + finalMessage = a.buildMessageWithImages(message, imagePaths) + } + } + a.logger.Info("sending prompt", zap.String("session_id", sessionID), - zap.String("operation_id", operationID)) + zap.String("operation_id", operationID), + zap.Int("attachments", len(attachments))) // Send user message - if err := client.SendUserMessage(message); err != nil { + if err := client.SendUserMessage(finalMessage); err != nil { a.mu.Lock() a.resultCh = nil a.mu.Unlock() @@ -328,6 +348,88 @@ func (a *Adapter) Prompt(ctx context.Context, message string) error { } } +// saveImageAttachments saves image attachments to temp files in the workspace. +// Returns the relative paths to the saved files. +func (a *Adapter) saveImageAttachments(workDir string, attachments []v1.MessageAttachment) ([]string, error) { + var imagePaths []string + + // Create temp directory for images + tempDir := filepath.Join(workDir, ".kandev", "temp", "images") + if err := os.MkdirAll(tempDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create temp directory: %w", err) + } + + for _, att := range attachments { + if att.Type != "image" { + continue + } + + // Decode base64 data + imageData, err := base64.StdEncoding.DecodeString(att.Data) + if err != nil { + a.logger.Warn("failed to decode image attachment", zap.Error(err)) + continue + } + + // Determine file extension from mime type + ext := ".png" // default + switch att.MimeType { + case "image/jpeg": + ext = ".jpg" + case "image/gif": + ext = ".gif" + case "image/webp": + ext = ".webp" + case "image/png": + ext = ".png" + } + + // Generate unique filename + filename := fmt.Sprintf("image-%s%s", uuid.New().String()[:8], ext) + filePath := filepath.Join(tempDir, filename) + + // Write image to file + if err := os.WriteFile(filePath, imageData, 0644); err != nil { + a.logger.Warn("failed to write image file", zap.Error(err), zap.String("path", filePath)) + continue + } + + // Store relative path from workspace root + relPath, err := filepath.Rel(workDir, filePath) + if err != nil { + relPath = filePath // Fall back to absolute path + } + + imagePaths = append(imagePaths, relPath) + a.logger.Info("saved image attachment", + zap.String("path", relPath), + zap.Int("size", len(imageData))) + } + + return imagePaths, nil +} + +// buildMessageWithImages prepends image file references to the user's message. +func (a *Adapter) buildMessageWithImages(message string, imagePaths []string) string { + var sb strings.Builder + + // Add image references at the beginning + if len(imagePaths) == 1 { + sb.WriteString(fmt.Sprintf("I've attached an image file that you should read and analyze: %s\n\n", imagePaths[0])) + } else { + sb.WriteString("I've attached image files that you should read and analyze:\n") + for _, path := range imagePaths { + sb.WriteString(fmt.Sprintf("- %s\n", path)) + } + sb.WriteString("\n") + } + + // Add the original message + sb.WriteString(message) + + return sb.String() +} + // Cancel interrupts the current operation. func (a *Adapter) Cancel(ctx context.Context) error { a.mu.RLock() diff --git a/apps/backend/internal/agentctl/server/api/agent.go b/apps/backend/internal/agentctl/server/api/agent.go index 61502c628f..ef5bd114e3 100644 --- a/apps/backend/internal/agentctl/server/api/agent.go +++ b/apps/backend/internal/agentctl/server/api/agent.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/websocket" "github.com/kandev/kandev/internal/agentctl/types" "github.com/kandev/kandev/internal/common/constants" + v1 "github.com/kandev/kandev/pkg/api/v1" ws "github.com/kandev/kandev/pkg/websocket" "go.uber.org/zap" ) @@ -216,7 +217,8 @@ func (s *Server) handleAgentLoadSession(c *gin.Context) { // PromptRequest is a request to send a prompt to the agent type PromptRequest struct { - Text string `json:"text"` // Simple text prompt + Text string `json:"text"` // Simple text prompt + Attachments []v1.MessageAttachment `json:"attachments,omitempty"` // Optional image attachments } // PromptResponse is the response to a prompt call @@ -258,7 +260,7 @@ func (s *Server) handleAgentPrompt(c *gin.Context) { ctx, cancel := context.WithTimeout(c.Request.Context(), constants.PromptTimeout) defer cancel() - err := adapter.Prompt(ctx, req.Text) + err := adapter.Prompt(ctx, req.Text, req.Attachments) if err != nil { s.logger.Error("prompt failed", zap.Error(err)) c.JSON(http.StatusInternalServerError, PromptResponse{ @@ -268,7 +270,7 @@ func (s *Server) handleAgentPrompt(c *gin.Context) { return } - s.logger.Info("prompt completed") + s.logger.Info("prompt completed", zap.Int("attachments", len(req.Attachments))) c.JSON(http.StatusOK, PromptResponse{ Success: true, diff --git a/apps/backend/internal/gateway/websocket/client.go b/apps/backend/internal/gateway/websocket/client.go index 5be24f9fa7..8c76502405 100644 --- a/apps/backend/internal/gateway/websocket/client.go +++ b/apps/backend/internal/gateway/websocket/client.go @@ -24,7 +24,8 @@ const ( pingPeriod = (pongWait * 9) / 10 // Maximum message size allowed from peer - maxMessageSize = 512 * 1024 // 512KB + // Increased to support image attachments (base64 encoded images are ~33% larger) + maxMessageSize = 32 * 1024 * 1024 // 32MB ) // Client represents a single WebSocket connection diff --git a/apps/backend/internal/integration/orchestrator_test.go b/apps/backend/internal/integration/orchestrator_test.go index e1717d437d..0149ca2036 100644 --- a/apps/backend/internal/integration/orchestrator_test.go +++ b/apps/backend/internal/integration/orchestrator_test.go @@ -299,7 +299,8 @@ func (s *SimulatedAgentManagerClient) StopAgent(ctx context.Context, agentExecut } // PromptAgent sends a follow-up prompt to a running agent -func (s *SimulatedAgentManagerClient) PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*executor.PromptResult, error) { +// Note: attachments parameter is accepted but not used in simulation +func (s *SimulatedAgentManagerClient) PromptAgent(ctx context.Context, agentExecutionID string, prompt string, _ []v1.MessageAttachment) (*executor.PromptResult, error) { s.mu.Lock() execution, exists := s.instances[agentExecutionID] s.mu.Unlock() diff --git a/apps/backend/internal/orchestrator/executor/executor.go b/apps/backend/internal/orchestrator/executor/executor.go index a052c72fbe..bed16b7380 100644 --- a/apps/backend/internal/orchestrator/executor/executor.go +++ b/apps/backend/internal/orchestrator/executor/executor.go @@ -44,7 +44,8 @@ type AgentManagerClient interface { // PromptAgent sends a prompt to a running agent // Returns PromptResult indicating if the agent needs input - PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*PromptResult, error) + // Attachments (images) are passed to the agent if provided + PromptAgent(ctx context.Context, agentExecutionID string, prompt string, attachments []v1.MessageAttachment) (*PromptResult, error) // CancelAgent interrupts the current agent turn without terminating the process. CancelAgent(ctx context.Context, sessionID string) error @@ -877,7 +878,8 @@ func (e *Executor) StopByTaskID(ctx context.Context, taskID string, reason strin // Prompt sends a follow-up prompt to a running agent for a task // Returns PromptResult indicating if the agent needs input -func (e *Executor) Prompt(ctx context.Context, taskID, sessionID string, prompt string) (*PromptResult, error) { +// Attachments (images) are passed to the agent if provided +func (e *Executor) Prompt(ctx context.Context, taskID, sessionID string, prompt string, attachments []v1.MessageAttachment) (*PromptResult, error) { session, err := e.repo.GetTaskSession(ctx, sessionID) if err != nil { return nil, ErrExecutionNotFound @@ -893,9 +895,10 @@ func (e *Executor) Prompt(ctx context.Context, taskID, sessionID string, prompt zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.String("agent_execution_id", session.AgentExecutionID), - zap.Int("prompt_length", len(prompt))) + zap.Int("prompt_length", len(prompt)), + zap.Int("attachments_count", len(attachments))) - return e.agentManager.PromptAgent(ctx, session.AgentExecutionID, prompt) + return e.agentManager.PromptAgent(ctx, session.AgentExecutionID, prompt, attachments) } // SwitchModel stops the current agent, restarts it with a new model, and sends the prompt. diff --git a/apps/backend/internal/orchestrator/executor/executor_test.go b/apps/backend/internal/orchestrator/executor/executor_test.go index f70e36d01b..6d07fcc18b 100644 --- a/apps/backend/internal/orchestrator/executor/executor_test.go +++ b/apps/backend/internal/orchestrator/executor/executor_test.go @@ -39,7 +39,7 @@ func (m *mockAgentManager) StopAgent(ctx context.Context, agentExecutionID strin return nil } -func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*PromptResult, error) { +func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string, _ []v1.MessageAttachment) (*PromptResult, error) { return nil, nil } diff --git a/apps/backend/internal/orchestrator/handlers/handlers.go b/apps/backend/internal/orchestrator/handlers/handlers.go index 5ba19fb155..2fea7d998a 100644 --- a/apps/backend/internal/orchestrator/handlers/handlers.go +++ b/apps/backend/internal/orchestrator/handlers/handlers.go @@ -243,7 +243,7 @@ func (h *Handlers) wsPromptTask(ctx context.Context, msg *ws.Message) (*ws.Messa return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "prompt is required", nil) } - result, err := h.service.PromptTask(ctx, req.TaskID, req.TaskSessionID, req.Prompt, req.Model, false) + result, err := h.service.PromptTask(ctx, req.TaskID, req.TaskSessionID, req.Prompt, req.Model, false, nil) if err != nil { h.logger.Error("failed to send prompt", zap.String("task_id", req.TaskID), zap.Error(err)) return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeInternalError, "Failed to send prompt: "+err.Error(), nil) diff --git a/apps/backend/internal/orchestrator/scheduler/scheduler_test.go b/apps/backend/internal/orchestrator/scheduler/scheduler_test.go index 77e942c959..5be4e219f8 100644 --- a/apps/backend/internal/orchestrator/scheduler/scheduler_test.go +++ b/apps/backend/internal/orchestrator/scheduler/scheduler_test.go @@ -51,7 +51,7 @@ func (m *mockAgentManager) StopAgent(ctx context.Context, agentExecutionID strin return nil } -func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string) (*executor.PromptResult, error) { +func (m *mockAgentManager) PromptAgent(ctx context.Context, agentExecutionID string, prompt string, _ []v1.MessageAttachment) (*executor.PromptResult, error) { return &executor.PromptResult{StopReason: "end_turn"}, nil } diff --git a/apps/backend/internal/orchestrator/task_operations.go b/apps/backend/internal/orchestrator/task_operations.go index 47d17ffa91..cae66e384b 100644 --- a/apps/backend/internal/orchestrator/task_operations.go +++ b/apps/backend/internal/orchestrator/task_operations.go @@ -390,7 +390,8 @@ func (s *Service) StartSessionForWorkflowStep(ctx context.Context, taskID, sessi // Send the prompt using PromptTask (with planMode from step) // Note: PromptTask internally uses context.WithoutCancel for the executor call - _, err = s.PromptTask(ctx, taskID, sessionID, effectivePrompt, "", step.PlanMode) + // No attachments for workflow-initiated prompts + _, err = s.PromptTask(ctx, taskID, sessionID, effectivePrompt, "", step.PlanMode, nil) if err != nil { return fmt.Errorf("failed to prompt session: %w", err) } @@ -527,13 +528,15 @@ func (s *Service) StopSession(ctx context.Context, sessionID string, reason stri // PromptTask sends a follow-up prompt to a running agent for a task session. // If planMode is true, a plan mode prefix is prepended to the prompt. -func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool) (*PromptResult, error) { +// Attachments (images) are passed through to the agent if provided. +func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prompt string, model string, planMode bool, attachments []v1.MessageAttachment) (*PromptResult, error) { s.logger.Debug("PromptTask called", zap.String("task_id", taskID), zap.String("session_id", sessionID), zap.Int("prompt_length", len(prompt)), zap.String("requested_model", model), - zap.Bool("plan_mode", planMode)) + zap.Bool("plan_mode", planMode), + zap.Int("attachments_count", len(attachments))) if sessionID == "" { return nil, fmt.Errorf("session_id is required") } @@ -610,7 +613,7 @@ func (s *Service) PromptTask(ctx context.Context, taskID, sessionID string, prom // Prompts can take a long time (minutes) while the WS request may timeout in 15 seconds. // We still want to log and respond, but the prompt should continue regardless. promptCtx := context.WithoutCancel(ctx) - result, err := s.executor.Prompt(promptCtx, taskID, sessionID, effectivePrompt) + result, err := s.executor.Prompt(promptCtx, taskID, sessionID, effectivePrompt, attachments) if err != nil { s.logger.Error("prompt failed", zap.String("task_id", taskID), diff --git a/apps/backend/internal/task/handlers/message_handlers.go b/apps/backend/internal/task/handlers/message_handlers.go index 3412a714eb..d92181d50e 100644 --- a/apps/backend/internal/task/handlers/message_handlers.go +++ b/apps/backend/internal/task/handlers/message_handlers.go @@ -23,7 +23,7 @@ import ( // OrchestratorService defines the interface for orchestrator operations type OrchestratorService interface { - PromptTask(ctx context.Context, taskID, sessionID, prompt, model string, planMode bool) (*orchestrator.PromptResult, error) + PromptTask(ctx context.Context, taskID, sessionID, prompt, model string, planMode bool, attachments []v1.MessageAttachment) (*orchestrator.PromptResult, error) ResumeTaskSession(ctx context.Context, taskID, taskSessionID string) error } @@ -115,12 +115,13 @@ func (h *MessageHandlers) httpListMessages(c *gin.Context) { // WS handlers type wsAddMessageRequest struct { - TaskID string `json:"task_id"` - TaskSessionID string `json:"session_id"` - Content string `json:"content"` - AuthorID string `json:"author_id,omitempty"` - Model string `json:"model,omitempty"` - PlanMode bool `json:"plan_mode,omitempty"` + TaskID string `json:"task_id"` + TaskSessionID string `json:"session_id"` + Content string `json:"content"` + AuthorID string `json:"author_id,omitempty"` + Model string `json:"model,omitempty"` + PlanMode bool `json:"plan_mode,omitempty"` + Attachments []v1.MessageAttachment `json:"attachments,omitempty"` } func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*ws.Message, error) { @@ -134,8 +135,9 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w if req.TaskID == "" { return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "task_id is required", nil) } - if req.Content == "" { - return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "content is required", nil) + // Content can be empty if there are attachments (image-only messages) + if req.Content == "" && len(req.Attachments) == 0 { + return ws.NewError(msg.ID, msg.Action, ws.ErrorCodeValidation, "content or attachments are required", nil) } // Check if session is currently processing a prompt (RUNNING state) @@ -171,12 +173,21 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w } } + // Build metadata with attachments if present + var metadata map[string]interface{} + if len(req.Attachments) > 0 { + metadata = map[string]interface{}{ + "attachments": req.Attachments, + } + } + message, err := h.messageController.CreateMessage(ctx, dto.CreateMessageRequest{ TaskSessionID: req.TaskSessionID, TaskID: req.TaskID, Content: req.Content, AuthorType: "user", AuthorID: req.AuthorID, + Metadata: metadata, }) if err != nil { h.logger.Error("failed to create message", zap.Error(err)) @@ -198,9 +209,10 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w content := req.Content model := req.Model planMode := req.PlanMode + attachments := req.Attachments go func() { promptCtx := context.WithoutCancel(ctx) - _, err := h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode) + _, err := h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode, attachments) if err != nil { if errors.Is(err, executor.ErrExecutionNotFound) { if resumeErr := h.orchestrator.ResumeTaskSession(promptCtx, taskID, sessionID); resumeErr != nil { @@ -211,7 +223,7 @@ func (h *MessageHandlers) wsAddMessage(ctx context.Context, msg *ws.Message) (*w } else { for attempt := 0; attempt < 3; attempt++ { time.Sleep(500 * time.Millisecond) - _, err = h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode) + _, err = h.orchestrator.PromptTask(promptCtx, taskID, sessionID, content, model, planMode, attachments) if err == nil { break } diff --git a/apps/backend/pkg/api/v1/task.go b/apps/backend/pkg/api/v1/task.go index c2242869e9..bc18104d49 100644 --- a/apps/backend/pkg/api/v1/task.go +++ b/apps/backend/pkg/api/v1/task.go @@ -119,6 +119,13 @@ type TaskEvent struct { CreatedAt time.Time `json:"created_at"` } +// MessageAttachment represents an attachment (image, file, etc.) to a message +type MessageAttachment struct { + Type string `json:"type"` // "image" + Data string `json:"data"` // Base64-encoded data + MimeType string `json:"mime_type"` // MIME type (e.g., "image/png") +} + // Message represents a message in a task session (user or agent) type Message struct { ID string `json:"id"` diff --git a/apps/web/components/task/chat/chat-input-container.tsx b/apps/web/components/task/chat/chat-input-container.tsx index 68b2144a98..aad6efcbff 100644 --- a/apps/web/components/task/chat/chat-input-container.tsx +++ b/apps/web/components/task/chat/chat-input-container.tsx @@ -21,11 +21,28 @@ import { SlashCommandMenu } from './slash-command-menu'; import { ClarificationInputOverlay } from './clarification-input-overlay'; import { CommentBlocksContainer } from './comment-block'; import { ChatInputFocusHint } from './chat-input-focus-hint'; +import { + ImageAttachmentPreview, + processImageFile, + MAX_IMAGES, + MAX_TOTAL_SIZE, + type ImageAttachment, +} from './image-attachment-preview'; import { useInlineMention } from '@/hooks/use-inline-mention'; import { useInlineSlash } from '@/hooks/use-inline-slash'; import type { Message } from '@/lib/types/http'; import type { DiffComment } from '@/lib/diff/types'; +// Re-export ImageAttachment type for consumers +export type { ImageAttachment } from './image-attachment-preview'; + +// Type for message attachments sent to backend +export type MessageAttachment = { + type: 'image'; + data: string; // Base64 data + mime_type: string; // MIME type +}; + // Memoized toolbar to prevent re-renders on input value changes type ChatInputToolbarProps = { planModeEnabled: boolean; @@ -146,7 +163,7 @@ export type ChatInputContainerHandle = { }; type ChatInputContainerProps = { - onSubmit: (message: string, reviewComments?: DiffComment[]) => void; + onSubmit: (message: string, reviewComments?: DiffComment[], attachments?: MessageAttachment[]) => void; sessionId: string | null; taskId: string | null; taskTitle?: string; @@ -207,6 +224,7 @@ export const ChatInputContainer = forwardRef([]); const inputRef = useRef(null); // Expose imperative handle for parent @@ -237,6 +255,75 @@ export const ChatInputContainer = forwardRef { + attachmentsRef.current = attachments; + }, [attachments]); + + // Handle image paste + const handlePaste = useCallback(async (e: ClipboardEvent) => { + const items = e.clipboardData?.items; + if (!items) return; + + // Check for image items + const imageItems: DataTransferItem[] = []; + for (const item of items) { + if (item.type.startsWith('image/')) { + imageItems.push(item); + } + } + + if (imageItems.length === 0) return; + + // Prevent default paste behavior for images + e.preventDefault(); + + // Check if we've hit the max images limit + if (attachments.length >= MAX_IMAGES) { + console.warn(`Maximum ${MAX_IMAGES} images allowed`); + return; + } + + // Calculate current total size + const currentTotalSize = attachments.reduce((sum, att) => sum + att.size, 0); + + // Process each image + for (const item of imageItems) { + if (attachments.length + imageItems.indexOf(item) >= MAX_IMAGES) break; + + const file = item.getAsFile(); + if (!file) continue; + + // Check total size limit + if (currentTotalSize + file.size > MAX_TOTAL_SIZE) { + console.warn('Total attachment size limit exceeded'); + break; + } + + const attachment = await processImageFile(file); + if (attachment) { + setAttachments(prev => [...prev, attachment]); + } + } + }, [attachments]); + + // Attach paste listener to the container + useEffect(() => { + const textarea = inputRef.current?.getTextareaElement(); + if (!textarea) return; + + textarea.addEventListener('paste', handlePaste); + return () => { + textarea.removeEventListener('paste', handlePaste); + }; + }, [handlePaste]); + + // Remove attachment handler + const handleRemoveAttachment = useCallback((id: string) => { + setAttachments(prev => prev.filter(att => att.id !== id)); + }, []); + // Mention menu hook const mention = useInlineMention(inputRef, value, setValue, sessionId); @@ -305,6 +392,7 @@ export const ChatInputContainer = forwardRef 0 ? allComments : undefined); + // Convert attachments to MessageAttachment format for backend + const messageAttachments: MessageAttachment[] = currentAttachments.map(att => ({ + type: 'image' as const, + data: att.data, + mime_type: att.mimeType, + })); + + onSubmit( + trimmed, + allComments.length > 0 ? allComments : undefined, + messageAttachments.length > 0 ? messageAttachments : undefined + ); setValue(''); + setAttachments([]); // Clear attachments after submit }, [onSubmit, isAgentBusy, isSending]); // Disable input when agent is busy (RUNNING state), starting, or sending a message @@ -383,6 +483,13 @@ export const ChatInputContainer = forwardRef )} + {/* Image attachments preview */} + + {/* Clarification inline (replaces input area when pending) */} {hasPendingClarification ? (
diff --git a/apps/web/components/task/chat/image-attachment-preview.tsx b/apps/web/components/task/chat/image-attachment-preview.tsx new file mode 100644 index 0000000000..04e7407417 --- /dev/null +++ b/apps/web/components/task/chat/image-attachment-preview.tsx @@ -0,0 +1,131 @@ +'use client'; + +import { memo } from 'react'; +import { IconX } from '@tabler/icons-react'; +import { cn } from '@/lib/utils'; + +export type ImageAttachment = { + id: string; + data: string; // Base64-encoded image data (without data: prefix) + mimeType: string; // "image/png", "image/jpeg", "image/gif", "image/webp" + preview: string; // Data URL for preview display + size: number; // File size in bytes +}; + +// Supported image types +export const SUPPORTED_IMAGE_TYPES = ['image/png', 'image/jpeg', 'image/gif', 'image/webp']; + +// Size limits +export const MAX_IMAGE_SIZE = 10 * 1024 * 1024; // 10MB per image +export const MAX_TOTAL_SIZE = 20 * 1024 * 1024; // 20MB total +export const MAX_IMAGES = 10; + +function formatBytes(bytes: number): string { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / Math.pow(k, i)).toFixed(1))} ${sizes[i]}`; +} + +type ImageAttachmentPreviewProps = { + attachments: ImageAttachment[]; + onRemove: (id: string) => void; + disabled?: boolean; +}; + +export const ImageAttachmentPreview = memo(function ImageAttachmentPreview({ + attachments, + onRemove, + disabled = false, +}: ImageAttachmentPreviewProps) { + if (attachments.length === 0) return null; + + return ( +
+ {attachments.map((attachment) => ( +
+ {/* Image thumbnail */} + Attachment preview + + {/* Remove button - inside the image bounds */} + {!disabled && ( + + )} +
+ ))} +
+ ); +}); + +/** + * Process a pasted or dropped file into an ImageAttachment + * Returns null if the file is invalid (wrong type, too large, etc.) + */ +export function processImageFile(file: File): Promise { + return new Promise((resolve) => { + // Validate type + if (!SUPPORTED_IMAGE_TYPES.includes(file.type)) { + console.warn(`Unsupported image type: ${file.type}`); + resolve(null); + return; + } + + // Validate size + if (file.size > MAX_IMAGE_SIZE) { + console.warn(`Image too large: ${formatBytes(file.size)} (max: ${formatBytes(MAX_IMAGE_SIZE)})`); + resolve(null); + return; + } + + const reader = new FileReader(); + reader.onload = (event) => { + const dataUrl = event.target?.result as string; + if (!dataUrl) { + resolve(null); + return; + } + + // Extract base64 data (remove "data:image/png;base64," prefix) + const base64 = dataUrl.split(',')[1]; + const mimeType = dataUrl.split(';')[0].split(':')[1]; + + resolve({ + id: crypto.randomUUID(), + data: base64, + mimeType, + preview: dataUrl, + size: file.size, + }); + }; + reader.onerror = () => { + console.error('Failed to read image file'); + resolve(null); + }; + reader.readAsDataURL(file); + }); +} + diff --git a/apps/web/components/task/chat/messages/chat-message.tsx b/apps/web/components/task/chat/messages/chat-message.tsx index 2f7bba90b8..c17b0ebff6 100644 --- a/apps/web/components/task/chat/messages/chat-message.tsx +++ b/apps/web/components/task/chat/messages/chat-message.tsx @@ -113,13 +113,44 @@ export const ChatMessage = memo(function ChatMessage({ comment, label, className // User message: right-aligned bubble with markdown support if (isUser) { + // Extract image attachments from metadata + const metadata = comment.metadata as { attachments?: Array<{ type: string; data: string; mime_type: string }> } | undefined; + const imageAttachments = (metadata?.attachments || []).filter(att => att.type === 'image'); + const hasContent = comment.content && comment.content.trim() !== ''; + const hasAttachments = imageAttachments.length > 0; + return (
-

- {comment.content ? renderContentWithFileRefs(comment.content) : '(empty)'} -

+ {/* Display image attachments */} + {hasAttachments && ( +
+ {imageAttachments.map((att, index) => ( + {`Attachment { + // Open image in new tab for full view + const win = window.open(); + if (win) { + win.document.write(``); + } + }} + /> + ))} +
+ )} + {/* Display text content */} + {hasContent ? ( +

+ {renderContentWithFileRefs(comment.content)} +

+ ) : !hasAttachments ? ( +

(empty)

+ ) : null}
diff --git a/apps/web/components/task/task-chat-panel.tsx b/apps/web/components/task/task-chat-panel.tsx index 3a69be583f..25231744f4 100644 --- a/apps/web/components/task/task-chat-panel.tsx +++ b/apps/web/components/task/task-chat-panel.tsx @@ -14,7 +14,7 @@ import { useSessionModel } from '@/hooks/domains/session/use-session-model'; import { useMessageHandler } from '@/hooks/use-message-handler'; import { TodoSummary } from '@/components/task/chat/todo-summary'; import { VirtualizedMessageList } from '@/components/task/chat/virtualized-message-list'; -import { ChatInputContainer, type ChatInputContainerHandle } from '@/components/task/chat/chat-input-container'; +import { ChatInputContainer, type ChatInputContainerHandle, type MessageAttachment } from '@/components/task/chat/chat-input-container'; import { formatReviewCommentsAsMarkdown } from '@/components/task/chat/messages/review-comments-attachment'; import { useDiffCommentsStore, @@ -167,7 +167,7 @@ export const TaskChatPanel = memo(function TaskChatPanel({ } }, [resolvedSessionId]); - const handleSubmit = useCallback(async (message: string, reviewComments?: DiffComment[]) => { + const handleSubmit = useCallback(async (message: string, reviewComments?: DiffComment[], attachments?: MessageAttachment[]) => { if (isSending) return; setIsSending(true); try { @@ -181,7 +181,7 @@ export const TaskChatPanel = memo(function TaskChatPanel({ if (onSend) { await onSend(finalMessage); } else { - await handleSendMessage(finalMessage); + await handleSendMessage(finalMessage, attachments); } // Mark comments as sent and clear pending diff --git a/apps/web/hooks/use-message-handler.ts b/apps/web/hooks/use-message-handler.ts index 5e383d55ff..0afe05d6b6 100644 --- a/apps/web/hooks/use-message-handler.ts +++ b/apps/web/hooks/use-message-handler.ts @@ -1,5 +1,6 @@ import { useCallback } from 'react'; import { getWebSocketClient } from '@/lib/ws/connection'; +import type { MessageAttachment } from '@/components/task/chat/chat-input-container'; export function useMessageHandler( resolvedSessionId: string | null, @@ -9,7 +10,7 @@ export function useMessageHandler( planMode: boolean = false ) { const handleSendMessage = useCallback( - async (message: string) => { + async (message: string, attachments?: MessageAttachment[]) => { if (!taskId || !resolvedSessionId) { console.error('No active task session. Start an agent before sending a message.'); return; @@ -20,6 +21,10 @@ export function useMessageHandler( // Include active model in the request if it differs from the session model const modelToSend = activeModel && activeModel !== sessionModel ? activeModel : undefined; + // Use longer timeout when sending attachments (images can be large) + const hasAttachments = attachments && attachments.length > 0; + const timeoutMs = hasAttachments ? 30000 : 10000; + await client.request( 'message.add', { @@ -28,8 +33,9 @@ export function useMessageHandler( content: message, ...(modelToSend && { model: modelToSend }), ...(planMode && { plan_mode: true }), + ...(hasAttachments && { attachments }), }, - 10000 + timeoutMs ); }, [resolvedSessionId, taskId, activeModel, sessionModel, planMode]