Files
kandev/apps/backend/internal/orchestrator/service.go
T

1024 lines
40 KiB
Go

// Package orchestrator provides the main orchestrator service that coordinates
// task execution across agents. It manages:
//
// - Task queuing and scheduling via the Scheduler
// - Agent lifecycle through the AgentManager
// - Event handling and propagation
// - Session management and resume
//
// The orchestrator acts as the central coordinator between the task service,
// agent lifecycle manager, and the event bus.
package orchestrator
import (
"context"
"errors"
"regexp"
"strings"
"sync"
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/agentctl/types/streams"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/events/bus"
"github.com/kandev/kandev/internal/orchestrator/executor"
"github.com/kandev/kandev/internal/orchestrator/messagequeue"
"github.com/kandev/kandev/internal/orchestrator/queue"
"github.com/kandev/kandev/internal/orchestrator/scheduler"
"github.com/kandev/kandev/internal/orchestrator/watcher"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/task/models"
"github.com/kandev/kandev/internal/workflow/engine"
wfmodels "github.com/kandev/kandev/internal/workflow/models"
v1 "github.com/kandev/kandev/pkg/api/v1"
)
// Common errors
var (
ErrServiceAlreadyRunning = errors.New("service is already running")
ErrServiceNotRunning = errors.New("service is not running")
)
// ServiceConfig holds orchestrator service configuration
type ServiceConfig struct {
Scheduler scheduler.SchedulerConfig
QueueSize int
QueueGroup string
}
// DefaultServiceConfig returns default configuration
func DefaultServiceConfig() ServiceConfig {
return ServiceConfig{
Scheduler: scheduler.DefaultSchedulerConfig(),
QueueSize: 1000,
QueueGroup: "orchestrator",
}
}
// MessageCreator is an interface for creating messages on tasks
type MessageCreator interface {
CreateAgentMessage(ctx context.Context, taskID, content, agentSessionID, turnID string) error
CreateUserMessage(ctx context.Context, taskID, content, agentSessionID, turnID string, metadata map[string]interface{}) error
// CreateToolCallMessage creates a message for a tool call.
// normalized contains the typed tool payload data.
// parentToolCallID is the parent Task tool call ID for subagent nesting (empty for top-level).
CreateToolCallMessage(ctx context.Context, taskID, toolCallID, parentToolCallID, title, status, agentSessionID, turnID string, normalized *streams.NormalizedPayload) error
// UpdateToolCallMessage updates a tool call message's status and optionally its normalized data.
// If the message doesn't exist, it creates it using taskID, turnID, and msgType.
// normalized contains the typed tool payload data.
// parentToolCallID is the parent Task tool call ID for subagent nesting (empty for top-level).
UpdateToolCallMessage(ctx context.Context, taskID, toolCallID, parentToolCallID, status, result, agentSessionID, title, turnID, msgType string, normalized *streams.NormalizedPayload) error
CreateSessionMessage(ctx context.Context, taskID, content, agentSessionID, messageType, turnID string, metadata map[string]interface{}, requestsInput bool) error
CreatePermissionRequestMessage(ctx context.Context, taskID, sessionID, pendingID, toolCallID, title, turnID string, options []map[string]interface{}, actionType string, actionDetails map[string]interface{}) (string, error)
UpdatePermissionMessage(ctx context.Context, sessionID, pendingID, status string) error
// CreateAgentMessageStreaming creates a new agent message with a pre-generated ID for streaming updates
CreateAgentMessageStreaming(ctx context.Context, messageID, taskID, content, agentSessionID, turnID string) error
// AppendAgentMessage appends additional content to an existing streaming message
AppendAgentMessage(ctx context.Context, messageID, additionalContent string) error
// CreateThinkingMessageStreaming creates a new thinking message with a pre-generated ID for streaming updates
CreateThinkingMessageStreaming(ctx context.Context, messageID, taskID, content, agentSessionID, turnID string) error
// AppendThinkingMessage appends additional content to an existing streaming thinking message
AppendThinkingMessage(ctx context.Context, messageID, additionalContent string) error
// InvalidateModelCache clears any cached model for a session, forcing the next
// message to re-read the model from the DB. Called after model switches.
InvalidateModelCache(sessionID string)
}
// TurnService is an interface for managing session turns
type TurnService interface {
StartTurn(ctx context.Context, sessionID string) (*models.Turn, error)
CompleteTurn(ctx context.Context, turnID string) error
GetActiveTurn(ctx context.Context, sessionID string) (*models.Turn, error)
}
// TaskEventPublisher is the orchestrator's collaborator for publishing
// task.updated events. Implemented by the task service (which owns the rich
// payload build — session counts, primary session info, repositories,
// metadata, parent_id). The orchestrator does not construct task.updated
// payloads itself.
type TaskEventPublisher interface {
PublishTaskUpdated(ctx context.Context, task *models.Task)
}
// WorkflowStepGetter retrieves workflow step information for prompt building.
type WorkflowStepGetter interface {
GetStep(ctx context.Context, stepID string) (*wfmodels.WorkflowStep, error)
GetNextStepByPosition(ctx context.Context, workflowID string, currentPosition int) (*wfmodels.WorkflowStep, error)
GetPreviousStepByPosition(ctx context.Context, workflowID string, currentPosition int) (*wfmodels.WorkflowStep, error)
GetWorkflowAgentProfileID(ctx context.Context, workflowID string) (string, error)
}
// repoStore is the repository interface accepted by NewService.
// It covers both the orchestrator's own needs (sessionExecutorStore) and
// the executor package's needs (executor.executorStore).
type repoStore interface {
sessionExecutorStore
// Additional methods needed by executor
UpdateTaskState(ctx context.Context, id string, state v1.TaskState) error
GetPrimaryTaskRepository(ctx context.Context, taskID string) (*models.TaskRepository, error)
ListTaskRepositories(ctx context.Context, taskID string) ([]*models.TaskRepository, error)
CreateTaskSession(ctx context.Context, session *models.TaskSession) error
UpdateTaskSession(ctx context.Context, session *models.TaskSession) error
ListActiveTaskSessions(ctx context.Context) ([]*models.TaskSession, error)
ListActiveTaskSessionsByTaskID(ctx context.Context, taskID string) ([]*models.TaskSession, error)
CreateTaskSessionWorktree(ctx context.Context, sessionWorktree *models.TaskSessionWorktree) error
ListSessionsWithBranches(ctx context.Context) ([]models.SessionBranchInfo, error)
GetRepository(ctx context.Context, id string) (*models.Repository, error)
UpdateRepository(ctx context.Context, repository *models.Repository) error
GetExecutorProfile(ctx context.Context, id string) (*models.ExecutorProfile, error)
// Multi-repo task environment children
CreateTaskEnvironmentRepo(ctx context.Context, repo *models.TaskEnvironmentRepo) error
ListTaskEnvironmentRepos(ctx context.Context, envID string) ([]*models.TaskEnvironmentRepo, error)
// Session history + plan (for context handover)
GetTaskPlan(ctx context.Context, taskID string) (*models.TaskPlan, error)
}
// sessionExecutorStore is the minimal repository interface needed by the orchestrator service.
type sessionExecutorStore interface {
// Session
GetTaskSession(ctx context.Context, id string) (*models.TaskSession, error)
GetActiveTaskSessionByTaskID(ctx context.Context, taskID string) (*models.TaskSession, error)
SetSessionPrimary(ctx context.Context, sessionID string) error
UpdateTaskSession(ctx context.Context, session *models.TaskSession) error
UpdateTaskSessionState(ctx context.Context, id string, state models.TaskSessionState, errorMessage string) error
UpdateTaskSessionBaseCommit(ctx context.Context, id string, baseCommitSHA string) error
UpdateTaskSessionWorktreeBranch(ctx context.Context, sessionID, branch string) error
UpdateSessionReviewStatus(ctx context.Context, sessionID string, status string) error
UpdateSessionMetadata(ctx context.Context, sessionID string, metadata map[string]interface{}) error
SetSessionMetadataKey(ctx context.Context, sessionID, key string, value interface{}) error
// Executor running state
ListExecutorsRunning(ctx context.Context) ([]*models.ExecutorRunning, error)
UpsertExecutorRunning(ctx context.Context, running *models.ExecutorRunning) error
GetExecutorRunningBySessionID(ctx context.Context, sessionID string) (*models.ExecutorRunning, error)
DeleteExecutorRunningBySessionID(ctx context.Context, sessionID string) error
HasExecutorRunningRow(ctx context.Context, sessionID string) (bool, error)
UpdateResumeToken(ctx context.Context, sessionID, expectedExecID, resumeToken, lastMessageUUID string) error
// Executor
GetExecutor(ctx context.Context, id string) (*models.Executor, error)
// Task
GetTask(ctx context.Context, id string) (*models.Task, error)
UpdateTask(ctx context.Context, task *models.Task) error
// Git snapshots and commits
GetLatestGitSnapshot(ctx context.Context, sessionID string) (*models.GitSnapshot, error)
CreateGitSnapshot(ctx context.Context, snapshot *models.GitSnapshot) error
UpsertLatestLiveGitSnapshot(ctx context.Context, snapshot *models.GitSnapshot) error
CreateSessionCommit(ctx context.Context, commit *models.SessionCommit) error
GetSessionCommits(ctx context.Context, sessionID string) ([]*models.SessionCommit, error)
DeleteSessionCommit(ctx context.Context, id string) error
// Session listing + delete
ListTaskSessions(ctx context.Context, taskID string) ([]*models.TaskSession, error)
DeleteTaskSession(ctx context.Context, id string) error
// Workspace
GetWorkspace(ctx context.Context, id string) (*models.Workspace, error)
// Task environment
GetTaskEnvironmentByTaskID(ctx context.Context, taskID string) (*models.TaskEnvironment, error)
CreateTaskEnvironment(ctx context.Context, env *models.TaskEnvironment) error
UpdateTaskEnvironment(ctx context.Context, env *models.TaskEnvironment) error
}
// Service is the main orchestrator service
type Service struct {
config ServiceConfig
logger *logger.Logger
eventBus bus.EventBus
taskRepo scheduler.TaskRepository
repo sessionExecutorStore
agentManager executor.AgentManagerClient
// Components
queue *queue.TaskQueue
executor *executor.Executor
scheduler *scheduler.Scheduler
watcher *watcher.Watcher
// Message queue service for queueing messages while agent is running
messageQueue *messagequeue.Service
// Message creator for saving agent responses
messageCreator MessageCreator
// Turn service for managing session turns
turnService TurnService
// Task event publisher for emitting task.updated events.
// Task service owns the rich payload; orchestrator delegates.
taskEvents TaskEventPublisher
// Workflow step getter for prompt building
workflowStepGetter WorkflowStepGetter
// Workflow engine for typed state-machine evaluation of step transitions
workflowEngine *engine.Engine
workflowStore *workflowStore
// GitHub service for PR auto-detection on push
githubService GitHubService
// Review task creator for auto-creating tasks from review watch PRs
reviewTaskCreator ReviewTaskCreator
// Issue task creator for auto-creating tasks from issue watch events
issueTaskCreator IssueTaskCreator
// Jira service for issue watch dedup operations
jiraService JiraService
// Repository resolver for cloning + finding/creating repos for review tasks
repositoryResolver RepositoryResolver
// Clarification canceller — cancels pending clarifications when agent's turn completes
clarificationCanceller ClarificationCanceller
// Push tracker: "<sessionID>|<repository_name>" -> last known ahead count.
// The repository_name segment is required for multi-repo: each repo emits
// its own git status events, and keying by sessionID alone made events
// from different repos overwrite each other's ahead counts (so only one
// push got detected per session). Single-repo / repo-less sessions key
// against an empty repository_name.
pushTracker sync.Map
// gitSnapshotCache throttles per-session writes of the live git status
// snapshot (used by handleGitStatusUpdate -> persistGitStatusSnapshot).
gitSnapshotCache *gitSnapshotCache
// Active turns map: sessionID -> turnID
activeTurns sync.Map
// Session reset flags: sessionID -> true while resetAgentContext is restarting process.
// Used to suppress stale ready events and avoid draining queued prompts mid-reset.
resetInProgressSessions sync.Map
// suppressToast: sessionID -> true. Set by failure handlers that create
// guidance messages with actions. Cleared by updateTaskSessionState which
// adds suppress_toast to the WS event so the frontend skips the error toast.
suppressToast sync.Map
// clarificationWatchdogs tracks active clarification primary-path resume watchdogs.
// key: "<session_id>::<pending_id>", value: *clarificationWatchdogEntry
clarificationWatchdogs sync.Map
// clarificationWatchdogTimeout controls how long to wait for post-clarification
// activity before triggering fallback resume.
clarificationWatchdogTimeout time.Duration
// cancelInFlight tracks sessionIDs whose CancelAgent call is currently in
// progress. Used to deduplicate impatient retries from the UI: while the
// first cancel is still propagating through the agent (which can take several
// seconds when a long-running tool like Claude's Monitor is being torn down),
// the user often clicks the button repeatedly. Without this guard each click
// would create another "Turn cancelled by user" message and lazily start a
// phantom turn just to host it.
cancelInFlight sync.Map
// Service state
mu sync.RWMutex
running bool
startedAt time.Time
}
// Status contains orchestrator status information
type Status struct {
Running bool `json:"running"`
ActiveAgents int `json:"active_agents"`
QueuedTasks int `json:"queued_tasks"`
TotalProcessed int64 `json:"total_processed"`
TotalFailed int64 `json:"total_failed"`
UptimeSeconds int64 `json:"uptime_seconds"`
LastHeartbeat time.Time `json:"last_heartbeat"`
}
// NewService creates a new orchestrator service
func NewService(
cfg ServiceConfig,
eventBus bus.EventBus,
agentManager executor.AgentManagerClient,
taskRepo scheduler.TaskRepository,
repo repoStore,
shellPrefs executor.ShellPreferenceProvider,
secretStore secrets.SecretStore,
log *logger.Logger,
) *Service {
svcLogger := log.WithFields(zap.String("component", "orchestrator"))
// Create the task queue with configured size
taskQueue := queue.NewTaskQueue(cfg.QueueSize)
// Create the executor with the agent manager client and repository for persistent sessions
execCfg := executor.ExecutorConfig{
ShellPrefs: shellPrefs,
SecretStore: secretStore,
}
exec := executor.NewExecutor(agentManager, repo, log, execCfg)
// Create the scheduler with queue, executor, and task repository
sched := scheduler.NewScheduler(taskQueue, exec, taskRepo, log, cfg.Scheduler)
// Create the message queue service
msgQueue := messagequeue.NewService(log)
// Create the service (watcher will be created after we have handlers)
s := &Service{
config: cfg,
logger: svcLogger,
eventBus: eventBus,
taskRepo: taskRepo,
repo: repo,
agentManager: agentManager,
queue: taskQueue,
executor: exec,
scheduler: sched,
messageQueue: msgQueue,
clarificationWatchdogTimeout: 15 * time.Second,
gitSnapshotCache: newGitSnapshotCache(),
}
// Wire executor state changes through the orchestrator so events are published
// (e.g. WebSocket notifications to the frontend). Must be set after service
// construction so the session callback can reference s.updateTaskSessionState.
exec.SetOnTaskStateChange(func(ctx context.Context, taskID string, state v1.TaskState) error {
return taskRepo.UpdateTaskState(ctx, taskID, state)
})
exec.SetOnSessionStateChange(func(ctx context.Context, taskID, sessionID string, state models.TaskSessionState, errorMessage string) error {
s.updateTaskSessionState(ctx, taskID, sessionID, state, errorMessage, true)
return nil
})
exec.SetOnLaunchFailed(s.handleSessionLaunchFailed)
exec.SetOnAgentStartFailed(s.handleAgentStartFailed)
if caps, ok := agentManager.(executor.ExecutorTypeCapabilities); ok {
exec.SetCapabilities(caps)
}
// Create the watcher with event handlers that wire everything together
handlers := watcher.EventHandlers{
OnTaskDeleted: s.handleTaskDeleted,
OnAgentRunning: s.handleAgentRunning,
OnAgentBootReady: s.handleAgentBootReady,
OnAgentReady: s.handleAgentReady,
OnAgentCompleted: s.handleAgentCompleted,
OnAgentFailed: s.handleAgentFailed,
OnAgentStopped: s.handleAgentStopped,
OnAgentStreamEvent: s.handleAgentStreamEvent,
OnACPSessionCreated: s.handleACPSessionCreated,
OnPermissionRequest: s.handlePermissionRequest,
OnGitEvent: s.handleGitEvent,
OnContextWindowUpdated: s.handleContextWindowUpdated,
OnTaskMoved: s.handleTaskMoved,
}
s.watcher = watcher.NewWatcher(eventBus, handlers, cfg.QueueGroup, log)
return s
}
// SetMessageCreator sets the message creator for saving agent responses to the database.
//
// This must be called before starting the orchestrator if you want agent messages, tool calls,
// and streaming content to be persisted to the database. The MessageCreator interface provides
// methods for creating and updating messages associated with task sessions.
//
// The MessageCreator is typically the task service, which owns the message persistence logic.
// Event handlers in the orchestrator call these methods when agent events occur:
// - AgentStreamEvent → CreateAgentMessage, AppendAgentMessage
// - Tool calls → CreateToolCallMessage, UpdateToolCallMessage
// - Permission requests → CreatePermissionRequestMessage
//
// When to call: During orchestrator initialization, after creating the task service.
//
// If not set: Agent messages won't be saved to the database (events will still be published).
func (s *Service) SetMessageCreator(mc MessageCreator) {
s.messageCreator = mc
}
// SetOnPrimarySessionSet sets a callback on the executor for when the first session
// of a task is marked primary. Used to publish a task.updated event so the frontend
// receives the primary_session_id.
func (s *Service) SetOnPrimarySessionSet(fn executor.PrimarySessionSetFunc) {
s.executor.SetOnPrimarySessionSet(fn)
}
// SetRepoCloner sets the repository cloner and updater on the executor, enabling automatic
// cloning of provider-backed repositories (e.g. from a GitHub URL) when they are launched
// for local/worktree execution and have no local path yet.
func (s *Service) SetRepoCloner(cloner executor.RepoCloner, updater executor.RepoUpdater) {
s.executor.SetRepoCloner(cloner, updater)
}
// SetTurnService sets the turn service for tracking conversation turns.
//
// A "turn" represents a single conversation round-trip: user prompt → agent response.
// The TurnService tracks turn timing and duration for analytics and UI display (e.g., showing
// how long each agent response took).
//
// The TurnService is typically the task service, which owns turn persistence logic.
// The orchestrator calls these methods:
// - StartTurn: When agent begins processing a prompt
// - CompleteTurn: When agent finishes and returns to ready state
// - GetActiveTurn: To associate messages with current turn
//
// When to call: During orchestrator initialization, after creating the task service.
//
// If not set: Turns won't be tracked (orchestrator continues functioning normally, but
// no timing data is recorded and turn IDs in messages will be empty).
func (s *Service) SetTurnService(turnService TurnService) {
s.turnService = turnService
}
// SetTaskEventPublisher wires the publisher used for task.updated events.
//
// The task service is the canonical publisher: it loads session counts,
// primary session info, repositories, metadata, and parent_id, and emits a
// single rich payload. Orchestrator code paths that mutate a task (workflow
// transitions, primary session assignment, workflow-step moves) delegate to
// this publisher instead of constructing their own partial payloads.
//
// If not set: orchestrator task.updated publishers no-op (nothing is emitted
// on those paths). Task service's own publishTaskEvent calls are unaffected.
func (s *Service) SetTaskEventPublisher(publisher TaskEventPublisher) {
s.taskEvents = publisher
}
// publishTaskUpdated forwards to the configured TaskEventPublisher.
// No-op when the publisher isn't wired (tests, or before SetTaskEventPublisher
// has been called during startup).
func (s *Service) publishTaskUpdated(ctx context.Context, task *models.Task) {
if s.taskEvents == nil || task == nil {
return
}
s.taskEvents.PublishTaskUpdated(ctx, task)
}
// SetWorkflowStepGetter sets the workflow step getter for prompt building.
//
// When workflow_step_id is provided to StartTask, the orchestrator uses this getter
// to retrieve the step's prompt_prefix, prompt_suffix, and plan_mode settings to
// build the effective prompt.
//
// If not set: workflow_step_id in StartTask is ignored and the prompt is used as-is.
func (s *Service) SetWorkflowStepGetter(getter WorkflowStepGetter) {
s.workflowStepGetter = getter
s.initWorkflowEngine()
}
// ClarificationCanceller cancels pending clarifications when an agent's turn completes.
type ClarificationCanceller interface {
CancelSessionAndNotify(ctx context.Context, sessionID string) int
}
// SetClarificationCanceller sets the canceller for cleaning up pending clarifications on turn complete.
func (s *Service) SetClarificationCanceller(c ClarificationCanceller) {
s.clarificationCanceller = c
}
// initWorkflowEngine creates the workflow engine with store and callbacks.
// Called after the workflow step getter is set. Safe to call multiple times.
func (s *Service) initWorkflowEngine() {
if s.workflowStepGetter == nil {
return
}
store := newWorkflowStore(s.repo, s.workflowStepGetter, s.agentManager, s.publishTaskUpdated, s.logger)
callbacks := buildWorkflowCallbacks(s)
s.workflowStore = store
s.workflowEngine = engine.New(store, callbacks)
}
// startTurnForSession ensures the session has an active turn and returns its ID.
//
// Idempotent: if an open turn already exists (either tracked in activeTurns or
// only present in the DB — the latter happens when service.CreateMessage lazily
// started a turn for an inbound user message before the orchestrator's prompt
// cycle began, or when the backend restarted with open turns in the DB), it is
// adopted rather than duplicated. A new turn is created only when none exists.
//
// This avoids the classic dual-creation leak: user message → service.CreateMessage
// starts turn X (DB only) → PromptTask → startTurnForSession → would create turn Y
// (DB + activeTurns), leaving X open forever because nothing tracks it.
func (s *Service) startTurnForSession(ctx context.Context, sessionID string) string {
if s.turnService == nil {
return ""
}
if turnIDVal, ok := s.activeTurns.Load(sessionID); ok {
if turnID, ok := turnIDVal.(string); ok && turnID != "" {
return turnID
}
}
if turn, err := s.turnService.GetActiveTurn(ctx, sessionID); turn != nil {
s.activeTurns.Store(sessionID, turn.ID)
return turn.ID
} else if err != nil {
// A real DB read failure here would otherwise be silently dropped, and
// we'd fall through to StartTurn — potentially writing a duplicate next
// to an existing open turn we couldn't see. Log it; the next sweep via
// completeTurnForSession will mop up any duplicate.
s.logger.Warn("failed to look up active turn before starting a new one",
zap.String("session_id", sessionID),
zap.Error(err))
}
turn, err := s.turnService.StartTurn(ctx, sessionID)
if err != nil {
s.logger.Warn("failed to start turn",
zap.String("session_id", sessionID),
zap.Error(err))
return ""
}
s.activeTurns.Store(sessionID, turn.ID)
return turn.ID
}
// completeTurnForSession closes any open turn for the session.
//
// The DB is the source of truth — activeTurns is just an in-memory cache that
// can drift (see startTurnForSession) or be wiped by a backend restart. We
// query the DB for any open turn and close it. Loops to mop up multiple
// zombies (e.g. left over from before this fix) with a small sanity bound.
func (s *Service) completeTurnForSession(ctx context.Context, sessionID string) {
if s.turnService == nil {
return
}
s.activeTurns.Delete(sessionID)
const maxIterations = 16
closed := 0
for closed < maxIterations {
turn, err := s.turnService.GetActiveTurn(ctx, sessionID)
if err != nil {
s.logger.Warn("failed to look up active turn",
zap.String("session_id", sessionID),
zap.Error(err))
return
}
if turn == nil {
return
}
if err := s.turnService.CompleteTurn(ctx, turn.ID); err != nil {
// GetActiveTurn returns the latest open turn — retrying here
// would just hit the same row and loop. Bail; the next
// completeTurnForSession call will pick it up.
s.logger.Warn("failed to complete turn; will retry on next sweep",
zap.String("session_id", sessionID),
zap.String("turn_id", turn.ID),
zap.Error(err))
return
}
closed++
}
// Only warn if turns are *still* accumulating after the cap. Closing
// exactly maxIterations turns and then finding the session clean is not a
// runaway.
if turn, err := s.turnService.GetActiveTurn(ctx, sessionID); err == nil && turn != nil {
s.logger.Warn("completeTurnForSession iteration cap hit; possible turn close loop",
zap.String("session_id", sessionID),
zap.Int("max_iterations", maxIterations))
}
}
// getActiveTurnID returns the active turn ID for a session.
// If no active turn exists and the session ID is provided, it will start a new turn.
// This ensures messages always have a valid turn ID even in edge cases like resumed sessions.
func (s *Service) getActiveTurnID(sessionID string) string {
if sessionID == "" {
return ""
}
turnIDVal, ok := s.activeTurns.Load(sessionID)
if ok {
turnID, _ := turnIDVal.(string)
if turnID != "" {
return turnID
}
}
// No active turn exists - start one lazily
// This handles edge cases like resumed sessions or race conditions
return s.startTurnForSession(context.Background(), sessionID)
}
func (s *Service) setSessionResetInProgress(sessionID string, inProgress bool) {
if sessionID == "" {
return
}
if inProgress {
s.resetInProgressSessions.Store(sessionID, true)
return
}
s.resetInProgressSessions.Delete(sessionID)
}
func (s *Service) isSessionResetInProgress(sessionID string) bool {
if sessionID == "" {
return false
}
v, ok := s.resetInProgressSessions.Load(sessionID)
if !ok {
return false
}
inProgress, _ := v.(bool)
return inProgress
}
// Start starts all orchestrator components
func (s *Service) Start(ctx context.Context) error {
s.mu.Lock()
if s.running {
s.mu.Unlock()
return ErrServiceAlreadyRunning
}
s.running = true
s.startedAt = time.Now()
s.mu.Unlock()
s.logger.Info("starting orchestrator service")
// Reconcile session state from persisted runtime state on startup.
// This does NOT launch any agent processes — sessions are recovered lazily
// when the user opens them (via task.session.status → task.session.resume).
s.reconcileSessionsOnStartup(ctx)
// Start the watcher first to begin receiving events
if err := s.watcher.Start(ctx); err != nil {
s.mu.Lock()
s.running = false
s.mu.Unlock()
return err
}
// Start the scheduler processing loop
if err := s.scheduler.Start(ctx); err != nil {
if stopErr := s.watcher.Stop(); stopErr != nil {
s.logger.Warn("failed to stop watcher after scheduler start failure", zap.Error(stopErr))
}
s.mu.Lock()
s.running = false
s.mu.Unlock()
return err
}
// Subscribe to GitHub integration events
s.subscribeGitHubEvents()
// Subscribe to JIRA integration events
s.subscribeJiraEvents()
// Subscribe to clarification events (cancel-and-resume flow)
s.subscribeClarificationEvents()
// Subscribe to prepare events (persist result in session metadata)
s.subscribePrepareEvents()
s.logger.Info("orchestrator service started successfully")
return nil
}
// Stop stops all orchestrator components
func (s *Service) Stop() error {
s.mu.Lock()
if !s.running {
s.mu.Unlock()
return ErrServiceNotRunning
}
s.running = false
s.mu.Unlock()
s.logger.Info("stopping orchestrator service")
// Stop components in reverse order
var errs []error
if err := s.scheduler.Stop(); err != nil {
s.logger.Error("failed to stop scheduler", zap.Error(err))
errs = append(errs, err)
}
if err := s.watcher.Stop(); err != nil {
s.logger.Error("failed to stop watcher", zap.Error(err))
errs = append(errs, err)
}
s.cancelAllClarificationWatchdogs()
if len(errs) > 0 {
return errs[0]
}
s.logger.Info("orchestrator service stopped successfully")
return nil
}
// reconcileSessionsOnStartup adjusts database state for sessions that were active before restart.
// It does NOT launch any agent processes — sessions are recovered lazily when the user opens them
// (via task.session.status → task.session.resume or by sending a prompt).
//
// Strategy:
//
// 1. Terminal states (Completed/Cancelled/Failed) → clean up executor record
// 2. Never-started (Created) → clean up executor record
// 3. Active states (Starting/Running/WaitingForInput) → set session to WAITING_FOR_INPUT,
// clear stale execution IDs, fix task state, preserve ExecutorRunning record
// 4. Pre-poll remote executor status for remote runtimes (sprites, remote_docker)
//
// Called by: Start() method during orchestrator initialization.
func (s *Service) reconcileSessionsOnStartup(ctx context.Context) {
runningExecutors, err := s.repo.ListExecutorsRunning(ctx)
if err != nil {
s.logger.Warn("failed to list executors running on startup", zap.Error(err))
return
}
if len(runningExecutors) == 0 {
s.logger.Info("no executors to reconcile on startup")
return
}
s.logger.Info("reconciling sessions on startup (lazy recovery)", zap.Int("count", len(runningExecutors)))
var remoteRecords []executor.RemoteStatusPollRequest
for _, running := range runningExecutors {
if models.IsRemoteExecutorType(models.ExecutorType(running.Runtime)) {
remoteRecords = append(remoteRecords, executor.RemoteStatusPollRequest{
SessionID: running.SessionID,
Runtime: running.Runtime,
AgentExecutionID: running.AgentExecutionID,
ContainerID: running.ContainerID,
Metadata: running.Metadata,
})
}
s.reconcileOneSessionOnStartup(ctx, running)
}
// Pre-poll remote executor status so task lists show accurate state
if len(remoteRecords) > 0 && s.agentManager != nil {
s.agentManager.PollRemoteStatusForRecords(ctx, remoteRecords)
}
}
// reconcileOneSessionOnStartup adjusts DB state for a single session without launching agents.
func (s *Service) reconcileOneSessionOnStartup(ctx context.Context, running *models.ExecutorRunning) {
sessionID := running.SessionID
if sessionID == "" {
return
}
session, err := s.repo.GetTaskSession(ctx, sessionID)
if err != nil {
s.logger.Warn("failed to load session for reconciliation",
zap.String("session_id", sessionID),
zap.Error(err))
return
}
previousState := session.State
// Handle terminal and never-started states (reuse existing cleanup logic)
if skip := s.handleTerminalSessionOnStartup(ctx, session, running, previousState); skip {
return
}
if previousState == models.TaskSessionStateCreated {
s.logger.Info("session was never started; cleaning up",
zap.String("session_id", sessionID))
if err := s.repo.DeleteExecutorRunningBySessionID(ctx, sessionID); err != nil {
s.logger.Warn("failed to remove executor record for unstarted session",
zap.String("session_id", sessionID),
zap.Error(err))
}
return
}
// Active states: STARTING, RUNNING, WAITING_FOR_INPUT
// Set session to WAITING_FOR_INPUT (idle, ready for lazy resume when user opens it)
if previousState != models.TaskSessionStateWaitingForInput {
if err := s.repo.UpdateTaskSessionState(ctx, sessionID, models.TaskSessionStateWaitingForInput, ""); err != nil {
s.logger.Warn("failed to set session to WAITING_FOR_INPUT on startup",
zap.String("session_id", sessionID),
zap.Error(err))
}
}
// PRESERVE executors_running.agent_execution_id post-restart. The in-memory
// process is gone, but the stored ID still serves as a "this session was
// previously launched" marker that drives resume detection in
// applyRunningRecordToResumeRequest → isResumedSession → ResumePassthroughSession
// (which adds the --resume flag). Clearing it here makes the next launch take
// the fresh-start path and skip the resume CLI flag, so passthrough TUIs lose
// their conversation context after every backend restart. The stale ID is
// harmless: persistExecutorRunning UPSERTs it atomically alongside the
// in-memory Add on the next launch, closing the divergence window we set out
// to fix in this refactor.
// Ensure task is in REVIEW state (not stuck IN_PROGRESS)
if running.TaskID != "" {
task, taskErr := s.taskRepo.GetTask(ctx, running.TaskID)
if taskErr == nil && task != nil && task.State == v1.TaskStateInProgress {
if updateErr := s.taskRepo.UpdateTaskState(ctx, running.TaskID, v1.TaskStateReview); updateErr != nil {
s.logger.Warn("failed to update task to REVIEW on startup",
zap.String("task_id", running.TaskID),
zap.Error(updateErr))
}
}
}
// PRESERVE the ExecutorRunning record — it holds the resume token and worktree info
// needed for lazy recovery when the user opens the session.
s.logger.Info("session reconciled for lazy recovery",
zap.String("session_id", sessionID),
zap.String("task_id", running.TaskID),
zap.String("previous_state", string(previousState)),
zap.Bool("has_resume_token", running.ResumeToken != ""),
zap.Bool("has_worktree", running.WorktreePath != ""))
}
// handleTerminalSessionOnStartup processes sessions in terminal states during startup.
// Returns true if the session should be skipped (no further processing needed).
func (s *Service) handleTerminalSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning, previousState models.TaskSessionState) bool {
sessionID := session.ID
switch previousState {
case models.TaskSessionStateCompleted, models.TaskSessionStateCancelled:
s.logger.Info("session in terminal state; cleaning up executor record",
zap.String("session_id", sessionID),
zap.String("task_id", session.TaskID),
zap.String("state", string(previousState)))
if err := s.repo.DeleteExecutorRunningBySessionID(ctx, sessionID); err != nil {
s.logger.Warn("failed to remove executor record for terminal session",
zap.String("session_id", sessionID),
zap.Error(err))
}
return true
case models.TaskSessionStateFailed:
s.handleFailedSessionOnStartup(ctx, session, running)
return true
}
return false
}
// handleFailedSessionOnStartup handles a failed session during startup recovery.
func (s *Service) handleFailedSessionOnStartup(ctx context.Context, session *models.TaskSession, running *models.ExecutorRunning) {
sessionID := session.ID
// If session failed, ensure task is in REVIEW state (not stuck IN_PROGRESS)
if session.TaskID != "" {
task, taskErr := s.taskRepo.GetTask(ctx, session.TaskID)
if taskErr == nil && task.State == v1.TaskStateInProgress {
s.logger.Info("fixing task state: session failed but task still IN_PROGRESS",
zap.String("task_id", session.TaskID),
zap.String("session_id", sessionID))
if updateErr := s.taskRepo.UpdateTaskState(ctx, session.TaskID, v1.TaskStateReview); updateErr != nil {
s.logger.Warn("failed to update task state to REVIEW",
zap.String("task_id", session.TaskID),
zap.Error(updateErr))
}
}
}
if canResumeRunning(running) {
s.logger.Info("preserving executor record for resumable failed session",
zap.String("session_id", sessionID),
zap.String("task_id", session.TaskID))
} else {
s.logger.Info("cleaning up executor record for non-resumable failed session",
zap.String("session_id", sessionID),
zap.String("task_id", session.TaskID))
if err := s.repo.DeleteExecutorRunningBySessionID(ctx, sessionID); err != nil {
s.logger.Warn("failed to remove executor record for failed session",
zap.String("session_id", sessionID),
zap.Error(err))
}
}
}
func canResumeRunning(running *models.ExecutorRunning) bool {
if running == nil || running.ResumeToken == "" {
return false
}
if running.Resumable {
return true
}
return models.IsAlwaysResumableRuntime(running.Runtime)
}
func isMissingMergedPRBranchError(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "couldn't find remote ref") ||
(strings.Contains(msg, "not found locally or on remote") && strings.Contains(msg, "branch")) ||
(strings.Contains(msg, "pathspec") && strings.Contains(msg, "did not match"))
}
var (
quotedBranchPattern = regexp.MustCompile(`branch "([^"]+)"`)
remoteRefPattern = regexp.MustCompile(`remote ref ([^\s]+)`)
pathspecBranchPattern = regexp.MustCompile(`pathspec '([^']+)'`)
)
func extractMissingBranchName(err error) string {
if err == nil {
return ""
}
msg := err.Error()
if match := quotedBranchPattern.FindStringSubmatch(msg); len(match) == 2 {
return strings.TrimSpace(match[1])
}
if match := remoteRefPattern.FindStringSubmatch(msg); len(match) == 2 {
return strings.TrimSpace(match[1])
}
if match := pathspecBranchPattern.FindStringSubmatch(msg); len(match) == 2 {
return strings.TrimSpace(match[1])
}
return ""
}
func (s *Service) handleSessionLaunchFailed(ctx context.Context, taskID, sessionID string, launchErr error) {
if s.messageCreator == nil || !isMissingMergedPRBranchError(launchErr) {
return
}
branch := extractMissingBranchName(launchErr)
content := "This task references a PR branch that no longer exists on remote (likely merged and deleted)."
if branch != "" {
content = "The remote PR branch \"" + branch + "\" no longer exists (likely merged and deleted)."
}
metadata := map[string]interface{}{
"variant": "warning",
"failure_kind": "missing_pr_branch",
"missing_branch": branch,
"actions": []map[string]interface{}{
{
"type": "archive_task",
"label": "Archive task",
"tooltip": "Keep task history and hide it from active work",
"icon": "archive",
"test_id": "missing-branch-archive-button",
},
{
"type": "delete_task",
"label": "Delete task",
"tooltip": "Permanently remove this task",
"variant": "destructive",
"icon": "trash",
"test_id": "missing-branch-delete-button",
},
},
}
s.suppressToast.Store(sessionID, true)
msgCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := s.messageCreator.CreateSessionMessage(
msgCtx,
taskID,
content,
sessionID,
string(v1.MessageTypeStatus),
"", // No turn — agent never started, avoid lazily creating a synthetic turn.
metadata,
false,
); err != nil {
s.logger.Warn("failed to create missing PR branch launch failure message",
zap.String("task_id", taskID),
zap.String("session_id", sessionID),
zap.Error(err))
}
}
// IsRunning returns true if the service is running
func (s *Service) IsRunning() bool {
s.mu.RLock()
defer s.mu.RUnlock()
return s.running
}
// GetStatus returns the orchestrator status
func (s *Service) GetStatus() *Status {
s.mu.RLock()
running := s.running
startedAt := s.startedAt
s.mu.RUnlock()
queueStatus := s.scheduler.GetQueueStatus()
var uptimeSeconds int64
if running {
uptimeSeconds = int64(time.Since(startedAt).Seconds())
}
return &Status{
Running: running,
ActiveAgents: queueStatus.ActiveExecutions,
QueuedTasks: queueStatus.QueuedTasks,
TotalProcessed: queueStatus.TotalProcessed,
TotalFailed: queueStatus.TotalFailed,
UptimeSeconds: uptimeSeconds,
LastHeartbeat: time.Now(),
}
}
// GetMessageQueue returns the message queue service
func (s *Service) GetMessageQueue() *messagequeue.Service {
return s.messageQueue
}
// GetEventBus returns the event bus
func (s *Service) GetEventBus() bus.EventBus {
return s.eventBus
}