Files
kandev/apps/backend/internal/office/scheduler/dispatch_routing.go
T
2026-05-17 20:16:15 +01:00

596 lines
21 KiB
Go

package scheduler
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/agent/runtime/routingerr"
"github.com/kandev/kandev/internal/office/models"
sqliterepo "github.com/kandev/kandev/internal/office/repository/sqlite"
"github.com/kandev/kandev/internal/office/routing"
)
// RouteAttemptOutcomeLaunched is the in-flight outcome string the
// dispatcher writes when an attempt is appended. It is overwritten with
// failed_provider_unavailable / failed_other on classifier verdict.
const (
RouteAttemptOutcomeLaunched = "launched"
RouteAttemptOutcomeFailedProviderUnavail = "failed_provider_unavailable"
RouteAttemptOutcomeFailedOther = "failed_other"
RouteAttemptOutcomeSkippedDegraded = "skipped_degraded"
RouteAttemptOutcomeSkippedUserAction = "skipped_user_action"
RouteAttemptOutcomeSkippedMissingMapping = "skipped_missing_mapping"
RouteAttemptOutcomeMaxAttempts = "skipped_max_attempts"
)
// MaxAttemptsPerRun caps the number of route_attempt rows a single run
// can accumulate. A pathological cycle (provider flapping, post-start
// fallback racing with re-park) could otherwise append rows forever.
// Once the cap is reached the dispatcher parks the run as
// blocked_provider_action_required so a human notices.
const MaxAttemptsPerRun = 20
// DispatchWithRouting attempts the configured providers in order. Three
// outcomes are possible:
// - launched=true, parked=false → run is in flight; caller skips legacy.
// - launched=false, parked=false, err=nil → routing disabled / not
// wired. Caller falls through to the legacy concrete-profile path.
// - launched=false, parked=true, err=nil → run parked under
// routing_blocked_status. Caller treats as handled.
// - launched=false, err!=nil → infrastructure failure; caller invokes
// existing HandleRunFailure.
//
// launch carries the Office-built prompt/env/profile/workflow context so
// routed launches behave identically to the legacy concrete-profile path
// for everything except the provider/model/mode selection.
func (ss *SchedulerService) DispatchWithRouting(
ctx context.Context, run *models.Run, agent *models.AgentInstance,
launch LaunchContext,
) (bool, bool, error) {
if ss.resolver == nil || ss.taskStarter == nil {
return false, false, nil
}
if agent == nil || run == nil {
return false, false, fmt.Errorf("dispatch: nil run or agent")
}
prior, err := ss.repo.ListRouteAttempts(ctx, run.ID)
if err != nil {
return false, false, fmt.Errorf("dispatch: list prior: %w", err)
}
if len(prior) >= MaxAttemptsPerRun {
if perr := ss.parkRunMaxAttempts(ctx, run, agent.WorkspaceID); perr != nil {
return false, false, perr
}
return false, true, nil
}
excluded := excludedFromAttempts(prior, run.RouteCycleBaselineSeq)
res, err := ss.resolver.Resolve(ctx, agent.WorkspaceID, *agent,
routing.ResolveOptions{
ExcludeProviders: excluded,
Reason: run.Reason,
})
if err != nil {
return false, false, fmt.Errorf("dispatch: resolve: %w", err)
}
if !res.Enabled {
return false, false, nil
}
if err := ss.persistRoutingDecision(ctx, run, res); err != nil {
return false, false, err
}
if len(res.Candidates) == 0 {
_, _, perr := ss.parkRunBlocked(ctx, run, agent.WorkspaceID, res)
return false, true, perr
}
launched, _, terr := ss.tryCandidates(ctx, run, agent, res, launch)
if terr != nil {
return false, false, terr
}
if launched {
return true, false, nil
}
return false, true, nil
}
// excludedFromAttempts returns the providers that have already failed
// for this run within the current retry cycle. Attempts with seq at or
// below baseline belong to an earlier cycle (a previous park-and-lift
// pass) and are intentionally NOT counted — when the scheduler lifts a
// parked run it bumps the baseline so the run can re-try every provider
// in its order. The dispatcher already loaded the attempts slice once
// (for the MaxAttemptsPerRun cap), so we walk that in-memory slice
// instead of re-querying.
func excludedFromAttempts(prior []models.RouteAttempt, baseline int) []routing.ProviderID {
excluded := make([]routing.ProviderID, 0, len(prior))
for _, a := range prior {
if a.Seq <= baseline {
continue
}
if a.Outcome == RouteAttemptOutcomeFailedProviderUnavail ||
a.Outcome == RouteAttemptOutcomeFailedOther {
excluded = append(excluded, routing.ProviderID(a.ProviderID))
}
}
return excluded
}
// parkRunMaxAttempts records a terminal `skipped_max_attempts` row and
// parks the run under blocked_provider_action_required. No earliest-retry
// is set: the cap exists to force human attention, not to schedule yet
// another retry. Operators clear the parked status manually (Retry now
// in the UI) once the underlying cause is fixed.
func (ss *SchedulerService) parkRunMaxAttempts(
ctx context.Context, run *models.Run, workspaceID string,
) error {
seq, err := ss.repo.IncrementRouteAttemptSeq(ctx, run.ID)
if err != nil {
return err
}
now := time.Now().UTC()
attempt := models.RouteAttempt{
RunID: run.ID,
Seq: seq,
Outcome: RouteAttemptOutcomeMaxAttempts,
ErrorCode: "max_attempts_exceeded",
StartedAt: now,
FinishedAt: &now,
}
if err := ss.repo.AppendRouteAttempt(ctx, &attempt); err != nil {
return err
}
ss.publishRouteAttemptAppended(ctx, run.ID, attempt)
if err := ss.repo.ParkRunForProviderCapacity(ctx,
run.ID, routing.StatusBlockedActionRequired, time.Time{}); err != nil {
return err
}
ss.logger.Warn("run parked after route attempts cap",
zap.String("run_id", run.ID),
zap.Int("cap", MaxAttemptsPerRun))
ss.recordRouteParked(workspaceID, run.ID, routing.StatusBlockedActionRequired)
return nil
}
// persistRoutingDecision snapshots the resolver's effective order + tier
// onto the run row on first dispatch. Subsequent post-start fallbacks
// keep the original snapshot (idempotent overwrite is safe).
func (ss *SchedulerService) persistRoutingDecision(
ctx context.Context, run *models.Run, res *routing.Resolution,
) error {
if run.LogicalProviderOrder != nil && *run.LogicalProviderOrder != "" {
return nil
}
orderJSON, err := json.Marshal(res.ProviderOrder)
if err != nil {
return fmt.Errorf("dispatch: encode order: %w", err)
}
if err := ss.repo.SetRunRoutingDecision(ctx,
run.ID, string(orderJSON), string(res.RequestedTier)); err != nil {
return err
}
orderStr := string(orderJSON)
tierStr := string(res.RequestedTier)
run.LogicalProviderOrder = &orderStr
run.RequestedTier = &tierStr
return nil
}
// parkRunBlocked records skip attempts and parks the run with the
// resolver's block reason. Called when len(Candidates)==0.
func (ss *SchedulerService) parkRunBlocked(
ctx context.Context, run *models.Run, workspaceID string, res *routing.Resolution,
) (bool, *routing.BlockReason, error) {
br := res.BlockReason
for _, sk := range res.SkippedDegraded {
seq, err := ss.repo.IncrementRouteAttemptSeq(ctx, run.ID)
if err != nil {
return false, nil, err
}
attempt := models.RouteAttempt{
RunID: run.ID,
Seq: seq,
ProviderID: string(sk.ProviderID),
Tier: string(res.RequestedTier),
Outcome: skipOutcome(sk.Reason),
ErrorCode: sk.ErrorCode,
RawExcerpt: sk.RawExcerpt,
StartedAt: time.Now().UTC(),
}
if err := ss.repo.AppendRouteAttempt(ctx, &attempt); err != nil {
return false, nil, err
}
ss.publishRouteAttemptAppended(ctx, run.ID, attempt)
}
if err := ss.repo.ParkRunForProviderCapacity(ctx,
run.ID, br.Status, br.EarliestRetry); err != nil {
return false, nil, err
}
ss.logger.Info("run parked for provider capacity",
zap.String("run_id", run.ID),
zap.String("status", br.Status),
zap.Time("earliest_retry_at", br.EarliestRetry))
ss.recordRouteParked(workspaceID, run.ID, br.Status)
return false, &br, nil
}
func skipOutcome(reason string) models.RouteAttemptOutcome {
switch reason {
case routing.SkipReasonDegraded:
return RouteAttemptOutcomeSkippedDegraded
case routing.SkipReasonUserAction:
return RouteAttemptOutcomeSkippedUserAction
case routing.SkipReasonMissingModelMapping:
return RouteAttemptOutcomeSkippedMissingMapping
}
return RouteAttemptOutcomeSkippedDegraded
}
// tryCandidates walks the resolved candidates in order, launching the
// first that does not classify as a fallback-allowed failure. When all
// candidates exhaust with fallback-allowed errors, the run is parked
// using the same block-reason aggregation as parkRunBlocked.
func (ss *SchedulerService) tryCandidates(
ctx context.Context, run *models.Run,
agent *models.AgentInstance, res *routing.Resolution,
launch LaunchContext,
) (bool, *routing.BlockReason, error) {
taskID := extractRunTaskID(run.Payload)
var prev *routing.Candidate
for i := range res.Candidates {
candidate := res.Candidates[i]
seq, err := ss.recordAttemptStart(ctx, run, candidate, res.RequestedTier)
if err != nil {
return false, nil, err
}
launchErr := ss.launchCandidate(ctx, taskID, agent.ID, candidate, launch)
if launchErr == nil {
if prev != nil {
// We walked past at least one prior candidate — that's
// a fallback hop. Recorded once per hop, regardless of
// whether the prior failure was fatal-skipped or
// fallback-allowed (both led us here).
ss.recordRouteFallback(agent.WorkspaceID,
string(prev.ProviderID), string(candidate.ProviderID),
"")
}
ss.recordRouteAttempt(agent.WorkspaceID,
string(candidate.ProviderID), metricOutcomeSuccess, "")
return ss.handleLaunchSuccess(ctx, run, agent, candidate)
}
// Emit the failure-side route_attempt before classifying. The
// outcome label is refined by handleLaunchFailure.
classified := classifyLaunchError(string(candidate.ProviderID), launchErr)
outcome := metricOutcomeFallbackErr
if !classified.FallbackAllowed {
outcome = metricOutcomeFatalErr
}
ss.recordRouteAttempt(agent.WorkspaceID,
string(candidate.ProviderID), outcome, string(classified.Code))
fatal, err := ss.handleLaunchFailure(ctx, run, agent, candidate, seq, launchErr)
if err != nil {
return false, nil, err
}
if fatal {
return false, nil, launchErr
}
// Remember this candidate so the next loop iteration can record
// the fallback hop on success.
c := candidate
prev = &c
}
return ss.exhaustedCandidates(ctx, run, agent, res.RequestedTier)
}
// recordAttemptStart increments the attempt sequence and appends the
// in-flight attempt row with outcome=launched.
func (ss *SchedulerService) recordAttemptStart(
ctx context.Context, run *models.Run,
candidate routing.Candidate, tier routing.Tier,
) (int, error) {
seq, err := ss.repo.IncrementRouteAttemptSeq(ctx, run.ID)
if err != nil {
return 0, err
}
run.CurrentRouteAttemptSeq = seq
attempt := models.RouteAttempt{
RunID: run.ID,
Seq: seq,
ProviderID: string(candidate.ProviderID),
Model: candidate.Model,
Tier: string(tier),
Outcome: RouteAttemptOutcomeLaunched,
StartedAt: time.Now().UTC(),
}
if err := ss.repo.AppendRouteAttempt(ctx, &attempt); err != nil {
return 0, err
}
ss.publishRouteAttemptAppended(ctx, run.ID, attempt)
return seq, nil
}
// launchCandidate invokes the task starter with the candidate's route
// override. Returns ErrRoutingNotSupported as-is so the caller can
// treat it as a configuration error rather than a provider failure.
//
// Honors KANDEV_PROVIDER_FAILURES (routingerr.InjectedCode) pre-launch
// so deterministic E2E specs can fail a specific provider without
// requiring the agent binary itself to fail. The injected error is
// returned as a *routingerr.Error wrapped via errors.As so
// classifyLaunchError picks it up unchanged.
//
// launch supplies the Office-built prompt/env/workflow-step/attachments
// so routed launches behave identically to the legacy launch path for
// everything except provider/model selection. Without this, the routed
// path would fall back to task.Description and lose role framing.
func (ss *SchedulerService) launchCandidate(
ctx context.Context, taskID, agentID string,
candidate routing.Candidate, launch LaunchContext,
) error {
if taskID == "" {
return fmt.Errorf("dispatch: empty task id in run payload")
}
if _, ok := routingerr.InjectedCode(string(candidate.ProviderID)); ok {
// Synthesize a launch failure via Classify so injection is
// honoured for deterministic E2E specs. Classify short-circuits
// to the injected code at the head of its decision chain.
return routingerr.Classify(routingerr.Input{
Phase: routingerr.PhaseProcessStart,
ProviderID: string(candidate.ProviderID),
})
}
return ss.taskStarter.StartTaskWithRoute(ctx, taskID, agentID, launch, RouteOverride{
ProviderID: string(candidate.ProviderID),
Model: candidate.Model,
Tier: string(candidate.Tier),
Mode: candidate.Mode,
Flags: candidate.Flags,
Env: candidate.Env,
})
}
// handleLaunchSuccess records the resolved provider/model on the run
// row and flips every health scope this candidate touched back to
// healthy. Returns (launched=true, nil, nil) on success.
func (ss *SchedulerService) handleLaunchSuccess(
ctx context.Context, run *models.Run,
agent *models.AgentInstance, candidate routing.Candidate,
) (bool, *routing.BlockReason, error) {
if err := ss.repo.SetRunResolvedProvider(ctx,
run.ID, string(candidate.ProviderID), candidate.Model); err != nil {
return false, nil, err
}
ss.markHealthScopes(ctx, agent.WorkspaceID, candidate)
return true, nil, nil
}
// markHealthScopes marks the provider, tier, and model scopes healthy
// for the launched candidate. Best-effort: errors are logged but do
// not fail the launch.
func (ss *SchedulerService) markHealthScopes(
ctx context.Context, workspaceID string, candidate routing.Candidate,
) {
scopes := []struct{ scope, value string }{
{sqliterepo.HealthScopeProvider, ""},
{sqliterepo.HealthScopeTier, string(candidate.Tier)},
{sqliterepo.HealthScopeModel, candidate.Model},
}
for _, s := range scopes {
if err := ss.repo.MarkProviderHealthy(ctx,
workspaceID, string(candidate.ProviderID), s.scope, s.value); err != nil {
ss.logger.Warn("mark healthy failed",
zap.String("provider", string(candidate.ProviderID)),
zap.String("scope", s.scope),
zap.Error(err))
continue
}
ss.publishProviderHealthy(ctx, workspaceID,
string(candidate.ProviderID), s.scope, s.value)
}
// Emit once per workspace+provider when the provider scope flips
// back to healthy, not once per scope, so a metrics consumer can
// observe the high-level recover event.
recordProviderRecovered(ss.logger, workspaceID, string(candidate.ProviderID))
}
// handleLaunchFailure classifies the launch error and either records
// the candidate as failed-with-fallback (continue loop) or as
// failed-other (fatal, escalate to caller). Returns (fatal, err).
func (ss *SchedulerService) handleLaunchFailure(
ctx context.Context, run *models.Run,
agent *models.AgentInstance, candidate routing.Candidate,
seq int, launchErr error,
) (bool, error) {
classified := classifyLaunchError(string(candidate.ProviderID), launchErr)
now := time.Now().UTC()
if !classified.FallbackAllowed {
return true, ss.finishAttempt(ctx, run.ID, seq,
RouteAttemptOutcomeFailedOther, classified, now)
}
if err := ss.degradeProviderForCandidate(ctx, agent.WorkspaceID,
candidate, classified, now); err != nil {
return false, err
}
return false, ss.finishAttempt(ctx, run.ID, seq,
RouteAttemptOutcomeFailedProviderUnavail, classified, now)
}
// classifyLaunchError runs routingerr.Classify on a launch error using
// the process-start phase. Adapter-classified errors (when threaded
// through) are honoured via errors.As lookup.
func classifyLaunchError(providerID string, launchErr error) *routingerr.Error {
var pre *routingerr.Error
if errors.As(launchErr, &pre) {
return pre
}
return routingerr.Classify(routingerr.Input{
Phase: routingerr.PhaseProcessStart,
ProviderID: providerID,
Stderr: launchErr.Error(),
})
}
// degradeProviderForCandidate writes a degraded health row for the
// scope (provider/tier/model) that matches the classifier code.
func (ss *SchedulerService) degradeProviderForCandidate(
ctx context.Context, workspaceID string,
candidate routing.Candidate, e *routingerr.Error, now time.Time,
) error {
scope, scopeVal := sqliterepo.ScopeFromCode(
string(e.Code), candidate.Model, candidate.Tier)
current, _ := ss.repo.GetProviderHealth(ctx,
workspaceID, string(candidate.ProviderID), scope, scopeVal)
var currentHealth models.ProviderHealth
if current != nil {
currentHealth = *current
}
retryAt, step := routing.Schedule(currentHealth, e, now)
row := models.ProviderHealth{
WorkspaceID: workspaceID,
ProviderID: string(candidate.ProviderID),
Scope: models.ProviderHealthScope(scope),
ScopeValue: scopeVal,
State: models.ProviderHealthState(healthStateFor(e)),
ErrorCode: string(e.Code),
RetryAt: timePtrIfNonZero(retryAt),
BackoffStep: step,
LastFailure: &now,
RawExcerpt: e.RawExcerpt,
}
if err := ss.repo.MarkProviderDegraded(ctx, row); err != nil {
return err
}
ss.publishProviderHealthChanged(ctx, row)
recordProviderDegraded(ss.logger,
workspaceID, string(candidate.ProviderID), string(e.Code))
return nil
}
func healthStateFor(e *routingerr.Error) string {
if e.UserAction && !e.AutoRetryable {
return sqliterepo.HealthStateUserActionRequired
}
return sqliterepo.HealthStateDegraded
}
func timePtrIfNonZero(t time.Time) *time.Time {
if t.IsZero() {
return nil
}
return &t
}
// finishAttempt updates the in-flight attempt row with the classifier
// outcome. The same path covers both fallback-allowed and fatal errors.
//
// After the DB write, the row is re-read so the WS publish carries a
// fully-hydrated attempt (provider_id / model / tier / started_at,
// which UpdateRouteAttemptOutcome does not touch). Without the
// re-read, the WS reducer on the frontend replaces the existing
// attempt entry with a partial one, clobbering the provider/model
// that was set when the launch attempt began.
func (ss *SchedulerService) finishAttempt(
ctx context.Context, runID string, seq int,
outcome string, e *routingerr.Error, finishedAt time.Time,
) error {
attempt := models.RouteAttempt{
RunID: runID,
Seq: seq,
Outcome: models.RouteAttemptOutcome(outcome),
ErrorCode: string(e.Code),
ErrorConfidence: models.ErrorConfidence(e.Confidence),
AdapterPhase: models.AdapterPhase(e.Phase),
ClassifierRule: e.ClassifierRule,
ExitCode: e.ExitCode,
RawExcerpt: e.RawExcerpt,
ResetHint: e.ResetHint,
FinishedAt: &finishedAt,
}
if err := ss.repo.UpdateRouteAttemptOutcome(ctx, &attempt); err != nil {
return err
}
hydrated := ss.hydrateAttempt(ctx, runID, seq, attempt)
ss.publishRouteAttemptAppended(ctx, runID, hydrated)
return nil
}
// hydrateAttempt re-reads the (run_id, seq) row so the WS publish has
// the full attempt shape. Falls back to the partial row when the
// re-read fails — better to broadcast something than nothing.
func (ss *SchedulerService) hydrateAttempt(
ctx context.Context, runID string, seq int, fallback models.RouteAttempt,
) models.RouteAttempt {
rows, err := ss.repo.ListRouteAttempts(ctx, runID)
if err != nil {
return fallback
}
for _, row := range rows {
if row.Seq == seq {
return row
}
}
return fallback
}
// exhaustedCandidates parks the run when every candidate failed with
// fallback-allowed errors. The block reason is rebuilt from the just-
// recorded health rows so the surfaced earliest-retry is current.
func (ss *SchedulerService) exhaustedCandidates(
ctx context.Context, run *models.Run,
agent *models.AgentInstance, tier routing.Tier,
) (bool, *routing.BlockReason, error) {
rows, err := ss.repo.ListProviderHealth(ctx, agent.WorkspaceID)
if err != nil {
return false, nil, err
}
br := blockReasonFromHealth(rows, tier)
if err := ss.repo.ParkRunForProviderCapacity(ctx,
run.ID, br.Status, br.EarliestRetry); err != nil {
return false, nil, err
}
ss.logger.Info("run parked after candidate exhaustion",
zap.String("run_id", run.ID),
zap.String("status", br.Status))
ss.recordRouteParked(agent.WorkspaceID, run.ID, br.Status)
return false, &br, nil
}
// blockReasonFromHealth aggregates a workspace's non-healthy rows into
// the same shape the resolver returns. Any auto-retryable code marks
// the run as waiting_for_provider_capacity; otherwise blocked.
func blockReasonFromHealth(
rows []models.ProviderHealth, _ routing.Tier,
) routing.BlockReason {
br := routing.BlockReason{Status: routing.StatusBlockedActionRequired}
for _, row := range rows {
if !routing.IsAutoRetryableCode(row.ErrorCode) {
continue
}
br.Status = routing.StatusWaitingForCapacity
if row.RetryAt != nil && !row.RetryAt.IsZero() {
if br.EarliestRetry.IsZero() || row.RetryAt.Before(br.EarliestRetry) {
br.EarliestRetry = *row.RetryAt
}
}
}
return br
}
// extractRunTaskID parses task_id from the run payload JSON. Returns
// "" when the payload is empty or missing the field.
func extractRunTaskID(payload string) string {
if payload == "" {
return ""
}
var p struct {
TaskID string `json:"task_id"`
}
if err := json.Unmarshal([]byte(payload), &p); err != nil {
return ""
}
return p.TaskID
}