Files
kandev/apps/backend/internal/office/repository/sqlite/failure.go
T
2026-09-07 13:47:30 +01:00

539 lines
17 KiB
Go

package sqlite
import (
"context"
"database/sql"
"fmt"
"time"
"github.com/kandev/kandev/internal/db/dialect"
"github.com/kandev/kandev/internal/office/models"
)
// GetRun fetches a single run row by id.
func (r *Repository) GetRun(
ctx context.Context, id string,
) (*models.Run, error) {
var req models.Run
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
SELECT * FROM runs WHERE id = ?
`), id).StructScan(&req)
if err != nil {
return nil, err
}
return &req, nil
}
// AgentPauseRecovery is the task snapshot captured when an agent enters
// auto-pause. The failed run identifies the failure streak entry that made
// the task eligible for recovery.
type AgentPauseRecovery struct {
AgentID string `db:"agent_id"`
TaskID string `db:"task_id"`
FailedRunID string `db:"failed_run_id"`
}
// ReplaceAgentPauseRecoveries replaces the active recovery snapshot for an
// agent with its newest failed runs. The limit is the consecutive-failure
// count, so older failures from a previous streak are not selected.
func (r *Repository) ReplaceAgentPauseRecoveries(
ctx context.Context, agentID string, limit int,
) error {
snapshot, err := r.pauseRecoverySnapshot(ctx, agentID, limit)
if err != nil {
return err
}
tx, err := r.db.BeginTxx(ctx, nil)
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
if _, err := tx.ExecContext(ctx, tx.Rebind(`
DELETE FROM office_agent_pause_recoveries WHERE agent_id = ?
`), agentID); err != nil {
return err
}
now := time.Now().UTC()
for _, recovery := range snapshot {
if _, err := tx.ExecContext(ctx, tx.Rebind(`
INSERT INTO office_agent_pause_recoveries
(agent_id, task_id, failed_run_id, captured_at)
VALUES (?, ?, ?, ?)
`), recovery.AgentID, recovery.TaskID, recovery.FailedRunID, now); err != nil {
return err
}
}
return tx.Commit()
}
func (r *Repository) pauseRecoverySnapshot(
ctx context.Context, agentID string, limit int,
) ([]AgentPauseRecovery, error) {
if limit <= 0 {
return nil, nil
}
taskIDExpr := dialect.JSONExtract(r.ro.DriverName(), "payload", "task_id")
rows, err := r.ro.QueryxContext(ctx, r.ro.Rebind(fmt.Sprintf(`
SELECT id AS failed_run_id, COALESCE(%s, '') AS task_id
FROM runs
WHERE agent_profile_id = ? AND status = 'failed'
ORDER BY COALESCE(finished_at, requested_at) DESC,
requested_at DESC, id DESC
LIMIT ?
`, taskIDExpr)), agentID, limit)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
seenTasks := make(map[string]struct{}, limit)
var snapshot []AgentPauseRecovery
for rows.Next() {
var row AgentPauseRecovery
if err := rows.StructScan(&row); err != nil {
return nil, err
}
if row.TaskID == "" {
continue
}
if _, seen := seenTasks[row.TaskID]; seen {
continue
}
seenTasks[row.TaskID] = struct{}{}
row.AgentID = agentID
snapshot = append(snapshot, row)
}
if err := rows.Err(); err != nil {
return nil, err
}
return snapshot, nil
}
// ListAgentPauseRecoveries returns the active task snapshot for an agent.
func (r *Repository) ListAgentPauseRecoveries(
ctx context.Context, agentID string,
) ([]AgentPauseRecovery, error) {
var rows []AgentPauseRecovery
err := r.ro.SelectContext(ctx, &rows, r.ro.Rebind(`
SELECT agent_id, task_id, failed_run_id
FROM office_agent_pause_recoveries
WHERE agent_id = ?
ORDER BY captured_at ASC, task_id ASC
`), agentID)
if rows == nil {
rows = []AgentPauseRecovery{}
}
return rows, err
}
// DeleteAgentPauseRecovery removes one completed or obsolete snapshot row.
func (r *Repository) DeleteAgentPauseRecovery(
ctx context.Context, agentID, taskID string,
) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
DELETE FROM office_agent_pause_recoveries
WHERE agent_id = ? AND task_id = ?
`), agentID, taskID)
return err
}
// GetLatestRunForAgentTask returns the newest run for an agent/task pair.
// It includes non-failed states so recovery can discard a snapshot when a
// newer run already succeeded or is already queued.
func (r *Repository) GetLatestRunForAgentTask(
ctx context.Context, agentID, taskID string,
) (*models.Run, error) {
taskIDExpr := dialect.JSONExtract(r.ro.DriverName(), "payload", "task_id")
var run models.Run
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(fmt.Sprintf(`
SELECT * FROM runs
WHERE agent_profile_id = ? AND %s = ?
ORDER BY COALESCE(finished_at, requested_at) DESC,
requested_at DESC, id DESC
LIMIT 1
`, taskIDExpr)), agentID, taskID).StructScan(&run)
if err == sql.ErrNoRows {
return nil, nil
}
if err != nil {
return nil, err
}
return &run, nil
}
// DefaultAgentFailureThreshold is the workspace-level default applied
// when neither a per-workspace nor per-agent override is set.
const DefaultAgentFailureThreshold = 3
// IncrementAgentConsecutiveFailures bumps the per-agent counter and
// returns the post-increment value so callers can decide whether the
// threshold has been crossed.
func (r *Repository) IncrementAgentConsecutiveFailures(
ctx context.Context, agentID string,
) (int, error) {
if _, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE agent_profiles
SET consecutive_failures = consecutive_failures + 1,
updated_at = ?
WHERE id = ?
`), time.Now().UTC(), agentID); err != nil {
return 0, err
}
var n int
if err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
SELECT consecutive_failures FROM agent_profiles WHERE id = ?
`), agentID).Scan(&n); err != nil {
return 0, err
}
return n, nil
}
// ResetAgentConsecutiveFailures clears the counter — called after any
// successful agent turn or when the user marks a paused agent as
// fixed via the inbox.
func (r *Repository) ResetAgentConsecutiveFailures(
ctx context.Context, agentID string,
) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE agent_profiles
SET consecutive_failures = 0, updated_at = ?
WHERE id = ?
`), time.Now().UTC(), agentID)
return err
}
// GetEffectiveFailureThreshold resolves the per-agent override, then
// the per-workspace override, then the global default.
//
// agent_profiles.failure_threshold is NOT NULL DEFAULT 3 under the
// merged schema; office stores 0 to mean "no override, use workspace
// default" (round-tripped to NULL on the AgentInstance struct).
func (r *Repository) GetEffectiveFailureThreshold(
ctx context.Context, agentID string,
) (int, error) {
var override sql.NullInt64
var workspaceID string
if err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
SELECT NULLIF(failure_threshold, 0), COALESCE(workspace_id, '')
FROM agent_profiles WHERE id = ?
`), agentID).Scan(&override, &workspaceID); err != nil {
return 0, err
}
if override.Valid && override.Int64 > 0 {
return int(override.Int64), nil
}
return r.GetWorkspaceAgentFailureThreshold(ctx, workspaceID)
}
// GetWorkspaceAgentFailureThreshold returns the workspace-level
// threshold, falling back to DefaultAgentFailureThreshold when the
// workspace has no row.
func (r *Repository) GetWorkspaceAgentFailureThreshold(
ctx context.Context, workspaceID string,
) (int, error) {
var n int
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
SELECT agent_failure_threshold
FROM office_workspace_settings WHERE workspace_id = ?
`), workspaceID).Scan(&n)
if err == sql.ErrNoRows {
return DefaultAgentFailureThreshold, nil
}
if err != nil {
return 0, err
}
if n <= 0 {
return DefaultAgentFailureThreshold, nil
}
return n, nil
}
// SetWorkspaceAgentFailureThreshold upserts the workspace setting.
func (r *Repository) SetWorkspaceAgentFailureThreshold(
ctx context.Context, workspaceID string, threshold int,
) error {
if threshold <= 0 {
threshold = DefaultAgentFailureThreshold
}
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
INSERT INTO office_workspace_settings (workspace_id, agent_failure_threshold, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(workspace_id) DO UPDATE SET
agent_failure_threshold = excluded.agent_failure_threshold,
updated_at = excluded.updated_at
`), workspaceID, threshold, time.Now().UTC())
return err
}
// MarkRunFailed sets a run row to status=failed with a verbatim
// error message and stamps finished_at. Idempotent: re-marking is a
// no-op when the run is already in the same state.
func (r *Repository) MarkRunFailed(
ctx context.Context, runID, errorMessage string,
) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE runs
SET status = 'failed',
error_message = ?,
finished_at = COALESCE(finished_at, ?)
WHERE id = ?
`), errorMessage, time.Now().UTC(), runID)
return err
}
// DismissInboxItem records a per-user dismissal. Idempotent — a second
// dismiss for the same (user, kind, item) is silently accepted.
func (r *Repository) DismissInboxItem(
ctx context.Context, userID, kind, itemID string,
) error {
_, err := r.db.ExecContext(ctx, r.db.Rebind(`
INSERT INTO office_inbox_dismissals (user_id, item_kind, item_id, dismissed_at)
VALUES (?, ?, ?, ?)
ON CONFLICT (user_id, item_kind, item_id) DO UPDATE SET
dismissed_at = excluded.dismissed_at
`), userID, kind, itemID, time.Now().UTC())
return err
}
// IsInboxItemDismissed reports whether the given (user, kind, item)
// has been dismissed.
func (r *Repository) IsInboxItemDismissed(
ctx context.Context, userID, kind, itemID string,
) (bool, error) {
var n int
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(`
SELECT COUNT(*) FROM office_inbox_dismissals
WHERE user_id = ? AND item_kind = ? AND item_id = ?
`), userID, kind, itemID).Scan(&n)
return n > 0, err
}
// ListDismissedInboxItemIDs returns the set of item ids dismissed by
// the given user for the given kind. Used by the inbox query to
// filter out already-dismissed entries in a single round-trip.
func (r *Repository) ListDismissedInboxItemIDs(
ctx context.Context, userID, kind string,
) ([]string, error) {
rows, err := r.ro.QueryxContext(ctx, r.ro.Rebind(`
SELECT item_id FROM office_inbox_dismissals
WHERE user_id = ? AND item_kind = ?
`), userID, kind)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, rows.Err()
}
// FailedRunInboxRow is a flat view of a failed run joined with
// its agent (so the inbox layer can render a row without a follow-up
// per-agent fetch). Excludes runs belonging to auto-paused agents
// since those are consolidated into the per-agent paused entry.
type FailedRunInboxRow struct {
RunID string `db:"run_id"`
AgentProfileID string `db:"agent_profile_id"`
AgentName string `db:"agent_name"`
WorkspaceID string `db:"workspace_id"`
TaskID string `db:"task_id"`
ErrorMessage string `db:"error_message"`
// FailedAt is COALESCE(finished_at, requested_at) — SQLite stores
// these as TEXT so we scan into a string and parse on the Go side.
FailedAtRaw string `db:"failed_at"`
FailedAt time.Time `db:"-"`
}
// ListFailedRunsForInbox returns one row per failed run that:
// - belongs to a workspace,
// - whose agent is NOT currently auto-paused (those are covered by
// the consolidated agent_paused_after_failures entry), and
// - has no dismissal row from the given user OR the auto-dismiss
// sentinel.
//
// Sorted by finished_at descending so the freshest failures float to
// the top of the inbox.
func (r *Repository) ListFailedRunsForInbox(
ctx context.Context, workspaceID, userID string,
) ([]*FailedRunInboxRow, error) {
rows, err := r.ro.QueryxContext(ctx, r.ro.Rebind(`
SELECT w.id AS run_id,
w.agent_profile_id,
a.name AS agent_name,
a.workspace_id,
COALESCE(json_extract(w.payload, '$.task_id'), '') AS task_id,
w.error_message,
COALESCE(w.finished_at, w.requested_at) AS failed_at
FROM runs w
JOIN agent_profiles a ON a.id = w.agent_profile_id
WHERE w.status = 'failed'
AND a.workspace_id = ?
AND (a.pause_reason IS NULL OR a.pause_reason NOT LIKE 'Auto-paused:%')
AND NOT EXISTS (
SELECT 1 FROM office_inbox_dismissals d
WHERE d.item_kind = 'agent_run_failed'
AND d.item_id = w.id
AND d.user_id IN (?, '_auto')
)
ORDER BY failed_at DESC
LIMIT 200
`), workspaceID, userID)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []*FailedRunInboxRow
for rows.Next() {
var row FailedRunInboxRow
if err := rows.StructScan(&row); err != nil {
return nil, err
}
row.FailedAt = parseSqliteTime(row.FailedAtRaw)
out = append(out, &row)
}
return out, rows.Err()
}
// parseSqliteTime parses the various string formats SQLite stores
// TIMESTAMP values in (RFC3339, with or without timezone, with or
// without nanoseconds). Falls back to zero time on parse failure
// rather than erroring, since the inbox doesn't break if a row's
// timestamp is unparseable.
func parseSqliteTime(s string) time.Time {
if s == "" {
return time.Time{}
}
for _, layout := range []string{
time.RFC3339Nano,
time.RFC3339,
"2006-01-02 15:04:05.999999999-07:00",
"2006-01-02 15:04:05.999999999Z07:00",
"2006-01-02 15:04:05-07:00",
"2006-01-02 15:04:05",
} {
if t, err := time.Parse(layout, s); err == nil {
return t.UTC()
}
}
return time.Time{}
}
// PausedAgentInboxRow is a flat view of an auto-paused agent with the
// snapshot of tasks affected (those whose most recent failed run
// is for this agent and isn't dismissed).
type PausedAgentInboxRow struct {
AgentID string `db:"agent_id"`
AgentName string `db:"agent_name"`
WorkspaceID string `db:"workspace_id"`
PauseReason string `db:"pause_reason"`
UpdatedAt time.Time `db:"updated_at"`
ConsecutiveFailures int `db:"consecutive_failures"`
}
// ListAutoPausedAgentsForInbox returns the agents auto-paused in the
// workspace whose paused-entry hasn't been user-dismissed.
func (r *Repository) ListAutoPausedAgentsForInbox(
ctx context.Context, workspaceID, userID string,
) ([]*PausedAgentInboxRow, error) {
rows, err := r.ro.QueryxContext(ctx, r.ro.Rebind(`
SELECT a.id AS agent_id,
a.name AS agent_name,
a.workspace_id,
a.pause_reason,
a.updated_at,
a.consecutive_failures
FROM agent_profiles a
WHERE a.workspace_id = ?
AND a.workspace_id != ''
AND a.deleted_at IS NULL
AND a.pause_reason LIKE 'Auto-paused:%'
AND NOT EXISTS (
SELECT 1 FROM office_inbox_dismissals d
WHERE d.item_kind = 'agent_paused_after_failures'
AND d.item_id = a.id
AND d.user_id IN (?, '_auto')
)
ORDER BY a.updated_at DESC
LIMIT 200
`), workspaceID, userID)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var out []*PausedAgentInboxRow
for rows.Next() {
var row PausedAgentInboxRow
if err := rows.StructScan(&row); err != nil {
return nil, err
}
out = append(out, &row)
}
return out, rows.Err()
}
// HasPriorTasklessFailedRun reports whether the same continuation scope for
// an agent already has a failed run with no task_id in its payload, other
// than excludeRunID. Used to cap inbox noise from routines that are taskless
// by design (the pre-installed "Coordinator heartbeat" fires every 5 minutes,
// WO-35). Agent-scoped callers also match legacy rows with an empty scope.
func (r *Repository) HasPriorTasklessFailedRun(
ctx context.Context, agentID, continuationScope, excludeRunID string,
) (bool, error) {
taskIDExpr := dialect.JSONExtract(r.ro.DriverName(), "payload", "task_id")
if continuationScope == "" {
continuationScope = "agent:" + agentID
}
scopeClause := "continuation_scope = ?"
if continuationScope == "agent:"+agentID {
// Rows created before continuation_scope was introduced have an
// empty value. Treat those rows as agent-scoped legacy failures.
scopeClause = "(continuation_scope = ? OR continuation_scope = '')"
}
var found int
err := r.ro.QueryRowxContext(ctx, r.ro.Rebind(fmt.Sprintf(`
SELECT 1 FROM runs
WHERE agent_profile_id = ?
AND status = 'failed'
AND id != ?
AND COALESCE(%s, '') = ''
AND %s
LIMIT 1
`, taskIDExpr, scopeClause)), agentID, excludeRunID, continuationScope).Scan(&found)
if err == sql.ErrNoRows {
return false, nil
}
return err == nil, err
}
// ListFailedRunsForAgent returns all failed runs for an agent. The assignee
// change handler uses this list to dismiss obsolete per-task inbox entries.
func (r *Repository) ListFailedRunsForAgent(
ctx context.Context, agentID string,
) ([]string, error) {
rows, err := r.ro.QueryxContext(ctx, r.ro.Rebind(`
SELECT id FROM runs
WHERE agent_profile_id = ? AND status = 'failed'
ORDER BY finished_at DESC
`), agentID)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
var ids []string
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, rows.Err()
}