Mattias Andersson
343 lines
13 KiB
Go
343 lines
13 KiB
Go
package automation
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"expvar"
|
|
"fmt"
|
|
"github.com/google/uuid"
|
|
"github.com/jmoiron/sqlx"
|
|
"github.com/kandev/kandev/internal/events"
|
|
"github.com/kandev/kandev/internal/events/bus"
|
|
"github.com/kandev/kandev/pkg/pluginsdk"
|
|
"go.uber.org/zap"
|
|
"time"
|
|
)
|
|
|
|
var webhookCounters = expvar.NewMap("automation_webhooks")
|
|
|
|
// ProcessWebhookReceipts retries persisted admissions and unclaimed dispatches.
|
|
// The backend has one automation worker; the DB claim also fences duplicate bus delivery.
|
|
func (s *Service) ProcessWebhookReceipts(ctx context.Context) error {
|
|
if s.pluginAutomation == nil {
|
|
return nil
|
|
}
|
|
if err := s.cleanupWebhookSecrets(ctx); err != nil {
|
|
webhookCounters.Add("cleanup_failures", 1)
|
|
s.logger.Warn("webhook secret cleanup failed", zap.Error(err))
|
|
}
|
|
receipts := []WebhookReceipt{}
|
|
if err := s.store.db.SelectContext(ctx, &receipts, s.store.db.Rebind(`SELECT * FROM automation_webhook_receipts WHERE state IN ('pending','dispatch') AND next_attempt_at <= ? ORDER BY next_attempt_at,created_at,id LIMIT 100`), time.Now().Unix()); err != nil {
|
|
return err
|
|
}
|
|
for i := range receipts {
|
|
ready, err := s.reserveWebhookAttempt(ctx, &receipts[i])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !ready {
|
|
continue
|
|
}
|
|
if err := s.processWebhookReceipt(ctx, &receipts[i]); err != nil {
|
|
webhookCounters.Add("processing_failures", 1)
|
|
s.logger.Warn("webhook receipt processing failed", zap.String("receipt_id", receipts[i].ID), zap.Error(err))
|
|
}
|
|
}
|
|
_, err := s.store.db.ExecContext(ctx, s.store.db.Rebind(`DELETE FROM automation_webhook_receipts WHERE state NOT IN ('pending','dispatch','processing') AND finished_at > 0 AND finished_at < ?`), time.Now().Add(-7*24*time.Hour).Unix())
|
|
return err
|
|
}
|
|
func (s *Service) processWebhookReceipt(ctx context.Context, r *WebhookReceipt) error {
|
|
b, err := s.store.webhookBinding(ctx, r.BindingID, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if b == nil || b.Revision != r.Revision {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "binding changed")
|
|
}
|
|
t, err := s.store.GetTrigger(ctx, b.TriggerID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if t == nil || !t.Enabled {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "condition disabled")
|
|
}
|
|
cfg, err := parsePluginEventConfig(t.Config)
|
|
if err != nil {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "condition changed")
|
|
}
|
|
_, generation, adapter, release, err := s.pluginAutomation.AcquireAutomationAdapter(cfg.PluginID, cfg.ConditionKey)
|
|
if err != nil {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "plugin unavailable")
|
|
}
|
|
defer release()
|
|
if b.Revision != bindingRevision(t, generation, b.ConnectionRevision)+":"+b.SecretID {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "adapter changed")
|
|
}
|
|
callCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
description, err := adapter.DescribeAutomationCondition(callCtx, &pluginsdk.AutomationConditionRequest{WorkspaceId: b.WorkspaceID, ConditionKey: cfg.ConditionKey, Config: cfg.Settings})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !webhookConnectionMatches(description, b) {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "connection changed")
|
|
}
|
|
return s.processCurrentWebhookReceipt(ctx, r, b, t)
|
|
}
|
|
func (s *Service) processCurrentWebhookReceipt(ctx context.Context, r *WebhookReceipt, b *WebhookBinding, t *AutomationTrigger) error {
|
|
unlock := s.automationRunLock(b.AutomationID)
|
|
defer unlock()
|
|
current, err := s.store.webhookBinding(ctx, b.ID, false)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if current == nil || current.Revision != b.Revision {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "binding changed")
|
|
}
|
|
a, err := s.store.GetAutomation(ctx, b.AutomationID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if a == nil || !a.Enabled {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "automation disabled")
|
|
}
|
|
valid, err := s.currentWebhookAuthority(ctx, b, t)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !valid {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "condition changed")
|
|
}
|
|
if r.State == "pending" {
|
|
if err := s.admitWebhookReceipt(ctx, a, t, r); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if r.State != "dispatch" {
|
|
return nil
|
|
}
|
|
run, err := s.store.GetRun(ctx, r.RunID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if run == nil {
|
|
return s.finishWebhookReceipt(ctx, r, "cancelled", "run history removed")
|
|
}
|
|
evt := &AutomationTriggeredEvent{RunID: r.RunID, AutomationID: a.ID, TriggerID: t.ID, TriggerType: TriggerTypePluginEvent, TriggerData: json.RawMessage(r.Payload), DedupKey: r.ID}
|
|
return s.eventBus.Publish(ctx, events.AutomationTriggered, bus.NewEvent(events.AutomationTriggered, "automation_webhook", evt))
|
|
}
|
|
func (s *Service) admitWebhookReceipt(ctx context.Context, a *Automation, t *AutomationTrigger, r *WebhookReceipt) error {
|
|
reason, _, full, err := s.automationCapacity(ctx, a)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if full {
|
|
r.State = "skipped"
|
|
return s.finishWebhookReceipt(ctx, r, "skipped", reason)
|
|
}
|
|
tx, err := s.store.db.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
runID := uuid.NewString()
|
|
result, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE automation_webhook_receipts SET state='dispatch',run_id=? WHERE id=? AND state='pending'`), runID, r.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n != 1 {
|
|
return nil
|
|
}
|
|
_, err = tx.ExecContext(ctx, tx.Rebind(`INSERT INTO automation_runs
|
|
(id,automation_id,trigger_id,trigger_type,status,dedup_key,trigger_data,display_title,created_at) VALUES (?,?,?,?,?,?,?,?,?)`), runID, a.ID, t.ID, TriggerTypePluginEvent, RunStatusTriggered, r.ID, r.Payload, RenderRunDisplayTitle(a, TriggerTypePluginEvent, json.RawMessage(r.Payload)), time.Now().UTC())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err = recordWebhookAdmission(ctx, tx, a.ID, t.ID); err != nil {
|
|
return err
|
|
}
|
|
if err = tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
r.RunID = runID
|
|
r.State = "dispatch"
|
|
return nil
|
|
}
|
|
func (s *Service) finishWebhookReceipt(ctx context.Context, r *WebhookReceipt, state, reason string) error {
|
|
tx, err := s.store.db.BeginTxx(ctx, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
result, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE automation_webhook_receipts SET state=?,reason=?,payload='{}',finished_at=? WHERE id=? AND state IN ('pending','dispatch')`), state, reason, time.Now().Unix(), r.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n > 0 && (state == "cancelled" || state == "failed") {
|
|
// Read the current run identity transactionally; the caller may hold a pre-admission snapshot.
|
|
_, err = tx.ExecContext(ctx, tx.Rebind(`UPDATE automation_runs SET status=?,error_message=? WHERE id=(SELECT run_id FROM automation_webhook_receipts WHERE id=?) AND status=?`), RunStatusFailed, reason, r.ID, RunStatusTriggered)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err = tx.Commit(); err != nil {
|
|
return err
|
|
}
|
|
if n > 0 {
|
|
webhookCounters.Add(state, n)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// ClaimPluginWebhookRun is consumed before task creation; a claimed run is never dispatched twice.
|
|
func (s *Service) ClaimPluginWebhookRun(ctx context.Context, runID string) (bool, error) {
|
|
r := &WebhookReceipt{}
|
|
err := s.store.db.GetContext(ctx, r, s.store.db.Rebind(`SELECT * FROM automation_webhook_receipts WHERE run_id=? AND state='dispatch' AND EXISTS (SELECT 1 FROM automation_runs WHERE id=?)`), runID, runID)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
b, err := s.store.webhookBinding(ctx, r.BindingID, false)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if b == nil {
|
|
return false, nil
|
|
}
|
|
t, err := s.store.GetTrigger(ctx, b.TriggerID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if t == nil {
|
|
return false, nil
|
|
}
|
|
return s.claimCurrentWebhookRun(ctx, r, b, t)
|
|
}
|
|
func (s *Service) claimCurrentWebhookRun(ctx context.Context, r *WebhookReceipt, b *WebhookBinding, t *AutomationTrigger) (bool, error) {
|
|
runID := r.RunID
|
|
cfg, err := parsePluginEventConfig(t.Config)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
_, generation, adapter, release, err := s.pluginAutomation.AcquireAutomationAdapter(cfg.PluginID, cfg.ConditionKey)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
defer release()
|
|
callCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
|
defer cancel()
|
|
info, err := adapter.DescribeAutomationCondition(callCtx, &pluginsdk.AutomationConditionRequest{WorkspaceId: b.WorkspaceID, ConditionKey: cfg.ConditionKey, Config: cfg.Settings})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
unlock := s.automationRunLock(b.AutomationID)
|
|
defer unlock()
|
|
current, err := s.store.webhookBinding(ctx, b.ID, false)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
valid, err := s.currentWebhookAuthority(ctx, b, t)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if !valid || current == nil || current.Revision != r.Revision || b.Revision != bindingRevision(t, generation, b.ConnectionRevision)+":"+b.SecretID || !webhookConnectionMatches(info, b) {
|
|
return false, s.finishWebhookReceipt(ctx, r, "cancelled", "authority changed")
|
|
}
|
|
result, err := s.store.db.ExecContext(ctx, s.store.db.Rebind(`UPDATE automation_webhook_receipts SET state='processing' WHERE run_id=? AND state='dispatch' AND EXISTS (SELECT 1 FROM automation_runs WHERE id=?)`), runID, runID)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
return n == 1, err
|
|
}
|
|
func (s *Service) CompletePluginWebhookRun(ctx context.Context, runID string) error {
|
|
run, err := s.store.GetRun(ctx, runID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if run == nil {
|
|
return fmt.Errorf("webhook run disappeared")
|
|
}
|
|
state := "admitted"
|
|
if run.Status == RunStatusFailed || run.TaskID == "" {
|
|
state = "failed"
|
|
}
|
|
result, err := s.store.db.ExecContext(ctx, s.store.db.Rebind(`UPDATE automation_webhook_receipts SET state=?,payload='{}',finished_at=? WHERE run_id=? AND state='processing'`), state, time.Now().Unix(), runID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
if err == nil && n > 0 {
|
|
webhookCounters.Add(state, n)
|
|
}
|
|
return err
|
|
}
|
|
func (s *Service) recoverWebhookClaims(ctx context.Context) error {
|
|
webhookCounters.Add("recovery_passes", 1)
|
|
// A crash after a durable dispatch claim has an indeterminate downstream outcome.
|
|
// Mark the receipt failed and expose the existing run; never repeat task creation.
|
|
_, err := s.store.db.ExecContext(ctx, s.store.db.Rebind(`UPDATE automation_webhook_receipts SET state='failed',reason='host stopped during dispatch',payload='{}',finished_at=? WHERE state='processing'`), time.Now().Unix())
|
|
return err
|
|
}
|
|
func (s *Service) hasPendingWebhookDispatch(ctx context.Context, runID string) (bool, error) {
|
|
var count int
|
|
err := s.store.db.GetContext(ctx, &count, s.store.db.Rebind(`SELECT COUNT(*) FROM automation_webhook_receipts WHERE run_id=? AND state='dispatch' AND EXISTS (SELECT 1 FROM automation_runs WHERE id=?)`), runID, runID)
|
|
return count > 0, err
|
|
}
|
|
func (s *Service) runWebhookWorker(ctx context.Context) {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-ticker.C:
|
|
if err := s.ProcessWebhookReceipts(ctx); err != nil {
|
|
s.logger.Warn("webhook worker failed", zap.Error(err))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func webhookConnectionMatches(info *pluginsdk.AutomationConditionResponse, b *WebhookBinding) bool {
|
|
return info != nil && info.Available && info.ConnectionId == b.ConnectionID && info.ConnectionRevision == b.ConnectionRevision
|
|
}
|
|
|
|
func recordWebhookAdmission(ctx context.Context, tx *sqlx.Tx, automationID, triggerID string) error {
|
|
now := time.Now().UTC()
|
|
if _, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE automations SET last_triggered_at=? WHERE id=?`), now, automationID); err != nil {
|
|
return err
|
|
}
|
|
_, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE automation_triggers SET last_evaluated_at=? WHERE id=?`), now, triggerID)
|
|
return err
|
|
}
|
|
|
|
const webhookMaxAttempts = 8
|
|
|
|
// Reserve the retry budget before any fallible dispatch, including an unclaimed publish.
|
|
func (s *Service) reserveWebhookAttempt(ctx context.Context, r *WebhookReceipt) (bool, error) {
|
|
if r.AttemptCount >= webhookMaxAttempts {
|
|
return false, s.finishWebhookReceipt(ctx, r, "failed", "delivery retry limit reached")
|
|
}
|
|
delay := min(5*time.Second*time.Duration(1<<r.AttemptCount), 5*time.Minute)
|
|
now := time.Now().Unix()
|
|
result, err := s.store.db.ExecContext(ctx, s.store.db.Rebind(`UPDATE automation_webhook_receipts SET attempt_count=attempt_count+1,next_attempt_at=? WHERE id=? AND state IN ('pending','dispatch') AND attempt_count=? AND next_attempt_at<=?`), now+int64(delay/time.Second), r.ID, r.AttemptCount, now)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n, err := result.RowsAffected()
|
|
return n == 1, err
|
|
}
|