Files
kandev/apps/backend/internal/executors/reachability/poller.go
T
2026-09-22 10:31:00 +01:00

298 lines
8.7 KiB
Go

package reachability
import (
"context"
"sync"
"time"
"go.uber.org/zap"
"golang.org/x/sync/singleflight"
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/task/models"
)
// probeFunc is the injectable seam over a single probe attempt. defaultProbe
// is the production implementation; tests substitute a fake so poller
// lifecycle/concurrency behavior can be verified under testing/synctest
// without real SSH or network I/O.
type probeFunc func(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome
// Poller sweeps every eligible SSH executor on intervalSeconds, probing each
// with bounded concurrency and persisting results through store. An
// intervalSeconds of 0 disables the scheduled sweep entirely — the poller
// still starts (so ProbeNow keeps working) but never runs a pass on its own.
//
// Start and Stop are idempotent. Stop cancels every in-flight unit of work
// registered on wg — the scheduled loop, the pass it is running, and any
// off-cycle ProbeNow call — and waits for all of them to drain before
// returning, so a probe cancelled by Stop is guaranteed to never be
// persisted (see probeAndPersist).
type Poller struct {
store *store
log *logger.Logger
intervalSeconds int
probe probeFunc
// publisher is optional: nil until SetPublisher wires the event bus, so
// task 03's own tests and callers that never call it keep working
// unchanged. Set once at construction time in production.
publisher *Publisher
// inflight coalesces concurrent ProbeAndWait callers for the same
// executor id into a single dial. Zero value is ready to use.
inflight singleflight.Group
// probeSem is owned by the poller rather than by a scheduled pass. This
// keeps scheduled, manual, and synchronous probes under one cap.
probeSem chan struct{}
// mu guards started/stopping/ctx/cancel/wg against concurrent
// Start/Stop/ProbeNow calls. acquire() registers new work on wg under mu
// so it can never race a concurrent Stop's wg.Wait() observing the
// counter transition from zero.
mu sync.Mutex
started bool
stopping bool
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
stopDone chan struct{}
// passMu guards passRunning, the single-flight gate for the scheduled
// pass: an overlapping tick is dropped and counted, never queued.
passMu sync.Mutex
passRunning bool
}
// New builds a Poller. intervalSeconds is clamped via ClampInterval; a
// clamp is logged once rather than refusing to start.
func New(repo Repository, intervalSeconds int, log *logger.Logger) *Poller {
effective, clamped := ClampInterval(intervalSeconds)
if clamped {
log.Warn("executor ssh reachability: interval clamped",
zap.Int("configured", intervalSeconds), zap.Int("effective", effective))
}
return &Poller{
store: &store{repo: repo, log: log},
log: log,
intervalSeconds: effective,
probe: defaultProbe,
probeSem: make(chan struct{}, passConcurrency),
}
}
// SetPublisher wires the event publisher used to announce a state/reason
// change. Optional — a Poller with no publisher still probes and persists
// exactly as before, it just never publishes.
func (p *Poller) SetPublisher(publisher *Publisher) {
p.publisher = publisher
}
// Start launches the background loop. Calling Start more than once without
// an intervening Stop is a no-op.
func (p *Poller) Start(ctx context.Context) {
p.mu.Lock()
if p.started {
p.mu.Unlock()
return
}
runCtx, cancel := context.WithCancel(ctx)
p.started = true
p.stopping = false
p.ctx = runCtx
p.cancel = cancel
p.stopDone = make(chan struct{})
if p.intervalSeconds > 0 {
p.wg.Add(1)
}
p.mu.Unlock()
if p.intervalSeconds > 0 {
go p.loop(runCtx)
}
}
// Stop cancels the loop and every in-flight probe, then waits for all of
// them to drain. Calling Stop before Start, or twice in a row, is a no-op.
func (p *Poller) Stop() {
p.mu.Lock()
if !p.started {
p.mu.Unlock()
return
}
stopDone := p.stopDone
if p.stopping {
p.mu.Unlock()
<-stopDone
return
}
p.stopping = true
cancel := p.cancel
p.mu.Unlock()
cancel()
p.wg.Wait()
p.mu.Lock()
p.started = false
p.stopping = false
p.ctx = nil
p.cancel = nil
if p.stopDone == stopDone {
close(stopDone)
p.stopDone = nil
}
p.mu.Unlock()
}
// ProbeNow runs one off-cycle probe for executor, outside the scheduled
// cadence, tracked on the same WaitGroup/context as the scheduled loop so
// Stop drains it too. Returns false without doing anything when the poller
// is not currently accepting new work.
func (p *Poller) ProbeNow(executor *models.Executor) bool {
ctx, ok := p.acquire()
if !ok {
return false
}
go func() {
defer p.wg.Done()
p.probeAndPersist(ctx, executor)
}()
return true
}
func (p *Poller) acquireProbeSlot(ctx context.Context) bool {
select {
case p.probeSem <- struct{}{}:
return true
case <-ctx.Done():
return false
}
}
func (p *Poller) releaseProbeSlot() {
<-p.probeSem
}
// acquire registers one unit of in-flight work on wg, gated by mu so it can
// never race a concurrent Stop's wg.Wait(). It refuses when the poller isn't
// started or a Stop is already underway.
func (p *Poller) acquire() (context.Context, bool) {
p.mu.Lock()
defer p.mu.Unlock()
if !p.started || p.stopping {
return nil, false
}
// loop already owns a WaitGroup reference, so this positive Add cannot
// race a Stop waiting for the counter to reach zero.
// Safe without passMu: dispatchPass is called only from loop, which holds
// its own WaitGroup slot, so Stop cannot observe zero before this Add.
p.wg.Add(1)
return p.ctx, true
}
func (p *Poller) loop(ctx context.Context) {
defer p.wg.Done()
ticker := time.NewTicker(time.Duration(p.intervalSeconds) * time.Second)
defer ticker.Stop()
p.dispatchPass(ctx)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
p.dispatchPass(ctx)
}
}
}
// dispatchPass starts one pass if none is currently running. An overlapping
// dispatch is dropped and counted rather than queued, so a slow pass never
// builds up a backlog of pending passes. The pass itself runs on its own
// WaitGroup unit so Stop drains it even after loop itself has returned.
func (p *Poller) dispatchPass(ctx context.Context) {
p.passMu.Lock()
if p.passRunning {
p.passMu.Unlock()
passSkippedTotal.Add(1)
return
}
p.passRunning = true
p.passMu.Unlock()
p.wg.Add(1)
go func() {
defer p.wg.Done()
defer func() {
p.passMu.Lock()
p.passRunning = false
p.passMu.Unlock()
}()
p.runPass(ctx)
}()
}
// runPass lists every eligible executor and probes each with bounded
// concurrency. A listing failure abandons the pass without writing anything.
// Cancellation stops dispatching new probes and waits only for the ones
// already started to finish or self-cancel.
func (p *Poller) runPass(ctx context.Context) {
start := time.Now()
executors, err := p.store.repo.ListSSHExecutorsForReachability(ctx)
if err != nil {
p.log.Warn("executor ssh reachability: list failed", zap.Error(err))
return
}
var wg sync.WaitGroup
for _, executor := range executors {
if !p.acquireProbeSlot(ctx) {
wg.Wait()
return
}
wg.Add(1)
go func(executor *models.Executor) {
defer wg.Done()
defer p.releaseProbeSlot()
p.probeAndPersistWithSlot(ctx, executor)
}(executor)
}
wg.Wait()
probeDurationMsLastRun.Set(time.Since(start).Milliseconds())
}
// probeAndPersist runs one probe and records it, unless the probe was
// cancelled — a cancelled outcome is not an observation about the host and
// must never be written (see agentruntime.SSHProbeOutcome.Cancelled).
func (p *Poller) probeAndPersist(ctx context.Context, executor *models.Executor) {
if !p.acquireProbeSlot(ctx) {
return
}
defer p.releaseProbeSlot()
p.probeAndPersistWithSlot(ctx, executor)
}
func (p *Poller) probeAndPersistWithSlot(ctx context.Context, executor *models.Executor) {
outcome := p.probe(ctx, executor)
if outcome.Cancelled {
probeDiscardedTotal.Add(1)
return
}
p.observeAndPublish(ctx, executor, outcome)
}
// observeAndPublish records outcome through store.Observe and, when the
// write actually changed the stored state or reason, announces it through
// the publisher — the one propagation path every probe primitive
// (scheduled pass, ProbeNow, ProbeAndWait) shares.
func (p *Poller) observeAndPublish(ctx context.Context, executor *models.Executor, outcome agentruntime.SSHProbeOutcome) observeResult {
result := p.store.Observe(ctx, executor, outcome, time.Now().UTC())
if result.Changed && p.publisher != nil {
p.publisher.PublishChanged(ctx, result.After, p.EffectiveIntervalSeconds())
}
return result
}