c7cc92382b
Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
446 lines
13 KiB
Go
446 lines
13 KiB
Go
package reachability
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"sort"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"testing/synctest"
|
|
"time"
|
|
|
|
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
|
"github.com/kandev/kandev/internal/common/logger"
|
|
"github.com/kandev/kandev/internal/task/models"
|
|
)
|
|
|
|
// fakeRepository is an in-memory Repository double for poller tests: it
|
|
// isolates poller mechanics (scheduling, concurrency, lifecycle) from the
|
|
// SQL-backed hysteresis contract, which store_test.go already covers
|
|
// end-to-end against a real database.
|
|
type fakeRepository struct {
|
|
mu sync.Mutex
|
|
executors []*models.Executor
|
|
records map[string]*models.ExecutorReachability
|
|
listErr error
|
|
upsertErr error
|
|
|
|
upsertCalls int32
|
|
}
|
|
|
|
// errRepoRefused simulates a repository-level write failure — a test double
|
|
// for the last-write-wins WHERE clause silently refusing a write, without
|
|
// needing a real SQLite race to reproduce it.
|
|
var errRepoRefused = errors.New("write refused")
|
|
|
|
func newFakeRepository(executors ...*models.Executor) *fakeRepository {
|
|
return &fakeRepository{executors: executors, records: map[string]*models.ExecutorReachability{}}
|
|
}
|
|
|
|
func (f *fakeRepository) ListSSHExecutorsForReachability(context.Context) ([]*models.Executor, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.listErr != nil {
|
|
return nil, f.listErr
|
|
}
|
|
out := append([]*models.Executor(nil), f.executors...)
|
|
sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
|
|
return out, nil
|
|
}
|
|
|
|
func (f *fakeRepository) GetExecutorReachability(_ context.Context, executorID string) (*models.ExecutorReachability, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
rec, ok := f.records[executorID]
|
|
if !ok {
|
|
return nil, models.ErrExecutorReachabilityNotFound
|
|
}
|
|
cp := *rec
|
|
return &cp, nil
|
|
}
|
|
|
|
func (f *fakeRepository) UpsertExecutorReachability(_ context.Context, obs models.ExecutorReachabilityObservation) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
atomic.AddInt32(&f.upsertCalls, 1)
|
|
if f.upsertErr != nil {
|
|
return f.upsertErr
|
|
}
|
|
state := obs.InitialState
|
|
failures := obs.InitialFailures
|
|
if existing, ok := f.records[obs.ExecutorID]; ok {
|
|
if obs.Reason == "" {
|
|
failures = 0
|
|
state = models.ExecutorReachabilityStateReachable
|
|
} else {
|
|
failures = existing.ConsecutiveFailures + 1
|
|
state = existing.State
|
|
if isStickyReason(obs.Reason) || failures >= obs.FailureThreshold {
|
|
state = models.ExecutorReachabilityStateUnreachable
|
|
}
|
|
}
|
|
}
|
|
f.records[obs.ExecutorID] = &models.ExecutorReachability{
|
|
ExecutorID: obs.ExecutorID, State: state, Reason: obs.Reason, Message: obs.Message,
|
|
ConsecutiveFailures: failures, Host: obs.Host, CheckedAt: timePtr(obs.CheckedAt), UpdatedAt: time.Now().UTC(),
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) ResetExecutorReachability(_ context.Context, executorID, host string, _ time.Time) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.records[executorID] = &models.ExecutorReachability{ExecutorID: executorID, State: models.ExecutorReachabilityStateUnknown, Host: host, UpdatedAt: time.Now().UTC()}
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeRepository) upsertCount() int32 { return atomic.LoadInt32(&f.upsertCalls) }
|
|
|
|
func timePtr(t time.Time) *time.Time { return &t }
|
|
|
|
func sshExecutor(id string) *models.Executor {
|
|
return &models.Executor{ID: id, Name: id, Type: models.ExecutorTypeSSH, Status: models.ExecutorStatusActive, Config: map[string]string{"ssh_host": id}}
|
|
}
|
|
|
|
// concurrencyProbe blocks on a gate until released, tracking the maximum
|
|
// number of simultaneous in-flight calls it observed.
|
|
type concurrencyProbe struct {
|
|
mu sync.Mutex
|
|
inFlight int
|
|
maxSeen int
|
|
release chan struct{}
|
|
}
|
|
|
|
func newConcurrencyProbe() *concurrencyProbe {
|
|
return &concurrencyProbe{release: make(chan struct{})}
|
|
}
|
|
|
|
func (c *concurrencyProbe) probe(ctx context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
c.mu.Lock()
|
|
c.inFlight++
|
|
if c.inFlight > c.maxSeen {
|
|
c.maxSeen = c.inFlight
|
|
}
|
|
c.mu.Unlock()
|
|
|
|
select {
|
|
case <-c.release:
|
|
case <-ctx.Done():
|
|
c.mu.Lock()
|
|
c.inFlight--
|
|
c.mu.Unlock()
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Cancelled: true}
|
|
}
|
|
|
|
c.mu.Lock()
|
|
c.inFlight--
|
|
c.mu.Unlock()
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
|
|
func TestPollerStart_RunsImmediatePass(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"), sshExecutor("b"))
|
|
p := New(repo, 60, logger.Default())
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
defer p.Stop()
|
|
|
|
synctest.Wait()
|
|
|
|
if got := repo.upsertCount(); got != 2 {
|
|
t.Fatalf("upsert calls = %d, want 2 (one immediate pass over two executors)", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerStart_IsIdempotent(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
p := New(repo, 60, logger.Default())
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
p.Start(ctx) // second call must be a no-op
|
|
|
|
synctest.Wait()
|
|
p.Stop()
|
|
|
|
if got := repo.upsertCount(); got != 1 {
|
|
t.Fatalf("upsert calls = %d, want exactly 1 (a second Start must not spawn a parallel loop)", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerStop_BeforeStart_IsNoOp(t *testing.T) {
|
|
p := New(newFakeRepository(), 60, logger.Default())
|
|
p.Stop() // must not block or panic
|
|
}
|
|
|
|
func TestPollerZeroInterval_DisablesScheduledPass(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
p := New(repo, 0, logger.Default())
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
defer p.Stop()
|
|
|
|
synctest.Wait()
|
|
// Advance well past what would have been several ticks at the
|
|
// default interval, to prove no pass ever runs.
|
|
time.Sleep(10 * time.Minute)
|
|
synctest.Wait()
|
|
|
|
if got := repo.upsertCount(); got != 0 {
|
|
t.Fatalf("upsert calls = %d, want 0 (interval=0 must run no scheduled pass)", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerZeroInterval_ProbeNowStillWorks(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
p := New(repo, 0, logger.Default())
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
defer p.Stop()
|
|
synctest.Wait()
|
|
|
|
if ok := p.ProbeNow(sshExecutor("a")); !ok {
|
|
t.Fatal("ProbeNow returned false while the poller is started")
|
|
}
|
|
synctest.Wait()
|
|
|
|
if got := repo.upsertCount(); got != 1 {
|
|
t.Fatalf("upsert calls = %d, want 1 from the off-cycle probe", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerPass_NeverExceedsPassConcurrency(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
var executors []*models.Executor
|
|
for i := 0; i < passConcurrency+3; i++ {
|
|
executors = append(executors, sshExecutor(string(rune('a'+i))))
|
|
}
|
|
repo := newFakeRepository(executors...)
|
|
p := New(repo, 60, logger.Default())
|
|
cp := newConcurrencyProbe()
|
|
p.probe = cp.probe
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
|
|
// Let every worker reach the gate.
|
|
synctest.Wait()
|
|
close(cp.release)
|
|
p.Stop()
|
|
|
|
cp.mu.Lock()
|
|
defer cp.mu.Unlock()
|
|
if cp.maxSeen > passConcurrency {
|
|
t.Fatalf("max simultaneous probes = %d, want at most passConcurrency (%d)", cp.maxSeen, passConcurrency)
|
|
}
|
|
if cp.maxSeen != passConcurrency {
|
|
t.Fatalf("max simultaneous probes = %d, want exactly passConcurrency (%d) since more executors than slots were queued", cp.maxSeen, passConcurrency)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerProbeEntrypointsShareConcurrencyLimit(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
var executors []*models.Executor
|
|
for i := 0; i < passConcurrency; i++ {
|
|
executors = append(executors, sshExecutor(string(rune('a'+i))))
|
|
}
|
|
repo := newFakeRepository(executors...)
|
|
p := New(repo, 60, logger.Default())
|
|
cp := newConcurrencyProbe()
|
|
p.probe = cp.probe
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
synctest.Wait()
|
|
|
|
if ok := p.ProbeNow(sshExecutor("manual")); !ok {
|
|
t.Fatal("ProbeNow returned false while the poller was running")
|
|
}
|
|
synctest.Wait()
|
|
|
|
cp.mu.Lock()
|
|
maxSeen := cp.maxSeen
|
|
cp.mu.Unlock()
|
|
close(cp.release)
|
|
p.Stop()
|
|
if maxSeen > passConcurrency {
|
|
t.Fatalf("max simultaneous probes = %d, want at most %d across scheduled and manual probes", maxSeen, passConcurrency)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerConcurrentStopWaitsForTheFirstStop(t *testing.T) {
|
|
repo := newFakeRepository()
|
|
p := New(repo, 0, logger.Default())
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
var inFlight atomic.Int32
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
inFlight.Add(1)
|
|
defer inFlight.Add(-1)
|
|
<-release
|
|
return agentruntime.SSHProbeOutcome{Success: true, Host: executor.ID}
|
|
}
|
|
t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) })
|
|
p.Start(context.Background())
|
|
if ok := p.ProbeNow(sshExecutor("stop-race")); !ok {
|
|
t.Fatal("ProbeNow returned false while the poller was running")
|
|
}
|
|
deadline := time.Now().Add(time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if inFlight.Load() == 1 {
|
|
break
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
|
|
firstDone := make(chan struct{})
|
|
go func() { p.Stop(); close(firstDone) }()
|
|
for time.Now().Before(deadline) {
|
|
p.mu.Lock()
|
|
stopping := p.stopping
|
|
p.mu.Unlock()
|
|
if stopping {
|
|
break
|
|
}
|
|
time.Sleep(time.Millisecond)
|
|
}
|
|
secondDone := make(chan struct{})
|
|
go func() { p.Stop(); close(secondDone) }()
|
|
select {
|
|
case <-secondDone:
|
|
t.Fatal("concurrent Stop returned before the first Stop drained the probe")
|
|
case <-time.After(20 * time.Millisecond):
|
|
}
|
|
releaseOnce.Do(func() { close(release) })
|
|
select {
|
|
case <-firstDone:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("first Stop did not finish")
|
|
}
|
|
select {
|
|
case <-secondDone:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("concurrent Stop did not finish after the first Stop")
|
|
}
|
|
}
|
|
|
|
func TestPollerPass_OverlappingTickIsSkippedNotQueued(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
p := New(repo, 60, logger.Default())
|
|
cp := newConcurrencyProbe()
|
|
p.probe = cp.probe
|
|
|
|
before := passSkippedTotal.Value()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
defer func() {
|
|
close(cp.release)
|
|
p.Stop()
|
|
}()
|
|
|
|
// The immediate pass is now blocked on the probe gate. Advance past
|
|
// a full tick while it is still running.
|
|
synctest.Wait()
|
|
time.Sleep(61 * time.Second)
|
|
synctest.Wait()
|
|
|
|
if got := passSkippedTotal.Value() - before; got != 1 {
|
|
t.Fatalf("pass_skipped_total delta = %d, want 1 (the overlapping tick must be dropped, not queued)", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerStop_DiscardsCancelledProbeResult(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
p := New(repo, 60, logger.Default())
|
|
cp := newConcurrencyProbe()
|
|
p.probe = cp.probe
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
|
|
// The immediate pass is now blocked on the probe gate (never
|
|
// released), so Stop must cancel it in flight.
|
|
synctest.Wait()
|
|
p.Stop()
|
|
|
|
if got := repo.upsertCount(); got != 0 {
|
|
t.Fatalf("upsert calls = %d, want 0 — a probe cancelled by Stop must never be written", got)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestPollerProbeNow_RefusedAfterStop(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
p := New(repo, 60, logger.Default())
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
p.Stop()
|
|
|
|
if ok := p.ProbeNow(sshExecutor("a")); ok {
|
|
t.Fatal("ProbeNow returned true after Stop; a stopped poller must refuse new work")
|
|
}
|
|
}
|
|
|
|
func TestPollerListFailure_AbandonsPassWithoutWriting(t *testing.T) {
|
|
synctest.Test(t, func(t *testing.T) {
|
|
repo := newFakeRepository(sshExecutor("a"))
|
|
repo.listErr = context.DeadlineExceeded
|
|
p := New(repo, 60, logger.Default())
|
|
p.probe = func(_ context.Context, executor *models.Executor) agentruntime.SSHProbeOutcome {
|
|
return agentruntime.SSHProbeOutcome{Host: executor.ID, Success: true}
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Start(ctx)
|
|
defer p.Stop()
|
|
synctest.Wait()
|
|
|
|
if got := repo.upsertCount(); got != 0 {
|
|
t.Fatalf("upsert calls = %d, want 0 when listing executors fails", got)
|
|
}
|
|
})
|
|
}
|