c7cc92382b
Co-authored-by: nova28 <17953305+nova28@users.noreply.github.com> Co-authored-by: Carlos Florencio <carlosmflorencio@gmail.com>
323 lines
12 KiB
Go
323 lines
12 KiB
Go
package reachability
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
agentruntime "github.com/kandev/kandev/internal/agent/runtime"
|
|
"github.com/kandev/kandev/internal/common/logger"
|
|
"github.com/kandev/kandev/internal/db"
|
|
"github.com/kandev/kandev/internal/task/models"
|
|
"github.com/kandev/kandev/internal/task/repository/sqlite"
|
|
)
|
|
|
|
func newStoreTestRepo(t *testing.T) *sqlite.Repository {
|
|
t.Helper()
|
|
path := filepath.Join(t.TempDir(), "reachability-store.db")
|
|
dbConn, err := db.OpenSQLite(path)
|
|
if err != nil {
|
|
t.Fatalf("open sqlite: %v", err)
|
|
}
|
|
sqlxDB := sqlx.NewDb(dbConn, "sqlite3")
|
|
repo, err := sqlite.NewWithDB(sqlxDB, sqlxDB, nil)
|
|
if err != nil {
|
|
_ = sqlxDB.Close()
|
|
t.Fatalf("new repository: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = sqlxDB.Close() })
|
|
return repo
|
|
}
|
|
|
|
func newStoreTestExecutor(t *testing.T, repo *sqlite.Repository) *models.Executor {
|
|
t.Helper()
|
|
executor := &models.Executor{
|
|
ID: "exec-" + t.Name(),
|
|
Name: "ssh-host",
|
|
Type: models.ExecutorTypeSSH,
|
|
Status: models.ExecutorStatusActive,
|
|
Config: map[string]string{"ssh_host": "10.0.0.1"},
|
|
}
|
|
if err := repo.CreateExecutor(context.Background(), executor); err != nil {
|
|
t.Fatalf("create executor: %v", err)
|
|
}
|
|
return executor
|
|
}
|
|
|
|
type microsecondReachabilityRepository struct {
|
|
*sqlite.Repository
|
|
}
|
|
|
|
func (r *microsecondReachabilityRepository) UpsertExecutorReachability(ctx context.Context, obs models.ExecutorReachabilityObservation) error {
|
|
obs.CheckedAt = obs.CheckedAt.UTC().Truncate(time.Microsecond)
|
|
return r.Repository.UpsertExecutorReachability(ctx, obs)
|
|
}
|
|
|
|
type failingGetReachabilityRepository struct {
|
|
*sqlite.Repository
|
|
err error
|
|
failAfter int
|
|
calls int
|
|
}
|
|
|
|
func (r *failingGetReachabilityRepository) GetExecutorReachability(ctx context.Context, executorID string) (*models.ExecutorReachability, error) {
|
|
r.calls++
|
|
if r.calls >= r.failAfter {
|
|
return nil, r.err
|
|
}
|
|
return r.Repository.GetExecutorReachability(ctx, executorID)
|
|
}
|
|
|
|
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9, AC-EXECUTORS-SSH-REACHABILITY-001.10
|
|
//
|
|
// Drives a success/fail/fail/success sequence and asserts the exact probe
|
|
// index on which each direction flips. A naive flip-on-first-result
|
|
// implementation would report unreachable after the very first failure
|
|
// (index 2) instead of the second (index 3), and would report a below-
|
|
// threshold failure as a state change even though the state stands still.
|
|
func TestStoreObserveHysteresisSequence(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
s := &store{repo: repo, log: logger.Default()}
|
|
ctx := context.Background()
|
|
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
|
|
type step struct {
|
|
label string
|
|
outcome agentruntime.SSHProbeOutcome
|
|
wantState models.ExecutorReachabilityState
|
|
wantChanged bool
|
|
wantFailures int
|
|
}
|
|
steps := []step{
|
|
{
|
|
label: "1: first success establishes reachable",
|
|
outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"},
|
|
wantState: models.ExecutorReachabilityStateReachable,
|
|
wantChanged: true,
|
|
wantFailures: 0,
|
|
},
|
|
{
|
|
label: "2: below-threshold failure leaves state reachable",
|
|
outcome: agentruntime.SSHProbeOutcome{
|
|
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
|
|
},
|
|
wantState: models.ExecutorReachabilityStateReachable,
|
|
wantChanged: false,
|
|
wantFailures: 1,
|
|
},
|
|
{
|
|
label: "3: failure reaching failureThreshold flips to unreachable",
|
|
outcome: agentruntime.SSHProbeOutcome{
|
|
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
|
|
},
|
|
wantState: models.ExecutorReachabilityStateUnreachable,
|
|
wantChanged: true,
|
|
wantFailures: 2,
|
|
},
|
|
{
|
|
label: "4: a single success flips back to reachable with a zero counter",
|
|
outcome: agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"},
|
|
wantState: models.ExecutorReachabilityStateReachable,
|
|
wantChanged: true,
|
|
wantFailures: 0,
|
|
},
|
|
}
|
|
|
|
for i, st := range steps {
|
|
checkedAt := base.Add(time.Duration(i) * time.Second)
|
|
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
|
result := s.Observe(ctx, executor, st.outcome, checkedAt)
|
|
if !result.Applied {
|
|
t.Fatalf("%s: write was refused, want applied", st.label)
|
|
}
|
|
if result.Current != st.wantState {
|
|
t.Fatalf("%s: state = %q, want %q", st.label, result.Current, st.wantState)
|
|
}
|
|
if result.StateChanged != st.wantChanged {
|
|
t.Fatalf("%s: changed = %v, want %v", st.label, result.StateChanged, st.wantChanged)
|
|
}
|
|
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
|
if err != nil {
|
|
t.Fatalf("%s: get: %v", st.label, err)
|
|
}
|
|
if got.ConsecutiveFailures != st.wantFailures {
|
|
t.Fatalf("%s: consecutive_failures = %d, want %d", st.label, got.ConsecutiveFailures, st.wantFailures)
|
|
}
|
|
}
|
|
}
|
|
|
|
// @covers AC-EXECUTORS-SSH-REACHABILITY-001.9
|
|
//
|
|
// A single config or host_key failure writes unreachable on that first
|
|
// probe, with the counter still below failureThreshold — sticky reasons
|
|
// bypass the threshold entirely.
|
|
func TestStoreObserveStickyFailureIsUnreachableOnFirstProbe(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
reason agentruntime.SSHReachabilityReason
|
|
}{
|
|
{"config", agentruntime.SSHReachabilityReasonConfig},
|
|
{"host_key", agentruntime.SSHReachabilityReasonHostKey},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
s := &store{repo: repo, log: logger.Default()}
|
|
ctx := context.Background()
|
|
|
|
result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
|
|
Host: "10.0.0.1", Reason: tt.reason, Message: "boom",
|
|
}, time.Now().UTC())
|
|
|
|
if !result.Applied || result.Current != models.ExecutorReachabilityStateUnreachable {
|
|
t.Fatalf("result = %+v, want applied unreachable on the first probe", result)
|
|
}
|
|
got, err := repo.GetExecutorReachability(ctx, executor.ID)
|
|
if err != nil {
|
|
t.Fatalf("get: %v", err)
|
|
}
|
|
if got.ConsecutiveFailures >= failureThreshold {
|
|
t.Fatalf("consecutive_failures = %d, want below failureThreshold (%d) — sticky must not need it", got.ConsecutiveFailures, failureThreshold)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
|
//
|
|
// The change-publish trigger is broader than StateChanged: a steady
|
|
// unreachable executor can still swap failure reasons (network, then
|
|
// timeout) between probes without ever leaving the unreachable state. A
|
|
// publisher that only watched StateChanged would miss this and never tell a
|
|
// client the diagnostic reason on screen just went stale.
|
|
func TestStoreObserveReportsChangedOnReasonChangeWithoutStateChange(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
s := &store{repo: repo, log: logger.Default()}
|
|
ctx := context.Background()
|
|
base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
|
|
// Two failures reach failureThreshold and flip to unreachable.
|
|
for i := 0; i < 2; i++ {
|
|
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
|
s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
|
|
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonNetwork, Message: "connection refused",
|
|
}, base.Add(time.Duration(i)*time.Second))
|
|
}
|
|
|
|
// A third failure with a different transient reason keeps the state
|
|
// unreachable (already above threshold) but changes the stored reason.
|
|
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
|
result := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{
|
|
Host: "10.0.0.1", Reason: agentruntime.SSHReachabilityReasonTimeout, Message: "handshake timed out",
|
|
}, base.Add(2*time.Second))
|
|
|
|
if result.StateChanged {
|
|
t.Fatalf("StateChanged = true, want false — state was already unreachable")
|
|
}
|
|
if !result.Changed {
|
|
t.Fatalf("Changed = false, want true — reason moved from network to timeout")
|
|
}
|
|
if result.After == nil || result.After.Reason != models.ExecutorReachabilityReasonTimeout {
|
|
t.Fatalf("After = %+v, want a record carrying reason %q", result.After, models.ExecutorReachabilityReasonTimeout)
|
|
}
|
|
}
|
|
|
|
// @covers AC-EXECUTORS-SSH-REACHABILITY-002.3
|
|
//
|
|
// A probe outcome that reproduces the same state and reason as the stored
|
|
// record must not be reported as a change — this is what keeps a steady host
|
|
// silent on the event bus.
|
|
func TestStoreObserveReportsUnchangedWhenStateAndReasonBothRepeat(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
s := &store{repo: repo, log: logger.Default()}
|
|
ctx := context.Background()
|
|
|
|
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
|
first := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC())
|
|
if !first.Changed {
|
|
t.Fatalf("first observation: Changed = false, want true (unknown -> reachable)")
|
|
}
|
|
|
|
executor.UpdatedAt = mustGetUpdatedAt(t, repo, executor.ID)
|
|
second := s.Observe(ctx, executor, agentruntime.SSHProbeOutcome{Success: true, Host: "10.0.0.1"}, time.Now().UTC().Add(time.Second))
|
|
if second.Changed {
|
|
t.Fatalf("second observation: Changed = true, want false — same state (reachable) and reason (success)")
|
|
}
|
|
if second.After == nil {
|
|
t.Fatalf("After = nil, want the current stored record")
|
|
}
|
|
}
|
|
|
|
func TestStoreObserveNormalizesDatabaseTimestampPrecision(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
s := &store{repo: µsecondReachabilityRepository{Repository: repo}, log: logger.Default()}
|
|
checkedAt := time.Date(2026, 1, 1, 0, 0, 0, 123456789, time.UTC)
|
|
|
|
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
|
|
Success: true, Host: "10.0.0.1",
|
|
}, checkedAt)
|
|
if !result.Applied {
|
|
t.Fatalf("result = %+v, want applied despite microsecond storage precision", result)
|
|
}
|
|
if result.After == nil || result.After.CheckedAt == nil {
|
|
t.Fatalf("After = %+v, want a persisted record", result.After)
|
|
}
|
|
if want := checkedAt.Truncate(time.Microsecond); !result.After.CheckedAt.Equal(want) {
|
|
t.Fatalf("CheckedAt = %v, want %v", result.After.CheckedAt, want)
|
|
}
|
|
}
|
|
|
|
func TestStoreObserveStopsOnUnexpectedInitialReadError(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
readErr := errors.New("database unavailable")
|
|
wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 1}
|
|
s := &store{repo: wrapped, log: logger.Default()}
|
|
|
|
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
|
|
Success: true, Host: "10.0.0.1",
|
|
}, time.Now().UTC())
|
|
if result.Applied || result.Observed != nil {
|
|
t.Fatalf("result = %+v, want no observation when the initial read fails", result)
|
|
}
|
|
if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); !errors.Is(err, models.ErrExecutorReachabilityNotFound) {
|
|
t.Fatalf("stored record error = %v, want no write", err)
|
|
}
|
|
}
|
|
|
|
func TestStoreObserveReturnsUnsavedObservationOnReadBackError(t *testing.T) {
|
|
repo := newStoreTestRepo(t)
|
|
executor := newStoreTestExecutor(t, repo)
|
|
readErr := errors.New("read-back unavailable")
|
|
wrapped := &failingGetReachabilityRepository{Repository: repo, err: readErr, failAfter: 2}
|
|
s := &store{repo: wrapped, log: logger.Default()}
|
|
|
|
result := s.Observe(context.Background(), executor, agentruntime.SSHProbeOutcome{
|
|
Success: true, Host: "10.0.0.1",
|
|
}, time.Now().UTC())
|
|
if result.Applied || result.Observed == nil || result.Observed.State != models.ExecutorReachabilityStateReachable {
|
|
t.Fatalf("result = %+v, want an unsaved reachable observation", result)
|
|
}
|
|
if _, err := repo.GetExecutorReachability(context.Background(), executor.ID); err != nil {
|
|
t.Fatalf("stored record lookup: %v, want the write to have landed", err)
|
|
}
|
|
}
|
|
|
|
func mustGetUpdatedAt(t *testing.T, repo *sqlite.Repository, id string) time.Time {
|
|
t.Helper()
|
|
executor, err := repo.GetExecutor(context.Background(), id)
|
|
if err != nil {
|
|
t.Fatalf("get executor: %v", err)
|
|
}
|
|
return executor.UpdatedAt
|
|
}
|