Files
kandev/apps/backend/internal/orchestrator/session_ceiling_test.go
T
2026-09-18 10:33:17 +01:00

369 lines
13 KiB
Go

package orchestrator
import (
"context"
"errors"
"fmt"
"sync"
"testing"
"go.uber.org/zap"
)
type fakeAdmittedLister struct {
mu sync.Mutex
ids []string
err error
}
func (f *fakeAdmittedLister) ListAdmittedSessionIDs(context.Context) ([]string, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.err != nil {
return nil, f.err
}
return append([]string(nil), f.ids...), nil
}
func (f *fakeAdmittedLister) set(ids ...string) {
f.mu.Lock()
defer f.mu.Unlock()
f.ids = ids
}
func (f *fakeAdmittedLister) fail(err error) {
f.mu.Lock()
defer f.mu.Unlock()
f.err = err
}
func newTestController(t *testing.T, ceiling int, lister *fakeAdmittedLister) *sessionCeilingController {
t.Helper()
c := newSessionCeilingController(ceiling, lister, zap.NewNop())
if c == nil {
t.Fatal("newSessionCeilingController returned nil")
}
return c
}
func mustPopulation(t *testing.T, c *sessionCeilingController) int {
t.Helper()
got, err := c.population(context.Background())
if err != nil {
t.Fatalf("population: %v", err)
}
return got
}
// The persisted rows alone are the population when nothing is in flight.
func TestPopulationCountsPersistedRows(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2", "s3")
c := newTestController(t, 10, lister)
if got := mustPopulation(t, c); got != 3 {
t.Fatalf("population = %d, want 3", got)
}
}
// A session holding both a row and a reservation is one session, not two. Counting
// it twice would refuse a launch the instance has room for.
func TestPopulationUnionsRowsWithReservationsBySessionID(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2")
c := newTestController(t, 10, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", origin: launchOriginAutomatic, seam: "seam2"})
if !d.admitted || d.reservationKey != "s9" {
t.Fatalf("admit = %+v, want admitted holding a reservation for s9", d)
}
if got := mustPopulation(t, c); got != 3 {
t.Fatalf("population = %d, want 3 (two rows plus one reservation)", got)
}
// s9's launch reaches STARTING: it now holds a row AND its reservation, and it
// is still one session. Counting both would refuse a launch there is room for.
lister.set("s1", "s2", "s9")
if got := mustPopulation(t, c); got != 3 {
t.Fatalf("population = %d, want 3 (s9's row and reservation are one session)", got)
}
}
// A seam-1 reservation has no session id yet and can collide with no row, so it
// adds to the population. Omitting it would make every seam-1 admission invisible
// to the ceiling until its session row lands.
func TestPopulationCountsLaunchScopedReservations(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1")
c := newTestController(t, 10, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", origin: launchOriginAutomatic, seam: "seam1"})
if !d.admitted {
t.Fatalf("admit refused below the ceiling: %+v", d)
}
if d.reservationKey == "" {
t.Fatal("admit returned no reservation key for a seam-1 launch")
}
if got := mustPopulation(t, c); got != 2 {
t.Fatalf("population = %d, want 2 (one row plus one launch-scoped reservation)", got)
}
}
func TestAdmitBelowCeilingReservesCapacity(t *testing.T) {
lister := &fakeAdmittedLister{}
c := newTestController(t, 2, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"})
if !d.admitted || d.manualOverride {
t.Fatalf("admit = %+v, want admitted without override", d)
}
if d.reservationKey != "s1" {
t.Fatalf("reservationKey = %q, want %q", d.reservationKey, "s1")
}
if d.ceiling != 2 || d.population != 0 || !d.populationKnown {
t.Fatalf("decision reported population %d/%d (known=%v), want 0/2 known", d.population, d.ceiling, d.populationKnown)
}
if got := mustPopulation(t, c); got != 1 {
t.Fatalf("population after admit = %d, want 1", got)
}
}
// A disabled ceiling must not make automatic admission depend on the
// population reader. The reader can be unavailable during startup or a
// transient database failure, but an unlimited controller still needs to keep
// the launch reservation for its later lifecycle callback.
func TestDisabledCeilingAdmitsWhenPopulationReadFails(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.fail(errors.New("population unavailable"))
c := newTestController(t, unlimitedSessionCeiling, lister)
decision := c.admit(context.Background(), admissionRequest{
taskID: "unlimited-task", sessionID: "unlimited-session",
origin: launchOriginAutomatic, seam: "disabled-test",
})
if !decision.admitted {
t.Fatalf("disabled ceiling refused an automatic launch: %+v", decision)
}
if decision.manualOverride || decision.reasonCode != "" {
t.Fatalf("disabled ceiling recorded an override/reason: %+v", decision)
}
if decision.reservationKey != "unlimited-session" {
t.Fatalf("reservation key = %q, want session reservation", decision.reservationKey)
}
}
func TestAdmitAtCeilingRefusesAutomaticLaunch(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2")
c := newTestController(t, 2, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", origin: launchOriginAutomatic, seam: "seam2"})
if d.admitted {
t.Fatalf("admit = %+v, want refused at the ceiling", d)
}
if d.reasonCode != ceilingReasonRefused {
t.Fatalf("reasonCode = %q, want %q", d.reasonCode, ceilingReasonRefused)
}
if d.reservationKey != "" {
t.Fatalf("a refusal took reservation %q, want none", d.reservationKey)
}
if got := mustPopulation(t, c); got != 2 {
t.Fatalf("population after a refusal = %d, want 2 unchanged", got)
}
}
// A manual launch is admitted over the ceiling and still counts, so the population
// rises above the ceiling and subsequent automatic launches stay refused.
func TestAdmitAtCeilingAdmitsManualLaunchAndCountsIt(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2")
c := newTestController(t, 2, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", origin: launchOriginManual, seam: "seam2"})
if !d.admitted || !d.manualOverride {
t.Fatalf("admit = %+v, want admitted as a manual override", d)
}
if d.reasonCode != ceilingReasonManualOverride {
t.Fatalf("reasonCode = %q, want %q", d.reasonCode, ceilingReasonManualOverride)
}
if got := mustPopulation(t, c); got != 3 {
t.Fatalf("population after a manual override = %d, want 3 (above the ceiling)", got)
}
next := c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s10", origin: launchOriginAutomatic, seam: "seam2"})
if next.admitted {
t.Fatalf("automatic launch admitted while the population is above the ceiling: %+v", next)
}
}
// A second request for a session that already holds a reservation consumes no
// second unit: one launch is capped at one unit of capacity however many seams it
// passes through.
func TestAdmitIsIdempotentPerSessionID(t *testing.T) {
lister := &fakeAdmittedLister{}
c := newTestController(t, 2, lister)
first := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam5"})
second := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"})
if !first.admitted || !second.admitted {
t.Fatalf("first = %+v, second = %+v, want both admitted", first, second)
}
if got := mustPopulation(t, c); got != 1 {
t.Fatalf("population after two admissions of one session = %d, want 1", got)
}
}
// A session already counted as a row is admitted without taking a reservation, so a
// duplicate retry of a running launch cannot consume capacity.
func TestAdmitShortCircuitsForAnAlreadyCountedSession(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2")
c := newTestController(t, 2, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam3"})
if !d.admitted {
t.Fatalf("admit = %+v, want admitted for an already-counted session even at the ceiling", d)
}
if got := mustPopulation(t, c); got != 2 {
t.Fatalf("population = %d, want 2 unchanged", got)
}
}
func TestReleaseIsIdempotentAndIgnoresUnknownKeys(t *testing.T) {
lister := &fakeAdmittedLister{}
c := newTestController(t, 4, lister)
c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"})
c.release("does-not-exist")
if got := mustPopulation(t, c); got != 1 {
t.Fatalf("population after releasing an unknown key = %d, want 1", got)
}
c.release("s1")
c.release("s1")
if got := mustPopulation(t, c); got != 0 {
t.Fatalf("population after a double release = %d, want 0", got)
}
}
// Zero is unlimited, matching the wip_limit convention.
func TestUnlimitedCeilingAdmitsEverything(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2", "s3", "s4", "s5")
c := newTestController(t, unlimitedSessionCeiling, lister)
for i := 0; i < 20; i++ {
d := c.admit(context.Background(), admissionRequest{taskID: "t", sessionID: fmt.Sprintf("new-%d", i), origin: launchOriginAutomatic, seam: "seam2"})
if !d.admitted || d.manualOverride {
t.Fatalf("admit %d = %+v, want plain admitted under an unlimited ceiling", i, d)
}
}
}
func TestCeilingOfOneAdmitsExactlyOneSession(t *testing.T) {
lister := &fakeAdmittedLister{}
c := newTestController(t, 1, lister)
if d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"}); !d.admitted {
t.Fatalf("first admit = %+v, want admitted", d)
}
if d := c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s2", origin: launchOriginAutomatic, seam: "seam2"}); d.admitted {
t.Fatalf("second admit = %+v, want refused", d)
}
}
// A failed population query fails CLOSED for automatic launches and open for manual
// ones, and neither surface may report a number it does not have.
func TestUnknownPopulationFailsClosedForAutomaticAndOpenForManual(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.fail(errors.New("database is locked"))
c := newTestController(t, 4, lister)
auto := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"})
if auto.admitted {
t.Fatalf("automatic admit = %+v, want refused when the population is unknown", auto)
}
if auto.reasonCode != ceilingReasonUnknownPopulation {
t.Fatalf("automatic reasonCode = %q, want %q", auto.reasonCode, ceilingReasonUnknownPopulation)
}
if auto.populationKnown {
t.Fatal("automatic decision reported the population as known")
}
manual := c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s2", origin: launchOriginManual, seam: "seam2"})
if !manual.admitted || !manual.manualOverride {
t.Fatalf("manual admit = %+v, want admitted as an override", manual)
}
if manual.populationKnown {
t.Fatal("manual decision reported the population as known")
}
if manual.reasonCode != ceilingReasonUnknownPopulation {
t.Fatalf("manual reasonCode = %q, want %q", manual.reasonCode, ceilingReasonUnknownPopulation)
}
}
// The failure concerns the counted rows only: reservations already held stay held.
func TestUnknownPopulationDoesNotReleaseHeldReservations(t *testing.T) {
lister := &fakeAdmittedLister{}
c := newTestController(t, 4, lister)
c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s1", origin: launchOriginAutomatic, seam: "seam2"})
lister.fail(errors.New("database is locked"))
c.admit(context.Background(), admissionRequest{taskID: "t2", sessionID: "s2", origin: launchOriginAutomatic, seam: "seam2"})
lister.fail(nil)
if got := mustPopulation(t, c); got != 1 {
t.Fatalf("population after a failed query = %d, want 1 (the held reservation survived)", got)
}
}
// An unlabelled launch is automatic, so it is refused and deferred rather than
// silently granted an override.
func TestMissingOriginIsTreatedAsAutomatic(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1", "s2")
c := newTestController(t, 2, lister)
d := c.admit(context.Background(), admissionRequest{taskID: "t1", sessionID: "s9", seam: "seam4"})
if d.admitted {
t.Fatalf("admit with no origin = %+v, want refused as automatic", d)
}
}
// Two launches racing for one free slot produce exactly one admission.
func TestConcurrentAdmissionsForOneFreeSlotAdmitExactlyOne(t *testing.T) {
lister := &fakeAdmittedLister{}
lister.set("s1")
c := newTestController(t, 2, lister)
const racers = 32
var wg sync.WaitGroup
results := make([]admissionDecision, racers)
start := make(chan struct{})
for i := 0; i < racers; i++ {
wg.Add(1)
go func(i int) {
defer wg.Done()
<-start
results[i] = c.admit(context.Background(), admissionRequest{
taskID: fmt.Sprintf("t%d", i),
sessionID: fmt.Sprintf("race-%d", i),
origin: launchOriginAutomatic,
seam: "seam2",
})
}(i)
}
close(start)
wg.Wait()
admitted := 0
for _, d := range results {
if d.admitted {
admitted++
}
}
if admitted != 1 {
t.Fatalf("%d of %d racers admitted, want exactly 1", admitted, racers)
}
if got := mustPopulation(t, c); got != 2 {
t.Fatalf("population = %d, want 2 (the ceiling)", got)
}
}