Files
kandev/apps/backend/internal/orchestrator/session_ceiling.go
T
2026-09-18 10:33:17 +01:00

579 lines
21 KiB
Go

package orchestrator
import (
"context"
"sync"
"time"
"github.com/google/uuid"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/common/constants"
)
// reservationExpiryAllowance is the slice of the backstop that follows the session's
// STARTING write: the start deadline inside the launch goroutine. The preparation
// phase before it is already bounded by constants.AgentLaunchTimeout, which is read
// rather than copied so this tracks an operator-raised preparation budget.
const reservationExpiryAllowance = 5 * time.Minute
// launchOrigin is the explicit automatic/manual classification threaded from the
// caller into the admission controller. It is never inferred from the transport or
// the handler, both of which carry human clicks and server-initiated automation.
type launchOrigin string
const (
launchOriginAutomatic launchOrigin = "automatic"
launchOriginManual launchOrigin = "manual"
)
// ceilingFieldReasonCode and ceilingFieldCeiling name the log field and card
// metadata key shared across every ceiling log line, message row and audit
// record, so the reason code and the ceiling value are always found under
// the same name regardless of which call site wrote them.
const (
ceilingFieldReasonCode = "reason_code"
ceilingFieldCeiling = "ceiling"
)
// Reason codes carried verbatim by the admission log line and the card surface.
const (
ceilingReasonRefused = "ceiling"
ceilingReasonManualOverride = "ceiling_manual_override"
ceilingReasonUnknownPopulation = "ceiling_unknown_population"
ceilingReasonSuperseded = "ceiling_superseded"
ceilingReasonDeferWriteFailed = "ceiling_defer_write_failed"
// ceilingReasonDroppedTaskIneligible covers AC-17b(a)-(d): the task is
// archived, cancelled, names a session that no longer exists, or moved
// to a step that no longer auto-starts.
ceilingReasonDroppedTaskIneligible = "ceiling_dropped_task_ineligible"
// ceilingReasonDroppedLaunchGateDeclined is AC-17b(e): the launch's own
// precondition (today, the terminal-PR guard) declined it at replay time.
ceilingReasonDroppedLaunchGateDeclined = "ceiling_dropped_launch_gate_declined"
// ceilingReasonDroppedUnreplayableRecord is AC-17b(f): the record's own
// ceiling_launch_kind is absent or outside the closed set.
ceilingReasonDroppedUnreplayableRecord = "ceiling_dropped_unreplayable_record"
// ceilingReasonSurfaceWriteFailed is AC-49g: the deferral itself persisted
// successfully, but the card note describing it could not be written.
ceilingReasonSurfaceWriteFailed = "ceiling_surface_write_failed"
)
// admittedSessionLister supplies the persisted half of the population. It returns
// ids rather than a count because the population unions rows with reservations by
// session id, so a session holding both is counted once.
type admittedSessionLister interface {
ListAdmittedSessionIDs(ctx context.Context) ([]string, error)
}
// ceilingReservation is an in-flight admission held until the session is observed
// running or the launch fails. sessionID is empty while the reservation is still
// keyed by a launch-scoped identifier, which is the window between admission at
// seam 1 and the session's creation.
type ceilingReservation struct {
sessionID string
takenAt time.Time
}
// admissionRequest is one launch asking for capacity.
type admissionRequest struct {
taskID string
sessionID string
origin launchOrigin
seam string
}
// admissionDecision is the controller's answer.
type admissionDecision struct {
admitted bool
manualOverride bool
reservationKey string
population int
populationKnown bool
ceiling int
reasonCode string
handedOff bool
}
// SessionCeilingObservation is a point-in-time view of the admission
// controller. It is deliberately separate from a launch deferral so callers
// can refresh displayed capacity without changing queue ownership or time.
type SessionCeilingObservation struct {
InUse int
Limit int
ObservedAt time.Time
Known bool
}
// sessionCeilingController is the single admission controller. Every mutation of
// the reservation set happens under its one mutex, together with the population
// read it is compared against.
type sessionCeilingController struct {
mu sync.Mutex
ceiling int
reservations map[string]*ceilingReservation
lister admittedSessionLister
logger *zap.Logger
now func() time.Time
newKey func() string
}
func newSessionCeilingController(ceiling int, lister admittedSessionLister, logger *zap.Logger) *sessionCeilingController {
if logger == nil {
logger = zap.NewNop()
}
return &sessionCeilingController{
ceiling: ceiling,
reservations: make(map[string]*ceilingReservation),
lister: lister,
logger: logger,
now: time.Now,
newKey: func() string { return "launch-" + uuid.NewString() },
}
}
// setCeiling changes the effective capacity without replacing the controller.
// Reservations therefore remain owned by the launch that created them across
// every Settings update.
func (c *sessionCeilingController) setCeiling(ceiling int) bool {
if c == nil {
return false
}
c.mu.Lock()
previous := c.ceiling
c.ceiling = ceiling
c.mu.Unlock()
return previous != ceiling
}
// population returns the admitted session population.
func (c *sessionCeilingController) population(ctx context.Context) (int, error) {
c.mu.Lock()
defer c.mu.Unlock()
counted, err := c.countedRowsLocked(ctx)
if err != nil {
return 0, err
}
return c.populationLocked(counted), nil
}
func (c *sessionCeilingController) observation(ctx context.Context) (SessionCeilingObservation, error) {
if c == nil {
return SessionCeilingObservation{}, nil
}
c.mu.Lock()
defer c.mu.Unlock()
counted, err := c.countedRowsLocked(ctx)
observedAt := time.Now().UTC()
if c.now != nil {
observedAt = c.now().UTC()
}
observation := SessionCeilingObservation{
Limit: c.ceiling,
ObservedAt: observedAt,
Known: err == nil,
}
if err != nil {
return observation, err
}
observation.InUse = c.populationLocked(counted)
return observation, nil
}
// CurrentSessionCeilingObservation exposes the controller's current bounded
// reading to composition-layer status projections. A population read failure
// returns the ceiling and a Known=false observation so queue ownership can
// still be displayed without claiming a count.
func (s *Service) CurrentSessionCeilingObservation(ctx context.Context) (SessionCeilingObservation, error) {
if s == nil || s.sessionCeiling == nil {
return SessionCeilingObservation{}, nil
}
return s.sessionCeiling.observation(ctx)
}
// SetSessionCapacity applies the effective install-wide capacity to the
// existing admission controller. A zero value disables refusal while keeping
// the controller and its reservations alive for later re-enablement.
func (s *Service) SetSessionCapacity(capacity int) {
if s == nil || s.sessionCeiling == nil {
return
}
changed := s.sessionCeiling.setCeiling(capacity)
if !changed {
return
}
if s.logger != nil {
s.logger.Info("session ceiling capacity applied",
zap.Int("ceiling", capacity),
zap.Bool("enabled", capacity != unlimitedSessionCeiling))
}
// A sweep both retries newly eligible work after an increase or disable and
// refreshes the task projection after any changed observation, including a
// decrease.
s.signalCeilingSweep()
}
// SessionCapacity returns the current effective limit for the live admission
// controller. Zero means automatic launches are unlimited.
func (s *Service) SessionCapacity() int {
if s == nil || s.sessionCeiling == nil {
return unlimitedSessionCeiling
}
s.sessionCeiling.mu.Lock()
defer s.sessionCeiling.mu.Unlock()
return s.sessionCeiling.ceiling
}
// countedRowsLocked reads the persisted half of the population. It is derived at
// decision time rather than kept as a running tally, so a restart, a panic or a
// missed release cannot leak capacity permanently.
func (c *sessionCeilingController) countedRowsLocked(ctx context.Context) (map[string]struct{}, error) {
if c.lister == nil {
return map[string]struct{}{}, nil
}
ids, err := c.lister.ListAdmittedSessionIDs(ctx)
if err != nil {
return nil, err
}
counted := make(map[string]struct{}, len(ids))
for _, id := range ids {
counted[id] = struct{}{}
}
return counted, nil
}
// populationLocked unions the persisted rows with the session-bound reservations by
// session id, then adds the launch-scoped reservations, each of which can collide
// with no row because no row for it exists yet. It is deliberately not the sum of
// two independent numbers.
func (c *sessionCeilingController) populationLocked(counted map[string]struct{}) int {
population := len(counted)
for _, reservation := range c.reservations {
if reservation.sessionID == "" {
population++
continue
}
if _, alreadyCounted := counted[reservation.sessionID]; !alreadyCounted {
population++
}
}
return population
}
// reserveLocked records an in-flight reservation, keyed by session id where one
// exists and by a launch-scoped identifier otherwise.
func (c *sessionCeilingController) reserveLocked(sessionID string) string {
key := sessionID
if key == "" {
key = c.newKey()
}
c.reservations[key] = &ceilingReservation{sessionID: sessionID, takenAt: c.now()}
return key
}
// admit decides one launch and, where it admits, records the reservation before
// returning. The population read and the reservation write share one critical
// section, so two launches arriving together cannot both see the same free slot.
func (c *sessionCeilingController) admit(ctx context.Context, req admissionRequest) admissionDecision {
if c == nil {
// A Service constructed without going through NewService (most
// commonly a test fixture built for narrow coverage of unrelated
// logic) has no ceiling resolved at all. Treat that the same as an
// explicitly unlimited ceiling rather than panicking on every launch.
return admissionDecision{admitted: true}
}
origin := req.origin
if origin != launchOriginManual && origin != launchOriginAutomatic {
origin = launchOriginAutomatic
c.logger.Warn("launch reached the session ceiling with no origin set; classified as automatic",
zap.String("seam", req.seam), zap.String("task_id", req.taskID), zap.String("session_id", req.sessionID))
}
c.mu.Lock()
defer c.mu.Unlock()
if c.ceiling == unlimitedSessionCeiling {
return c.admitUnlimitedLocked(req, origin)
}
counted, err := c.countedRowsLocked(ctx)
if err != nil {
return c.decideUnknownPopulationLocked(req, origin, err)
}
return c.decideLocked(req, origin, counted)
}
// admitUnlimitedLocked keeps launch ownership and callback accounting intact
// while avoiding a population read that cannot affect an unlimited decision.
func (c *sessionCeilingController) admitUnlimitedLocked(req admissionRequest, origin launchOrigin) admissionDecision {
decision := admissionDecision{admitted: true, ceiling: unlimitedSessionCeiling}
if req.sessionID != "" {
if _, held := c.reservations[req.sessionID]; held {
decision.reservationKey = req.sessionID
return decision
}
}
decision.reservationKey = c.reserveLocked(req.sessionID)
c.logDecision(req, origin, decision)
return decision
}
// decideLocked is the ordinary admission decision, taken against a population the
// caller has already read inside the critical section.
func (c *sessionCeilingController) decideLocked(req admissionRequest, origin launchOrigin, counted map[string]struct{}) admissionDecision {
population := c.populationLocked(counted)
if decision, ok := c.alreadyAdmittedLocked(req, counted, population); ok {
return decision
}
decision := admissionDecision{
population: population,
populationKnown: true,
ceiling: c.ceiling,
}
switch {
case c.ceiling == unlimitedSessionCeiling || population < c.ceiling:
decision.admitted = true
decision.reservationKey = c.reserveLocked(req.sessionID)
case origin == launchOriginManual:
decision.admitted = true
decision.manualOverride = true
decision.reasonCode = ceilingReasonManualOverride
decision.reservationKey = c.reserveLocked(req.sessionID)
default:
decision.reasonCode = ceilingReasonRefused
}
c.logDecision(req, origin, decision)
return decision
}
// alreadyAdmittedLocked answers a request for a session that already holds a
// reservation or is already counted, so one launch consumes at most one unit of
// capacity however many seams it passes through.
func (c *sessionCeilingController) alreadyAdmittedLocked(req admissionRequest, counted map[string]struct{}, population int) (admissionDecision, bool) {
if req.sessionID == "" {
return admissionDecision{}, false
}
decision := admissionDecision{
admitted: true,
population: population,
populationKnown: true,
ceiling: c.ceiling,
}
if _, held := c.reservations[req.sessionID]; held {
decision.reservationKey = req.sessionID
return decision, true
}
if _, isCounted := counted[req.sessionID]; isCounted {
return decision, true
}
return admissionDecision{}, false
}
// decideUnknownPopulationLocked fails closed for automatic launches and open for
// manual ones. Reservations already held are untouched: the failure concerns the
// counted rows only.
func (c *sessionCeilingController) decideUnknownPopulationLocked(req admissionRequest, origin launchOrigin, err error) admissionDecision {
c.logger.Error("session ceiling could not read the admitted session population",
zap.String("seam", req.seam), zap.String("task_id", req.taskID),
zap.String("session_id", req.sessionID), zap.String("origin", string(origin)), zap.Error(err))
decision := admissionDecision{
ceiling: c.ceiling,
reasonCode: ceilingReasonUnknownPopulation,
}
if origin == launchOriginManual {
decision.admitted = true
decision.manualOverride = true
decision.reservationKey = c.reserveLocked(req.sessionID)
}
c.logDecision(req, origin, decision)
return decision
}
// logDecision emits the admission log line, which carries every reason code and is
// the only universal observable of a decision.
func (c *sessionCeilingController) logDecision(req admissionRequest, origin launchOrigin, decision admissionDecision) {
fields := []zap.Field{
zap.String("seam", req.seam),
zap.String("task_id", req.taskID),
zap.String("session_id", req.sessionID),
zap.String("origin", string(origin)),
zap.Int(ceilingFieldCeiling, decision.ceiling),
zap.Bool("admitted", decision.admitted),
}
if decision.populationKnown {
fields = append(fields, zap.Int("population", decision.population))
}
if decision.reasonCode != "" {
fields = append(fields, zap.String(ceilingFieldReasonCode, decision.reasonCode))
}
c.logger.Info("session ceiling admission decision", fields...)
}
// release drops a reservation by its key. Releasing an unknown or already-released
// key succeeds and decrements nothing.
func (c *sessionCeilingController) release(key string) {
if key == "" {
return
}
c.mu.Lock()
defer c.mu.Unlock()
delete(c.reservations, key)
}
// rebind moves a launch-scoped reservation onto the session id that launch just
// created, as one operation rather than a release followed by an acquire, so the
// population never momentarily drops and no concurrent admission can take the
// freed unit. Where a reservation already occupies that session id — two
// launches racing to the same session, such as an Office identity-owned
// session two concurrent starts converge onto via
// EnsureSessionForAgentWithCreation — the launch-scoped reservation is
// released instead of overwriting the winner's slot: this is still one
// launch, mirroring rekey's own collision handling. Returning false leaves the
// caller's key pointing at the now-deleted launch-scoped entry, so its own
// later release is a harmless no-op rather than deleting the winner's still
// in-flight reservation.
func (c *sessionCeilingController) rebind(launchKey, sessionID string) bool {
if launchKey == "" || sessionID == "" {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
reservation, held := c.reservations[launchKey]
if !held {
return false
}
delete(c.reservations, launchKey)
if _, collision := c.reservations[sessionID]; collision {
return false
}
reservation.sessionID = sessionID
c.reservations[sessionID] = reservation
return true
}
// rekey moves a reservation from the session that was gated onto the replacement
// session that will actually be launched. Where the replacement is already counted
// or already reserved, the original is released and no second unit is consumed:
// this is still one launch.
func (c *sessionCeilingController) rekey(ctx context.Context, fromSessionID, toSessionID string) bool {
if fromSessionID == "" || toSessionID == "" || fromSessionID == toSessionID {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
reservation, held := c.reservations[fromSessionID]
if !held {
return false
}
delete(c.reservations, fromSessionID)
if _, alreadyReserved := c.reservations[toSessionID]; alreadyReserved {
// Do not transfer ownership onto a reservation the caller does not
// hold: the source key is already gone (deleted above), so the
// caller's later releaseIfNotConsumed becomes a no-op instead of
// deleting the actual holder's reservation and undercounting the
// population, matching rebind's collision behavior.
return false
}
if counted, err := c.countedRowsLocked(context.WithoutCancel(ctx)); err == nil {
if _, alreadyCounted := counted[toSessionID]; alreadyCounted {
return true
}
}
reservation.sessionID = toSessionID
c.reservations[toSessionID] = reservation
return true
}
// handOffOrAdmit serves the dynamic-route relaunch seam. Where the session is still
// in the population, its counted membership is converted into a reservation under
// the same mutex: a relaunch replaces one agent process with another rather than
// adding one, so the hand-off is never refused and never changes the count.
// Where the session is not counted there is no slot to hand off and this is an
// ordinary admission request.
func (c *sessionCeilingController) handOffOrAdmit(ctx context.Context, req admissionRequest) admissionDecision {
if c == nil {
return admissionDecision{admitted: true}
}
origin := req.origin
if origin != launchOriginManual && origin != launchOriginAutomatic {
origin = launchOriginAutomatic
}
c.mu.Lock()
defer c.mu.Unlock()
if c.ceiling == unlimitedSessionCeiling {
return c.admitUnlimitedLocked(req, origin)
}
counted, err := c.countedRowsLocked(ctx)
if err != nil {
return c.decideUnknownPopulationLocked(req, origin, err)
}
if _, isCounted := counted[req.sessionID]; req.sessionID == "" || !isCounted {
return c.decideLocked(req, origin, counted)
}
decision := admissionDecision{
admitted: true,
handedOff: true,
reservationKey: req.sessionID,
population: c.populationLocked(counted),
populationKnown: true,
ceiling: c.ceiling,
}
c.reservations[req.sessionID] = &ceilingReservation{sessionID: req.sessionID, takenAt: c.now()}
c.logDecision(req, origin, decision)
return decision
}
// isSessionCeilingBacked implements AC-41: whether a session_id is currently
// backed by an in-flight reservation or a counted AC-1 population row. A nil
// controller (a Service built without one resolved, e.g. narrow test
// fixtures) reports every session as backed, the same "no ceiling in effect"
// posture admit/handOffOrAdmit already take. This method is consulted only
// by Executor's observation-only bypass detector (AC-41a) and never gates,
// delays or fails a launch itself.
func (c *sessionCeilingController) isSessionCeilingBacked(ctx context.Context, sessionID string) (bool, error) {
if c == nil || sessionID == "" {
return true, nil
}
c.mu.Lock()
defer c.mu.Unlock()
if _, reserved := c.reservations[sessionID]; reserved {
return true, nil
}
counted, err := c.countedRowsLocked(ctx)
if err != nil {
return false, err
}
_, ok := counted[sessionID]
return ok, nil
}
// expireStaleReservations releases reservations whose launch neither reached a
// counted state nor reported failure inside the launch budget. It is the backstop,
// not the primary release edge.
func (c *sessionCeilingController) expireStaleReservations() int {
budget := constants.AgentLaunchTimeout + reservationExpiryAllowance
now := c.now()
c.mu.Lock()
defer c.mu.Unlock()
released := 0
for key, reservation := range c.reservations {
if now.Sub(reservation.takenAt) <= budget {
continue
}
delete(c.reservations, key)
released++
c.logger.Warn("session ceiling released a reservation whose launch never reached a counted state",
zap.String("reservation_key", key),
zap.String("session_id", reservation.sessionID),
zap.Duration("held_for", now.Sub(reservation.takenAt)),
zap.Duration("budget", budget))
}
return released
}