Files
kandev/apps/backend/internal/startup/step_lifecycle.go
T

339 lines
8.5 KiB
Go

package startup
import (
"context"
"math"
"time"
"go.uber.org/zap"
)
const (
stallThresholdMS = 120000
rateWindow = 30 * time.Second
maxETAMs = 86400000
maxRateSamples = 64
)
type rateSample struct {
at time.Time
delta int64
}
// stepActivation is the mutable state of one BeginStep..EndStep lifetime. A
// fresh one is created on every BeginStep, so nothing here survives past the
// next activation of the same step.
type stepActivation struct {
spec StepSpec
startedAt time.Time
measure Measure
promotionBlocked bool
hasTotal bool
total int64
done int64
anchorDone int64
hasAdvancedOnce bool
lastAdvanceAt time.Time
samples []rateSample
warned map[string]bool
}
// BeginStep ends the currently active step, if any, and opens id as the new
// active step, as a single sequence-number transition. A call naming the
// already-active step is a no-op. A call naming an unregistered identifier
// opens no step and records one warning.
func (r *Reporter) BeginStep(id StepID) {
r.mu.Lock()
defer r.mu.Unlock()
spec, ok := lookupStep(id)
if !ok {
r.warnUnregisteredLocked(id)
return
}
if r.active != nil && r.active.spec.ID == id {
return
}
r.endActiveLocked()
a := &stepActivation{spec: spec, startedAt: time.Now(), measure: MeasureOpaque, warned: map[string]bool{}}
if spec.Measure == MeasureCounting {
a.measure = MeasureCounting
}
r.active = a
r.seq++
if spec.Phase != r.phase {
r.warnActivationLocked(a, "phase_mismatch")
}
if r.log != nil {
r.log.Info("Startup step started", zap.String("step", string(id)), zap.String("measure", string(a.measure)))
}
}
// EndStep closes the active step if it is the one named. A call naming a
// step that is not active changes nothing.
func (r *Reporter) EndStep(id StepID) {
r.mu.Lock()
defer r.mu.Unlock()
if r.active == nil || r.active.spec.ID != id {
return
}
r.endActiveLocked()
r.seq++
}
func (r *Reporter) endActiveLocked() {
a := r.active
if a == nil {
return
}
if r.log != nil {
r.log.Info("Startup step completed",
zap.String("step", string(a.spec.ID)), zap.String("measure", string(a.measure)),
zap.Int64("done", a.done), zap.Int64("total", a.total))
if a.measure == MeasureCounted && a.hasTotal && a.done < a.total {
r.log.Warn("Startup step ended below its total",
zap.String("step", string(a.spec.ID)), zap.Int64("done", a.done), zap.Int64("total", a.total))
}
}
r.active = nil
}
// SeedDone assigns the active step's done count outright from durably
// applied work, without appending a rate sample or touching the
// last-advance clock. Rejected once the total is set or the step has
// advanced.
func (r *Reporter) SeedDone(id StepID, done int64) {
r.mu.Lock()
defer r.mu.Unlock()
a := r.active
if a == nil || a.spec.ID != id {
r.warnMismatchLocked(id, "seed_done")
return
}
if done < 0 {
r.warnActivationLocked(a, "seed_done_negative")
return
}
if a.hasTotal || a.hasAdvancedOnce {
r.warnActivationLocked(a, "seed_done_rejected")
return
}
if done == a.done {
return
}
a.done = done
a.anchorDone = done
}
// SetTotal accepts the active step's total, the single promotion point for a
// step whose registry entry declares counted. A total of zero seals the
// activation opaque; any other accepted total promotes it.
func (r *Reporter) SetTotal(id StepID, total int64) {
r.mu.Lock()
defer r.mu.Unlock()
a := r.active
if a == nil || a.spec.ID != id {
r.warnMismatchLocked(id, "set_total")
return
}
if total < 0 {
r.warnActivationLocked(a, "set_total_negative")
return
}
if a.hasTotal {
if total != a.total {
r.warnActivationLocked(a, "set_total_rejected")
}
return
}
a.hasTotal = true
a.total = total
if a.done > total {
a.done = total
}
if total == 0 {
a.promotionBlocked = true
return
}
if !a.promotionBlocked && a.spec.Measure == MeasureCounted {
a.measure = MeasureCounted
}
}
// Advance adds delta to the active step's done count, saturating at the
// total when one is known. Only the actual increase - never the requested
// delta - resets the stall clock and feeds the rate sample.
func (r *Reporter) Advance(id StepID, delta int64) {
r.mu.Lock()
defer r.mu.Unlock()
a := r.active
if a == nil || a.spec.ID != id {
r.warnMismatchLocked(id, "advance")
return
}
if delta <= 0 {
return
}
newDone := saturatingAdd(a.done, delta)
if a.hasTotal && newDone > a.total {
newDone = a.total
r.warnActivationLocked(a, "advance_exceeds_total")
}
increase := newDone - a.done
a.done = newDone
if increase <= 0 {
return
}
now := time.Now()
a.hasAdvancedOnce = true
a.lastAdvanceAt = now
samples, droppedDelta := appendSample(a.samples, rateSample{at: now, delta: increase})
a.samples = samples
a.anchorDone += droppedDelta
}
// Degrade moves the active step's measure down one rung (counted->counting,
// counting->opaque) and permanently blocks re-promotion for the rest of this
// activation, including when it is called while already opaque.
func (r *Reporter) Degrade(id StepID) {
r.mu.Lock()
defer r.mu.Unlock()
a := r.active
if a == nil || a.spec.ID != id {
r.warnMismatchLocked(id, "degrade")
return
}
switch a.measure {
case MeasureCounted:
a.measure = MeasureCounting
case MeasureCounting:
a.measure = MeasureOpaque
case MeasureOpaque:
// already at the floor; only the latch below changes.
}
a.promotionBlocked = true
r.warnActivationLocked(a, "degrade")
}
func (r *Reporter) warnActivationLocked(a *stepActivation, condition string) {
if a.warned[condition] {
return
}
a.warned[condition] = true
if r.log != nil {
r.log.Warn("Startup step warning", zap.String("step", string(a.spec.ID)), zap.String("condition", condition))
}
}
func (r *Reporter) warnUnregisteredLocked(id StepID) {
if r.unregisteredWarned == nil {
r.unregisteredWarned = map[StepID]bool{}
}
if r.unregisteredWarned[id] {
return
}
r.unregisteredWarned[id] = true
if r.log != nil {
r.log.Warn("Startup step begin for unregistered identifier", zap.String("step", string(id)))
}
}
// mismatchKey dedupes a "called for a step that isn't active" warning per
// (target step, operation): a caller bug of this shape typically repeats on
// every record in a loop, and only the first occurrence is informative.
type mismatchKey struct {
id StepID
op string
}
func (r *Reporter) warnMismatchLocked(id StepID, op string) {
if r.mismatchWarned == nil {
r.mismatchWarned = map[mismatchKey]bool{}
}
key := mismatchKey{id: id, op: op}
if r.mismatchWarned[key] {
return
}
r.mismatchWarned[key] = true
if r.log != nil {
var active StepID
if r.active != nil {
active = r.active.spec.ID
}
r.log.Warn("Startup step call for a step that is not active",
zap.String("step", string(id)), zap.String("op", op), zap.String("active_step", string(active)))
}
}
func saturatingAdd(a, b int64) int64 {
sum := a + b
if sum < a {
return math.MaxInt64
}
return sum
}
// appendSample appends s to samples, trimming from the front once the ring
// exceeds maxRateSamples. It also returns the sum of the trimmed samples'
// deltas so the caller can fold it into anchorDone: rateWindowStart's
// cumulative-sum walk assumes anchorDone is the done value immediately
// before the first sample still in the slice, and silently dropping old
// samples without that adjustment understates the window's baseline,
// overstating the rate it produces.
func appendSample(samples []rateSample, s rateSample) ([]rateSample, int64) {
samples = append(samples, s)
var droppedDelta int64
if len(samples) > maxRateSamples {
trimCount := len(samples) - maxRateSamples
for _, dropped := range samples[:trimCount] {
droppedDelta += dropped.delta
}
samples = samples[trimCount:]
}
return samples, droppedDelta
}
// Package-level helpers follow the existing SetPhase pattern: a no-op when
// ctx carries no reporter, which keeps every call site safe in tests and
// embedded use.
func BeginStep(ctx context.Context, id StepID) {
if r := FromContext(ctx); r != nil {
r.BeginStep(id)
}
}
func EndStep(ctx context.Context, id StepID) {
if r := FromContext(ctx); r != nil {
r.EndStep(id)
}
}
func SeedDone(ctx context.Context, id StepID, done int64) {
if r := FromContext(ctx); r != nil {
r.SeedDone(id, done)
}
}
func SetTotal(ctx context.Context, id StepID, total int64) {
if r := FromContext(ctx); r != nil {
r.SetTotal(id, total)
}
}
func Advance(ctx context.Context, id StepID, delta int64) {
if r := FromContext(ctx); r != nil {
r.Advance(id, delta)
}
}
func Degrade(ctx context.Context, id StepID) {
if r := FromContext(ctx); r != nil {
r.Degrade(id)
}
}