dreampiler/kandev
1
0
Fork 0
Code Issues 43 Pull Requests Actions 6 Packages Projects Releases Wiki Activity

fix: keep large database startup alive (#3624)

Browse Source
This commit is contained in:
Carlos Florêncio
2026-09-12 16:29:38 +01:00
committed by GitHub
parent 3d3209aa1a
commit b6571a9854
58 changed files with 2781 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/backend/internal/backendapp/gateway.go
+1 -1
View File
@@ -293,7 +293,7 @@ func provideGateway(
portHandlers.RegisterHandlers(gateway.Dispatcher)
}
go gateway.Hub.Run(ctx)
go gateway.Hub.Run(processRuntimeContext(ctx))
gateways.RegisterTaskNotifications(ctx, eventBus, gateway.Hub, log)
if taskRepo != nil && eventBus != nil {
var loadPullRequests statussummary.PullRequestLoader
apps/backend/internal/backendapp/httpserver.go
+11 -3
View File
@@ -12,6 +12,7 @@ import (
"github.com/kandev/kandev/internal/common/config"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/startup"
"go.uber.org/zap"
)
@@ -57,7 +58,11 @@ func (hs *handlerSwitch) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// GET /health's body is identical to healthHandler's in helpers.go (status
// "ok", service, mode, version) since /health is a pure liveness probe and
// its shape must not depend on startup progress.
func newBootstrapHandler(version string) http.Handler {
func newBootstrapHandler(version string, reporters ...*startup.Reporter) http.Handler {
progress := startup.New(nil)
if len(reporters) > 0 {
progress = reporters[0]
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet || r.URL.Path != "/health" {
body := map[string]any{
@@ -65,6 +70,9 @@ func newBootstrapHandler(version string) http.Handler {
serviceFieldKey: kandevName,
versionFieldKey: version,
}
if r.Method == http.MethodGet && r.URL.Path == "/ready" {
body["startup"] = progress.Snapshot()
}
writeBootstrapJSON(w, http.StatusServiceUnavailable, body)
return
}
@@ -91,8 +99,8 @@ func writeBootstrapJSON(w http.ResponseWriter, status int, body map[string]any)
// The returned handlerSwitch is later Store()d with the real router; the
// returned *http.Server and *serverListeners are the same instances that
// must flow unmodified into awaitShutdown, so shutdown needs no changes.
func bindBootstrapListeners(cfg *config.Config, log *logger.Logger, version string) (*handlerSwitch, *http.Server, *serverListeners, error) {
handler := newHandlerSwitch(newBootstrapHandler(version))
func bindBootstrapListeners(cfg *config.Config, log *logger.Logger, version string, reporters ...*startup.Reporter) (*handlerSwitch, *http.Server, *serverListeners, error) {
handler := newHandlerSwitch(newBootstrapHandler(version, reporters...))
server := &http.Server{
Handler: handler,
ReadTimeout: cfg.Server.ReadTimeoutDuration(),
apps/backend/internal/backendapp/main.go
+56 -27
View File
@@ -37,6 +37,7 @@ import (
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/common/subproc"
"github.com/kandev/kandev/internal/profiles"
"github.com/kandev/kandev/internal/startup"
// Event bus
"github.com/kandev/kandev/internal/events"
@@ -339,7 +340,14 @@ func run(cfg *config.Config, log *logger.Logger, cleanups *[]func() error, runCl
ctx, cancel := context.WithCancel(context.Background())
addCleanup(func() error { cancel(); return nil })
// 4. Initialize event bus (in-memory for unified mode, or NATS if configured)
return runWithBootstrap(ctx, cfg, log, func(ctx context.Context) bool {
return initializeApplication(ctx, cfg, log, addCleanup, runCleanups)
})
}
func initializeApplication(ctx context.Context, cfg *config.Config, log *logger.Logger,
addCleanup func(func() error), runCleanups func()) bool {
// Initialize the event bus only after the liveness listener is available.
eventBusProvider, cleanup, err := events.Provide(cfg, log)
if err != nil {
log.Error("Failed to initialize event bus", zap.Error(err))
@@ -348,7 +356,7 @@ func run(cfg *config.Config, log *logger.Logger, cleanups *[]func() error, runCl
addCleanup(cleanup)
eventBus := eventBusProvider.Bus
return startServices(ctx, cfg, log, addCleanup, eventBus, runCleanups, cancel)
return startServices(ctx, cfg, log, addCleanup, eventBus, runCleanups, workerCancelFromContext(ctx))
}
// applyStartupRuntimeFlags resolves persisted runtime-flag overrides and
@@ -376,7 +384,7 @@ func startServices( //nolint:cyclop
addCleanup func(func() error),
eventBus bus.EventBus,
runCleanups func(),
cancelContext context.CancelFunc,
cancelWorkers context.CancelFunc,
) bool {
// ============================================
// TASK SERVICE
@@ -385,7 +393,7 @@ func startServices( //nolint:cyclop
dbPool, repos, repoCleanups, err := provideRepositories(ctx, cfg, log, Version)
if err != nil {
log.Error("Failed to initialize repositories", zap.Error(err))
log.Error("Failed to initialize repositories", zap.Error(err), zap.String("phase", string(startup.FromContext(ctx).Snapshot().Phase)))
return false
}
for _, c := range repoCleanups {
@@ -525,7 +533,7 @@ func startServices( //nolint:cyclop
}
return startAgentInfrastructure(ctx, cfg, log, addCleanup, eventBus, agentRuntimeAvailability,
dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups, cancelContext)
dbPool, repos, services, agentSettingsController, agentRegistry, agentctlBinaryPath, runCleanups, cancelWorkers)
}
// startAgentInfrastructure initializes the agent lifecycle manager, worktree, orchestrator,
@@ -546,7 +554,7 @@ func startAgentInfrastructure(
agentRegistry *registry.Registry,
agentctlBinaryPath string,
runCleanups func(),
cancelContext context.CancelFunc,
cancelWorkers context.CancelFunc,
) bool {
restoreCleanups := make([]func() error, 0)
var databaseQuiesce func() error
@@ -858,7 +866,7 @@ func startAgentInfrastructure(
return startGatewayAndServe(ctx, cfg, log, eventBus, agentRuntimeAvailability, dbPool, repos, services,
agentSettingsController, lifecycleMgr, agentRegistry, orchestratorSvc, msgCreator, repoCloner, agentctlBinaryPath,
func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelContext, restoreCleanups, databaseQuiesce)
func(fn func() error) { addRuntimeCleanup(fn) }, runCleanups, cancelWorkers, restoreCleanups, databaseQuiesce)
}
// startOrchestratorAndAutomationConsumers establishes the startup chain in
@@ -896,7 +904,7 @@ func closeBoundListeners(server *http.Server, listeners *serverListeners, log *l
return
}
listeners.Stop()
if err := server.Close(); err != nil {
if err := server.Close(); err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Warn("failed to close HTTP listeners after startup failure", zap.Error(err))
}
}
@@ -923,7 +931,7 @@ func startGatewayAndServe(
agentctlBinaryPath string,
addCleanup func(func() error),
runCleanups func(),
cancelContext context.CancelFunc,
cancelWorkers context.CancelFunc,
restoreCleanups []func() error,
databaseQuiesce func() error,
) bool {
@@ -999,19 +1007,9 @@ func startGatewayAndServe(
services.Plugins.SetUtilityAgent(pluginsUtilityAgentAdapter{svc: services.Utility, userSvc: services.User}, pluginsHostUtilityAdapter{mgr: hostUtilityMgr})
}
var (
handler *handlerSwitch
server *http.Server
listeners *serverListeners
)
bindListeners := func() error {
h, s, l, err := bindBootstrapListeners(cfg, log, Version)
if err != nil {
return err
}
handler, server, listeners = h, s, l
return nil
}
bootstrap := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime)
handler, server, listeners := bootstrap.handler, bootstrap.server, bootstrap.listeners
bindListeners := func() error { startup.SetPhase(ctx, startup.RecoveringSessions); return ctx.Err() }
if err := startOrchestratorAndAutomationConsumers(
bindListeners,
@@ -1035,7 +1033,7 @@ func startGatewayAndServe(
log.Info("GitHub poller started")
},
); err != nil {
if !errors.Is(err, errServerBindFailed) {
if shouldLogStartupOrchestratorError(err) {
log.Error("Failed to start orchestrator", zap.Error(err))
}
closeBoundListeners(server, listeners, log)
@@ -1082,7 +1080,7 @@ func startGatewayAndServe(
workers = append(workers, restoreCleanups[i])
}
restoreQuiesceErr = quiesceForRestore(
cancelContext,
cancelWorkers,
scheduling.Stop,
orchestratorSvc.Stop,
func() error { return stopLifecycleManager(lifecycleMgr, log) },
@@ -1203,7 +1201,7 @@ func startGatewayAndServe(
}
systemSvc.StartBackground(ctx)
addCleanup(func() error { systemSvc.StopBackground(); return nil })
gateways.RegisterSystemNotifications(ctx, eventBus, gateway.Hub, log)
gateways.RegisterSystemNotifications(processRuntimeContext(ctx), eventBus, gateway.Hub, log)
gateways.RegisterAgentRuntimeNotifications(ctx, eventBus, gateway.Hub, func() (any, bool) {
if agentRuntimeAvailability == nil {
return nil, false
@@ -1253,9 +1251,16 @@ func startGatewayAndServe(
// publishReadiness for why the order matters and
// TestPublishReadinessFlipsReadyBeforeSwappingHandler for the regression
// test pinning it.
publishReadiness(func() { ready.Store(true) }, func() { handler.Store(builtServer.Handler) })
if !bootstrap.beginReadinessPublication(ctx) {
return false
}
publishReadiness(func() {
ready.Store(true)
bootstrap.ready.Store(true)
}, func() { handler.Store(builtServer.Handler) })
awaitShutdown(server, listeners, scheduling, orchestratorSvc, lifecycleMgr, runCleanups, log)
startup.SetPhase(ctx, startup.Ready)
awaitShutdown(ctx, server, listeners, scheduling, orchestratorSvc, lifecycleMgr, runCleanups, log)
return true
}
@@ -2429,6 +2434,7 @@ func buildHTTPServer(
// awaitShutdown waits for an OS signal then performs graceful shutdown.
func awaitShutdown(
ctx context.Context,
server *http.Server,
listeners *serverListeners,
scheduling *schedulingRuntime,
@@ -2440,6 +2446,23 @@ func awaitShutdown(
// ============================================
// GRACEFUL SHUTDOWN
// ============================================
shutdownCtx := processContextFromContext(ctx)
if shutdownCtx == nil {
shutdownCtx = ctx
}
if controller := startupSignalControllerFromContext(ctx); controller != nil {
sig := controller.wait(shutdownCtx)
if sig == nil {
log.Info("Shutdown context canceled without OS signal", zap.Int("pid", os.Getpid()))
return
}
log.Info("Received shutdown signal",
zap.String("signal", sig.String()),
zap.Int("pid", os.Getpid()))
runGracefulShutdown(server, listeners, scheduling, orchestratorSvc, lifecycleMgr, runCleanups, log)
return
}
quit := make(chan os.Signal, 2)
signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
log.Debug("shutdown signal handler armed",
@@ -2461,6 +2484,12 @@ func awaitShutdown(
runGracefulShutdown(server, listeners, scheduling, orchestratorSvc, lifecycleMgr, runCleanups, log)
}
func shouldLogStartupOrchestratorError(err error) bool {
return !errors.Is(err, errServerBindFailed) &&
!errors.Is(err, context.Canceled) &&
!errors.Is(err, context.DeadlineExceeded)
}
// migrateDefaultUtilityProfile upgrades the portable user's legacy default
// pair only when it identifies one eligible profile. Ambiguous and missing
// values remain untouched for a later retry or user repair.
apps/backend/internal/backendapp/restore_quiesce.go
+10 -6
View File
@@ -6,18 +6,22 @@ import (
)
// quiesceForRestore stops every runtime that can write to the shared database
// before the restore path checkpoints and closes its pool. It keeps going
// after an individual stop failure so callers receive the complete failure
// set and no remaining worker is left running by an early return.
// before the restore path checkpoints and closes its pool. cancelWorkers is
// deliberately limited to worker/runtime contexts: it must not cancel the
// process context that owns the HTTP listener and restore job lifetime. The
// caller therefore remains able to publish restart_required and wait for an
// explicit process shutdown after the restore completes. The function keeps
// going after an individual stop failure so callers receive the complete
// failure set and no remaining worker is left running by an early return.
func quiesceForRestore(
cancelContext func(),
cancelWorkers func(),
stopScheduling func() error,
stopOrchestrator func() error,
stopAgents func() error,
stopWorkers []func() error,
) error {
if cancelContext != nil {
cancelContext()
if cancelWorkers != nil {
cancelWorkers()
}
var errs []error
if stopScheduling != nil {
apps/backend/internal/backendapp/startup.go
+221
View File
@@ -0,0 +1,221 @@
package backendapp
import (
"context"
"net/http"
"os"
"os/signal"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/kandev/kandev/internal/common/config"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/startup"
"go.uber.org/zap"
)
type bootstrapContextKey struct{}
type bootstrapRuntime struct {
handler *handlerSwitch
server *http.Server
listeners *serverListeners
ready atomic.Bool
readinessMu sync.Mutex
readinessPublishing bool
startupClosing bool
}
type startupSignalContextKey struct{}
type processContextKey struct{}
type workerCancelContextKey struct{}
type startupSignalController struct {
first chan os.Signal
stop chan struct{}
done chan struct{}
signals chan os.Signal
once sync.Once
}
func newStartupSignalController(ctx context.Context, cancel context.CancelFunc, log *logger.Logger) *startupSignalController {
controller := &startupSignalController{
first: make(chan os.Signal, 1),
stop: make(chan struct{}),
done: make(chan struct{}),
signals: make(chan os.Signal, 2),
}
signal.Notify(controller.signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
if log != nil {
log.Debug("shutdown signal handler armed",
zap.Int("pid", os.Getpid()),
zap.Int("ppid", os.Getppid()))
}
go func() {
defer close(controller.done)
select {
case sig := <-controller.signals:
controller.first <- sig
cancel()
select {
case second := <-controller.signals:
if log != nil {
log.Warn("Received second shutdown signal, forcing exit", zap.String("signal", second.String()))
_ = log.Close()
}
os.Exit(1)
case <-controller.stop:
}
case <-ctx.Done():
case <-controller.stop:
}
}()
return controller
}
func withStartupSignalController(ctx context.Context, controller *startupSignalController) context.Context {
return context.WithValue(ctx, startupSignalContextKey{}, controller)
}
func withProcessContext(ctx, process context.Context) context.Context {
return context.WithValue(ctx, processContextKey{}, process)
}
func processContextFromContext(ctx context.Context) context.Context {
process, _ := ctx.Value(processContextKey{}).(context.Context)
return process
}
// processRuntimeContext returns the process lifetime carried by startup
// workers. Restore cancels workers while the process remains alive to finish
// the restore and deliver its terminal job update.
func processRuntimeContext(ctx context.Context) context.Context {
if process := processContextFromContext(ctx); process != nil {
return process
}
return ctx
}
// beginReadinessPublication serializes the readiness handoff with the startup
// cancellation watcher. Once startup cancellation claims listener closure,
// initialization cannot publish a router into a closing server.
func (runtime *bootstrapRuntime) beginReadinessPublication(ctx context.Context) bool {
runtime.readinessMu.Lock()
defer runtime.readinessMu.Unlock()
if runtime.startupClosing || ctx.Err() != nil {
return false
}
runtime.readinessPublishing = true
return true
}
func (runtime *bootstrapRuntime) claimStartupClose() bool {
runtime.readinessMu.Lock()
defer runtime.readinessMu.Unlock()
if runtime.ready.Load() || runtime.readinessPublishing {
return false
}
runtime.startupClosing = true
return true
}
func withWorkerCancel(ctx context.Context, cancel context.CancelFunc) context.Context {
return context.WithValue(ctx, workerCancelContextKey{}, cancel)
}
func workerCancelFromContext(ctx context.Context) context.CancelFunc {
cancel, _ := ctx.Value(workerCancelContextKey{}).(context.CancelFunc)
return cancel
}
func startupSignalControllerFromContext(ctx context.Context) *startupSignalController {
controller, _ := ctx.Value(startupSignalContextKey{}).(*startupSignalController)
return controller
}
func (controller *startupSignalController) wait(ctx context.Context) os.Signal {
select {
case sig := <-controller.first:
return sig
default:
}
select {
case sig := <-controller.first:
return sig
case <-ctx.Done():
select {
case sig := <-controller.first:
return sig
default:
return nil
}
}
}
func (controller *startupSignalController) stopAndWait() {
if controller == nil {
return
}
controller.once.Do(func() {
signal.Stop(controller.signals)
close(controller.stop)
<-controller.done
})
}
// runWithBootstrap owns the listener for both initialization and application use.
// Constructors run synchronously so resource cleanup cannot race initialization.
// The process context remains live while worker contexts are quiesced, which
// lets restore finish and deliver its job result before explicit shutdown.
func runWithBootstrap(ctx context.Context, cfg *config.Config, log *logger.Logger, initialize func(context.Context) bool) bool {
startupCtx, cancel := context.WithCancel(ctx)
defer cancel()
controller := newStartupSignalController(startupCtx, cancel, log)
defer controller.stopAndWait()
if err := startupCtx.Err(); err != nil {
return false
}
progress := startup.New(log)
startupCtx = startup.WithReporter(startupCtx, progress)
startupCtx = withStartupSignalController(startupCtx, controller)
workerCtx, cancelWorkers := context.WithCancel(startupCtx)
defer cancelWorkers()
workerCtx = withProcessContext(workerCtx, startupCtx)
workerCtx = withWorkerCancel(workerCtx, cancelWorkers)
ready.Store(false)
handler, server, listeners, err := bindBootstrapListeners(cfg, log, Version, progress)
if err != nil {
return false
}
defer closeBoundListeners(server, listeners, log)
runtime := &bootstrapRuntime{handler: handler, server: server, listeners: listeners}
done := make(chan struct{})
stopped := make(chan struct{})
go func() {
defer close(stopped)
ticker := time.NewTicker(15 * time.Second)
defer ticker.Stop()
progress.Report()
for {
select {
case <-startupCtx.Done():
if runtime.claimStartupClose() {
closeBoundListeners(server, listeners, log)
}
return
case <-done:
return
case <-ticker.C:
progress.Report()
}
}
}()
defer func() { close(done); <-stopped }()
ok := initialize(context.WithValue(workerCtx, bootstrapContextKey{}, runtime))
return ok && (startupCtx.Err() == nil || runtime.ready.Load())
}
apps/backend/internal/backendapp/startup_database_test.go
+616
View File
@@ -0,0 +1,616 @@
package backendapp
import (
"bufio"
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"runtime"
"sync"
"syscall"
"testing"
"time"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
"go.uber.org/zap/zaptest/observer"
"github.com/kandev/kandev/internal/common/config"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/persistence"
"github.com/kandev/kandev/internal/startup"
"github.com/kandev/kandev/internal/system/backups"
"github.com/kandev/kandev/internal/system/jobs"
)
// @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.1, .5
func TestRunBindsBeforeDatabaseFailure(t *testing.T) {
core, logs := observer.New(zap.InfoLevel)
log, err := logger.NewFromZap(zap.New(core))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = log.Close() })
cfg := &config.Config{}
cfg.Server.Host = "127.0.0.1"
cfg.Server.Port = freePort(t)
cfg.HomeDir = t.TempDir()
cfg.Database.Path = filepath.Join(cfg.HomeDir, "data", "kandev.db")
if err := os.MkdirAll(cfg.Database.Path, 0700); err != nil {
t.Fatal(err)
} // Directory, not a database.
var cleanups []func() error
cleanup := func() {
for i := len(cleanups) - 1; i >= 0; i-- {
if cleanups[i] != nil {
_ = cleanups[i]()
}
}
cleanups = nil
}
defer cleanup()
if run(cfg, log, &cleanups, cleanup) {
t.Fatal("database failure reported success")
}
if logs.FilterMessage("HTTP listener bound; serving liveness probe while startup continues").Len() != 1 {
t.Fatal("database initialization ran before the liveness listener bound")
}
if _, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/health"); err == nil {
t.Fatal("failed initializer left healthy listener")
}
}
func TestBootstrapReportsStartupPhase(t *testing.T) {
w := httptest.NewRecorder()
newBootstrapHandler("test").ServeHTTP(w, httptest.NewRequest("GET", "/ready", nil))
var body map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
startupBody, ok := body["startup"].(map[string]any)
if !ok {
t.Fatalf("startup body = %#v, want an object", body["startup"])
}
if startupBody["phase"] != string(startup.OpeningDatabase) {
t.Fatalf("startup phase = %v, want %q", startupBody["phase"], startup.OpeningDatabase)
}
for _, field := range []string{"elapsed_ms", "phase_elapsed_ms"} {
value, ok := startupBody[field].(float64)
if !ok || value < 0 {
t.Fatalf("startup %s = %v, want a non-negative number", field, startupBody[field])
}
}
}
// @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.2, .6
func TestBootstrapCancellationClosesListenerBeforeInitializerReturns(t *testing.T) {
cfg := &config.Config{}
cfg.Server.Host = "127.0.0.1"
cfg.Server.Port = freePort(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
entered := make(chan struct{})
release := make(chan struct{})
var releaseOnce sync.Once
done := make(chan bool, 1)
defer releaseOnce.Do(func() { close(release) })
go func() {
done <- runWithBootstrap(ctx, cfg, testLogger(t), func(ctx context.Context) bool {
close(entered)
<-release
return true
})
}()
<-entered
for _, path := range []string{"/ready", "/api/v1/tasks"} {
status, _, err := getJSON("127.0.0.1", cfg.Server.Port, path)
if err != nil || status != http.StatusServiceUnavailable {
t.Fatalf("%s: %d %v", path, status, err)
}
}
cancel()
deadline := time.After(2 * time.Second)
for {
_, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/health")
if err != nil {
break
}
select {
case <-deadline:
t.Fatal("canceled initialization left listener healthy")
default:
}
}
// The wrapper must join initialization instead of racing resource cleanup.
select {
case <-done:
t.Fatal("returned before initializer drained")
default:
}
releaseOnce.Do(func() { close(release) })
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("initializer did not drain after release")
}
}
// @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.1, .2, .3, .6
func TestBootstrapDelayedSuccessKeepsLivenessUntilReadiness(t *testing.T) {
cfg := &config.Config{}
cfg.Server.Host = "127.0.0.1"
cfg.Server.Port = freePort(t)
cfg.Launcher.HealthTimeoutMs = 100
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
entered := make(chan struct{})
release := make(chan struct{})
published := make(chan struct{})
done := make(chan bool, 1)
var releaseOnce sync.Once
var publishOnce sync.Once
defer releaseOnce.Do(func() { close(release) })
go func() {
done <- runWithBootstrap(ctx, cfg, testLogger(t), func(ctx context.Context) bool {
runtime := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime)
close(entered)
<-release
application := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/health" {
writeBootstrapJSON(w, http.StatusOK, map[string]any{
statusKey: "ok",
serviceFieldKey: kandevName,
"mode": "websocket+http",
versionFieldKey: Version,
})
return
}
if r.URL.Path == "/ready" || r.URL.Path == "/api/v1/tasks" {
writeBootstrapJSON(w, http.StatusOK, map[string]any{statusKey: "ok"})
return
}
writeBootstrapJSON(w, http.StatusNotFound, map[string]any{statusKey: "not_found"})
})
if !runtime.beginReadinessPublication(ctx) {
return false
}
publishReadiness(func() {
ready.Store(true)
runtime.ready.Store(true)
publishOnce.Do(func() { close(published) })
}, func() { runtime.handler.Store(application) })
<-processContextFromContext(ctx).Done()
return true
})
}()
<-entered
// Keep the bootstrap initializer behind the same short health budget the
// launcher uses for its keep-or-kill decision. Readiness has no matching
// deadline: it must keep polling while recovery is still running.
healthDeadline := time.Now().Add(time.Duration(cfg.Launcher.HealthTimeoutMs) * time.Millisecond)
for time.Now().Before(healthDeadline) {
status, body, err := getJSON("127.0.0.1", cfg.Server.Port, "/health")
if err != nil || status != http.StatusOK {
t.Fatalf("bootstrap /health during delayed initialization = %d, %v; want 200", status, err)
}
if body[statusKey] != "ok" || body[serviceFieldKey] != kandevName || body[versionFieldKey] != Version {
t.Fatalf("bootstrap /health identity = %#v, want status=ok service=%q version=%q", body, kandevName, Version)
}
status, _, err = getJSON("127.0.0.1", cfg.Server.Port, "/ready")
if err != nil || status != http.StatusServiceUnavailable {
t.Fatalf("bootstrap /ready during delayed initialization = %d, %v; want 503", status, err)
}
status, _, err = getJSON("127.0.0.1", cfg.Server.Port, "/api/v1/tasks")
if err != nil || status != http.StatusServiceUnavailable {
t.Fatalf("bootstrap application route during delayed initialization = %d, %v; want 503", status, err)
}
}
select {
case result := <-done:
t.Fatalf("bootstrap exited before readiness handoff: %v", result)
default:
}
select {
case <-published:
t.Fatal("readiness published before initialization gate released")
default:
}
readyDone := make(chan error, 1)
go func() {
for {
status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/ready")
if err != nil {
readyDone <- err
return
}
if status == http.StatusOK {
readyDone <- nil
return
}
time.Sleep(10 * time.Millisecond)
}
}()
select {
case err := <-readyDone:
t.Fatalf("unbounded readiness wait completed before initialization release: %v", err)
case <-time.After(50 * time.Millisecond):
}
releaseOnce.Do(func() { close(release) })
select {
case <-published:
case <-time.After(2 * time.Second):
t.Fatal("initializer did not publish readiness after release")
}
select {
case err := <-readyDone:
if err != nil {
t.Fatalf("readiness wait after release: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("readiness wait did not observe router handoff")
}
if status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/api/v1/tasks"); err != nil || status != http.StatusOK {
t.Fatalf("application route after readiness = %d, %v; want 200", status, err)
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("bootstrap did not stop after explicit process cancellation")
}
}
// @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.3, .6
func TestBootstrapRestoreKeepsProcessAliveUntilResultAndShutdown(t *testing.T) {
root := t.TempDir()
databasePath := filepath.Join(root, "kandev.db")
backupsDir := filepath.Join(root, "backups")
if err := os.MkdirAll(backupsDir, 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(databasePath, []byte("before"), 0o600); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(backupsDir, "manual-restore.db"), []byte("after"), 0o600); err != nil {
t.Fatal(err)
}
cfg := &config.Config{}
cfg.Server.Host = "127.0.0.1"
cfg.Server.Port = freePort(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
restoreBlocked := make(chan struct{})
restoreRelease := make(chan struct{})
jobIDReady := make(chan string, 1)
resultReady := make(chan *jobs.Job, 1)
bootstrapReady := make(chan struct{})
done := make(chan bool, 1)
var restoreBlockedOnce sync.Once
var bootstrapReadyOnce sync.Once
var restoreReleaseOnce sync.Once
defer restoreReleaseOnce.Do(func() { close(restoreRelease) })
go func() {
done <- runWithBootstrap(ctx, cfg, testLogger(t), func(ctx context.Context) bool {
runtime := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime)
tracker := jobs.NewTracker(nil, testLogger(t))
svc := backups.NewService(databasePath, nil, tracker, testLogger(t))
svc.RestoreQuiesce = func() error {
return quiesceForRestore(
workerCancelFromContext(ctx),
func() error {
restoreBlockedOnce.Do(func() { close(restoreBlocked) })
<-restoreRelease
return nil
}, nil, nil, nil,
)
}
application := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/health" || r.URL.Path == "/ready" {
writeBootstrapJSON(w, http.StatusOK, map[string]any{statusKey: "ok"})
return
}
writeBootstrapJSON(w, http.StatusNotFound, map[string]any{statusKey: "not_found"})
})
if !runtime.beginReadinessPublication(ctx) {
return false
}
publishReadiness(func() {
ready.Store(true)
runtime.ready.Store(true)
bootstrapReadyOnce.Do(func() { close(bootstrapReady) })
}, func() { runtime.handler.Store(application) })
jobID, err := svc.Restore(context.Background(), "manual-restore.db", backups.RestoreConfirmToken)
if err != nil {
return false
}
jobIDReady <- jobID
<-restoreBlocked
<-restoreRelease
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
if job := tracker.Get(jobID); job != nil && job.State == jobs.StateSucceeded {
resultReady <- job
<-processContextFromContext(ctx).Done()
return true
}
time.Sleep(5 * time.Millisecond)
}
return false
})
}()
select {
case <-bootstrapReady:
case <-time.After(2 * time.Second):
t.Fatal("bootstrap did not publish readiness")
}
select {
case <-jobIDReady:
case <-time.After(2 * time.Second):
t.Fatal("restore job was not submitted")
}
select {
case <-restoreBlocked:
case <-time.After(2 * time.Second):
t.Fatal("restore did not reach the blocked quiesce boundary")
}
if status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/health"); err != nil || status != http.StatusOK {
t.Fatalf("/health while restore quiesce is blocked = %d, %v; want 200", status, err)
}
if status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/ready"); err != nil || status != http.StatusOK {
t.Fatalf("/ready while restore quiesce is blocked = %d, %v; want 200", status, err)
}
select {
case result := <-done:
t.Fatalf("backend exited while restore was blocked: %v", result)
default:
}
restoreReleaseOnce.Do(func() { close(restoreRelease) })
select {
case job := <-resultReady:
if got, _ := job.Result["restart_required"].(bool); !got {
t.Fatalf("restore result = %#v, want restart_required=true", job.Result)
}
case <-time.After(2 * time.Second):
t.Fatal("restore result was not delivered")
}
select {
case <-done:
t.Fatal("backend exited before explicit shutdown after restore result")
default:
}
cancel()
select {
case <-done:
case <-time.After(2 * time.Second):
t.Fatal("backend did not exit after explicit shutdown")
}
}
// @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.5, .7
func TestRepositoryBackupFailureReportsPhaseAndPreventsMigration(t *testing.T) {
dir := t.TempDir()
dataDir := filepath.Join(dir, "data")
if err := os.MkdirAll(dataDir, 0700); err != nil {
t.Fatal(err)
}
path := filepath.Join(dataDir, "kandev.db")
db, err := sqlx.Open("sqlite3", path)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
if _, err = db.Exec("CREATE TABLE sentinel(value TEXT); INSERT INTO sentinel VALUES ('original')"); err != nil {
t.Fatal(err)
}
if err = os.WriteFile(filepath.Join(dataDir, "backups"), []byte("blocked"), 0600); err != nil {
t.Fatal(err)
}
cfg := &config.Config{HomeDir: dir}
cfg.Database.Path = path
progress := startup.New(nil)
ctx := startup.WithReporter(context.Background(), progress)
_, _, _, err = provideRepositories(ctx, cfg, testLogger(t), "upgrade-test")
if err == nil {
t.Fatal("backup failure did not fail initialization")
}
if got := progress.Snapshot().Phase; got != startup.BackingUpDatabase {
t.Fatalf("phase=%s, want backup", got)
}
var count int
if err = db.Get(&count, "SELECT count(*) FROM sqlite_master WHERE name='tasks'"); err != nil {
t.Fatal(err)
}
if count != 0 {
t.Fatal("migration ran after failed backup")
}
}
func TestCanceledPersistenceDoesNotStartBackup(t *testing.T) {
dir := t.TempDir()
cfg := &config.Config{HomeDir: dir}
cfg.Database.Path = filepath.Join(dir, "test.db")
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, cleanup, err := persistence.ProvideContext(ctx, cfg, nil, "test")
if cleanup != nil {
defer func() { _ = cleanup() }()
}
if err == nil {
t.Fatal("canceled initialization opened persistence")
}
}
func TestRepositoryFailureClosesDatabase(t *testing.T) {
if _, err := os.Stat("/proc/self/fd"); err != nil {
t.Skip("requires Linux descriptor inspection")
}
dir := t.TempDir()
data := filepath.Join(dir, "data")
if err := os.MkdirAll(data, 0700); err != nil {
t.Fatal(err)
}
path := filepath.Join(data, "kandev.db")
db, err := sqlx.Open("sqlite3", path)
if err != nil {
t.Fatal(err)
}
if _, err = db.Exec("CREATE TABLE tasks(id TEXT)"); err != nil {
t.Fatal(err)
}
if err = db.Close(); err != nil {
t.Fatal(err)
}
cfg := &config.Config{HomeDir: dir}
cfg.Database.Path = path
_, _, cleanups, err := provideRepositories(context.Background(), cfg, testLogger(t), "")
for _, cleanup := range cleanups {
if cleanup != nil {
_ = cleanup()
}
}
if err == nil {
t.Fatal("invalid schema unexpectedly initialized")
}
fds, err := os.ReadDir("/proc/self/fd")
if err != nil {
t.Fatal(err)
}
for _, fd := range fds {
target, _ := os.Readlink(filepath.Join("/proc/self/fd", fd.Name()))
if target == path {
t.Fatalf("failed migration leaked database descriptor %s", fd.Name())
}
}
}
func TestBootstrapSignalDuringInitialization(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("POSIX signals")
}
if os.Getenv("KANDEV_TEST_BOOTSTRAP_SIGNAL") == "1" {
cfg := &config.Config{}
cfg.Server.Host = "127.0.0.1"
cfg.Server.Port = freePort(t)
ok := runWithBootstrap(context.Background(), cfg, testLogger(t), func(ctx context.Context) bool {
_, _ = os.Stdout.WriteString("BOUND\n")
<-ctx.Done()
return false
})
if ok {
t.Fatal("canceled startup succeeded")
}
return
}
exe, err := os.Executable()
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, exe, "-test.run=^TestBootstrapSignalDuringInitialization$")
cmd.Env = append(os.Environ(), "KANDEV_TEST_BOOTSTRAP_SIGNAL=1")
stdout, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
if err = cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() { _ = cmd.Process.Kill() }()
scan := bufio.NewScanner(stdout)
bound := false
for scan.Scan() {
if scan.Text() == "BOUND" {
bound = true
break
}
}
if !bound {
t.Fatal("child did not bind")
}
if err = cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
if err = cmd.Wait(); err != nil {
t.Fatalf("initialization did not drain on SIGTERM: %v", err)
}
}
// @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.3, .6
func TestBootstrapSignalAfterReadiness(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("POSIX signals")
}
if os.Getenv("KANDEV_TEST_BOOTSTRAP_SIGNAL_READY") == "1" {
cfg := &config.Config{}
cfg.Server.Host = "127.0.0.1"
cfg.Server.Port = freePort(t)
ok := runWithBootstrap(context.Background(), cfg, testLogger(t), func(ctx context.Context) bool {
runtime := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime)
application := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
writeBootstrapJSON(w, http.StatusOK, map[string]any{statusKey: "ok"})
})
if !runtime.beginReadinessPublication(ctx) {
t.Fatal("readiness publication rejected before signal test")
}
publishReadiness(func() {
ready.Store(true)
runtime.ready.Store(true)
}, func() { runtime.handler.Store(application) })
_, _ = os.Stdout.WriteString("READY\n")
awaitShutdown(ctx, runtime.server, runtime.listeners, &schedulingRuntime{}, nil, nil, func() {}, testLogger(t))
return true
})
if !ok {
t.Fatal("ready bootstrap did not complete graceful shutdown")
}
return
}
exe, err := os.Executable()
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, exe, "-test.run=^TestBootstrapSignalAfterReadiness$")
cmd.Env = append(os.Environ(), "KANDEV_TEST_BOOTSTRAP_SIGNAL_READY=1")
stdout, err := cmd.StdoutPipe()
if err != nil {
t.Fatal(err)
}
if err = cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() { _ = cmd.Process.Kill() }()
scan := bufio.NewScanner(stdout)
readyOutput := false
for scan.Scan() {
if scan.Text() == "READY" {
readyOutput = true
break
}
}
if !readyOutput {
t.Fatal("child did not publish readiness")
}
if err = cmd.Process.Signal(syscall.SIGTERM); err != nil {
t.Fatal(err)
}
if err = cmd.Wait(); err != nil {
t.Fatalf("ready bootstrap did not shut down on SIGTERM: %v", err)
}
}
apps/backend/internal/backendapp/startup_order_test.go
+64
View File
@@ -1,7 +1,9 @@
package backendapp
import (
"context"
"errors"
"fmt"
"reflect"
"testing"
)
@@ -92,3 +94,65 @@ func TestPublishReadinessFlipsReadyBeforeSwappingHandler(t *testing.T) {
t.Fatalf("publishReadiness order = %v, want %v", order, want)
}
}
func TestShouldLogStartupOrchestratorErrorSkipsExpectedShutdownErrors(t *testing.T) {
for _, test := range []struct {
name string
err error
want bool
}{
{name: "bind failure", err: errServerBindFailed, want: false},
{name: "canceled", err: context.Canceled, want: false},
{name: "wrapped canceled", err: fmt.Errorf("start: %w", context.Canceled), want: false},
{name: "deadline", err: context.DeadlineExceeded, want: false},
{name: "unexpected", err: errors.New("orchestrator unavailable"), want: true},
} {
t.Run(test.name, func(t *testing.T) {
if got := shouldLogStartupOrchestratorError(test.err); got != test.want {
t.Fatalf("shouldLogStartupOrchestratorError(%v) = %v, want %v", test.err, got, test.want)
}
})
}
}
func TestBootstrapRuntimeReadinessHandoffSerializesStartupClose(t *testing.T) {
runtime := &bootstrapRuntime{}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
if !runtime.beginReadinessPublication(ctx) {
t.Fatal("beginReadinessPublication() = false, want true")
}
cancel()
if runtime.claimStartupClose() {
t.Fatal("claimStartupClose() = true after readiness publication began")
}
}
func TestBootstrapRuntimeStartupCloseWinsBeforeReadiness(t *testing.T) {
runtime := &bootstrapRuntime{}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if runtime.beginReadinessPublication(ctx) {
t.Fatal("beginReadinessPublication() = true after cancellation")
}
if !runtime.claimStartupClose() {
t.Fatal("claimStartupClose() = false before readiness publication")
}
}
func TestProcessRuntimeContextSurvivesWorkerCancellation(t *testing.T) {
processCtx, cancelProcess := context.WithCancel(context.Background())
defer cancelProcess()
workerCtx, cancelWorker := context.WithCancel(context.Background())
workerCtx = withProcessContext(workerCtx, processCtx)
cancelWorker()
if got := processRuntimeContext(workerCtx); got != processCtx {
t.Fatal("processRuntimeContext() did not return the process context")
}
if err := processRuntimeContext(workerCtx).Err(); err != nil {
t.Fatalf("process runtime context error = %v, want nil", err)
}
}
apps/backend/internal/backendapp/storage.go
+109 -17
View File
@@ -18,6 +18,7 @@ import (
"github.com/kandev/kandev/internal/persistence/requiredstores"
quickterminalrepository "github.com/kandev/kandev/internal/quickterminal/repository"
"github.com/kandev/kandev/internal/secrets"
"github.com/kandev/kandev/internal/startup"
systemsettings "github.com/kandev/kandev/internal/system/settings"
"github.com/kandev/kandev/internal/task/repository"
"github.com/kandev/kandev/internal/telemetrycontract"
@@ -34,88 +35,152 @@ import (
)
func provideRepositories(ctx context.Context, cfg *config.Config, log *logger.Logger, version string) (*db.Pool, *Repositories, []func() error, error) {
// Each check is an admission barrier. Constructors that predate context
// support may finish one in-flight statement after cancellation, but the
// next store is never admitted and cleanup runs only after this synchronous
// initializer has returned.
cleanups := make([]func() error, 0, 12)
initialized := false
defer func() {
if !initialized {
for i := len(cleanups) - 1; i >= 0; i-- {
if cleanups[i] != nil {
_ = cleanups[i]()
}
}
}
}()
tracker, err := requiredstores.NewCatalogTracker()
if err != nil {
return nil, nil, nil, err
}
pool, cleanup, err := persistence.Provide(cfg, log, version)
pool, cleanup, err := persistence.ProvideContext(ctx, cfg, log, version)
if err != nil {
return nil, nil, nil, err
}
cleanups = append(cleanups, cleanup)
startup.SetPhase(ctx, startup.ApplyingMigrations)
writer, reader := pool.Writer(), pool.Reader()
if err := checkStartupContext(ctx, "schema metadata"); err != nil {
return nil, nil, nil, err
}
if err := recordRequiredStore(tracker, "schema-meta", nil); err != nil {
return nil, nil, nil, err
}
taskRepoImpl, cleanup, err := repository.Provide(writer, reader, log)
if err := checkStartupContext(ctx, "task repository"); err != nil {
return nil, nil, nil, err
}
taskRepoImpl, cleanup, err := repository.ProvideContext(ctx, writer, reader, log)
if err := recordRequiredStore(tracker, "task", err); err != nil {
return nil, nil, nil, fmt.Errorf("task store: %w", err)
}
cleanups = append(cleanups, cleanup)
// Workflow repo must be initialized before analytics repo because
// analytics creates indexes on the workflow_steps table.
if err := checkStartupContext(ctx, "workflow repository"); err != nil {
return nil, nil, nil, err
}
workflowRepo, err := workflowrepository.NewWithDB(writer, reader, log)
if err := recordRequiredStore(tracker, "workflow", err); err != nil {
return nil, nil, nil, fmt.Errorf("workflow store: %w", err)
}
if err := checkStartupContext(ctx, "analytics repository"); err != nil {
return nil, nil, nil, err
}
analyticsRepo, cleanup, err := analyticsrepository.Provide(writer, reader)
if err := recordRequiredStore(tracker, "analytics", err); err != nil {
return nil, nil, nil, fmt.Errorf("analytics store: %w", err)
}
cleanups = append(cleanups, cleanup)
if err := checkStartupContext(ctx, "agent settings repository"); err != nil {
return nil, nil, nil, err
}
agentSettingsRepo, cleanup, err := settingsstore.Provide(writer, reader, log)
if err := recordRequiredStore(tracker, "agent-settings", err); err != nil {
return nil, nil, nil, fmt.Errorf("agent settings store: %w", err)
}
cleanups = append(cleanups, cleanup)
if err := checkStartupContext(ctx, "support repositories"); err != nil {
return nil, nil, nil, err
}
supportRepos, supportCleanups, err := provideSupportRepos(ctx, writer, reader, tracker)
if err != nil {
cleanups = append(cleanups, supportCleanups...)
return nil, nil, nil, err
}
cleanups = append(cleanups, supportCleanups...)
if err := checkStartupContext(ctx, "office repository"); err != nil {
return nil, nil, nil, err
}
officeRepo, officeCleanup, err := office.Provide(writer, reader, log)
if err := recordRequiredStore(tracker, "office", err); err != nil {
return nil, nil, nil, fmt.Errorf("office repo: %w", err)
}
cleanups = append(cleanups, officeCleanup)
if err := checkStartupContext(ctx, "terminal repositories"); err != nil {
return nil, nil, nil, err
}
terminalRepoImpl, err := terminalrepo.NewWithDB(writer, reader, log)
if err := recordRequiredStore(tracker, "terminal", err); err != nil {
return nil, nil, nil, fmt.Errorf("terminal repo: %w", err)
}
if err := checkStartupContext(ctx, "quick terminal repository"); err != nil {
return nil, nil, nil, err
}
quickTerminalRepoImpl, err := quickterminalrepository.NewWithDB(writer, reader)
if err := recordRequiredStore(tracker, "quick-terminal", err); err != nil {
return nil, nil, nil, fmt.Errorf("quick terminal repo: %w", err)
}
if err := checkStartupContext(ctx, "runtime flags repository"); err != nil {
return nil, nil, nil, err
}
runtimeFlagsStore, err := runtimeflags.NewSQLiteStore(writer, reader)
if err := recordRequiredStore(tracker, "runtime-flags", err); err != nil {
return nil, nil, nil, fmt.Errorf("runtime flags store: %w", err)
}
if err := checkStartupContext(ctx, "auth repository"); err != nil {
return nil, nil, nil, err
}
authRepo, err := authstore.New(writer, reader)
if err := recordRequiredStore(tracker, "auth", err); err != nil {
return nil, nil, nil, fmt.Errorf("auth store: %w", err)
}
if err := checkStartupContext(ctx, "secret repository"); err != nil {
return nil, nil, nil, err
}
masterKeyProvider, err := secrets.NewMasterKeyProvider(cfg.ResolvedDataDir())
if err != nil {
return nil, nil, nil, fmt.Errorf("master key: %w", err)
}
if err := checkStartupContext(ctx, "secret store"); err != nil {
return nil, nil, nil, err
}
secretStore, cleanup, err := secrets.Provide(writer, reader, masterKeyProvider)
if err := recordRequiredStore(tracker, "secrets", err); err != nil {
return nil, nil, nil, fmt.Errorf("secret store: %w", err)
}
cleanups = append(cleanups, cleanup)
if err := checkStartupContext(ctx, "system settings and hostname repositories"); err != nil {
return nil, nil, nil, err
}
systemSettings, err := systemsettings.NewStore(pool)
if err := recordRequiredStore(tracker, "system-settings", err); err != nil {
return nil, nil, nil, fmt.Errorf("system settings store: %w", err)
}
if err := checkStartupContext(ctx, "hostname repository"); err != nil {
return nil, nil, nil, err
}
hostnameCache, err := hostnames.NewStore(writer, reader)
if err := recordRequiredStore(tracker, "auth-hostnames", err); err != nil {
return nil, nil, nil, fmt.Errorf("hostname cache store: %w", err)
}
if err := checkStartupContext(ctx, "telemetry contract repository"); err != nil {
return nil, nil, nil, err
}
telemetryStore, err := telemetrycontract.NewWithDB(writer, reader)
if err := recordRequiredStore(tracker, "telemetry-contract", err); err != nil {
return nil, nil, nil, fmt.Errorf("telemetry contract store: %w", err)
@@ -143,9 +208,21 @@ func provideRepositories(ctx context.Context, cfg *config.Config, log *logger.Lo
HostnameCache: hostnameCache,
SystemSettings: systemSettings,
}
if err := ctx.Err(); err != nil {
return nil, nil, nil, err
}
initialized = true
startup.SetPhase(ctx, startup.InitializingServices)
return pool, repos, cleanups, nil
}
func checkStartupContext(ctx context.Context, stage string) error {
if err := ctx.Err(); err != nil {
return fmt.Errorf("startup canceled before %s: %w", stage, err)
}
return nil
}
// supportRepositorySet groups the lighter-weight support repositories
// that share a common (writer, reader) wire-up pattern.
type supportRepositorySet struct {
@@ -164,41 +241,56 @@ func provideSupportRepos(ctx context.Context, writer, reader *sqlx.DB, tracker *
var cleanups []func() error
var repos supportRepositorySet
userRepo, cleanup, err := userstore.Provide(writer, reader)
if err := recordRequiredStore(tracker, "user", err); err != nil {
return repos, nil, err
if err := checkStartupContext(ctx, "user repository"); err != nil {
return repos, cleanups, err
}
userRepo, cleanup, err := userstore.Provide(writer, reader)
cleanups = append(cleanups, cleanup)
if err := recordRequiredStore(tracker, "user", err); err != nil {
return repos, cleanups, err
}
repos.user = userRepo
// Same concrete store, account-management view (used by internal/auth).
repos.userAccounts = userRepo
notificationRepo, cleanup, err := notificationstore.Provide(ctx, writer, reader)
if err := recordRequiredStore(tracker, "notification", err); err != nil {
return repos, nil, err
if err := checkStartupContext(ctx, "notification repository"); err != nil {
return repos, cleanups, err
}
notificationRepo, cleanup, err := notificationstore.Provide(ctx, writer, reader)
cleanups = append(cleanups, cleanup)
if err := recordRequiredStore(tracker, "notification", err); err != nil {
return repos, cleanups, err
}
repos.notification = notificationRepo
editorRepo, cleanup, err := editorstore.Provide(writer, reader)
if err := recordRequiredStore(tracker, "editor", err); err != nil {
return repos, nil, err
if err := checkStartupContext(ctx, "editor repository"); err != nil {
return repos, cleanups, err
}
editorRepo, cleanup, err := editorstore.Provide(writer, reader)
cleanups = append(cleanups, cleanup)
if err := recordRequiredStore(tracker, "editor", err); err != nil {
return repos, cleanups, err
}
repos.editor = editorRepo
promptRepo, cleanup, err := promptstore.Provide(writer, reader)
if err := recordRequiredStore(tracker, "prompts", err); err != nil {
return repos, nil, err
if err := checkStartupContext(ctx, "prompt repository"); err != nil {
return repos, cleanups, err
}
promptRepo, cleanup, err := promptstore.Provide(writer, reader)
cleanups = append(cleanups, cleanup)
if err := recordRequiredStore(tracker, "prompts", err); err != nil {
return repos, cleanups, err
}
repos.prompts = promptRepo
utilityRepo, cleanup, err := utilitystore.Provide(writer, reader)
if err := recordRequiredStore(tracker, "utility", err); err != nil {
return repos, nil, err
if err := checkStartupContext(ctx, "utility repository"); err != nil {
return repos, cleanups, err
}
utilityRepo, cleanup, err := utilitystore.Provide(writer, reader)
cleanups = append(cleanups, cleanup)
if err := recordRequiredStore(tracker, "utility", err); err != nil {
return repos, cleanups, err
}
repos.utility = utilityRepo
return repos, cleanups, nil
apps/backend/internal/db/migratelog.go
+70 -14
View File
@@ -1,6 +1,8 @@
package db
import (
"context"
"errors"
"fmt"
"github.com/jmoiron/sqlx"
@@ -17,6 +19,7 @@ type MigrateLogger struct {
db *sqlx.DB
log *logger.Logger
strict bool
ctx context.Context
firstErr error
}
@@ -33,8 +36,24 @@ func NewRequiredMigrateLogger(db *sqlx.DB, log *logger.Logger) *MigrateLogger {
return newMigrateLogger(db, log, true)
}
// NewRequiredMigrateLoggerContext creates a required migration logger whose
// statements observe ctx. The context is checked between migration steps and
// by the SQL driver while a statement is running. Once a required statement
// fails, later Apply calls become admission no-ops until the owning
// constructor returns the first error.
func NewRequiredMigrateLoggerContext(db *sqlx.DB, log *logger.Logger, ctx context.Context) *MigrateLogger {
if ctx == nil {
ctx = context.Background()
}
return newMigrateLoggerContext(db, log, true, ctx)
}
func newMigrateLogger(db *sqlx.DB, log *logger.Logger, strict bool) *MigrateLogger {
return &MigrateLogger{db: db, log: log, strict: strict}
return newMigrateLoggerContext(db, log, strict, context.Background())
}
func newMigrateLoggerContext(db *sqlx.DB, log *logger.Logger, strict bool, ctx context.Context) *MigrateLogger {
return &MigrateLogger{db: db, log: log, strict: strict, ctx: ctx}
}
// Apply executes stmt and classifies the result:
@@ -45,22 +64,34 @@ func newMigrateLogger(db *sqlx.DB, log *logger.Logger, strict bool) *MigrateLogg
// In strict mode, the first unexpected error is retained and returned. In
// compatibility mode, unexpected errors are logged and swallowed as before.
func (m *MigrateLogger) Apply(name, stmt string) error {
if _, err := m.db.Exec(stmt); err != nil {
return m.ApplyContext(m.ctx, name, stmt)
}
// ApplyContext executes a migration statement with an explicit context.
func (m *MigrateLogger) ApplyContext(ctx context.Context, name, stmt string) error {
if m.strict && m.firstErr != nil {
return m.firstErr
}
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return m.recordFailure(name, err)
}
if _, err := m.db.ExecContext(ctx, stmt); err != nil {
if ctxErr := ctx.Err(); ctxErr != nil {
return m.recordFailure(name, errors.Join(err, ctxErr))
}
if IsAlreadyExistsError(err) {
return nil
}
wrapped := fmt.Errorf("migration %q failed: %w", name, err)
if m.strict && m.firstErr == nil {
m.firstErr = wrapped
}
if m.log != nil {
m.log.Warn("migration failed",
zap.String("name", name), zap.Error(wrapped))
}
if m.strict {
return wrapped
}
return nil
return m.recordFailure(name, err)
}
// SQLite can finish an interruptible statement successfully at the same
// time that its context is canceled. Treat cancellation observed at this
// boundary as a failed migration so the next store is never admitted.
if err := ctx.Err(); err != nil {
return m.recordFailure(name, err)
}
if m.log != nil {
m.log.Info("migration applied", zap.String("name", name))
@@ -68,6 +99,31 @@ func (m *MigrateLogger) Apply(name, stmt string) error {
return nil
}
func (m *MigrateLogger) recordFailure(name string, err error) error {
wrapped := fmt.Errorf("migration %q failed: %w", name, err)
if m.strict && m.firstErr == nil {
m.firstErr = wrapped
}
if m.log != nil {
m.log.Warn("migration failed",
zap.String("name", name), zap.Error(wrapped))
}
if m.strict {
return wrapped
}
return nil
}
// Context returns the context used by this migration logger. It is used by
// legacy migration helpers that need to switch a long-running query to
// ExecContext without changing their public constructor signatures.
func (m *MigrateLogger) Context() context.Context {
if m.ctx == nil {
return context.Background()
}
return m.ctx
}
// Err returns the first unexpected migration failure observed by this
// logger. Idempotent duplicate errors are not reported.
func (m *MigrateLogger) Err() error {
apps/backend/internal/db/migratelog_test.go
+71 -1
View File
@@ -1,10 +1,14 @@
package db
import (
"context"
"errors"
"sync"
"testing"
"time"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"github.com/mattn/go-sqlite3"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"go.uber.org/zap/zaptest/observer"
@@ -139,3 +143,69 @@ func TestMigrateLogger_Apply_NilLog(t *testing.T) {
m.Apply("t.col", `ALTER TABLE t ADD COLUMN col TEXT DEFAULT ''`) // idempotent
m.Apply("bad", `ALTER TABLE missing ADD COLUMN x TEXT`)
}
// TestRequiredMigrateLoggerContextCancelsActiveStatement proves cancellation
// reaches a migration after the driver has started executing it. The custom
// SQLite function gives the test a deterministic statement boundary instead
// of relying on a scheduler sleep or a large-data timing guess.
func TestRequiredMigrateLoggerContextCancelsActiveStatement(t *testing.T) {
db := memDB(t)
db.SetMaxOpenConns(1)
if _, err := db.Exec(`CREATE TABLE t (value INTEGER NOT NULL)`); err != nil {
t.Fatalf("create table: %v", err)
}
if _, err := db.Exec(`INSERT INTO t(value) VALUES (0)`); err != nil {
t.Fatalf("insert row: %v", err)
}
started := make(chan struct{})
release := make(chan struct{})
var startedOnce sync.Once
conn, err := db.Conn(context.Background())
if err != nil {
t.Fatalf("reserve SQLite connection: %v", err)
}
if err := conn.Raw(func(driverConn any) error {
sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn)
if !ok {
t.Fatalf("driver connection = %T, want *sqlite3.SQLiteConn", driverConn)
}
return sqliteConn.RegisterFunc("wait_for_release", func() int64 {
startedOnce.Do(func() { close(started) })
<-release
return 1
}, true)
}); err != nil {
_ = conn.Close()
t.Fatalf("register SQLite function: %v", err)
}
if err := conn.Close(); err != nil {
t.Fatalf("release SQLite connection: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
done <- NewRequiredMigrateLoggerContext(db, nil, ctx).ApplyContext(
ctx,
"active.statement",
`UPDATE t SET value = wait_for_release()`,
)
}()
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("migration statement did not enter the SQLite function")
}
cancel()
close(release)
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("active migration error = %v, want context.Canceled", err)
}
case <-time.After(2 * time.Second):
t.Fatal("active migration did not stop after cancellation")
}
}
apps/backend/internal/db/schema.go
+20
View File
@@ -98,6 +98,26 @@ func ColumnExists(conn SchemaQuerier, table, column string) (bool, error) {
return columns[column], nil
}
// ColumnExistsContext is the cancellation-aware form of ColumnExists. It uses
// a point-existence query instead of loading every column name, which keeps
// startup schema probes bounded even on a wide catalog.
func ColumnExistsContext(ctx context.Context, conn ContextSchemaQuerier, table, column string) (bool, error) {
query := `SELECT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = ? AND column_name = ?
)`
if !isPostgresSchema(conn) {
query = `SELECT EXISTS (
SELECT 1 FROM pragma_table_info(?) WHERE name = ?
)`
}
var exists bool
if err := conn.QueryRowContext(ctx, conn.Rebind(query), table, column).Scan(&exists); err != nil {
return false, err
}
return exists, nil
}
func isPostgresSchema(conn SchemaQuerier) bool {
return conn.DriverName() == "pgx"
}
apps/backend/internal/db/sqlguard/exemptions.json
+1 -4
View File
@@ -17,6 +17,7 @@
{"file": "internal/backendapp/e2e_reset.go", "symbol": "handleE2ECreateAutomation", "rule": "raw-placeholder", "reason": "The E2E seed endpoint operates on the SQLite task repository"},
{"file": "internal/db/schema.go", "symbol": "tableExistsQuery", "rule": "sqlite-catalog", "reason": "This is the central dialect-aware schema inspection helper"},
{"file": "internal/db/schema.go", "symbol": "TableColumns", "rule": "sqlite-catalog", "reason": "This is the central dialect-aware schema inspection helper"},
{"file": "internal/db/schema.go", "symbol": "ColumnExistsContext", "rule": "sqlite-catalog", "reason": "This is the central dialect-aware schema inspection helper"},
{"file": "internal/delivery/store.go", "symbol": "tableExists", "rule": "sqlite-catalog", "reason": "The SQLite branch is the explicit dialect boundary for the schema probe"},
{"file": "internal/delivery/store.go", "symbol": "tableExists", "rule": "raw-placeholder", "reason": "The placeholder is in the explicit SQLite branch of the schema probe"},
{"file": "internal/github/store.go", "symbol": "createTablesSQL", "rule": "boolean-integer", "reason": "The provider schema template is rendered by dialect.MustRenderSchema"},
@@ -96,10 +97,6 @@
{"file": "internal/task/repository/sqlite/base_migrations.go", "symbol": "backfillExecutorsRunningFromTaskSessions", "rule": "raw-placeholder", "reason": "This backfill is a SQLite-only legacy migration"},
{"file": "internal/task/repository/sqlite/base_migrations.go", "symbol": "healDuplicateTaskEnvForTask", "rule": "raw-placeholder", "reason": "This repair is a SQLite-only legacy migration"},
{"file": "internal/task/repository/sqlite/session.go", "symbol": "DismissLastAgentError", "rule": "sqlite-date-function", "reason": "The task SQLite repository owns this SQLite date expression"},
{"file": "internal/task/repository/sqlite/worktree_ownership_migration.go", "symbol": "tableExists", "rule": "sqlite-catalog", "reason": "SQLite-only worktree ownership migration"},
{"file": "internal/task/repository/sqlite/worktree_ownership_migration.go", "symbol": "tableExists", "rule": "raw-placeholder", "reason": "SQLite-only worktree ownership migration"},
{"file": "internal/task/repository/sqlite/worktree_ownership_migration.go", "symbol": "columnExists", "rule": "sqlite-catalog", "reason": "SQLite-only worktree ownership migration"},
{"file": "internal/task/repository/sqlite/worktree_ownership_migration.go", "symbol": "columnExists", "rule": "raw-placeholder", "reason": "SQLite-only worktree ownership migration"},
{"file": "internal/task/repository/sqlite/worktree_ownership_targets.go", "symbol": "tableColumns", "rule": "sqlite-catalog", "reason": "SQLite-only worktree ownership migration"},
{"file": "internal/task/repository/sqlite/worktree_ownership_targets.go", "symbol": "tableColumns", "rule": "raw-placeholder", "reason": "SQLite-only worktree ownership migration"},
{"file": "internal/workflow/repository/phase2_sqlite.go", "symbol": "backfillParticipantsCreatedAtFromRowid", "rule": "sqlite-date-function", "reason": "Guarded by dialect.IsPostgres and returns before this statement on every non-SQLite driver"},
apps/backend/internal/gateway/websocket/agent_runtime_notifications_test.go
+53
View File
@@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/kandev/kandev/internal/auth/authn"
"github.com/kandev/kandev/internal/events"
"github.com/kandev/kandev/internal/events/bus"
storagepkg "github.com/kandev/kandev/internal/system/storage"
@@ -78,6 +79,58 @@ func TestSystemEventBroadcasterCloseIsIdempotent(t *testing.T) {
eventBus.Close()
}
func TestSystemEventBroadcasterSurvivesWorkerCancellation(t *testing.T) {
log := testLoggerForUserNotifications(t)
eventBus := bus.NewMemoryEventBus(log)
processCtx, cancelProcess := context.WithCancel(context.Background())
t.Cleanup(cancelProcess)
workerCtx, cancelWorker := context.WithCancel(processCtx)
hub := newTestHub(t)
hubDone := make(chan struct{})
go func() {
defer close(hubDone)
hub.Run(processCtx)
}()
t.Cleanup(func() {
cancelProcess()
<-hubDone
eventBus.Close()
})
broadcaster := RegisterSystemNotifications(processCtx, eventBus, hub, log)
t.Cleanup(broadcaster.Close)
client := NewClient("restore-auth", authn.Identity{UserID: "restore-user", Role: authn.RoleMember}, nil, hub, log)
registerTestClient(hub, client)
// Restore quiescing cancels worker-owned services. The process context
// still owns the authenticated WebSocket and system-job broadcaster until
// the terminal restart-required result is delivered.
cancelWorker()
if workerCtx.Err() != context.Canceled {
t.Fatalf("worker context error = %v, want context.Canceled", workerCtx.Err())
}
if err := eventBus.Publish(processCtx, events.SystemJobUpdate,
bus.NewEvent(events.SystemJobUpdate, "restore", map[string]any{
"job_id": "restore-job",
"state": "succeeded",
})); err != nil {
t.Fatalf("publish restore result: %v", err)
}
select {
case frame := <-client.send:
var message ws.Message
if err := json.Unmarshal(frame, &message); err != nil {
t.Fatalf("decode restore notification: %v", err)
}
if message.Action != ws.ActionSystemJobUpdate {
t.Fatalf("restore notification action = %q, want %q", message.Action, ws.ActionSystemJobUpdate)
}
case <-time.After(time.Second):
t.Fatal("timed out waiting for restore notification after worker cancellation")
}
}
func TestSystemStorageAnalysisNotificationsBroadcastProgressState(t *testing.T) {
hub := newTestHub(t)
ctx, cancel := context.WithCancel(context.Background())
apps/backend/internal/launcher/dev.go
+1
View File
@@ -73,6 +73,7 @@ func runDev(ctx context.Context, opts Options, build BuildInfo) int {
}
if err := waitForReadyFn(ctx, cfg.ports.BackendURL, backend); err != nil {
supervisor.shutdown("backend readiness failure")
dumpLogs()
fmt.Fprintln(os.Stderr, "[kandev] "+err.Error())
return 1
}
apps/backend/internal/launcher/health.go
+39 -7
View File
@@ -2,6 +2,7 @@ package launcher
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -14,6 +15,7 @@ import (
"time"
"github.com/kandev/kandev/internal/common/config"
"github.com/kandev/kandev/internal/startup"
)
const (
@@ -362,16 +364,32 @@ func safeProbeError(err error) string {
// exists to fix.
func waitForReady(ctx context.Context, baseURL string, proc childState) error {
readyURL := baseURL + "/ready"
var last startup.Snapshot
var reported time.Time
exitError := func(code int) error {
if label := last.Phase.Label(); label != "" {
return fmt.Errorf("backend exited (code %d) during %s (elapsed %.1fs); check backend logs for the initialization error", code, label, float64(last.ElapsedMS)/1000)
}
return fmt.Errorf("backend exited (code %d) before it reported ready; check backend logs for the initialization error", code)
}
for ctx.Err() == nil {
if exited, code := proc.Exited(); exited {
return fmt.Errorf("backend exited (code %d) before it reported ready", code)
return exitError(code)
}
if probeReady(ctx, readyURL) {
isReady, snapshot := probeReadyStatus(ctx, readyURL)
if snapshot.Phase.Label() != "" {
if snapshot.Phase != last.Phase || time.Since(reported) >= 15*time.Second {
fmt.Fprintf(os.Stderr, "[kandev] %s (elapsed %.1fs; phase %.1fs)\n", snapshot.Phase.Label(), float64(snapshot.ElapsedMS)/1000, float64(snapshot.PhaseElapsedMS)/1000)
reported = time.Now()
}
last = snapshot
}
if isReady {
if err := ctx.Err(); err != nil {
return fmt.Errorf("backend readiness wait canceled at %s: %w", readyURL, err)
}
if exited, code := proc.Exited(); exited {
return fmt.Errorf("backend exited (code %d) before it reported ready", code)
return exitError(code)
}
return nil
}
@@ -388,17 +406,31 @@ func waitForReady(ctx context.Context, baseURL string, proc childState) error {
// sets X-Kandev-Desktop-Health-Token (that header is /health-only, see
// docs/specs/health-endpoint-version/spec.md AC-21).
func probeReady(ctx context.Context, readyURL string) bool {
ready, _ := probeReadyStatus(ctx, readyURL)
return ready
}
func probeReadyStatus(ctx context.Context, readyURL string) (bool, startup.Snapshot) {
var body struct {
Startup startup.Snapshot `json:"startup"`
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, readyURL, nil)
if err != nil {
return false
return false, body.Startup
}
resp, err := healthProbeClient.Do(req)
if err != nil {
return false
return false, body.Startup
}
defer func() { _ = resp.Body.Close() }()
_, _ = io.Copy(io.Discard, resp.Body)
return resp.StatusCode >= 200 && resp.StatusCode < 300
if resp.StatusCode == http.StatusServiceUnavailable {
if err := json.NewDecoder(io.LimitReader(resp.Body, 8192)).Decode(&body); err != nil || body.Startup.Phase.Label() == "" || body.Startup.ElapsedMS < 0 || body.Startup.PhaseElapsedMS < 0 {
body.Startup = startup.Snapshot{}
}
} else {
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 8192))
}
return resp.StatusCode >= 200 && resp.StatusCode < 300, body.Startup
}
func healthPort(baseURL string) string {
apps/backend/internal/launcher/health_test.go
+81
View File
@@ -461,6 +461,72 @@ func TestWaitForReadyReturnsNilOnReadyResponse(t *testing.T) {
}
}
func TestWaitForHealthThenReadyKeepsDelayedBootstrapAlive(t *testing.T) {
const healthToken = "delayed-bootstrap-token"
var initialized atomic.Bool
readyStarted := make(chan struct{})
var readyStartedOnce sync.Once
var readyCalls atomic.Int32
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/health":
w.Header().Set("X-Kandev-Desktop-Health-Token", healthToken)
w.WriteHeader(http.StatusOK)
case "/ready":
readyCalls.Add(1)
readyStartedOnce.Do(func() { close(readyStarted) })
if !initialized.Load() {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"startup":{"phase":"opening_database","elapsed_ms":1,"phase_elapsed_ms":1}}`))
return
}
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"status":"ok"}`))
default:
http.NotFound(w, r)
}
}))
t.Cleanup(srv.Close)
// The launcher must decide that the process is healthy within this short
// budget, then keep waiting for readiness while initialization remains
// gated beyond that budget.
if readyURL, err := waitForHealth(context.Background(), singleHealthTarget(srv.URL), fakeChild{}, 50*time.Millisecond, healthToken, nil); err != nil {
t.Fatalf("waitForHealth() = %v, want healthy bootstrap", err)
} else if readyURL != srv.URL {
t.Fatalf("waitForHealth() URL = %q, want %q", readyURL, srv.URL)
}
readyCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
readyDone := make(chan error, 1)
go func() { readyDone <- waitForReady(readyCtx, srv.URL, fakeChild{}) }()
select {
case <-readyStarted:
case <-time.After(time.Second):
t.Fatal("waitForReady did not probe the bootstrap readiness endpoint")
}
select {
case err := <-readyDone:
t.Fatalf("waitForReady returned before initialization release: %v", err)
case <-time.After(100 * time.Millisecond):
}
initialized.Store(true)
select {
case err := <-readyDone:
if err != nil {
t.Fatalf("waitForReady after initialization release: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("waitForReady did not observe the released readiness endpoint")
}
if readyCalls.Load() < 1 {
t.Fatal("waitForReady did not issue a readiness request")
}
}
func TestWaitForReadyRejectsChildExitDuringSuccessfulProbe(t *testing.T) {
requestStarted := make(chan struct{})
responseRelease := make(chan struct{})
@@ -617,3 +683,18 @@ func TestWaitForHealthFailsFastWhenBackendExited(t *testing.T) {
t.Fatalf("onFailure called %d times, want 1", failures)
}
}
func TestReadinessExitIncludesLastStartupPhase(t *testing.T) {
var child toggledChild
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"status":"starting","startup":{"phase":"backing_up_database","elapsed_ms":1234,"phase_elapsed_ms":1000}}`))
child.code.Store(3)
child.exited.Store(true)
}))
defer srv.Close()
err := waitForReady(context.Background(), srv.URL, &child)
if err == nil || !strings.Contains(err.Error(), "Backing up database") {
t.Fatalf("missing startup phase in error: %v", err)
}
}
apps/backend/internal/launcher/start.go
+1
View File
@@ -179,6 +179,7 @@ func runManagedApp(ctx context.Context, cfg managedAppConfig) int {
shutdownDebugf("runManagedApp backend healthy, waiting for readiness")
if err := waitForReadyFn(ctx, cfg.Ports.BackendURL, backend); err != nil {
supervisor.shutdown("backend readiness failure")
dumpLogs()
fmt.Fprintln(os.Stderr, "[kandev] "+err.Error())
return 1
}
apps/backend/internal/persistence/provider.go
+49 -5
View File
@@ -1,6 +1,7 @@
package persistence
import (
"context"
"fmt"
"os"
"path/filepath"
@@ -12,6 +13,7 @@ import (
"github.com/kandev/kandev/internal/common/config"
"github.com/kandev/kandev/internal/common/logger"
"github.com/kandev/kandev/internal/db"
"github.com/kandev/kandev/internal/startup"
)
// Provide creates the database connection pool used by repositories.
@@ -23,6 +25,15 @@ import (
// compared against the stored kandev_version to decide whether to take a
// pre-migration backup. Pass "" in tests that do not care about snapshots.
func Provide(cfg *config.Config, log *logger.Logger, version string) (*db.Pool, func() error, error) {
return ProvideContext(context.Background(), cfg, log, version)
}
// ProvideContext opens persistence for a cancellable backend initialization.
func ProvideContext(ctx context.Context, cfg *config.Config, log *logger.Logger, version string) (*db.Pool, func() error, error) {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
startup.SetPhase(ctx, startup.OpeningDatabase)
driver := cfg.Database.Driver
if driver == "" {
driver = "sqlite"
@@ -30,7 +41,7 @@ func Provide(cfg *config.Config, log *logger.Logger, version string) (*db.Pool,
switch driver {
case "sqlite":
return provideSQLite(cfg, log, version)
return provideSQLite(ctx, cfg, log, version)
case "postgres":
return providePostgres(cfg, log)
default:
@@ -38,7 +49,7 @@ func Provide(cfg *config.Config, log *logger.Logger, version string) (*db.Pool,
}
}
func provideSQLite(cfg *config.Config, log *logger.Logger, version string) (*db.Pool, func() error, error) {
func provideSQLite(ctx context.Context, cfg *config.Config, log *logger.Logger, version string) (*db.Pool, func() error, error) {
selection, err := selectSQLiteDatabase(cfg, log)
if err != nil {
return nil, nil, fmt.Errorf("select sqlite database: %w", err)
@@ -87,16 +98,17 @@ func provideSQLite(cfg *config.Config, log *logger.Logger, version string) (*db.
}
if shouldBackup(storedVersion, version, userTables) {
startup.SetPhase(ctx, startup.BackingUpDatabase)
started := time.Now()
backupDir := filepath.Join(filepath.Dir(dbPath), "backups")
if err := os.MkdirAll(backupDir, 0o755); err != nil {
_ = pool.Close()
return nil, nil, fmt.Errorf("create backup dir: %w", err)
}
path := snapshotPath(backupDir, storedVersion)
size, err := snapshotSQLite(writer, path)
path, size, err := createPreMigrationBackup(ctx, writer, backupDir, storedVersion)
if err != nil {
_ = pool.Close()
return nil, nil, fmt.Errorf("pre-migration backup failed: %w", err)
return nil, nil, err
}
if log != nil {
log.Info("pre-migration backup taken",
@@ -104,6 +116,7 @@ func provideSQLite(cfg *config.Config, log *logger.Logger, version string) (*db.
zap.String("to_version", version),
zap.String("path", path),
zap.Int64("size_bytes", size),
zap.Duration("duration", time.Since(started)),
)
}
_ = pruneBackups(backupDir, 2)
@@ -128,6 +141,37 @@ func provideSQLite(cfg *config.Config, log *logger.Logger, version string) (*db.
return pool, cleanup, nil
}
func createPreMigrationBackup(ctx context.Context, writer *sqlx.DB, backupDir, storedVersion string) (string, int64, error) {
path := snapshotPath(backupDir, storedVersion)
stagingDir, err := os.MkdirTemp(backupDir, ".kandev-backup-*")
if err != nil {
return "", 0, fmt.Errorf("create private backup staging directory: %w", err)
}
defer func() { _ = os.RemoveAll(stagingDir) }()
stagedPath := filepath.Join(stagingDir, filepath.Base(path))
size, err := snapshotSQLiteContext(ctx, writer, stagedPath)
if err != nil {
return "", 0, fmt.Errorf("pre-migration backup failed: %w", err)
}
if err := ctx.Err(); err != nil {
return "", 0, fmt.Errorf("startup canceled after pre-migration backup: %w", err)
}
if err := os.Chmod(stagedPath, 0o600); err != nil {
return "", 0, fmt.Errorf("protect staged pre-migration backup: %w", err)
}
if _, err := inspectSQLiteCandidate(stagedPath); err != nil {
return "", 0, fmt.Errorf("validate staged pre-migration backup: %w", err)
}
if err := ctx.Err(); err != nil {
return "", 0, fmt.Errorf("startup canceled before installing pre-migration backup: %w", err)
}
if err := installStagedSQLite(stagedPath, path); err != nil {
return "", 0, fmt.Errorf("install pre-migration backup: %w", err)
}
return path, size, nil
}
func providePostgres(cfg *config.Config, log *logger.Logger) (*db.Pool, func() error, error) {
dsn := cfg.Database.DSN()
dbConn, err := db.OpenPostgres(dsn, cfg.Database.MaxConns, cfg.Database.MinConns)
apps/backend/internal/persistence/snapshot.go
+27 -1
View File
@@ -1,6 +1,7 @@
package persistence
import (
"context"
"fmt"
"os"
"path/filepath"
@@ -36,13 +37,38 @@ func SnapshotSQLite(writer *sqlx.DB, path string) (int64, error) {
// which produces a clean, defragmented snapshot including all WAL frames.
// Returns the size of the created file in bytes.
func snapshotSQLite(writer *sqlx.DB, path string) (int64, error) {
if _, err := writer.Exec(`VACUUM INTO ?`, path); err != nil {
return snapshotSQLiteContext(context.Background(), writer, path)
}
func snapshotSQLiteContext(ctx context.Context, writer *sqlx.DB, path string) (int64, error) {
_, statErr := os.Lstat(path)
if statErr != nil && !os.IsNotExist(statErr) {
return 0, fmt.Errorf("inspect snapshot destination %s: %w", path, statErr)
}
pathExisted := statErr == nil
installed := false
defer func() {
if !installed && !pathExisted {
// VACUUM INTO writes the destination incrementally. A canceled or
// failed statement can leave a partial SQLite file behind, which
// must not be mistaken for a usable backup on retry.
_ = os.Remove(path)
}
}()
if _, err := writer.ExecContext(ctx, `VACUUM INTO ?`, path); err != nil {
return 0, fmt.Errorf("vacuum into %s: %w", path, err)
}
if err := ctx.Err(); err != nil {
return 0, fmt.Errorf("snapshot canceled after vacuum into %s: %w", path, err)
}
info, err := os.Stat(path)
if err != nil {
return 0, fmt.Errorf("stat snapshot %s: %w", path, err)
}
if err := ctx.Err(); err != nil {
return 0, fmt.Errorf("snapshot canceled before installing %s: %w", path, err)
}
installed = true
return info.Size(), nil
}
apps/backend/internal/persistence/snapshot_test.go
+21
View File
@@ -1,6 +1,7 @@
package persistence
import (
"context"
"os"
"path/filepath"
"testing"
@@ -59,6 +60,26 @@ func TestSnapshotSQLite_CreatesReadableCopy(t *testing.T) {
}
}
func TestSnapshotSQLite_CanceledRemovesDestination(t *testing.T) {
dir := t.TempDir()
srcPath := filepath.Join(dir, "src.db")
dstPath := filepath.Join(dir, "snap.db")
src := fileSQLiteDB(t, srcPath)
if _, err := src.Exec(`CREATE TABLE things (id INTEGER PRIMARY KEY, name TEXT)`); err != nil {
t.Fatalf("create table: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := snapshotSQLiteContext(ctx, src, dstPath); err == nil {
t.Fatal("canceled snapshot succeeded")
}
if _, err := os.Stat(dstPath); !os.IsNotExist(err) {
t.Fatalf("canceled snapshot destination stat error = %v, want not exist", err)
}
}
// TestPruneBackups verifies that pruneBackups keeps exactly the N newest files.
func TestPruneBackups_KeepsNewest(t *testing.T) {
dir := t.TempDir()
apps/backend/internal/persistence/upgrade_test.go
+7
View File
@@ -82,6 +82,13 @@ func TestProvide_TakesSnapshotOnUpgrade(t *testing.T) {
// The backup should be a readable SQLite DB containing the seeded row.
snapPath := filepath.Join(backupDir, entries[0].Name())
info, err := os.Stat(snapPath)
if err != nil {
t.Fatalf("stat snapshot: %v", err)
}
if got := info.Mode().Perm(); got != 0o600 {
t.Fatalf("snapshot mode = %#o, want 0600", got)
}
snap, err := sqlx.Open("sqlite3", snapPath)
if err != nil {
t.Fatalf("open snapshot: %v", err)
apps/backend/internal/startup/progress.go
+100
View File
@@ -0,0 +1,100 @@
// Package startup reports process initialization without exposing private diagnostics.
package startup
import (
"context"
"sync"
"time"
"github.com/kandev/kandev/internal/common/logger"
"go.uber.org/zap"
)
type Phase string
const (
OpeningDatabase Phase = "opening_database"
BackingUpDatabase Phase = "backing_up_database"
ApplyingMigrations Phase = "applying_migrations"
InitializingServices Phase = "initializing_services"
RecoveringSessions Phase = "recovering_sessions"
Ready Phase = "ready"
)
// Label accepts only protocol-owned phase names, never arbitrary server text.
func (p Phase) Label() string {
switch p {
case OpeningDatabase:
return "Opening database"
case BackingUpDatabase:
return "Backing up database"
case ApplyingMigrations:
return "Applying migrations"
case InitializingServices:
return "Initializing services"
case RecoveringSessions:
return "Recovering sessions"
case Ready:
return "Ready"
default:
return ""
}
}
type Snapshot struct {
Phase Phase `json:"phase"`
ElapsedMS int64 `json:"elapsed_ms"`
PhaseElapsedMS int64 `json:"phase_elapsed_ms"`
}
type Reporter struct {
mu sync.Mutex
started, phaseStarted time.Time
phase Phase
log *logger.Logger
}
type contextKey struct{}
func New(log *logger.Logger) *Reporter {
now := time.Now()
return &Reporter{started: now, phaseStarted: now, phase: OpeningDatabase, log: log}
}
func WithReporter(ctx context.Context, r *Reporter) context.Context {
return context.WithValue(ctx, contextKey{}, r)
}
func FromContext(ctx context.Context) *Reporter {
r, _ := ctx.Value(contextKey{}).(*Reporter)
return r
}
func SetPhase(ctx context.Context, phase Phase) {
if r := FromContext(ctx); r != nil {
r.Set(phase)
}
}
func (r *Reporter) Snapshot() Snapshot {
r.mu.Lock()
defer r.mu.Unlock()
return Snapshot{Phase: r.phase, ElapsedMS: time.Since(r.started).Milliseconds(), PhaseElapsedMS: time.Since(r.phaseStarted).Milliseconds()}
}
func (r *Reporter) Set(phase Phase) {
r.mu.Lock()
defer r.mu.Unlock()
if r.phase == phase {
return
}
if r.log != nil {
r.log.Info("Startup phase completed", zap.String("phase", string(r.phase)), zap.Duration("duration", time.Since(r.phaseStarted)), zap.Duration("elapsed", time.Since(r.started)))
}
r.phase = phase
r.phaseStarted = time.Now()
if r.log != nil {
r.log.Info("Startup phase started", zap.String("phase", string(phase)), zap.Duration("elapsed", time.Since(r.started)))
}
}
func (r *Reporter) Report() {
s := r.Snapshot()
if r.log != nil && s.Phase != Ready {
r.log.Info("Startup in progress", zap.String("phase", string(s.Phase)), zap.Int64("elapsed_ms", s.ElapsedMS), zap.Int64("phase_elapsed_ms", s.PhaseElapsedMS))
}
}
apps/backend/internal/startup/progress_test.go
+33
View File
@@ -0,0 +1,33 @@
package startup
import (
"context"
"testing"
)
func TestReporterStartsWithOpeningDatabase(t *testing.T) {
reporter := New(nil)
snapshot := reporter.Snapshot()
if snapshot.Phase != OpeningDatabase {
t.Fatalf("initial phase = %q, want %q", snapshot.Phase, OpeningDatabase)
}
if snapshot.ElapsedMS < 0 || snapshot.PhaseElapsedMS < 0 {
t.Fatalf("negative startup durations: %#v", snapshot)
}
}
func TestReporterTransitionsAndKeepsProtocolLabels(t *testing.T) {
reporter := New(nil)
ctx := WithReporter(context.Background(), reporter)
SetPhase(ctx, BackingUpDatabase)
snapshot := reporter.Snapshot()
if snapshot.Phase != BackingUpDatabase {
t.Fatalf("phase = %q, want %q", snapshot.Phase, BackingUpDatabase)
}
if snapshot.Phase.Label() != "Backing up database" {
t.Fatalf("phase label = %q, want %q", snapshot.Phase.Label(), "Backing up database")
}
if Phase("private_error").Label() != "" {
t.Fatal("unknown phase exposed a label")
}
}
apps/backend/internal/task/repository/provider.go
+9 -1
View File
@@ -1,6 +1,8 @@
package repository
import (
"context"
"github.com/jmoiron/sqlx"
"github.com/kandev/kandev/internal/common/logger"
@@ -9,7 +11,13 @@ import (
// Provide creates the SQLite repository using separate writer and reader pools.
func Provide(writer, reader *sqlx.DB, log *logger.Logger) (*sqlite.Repository, func() error, error) {
repo, err := sqlite.NewWithDB(writer, reader, log)
return ProvideContext(context.Background(), writer, reader, log)
}
// ProvideContext creates the SQLite repository while observing startup
// cancellation between schema steps and during context-aware migrations.
func ProvideContext(ctx context.Context, writer, reader *sqlx.DB, log *logger.Logger) (*sqlite.Repository, func() error, error) {
repo, err := sqlite.NewWithDBContext(ctx, writer, reader, log)
if err != nil {
return nil, nil, err
}
apps/backend/internal/task/repository/sqlite/attachment.go
+1 -1
View File
@@ -556,7 +556,7 @@ func (r *Repository) deleteClaimedMessageAttachments(
if len(ids) == 0 {
return nil, nil
}
queueTablePresent, err := r.tableExists("queued_messages")
queueTablePresent, err := r.tableExistsContext(ctx, "queued_messages")
if err != nil {
return nil, fmt.Errorf("probe queue attachment references: %w", err)
}
apps/backend/internal/task/repository/sqlite/base.go
+29 -3
View File
@@ -108,6 +108,13 @@ func (r *Repository) nowUTC() time.Time {
return time.Now().UTC()
}
func (r *Repository) migrationContext() context.Context {
if r.migrate == nil {
return context.Background()
}
return r.migrate.Context()
}
// SetTaskQueuePurger registers the orchestrator-owned queue cleanup for
// in-memory queues. SQLite queues are also purged in the task mutation
// transaction; this callback keeps the explicitly ephemeral queue equivalent.
@@ -178,18 +185,37 @@ func (r *Repository) notifyTaskSessionQueuePurged(ctx context.Context, taskID, s
// NewWithDB creates a new SQLite repository with an existing database connection (shared ownership).
func NewWithDB(writer, reader *sqlx.DB, log *logger.Logger) (*Repository, error) {
return newRepository(writer, reader, log, false)
return NewWithDBContext(context.Background(), writer, reader, log)
}
// NewWithDBContext creates a repository with an existing database connection
// while observing cancellation at schema-step boundaries and in
// context-aware migration statements. A statement that has already entered
// the database driver can finish before that driver reports cancellation; the
// caller retains pool ownership until this constructor returns.
func NewWithDBContext(ctx context.Context, writer, reader *sqlx.DB, log *logger.Logger) (*Repository, error) {
return newRepositoryContext(ctx, writer, reader, log, false)
}
func newRepository(writer, reader *sqlx.DB, log *logger.Logger, ownsDB bool) (*Repository, error) {
return newRepositoryContext(context.Background(), writer, reader, log, ownsDB)
}
func newRepositoryContext(ctx context.Context, writer, reader *sqlx.DB, log *logger.Logger, ownsDB bool) (*Repository, error) {
if ctx == nil {
ctx = context.Background()
}
if err := ctx.Err(); err != nil {
return nil, err
}
repo := &Repository{
db: writer,
ro: reader,
ownsDB: ownsDB,
log: log,
migrate: db.NewRequiredMigrateLogger(writer, log),
migrate: db.NewRequiredMigrateLoggerContext(writer, log, ctx),
}
if err := repo.initSchema(); err != nil {
if err := repo.initSchemaContext(ctx); err != nil {
if ownsDB {
if closeErr := writer.Close(); closeErr != nil {
return nil, fmt.Errorf("failed to close database after schema error: %w", closeErr)
apps/backend/internal/task/repository/sqlite/base_migrations.go
+38 -33
View File
@@ -2,6 +2,7 @@
package sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
@@ -391,7 +392,8 @@ func (r *Repository) runMigrations() error {
return nil
}
func (r *Repository) backfillTaskSessionQueueIncarnations() error {
rows, err := r.db.Queryx(`SELECT id FROM task_sessions WHERE queue_incarnation_id = ''`)
ctx := r.migrationContext()
rows, err := r.db.QueryxContext(ctx, `SELECT id FROM task_sessions WHERE queue_incarnation_id = ''`)
if err != nil {
return fmt.Errorf("list sessions missing queue incarnation: %w", err)
}
@@ -411,7 +413,7 @@ func (r *Repository) backfillTaskSessionQueueIncarnations() error {
return fmt.Errorf("iterate sessions missing queue incarnation: %w", err)
}
for _, sessionID := range sessionIDs {
if _, err := r.db.Exec(r.db.Rebind(`
if _, err := r.db.ExecContext(ctx, r.db.Rebind(`
UPDATE task_sessions
SET queue_incarnation_id = ?
WHERE id = ? AND queue_incarnation_id = ''
@@ -440,7 +442,8 @@ func (r *Repository) backfillPromptSeq() error {
AND (%s < %s OR (%s = %s AND u.id <= task_session_messages.id))
)
WHERE author_type = 'user' AND prompt_seq = 0`, nmU, nmM, nmU, nmM)
if _, err := r.db.Exec(update); err != nil {
ctx := r.migrationContext()
if _, err := r.db.ExecContext(ctx, update); err != nil {
return fmt.Errorf("backfill prompt_seq: %w", err)
}
seed := `
@@ -449,7 +452,7 @@ func (r *Repository) backfillPromptSeq() error {
WHERE author_type = 'user' AND prompt_seq > 0
GROUP BY task_session_id
ON CONFLICT(task_session_id) DO NOTHING`
if _, err := r.db.Exec(seed); err != nil {
if _, err := r.db.ExecContext(ctx, seed); err != nil {
return fmt.Errorf("seed prompt sequence counters: %w", err)
}
return nil
@@ -602,14 +605,14 @@ func jsonBoolToInt(postgres bool, column, parent, key string) string {
// the initial implementation, reconciles legacy bootstrap duplicates, then
// enforces uniqueness only for the two hidden workflows created by this flow.
func (r *Repository) ensureImproveKandevWorkflowTemplateUniqueness() error {
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return fmt.Errorf("begin improve kandev workflow migration: %w", err)
}
// Keep this template list synchronized with every hidden Improve Kandev
// workflow bootstrapped by internal/improvekandev. Add new IDs to both this
// reconciliation query and the partial-index predicate below.
if _, err := tx.Exec(`
if _, err := tx.ExecContext(r.migrationContext(), `
UPDATE workflows
SET workflow_template_id = ''
WHERE id IN (
@@ -631,11 +634,11 @@ func (r *Repository) ensureImproveKandevWorkflowTemplateUniqueness() error {
_ = tx.Rollback()
return fmt.Errorf("reconcile improve kandev workflow duplicates: %w", err)
}
if _, err := tx.Exec(`DROP INDEX IF EXISTS uniq_workflows_workspace_template_hidden`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `DROP INDEX IF EXISTS uniq_workflows_workspace_template_hidden`); err != nil {
_ = tx.Rollback()
return fmt.Errorf("drop broad workflow template index: %w", err)
}
if _, err := tx.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS uniq_improve_kandev_workflows
if _, err := tx.ExecContext(r.migrationContext(), `CREATE UNIQUE INDEX IF NOT EXISTS uniq_improve_kandev_workflows
ON workflows(workspace_id, workflow_template_id, hidden)
WHERE workflow_template_id IN ('improve-kandev', 'report-kandev-issue')`); err != nil {
_ = tx.Rollback()
@@ -651,7 +654,7 @@ func (r *Repository) ensureImproveKandevWorkflowTemplateUniqueness() error {
// folder attachments existed. CREATE TABLE/INDEX IF NOT EXISTS is replay-safe
// on SQLite and Postgres.
func (r *Repository) ensureTaskWorkspaceFoldersSchema() error {
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_workspace_folders (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL,
@@ -676,7 +679,7 @@ func (r *Repository) ensureTaskWorkspaceFoldersSchema() error {
// existed. It replays the same DDL as the schema-init step; CREATE TABLE/INDEX
// IF NOT EXISTS is replay-safe on SQLite and Postgres.
func (r *Repository) ensureRepositorySetsSchema() error {
if _, err := r.db.Exec(repositorySetsSchemaDDL); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), repositorySetsSchemaDDL); err != nil {
return fmt.Errorf("create repository sets schema: %w", err)
}
return nil
@@ -685,7 +688,7 @@ func (r *Repository) ensureRepositorySetsSchema() error {
// ensureRepositoryBranchPoliciesSchema replays the policy DDL for databases
// created before repository branch policies existed.
func (r *Repository) ensureRepositoryBranchPoliciesSchema() error {
if _, err := r.db.Exec(repositoryBranchPoliciesSchemaDDL); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), repositoryBranchPoliciesSchemaDDL); err != nil {
return fmt.Errorf("create repository branch policies schema: %w", err)
}
return nil
@@ -705,7 +708,7 @@ func (r *Repository) recreateTable(tableName, triggerPhrase string, statements [
}
var tableSql string
err := r.db.QueryRow(`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`, tableName).Scan(&tableSql)
err := r.db.QueryRowContext(r.migrationContext(), `SELECT sql FROM sqlite_master WHERE type='table' AND name=?`, tableName).Scan(&tableSql)
if errors.Is(err, sql.ErrNoRows) {
return false, nil // Table doesn't exist yet; migration not applicable
}
@@ -716,19 +719,21 @@ func (r *Repository) recreateTable(tableName, triggerPhrase string, statements [
return false, nil // Trigger phrase absent; migration already applied or not needed
}
if _, err := r.db.Exec(`PRAGMA foreign_keys=OFF`); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), `PRAGMA foreign_keys=OFF`); err != nil {
return false, fmt.Errorf("disable foreign keys: %w", err)
}
defer func() { _, _ = r.db.Exec(`PRAGMA foreign_keys=ON`) }()
// This restoration is cleanup, so it must not inherit a canceled startup
// context and leave the shared SQLite writer with foreign-key checks off.
defer func() { _, _ = r.db.ExecContext(context.Background(), `PRAGMA foreign_keys=ON`) }()
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return false, fmt.Errorf("begin migration transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
for _, stmt := range statements {
if _, err := tx.Exec(stmt); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), stmt); err != nil {
return false, fmt.Errorf("migration %s failed: %w", tableName, err)
}
}
@@ -806,7 +811,7 @@ func (r *Repository) backfillExecutorsRunningFromTaskSessions() error {
// Check whether task_sessions still has the column. If migration already ran,
// the column is gone and there's nothing to backfill.
var tableSql string
if err := r.db.QueryRow(`SELECT sql FROM sqlite_master WHERE type='table' AND name='task_sessions'`).Scan(&tableSql); err != nil {
if err := r.db.QueryRowContext(r.migrationContext(), `SELECT sql FROM sqlite_master WHERE type='table' AND name='task_sessions'`).Scan(&tableSql); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil
}
@@ -820,7 +825,7 @@ func (r *Repository) backfillExecutorsRunningFromTaskSessions() error {
// SELECT … LEFT JOIN to find sessions with execution data but no executors_running row.
// Insert with the minimum field set; runtime/status are best-effort defaults
// (subsequent Launch / Resume will overwrite via the lifecycle manager's persistence).
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
INSERT INTO executors_running (
id, session_id, task_id, executor_id, runtime, status, resumable,
resume_token, last_message_uuid, agent_execution_id, container_id,
@@ -887,7 +892,7 @@ const commitCaptureActivatedAtMetaKey = "commit_capture_activated_at"
// exist, so a failure here must abort boot (propagated below) rather than
// leave every future commit insert failing silently forever.
func (r *Repository) migrateSessionCommitsDedupeAndActivation() error {
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
DELETE FROM task_session_commits
WHERE id NOT IN (
SELECT id FROM (
@@ -903,7 +908,7 @@ func (r *Repository) migrateSessionCommitsDedupeAndActivation() error {
`); err != nil {
return fmt.Errorf("dedupe task_session_commits: %w", err)
}
if _, err := r.db.Exec(
if _, err := r.db.ExecContext(r.migrationContext(),
`CREATE UNIQUE INDEX IF NOT EXISTS uniq_session_commits_session_sha ON task_session_commits(session_id, commit_sha)`,
); err != nil {
return fmt.Errorf("create uniq_session_commits_session_sha: %w", err)
@@ -916,7 +921,7 @@ func (r *Repository) migrateSessionCommitsDedupeAndActivation() error {
// production (persistence.Provide creates it before opening any
// repository), but repo-level tests build a bare DB via NewWithDB where
// it does not, so recreate it defensively.
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS kandev_meta (
key TEXT PRIMARY KEY,
value TEXT NOT NULL DEFAULT ''
@@ -924,7 +929,7 @@ func (r *Repository) migrateSessionCommitsDedupeAndActivation() error {
return fmt.Errorf("ensure kandev_meta: %w", err)
}
now := time.Now().UTC().Format(time.RFC3339Nano)
if _, err := r.db.Exec(r.db.Rebind(`
if _, err := r.db.ExecContext(r.migrationContext(), r.db.Rebind(`
INSERT INTO kandev_meta (key, value) VALUES (?, ?)
ON CONFLICT(key) DO NOTHING
`), commitCaptureActivatedAtMetaKey, now); err != nil {
@@ -1081,10 +1086,10 @@ func (r *Repository) migrateTaskEnvironmentReposAllowMultiBranch() error {
}
func (r *Repository) migrateTaskEnvironmentReposAllowMultiBranchPostgres() error {
if _, err := r.db.Exec(`ALTER TABLE task_environment_repos ADD COLUMN IF NOT EXISTS branch_slug TEXT NOT NULL DEFAULT ''`); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), `ALTER TABLE task_environment_repos ADD COLUMN IF NOT EXISTS branch_slug TEXT NOT NULL DEFAULT ''`); err != nil {
return fmt.Errorf("add task_environment_repos.branch_slug: %w", err)
}
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
DO $$
DECLARE
old_constraint_name text;
@@ -1254,13 +1259,13 @@ func (r *Repository) migrateSessionsRemoveWorkflowStepID() error {
// Runs before ensureTaskEnvironmentTaskUniqueIndex so the unique constraint
// can be added cleanly. Idempotent — a no-op once the data is healed.
func (r *Repository) healDuplicateTaskEnvironments() error {
tx, err := r.db.Begin()
tx, err := r.db.BeginTx(r.migrationContext(), nil)
if err != nil {
return fmt.Errorf("heal duplicate envs: begin tx: %w", err)
}
defer tx.Rollback() //nolint:errcheck
rows, err := tx.Query(`
rows, err := tx.QueryContext(r.migrationContext(), `
SELECT task_id
FROM task_environments
GROUP BY task_id
@@ -1285,7 +1290,7 @@ func (r *Repository) healDuplicateTaskEnvironments() error {
_ = rows.Close()
for _, taskID := range taskIDs {
if err := healDuplicateTaskEnvForTask(tx, taskID); err != nil {
if err := healDuplicateTaskEnvForTask(r.migrationContext(), tx, taskID); err != nil {
return err
}
}
@@ -1294,9 +1299,9 @@ func (r *Repository) healDuplicateTaskEnvironments() error {
// healDuplicateTaskEnvForTask keeps the most recently updated env for a task,
// re-points sessions on the loser rows to the winner, then deletes losers.
func healDuplicateTaskEnvForTask(tx *sql.Tx, taskID string) error {
func healDuplicateTaskEnvForTask(ctx context.Context, tx *sql.Tx, taskID string) error {
var winnerID string
if err := tx.QueryRow(`
if err := tx.QueryRowContext(ctx, `
SELECT id FROM task_environments
WHERE task_id = ?
ORDER BY updated_at DESC, created_at DESC
@@ -1305,7 +1310,7 @@ func healDuplicateTaskEnvForTask(tx *sql.Tx, taskID string) error {
return fmt.Errorf("heal duplicate envs: find winner for task %s: %w", taskID, err)
}
if _, err := tx.Exec(`
if _, err := tx.ExecContext(ctx, `
UPDATE task_sessions
SET task_environment_id = ?
WHERE task_id = ?
@@ -1314,7 +1319,7 @@ func healDuplicateTaskEnvForTask(tx *sql.Tx, taskID string) error {
return fmt.Errorf("heal duplicate envs: relink sessions for task %s: %w", taskID, err)
}
if _, err := tx.Exec(`
if _, err := tx.ExecContext(ctx, `
DELETE FROM task_environments
WHERE task_id = ?
AND id != ?
@@ -1329,7 +1334,7 @@ func healDuplicateTaskEnvForTask(tx *sql.Tx, taskID string) error {
// instead of silently producing two rows for the same task. Must run AFTER
// healDuplicateTaskEnvironments, which collapses any pre-existing duplicates.
func (r *Repository) ensureTaskEnvironmentTaskUniqueIndex() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE UNIQUE INDEX IF NOT EXISTS uniq_task_environments_task_id
ON task_environments(task_id)
`)
@@ -1351,7 +1356,7 @@ func (r *Repository) healSessionTaskEnvironmentIDs() error {
// ensureTaskEnvironmentTaskUniqueIndex guarantees ≤1 row per task at
// runtime, but the SQL reads as non-deterministic in isolation. Belt
// and suspenders.
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
UPDATE task_sessions
SET task_environment_id = (
SELECT te.id FROM task_environments te WHERE te.task_id = task_sessions.task_id LIMIT 1
apps/backend/internal/task/repository/sqlite/base_schema.go
+55 -35
View File
@@ -2,6 +2,7 @@
package sqlite
import (
"context"
"fmt"
"github.com/google/uuid"
@@ -14,6 +15,13 @@ import (
// table so the call list stays readable as new schema/migration steps are
// added without growing the function's cyclomatic complexity.
func (r *Repository) initSchema() error {
return r.initSchemaContext(context.Background())
}
func (r *Repository) initSchemaContext(ctx context.Context) error {
if ctx == nil {
ctx = context.Background()
}
steps := []func() error{
r.initDesktopDiscoverySchema,
r.initCoreSchema,
@@ -51,10 +59,22 @@ func (r *Repository) initSchema() error {
r.ensureMessageMetadataIndexes,
r.ensurePromptOrderIndex,
}
for _, step := range steps {
// Every boundary is checked before and after its step. The task repository
// passes the same context to startup SQL through migrationContext, so a
// cancellation stops later statements and later repository admission. A
// statement already executing is bounded by the database driver's
// cancellation behavior; the constructor never closes the shared pool
// until this function returns, so that in-flight work cannot race cleanup.
for index, step := range steps {
if err := ctx.Err(); err != nil {
return fmt.Errorf("startup schema canceled before step %d: %w", index, err)
}
if err := step(); err != nil {
return err
}
if err := ctx.Err(); err != nil {
return fmt.Errorf("startup schema canceled after step %d: %w", index, err)
}
}
return nil
}
@@ -89,7 +109,7 @@ func (r *Repository) ensureTaskEnvironmentRecoveryClaimsSchema() error {
}
func (r *Repository) initDynamicRoutingSchema() error {
_, err := r.db.Exec(fmt.Sprintf(`
_, err := r.db.ExecContext(r.migrationContext(), fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS dynamic_route_states (
session_id TEXT PRIMARY KEY,
logical_profile_id TEXT NOT NULL,
@@ -161,19 +181,19 @@ const taskResourceCleanupSchemaDDL = `
`
func (r *Repository) initTaskResourceCleanupSchema() error {
_, err := r.db.Exec(taskResourceCleanupSchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), taskResourceCleanupSchemaDDL)
return err
}
// ensureWorkspaceIndexes creates workspace-related indexes
func (r *Repository) ensureWorkspaceIndexes() error {
if _, err := r.db.Exec(`CREATE INDEX IF NOT EXISTS idx_tasks_workspace_id ON tasks(workspace_id)`); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), `CREATE INDEX IF NOT EXISTS idx_tasks_workspace_id ON tasks(workspace_id)`); err != nil {
return err
}
if _, err := r.db.Exec(`CREATE INDEX IF NOT EXISTS idx_workflows_workspace_id ON workflows(workspace_id)`); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), `CREATE INDEX IF NOT EXISTS idx_workflows_workspace_id ON workflows(workspace_id)`); err != nil {
return err
}
if _, err := r.db.Exec(`CREATE INDEX IF NOT EXISTS idx_tasks_workspace_archived ON tasks(workspace_id, archived_at)`); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), `CREATE INDEX IF NOT EXISTS idx_tasks_workspace_archived ON tasks(workspace_id, archived_at)`); err != nil {
return err
}
return nil
@@ -186,14 +206,14 @@ func (r *Repository) ensureMessageMetadataIndexes() error {
`CREATE INDEX IF NOT EXISTS idx_messages_metadata_tool_call_id ON task_session_messages(task_session_id, (%s))`,
dialect.JSONExtract(driver, "metadata", "tool_call_id"),
)
if _, err := r.db.Exec(toolCallIndex); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), toolCallIndex); err != nil {
return err
}
pendingIndex := fmt.Sprintf(
`CREATE INDEX IF NOT EXISTS idx_messages_metadata_pending_id ON task_session_messages(task_session_id, (%s))`,
dialect.JSONExtract(driver, "metadata", "pending_id"),
)
if _, err := r.db.Exec(pendingIndex); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), pendingIndex); err != nil {
return err
}
// idx_messages_metadata_pending_id leads with task_session_id, so it
@@ -204,7 +224,7 @@ func (r *Repository) ensureMessageMetadataIndexes() error {
`CREATE INDEX IF NOT EXISTS idx_messages_metadata_pending_id_lookup ON task_session_messages((%s))`,
dialect.JSONExtract(driver, "metadata", "pending_id"),
)
if _, err := r.db.Exec(pendingIndexLookup); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), pendingIndexLookup); err != nil {
return err
}
lookupIndex := dialect.PendingIDLookupIndexDDL(
@@ -212,7 +232,7 @@ func (r *Repository) ensureMessageMetadataIndexes() error {
"idx_messages_metadata_pending_id_lookup_ordered",
"task_session_messages",
)
if _, err := r.db.Exec(lookupIndex); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), lookupIndex); err != nil {
return err
}
return nil
@@ -223,13 +243,13 @@ func (r *Repository) ensureMessageMetadataIndexes() error {
// filtered prompt pages skip interleaved agent and tool rows.
func (r *Repository) ensurePromptOrderIndex() error {
ddl := dialect.PromptOrderIndexDDL(r.db.DriverName(), "idx_messages_prompt_order", "task_session_messages")
if _, err := r.db.Exec(ddl); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), ddl); err != nil {
return fmt.Errorf("create prompt-order index: %w", err)
}
userDDL := dialect.PromptUserOrderIndexDDL(
r.db.DriverName(), "idx_messages_prompt_user_order", "task_session_messages",
)
if _, err := r.db.Exec(userDDL); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), userDDL); err != nil {
return fmt.Errorf("create prompt-user-order index: %w", err)
}
return nil
@@ -249,7 +269,7 @@ func (r *Repository) ensureRunnerProjectionTables() error {
// repo so workflow.NewWithDB's later ALTER ADD COLUMNs become no-ops
// (only column-already-exists errors are tolerated). Mirrors
// internal/workflow/repository/sqlite.go (the canonical owner).
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS workflow_steps (
id TEXT PRIMARY KEY,
workflow_id TEXT NOT NULL DEFAULT '',
@@ -275,7 +295,7 @@ func (r *Repository) ensureRunnerProjectionTables() error {
)`); err != nil {
return fmt.Errorf("create workflow_steps projection table: %w", err)
}
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS workflow_step_participants (
id TEXT PRIMARY KEY,
step_id TEXT NOT NULL DEFAULT '',
@@ -319,7 +339,7 @@ const taskStatusSummarySchemaDDL = `
`
func (r *Repository) initTaskStatusSummarySchema() error {
_, err := r.db.Exec(taskStatusSummarySchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), taskStatusSummarySchemaDDL)
return err
}
@@ -441,12 +461,12 @@ const infraSchemaDDL = `
`
func (r *Repository) initInfraSchema() error {
_, err := r.db.Exec(infraSchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), infraSchemaDDL)
return err
}
func (r *Repository) initTaskSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS tasks (
id TEXT PRIMARY KEY,
workspace_id TEXT NOT NULL DEFAULT '',
@@ -601,7 +621,7 @@ const repositorySetsSchemaDDL = `
`
func (r *Repository) initRepositorySetsSchema() error {
_, err := r.db.Exec(repositorySetsSchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), repositorySetsSchemaDDL)
return err
}
@@ -625,12 +645,12 @@ const repositoryBranchPoliciesSchemaDDL = `
`
func (r *Repository) initRepositoryBranchPoliciesSchema() error {
_, err := r.db.Exec(repositoryBranchPoliciesSchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), repositoryBranchPoliciesSchemaDDL)
return err
}
func (r *Repository) initCoreIndexes() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE INDEX IF NOT EXISTS idx_tasks_workflow_id ON tasks(workflow_id);
CREATE INDEX IF NOT EXISTS idx_tasks_workflow_step_id ON tasks(workflow_step_id);
CREATE INDEX IF NOT EXISTS idx_tasks_archived_at ON tasks(archived_at);
@@ -645,7 +665,7 @@ func (r *Repository) initCoreIndexes() error {
}
func (r *Repository) initPlansSchema() error {
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_plans (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL UNIQUE,
@@ -692,7 +712,7 @@ func (r *Repository) initPlansSchema() error {
`); err != nil {
return err
}
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_plan_revisions (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL,
@@ -729,7 +749,7 @@ func (r *Repository) initPlansSchema() error {
// as a JSON array in a single column (read/written whole — there is no
// per-step query path), keeping the artifact one row per task like task_plans.
func (r *Repository) initWalkthroughsSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_walkthroughs (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL UNIQUE,
@@ -748,7 +768,7 @@ func (r *Repository) initWalkthroughsSchema() error {
// backfillInitialPlanRevisions ensures every existing task_plans row has at least
// one corresponding revision. Runs once at startup and is idempotent.
func (r *Repository) backfillInitialPlanRevisions() error {
rows, err := r.db.Query(`
rows, err := r.db.QueryContext(r.migrationContext(), `
SELECT p.id, p.task_id, p.title, p.content, p.created_by, p.created_at, p.updated_at
FROM task_plans p
WHERE NOT EXISTS (
@@ -781,7 +801,7 @@ func (r *Repository) backfillInitialPlanRevisions() error {
if authorKind != "user" && authorKind != authorKindAgent {
authorKind = authorKindAgent
}
_, err := r.db.Exec(r.db.Rebind(`
_, err := r.db.ExecContext(r.migrationContext(), r.db.Rebind(`
INSERT INTO task_plan_revisions
(id, task_id, revision_number, title, content, author_kind, author_name, revert_of_revision_id, created_at, updated_at)
VALUES (?, ?, 1, ?, ?, ?, 'legacy', NULL, ?, ?)
@@ -796,7 +816,7 @@ func (r *Repository) backfillInitialPlanRevisions() error {
// initDocumentsSchema creates the task_documents and task_document_revisions tables.
// These tables generalize task_plans: documents have a key (e.g., "plan", "spec") and type.
func (r *Repository) initDocumentsSchema() error {
if _, err := r.db.Exec(`
if _, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_documents (
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL,
@@ -844,7 +864,7 @@ func (r *Repository) initDocumentsSchema() error {
// attachments. The storage key is intentionally opaque and never exposed in
// API responses; bytes are owned by the attachment service.
func (r *Repository) initAttachmentsSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_message_attachments (
id TEXT PRIMARY KEY,
owner_id TEXT NOT NULL DEFAULT '',
@@ -901,7 +921,7 @@ func (r *Repository) initSessionSchema() error {
// row instead of clobbering a later execution's (AC-32). Historical message
// rows are handled separately by the one-time backfill migration.
func (r *Repository) initSubagentContextSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_session_subagents (
id TEXT PRIMARY KEY,
task_session_id TEXT NOT NULL,
@@ -947,9 +967,9 @@ func (r *Repository) initSubagentContextSchema() error {
// now-deleted step must survive that deletion.
func (r *Repository) initStepTransitionsSchema() error {
idCol := dialect.AutoIncrementIDColumn(r.db.DriverName())
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_step_transitions (
` + idCol + `,
`+idCol+`,
task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
session_id TEXT REFERENCES task_sessions(id) ON DELETE SET NULL,
from_workflow_id TEXT,
@@ -974,7 +994,7 @@ func (r *Repository) initStepTransitionsSchema() error {
}
func (r *Repository) initMessageTurnSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_session_turns (
id TEXT PRIMARY KEY,
task_session_id TEXT NOT NULL,
@@ -1126,12 +1146,12 @@ const sessionWorktreeSchemaDDL = `
`
func (r *Repository) initSessionWorktreeSchema() error {
_, err := r.db.Exec(sessionWorktreeSchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), sessionWorktreeSchemaDDL)
return err
}
func (r *Repository) initGitSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_session_git_snapshots (
id TEXT PRIMARY KEY,
task_environment_id TEXT NOT NULL,
@@ -1179,7 +1199,7 @@ func (r *Repository) initGitSchema() error {
}
func (r *Repository) initReviewSchema() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS session_file_reviews (
id TEXT PRIMARY KEY,
session_id TEXT NOT NULL,
@@ -1253,6 +1273,6 @@ const taskReviewSchemaDDL = `
`
func (r *Repository) initTaskReviewSchema() error {
_, err := r.db.Exec(taskReviewSchemaDDL)
_, err := r.db.ExecContext(r.migrationContext(), taskReviewSchemaDDL)
return err
}
apps/backend/internal/task/repository/sqlite/builtin_workflow_reconcile.go
+3 -3
View File
@@ -19,14 +19,14 @@ func (r *Repository) tryHealWorkflowStepEvents(
stepID, operation string,
mutate func(*wfmodels.StepEvents) bool,
) (applied, retry bool, err error) {
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return false, false, fmt.Errorf("begin %s reconciliation tx: %w", operation, err)
}
defer func() { _ = tx.Rollback() }()
var rawEvents sql.NullString
if err := tx.QueryRow(tx.Rebind(`SELECT events FROM workflow_steps WHERE id = ?`), stepID).Scan(&rawEvents); err != nil {
if err := tx.QueryRowContext(r.migrationContext(), tx.Rebind(`SELECT events FROM workflow_steps WHERE id = ?`), stepID).Scan(&rawEvents); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return true, false, nil
}
@@ -60,7 +60,7 @@ func (r *Repository) tryHealWorkflowStepEvents(
if guardArg != nil {
args = append(args, guardArg)
}
res, err := tx.Exec(tx.Rebind(`
res, err := tx.ExecContext(r.migrationContext(), tx.Rebind(`
UPDATE workflow_steps SET events = ?, updated_at = ?
WHERE id = ? AND `+guardClause), args...)
if err != nil {
apps/backend/internal/task/repository/sqlite/builtin_workflow_step_rows.go
+1 -1
View File
@@ -15,7 +15,7 @@ type systemWorkflowStepRow struct{ stepID, workflowID string }
// transactional per-row retry loops never run with an open read cursor over
// the same table.
func (r *Repository) findSystemOwnedWorkflowSteps(templateID, stepName string) ([]systemWorkflowStepRow, error) {
rows, err := r.db.Query(r.db.Rebind(`
rows, err := r.db.QueryContext(r.migrationContext(), r.db.Rebind(`
SELECT ws.id, ws.workflow_id FROM workflow_steps ws
JOIN workflows w ON w.id = ws.workflow_id
WHERE w.is_system = 1 AND w.workflow_template_id = ? AND ws.name = ?
apps/backend/internal/task/repository/sqlite/defaults.go
+4 -4
View File
@@ -58,7 +58,7 @@ func (r *Repository) hideBuiltinWorkflows() error {
if err != nil {
return fmt.Errorf("build hidden workflow reconciliation: %w", err)
}
if _, err := r.db.Exec(r.db.Rebind(query), args...); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), r.db.Rebind(query), args...); err != nil {
return fmt.Errorf("hide builtin workflows: %w", err)
}
return nil
@@ -79,7 +79,7 @@ func (r *Repository) healBuiltinWorkflowStepFlags() error {
for _, tmpl := range templates {
for _, step := range tmpl.Steps {
want := dialect.BoolToInt(step.AutoAdvanceRequiresSignal)
if _, err := r.db.Exec(r.db.Rebind(`
if _, err := r.db.ExecContext(r.migrationContext(), r.db.Rebind(`
UPDATE workflow_steps SET auto_advance_requires_signal = ?, updated_at = ?
WHERE name = ? AND auto_advance_requires_signal != ?
AND workflow_id IN (SELECT id FROM workflows WHERE is_system = 1 AND workflow_template_id = ?)
@@ -93,7 +93,7 @@ func (r *Repository) healBuiltinWorkflowStepFlags() error {
// ensureDefaultWorkspace creates a default workspace if none exists
func (r *Repository) ensureDefaultWorkspace() error {
ctx := context.Background()
ctx := r.migrationContext()
var count int
if err := r.db.QueryRowContext(ctx, "SELECT COUNT(1) FROM workspaces").Scan(&count); err != nil {
@@ -215,7 +215,7 @@ func (r *Repository) stampWorkspaceOfficeWorkflow(ctx context.Context, workspace
// ensureDefaultExecutorsAndEnvironments creates default executors and environments if none exist
func (r *Repository) ensureDefaultExecutorsAndEnvironments() error {
ctx := context.Background()
ctx := r.migrationContext()
if err := r.ensureDefaultExecutors(ctx); err != nil {
return err
}
apps/backend/internal/task/repository/sqlite/desktop_discovery_root.go
+2 -2
View File
@@ -39,14 +39,14 @@ func (r *Repository) initDesktopDiscoverySchema() error {
if err != nil {
return err
}
if _, err := r.db.Exec(desktopDiscoverySchemaDDL); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), desktopDiscoverySchemaDDL); err != nil {
return err
}
required := 0
if legacyInstallation {
required = 1
}
_, err = r.db.Exec(r.db.Rebind(`
_, err = r.db.ExecContext(r.migrationContext(), r.db.Rebind(`
INSERT INTO desktop_discovery_migration (id, home_confirmation_required, updated_at)
VALUES (1, ?, ?)
ON CONFLICT (id) DO NOTHING
apps/backend/internal/task/repository/sqlite/dynamic_route_legacy_active_migration.go
+15 -13
View File
@@ -1,6 +1,7 @@
package sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
@@ -54,14 +55,15 @@ const dynamicRouteLegacyActiveBackfillLockID int64 = 0x4B44_4452_4142 // "KDDRAB
// is exactly the orphan the startup sweep exists to catch. Re-running a
// value-based backfill on every boot would silently swallow it instead.
func (r *Repository) backfillLegacyActiveDynamicRoutes() error {
exists, err := db.ColumnExists(r.db, "dynamic_route_states", dynamicRouteLegacyActiveBackfillColumn)
ctx := r.migrationContext()
exists, err := db.ColumnExistsContext(ctx, r.db, "dynamic_route_states", dynamicRouteLegacyActiveBackfillColumn)
if err != nil {
return fmt.Errorf("dynamic route legacy backfill: probe marker column: %w", err)
}
if exists {
return nil
}
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return fmt.Errorf("dynamic route legacy backfill: begin tx: %w", err)
}
@@ -74,20 +76,20 @@ func (r *Repository) backfillLegacyActiveDynamicRoutes() error {
// Re-probe inside the lock: a concurrent PostgreSQL initializer may have
// already run and committed the backfill between the cheap probe above
// and this transaction acquiring the advisory lock.
exists, err = r.columnExists(tx, "dynamic_route_states", dynamicRouteLegacyActiveBackfillColumn)
exists, err = db.ColumnExistsContext(ctx, tx, "dynamic_route_states", dynamicRouteLegacyActiveBackfillColumn)
if err != nil {
return fmt.Errorf("dynamic route legacy backfill: re-probe marker column: %w", err)
}
if exists {
return nil
}
eligible, err := legacyActiveBackfillVersionAllowed(tx)
eligible, err := legacyActiveBackfillVersionAllowed(ctx, tx)
if err != nil {
return fmt.Errorf("dynamic route legacy backfill: read stored version: %w", err)
}
if eligible {
if _, err := tx.Exec(`
if _, err := tx.ExecContext(r.migrationContext(), `
UPDATE dynamic_route_states
SET state = 'active'
WHERE state = 'starting'
@@ -105,7 +107,7 @@ func (r *Repository) backfillLegacyActiveDynamicRoutes() error {
// Scoped to sessions whose dynamic_route_states row is now 'active' and
// whose generation still matches. A session whose authoritative route row
// or generation differs must keep its projection available for recovery.
if _, err := tx.Exec(`
if _, err := tx.ExecContext(r.migrationContext(), `
UPDATE task_sessions
SET route_state = 'active'
WHERE state = 'IDLE' AND route_state = 'starting'
@@ -120,8 +122,8 @@ func (r *Repository) backfillLegacyActiveDynamicRoutes() error {
return fmt.Errorf("dynamic route legacy backfill: backfill task_sessions: %w", err)
}
}
if _, err := tx.Exec(`ALTER TABLE dynamic_route_states ADD COLUMN ` +
dynamicRouteLegacyActiveBackfillColumn + ` INTEGER NOT NULL DEFAULT 1`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `ALTER TABLE dynamic_route_states ADD COLUMN `+
dynamicRouteLegacyActiveBackfillColumn+` INTEGER NOT NULL DEFAULT 1`); err != nil {
return fmt.Errorf("dynamic route legacy backfill: add marker column: %w", err)
}
if err := tx.Commit(); err != nil {
@@ -130,8 +132,8 @@ func (r *Repository) backfillLegacyActiveDynamicRoutes() error {
return nil
}
func legacyActiveBackfillVersionAllowed(conn db.SchemaQuerier) (bool, error) {
exists, err := db.TableExists(conn, "kandev_meta")
func legacyActiveBackfillVersionAllowed(ctx context.Context, conn db.ContextSchemaQuerier) (bool, error) {
exists, err := db.TableExistsContext(ctx, conn, "kandev_meta")
if err != nil {
return false, err
}
@@ -139,7 +141,7 @@ func legacyActiveBackfillVersionAllowed(conn db.SchemaQuerier) (bool, error) {
return false, nil
}
var storedVersion string
err = conn.QueryRow(conn.Rebind(
err = conn.QueryRowContext(ctx, conn.Rebind(
`SELECT value FROM kandev_meta WHERE key = 'kandev_version'`,
)).Scan(&storedVersion)
if errors.Is(err, sql.ErrNoRows) {
@@ -174,10 +176,10 @@ func (r *Repository) acquireDynamicRouteLegacyActiveBackfillLock(tx *sqlx.Tx) er
if !dialect.IsPostgres(r.db.DriverName()) {
return nil
}
if _, err := tx.Exec(`SET LOCAL lock_timeout = '30s'`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `SET LOCAL lock_timeout = '30s'`); err != nil {
return fmt.Errorf("dynamic route legacy backfill: set lock timeout: %w", err)
}
if _, err := tx.Exec(`SELECT pg_advisory_xact_lock($1)`, dynamicRouteLegacyActiveBackfillLockID); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `SELECT pg_advisory_xact_lock($1)`, dynamicRouteLegacyActiveBackfillLockID); err != nil {
return fmt.Errorf("dynamic route legacy backfill: acquire migration advisory lock: %w", err)
}
return nil
apps/backend/internal/task/repository/sqlite/git_snapshot_environment_migration.go
+10 -10
View File
@@ -28,7 +28,7 @@ func (r *Repository) migrateGitSnapshotOwnership() error {
return nil
}
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return fmt.Errorf("git snapshot cutover: begin tx: %w", err)
}
@@ -70,10 +70,10 @@ func (r *Repository) replayGitSnapshotFinalShape(tx *sqlx.Tx) error {
}
func (r *Repository) prepareGitSnapshotCutover(tx *sqlx.Tx) error {
if _, err := tx.Exec("DROP TABLE IF EXISTS " + gitSnapshotCutoverShadowTable); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), "DROP TABLE IF EXISTS "+gitSnapshotCutoverShadowTable); err != nil {
return fmt.Errorf("git snapshot cutover: remove stale shadow: %w", err)
}
if _, err := tx.Exec(finalGitSnapshotDDL(gitSnapshotCutoverShadowTable)); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), finalGitSnapshotDDL(gitSnapshotCutoverShadowTable)); err != nil {
return fmt.Errorf("git snapshot cutover: create shadow: %w", err)
}
if err := r.maybeFailGitSnapshotCutover("create_shadow"); err != nil {
@@ -99,10 +99,10 @@ func (r *Repository) prepareGitSnapshotCutover(tx *sqlx.Tx) error {
}
func (r *Repository) swapGitSnapshotCutover(tx *sqlx.Tx) error {
if _, err := tx.Exec("DROP TABLE task_session_git_snapshots"); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), "DROP TABLE task_session_git_snapshots"); err != nil {
return fmt.Errorf("git snapshot cutover: drop legacy table: %w", err)
}
if _, err := tx.Exec("ALTER TABLE " + gitSnapshotCutoverShadowTable + " RENAME TO task_session_git_snapshots"); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), "ALTER TABLE "+gitSnapshotCutoverShadowTable+" RENAME TO task_session_git_snapshots"); err != nil {
return fmt.Errorf("git snapshot cutover: rename shadow: %w", err)
}
if err := r.maybeFailGitSnapshotCutover("swap"); err != nil {
@@ -143,13 +143,13 @@ func (r *Repository) acquireGitSnapshotCutoverLocks(tx *sqlx.Tx) error {
if !dialect.IsPostgres(r.db.DriverName()) {
return nil
}
if _, err := tx.Exec("SET LOCAL lock_timeout = '30s'"); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), "SET LOCAL lock_timeout = '30s'"); err != nil {
return fmt.Errorf("git snapshot cutover: set lock timeout: %w", err)
}
if _, err := tx.Exec("SELECT pg_advisory_xact_lock($1)", gitSnapshotCutoverLockID); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), "SELECT pg_advisory_xact_lock($1)", gitSnapshotCutoverLockID); err != nil {
return fmt.Errorf("git snapshot cutover: acquire migration advisory lock: %w", err)
}
if _, err := tx.Exec(`
if _, err := tx.ExecContext(r.migrationContext(), `
LOCK TABLE task_session_git_snapshots, task_sessions, task_environments
IN ACCESS EXCLUSIVE MODE`); err != nil {
return fmt.Errorf("git snapshot cutover: lock ownership tables: %w", err)
@@ -195,7 +195,7 @@ func (r *Repository) copyGitSnapshotWinners(tx *sqlx.Tx) error {
) ranked
WHERE row_number = 1
`
if _, err := tx.Exec(r.db.Rebind(query)); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), r.db.Rebind(query)); err != nil {
return fmt.Errorf("git snapshot cutover: copy current winners: %w", err)
}
return nil
@@ -221,7 +221,7 @@ func (r *Repository) ensureGitSnapshotIndexes(tx *sqlx.Tx) error {
`CREATE INDEX IF NOT EXISTS idx_git_snapshots_session ON task_session_git_snapshots(session_id, created_at DESC)`,
`CREATE INDEX IF NOT EXISTS idx_git_snapshots_type ON task_session_git_snapshots(session_id, snapshot_type)`,
} {
if _, err := tx.Exec(query); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), query); err != nil {
return fmt.Errorf("git snapshot cutover: create index: %w", err)
}
}
apps/backend/internal/task/repository/sqlite/startup_cost_test.go
+434
View File
@@ -0,0 +1,434 @@
package sqlite
import (
"context"
"errors"
"os"
"path/filepath"
"runtime"
"strconv"
"sync"
"testing"
"time"
"github.com/jmoiron/sqlx"
dbpkg "github.com/kandev/kandev/internal/db"
"github.com/kandev/kandev/internal/persistence"
"github.com/mattn/go-sqlite3"
)
// TestStartupMigrationCosts records fresh-schema and replay costs on an
// isolated SQLite file. The output is evidence for startup budgeting; it does
// not impose a machine-specific duration threshold.
func TestStartupMigrationCosts(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "startup-costs.db")
freshDB := openStartupCostDB(t, dbPath)
t.Cleanup(func() { _ = freshDB.Close() })
freshStarted := time.Now()
if _, err := NewWithDB(freshDB, freshDB, nil); err != nil {
t.Fatalf("initialize fresh schema: %v", err)
}
freshDuration := time.Since(freshStarted)
if err := freshDB.Close(); err != nil {
t.Fatalf("close fresh database: %v", err)
}
replayDB := openStartupCostDB(t, dbPath)
t.Cleanup(func() { _ = replayDB.Close() })
replayStarted := time.Now()
if _, err := NewWithDB(replayDB, replayDB, nil); err != nil {
t.Fatalf("replay schema: %v", err)
}
replayDuration := time.Since(replayStarted)
if err := replayDB.Close(); err != nil {
t.Fatalf("close replay database: %v", err)
}
t.Logf("startup migration cost: fresh=%s replay=%s", freshDuration, replayDuration)
}
// TestStartupMigrationCostsPopulated measures startup against a repeatable
// legacy-shaped fixture with enough messages for the two recurring backfills
// to do real work. The fixture is intentionally local and synthetic: its
// purpose is to expose the relative costs and the backup/liveness boundary,
// not to predict a particular production machine's total startup time.
func TestStartupMigrationCostsPopulated(t *testing.T) {
const (
sessionCount = 24
messagesPerSession = 250
)
dbPath := filepath.Join(t.TempDir(), "startup-costs-populated.db")
freshDB := openStartupCostDB(t, dbPath)
freshStarted := time.Now()
if _, err := NewWithDB(freshDB, freshDB, nil); err != nil {
t.Fatalf("initialize fresh schema: %v", err)
}
freshDuration := time.Since(freshStarted)
if err := freshDB.Close(); err != nil {
t.Fatalf("close fresh database: %v", err)
}
seedDB := openStartupCostDB(t, dbPath)
if err := makePopulatedLegacyMessageFixture(seedDB, sessionCount, messagesPerSession); err != nil {
_ = seedDB.Close()
t.Fatalf("prepare populated legacy fixture: %v", err)
}
messageCount := sessionCount * messagesPerSession
var measuredMessageCount int
if err := seedDB.Get(&measuredMessageCount, `SELECT COUNT(*) FROM task_session_messages`); err != nil {
_ = seedDB.Close()
t.Fatalf("count fixture messages: %v", err)
}
if measuredMessageCount != messageCount {
_ = seedDB.Close()
t.Fatalf("fixture message count = %d, want %d", measuredMessageCount, messageCount)
}
dbInfo, err := os.Stat(dbPath)
if err != nil {
_ = seedDB.Close()
t.Fatalf("stat populated fixture: %v", err)
}
backupPath := filepath.Join(t.TempDir(), "startup-costs-backup.db")
backupStarted := time.Now()
backupBytes, err := persistence.SnapshotSQLite(seedDB, backupPath)
backupDuration := time.Since(backupStarted)
if err != nil {
_ = seedDB.Close()
t.Fatalf("snapshot populated fixture: %v", err)
}
if err := seedDB.Close(); err != nil {
t.Fatalf("close seeded database: %v", err)
}
replayDB := openStartupCostDB(t, dbPath)
replayStarted := time.Now()
replayRepo, err := NewWithDB(replayDB, replayDB, nil)
if err != nil {
_ = replayDB.Close()
t.Fatalf("replay populated schema: %v", err)
}
replayDuration := time.Since(replayStarted)
// Measure the two recurring data migrations separately on the same
// populated database. Reset statements are excluded from the timings.
if _, err := replayDB.Exec(`UPDATE task_session_messages SET updated_at = NULL`); err != nil {
_ = replayDB.Close()
t.Fatalf("reset updated_at fixture: %v", err)
}
updatedAtStarted := time.Now()
if err := dbpkg.NewRequiredMigrateLogger(replayDB, nil).Apply(
"evidence.task_session_messages.updated_at.backfill",
`UPDATE task_session_messages SET updated_at = created_at WHERE updated_at IS NULL`,
); err != nil {
_ = replayDB.Close()
t.Fatalf("measure updated_at backfill: %v", err)
}
updatedAtDuration := time.Since(updatedAtStarted)
if _, err := replayDB.Exec(`UPDATE task_session_messages SET prompt_seq = 0`); err != nil {
_ = replayDB.Close()
t.Fatalf("reset prompt sequence fixture: %v", err)
}
if _, err := replayDB.Exec(`DELETE FROM task_session_prompt_seq`); err != nil {
_ = replayDB.Close()
t.Fatalf("reset prompt sequence counters: %v", err)
}
promptSeqStarted := time.Now()
if err := replayRepo.backfillPromptSeq(); err != nil {
_ = replayDB.Close()
t.Fatalf("measure prompt sequence backfill: %v", err)
}
promptSeqDuration := time.Since(promptSeqStarted)
if err := replayDB.Close(); err != nil {
t.Fatalf("close replay database: %v", err)
}
t.Logf("populated startup evidence: sessions=%d messages=%d db_bytes=%d backup_bytes=%d fresh_schema=%s backup=%s replay=%s updated_at_backfill=%s prompt_seq_backfill=%s environment=%s/%s go=%s cpus=%d",
sessionCount, messageCount, dbInfo.Size(), backupBytes, freshDuration, backupDuration,
replayDuration, updatedAtDuration, promptSeqDuration, runtime.GOOS, runtime.GOARCH,
runtime.Version(), runtime.NumCPU())
}
// makePopulatedLegacyMessageFixture turns the current empty schema into the
// shape that exercises the recurring startup migrations: updated_at is present
// but nullable, and prompt_seq is absent. The rebuild is isolated to this
// measurement test and does not alter production migration behavior.
func makePopulatedLegacyMessageFixture(db *sqlx.DB, sessionCount, messagesPerSession int) error {
for _, index := range []string{
"idx_messages_session_id",
"idx_messages_created_at",
"idx_messages_session_created",
"idx_messages_turn_id",
"idx_messages_task_author_created",
"idx_messages_session_updated",
"idx_messages_metadata_tool_call_id",
"idx_messages_metadata_pending_id",
"idx_messages_metadata_pending_id_lookup",
"idx_messages_metadata_pending_id_lookup_ordered",
} {
if _, err := db.Exec("DROP INDEX IF EXISTS " + index); err != nil {
return err
}
}
if _, err := db.Exec(`PRAGMA foreign_keys=OFF`); err != nil {
return err
}
if _, err := db.Exec(`ALTER TABLE task_session_messages RENAME TO startup_messages_current`); err != nil {
return err
}
if _, err := db.Exec(`
CREATE TABLE task_session_messages (
id TEXT PRIMARY KEY,
task_session_id TEXT NOT NULL,
task_id TEXT DEFAULT '',
turn_id TEXT NOT NULL,
author_type TEXT NOT NULL DEFAULT 'user',
author_id TEXT DEFAULT '',
content TEXT NOT NULL,
requests_input INTEGER DEFAULT 0,
type TEXT NOT NULL DEFAULT 'message',
metadata TEXT DEFAULT '{}',
created_at TIMESTAMP NOT NULL,
updated_at TIMESTAMP,
FOREIGN KEY (task_session_id) REFERENCES task_sessions(id) ON DELETE CASCADE,
FOREIGN KEY (turn_id) REFERENCES task_session_turns(id) ON DELETE CASCADE
)`); err != nil {
return err
}
if _, err := db.Exec(`
INSERT INTO task_session_messages
(id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at)
SELECT id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at
FROM startup_messages_current`); err != nil {
return err
}
if _, err := db.Exec(`DROP TABLE startup_messages_current`); err != nil {
return err
}
if _, err := db.Exec(`PRAGMA foreign_keys=ON`); err != nil {
return err
}
var workspaceID, workflowID string
if err := db.QueryRow(`SELECT id FROM workspaces ORDER BY created_at LIMIT 1`).Scan(&workspaceID); err != nil {
return err
}
if err := db.QueryRow(`SELECT id FROM workflows ORDER BY created_at LIMIT 1`).Scan(&workflowID); err != nil {
return err
}
tx, err := db.Beginx()
if err != nil {
return err
}
defer func() { _ = tx.Rollback() }()
baseTime := time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC)
for sessionIndex := 0; sessionIndex < sessionCount; sessionIndex++ {
taskID := "startup-evidence-task-" + strconv.Itoa(sessionIndex)
sessionID := "startup-evidence-session-" + strconv.Itoa(sessionIndex)
turnID := "startup-evidence-turn-" + strconv.Itoa(sessionIndex)
now := baseTime.Add(time.Duration(sessionIndex) * time.Hour)
if _, err := tx.Exec(`INSERT INTO tasks (id, workspace_id, workflow_id, title, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`, taskID, workspaceID, workflowID, taskID, now, now); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO task_sessions (id, task_id, started_at, updated_at) VALUES (?, ?, ?, ?)`, sessionID, taskID, now, now); err != nil {
return err
}
if _, err := tx.Exec(`INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)`, turnID, sessionID, taskID, now, now, now); err != nil {
return err
}
for messageIndex := 0; messageIndex < messagesPerSession; messageIndex++ {
authorType := "agent"
if messageIndex%3 == 0 {
authorType = "user"
}
createdAt := now.Add(time.Duration(messageIndex) * time.Microsecond)
messageID := "startup-evidence-message-" + strconv.Itoa(sessionIndex) + "-" + strconv.Itoa(messageIndex)
if _, err := tx.Exec(`
INSERT INTO task_session_messages
(id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, '', 'startup evidence', 0, 'message', '{}', ?, NULL)
`, messageID, sessionID, taskID, turnID, authorType, createdAt); err != nil {
return err
}
}
}
return tx.Commit()
}
func openStartupCostDB(t *testing.T, path string) *sqlx.DB {
t.Helper()
conn, err := dbpkg.OpenSQLite(path)
if err != nil {
t.Fatalf("open SQLite database: %v", err)
}
return sqlx.NewDb(conn, "sqlite3")
}
// TestBackfillPromptSeqStopsOnCanceledContext exercises a real recurring
// migration while its UPDATE is executing. The trigger blocks the statement
// at the SQLite driver boundary so cancellation is deterministic and does not
// depend on a timer racing a particular database size.
func TestBackfillPromptSeqStopsOnCanceledContext(t *testing.T) {
db := openStartupCostDB(t, filepath.Join(t.TempDir(), "prompt-cancel.db"))
t.Cleanup(func() { _ = db.Close() })
db.SetMaxOpenConns(1)
if _, err := db.Exec(`
CREATE TABLE task_session_messages (
id TEXT PRIMARY KEY,
task_session_id TEXT NOT NULL,
author_type TEXT NOT NULL,
created_at TIMESTAMP NOT NULL,
prompt_seq INTEGER NOT NULL DEFAULT 0
);
CREATE TABLE task_session_prompt_seq (
task_session_id TEXT PRIMARY KEY,
last_seq INTEGER NOT NULL
);
INSERT INTO task_session_messages(id, task_session_id, author_type, created_at, prompt_seq)
VALUES ('message-1', 'session-1', 'user', '2026-01-01T00:00:00Z', 0);
`); err != nil {
t.Fatalf("seed prompt sequence fixture: %v", err)
}
started := make(chan struct{})
release := make(chan struct{})
var startedOnce sync.Once
conn, err := db.Conn(context.Background())
if err != nil {
t.Fatalf("reserve SQLite connection: %v", err)
}
if err := conn.Raw(func(driverConn any) error {
sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn)
if !ok {
t.Fatalf("driver connection = %T, want *sqlite3.SQLiteConn", driverConn)
}
return sqliteConn.RegisterFunc("wait_for_prompt_release", func() int64 {
startedOnce.Do(func() { close(started) })
<-release
return 1
}, true)
}); err != nil {
_ = conn.Close()
t.Fatalf("register SQLite function: %v", err)
}
if err := conn.Close(); err != nil {
t.Fatalf("release SQLite connection: %v", err)
}
if _, err := db.Exec(`
CREATE TRIGGER block_prompt_sequence_update
BEFORE UPDATE OF prompt_seq ON task_session_messages
BEGIN
SELECT wait_for_prompt_release();
END;
`); err != nil {
t.Fatalf("create blocking trigger: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
repo := &Repository{
db: db,
migrate: dbpkg.NewRequiredMigrateLoggerContext(db, nil, ctx),
}
done := make(chan error, 1)
go func() { done <- repo.backfillPromptSeq() }()
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("prompt sequence migration did not enter the active statement")
}
cancel()
close(release)
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("prompt sequence migration error = %v, want context.Canceled", err)
}
case <-time.After(2 * time.Second):
t.Fatal("prompt sequence migration did not stop after cancellation")
}
}
// TestNewWithDBContextStopsActiveMigration proves that cancellation reaches
// the real repository constructor at the shared startup boundary. The
// trigger blocks the production prompt-sequence UPDATE, then cancellation
// must abort that constructor before it can admit a later schema step.
func TestNewWithDBContextStopsActiveMigration(t *testing.T) {
db := openStartupCostDB(t, filepath.Join(t.TempDir(), "constructor-cancel.db"))
t.Cleanup(func() { _ = db.Close() })
db.SetMaxOpenConns(1)
repo, err := NewWithDB(db, db, nil)
if err != nil {
t.Fatalf("initialize schema: %v", err)
}
seedForMsgTest(t, repo, "constructor-task", "constructor-session", "constructor-turn")
var workspaceID, workflowID string
if err := db.QueryRow(`SELECT id FROM workspaces ORDER BY created_at LIMIT 1`).Scan(&workspaceID); err != nil {
t.Fatalf("read workspace: %v", err)
}
if err := db.QueryRow(`SELECT id FROM workflows ORDER BY created_at LIMIT 1`).Scan(&workflowID); err != nil {
t.Fatalf("read workflow: %v", err)
}
if _, err := db.Exec(`UPDATE tasks SET workspace_id = ?, workflow_id = ? WHERE id = ?`, workspaceID, workflowID, "constructor-task"); err != nil {
t.Fatalf("make constructor fixture replayable: %v", err)
}
insertPromptRow(t, repo, "constructor-message", "constructor-session", "constructor-turn", "user", "startup cancellation", time.Now().UTC())
started := make(chan struct{})
release := make(chan struct{})
var startedOnce sync.Once
conn, err := db.Conn(context.Background())
if err != nil {
t.Fatalf("reserve SQLite connection: %v", err)
}
if err := conn.Raw(func(driverConn any) error {
sqliteConn, ok := driverConn.(*sqlite3.SQLiteConn)
if !ok {
t.Fatalf("driver connection = %T, want *sqlite3.SQLiteConn", driverConn)
}
return sqliteConn.RegisterFunc("wait_for_constructor_release", func() int64 {
startedOnce.Do(func() { close(started) })
<-release
return 1
}, true)
}); err != nil {
_ = conn.Close()
t.Fatalf("register SQLite function: %v", err)
}
if err := conn.Close(); err != nil {
t.Fatalf("release SQLite connection: %v", err)
}
if _, err := db.Exec(`
CREATE TRIGGER block_constructor_prompt_sequence
BEFORE UPDATE OF prompt_seq ON task_session_messages
BEGIN
SELECT wait_for_constructor_release();
END;
`); err != nil {
t.Fatalf("create blocking trigger: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
_, err := NewWithDBContext(ctx, db, db, nil)
done <- err
}()
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("repository constructor did not enter the active migration")
}
cancel()
close(release)
select {
case err := <-done:
if !errors.Is(err, context.Canceled) {
t.Fatalf("constructor error = %v, want context.Canceled", err)
}
case <-time.After(2 * time.Second):
t.Fatal("repository constructor did not stop after cancellation")
}
}
apps/backend/internal/task/repository/sqlite/step_entries.go
+3 -3
View File
@@ -71,9 +71,9 @@ func isStepEntryMarkerUniqueViolation(err error) bool {
// initStepTransitionsSchema's comment for why that rule doesn't apply here.
func (r *Repository) initStepEntriesSchema() error {
idCol := dialect.AutoIncrementIDColumn(r.db.DriverName())
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS workflow_step_entries (
` + idCol + `,
`+idCol+`,
task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
step_id TEXT NOT NULL,
entry_seq INTEGER NOT NULL,
@@ -85,7 +85,7 @@ func (r *Repository) initStepEntriesSchema() error {
ON workflow_step_entries(task_id, step_id);
CREATE TABLE IF NOT EXISTS workflow_step_entry_markers (
` + idCol + `,
`+idCol+`,
entry_id INTEGER NOT NULL REFERENCES workflow_step_entries(id) ON DELETE CASCADE,
position INTEGER NOT NULL,
kind TEXT NOT NULL,
apps/backend/internal/task/repository/sqlite/subagent_context_activation.go
+1 -1
View File
@@ -24,7 +24,7 @@ func subagentContextActivationKeyInsertSQL(key, value string) string {
// empty string as this key's legitimate sentinel value, not "absent".
func (r *Repository) subagentContextActivationKeyPresent(key string) (bool, error) {
var value string
err := r.db.QueryRow(r.db.Rebind(`SELECT value FROM kandev_meta WHERE key = ?`), key).Scan(&value)
err := r.db.QueryRowContext(r.migrationContext(), r.db.Rebind(`SELECT value FROM kandev_meta WHERE key = ?`), key).Scan(&value)
if err == nil {
return true, nil
}
apps/backend/internal/task/repository/sqlite/subagent_context_backfill.go
+10 -8
View File
@@ -2,6 +2,7 @@
package sqlite
import (
"context"
"database/sql"
"fmt"
"time"
@@ -89,16 +90,17 @@ func (r *Repository) migrateSubagentContextBackfill() {
// trigger, or from a pre-amendment partial pair — is preserved verbatim
// (AC-24 point 5, AC-24d revision 4).
func (r *Repository) migrateSubagentContextBackfillUnsafe() error {
ctx := r.migrationContext()
postgres := dialect.IsPostgres(r.db.DriverName())
predicate, selectCols := subagentContextBackfillPredicateAndColumns(postgres)
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(ctx, nil)
if err != nil {
return fmt.Errorf("begin subagent context backfill transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
backfillThrough, err := subagentContextBackfillHighWaterMark(tx, postgres, predicate)
backfillThrough, err := subagentContextBackfillHighWaterMark(ctx, tx, postgres, predicate)
if err != nil {
return err
}
@@ -148,15 +150,15 @@ func (r *Repository) migrateSubagentContextBackfillUnsafe() error {
WHERE m.rn = 1
ON CONFLICT (task_session_id, agent_execution_id, tool_call_id) DO NOTHING
`
if _, err := tx.Exec(insertSQL); err != nil {
if _, err := tx.ExecContext(ctx, insertSQL); err != nil {
return fmt.Errorf("subagent context backfill insert: %w", err)
}
captureSince := time.Now().UTC().Format(time.RFC3339Nano)
if _, err := tx.Exec(subagentContextActivationKeyInsertSQL(subagentContextCaptureSinceKey, captureSince)); err != nil {
if _, err := tx.ExecContext(ctx, subagentContextActivationKeyInsertSQL(subagentContextCaptureSinceKey, captureSince)); err != nil {
return fmt.Errorf("write subagent_context_capture_since: %w", err)
}
if _, err := tx.Exec(subagentContextActivationKeyInsertSQL(subagentContextBackfillThroughKey, backfillThrough)); err != nil {
if _, err := tx.ExecContext(ctx, subagentContextActivationKeyInsertSQL(subagentContextBackfillThroughKey, backfillThrough)); err != nil {
return fmt.Errorf("write subagent_context_backfill_through: %w", err)
}
@@ -176,11 +178,11 @@ func (r *Repository) migrateSubagentContextBackfillUnsafe() error {
// see the declared column type; wrapping it in MAX() loses that, so the scan
// target must be a string on that dialect (see parseLegacyTimestamp's own
// comment for the same fact). PostgreSQL's driver has no such gap.
func subagentContextBackfillHighWaterMark(tx *sqlx.Tx, postgres bool, predicate string) (string, error) {
func subagentContextBackfillHighWaterMark(ctx context.Context, tx *sqlx.Tx, postgres bool, predicate string) (string, error) {
query := `SELECT MAX(m.created_at) FROM task_session_messages m WHERE ` + predicate
if postgres {
var maxCreatedAt sql.NullTime
if err := tx.QueryRow(query).Scan(&maxCreatedAt); err != nil {
if err := tx.QueryRowContext(ctx, query).Scan(&maxCreatedAt); err != nil {
return "", fmt.Errorf("compute subagent context backfill high-water mark: %w", err)
}
if !maxCreatedAt.Valid {
@@ -189,7 +191,7 @@ func subagentContextBackfillHighWaterMark(tx *sqlx.Tx, postgres bool, predicate
return maxCreatedAt.Time.UTC().Format(time.RFC3339Nano), nil
}
var maxCreatedAt sql.NullString
if err := tx.QueryRow(query).Scan(&maxCreatedAt); err != nil {
if err := tx.QueryRowContext(ctx, query).Scan(&maxCreatedAt); err != nil {
return "", fmt.Errorf("compute subagent context backfill high-water mark: %w", err)
}
if !maxCreatedAt.Valid || maxCreatedAt.String == "" {
apps/backend/internal/task/repository/sqlite/task_priority_postgres.go
+3 -4
View File
@@ -1,7 +1,6 @@
package sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
@@ -25,7 +24,7 @@ func (r *Repository) migrateTaskPriorityToTextPostgres() error {
}
var dataType string
err := r.db.Get(&dataType, `
err := r.db.GetContext(r.migrationContext(), &dataType, `
SELECT data_type
FROM information_schema.columns
WHERE table_schema = current_schema()
@@ -42,7 +41,7 @@ func (r *Repository) migrateTaskPriorityToTextPostgres() error {
return nil
}
tx, err := r.db.BeginTxx(context.Background(), nil)
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return fmt.Errorf("begin tasks.priority migration: %w", err)
}
@@ -59,7 +58,7 @@ func (r *Repository) migrateTaskPriorityToTextPostgres() error {
{"add constraint", `ALTER TABLE tasks ADD CONSTRAINT tasks_priority_check CHECK (priority IN ('critical','high','medium','low'))`},
}
for _, statement := range statements {
if _, err := tx.Exec(statement.sql); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), statement.sql); err != nil {
return fmt.Errorf("tasks.priority migration (%s): %w", statement.name, err)
}
}
apps/backend/internal/task/repository/sqlite/team_access_migration.go
+1 -1
View File
@@ -45,7 +45,7 @@ func (r *Repository) ensureTeamAccessSchema() error {
// visible to everyone until the setup wizard claims them, and inventing an
// owner here would lock the single user out of their own data.
func (r *Repository) backfillWorkspaceOwnerMembers() error {
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
INSERT INTO workspace_members (workspace_id, user_id, role, added_by, created_at)
SELECT w.id, w.owner_id, 'owner', '', w.created_at
FROM workspaces w
apps/backend/internal/task/repository/sqlite/usage_events_schema.go
+2 -2
View File
@@ -34,9 +34,9 @@ func (r *Repository) initTaskUsageEventsSchema() error {
if dialect.IsPostgres(r.db.DriverName()) {
idCol = "id BIGSERIAL PRIMARY KEY"
}
_, err := r.db.Exec(`
_, err := r.db.ExecContext(r.migrationContext(), `
CREATE TABLE IF NOT EXISTS task_usage_events (
` + idCol + `,
`+idCol+`,
usage_event_id TEXT NOT NULL,
task_id TEXT NOT NULL REFERENCES tasks(id) ON DELETE CASCADE,
session_id TEXT REFERENCES task_sessions(id) ON DELETE SET NULL,
apps/backend/internal/task/repository/sqlite/worktree_ownership_git_snapshot.go
+10 -6
View File
@@ -1,6 +1,7 @@
package sqlite
import (
"context"
"fmt"
"github.com/jmoiron/sqlx"
@@ -17,10 +18,13 @@ func (r *Repository) prepareTaskWorktreeCutoverForeignKeys() (func(), error) {
if dialect.IsPostgres(r.db.DriverName()) {
return func() {}, nil
}
if _, err := r.db.Exec(`PRAGMA foreign_keys=OFF`); err != nil {
if _, err := r.db.ExecContext(r.migrationContext(), `PRAGMA foreign_keys=OFF`); err != nil {
return nil, fmt.Errorf("cutover: disable sqlite foreign keys: %w", err)
}
return func() { _, _ = r.db.Exec(`PRAGMA foreign_keys=ON`) }, nil
// Cleanup must restore the connection setting even after startup context
// cancellation; otherwise the shared writer would retain foreign-key checks
// disabled for the next operation.
return func() { _, _ = r.db.ExecContext(context.Background(), `PRAGMA foreign_keys=ON`) }, nil
}
// rebindGitSnapshotEnvironmentForeignKey preserves an already environment-
@@ -41,7 +45,7 @@ func (r *Repository) rebindGitSnapshotEnvironmentForeignKey(c *worktreeCutover,
if !dialect.IsPostgres(r.db.DriverName()) {
return nil
}
if _, err := tx.Exec(postgresGitSnapshotShadowForeignKeyDDL); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), postgresGitSnapshotShadowForeignKeyDDL); err != nil {
return fmt.Errorf("cutover: rebind git snapshot environment foreign key: %w", err)
}
return nil
@@ -53,7 +57,7 @@ func (r *Repository) rehomeGitSnapshotsForCutover(c *worktreeCutover, tx *sqlx.T
if survivingID == "" || survivingID == environmentID {
continue
}
if _, err := tx.Exec(tx.Rebind(`
if _, err := tx.ExecContext(r.migrationContext(), tx.Rebind(`
UPDATE task_session_git_snapshots
SET task_environment_id = ?
WHERE task_environment_id = ?
@@ -81,7 +85,7 @@ func (r *Repository) rebindTaskEnvironmentRecoveryClaimForeignKey(c *worktreeCut
if survivingID == "" || survivingID == environmentID {
continue
}
if _, err := tx.Exec(tx.Rebind(`
if _, err := tx.ExecContext(r.migrationContext(), tx.Rebind(`
UPDATE task_environment_recovery_claims
SET task_environment_id = ?
WHERE task_environment_id = ?
@@ -92,7 +96,7 @@ func (r *Repository) rebindTaskEnvironmentRecoveryClaimForeignKey(c *worktreeCut
if !dialect.IsPostgres(r.db.DriverName()) {
return nil
}
if _, err := tx.Exec(postgresRecoveryClaimShadowForeignKeyDDL); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), postgresRecoveryClaimShadowForeignKeyDDL); err != nil {
return fmt.Errorf("cutover: rebind recovery claim environment foreign key: %w", err)
}
return nil
apps/backend/internal/task/repository/sqlite/worktree_ownership_migration.go
+14 -31
View File
@@ -1,11 +1,13 @@
package sqlite
import (
"context"
"fmt"
"github.com/jmoiron/sqlx"
"go.uber.org/zap"
"github.com/kandev/kandev/internal/db"
"github.com/kandev/kandev/internal/db/dialect"
)
@@ -107,7 +109,7 @@ func (r *Repository) normalizeTaskWorktreeOwnership() error {
}
defer restoreForeignKeys()
tx, err := r.db.Beginx()
tx, err := r.db.BeginTxx(r.migrationContext(), nil)
if err != nil {
return fmt.Errorf("cutover: begin tx: %w", err)
}
@@ -129,6 +131,7 @@ func (r *Repository) normalizeTaskWorktreeOwnership() error {
return err
}
cut := &worktreeCutover{
ctx: r.migrationContext(),
envs: make(map[string]*legacyEnv),
taskEnvs: make(map[string][]*legacyEnv),
sessions: make(map[string]*legacySession),
@@ -211,10 +214,10 @@ func (r *Repository) cutoverAcquireLocks(tx *sqlx.Tx) error {
if !dialect.IsPostgres(r.db.DriverName()) {
return nil
}
if _, err := tx.Exec(`SET LOCAL lock_timeout = '30s'`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `SET LOCAL lock_timeout = '30s'`); err != nil {
return fmt.Errorf("cutover: set lock timeout: %w", err)
}
if _, err := tx.Exec(`SELECT pg_advisory_xact_lock($1)`, taskWorktreeCutoverLockID); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `SELECT pg_advisory_xact_lock($1)`, taskWorktreeCutoverLockID); err != nil {
return fmt.Errorf("cutover: acquire migration advisory lock: %w", err)
}
claimTable, err := r.columnExists(tx, "task_environment_recovery_claims", "task_environment_id")
@@ -227,7 +230,7 @@ func (r *Repository) cutoverAcquireLocks(tx *sqlx.Tx) error {
if claimTable {
lockTables += `, task_environment_recovery_claims`
}
if _, err := tx.Exec("LOCK TABLE " + lockTables + " IN ACCESS EXCLUSIVE MODE"); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), "LOCK TABLE "+lockTables+" IN ACCESS EXCLUSIVE MODE"); err != nil {
return fmt.Errorf("cutover: lock ownership tables: %w", err)
}
return nil
@@ -235,19 +238,11 @@ func (r *Repository) cutoverAcquireLocks(tx *sqlx.Tx) error {
// tableExists reports whether a table exists on either database engine.
func (r *Repository) tableExists(name string) (bool, error) {
var exists bool
var err error
if dialect.IsPostgres(r.db.DriverName()) {
err = r.db.QueryRow(`
SELECT EXISTS (
SELECT 1 FROM information_schema.tables
WHERE table_name = $1 AND table_schema = current_schema())`,
name).Scan(&exists)
} else {
err = r.db.QueryRow(
`SELECT EXISTS (SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?)`,
name).Scan(&exists)
}
return db.TableExists(r.db, name)
}
func (r *Repository) tableExistsContext(ctx context.Context, name string) (bool, error) {
exists, err := db.TableExistsContext(ctx, r.db, name)
if err != nil {
return false, fmt.Errorf("probe table %s: %w", name, err)
}
@@ -266,7 +261,7 @@ func (r *Repository) cutoverEnsureLegacyBranchSlug(tx *sqlx.Tx) error {
if has {
return nil
}
if _, err := tx.Exec(`ALTER TABLE task_session_worktrees ADD COLUMN branch_slug TEXT NOT NULL DEFAULT ''`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `ALTER TABLE task_session_worktrees ADD COLUMN branch_slug TEXT NOT NULL DEFAULT ''`); err != nil {
return fmt.Errorf("cutover: add legacy branch_slug: %w", err)
}
return nil
@@ -274,19 +269,7 @@ func (r *Repository) cutoverEnsureLegacyBranchSlug(tx *sqlx.Tx) error {
// columnExists reports whether a column exists on either database engine.
func (r *Repository) columnExists(tx *sqlx.Tx, table, column string) (bool, error) {
var exists bool
var err error
if dialect.IsPostgres(r.db.DriverName()) {
err = tx.QueryRow(`
SELECT EXISTS (
SELECT 1 FROM information_schema.columns
WHERE table_name = $1 AND column_name = $2 AND table_schema = current_schema())`,
table, column).Scan(&exists)
} else {
err = tx.QueryRow(`SELECT EXISTS (
SELECT 1 FROM pragma_table_info(?) WHERE name = ?)`,
table, column).Scan(&exists)
}
exists, err := db.ColumnExistsContext(r.migrationContext(), tx, table, column)
if err != nil {
return false, fmt.Errorf("probe column %s.%s: %w", table, column, err)
}
apps/backend/internal/task/repository/sqlite/worktree_ownership_normalize.go
+7 -5
View File
@@ -1,6 +1,7 @@
package sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
@@ -20,6 +21,7 @@ const (
// worktreeCutover holds the legacy inventory and the normalized result.
type worktreeCutover struct {
ctx context.Context
envs map[string]*legacyEnv
taskEnvs map[string][]*legacyEnv
sessions map[string]*legacySession
@@ -93,7 +95,7 @@ func (c *worktreeCutover) loadLegacy(tx *sqlx.Tx, legacyEnvColumns, legacyRepoCo
func (c *worktreeCutover) loadLegacyEnvs(tx *sqlx.Tx, columns map[string]bool) error {
// Every interpolated expression comes from the fixed deprecated-column allowlist.
//nolint:gosec
rows, err := tx.Query(fmt.Sprintf(`
rows, err := tx.QueryContext(c.ctx, fmt.Sprintf(`
SELECT id, task_id, %s, executor_type, executor_id,
executor_profile_id, control_port, status,
%s, %s, %s, COALESCE(workspace_path, ''),
@@ -137,7 +139,7 @@ func (c *worktreeCutover) loadLegacySessions(tx *sqlx.Tx) error {
// query (SQLite is lenient about functional dependency). started_at is
// scanned as a string because MIN() makes the driver return the raw
// TEXT value instead of a time.
rows, err := tx.Query(`
rows, err := tx.QueryContext(c.ctx, `
SELECT ts.id, ts.task_id, COALESCE(ts.task_environment_id, ''), ts.state,
COALESCE(ts.executor_id, ''), COALESCE(ts.executor_profile_id, ''),
COALESCE(ts.repository_id, ''), COALESCE(er.container_id, ''),
@@ -186,7 +188,7 @@ func parseLegacyTimestamp(raw string) time.Time {
func (c *worktreeCutover) loadLegacyEnvRepos(tx *sqlx.Tx, columns map[string]bool) error {
// Every interpolated expression comes from the fixed lifecycle-column allowlist.
//nolint:gosec
rows, err := tx.Query(fmt.Sprintf(`
rows, err := tx.QueryContext(c.ctx, fmt.Sprintf(`
SELECT id, task_environment_id, repository_id, COALESCE(branch_slug, ''),
COALESCE(worktree_id, ''), COALESCE(worktree_path, ''),
COALESCE(worktree_branch, ''), position, COALESCE(error_message, ''),
@@ -228,7 +230,7 @@ func legacyRepoColumnExpr(columns map[string]bool, column, fallback string) stri
}
func (c *worktreeCutover) loadLegacySessionWorktrees(tx *sqlx.Tx) error {
rows, err := tx.Query(`
rows, err := tx.QueryContext(c.ctx, `
SELECT session_id, worktree_id, repository_id, COALESCE(branch_slug, ''),
position, COALESCE(worktree_path, ''), COALESCE(worktree_branch, ''),
status, created_at, updated_at, merged_at, deleted_at
@@ -710,7 +712,7 @@ func (c *worktreeCutover) resolveExecutorType(tx *sqlx.Tx, executorID string) (s
return cached, nil
}
var executorType string
err := tx.QueryRow(tx.Rebind(`SELECT type FROM executors WHERE id = ?`), executorID).Scan(&executorType)
err := tx.QueryRowContext(c.ctx, tx.Rebind(`SELECT type FROM executors WHERE id = ?`), executorID).Scan(&executorType)
if errors.Is(err, sql.ErrNoRows) {
c.executorTypes[executorID] = executorTypeLocalPC
return executorTypeLocalPC, nil
apps/backend/internal/task/repository/sqlite/worktree_ownership_targets.go
+16 -16
View File
@@ -220,10 +220,10 @@ func (t *envRepoTarget) verifyWorktree(worktreeID, path, branch string) error {
// cutoverBuildShadows creates the final-shape shadow tables.
func (r *Repository) cutoverBuildShadows(tx *sqlx.Tx) error {
if _, err := tx.Exec(finalTaskEnvironmentsDDL("task_environments_shadow")); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), finalTaskEnvironmentsDDL("task_environments_shadow")); err != nil {
return fmt.Errorf("cutover: create task_environments_shadow: %w", err)
}
if _, err := tx.Exec(finalTaskEnvironmentReposDDL("task_environment_repos_shadow", "task_environments_shadow")); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), finalTaskEnvironmentReposDDL("task_environment_repos_shadow", "task_environments_shadow")); err != nil {
return fmt.Errorf("cutover: create task_environment_repos_shadow: %w", err)
}
return r.maybeFailCutover("create_shadow")
@@ -237,7 +237,7 @@ func (r *Repository) insertNormalized(c *worktreeCutover, tx *sqlx.Tx) error {
if !ok {
return fmt.Errorf("cutover: internal: environment %s for task %s missing", envID, taskID)
}
if _, err := tx.Exec(tx.Rebind(`
if _, err := tx.ExecContext(r.migrationContext(), tx.Rebind(`
INSERT INTO task_environments_shadow (
id, task_id, ownership_generation, executor_type, executor_id, executor_profile_id,
control_port, status, materialization_session_id, workspace_path, container_id,
@@ -270,7 +270,7 @@ func (r *Repository) insertNormalized(c *worktreeCutover, tx *sqlx.Tx) error {
if envID == "" {
continue
}
if _, err := tx.Exec(tx.Rebind(
if _, err := tx.ExecContext(r.migrationContext(), tx.Rebind(
`UPDATE task_sessions SET task_environment_id = ? WHERE id = ?`),
envID, sessionID); err != nil {
return fmt.Errorf("cutover: link session %s to environment %s: %w", sessionID, envID, err)
@@ -293,7 +293,7 @@ func (c *worktreeCutover) insertRepoRow(tx *sqlx.Tx, taskID, envID string, targe
if target.deletedAt != nil {
deletedAt = *target.deletedAt
}
if _, err := tx.Exec(tx.Rebind(`
if _, err := tx.ExecContext(c.ctx, tx.Rebind(`
INSERT INTO task_environment_repos_shadow (
id, task_environment_id, repository_id, branch_slug,
worktree_id, worktree_path, worktree_branch, position,
@@ -462,11 +462,11 @@ func (r *Repository) tableColumns(tx *sqlx.Tx, table string) (map[string]bool, e
var rows *sql.Rows
var err error
if dialect.IsPostgres(r.db.DriverName()) {
rows, err = tx.Query(`
rows, err = tx.QueryContext(r.migrationContext(), `
SELECT column_name FROM information_schema.columns
WHERE table_name = $1 AND table_schema = current_schema()`, table)
} else {
rows, err = tx.Query(`SELECT name FROM pragma_table_info(?)`, table)
rows, err = tx.QueryContext(r.migrationContext(), `SELECT name FROM pragma_table_info(?)`, table)
}
if err != nil {
return nil, fmt.Errorf("cutover: read columns of %s: %w", table, err)
@@ -490,18 +490,18 @@ func (r *Repository) tableColumns(tx *sqlx.Tx, table string) (map[string]bool, e
// the swap. Postgres constraint names are restored to their canonical form
// afterwards.
func (r *Repository) cutoverSwap(c *worktreeCutover, tx *sqlx.Tx) error {
if _, err := tx.Exec(`DROP TABLE task_session_worktrees`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `DROP TABLE task_session_worktrees`); err != nil {
return fmt.Errorf("cutover: drop task_session_worktrees: %w", err)
}
// Preview-build session-delete cleanup jobs are invalid under the
// task-owned model; purge them before any cleanup worker can claim them.
if _, err := tx.Exec(tx.Rebind(`DELETE FROM task_resource_cleanup_jobs WHERE trigger = ?`), "session_delete"); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), tx.Rebind(`DELETE FROM task_resource_cleanup_jobs WHERE trigger = ?`), "session_delete"); err != nil {
return fmt.Errorf("cutover: purge preview session_delete cleanup jobs: %w", err)
}
if err := r.maybeFailCutover("drop_legacy"); err != nil {
return err
}
if _, err := tx.Exec(`DROP TABLE task_environment_repos`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `DROP TABLE task_environment_repos`); err != nil {
return fmt.Errorf("cutover: drop legacy task_environment_repos: %w", err)
}
if err := r.rebindGitSnapshotEnvironmentForeignKey(c, tx); err != nil {
@@ -510,13 +510,13 @@ func (r *Repository) cutoverSwap(c *worktreeCutover, tx *sqlx.Tx) error {
if err := r.rebindTaskEnvironmentRecoveryClaimForeignKey(c, tx); err != nil {
return err
}
if _, err := tx.Exec(`DROP TABLE task_environments`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `DROP TABLE task_environments`); err != nil {
return fmt.Errorf("cutover: drop legacy task_environments: %w", err)
}
if _, err := tx.Exec(`ALTER TABLE task_environments_shadow RENAME TO task_environments`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `ALTER TABLE task_environments_shadow RENAME TO task_environments`); err != nil {
return fmt.Errorf("cutover: rename task_environments_shadow: %w", err)
}
if _, err := tx.Exec(`ALTER TABLE task_environment_repos_shadow RENAME TO task_environment_repos`); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), `ALTER TABLE task_environment_repos_shadow RENAME TO task_environment_repos`); err != nil {
return fmt.Errorf("cutover: rename task_environment_repos_shadow: %w", err)
}
if err := r.maybeFailCutover("swap"); err != nil {
@@ -533,7 +533,7 @@ func (r *Repository) cutoverSwap(c *worktreeCutover, tx *sqlx.Tx) error {
`CREATE INDEX IF NOT EXISTS idx_task_environment_repos_env_id ON task_environment_repos(task_environment_id)`,
`CREATE INDEX IF NOT EXISTS idx_task_environment_repos_repository_id ON task_environment_repos(repository_id)`,
} {
if _, err := tx.Exec(stmt); err != nil {
if _, err := tx.ExecContext(r.migrationContext(), stmt); err != nil {
return fmt.Errorf("cutover: recreate index: %w", err)
}
}
@@ -556,7 +556,7 @@ func (r *Repository) cutoverRenamePostgresConstraints(tx *sqlx.Tx) error {
}
for _, rename := range renames {
var constraintName string
err := tx.QueryRow(tx.Rebind(`
err := tx.QueryRowContext(r.migrationContext(), tx.Rebind(`
SELECT conname FROM pg_constraint
WHERE conrelid = (SELECT oid FROM pg_class WHERE relname = ? AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = current_schema()))
AND conname LIKE ?
@@ -564,7 +564,7 @@ func (r *Repository) cutoverRenamePostgresConstraints(tx *sqlx.Tx) error {
if err != nil {
return fmt.Errorf("cutover: find postgres constraint on %s: %w", rename.table, err)
}
if _, err := tx.Exec(fmt.Sprintf(
if _, err := tx.ExecContext(r.migrationContext(), fmt.Sprintf(
`ALTER TABLE %s RENAME CONSTRAINT %s TO %s`, rename.table, constraintName, rename.canonical)); err != nil {
return fmt.Errorf("cutover: rename postgres constraint %s: %w", constraintName, err)
}
docs/plans/large-database-startup/plan.md
+84
View File
@@ -0,0 +1,84 @@
---
created: 2026-09-11
status: complete
requirements:
- REQ-PLATFORM-STARTUP-LIFECYCLE-001
system_design:
- ../../specs/platform/system-design/startup-lifecycle.md
legacy_specs: []
---
# Large database startup implementation plan
## Root cause
`provideRepositories` performs the upgrade backup and migrations before
`startGatewayAndServe` binds the bootstrap listener. The reported 42.5-second
backup consumes almost all of the 45-second listener budget.
## Scope and approach
Move shared bootstrap ownership before initialization. Add typed phase reporting
and cancellation-aware backup execution. Preserve version gates, retention,
health identity, binding semantics, and background recovery ordering.
## Work orders
- [x] [01: Startup lifecycle](task-01-lifecycle.md)
- [x] [02: Startup measurements and documentation](task-02-evidence.md)
## Verification
Backend commands run from `apps/backend`; script and diff commands run from root.
- `rtk go test -race ./internal/backendapp ./internal/persistence ./internal/launcher ./internal/common/config ./internal/db`
- `rtk go test ./internal/task/repository/sqlite -run 'Test(StartupMigrationCosts|StartupMigrationCostsPopulated|NewWithDBContextStopsActiveMigration|BackfillPromptSeqStopsOnCanceledContext)' -v`
- `rtk proxy python3 scripts/lint-spec-files.py --all`
- `rtk proxy node --test scripts/validate-public-docs.test.mjs`
- `rtk proxy node scripts/validate-public-docs.mjs`
- `rtk git diff --check`
## End-to-end evidence
Use real HTTP listeners with controlled initialization and short launcher health
deadlines. Cover success, failure, route isolation, cancellation, and bind failure.
No browser UI changes or live database access are needed.
## Risks
Legacy constructors include non-context SQL. Shutdown must not race cleanup with
those constructors. Retry retention can discard the original snapshot; no reuse
or retention changes are authorized in this package.
## Results
Implemented early bootstrap ownership, typed startup phases, cancellation-aware
SQLite backup and task migration execution, worker-only restore quiescing,
launcher phase diagnostics, populated startup measurements, and public
troubleshooting documentation.
Validation passed:
- `env -u KANDEV_INTERNAL_CONFIG_FILE -u KANDEV_INTERNAL_CONFIG_HOME_FILE -u KANDEV_INTERNAL_AGENTCTL_STARTUP_CONFIG -u KANDEV_HEALTH_TIMEOUT_MS go test -race ./internal/backendapp ./internal/persistence ./internal/launcher ./internal/common/config ./internal/db -count=1`
- `go test -race ./internal/startup -count=1`
- `go test -race ./internal/persistence/storeconformance -count=1`
- `go test ./internal/task/repository/sqlite -run 'Test(StartupMigrationCosts|StartupMigrationCostsPopulated|NewWithDBContextStopsActiveMigration|BackfillPromptSeqStopsOnCanceledContext)$' -v -count=1`
- `make -C apps/backend build`
- `make -C apps/backend lint` (0 issues)
- `go run ./cmd/sqlguard ./internal`
- `python3 scripts/lint-spec-files.py --all`
- `node --test scripts/validate-public-docs.test.mjs` (62 passed)
- `node scripts/validate-public-docs.mjs` (46 published pages)
- `git diff --check`
The populated isolated SQLite evidence is recorded in
[task-02-evidence](task-02-evidence.md). It covers a 929,792-byte database,
6,000 messages, a 2,252,800-byte `VACUUM INTO` snapshot, 285.06 ms replay,
and separate recurring backfill timings on Linux amd64 with Go 1.26.0. The
measurement remains synthetic and does not predict production startup time.
Deferred risks remain outside this package: backup retry retention and the
backup-registry redesign are unchanged. Constructors for non-task stores still
have bounded admission barriers rather than a new shared context API; an
already admitted statement can finish according to its driver's cancellation
behavior before the next store is rejected.
docs/plans/large-database-startup/task-01-lifecycle.md
+75
View File
@@ -0,0 +1,75 @@
---
id: "01-lifecycle"
title: "Startup lifecycle"
status: done
wave: 1
depends_on: []
plan: plan.md
requirements:
- REQ-PLATFORM-STARTUP-LIFECYCLE-001
acceptance_criteria:
- AC-PLATFORM-STARTUP-LIFECYCLE-001.1
- AC-PLATFORM-STARTUP-LIFECYCLE-001.2
- AC-PLATFORM-STARTUP-LIFECYCLE-001.3
- AC-PLATFORM-STARTUP-LIFECYCLE-001.4
- AC-PLATFORM-STARTUP-LIFECYCLE-001.5
- AC-PLATFORM-STARTUP-LIFECYCLE-001.6
- AC-PLATFORM-STARTUP-LIFECYCLE-001.7
- AC-PLATFORM-STARTUP-LIFECYCLE-001.8
system_design:
- ../../specs/platform/system-design/startup-lifecycle.md
---
# Startup lifecycle
## Scope
Early bootstrap ownership, phase reporting, cancellation, backup ordering, and launcher diagnostics.
## Exclusions
No backup mechanism replacement, registry conversion, retention redesign, or publication.
## Acceptance
- Meet the referenced lifecycle criteria with isolated regression evidence.
- Preserve health identity, timeout overrides, and required persistence gates.
- Record exact verification results and measurement limitations.
## Files likely touched
`apps/backend/internal/backendapp`, `internal/persistence`, `internal/launcher`,
`internal/db`, task repository measurement tests, and owning/public documentation.
## Verification
Use the corresponding commands in [plan](plan.md#verification), from their stated directories.
## Dependencies
None.
## Parallelism
Sequential. No delegation.
## Risks
See plan and system design for cancellation and retry retention constraints.
## Results
Implemented and tested the lifecycle changes. The bootstrap listener now binds
before database opening, backup, migrations, service initialization, and
session recovery. `/health` is live during initialization, `/ready` reports a
typed phase with elapsed measurements, cancellation closes listeners while
initialization drains, and the launcher reports the last phase on early exit.
The backend lifecycle and launcher race suites passed. Backup failure remains
before migrations, canceled persistence does not start backup work, and
restore quiescing cancels workers without ending the process lifetime before
the restore result is delivered. Repository startup checks cancellation at
schema and store admission boundaries; task migrations pass cancellation into
their data statements and table rebuilds. An in-flight database statement can
still finish according to the driver's cancellation semantics, and cleanup
waits for the synchronous initializer to return.
docs/plans/large-database-startup/task-02-evidence.md
+107
View File
@@ -0,0 +1,107 @@
---
id: "02-evidence"
title: "Startup evidence"
status: done
wave: 2
depends_on: ["01-lifecycle"]
plan: plan.md
requirements:
- REQ-PLATFORM-STARTUP-LIFECYCLE-001
acceptance_criteria:
- AC-PLATFORM-STARTUP-LIFECYCLE-001.1
- AC-PLATFORM-STARTUP-LIFECYCLE-001.2
- AC-PLATFORM-STARTUP-LIFECYCLE-001.3
- AC-PLATFORM-STARTUP-LIFECYCLE-001.4
- AC-PLATFORM-STARTUP-LIFECYCLE-001.5
- AC-PLATFORM-STARTUP-LIFECYCLE-001.6
- AC-PLATFORM-STARTUP-LIFECYCLE-001.7
- AC-PLATFORM-STARTUP-LIFECYCLE-001.8
system_design:
- ../../specs/platform/system-design/startup-lifecycle.md
---
# Startup evidence
## Scope
Isolated populated SQLite backup and migration timing, deferred registry and
retention findings, public troubleshooting documentation.
## Exclusions
No backup mechanism replacement, registry conversion, retention redesign, or publication.
## Acceptance
- Meet the referenced lifecycle criteria with isolated regression evidence.
- Preserve health identity, timeout overrides, and required persistence gates.
- Record exact verification results and measurement limitations.
## Files likely touched
`apps/backend/internal/backendapp`, `internal/persistence`, `internal/launcher`,
`internal/db`, task repository measurement tests, and owning/public documentation.
## Verification
Use the corresponding commands in [plan](plan.md#verification), from their stated directories.
## Dependencies
Task 01.
## Parallelism
Sequential. No delegation.
## Risks
See plan and system design for cancellation and retry retention constraints.
## Results
`TestStartupMigrationCostsPopulated` creates a fresh schema, then rebuilds
only the message table into the legacy shape needed by the recurring startup
backfills. It inserts 24 task sessions and 6,000 messages (one third user
messages), leaves `updated_at` nullable and empty, and omits `prompt_seq` so a
replay exercises both migration paths. The test also runs the same
`VACUUM INTO` snapshot used by production and measures the two backfills
separately after replay. The fixture is isolated under `t.TempDir()` and can be
reproduced with:
```text
go test ./internal/task/repository/sqlite -run '^TestStartupMigrationCostsPopulated$' -v -count=1
```
One recorded run on Linux amd64, Go 1.26.0, 11 CPUs, reported:
| Measurement | Result |
| --- | ---: |
| Populated database file before replay | 929,792 bytes |
| `VACUUM INTO` backup | 2,252,800 bytes in 18.99 ms |
| Fresh schema initialization | 48.75 ms |
| Populated replay | 285.06 ms |
| `task_session_messages.updated_at` backfill | 19.25 ms |
| Prompt sequence update plus counter aggregation | 41.52 ms |
The values are observations from one local run, not startup budgets. The
fixture does not model a production database's row distribution, WAL state,
filesystem, CPU contention, provider recovery, or full backup directory. It
does distinguish the liveness and readiness contract: the bootstrap tests keep
`/health` at 200 while `/ready` and application routes remain 503 until the
initializer publishes the router, and the launcher health deadline is not
used as the readiness deadline. The evidence does not change `VACUUM INTO`,
backup retry behavior, or backup-registry and retention design.
Lifecycle regression coverage now includes delayed success past a short health
deadline, active SQL cancellation through `NewWithDBContext`, worker-only
restore quiescing with `restart_required`, explicit post-readiness signal
handling, and initializer joining after cancellation.
PR review follow-up also verifies the retry boundaries: pre-migration snapshots
are created in a private staging directory, validated, set to mode `0600`, and
installed only after validation; a failed `VACUUM INTO` destination is removed
when the path did not exist before the attempt. The authenticated system-job
notification path remains on the process lifetime while restore workers are
quiesced. These changes do not alter backup retention or the deferred backup
registry redesign.
docs/public/cli.md
+5
View File
@@ -132,6 +132,11 @@ On a normal start, the launcher:
7. waits for `/ready` (the backend has finished startup recovery and can serve real requests); this wait is unbounded by design, since recovery can legitimately take much longer than 45 seconds; and
8. opens the reachable access URL in the default browser.
While `/ready` returns 503, it reports the current startup phase and elapsed
time. The launcher prints phase changes and periodic status, so a long backup,
migration, or recovery can be distinguished from an unreachable backend. These
values describe elapsed work and do not estimate completion.
The launcher remains in the foreground. Press `Ctrl+C` or terminate it to stop the backend and its managed children cleanly. A force-kill can leave worktree processes or containers running; inspect them before deleting data.
Use headless mode for SSH sessions, containers, or an external reverse proxy:
docs/public/docker.md
+1 -1
View File
@@ -263,7 +263,7 @@ Remote Docker profiles are not a workaround: that executor runtime is currently
## Health and observability
`GET /health` returns 200 as soon as the listener is accepting connections, even mid-startup; `GET /ready` returns 503 during startup and 200 after routes are registered:
`GET /health` returns 200 as soon as the listener is accepting connections, even mid-startup. `GET /ready` returns 503 during startup and includes the current startup phase and elapsed time; it returns 200 after routes are registered:
```bash
curl --fail http://localhost:38429/ready
docs/public/k8s.md
+1 -1
View File
@@ -489,7 +489,7 @@ The universal image already runs as `kandev`; use `kubectl exec -it deployment/k
The example requests 250 millicores and 512 MiB, with limits of 2 CPU and 2 GiB. Those are placeholders, not capacity recommendations. Local/Worktree agents share the pod limit with the control plane and can exceed it during builds. Measure workload memory, CPU, ephemeral storage, PVC growth, and process counts; then set requests/limits accordingly.
The example liveness probe calls `/health`; the example readiness probe calls `/ready`. `/health` returns 200 as soon as the TCP listener accepts connections, before startup finishes: it confirms the process is alive, not that it can serve real traffic, so gating liveness on it never restarts a pod that is merely still starting up. `/ready` returns 503 until routes are wired, the agent registry is seeded, and (in e2e builds) the mock-harness routes are mounted, then 200. Neither is a deep check of database, repository, Docker, provider, or agent health.
The example liveness probe calls `/health`; the example readiness probe calls `/ready`. `/health` returns 200 as soon as the TCP listener accepts connections, before startup finishes: it confirms the process is alive, not that it can serve real traffic, so gating liveness on it never restarts a pod that is merely still starting up. `/ready` returns 503 until routes are wired, the agent registry is seeded, and (in e2e builds) the mock-harness routes are mounted, then 200. While it returns 503, its `startup` object identifies the current database, migration, service, or recovery phase and reports elapsed time without estimating completion. Neither is a deep check of database, repository, Docker, provider, or agent health.
Long migrations or slow storage may need a startup probe to prevent premature liveness restarts:
docs/public/operations.md
+20 -2
View File
@@ -83,7 +83,25 @@ It returns HTTP 200 as soon as the listener accepts connections, with:
curl -fsS http://127.0.0.1:38429/ready
```
It returns HTTP 503 with `status: "starting"` (plus the same `version`) until routes, the agent registry, and the listener are ready, then HTTP 200. The supplied Kubernetes liveness probe uses `/health`; the readiness probe uses `/ready`. Both are unauthenticated even when auth is enabled, so they're also a credential-free way for monitoring to read the running version; there is no need to authenticate to **System > About** just to check what build is deployed.
It returns HTTP 503 with `status: "starting"` (plus the same `version`) until routes, the agent registry, and the listener are ready, then HTTP 200. During startup, the response also includes a bounded `startup` object:
```json
{
"phase": "backing_up_database",
"elapsed_ms": 42000,
"phase_elapsed_ms": 18000
}
```
The phase is a label such as `opening_database`, `backing_up_database`,
`applying_migrations`, `initializing_services`, or `recovering_sessions`.
Elapsed values are status measurements, not progress estimates. The launcher
prints phase changes and periodic elapsed status, and names the last phase if
the backend exits before readiness. The supplied Kubernetes liveness probe uses
`/health`; the readiness probe uses `/ready`. Both are unauthenticated even
when auth is enabled, so they're also a credential-free way for monitoring to
read the running version; there is no need to authenticate to **System > About**
just to check what build is deployed.
For application diagnostics, open **Settings > System > Status** or request:
@@ -422,7 +440,7 @@ The UI flow applies to the configured SQLite path:
1. Stop or finish active agent sessions and preserve unpushed work.
2. Open **Settings > System > Backups**, choose **Restore**, type `RESTORE`, and confirm.
3. Kandev stops scheduling, active executions, and database-backed workers. It copies the selected snapshot to `<configured-database-path>.new`, validates the SQLite checkpoint result, and closes the pool. It then quarantines the configured database and its `-wal`/`-shm` sidecars before installing the staged file. If installation fails, Kandev restores the quarantined files.
3. Kandev stops scheduling, active executions, and database-backed workers. It copies the selected snapshot to `<configured-database-path>.new`, validates the SQLite checkpoint result, and closes the pool. It then quarantines the configured database and its `-wal`/`-shm` sidecars before installing the staged file. If installation fails, Kandev restores the quarantined files. The restore job remains available long enough to report `restart_required` before process shutdown.
4. Click **Restart Kandev** in the success dialog. If automatic restart is unavailable, quit and relaunch Kandev manually. The backend must restart before database-backed work resumes.
5. Check `/ready`, **System > Status**, database schema version, secrets, and representative tasks.
docs/public/run-as-a-service.md
+1 -1
View File
@@ -58,7 +58,7 @@ The installer writes the managed unit or plist and starts the process. It does *
curl --fail http://127.0.0.1:43127/ready
```
`/ready` reports backend readiness after routes and the agent registry are initialized and the HTTP listener is accepting connections. It is not a deep health check of the database, message bus, executors, or remote providers.
`/ready` reports backend readiness after routes and the agent registry are initialized and the HTTP listener is accepting connections. While it returns 503, its `startup` object reports the current phase and elapsed time for database opening, backup, migrations, service initialization, or session recovery. It is not a deep health check of the database, message bus, executors, or remote providers.
### System service
docs/specs/platform/README.md
+4
View File
@@ -35,6 +35,8 @@ retains ownership of its values, validation, authority, and persistence.
### Requirements
- [Startup lifecycle](requirements/startup-lifecycle.md)
- [Agent-accessible Kandev settings](requirements/agent-settings-parity.md)
@@ -80,6 +82,8 @@ retains ownership of its values, validation, authority, and persistence.
### System design
- [Startup lifecycle](system-design/startup-lifecycle.md)
- [Agent-accessible Kandev settings](system-design/agent-settings-parity.md)
- [Core settings domain adoption](system-design/agent-settings-domains.md)
docs/specs/platform/requirements/health-endpoint-version.md
+2 -2
View File
@@ -65,11 +65,11 @@ resolved by explicit decision — see [Decisions](#decisions).
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.1:** `GET /health` SHALL include the running Kandev version in its JSON response body.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.2:** The field SHALL be named **`version`**, matching `GET /api/v1/system/info`.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.3:** The field SHALL be present in **both** the ready (200) and not-ready (503) responses, so an operator can identify the build of a backend that is stuck starting.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.3:** The field SHALL be present in both `/health` and bootstrap `/ready` responses, so an operator can identify the build of a backend that is stuck starting.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.4:** The field value SHALL be the same string `GET /api/v1/system/info` reports as `version` for the same running process.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.5:** The field SHALL be served to unauthenticated callers, exactly as the rest of the `/health` payload already is. This is a deliberate, accepted disclosure — see [Security](#security-and-permissions).
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.6:** Every existing field (`status`, `service`, `mode`) SHALL retain its current name, value, and semantics. This is a purely additive change.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.7:** The existing HTTP status semantics (200 when ready, 503 while starting) SHALL be unchanged.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.7:** `/health` SHALL return 200 once bound, including during initialization. `/ready` SHALL return 503 until application initialization finishes.
- **AC-PLATFORM-HEALTH-ENDPOINT-VERSION-001.8:** The desktop health-token response header SHALL continue to be set on the 200 path only, unchanged.
## System design
docs/specs/platform/requirements/startup-lifecycle.md
+39
View File
@@ -0,0 +1,39 @@
---
status: active
system: platform
created: 2026-09-11
owners:
- kandev
---
# Startup lifecycle requirements
## Overview
Platform owns process startup, liveness, readiness, and shutdown across native launch modes and desktop.
Large databases must complete initialization without a listener-health deadline interrupting valid work.
## Requirements
### REQ-PLATFORM-STARTUP-LIFECYCLE-001: Observable initialization
**Intent:** Separate process liveness from application readiness while preserving upgrade safety.
#### Acceptance criteria
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.1:** During database opening, backup, migration, and required recovery, `/health` shall answer successfully with existing version and identity semantics.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.2:** Until required initialization and router installation finish, `/ready` and application endpoints shall remain unsuccessful.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.3:** After liveness succeeds, the launcher shall wait for readiness without an initialization deadline. Explicit listener-timeout overrides shall remain effective.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.4:** Startup shall report phase transitions and elapsed time, with bounded periodic status during long phases and no estimated percentages.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.5:** Failed initialization shall report its phase and underlying error, release resources, and exit unsuccessfully. The launcher shall detect child exit.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.6:** Cancellation shall close startup listeners and stop further initialization admission at safe schema and store boundaries. Signal handling shall remain available throughout startup and shutdown. Restore worker quiescing shall not cancel the process and HTTP lifetime before the restore job can publish its result.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.7:** A required upgrade backup shall complete before application migrations mutate the database. Backup failure shall prevent migrations.
- **AC-PLATFORM-STARTUP-LIFECYCLE-001.8:** A listener bind failure shall stop startup promptly and identify the bind error.
## Exclusions
Backup reuse, retention redesign, migration-registry conversion, and background lifecycle-sweep scheduling are outside this change.
## System design
See [startup lifecycle](../system-design/startup-lifecycle.md).
docs/specs/platform/system-design/startup-lifecycle.md
+105
View File
@@ -0,0 +1,105 @@
---
status: current
system: platform
requirements:
- REQ-PLATFORM-STARTUP-LIFECYCLE-001
---
# Startup lifecycle design
## Ownership and flow
Platform owns this shared backend lifecycle. Configuration, runtime-state ownership,
logging, and startup security validation precede the bootstrap listener.
The listener precedes event-bus and database initialization. One handler switch,
HTTP server, and listener set survive through application-router installation.
Only the bootstrap handler serves traffic during initialization.
`backendapp.run` owns the listener and cancellation lifetime. Startup state travels
through context to persistence and router composition. Initialization remains
sequential; cleanup must not race constructors or database mutation.
Signal handling begins before database work. The first signal cancels startup;
the existing second-signal forced-exit behavior remains available.
The bootstrap owns separate process and worker cancellation contexts. A startup
signal cancels the process context and closes the bootstrap listener while the
initializer drains. Restore quiescing cancels only worker and runtime contexts;
the process context, HTTP listener, and job tracker remain alive until restore
has installed the staged database and published `restart_required`, after
which an explicit shutdown can close the process.
## Readiness and status
`/health` retains its existing body and desktop token header. Bootstrap `/ready`
returns 503 with a `startup` object containing a fixed phase name, total elapsed
milliseconds, and phase elapsed milliseconds. No paths, SQL, credentials, or raw
errors enter this unauthenticated payload. Application routes remain 503.
Phases cover database opening, backup, migrations, service initialization,
required session recovery, and ready. Structured logs record transitions and
phase duration. Launcher readiness polling reads JSON, reports transitions and
periodic elapsed status, and remembers the last phase for child-exit diagnostics.
Underlying errors remain in backend diagnostic output. Readiness waits retain
cancellation and child-exit checks without a separate deadline.
Router publication occurs only after required stores, recovery gates, final
persistence checks, and router construction succeed. The existing background
lifecycle-token sweep remains outside readiness: its watcher/scheduler ordering
and deadline are unchanged.
## Timeout policy
Keep release listener-health timeout at 45000 ms, development/E2E at 600000 ms.
Keep environment and YAML overrides. A 300000 ms default gives no database-work
benefit after early binding; it delays genuine pre-listener failure detection.
Desktop retains its existing independent 60-second liveness budget and unlimited
readiness wait. No timeout default changes are required.
Initialization checks cancellation before and after each repository schema step
and before each required store admission. The task repository passes the same
context to its migration SQL, including data backfills and transactional table
rebuilds. Other legacy constructors can finish an already admitted statement
because they do not expose a context API; the next admission barrier prevents
later stores from opening. A statement already executing in a context-aware
driver is also subject to that driver's cancellation behavior, so cancellation
does not promise an immediate interrupt at every SQLite virtual-machine or
user-function instruction. Cleanup waits until the synchronous initializer
returns, avoiding a pool-close race with in-flight SQL.
## Persistence safety
Keep `VACUUM INTO`, version-based backup eligibility, and two-snapshot retention.
Context cancellation applies to the backup SQL. Repository initialization failure
must release previously constructed resources; a completed backup remains intact.
The successful version marker stays behind the final persistence/router gates.
Pending migrations cannot be inferred from `MigrateLogger`: owners also execute
DDL, table rebuilds, data repairs, and seed changes directly, including stores
constructed after `provideRepositories`. A future registry must account for all
required stores, late constructors, dialect-specific DDL, data backfills, and
seed migrations, with an immutable applied ledger and completeness tests.
Only then can backup eligibility safely depend on pending work.
Retries before version recording take another backup. After partial migration,
that snapshot is not necessarily a pre-upgrade image. Two successful retries can
prune the original snapshot. Filename timestamps have one-second resolution and
can also collide on rapid retries. This existing risk requires a separate recovery
design: database identity, durable attempt state, original-snapshot pinning,
crash-safe retention, and restore tests. Never reuse by filename or target version.
## Requirement mapping
| Criteria | Mechanism |
| --- | --- |
| .1, .2, .3, .8 | Early bootstrap binding, atomic router handoff, existing launcher waits |
| .4, .5 | Typed phase snapshot, structured timing, child-exit diagnostics |
| .6, .7 | Separate process/worker cancellation ownership, schema/store admission barriers, sequential initialization, backup gate |
## Decisions
This extends the existing bootstrap architecture rather than creating a new
persistence contract. See [database upgrade safety](../../../decisions/0008-db-upgrade-safety.md).
## Implementation plans
- [Large database startup](../../../plans/large-database-startup/plan.md)
docs/specs/startup-listener-before-recovery/spec.md
+7 -163
View File
@@ -1,165 +1,9 @@
---
status: draft
created: 2026-08-22
revised: 2026-09-05
type: repair
---
# Startup listener before recovery
# The Backend Must Bind and Answer Liveness Before Startup Recovery Runs
The authoritative lifecycle contract is now [startup lifecycle requirements](../platform/requirements/startup-lifecycle.md)
and [startup lifecycle design](../platform/system-design/startup-lifecycle.md).
## Problem
The backend does not bind its HTTP socket until orchestrator startup
reconciliation has finished, and that reconciliation can wait on agent
processes for hours. Under the launcher's health window the process is killed
before it ever listens, restarted, and the cycle repeats without converging.
`backendapp.startGatewayAndServe` calls `orchestrator.Service.Start` at
`main.go:877` and reaches the first `net.Listen` only via `main.go:1064` ->
`httpserver.go:76`. `/health` is a route on the server built after that point
(`helpers.go:716`); no earlier path serves it. So while `Start` runs there is
no listener at all, which is exactly what `lsof` showed.
`Start` runs `reconcileTaskLifecycleTokens` synchronously
(`service.go:2255`), fanning four workers over every task with a pending
lifecycle token and waiting on a `sync.WaitGroup`. The per-task work reaches
`autoStartStepPrompt` -> `PromptTask` -> `ensureSessionRunning` ->
`attemptColdResume`.
**The per-task cost is far larger than the prompt-ready timeout suggests.**
`attemptColdResume` first waits for session readiness via `waitForSessionReady`
(`task_operations.go:2498`), whose budget is `constants.AgentLaunchTimeout` =
**15 minutes** (`timeouts.go:44`, 10m setup script + 5m allowance). Only after
that does the 30s `waitForAgentPromptReady` apply, and the attempt is retried
once. Worst case is roughly 31 minutes per task, four at a time.
Measured on the affected instance: 424 tasks, 32 carrying
`manual_move_lifecycle` metadata, on a 1.0 GB database. A boot given a full
10-minute health window still never bound, and produced only 2 successful ACP
session creations in those 10 minutes. That is the signature of workers parked
in the 15-minute session-ready wait, not of workers cycling through tasks.
Three consequences, in severity order:
1. **Socket liveness is coupled to agent readiness.** The process cannot answer
a health probe, serve the UI, or expose the API until agents that may never
become ready are waited out.
2. **The failure is self-sustaining.** `healthTimeoutReleaseMS` is 45s
(`internal/launcher/constants.go`). The launcher declares a backend health
failure and stops the process; a supervisor restarts it; the same tokens are
still pending because none were cleared, and the runtimes orphaned by the
kill are added to the next boot's work. Ten consecutive boots were observed,
none of which bound a socket.
3. **The wait cannot be bounded from outside.** `attemptColdResume` and
`reapAndReloadSession` deliberately use `context.WithoutCancel`
(`task_operations.go:2308`, `task_operations.go:2255`) so a resume survives
its caller's cancellation. A deadline handed to the sweep therefore does not
stop an in-flight resume. Any fix that relies on cancelling the sweep to
protect startup is ineffective.
This is startup ordering, not a defect in the recovery logic. The same frames
are reached whenever a lifecycle token is pending.
## Correction (2026-09-05)
This spec's original Desired-behaviour and Out-of-scope sections asserted
that the sweep's position *before* `watcher.Start` was load-bearing and that
moving it would introduce races with live traffic. Runtime evidence gathered
after this spec's socket-bind fix shipped disproves that: with the socket-bind
fix in place, boots with pending lifecycle tokens still never reached `API
configured` — the sweep itself was still holding the readiness path open
indefinitely, just behind the new bootstrap 503 handler instead of behind a
closed socket.
The reason is a **self-inflicted event blackout**, not merely a slow
individual resume. `Watcher.Start` is what registers
`{events.AgentBootReady, w.handlers.OnAgentBootReady}` (the *only* handler
that flips a launching session to `WAITING_FOR_INPUT` on the resume path).
The event bus used here has no replay, so any `agent.boot_ready` published
while the sweep runs *before* `watcher.Start` is dropped on the floor.
`waitForSessionReady`'s poll can then never observe the state change it is
waiting for, and burns its full `constants.AgentLaunchTimeout` (15 minutes)
on every recovered task, regardless of whether the underlying agent process
is actually healthy. Four workers wide, retried up to three times, this is
indistinguishable from a hang and gates the entire router.
The fix inverts the ordering this spec forbade: the sweep now runs in a
background goroutine *after* `watcher.Start` and `scheduler.Start` succeed,
so the events its own resume path depends on are deliverable from the
moment recovery starts. Readiness is not gated on the sweep at all — the
socket-bind / bootstrap-503 behavior this spec introduced is unaffected and
still the mechanism that makes an in-progress sweep observable as "starting"
rather than a hang. The Desired-behaviour and Out-of-scope sections below
are amended accordingly; the regression scenarios still hold, since a bound
socket does not depend on this ordering.
## Broken behavior
- No socket is bound, and `/health` cannot be answered, until orchestrator
startup reconciliation returns.
- A backend whose recovery exceeds the health window is killed and restarted
with at least as much recovery work as before, indefinitely.
## Desired behavior
- The backend binds its socket and answers a liveness probe within the release
health window, regardless of how much recovery work is pending.
- Startup recovery runs in the background *after* the watcher and scheduler
start (amended 2026-09-05 — see Correction above): its own resume path
depends on events only the watcher's subscriptions deliver, so running it
earlier does not protect anything and instead guarantees every recovered
task times out. Running concurrently with live watcher/scheduler traffic is
safe because the sweep's operations are already built for concurrent
delivery — per-task locks (`queuedMoveLifecycleLocks`) and one-shot tokens
claimed by an atomic metadata-key removal (`claimTaskEventMetadata`) —
confirmed by
`TestLifecycleSweepConcurrentWithLiveTaskMovedRunsManualMoveSideEffectsOnce`,
which runs the sweep's recovery path concurrently with a live `task.moved`
redelivery of the same admitted manual move and asserts the step exit/enter
side effects run exactly once.
- Requests that need a started application are answered with an explicit
"starting" response rather than by a hung connection or a refused socket.
- Readiness is observable and distinct from liveness: an operator can tell
"bound, still starting" from "fully started".
## Regression scenarios
- **GIVEN** a backend with pending lifecycle tokens whose agent sessions never
reach ready, **WHEN** it starts, **THEN** the socket is bound and the
liveness probe returns success well inside `healthTimeoutReleaseMS`, and the
launcher does not shut the process down.
- **GIVEN** the same backend while startup reconciliation is still running,
**WHEN** an API route is requested, **THEN** it returns an explicit starting
status rather than hanging, and readiness reports not-ready while liveness
reports healthy.
- **GIVEN** startup reconciliation that has completed, **WHEN** readiness is
requested, **THEN** it reports ready and all routes serve normally.
## Out of scope
- Changing WIP promotion, feeder pull, or manual-move lifecycle semantics.
- Changing `AgentLaunchTimeout`, `agentPromptReadyTimeout`, or the cold-resume
retry shape (`context.WithoutCancel`). See the follow-up note below — a
caller-side deadline on the sweep bounds admission and reporting, not an
in-flight resume, precisely because of this contract.
- Raising `healthTimeoutReleaseMS`, which hides the coupling instead of fixing
it. Empirically a 10-minute window did not help.
## Known follow-up, addressed 2026-09-05
The note below originally described work deliberately deferred past this
spec's socket-bind fix. It has since been addressed in the same change that
corrected the ordering above (see Correction): the sweep now runs off the
readiness path entirely (background goroutine, not gating `API configured`),
carries its own bounded overall deadline separate from the interactive
`AgentLaunchTimeout`, and no longer accumulates already-completed
`manual_move_lifecycle_completed` tokens as permanent per-boot work. The
`context.WithoutCancel` resume contract itself is unchanged, so an
individual in-flight resume still cannot be aborted early — the sweep's
deadline bounds its own dispatch loop returning, not that resume.
> Fixing the ordering stops the crash loop but does not make a large board
> start quickly: a backend with many pending tokens still spends a long time
> in "starting" because bulk recovery inherits the interactive 15-minute
> `AgentLaunchTimeout` per task. A bulk startup sweep needs its own, much
> smaller per-task budget, and a resume path that can actually be abandoned.
The background lifecycle-token sweep still starts after watcher and scheduler
subscriptions. It does not gate readiness. Its per-task locks and atomic metadata
claims prevent duplicate effects during live event delivery. This change does not
alter its bounded dispatch deadline or in-flight resume behavior.