package backendapp import ( "bufio" "context" "encoding/json" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" "runtime" "sync" "syscall" "testing" "time" "github.com/jmoiron/sqlx" "go.uber.org/zap" "go.uber.org/zap/zaptest/observer" "github.com/kandev/kandev/internal/common/config" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/persistence" "github.com/kandev/kandev/internal/startup" "github.com/kandev/kandev/internal/system/backups" "github.com/kandev/kandev/internal/system/jobs" ) // @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.1, .5 func TestRunBindsBeforeDatabaseFailure(t *testing.T) { core, logs := observer.New(zap.InfoLevel) log, err := logger.NewFromZap(zap.New(core)) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = log.Close() }) cfg := &config.Config{} cfg.Server.Host = "127.0.0.1" cfg.Server.Port = freePort(t) cfg.HomeDir = t.TempDir() cfg.Database.Path = filepath.Join(cfg.HomeDir, "data", "kandev.db") if err := os.MkdirAll(cfg.Database.Path, 0700); err != nil { t.Fatal(err) } // Directory, not a database. var cleanups []func() error cleanup := func() { for i := len(cleanups) - 1; i >= 0; i-- { if cleanups[i] != nil { _ = cleanups[i]() } } cleanups = nil } defer cleanup() if run(cfg, log, &cleanups, cleanup) { t.Fatal("database failure reported success") } if logs.FilterMessage("HTTP listener bound; serving liveness probe while startup continues").Len() != 1 { t.Fatal("database initialization ran before the liveness listener bound") } if _, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/health"); err == nil { t.Fatal("failed initializer left healthy listener") } } func TestBootstrapReportsStartupPhase(t *testing.T) { w := httptest.NewRecorder() newBootstrapHandler("test").ServeHTTP(w, httptest.NewRequest("GET", "/ready", nil)) var body map[string]any if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatal(err) } startupBody, ok := body["startup"].(map[string]any) if !ok { t.Fatalf("startup body = %#v, want an object", body["startup"]) } if startupBody["phase"] != string(startup.OpeningDatabase) { t.Fatalf("startup phase = %v, want %q", startupBody["phase"], startup.OpeningDatabase) } for _, field := range []string{"elapsed_ms", "phase_elapsed_ms"} { value, ok := startupBody[field].(float64) if !ok || value < 0 { t.Fatalf("startup %s = %v, want a non-negative number", field, startupBody[field]) } } } // @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.2, .6 func TestBootstrapCancellationClosesListenerBeforeInitializerReturns(t *testing.T) { cfg := &config.Config{} cfg.Server.Host = "127.0.0.1" cfg.Server.Port = freePort(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() entered := make(chan struct{}) release := make(chan struct{}) var releaseOnce sync.Once done := make(chan bool, 1) defer releaseOnce.Do(func() { close(release) }) go func() { done <- runWithBootstrap(ctx, cfg, testLogger(t), func(ctx context.Context) bool { close(entered) <-release return true }) }() <-entered for _, path := range []string{"/ready", "/api/v1/tasks"} { status, _, err := getJSON("127.0.0.1", cfg.Server.Port, path) if err != nil || status != http.StatusServiceUnavailable { t.Fatalf("%s: %d %v", path, status, err) } } cancel() deadline := time.After(2 * time.Second) for { _, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/health") if err != nil { break } select { case <-deadline: t.Fatal("canceled initialization left listener healthy") default: } } // The wrapper must join initialization instead of racing resource cleanup. select { case <-done: t.Fatal("returned before initializer drained") default: } releaseOnce.Do(func() { close(release) }) select { case <-done: case <-time.After(2 * time.Second): t.Fatal("initializer did not drain after release") } } // @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.1, .2, .3, .6 func TestBootstrapDelayedSuccessKeepsLivenessUntilReadiness(t *testing.T) { cfg := &config.Config{} cfg.Server.Host = "127.0.0.1" cfg.Server.Port = freePort(t) cfg.Launcher.HealthTimeoutMs = 100 ctx, cancel := context.WithCancel(context.Background()) defer cancel() entered := make(chan struct{}) release := make(chan struct{}) published := make(chan struct{}) done := make(chan bool, 1) var releaseOnce sync.Once var publishOnce sync.Once defer releaseOnce.Do(func() { close(release) }) go func() { done <- runWithBootstrap(ctx, cfg, testLogger(t), func(ctx context.Context) bool { runtime := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime) close(entered) <-release application := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/health" { writeBootstrapJSON(w, http.StatusOK, map[string]any{ statusKey: "ok", serviceFieldKey: kandevName, "mode": "websocket+http", versionFieldKey: Version, }) return } if r.URL.Path == "/ready" || r.URL.Path == "/api/v1/tasks" { writeBootstrapJSON(w, http.StatusOK, map[string]any{statusKey: "ok"}) return } writeBootstrapJSON(w, http.StatusNotFound, map[string]any{statusKey: "not_found"}) }) if !runtime.beginReadinessPublication(ctx) { return false } publishReadiness(func() { ready.Store(true) runtime.ready.Store(true) publishOnce.Do(func() { close(published) }) }, func() { runtime.handler.Store(application) }) <-processContextFromContext(ctx).Done() return true }) }() <-entered // Keep the bootstrap initializer behind the same short health budget the // launcher uses for its keep-or-kill decision. Readiness has no matching // deadline: it must keep polling while recovery is still running. healthDeadline := time.Now().Add(time.Duration(cfg.Launcher.HealthTimeoutMs) * time.Millisecond) for time.Now().Before(healthDeadline) { status, body, err := getJSON("127.0.0.1", cfg.Server.Port, "/health") if err != nil || status != http.StatusOK { t.Fatalf("bootstrap /health during delayed initialization = %d, %v; want 200", status, err) } if body[statusKey] != "ok" || body[serviceFieldKey] != kandevName || body[versionFieldKey] != Version { t.Fatalf("bootstrap /health identity = %#v, want status=ok service=%q version=%q", body, kandevName, Version) } status, _, err = getJSON("127.0.0.1", cfg.Server.Port, "/ready") if err != nil || status != http.StatusServiceUnavailable { t.Fatalf("bootstrap /ready during delayed initialization = %d, %v; want 503", status, err) } status, _, err = getJSON("127.0.0.1", cfg.Server.Port, "/api/v1/tasks") if err != nil || status != http.StatusServiceUnavailable { t.Fatalf("bootstrap application route during delayed initialization = %d, %v; want 503", status, err) } } select { case result := <-done: t.Fatalf("bootstrap exited before readiness handoff: %v", result) default: } select { case <-published: t.Fatal("readiness published before initialization gate released") default: } readyDone := make(chan error, 1) go func() { for { status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/ready") if err != nil { readyDone <- err return } if status == http.StatusOK { readyDone <- nil return } time.Sleep(10 * time.Millisecond) } }() select { case err := <-readyDone: t.Fatalf("unbounded readiness wait completed before initialization release: %v", err) case <-time.After(50 * time.Millisecond): } releaseOnce.Do(func() { close(release) }) select { case <-published: case <-time.After(2 * time.Second): t.Fatal("initializer did not publish readiness after release") } select { case err := <-readyDone: if err != nil { t.Fatalf("readiness wait after release: %v", err) } case <-time.After(2 * time.Second): t.Fatal("readiness wait did not observe router handoff") } if status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/api/v1/tasks"); err != nil || status != http.StatusOK { t.Fatalf("application route after readiness = %d, %v; want 200", status, err) } cancel() select { case <-done: case <-time.After(2 * time.Second): t.Fatal("bootstrap did not stop after explicit process cancellation") } } // @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.3, .6 func TestBootstrapRestoreKeepsProcessAliveUntilResultAndShutdown(t *testing.T) { root := t.TempDir() databasePath := filepath.Join(root, "kandev.db") backupsDir := filepath.Join(root, "backups") if err := os.MkdirAll(backupsDir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(databasePath, []byte("before"), 0o600); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(backupsDir, "manual-restore.db"), []byte("after"), 0o600); err != nil { t.Fatal(err) } cfg := &config.Config{} cfg.Server.Host = "127.0.0.1" cfg.Server.Port = freePort(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() restoreBlocked := make(chan struct{}) restoreRelease := make(chan struct{}) jobIDReady := make(chan string, 1) resultReady := make(chan *jobs.Job, 1) bootstrapReady := make(chan struct{}) done := make(chan bool, 1) var restoreBlockedOnce sync.Once var bootstrapReadyOnce sync.Once var restoreReleaseOnce sync.Once defer restoreReleaseOnce.Do(func() { close(restoreRelease) }) go func() { done <- runWithBootstrap(ctx, cfg, testLogger(t), func(ctx context.Context) bool { runtime := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime) tracker := jobs.NewTracker(nil, testLogger(t)) svc := backups.NewService(databasePath, nil, tracker, testLogger(t)) svc.RestoreQuiesce = func() error { return quiesceForRestore( workerCancelFromContext(ctx), func() error { restoreBlockedOnce.Do(func() { close(restoreBlocked) }) <-restoreRelease return nil }, nil, nil, nil, ) } application := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/health" || r.URL.Path == "/ready" { writeBootstrapJSON(w, http.StatusOK, map[string]any{statusKey: "ok"}) return } writeBootstrapJSON(w, http.StatusNotFound, map[string]any{statusKey: "not_found"}) }) if !runtime.beginReadinessPublication(ctx) { return false } publishReadiness(func() { ready.Store(true) runtime.ready.Store(true) bootstrapReadyOnce.Do(func() { close(bootstrapReady) }) }, func() { runtime.handler.Store(application) }) jobID, err := svc.Restore(context.Background(), "manual-restore.db", backups.RestoreConfirmToken) if err != nil { return false } jobIDReady <- jobID <-restoreBlocked <-restoreRelease deadline := time.Now().Add(2 * time.Second) for time.Now().Before(deadline) { if job := tracker.Get(jobID); job != nil && job.State == jobs.StateSucceeded { resultReady <- job <-processContextFromContext(ctx).Done() return true } time.Sleep(5 * time.Millisecond) } return false }) }() select { case <-bootstrapReady: case <-time.After(2 * time.Second): t.Fatal("bootstrap did not publish readiness") } select { case <-jobIDReady: case <-time.After(2 * time.Second): t.Fatal("restore job was not submitted") } select { case <-restoreBlocked: case <-time.After(2 * time.Second): t.Fatal("restore did not reach the blocked quiesce boundary") } if status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/health"); err != nil || status != http.StatusOK { t.Fatalf("/health while restore quiesce is blocked = %d, %v; want 200", status, err) } if status, _, err := getJSON("127.0.0.1", cfg.Server.Port, "/ready"); err != nil || status != http.StatusOK { t.Fatalf("/ready while restore quiesce is blocked = %d, %v; want 200", status, err) } select { case result := <-done: t.Fatalf("backend exited while restore was blocked: %v", result) default: } restoreReleaseOnce.Do(func() { close(restoreRelease) }) select { case job := <-resultReady: if got, _ := job.Result["restart_required"].(bool); !got { t.Fatalf("restore result = %#v, want restart_required=true", job.Result) } case <-time.After(2 * time.Second): t.Fatal("restore result was not delivered") } select { case <-done: t.Fatal("backend exited before explicit shutdown after restore result") default: } cancel() select { case <-done: case <-time.After(2 * time.Second): t.Fatal("backend did not exit after explicit shutdown") } } // @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.5, .7 func TestRepositoryBackupFailureReportsPhaseAndPreventsMigration(t *testing.T) { dir := t.TempDir() dataDir := filepath.Join(dir, "data") if err := os.MkdirAll(dataDir, 0700); err != nil { t.Fatal(err) } path := filepath.Join(dataDir, "kandev.db") db, err := sqlx.Open("sqlite3", path) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = db.Close() }) if _, err = db.Exec("CREATE TABLE sentinel(value TEXT); INSERT INTO sentinel VALUES ('original')"); err != nil { t.Fatal(err) } if err = os.WriteFile(filepath.Join(dataDir, "backups"), []byte("blocked"), 0600); err != nil { t.Fatal(err) } cfg := &config.Config{HomeDir: dir} cfg.Database.Path = path progress := startup.New(nil) ctx := startup.WithReporter(context.Background(), progress) _, _, _, err = provideRepositories(ctx, cfg, testLogger(t), "upgrade-test") if err == nil { t.Fatal("backup failure did not fail initialization") } if got := progress.Snapshot().Phase; got != startup.BackingUpDatabase { t.Fatalf("phase=%s, want backup", got) } var count int if err = db.Get(&count, "SELECT count(*) FROM sqlite_master WHERE name='tasks'"); err != nil { t.Fatal(err) } if count != 0 { t.Fatal("migration ran after failed backup") } } func TestCanceledPersistenceDoesNotStartBackup(t *testing.T) { dir := t.TempDir() cfg := &config.Config{HomeDir: dir} cfg.Database.Path = filepath.Join(dir, "test.db") ctx, cancel := context.WithCancel(context.Background()) cancel() _, cleanup, err := persistence.ProvideContext(ctx, cfg, nil, "test") if cleanup != nil { defer func() { _ = cleanup() }() } if err == nil { t.Fatal("canceled initialization opened persistence") } } func TestRepositoryFailureClosesDatabase(t *testing.T) { if _, err := os.Stat("/proc/self/fd"); err != nil { t.Skip("requires Linux descriptor inspection") } dir := t.TempDir() data := filepath.Join(dir, "data") if err := os.MkdirAll(data, 0700); err != nil { t.Fatal(err) } path := filepath.Join(data, "kandev.db") db, err := sqlx.Open("sqlite3", path) if err != nil { t.Fatal(err) } if _, err = db.Exec("CREATE TABLE tasks(id TEXT)"); err != nil { t.Fatal(err) } if err = db.Close(); err != nil { t.Fatal(err) } cfg := &config.Config{HomeDir: dir} cfg.Database.Path = path _, _, cleanups, err := provideRepositories(context.Background(), cfg, testLogger(t), "") for _, cleanup := range cleanups { if cleanup != nil { _ = cleanup() } } if err == nil { t.Fatal("invalid schema unexpectedly initialized") } fds, err := os.ReadDir("/proc/self/fd") if err != nil { t.Fatal(err) } for _, fd := range fds { target, _ := os.Readlink(filepath.Join("/proc/self/fd", fd.Name())) if target == path { t.Fatalf("failed migration leaked database descriptor %s", fd.Name()) } } } func TestBootstrapSignalDuringInitialization(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("POSIX signals") } if os.Getenv("KANDEV_TEST_BOOTSTRAP_SIGNAL") == "1" { cfg := &config.Config{} cfg.Server.Host = "127.0.0.1" cfg.Server.Port = freePort(t) ok := runWithBootstrap(context.Background(), cfg, testLogger(t), func(ctx context.Context) bool { _, _ = os.Stdout.WriteString("BOUND\n") <-ctx.Done() return false }) if ok { t.Fatal("canceled startup succeeded") } return } exe, err := os.Executable() if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, exe, "-test.run=^TestBootstrapSignalDuringInitialization$") cmd.Env = append(os.Environ(), "KANDEV_TEST_BOOTSTRAP_SIGNAL=1") stdout, err := cmd.StdoutPipe() if err != nil { t.Fatal(err) } if err = cmd.Start(); err != nil { t.Fatal(err) } defer func() { _ = cmd.Process.Kill() }() scan := bufio.NewScanner(stdout) bound := false for scan.Scan() { if scan.Text() == "BOUND" { bound = true break } } if !bound { t.Fatal("child did not bind") } if err = cmd.Process.Signal(syscall.SIGTERM); err != nil { t.Fatal(err) } if err = cmd.Wait(); err != nil { t.Fatalf("initialization did not drain on SIGTERM: %v", err) } } // @covers AC-PLATFORM-STARTUP-LIFECYCLE-001.3, .6 func TestBootstrapSignalAfterReadiness(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("POSIX signals") } if os.Getenv("KANDEV_TEST_BOOTSTRAP_SIGNAL_READY") == "1" { cfg := &config.Config{} cfg.Server.Host = "127.0.0.1" cfg.Server.Port = freePort(t) ok := runWithBootstrap(context.Background(), cfg, testLogger(t), func(ctx context.Context) bool { runtime := ctx.Value(bootstrapContextKey{}).(*bootstrapRuntime) application := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { writeBootstrapJSON(w, http.StatusOK, map[string]any{statusKey: "ok"}) }) if !runtime.beginReadinessPublication(ctx) { t.Fatal("readiness publication rejected before signal test") } publishReadiness(func() { ready.Store(true) runtime.ready.Store(true) }, func() { runtime.handler.Store(application) }) _, _ = os.Stdout.WriteString("READY\n") awaitShutdown(ctx, runtime.server, runtime.listeners, &schedulingRuntime{}, nil, nil, func() {}, testLogger(t)) return true }) if !ok { t.Fatal("ready bootstrap did not complete graceful shutdown") } return } exe, err := os.Executable() if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, exe, "-test.run=^TestBootstrapSignalAfterReadiness$") cmd.Env = append(os.Environ(), "KANDEV_TEST_BOOTSTRAP_SIGNAL_READY=1") stdout, err := cmd.StdoutPipe() if err != nil { t.Fatal(err) } if err = cmd.Start(); err != nil { t.Fatal(err) } defer func() { _ = cmd.Process.Kill() }() scan := bufio.NewScanner(stdout) readyOutput := false for scan.Scan() { if scan.Text() == "READY" { readyOutput = true break } } if !readyOutput { t.Fatal("child did not publish readiness") } if err = cmd.Process.Signal(syscall.SIGTERM); err != nil { t.Fatal(err) } if err = cmd.Wait(); err != nil { t.Fatalf("ready bootstrap did not shut down on SIGTERM: %v", err) } }