diff --git a/apps/backend/cmd/plugin-fixture/fixture-package/ui/bundle.js b/apps/backend/cmd/plugin-fixture/fixture-package/ui/bundle.js index 4aa8cad07c..de2bf211bd 100644 --- a/apps/backend/cmd/plugin-fixture/fixture-package/ui/bundle.js +++ b/apps/backend/cmd/plugin-fixture/fixture-package/ui/bundle.js @@ -34,7 +34,7 @@ * a tiny listener set, so it survives across route navigations (the page * component unmounts/remounts as the user navigates away and back). */ -/* eslint-disable max-lines, max-lines-per-function, sonarjs/no-duplicate-string -- The fixture is the literal self-contained, dependency-free browser bundle exercised and hashed by E2E. */ +/* eslint-disable max-lines, max-lines-per-function -- The fixture is the literal self-contained, dependency-free browser bundle exercised and hashed by E2E. */ (function () { var moduleCount = 0; var listeners = new Set(); @@ -70,6 +70,9 @@ return count; } + var FIXTURE_REPOSITORY_ID = "fixture-repository"; + var FIXTURE_HELLO_PATH = "/plugins/e2e-hello"; + var FIXTURE_SIDEBAR_SECTION = "sidebar-footer"; var PROVIDER_ID = "fixture-source-control"; var PULL_REQUEST_URL = "https://bitbucket.example.test/projects/TEAM/repos/fixture/pull-requests/42"; @@ -77,8 +80,8 @@ function fixtureRepository() { return { - id: "fixture-repository", - repositoryId: "fixture-repository", + id: FIXTURE_REPOSITORY_ID, + repositoryId: FIXTURE_REPOSITORY_ID, owner: "TEAM", ownerOrProject: "TEAM", name: "fixture", @@ -731,14 +734,14 @@ registry.registerNavItem({ id: "e2e-hello", label: "Hello E2E", - path: "/plugins/e2e-hello", + path: FIXTURE_HELLO_PATH, section: "main", }); registry.registerNavItem({ id: "e2e-insights-tools", label: "E2E Insights Tools", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); // Three more sidebar-footer items so this one plugin install alone // produces P = 4 (budget MAX_INLINE_PLUGIN_FOOTER_ITEMS = 3, plus one @@ -754,22 +757,22 @@ registry.registerNavItem({ id: "e2e-insights-tools-2", label: "E2E Overflow Item 2", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); registry.registerNavItem({ id: "e2e-insights-tools-3", label: "E2E Overflow Item 3", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); registry.registerNavItem({ id: "e2e-insights-tools-4", label: "E2E Overflow Item 4", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); - registry.registerRoute("/plugins/e2e-hello", PluginPage); + registry.registerRoute(FIXTURE_HELLO_PATH, PluginPage); registry.registerComponent("task-sidebar", SidebarSlot); registry.registerComponent("main-top-bar", MainTopBarSlot); registry.registerComponent("app-status-bar-left", StatusSlot); @@ -801,7 +804,7 @@ providerId: PROVIDER_ID, providerHost: "bitbucket.example.test", ownerOrProject: "TEAM", - repositoryId: "fixture-repository", + repositoryId: FIXTURE_REPOSITORY_ID, repositoryName: "fixture", cloneUrl: REPOSITORY_URL, defaultBranch: "main", @@ -835,7 +838,7 @@ title: "Bitbucket Pull Request #42", url: PULL_REQUEST_URL, connectionScope: "https://bitbucket.example.test", - repositoryId: "fixture-repository", + repositoryId: FIXTURE_REPOSITORY_ID, changeRequestNumber: 42, state: "OPEN", statusBadge: { label: "Open" }, @@ -847,7 +850,7 @@ title: "Bitbucket Pull Request #43", url: "https://bitbucket.example.test/projects/TEAM/repos/fixture/pull-requests/43", connectionScope: "https://bitbucket.example.test", - repositoryId: "fixture-repository", + repositoryId: FIXTURE_REPOSITORY_ID, changeRequestNumber: 43, state: "OPEN", statusBadge: { label: "Open" }, diff --git a/apps/backend/internal/backendapp/main.go b/apps/backend/internal/backendapp/main.go index b552235fb9..a8fcae74e2 100644 --- a/apps/backend/internal/backendapp/main.go +++ b/apps/backend/internal/backendapp/main.go @@ -1013,6 +1013,7 @@ func startGatewayAndServe( } gateway.Hub.SetSessionDataProvider(buildSessionDataProvider(repos.Task, lifecycleMgr, orchestratorSvc, log)) gateway.Hub.SetSessionGitDataProvider(buildSessionGitDataProvider(repos.Task, lifecycleMgr, log)) + gateway.Hub.SetConversationSourceReader(services.Task) log.Info("Session data provider configured for session subscriptions (git status from snapshots)") // WS gateway per-user scoping (opt-in auth): connection auth on upgrade diff --git a/apps/backend/internal/gateway/websocket/client.go b/apps/backend/internal/gateway/websocket/client.go index 1bd99e46aa..d11f8d7988 100644 --- a/apps/backend/internal/gateway/websocket/client.go +++ b/apps/backend/internal/gateway/websocket/client.go @@ -3,16 +3,13 @@ package websocket import ( "context" "encoding/json" - "errors" "strings" "sync" "time" - "github.com/google/uuid" "github.com/gorilla/websocket" "github.com/kandev/kandev/internal/auth/authn" "github.com/kandev/kandev/internal/common/logger" - "github.com/kandev/kandev/internal/plugins" "github.com/kandev/kandev/internal/user/store" ws "github.com/kandev/kandev/pkg/websocket" "go.uber.org/zap" @@ -36,8 +33,6 @@ const ( // high-volume session notifications cannot fill the queue and make a user // action appear to time out. controlSendBufferSize = 256 - orderedConsumerPlugin = "plugin" - orderedConsumerCore = "core" responseErrorKey = "error" sessionIDPayloadKey = "session_id" eventTypePayloadKey = "type" @@ -52,21 +47,21 @@ type Client struct { // anonymous connections (auth disabled and no synthetic identity set by // the HTTP middleware — e.g. direct hub tests); synthetic in disabled // mode; a real user when auth is enabled. - identity authn.Identity - conn *websocket.Conn - hub *Hub - send chan []byte - controlSend chan []byte - subscriptions map[string]bool // Task IDs this client is subscribed to - sessionSubscriptions map[string]bool // Session IDs this client is subscribed to - sessionFocus map[string]bool // Session IDs this client has focused (a strict subset of subscriptions, conceptually — see hub_session_mode.go) - orderedSessionSubscriptions map[string]map[string]plugins.SessionDeliveryCursorKey - userSubscriptions map[string]bool // User IDs this client is subscribed to - runSubscriptions map[string]bool // Office run IDs this client is subscribed to (for run.event.appended) - systemMetricsSubscribed bool - mu sync.RWMutex - closed bool - logger *logger.Logger + identity authn.Identity + conn *websocket.Conn + hub *Hub + send chan []byte + controlSend chan []byte + subscriptions map[string]bool // Task IDs this client is subscribed to + sessionSubscriptions map[string]bool // Session IDs this client is subscribed to + sessionFocus map[string]bool // Session IDs this client has focused (a strict subset of subscriptions, conceptually — see hub_session_mode.go) + conversationSubscriptions map[string]conversationSubscription + userSubscriptions map[string]bool // User IDs this client is subscribed to + runSubscriptions map[string]bool // Office run IDs this client is subscribed to (for run.event.appended) + systemMetricsSubscribed bool + mu sync.RWMutex + closed bool + logger *logger.Logger // Replaceable session.message.updated traffic is scheduled separately from // semantic notifications so one noisy session cannot fill the shared FIFO. @@ -92,23 +87,23 @@ type Client struct { // NewClient creates a new WebSocket client func NewClient(id string, identity authn.Identity, conn *websocket.Conn, hub *Hub, log *logger.Logger) *Client { return &Client{ - ID: id, - identity: identity, - conn: conn, - hub: hub, - send: make(chan []byte, 256), - controlSend: make(chan []byte, controlSendBufferSize), - subscriptions: make(map[string]bool), - sessionSubscriptions: make(map[string]bool), - sessionFocus: make(map[string]bool), - orderedSessionSubscriptions: make(map[string]map[string]plugins.SessionDeliveryCursorKey), - userSubscriptions: make(map[string]bool), - runSubscriptions: make(map[string]bool), - replaceableByKey: make(map[queuedReplaceableKey]outboundNotification), - replaceableBySession: make(map[string][]sessionNotificationQueueItem), - replaceableCurrentByKey: make(map[replaceableNotificationKey]queuedReplaceableKey), - notificationWake: make(chan struct{}, 1), - logger: log.WithFields(zap.String("client_id", id)), + ID: id, + identity: identity, + conn: conn, + hub: hub, + send: make(chan []byte, 256), + controlSend: make(chan []byte, controlSendBufferSize), + subscriptions: make(map[string]bool), + sessionSubscriptions: make(map[string]bool), + sessionFocus: make(map[string]bool), + conversationSubscriptions: make(map[string]conversationSubscription), + userSubscriptions: make(map[string]bool), + runSubscriptions: make(map[string]bool), + replaceableByKey: make(map[queuedReplaceableKey]outboundNotification), + replaceableBySession: make(map[string][]sessionNotificationQueueItem), + replaceableCurrentByKey: make(map[replaceableNotificationKey]queuedReplaceableKey), + notificationWake: make(chan struct{}, 1), + logger: log.WithFields(zap.String("client_id", id)), } } @@ -205,11 +200,11 @@ func (c *Client) handleMessage(msg *ws.Message) { case ws.ActionSessionUnsubscribe: c.handleSessionUnsubscribe(msg) return - case ws.ActionSessionAck: - c.handleSessionAck(msg) + case ws.ActionSessionConversationSubscribe: + c.handleConversationSubscribe(msg) return - case ws.ActionSessionPoisonRequeue: - c.handleSessionPoisonRequeue(msg) + case ws.ActionSessionConversationUnsubscribe: + c.handleConversationUnsubscribe(msg) return case ws.ActionSessionFocus: c.handleSessionFocus(msg) @@ -379,7 +374,7 @@ func (c *Client) handleSessionSubscribe(msg *ws.Message) { return } if req.ConsumerKind != "" { - c.handleOrderedSessionSubscribe(msg, req) + c.sendError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "legacy session stream subscriptions are no longer supported", nil) return } @@ -401,500 +396,6 @@ func (c *Client) handleSessionSubscribe(msg *ws.Message) { } } -func (c *Client) handleOrderedSessionSubscribe(msg *ws.Message, req SessionSubscribeRequest) { - service := c.hub.pluginConversationService - if service == nil { - c.sendSessionStreamFailure(msg, req.SessionID, "upstream_failure", "session stream unavailable", true) - return - } - userID := c.ownUserTopic() - if req.ConsumerKind == orderedConsumerPlugin { - lastSeen := uint64(0) - if req.LastSeenSequence != nil { - lastSeen = *req.LastSeenSequence - } - candidate := plugins.SessionDeliveryCursorKey{ - SessionID: req.SessionID, ConsumerKind: req.ConsumerKind, ConsumerID: req.ConsumerID, - WireID: req.WireID, PluginID: req.PluginID, Generation: req.Generation, UserID: userID, - } - if req.ResumeToken == "" || service.ValidateSessionResume(req.ResumeToken, candidate, lastSeen) != nil { - req.ConsumerID = uuid.NewString() - } - } - key := plugins.SessionDeliveryCursorKey{ - SessionID: req.SessionID, ConsumerKind: req.ConsumerKind, - ConsumerID: req.ConsumerID, WireID: req.WireID, PluginID: req.PluginID, - Generation: req.Generation, UserID: userID, - } - if !c.authorizeOrderedSessionConsumer(msg, req, service, userID) { - return - } - if !c.maySubscribeOrderedSession(msg, req, key, service) { - return - } - c.hub.orderedSessionMu.Lock() - defer c.hub.orderedSessionMu.Unlock() - committedEvents, err := service.SyncCommittedSessionEvents(context.Background(), req.SessionID) - if err != nil { - c.sendSessionStreamFailure(msg, req.SessionID, "upstream_failure", "session stream unavailable", true) - return - } - for _, event := range committedEvents { - c.hub.broadcastCommittedOrderedSessionEvent(service, event) - } - replay := resolveOrderedSessionReplay(service, req, key) - if replay.terminal && (replay.resumeValid || service.SessionEvents().HasCursor(key)) { - c.sendSessionStreamFailure(msg, req.SessionID, "session_removed", "session was removed", false) - return - } - c.acceptOrderedSessionSubscription(msg, req, key, userID, replay) -} - -type orderedSessionReplay struct { - events []plugins.SessionEvent - watermark uint64 - cursorSequence uint64 - result string - resumeValid bool - terminal bool -} - -//nolint:goconst // Result values are protocol literals. -func resolveOrderedSessionReplay( - service *plugins.Service, - req SessionSubscribeRequest, - key plugins.SessionDeliveryCursorKey, -) orderedSessionReplay { - lastSeen := uint64(0) - lastSeenSupplied := req.LastSeenSequence != nil - if lastSeenSupplied { - lastSeen = *req.LastSeenSequence - } - events, watermark, terminal := service.SessionEvents().ReplayState(req.SessionID, lastSeen) - // A retained replay that starts past lastSeen+1 - or has no retained rows - // at all while the partition watermark is still ahead of the cursor - - // means intermediate rows aged out on both sides; replaying it as if - // contiguous would hide lost history. Rebind to the watermark instead - // (same replacement-cursor machinery as invalid_resume) so the client - // reconciles from the authoritative snapshot boundary. - retainedGap := lastSeenSupplied && - lastSeen < watermark && - (len(events) == 0 || events[0].Sequence > lastSeen+1) - if retainedGap { - return orderedSessionReplay{ - watermark: watermark, cursorSequence: watermark, - result: "invalid_resume", terminal: terminal, - } - } - if req.ResumeToken != "" { - if err := service.ValidateSessionResume(req.ResumeToken, key, lastSeen); err != nil { - return orderedSessionReplay{ - watermark: watermark, cursorSequence: watermark, - result: "invalid_resume", terminal: terminal, - } - } - result := "fresh" - if lastSeenSupplied && len(events) > 0 { - result = "replay" - } - return orderedSessionReplay{ - events: events, watermark: watermark, cursorSequence: lastSeen, - result: result, resumeValid: true, terminal: terminal, - } - } - if !lastSeenSupplied { - return orderedSessionReplay{ - watermark: watermark, cursorSequence: watermark, - result: "fresh", terminal: terminal, - } - } - result := "fresh" - if len(events) > 0 { - result = "replay" - } - return orderedSessionReplay{ - events: events, watermark: watermark, cursorSequence: lastSeen, - result: result, terminal: terminal, - } -} - -func (c *Client) acceptOrderedSessionSubscription( - msg *ws.Message, - req SessionSubscribeRequest, - key plugins.SessionDeliveryCursorKey, - userID string, - replay orderedSessionReplay, -) bool { - service := c.hub.pluginConversationService - replay, claims, err := prepareOrderedReplayDelivery(service, replay, time.Now().UTC()) - if err != nil { - c.sendSessionStreamFailure(msg, req.SessionID, "upstream_failure", "cannot claim session event delivery", true) - return false - } - cursorSequence := replay.cursorSequence - if req.ReplaceCursor || replay.result == "invalid_resume" { - cursorSequence = replay.watermark - } - snapshotToken, resumeToken, expiresAt, err := service.MintSessionStreamGrant( - req.PluginID, - userID, - req.Generation, - req.SessionID, - req.ConsumerID, - req.WireID, - replay.watermark, - cursorSequence, - ) - if err != nil { - c.releaseOrderedReplayClaims(service, claims) - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_binding", "cannot mint session stream grant", true) - return false - } - var cursorErr error - if req.ReplaceCursor || replay.result == "invalid_resume" { - cursorErr = service.SessionEvents().ReplaceCursor(key, cursorSequence) - } else { - cursorErr = service.SessionEvents().RegisterCursor(key, cursorSequence) - } - if cursorErr != nil { - c.releaseOrderedReplayClaims(service, claims) - c.sendSessionStreamFailure(msg, req.SessionID, "upstream_failure", "cannot register session cursor", true) - return false - } - payload := orderedSessionSubscribePayload(req, replay, snapshotToken, resumeToken, expiresAt) - c.storeOrderedSessionSubscription(req, key) - response, _ := ws.NewResponse(msg.ID, msg.Action, payload) - c.sendMessage(response) - for _, event := range replay.events { - queued := c.sendOrderedSessionEvent(event) - if claim, ok := claims[event.ID]; ok { - if err := service.SessionDelivery().Complete(claim, queued, time.Now().UTC()); err != nil && c.logger != nil { - c.logger.Error("complete ordered replay poison", zap.String("event_id", event.ID), zap.Error(err)) - } - } - } - return true -} - -func (c *Client) releaseOrderedReplayClaims( - service *plugins.Service, - claims map[string]plugins.SessionDeliveryClaim, -) { - if err := releaseOrderedReplayClaims(service, claims); err != nil && c.logger != nil { - c.logger.Error("release ordered replay claims", zap.Error(err)) - } -} - -func prepareOrderedReplayDelivery( - service *plugins.Service, - replay orderedSessionReplay, - now time.Time, -) (orderedSessionReplay, map[string]plugins.SessionDeliveryClaim, error) { - claims := make(map[string]plugins.SessionDeliveryClaim) - for _, event := range replay.events { - claim, err := service.SessionDelivery().Claim(event.SessionID, event.ID, now) - if err != nil { - releaseErr := releaseOrderedReplayClaims(service, claims) - return replay, nil, errors.Join(err, releaseErr) - } - switch claim.Disposition { - case plugins.SessionDeliveryUntracked: - case plugins.SessionDeliveryClaimed: - claims[event.ID] = claim - default: - if err := releaseOrderedReplayClaims(service, claims); err != nil { - return replay, nil, err - } - replay.events = nil - replay.result = "invalid_resume" - replay.cursorSequence = replay.watermark - return replay, nil, nil - } - } - return replay, claims, nil -} - -func releaseOrderedReplayClaims( - service *plugins.Service, - claims map[string]plugins.SessionDeliveryClaim, -) error { - now := time.Now().UTC() - var result error - for _, claim := range claims { - result = errors.Join(result, service.SessionDelivery().Complete(claim, false, now)) - } - return result -} - -func orderedSessionSubscribePayload( - req SessionSubscribeRequest, - replay orderedSessionReplay, - snapshotToken string, - resumeToken string, - expiresAt time.Time, -) map[string]any { - payload := map[string]any{ - "success": true, sessionIDPayloadKey: req.SessionID, "result": replay.result, - "event_watermark": replay.watermark, "snapshot_cutoff": replay.watermark, - "snapshot_token": snapshotToken, "resume_token": resumeToken, "expires_at": expiresAt, - } - if req.ConsumerKind == orderedConsumerPlugin { - payload["consumer_id"] = req.ConsumerID - } else { - payload["wire_id"] = req.WireID - } - if replay.result == "replay" && len(replay.events) > 0 { - payload["replay_from"] = replay.events[0].Sequence - payload["replay_to"] = replay.events[len(replay.events)-1].Sequence - } - return payload -} - -func (c *Client) storeOrderedSessionSubscription( - req SessionSubscribeRequest, - key plugins.SessionDeliveryCursorKey, -) { - subscriptionID := req.ConsumerID - if subscriptionID == "" { - subscriptionID = req.WireID - } - c.mu.Lock() - defer c.mu.Unlock() - byConsumer := c.orderedSessionSubscriptions[req.SessionID] - if byConsumer == nil { - byConsumer = make(map[string]plugins.SessionDeliveryCursorKey) - c.orderedSessionSubscriptions[req.SessionID] = byConsumer - } - byConsumer[subscriptionID] = key -} - -func (c *Client) authorizeOrderedSessionConsumer( - msg *ws.Message, - req SessionSubscribeRequest, - service *plugins.Service, - userID string, -) bool { - switch req.ConsumerKind { - case orderedConsumerPlugin: - return c.authorizeOrderedPluginConsumer(msg, req, service, userID) - case orderedConsumerCore: - if req.WireID != "" && req.ConsumerID == "" && req.PluginID == "" && - req.Generation == 0 && req.BindingToken == "" { - return true - } - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "invalid core consumer identity", false) - default: - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "unsupported consumer kind", false) - } - return false -} - -func (c *Client) authorizeOrderedPluginConsumer( - msg *ws.Message, - req SessionSubscribeRequest, - service *plugins.Service, - userID string, -) bool { - if req.ConsumerID == "" || req.WireID != "" || req.PluginID == "" || req.Generation == 0 { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "invalid plugin consumer identity", false) - return false - } - err := service.AuthorizeConversationConsumer( - req.PluginID, - userID, - req.Generation, - req.BindingToken, - ) - if err == nil { - return true - } - code := "invalid_binding" - retryable := false - if errors.Is(err, plugins.ErrConversationGenerationSuperseded) { - code = "generation_superseded" - retryable = true - } - c.sendSessionStreamFailure(msg, req.SessionID, code, "plugin conversation binding rejected", retryable) - return false -} - -func (c *Client) sendSessionStreamFailure( - msg *ws.Message, - sessionID string, - code string, - message string, - retryable bool, -) { - payload := map[string]any{ - "success": false, - responseErrorKey: map[string]any{"code": code, "message": message, "retryable": retryable}, - } - if sessionID != "" { - payload["session_id"] = sessionID - } - response, _ := ws.NewResponse(msg.ID, msg.Action, payload) - c.sendMessage(response) -} - -func (c *Client) sendOrderedSessionEvent(event plugins.SessionEvent) bool { - frame, err := sessionEventFrame(event) - if err != nil { - c.closeSend() - return false - } - if c.sendNotification(frame, "session.event") { - return true - } - // An ordered frame that is not queued cannot be acknowledged safely. End - // this connection so the client reconnects and replays from its cursor. - c.closeSend() - return false -} - -type SessionAckRequest struct { - SessionID string `json:"session_id"` - ConsumerKind string `json:"consumer_kind"` - PluginID string `json:"plugin_id,omitempty"` - Generation int64 `json:"generation,omitempty"` - Sequence uint64 `json:"sequence"` - ResumeToken string `json:"resume_token,omitempty"` - ConsumerID string `json:"consumer_id,omitempty"` - WireID string `json:"wire_id,omitempty"` -} - -func (c *Client) handleSessionAck(msg *ws.Message) { - var req SessionAckRequest - if err := msg.ParsePayload(&req); err != nil { - c.sendSessionStreamFailure(msg, "", "invalid_request", "invalid acknowledgement", false) - return - } - service := c.hub.pluginConversationService - identity, validIdentity := sessionAckSubscriptionID(req) - if !validIdentity { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "invalid session consumer identity", false) - return - } - c.mu.RLock() - key, ok := c.orderedSessionSubscriptions[req.SessionID][identity] - c.mu.RUnlock() - if service == nil || !ok || !sessionAckMatchesKey(req, key) { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "unknown session consumer", false) - return - } - if err := service.ValidateSessionResume(req.ResumeToken, key, req.Sequence); err != nil { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_binding", "invalid resume token", false) - return - } - if err := service.SessionEvents().Acknowledge(key, req.Sequence); err != nil { - code := "forward_gap" - retryable := true - if errors.Is(err, plugins.ErrPoisonEvent) { - code = "invalid_request" - retryable = false - } - c.sendSessionStreamFailure(msg, req.SessionID, code, "acknowledgement rejected", retryable) - return - } - _, resumeToken, _, err := service.MintSessionStreamGrant( - key.PluginID, - key.UserID, - key.Generation, - key.SessionID, - key.ConsumerID, - key.WireID, - req.Sequence, - req.Sequence, - ) - if err != nil { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_binding", "cannot mint resume token", true) - return - } - payload := map[string]any{ - "success": true, "session_id": req.SessionID, - "acknowledged_sequence": req.Sequence, "resume_token": resumeToken, - } - if key.ConsumerKind == orderedConsumerPlugin { - payload["consumer_id"] = key.ConsumerID - } else { - payload["wire_id"] = key.WireID - } - response, _ := ws.NewResponse(msg.ID, msg.Action, payload) - c.sendMessage(response) -} - -func sessionAckSubscriptionID(req SessionAckRequest) (string, bool) { - switch req.ConsumerKind { - case orderedConsumerPlugin: - return req.ConsumerID, req.ConsumerID != "" && req.WireID == "" - case orderedConsumerCore: - return req.WireID, req.WireID != "" && req.ConsumerID == "" - default: - return "", false - } -} - -func sessionAckMatchesKey(req SessionAckRequest, key plugins.SessionDeliveryCursorKey) bool { - if req.SessionID != key.SessionID || req.ConsumerKind != key.ConsumerKind { - return false - } - switch req.ConsumerKind { - case orderedConsumerPlugin: - return req.ConsumerID == key.ConsumerID && - req.PluginID == key.PluginID && - req.Generation == key.Generation - case orderedConsumerCore: - return req.WireID == key.WireID && req.PluginID == "" && req.Generation == 0 - default: - return false - } -} - -type sessionPoisonRequeueRequest struct { - SessionID string `json:"session_id"` - EventID string `json:"event_id"` - ExpectedOwnerEpoch uint64 `json:"expected_owner_epoch"` -} - -const sessionPoisonStateKey = "state" - -func (c *Client) handleSessionPoisonRequeue(msg *ws.Message) { - var req sessionPoisonRequeueRequest - if err := msg.ParsePayload(&req); err != nil || - req.SessionID == "" || - req.EventID == "" || - req.ExpectedOwnerEpoch == 0 { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "invalid poison requeue request", false) - return - } - if !canRequeueSessionEvents(c.identity) || c.hub.pluginConversationService == nil { - c.sendSessionStreamFailure(msg, req.SessionID, "unauthorized", "session_events:requeue required", false) - return - } - err := c.hub.pluginConversationService.SessionDelivery().Requeue( - req.SessionID, - req.EventID, - req.ExpectedOwnerEpoch, - c.ownUserTopic(), - time.Now().UTC(), - ) - if err != nil { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "poison requeue rejected", false) - return - } - response, _ := ws.NewResponse(msg.ID, msg.Action, map[string]any{ - "success": true, "session_id": req.SessionID, "event_id": req.EventID, - sessionPoisonStateKey: plugins.SessionPoisonPending, - }) - c.sendMessage(response) -} - -func canRequeueSessionEvents(identity authn.Identity) bool { - if identity.UserID == "" { - return false - } - return identity.Instance || identity.Synthetic -} - // maySubscribeSession applies the per-user session scoping check, emitting the // forbidden error itself. Returns true when the subscription may proceed. func (c *Client) maySubscribeSession(msg *ws.Message, sessionID string) bool { @@ -909,27 +410,6 @@ func (c *Client) maySubscribeSession(msg *ws.Message, sessionID string) bool { return true } -func (c *Client) maySubscribeOrderedSession( - msg *ws.Message, - req SessionSubscribeRequest, - key plugins.SessionDeliveryCursorKey, - service *plugins.Service, -) bool { - check := c.hub.authPolicy.Subscriptions.Session - if check == nil { - return true - } - if err := check(c.dispatchContext(), req.SessionID); err == nil { - return true - } - _, _, terminal := service.SessionEvents().ReplayState(req.SessionID, 0) - if terminal && service.SessionEvents().HasCursor(key) { - return true - } - c.sendError(msg.ID, msg.Action, ws.ErrorCodeForbidden, "cannot subscribe to this session", nil) - return false -} - func (c *Client) handleUserUnsubscribe(msg *ws.Message) { var req UserSubscribeRequest if err := msg.ParsePayload(&req); err != nil { @@ -1116,7 +596,7 @@ func (c *Client) handleSessionUnsubscribe(msg *ws.Message) { return } if req.ConsumerKind != "" { - c.handleOrderedSessionUnsubscribe(msg, req) + c.sendError(msg.ID, msg.Action, ws.ErrorCodeBadRequest, "legacy session stream subscriptions are no longer supported", nil) return } c.hub.UnsubscribeFromSession(c, req.SessionID) @@ -1127,55 +607,6 @@ func (c *Client) handleSessionUnsubscribe(msg *ws.Message) { c.sendMessage(resp) } -func (c *Client) handleOrderedSessionUnsubscribe(msg *ws.Message, req SessionSubscribeRequest) { - key, removed := c.removeOrderedSessionSubscription(req) - if !removed { - c.sendSessionStreamFailure(msg, req.SessionID, "invalid_request", "ordered subscription not found", false) - return - } - if service := c.hub.pluginConversationService; service != nil { - if err := service.SessionEvents().ReleaseCursor(key); err != nil { - c.storeOrderedSessionSubscription(req, key) - c.sendSessionStreamFailure(msg, req.SessionID, "upstream_failure", "cannot release session cursor", true) - return - } - } - payload := map[string]any{"success": true, sessionIDPayloadKey: req.SessionID} - if req.ConsumerKind == orderedConsumerPlugin { - payload["consumer_id"] = req.ConsumerID - } else { - payload["wire_id"] = req.WireID - } - response, _ := ws.NewResponse(msg.ID, msg.Action, payload) - c.sendMessage(response) -} - -func (c *Client) removeOrderedSessionSubscription( - req SessionSubscribeRequest, -) (plugins.SessionDeliveryCursorKey, bool) { - subscriptionID := req.ConsumerID - if req.ConsumerKind == orderedConsumerCore { - subscriptionID = req.WireID - } - c.mu.Lock() - defer c.mu.Unlock() - byConsumer := c.orderedSessionSubscriptions[req.SessionID] - key, subscribed := byConsumer[subscriptionID] - if !subscribed || - key.ConsumerKind != req.ConsumerKind || - key.ConsumerID != req.ConsumerID || - key.WireID != req.WireID || - key.PluginID != req.PluginID || - key.Generation != req.Generation { - return plugins.SessionDeliveryCursorKey{}, false - } - delete(byConsumer, subscriptionID) - if len(byConsumer) == 0 { - delete(c.orderedSessionSubscriptions, req.SessionID) - } - return key, true -} - // handleSessionFocus handles session.focus — marks the session as actively // viewed by this client, lifting backend polling to fast mode for the workspace. // It intentionally sends only the control acknowledgement. Snapshot/data diff --git a/apps/backend/internal/gateway/websocket/client_session_ack_test.go b/apps/backend/internal/gateway/websocket/client_session_ack_test.go deleted file mode 100644 index 6df758191f..0000000000 --- a/apps/backend/internal/gateway/websocket/client_session_ack_test.go +++ /dev/null @@ -1,484 +0,0 @@ -package websocket - -import ( - "encoding/json" - "testing" - "time" - - "github.com/kandev/kandev/internal/auth/authn" - "github.com/kandev/kandev/internal/plugins" - ws "github.com/kandev/kandev/pkg/websocket" -) - -func TestSessionAckIdentityMatchesExactConsumerBranch(t *testing.T) { - pluginKey := plugins.SessionDeliveryCursorKey{ - SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", - } - coreKey := plugins.SessionDeliveryCursorKey{ - SessionID: "session-1", ConsumerKind: orderedConsumerCore, WireID: "wire-1", - } - tests := []struct { - name string - req SessionAckRequest - key plugins.SessionDeliveryCursorKey - want bool - }{ - { - name: "plugin identity", - req: SessionAckRequest{SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1"}, - key: pluginKey, want: true, - }, - { - name: "plugin cannot also send wire identity", - req: SessionAckRequest{SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", WireID: "wire-1"}, - key: pluginKey, - }, - { - name: "plugin id must match", - req: SessionAckRequest{SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "other", Generation: 7, ConsumerID: "consumer-1"}, - key: pluginKey, - }, - { - name: "core identity", - req: SessionAckRequest{SessionID: "session-1", ConsumerKind: orderedConsumerCore, - WireID: "wire-1"}, - key: coreKey, want: true, - }, - { - name: "core cannot send plugin fields", - req: SessionAckRequest{SessionID: "session-1", ConsumerKind: orderedConsumerCore, - WireID: "wire-1", PluginID: "plugin-1", Generation: 7}, - key: coreKey, - }, - { - name: "core wire must match", - req: SessionAckRequest{SessionID: "session-1", ConsumerKind: orderedConsumerCore, - WireID: "other"}, - key: coreKey, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - identity, valid := sessionAckSubscriptionID(test.req) - if test.want { - if !valid || identity == "" || !sessionAckMatchesKey(test.req, test.key) { - t.Fatalf("valid ACK identity rejected: valid=%v identity=%q", valid, identity) - } - return - } - if valid && sessionAckMatchesKey(test.req, test.key) { - t.Fatal("malformed or conflicting ACK identity accepted") - } - }) - } -} - -func TestPoisonRequeueRequiresOperatorCapability(t *testing.T) { - tests := []struct { - name string - identity authn.Identity - want bool - }{ - {name: "instance operator", identity: authn.Identity{UserID: "operator-1", Instance: true}, want: true}, - {name: "single user operator", identity: authn.Identity{UserID: "default-user", Role: authn.RoleAdmin, Synthetic: true}, want: true}, - {name: "organization admin has no operator capability", identity: authn.Identity{UserID: "admin-1", Role: authn.RoleAdmin}}, - {name: "anonymous instance bit is insufficient", identity: authn.Identity{Instance: true}}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if got := canRequeueSessionEvents(test.identity); got != test.want { - t.Fatalf("canRequeueSessionEvents() = %v, want %v", got, test.want) - } - }) - } -} - -func TestOrderedReplayIsExplicitOnly(t *testing.T) { - service := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - _, err := service.SessionEvents().Append( - "session-1", - nil, - "message.deleted", - json.RawMessage(`{"type":"message.deleted","session_id":"session-1","message_id":"message-1"}`), - ) - if err != nil { - t.Fatalf("append event: %v", err) - } - key := plugins.SessionDeliveryCursorKey{ - SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", UserID: "user-1", - } - - fresh := resolveOrderedSessionReplay(service, SessionSubscribeRequest{ - SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", - }, key) - if fresh.result != "fresh" || fresh.watermark != 1 || len(fresh.events) != 0 { - t.Fatalf("fresh result = %+v, want no replay at watermark 1", fresh) - } - - zero := uint64(0) - replay := resolveOrderedSessionReplay(service, SessionSubscribeRequest{ - SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", - LastSeenSequence: &zero, - }, key) - if replay.result != "replay" || len(replay.events) != 1 { - t.Fatalf("explicit replay result = %+v, want one replay event", replay) - } - - invalid := resolveOrderedSessionReplay(service, SessionSubscribeRequest{ - SessionID: "session-1", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", - LastSeenSequence: &zero, ResumeToken: "invalid", - }, key) - if invalid.result != "invalid_resume" || len(invalid.events) != 0 { - t.Fatalf("invalid resume result = %+v, want replacement without replay", invalid) - } -} - -func TestOrderedReplayGrantAcceptsEveryContiguousAck(t *testing.T) { - tests := []struct { - name string - consumerKind string - consumerID string - wireID string - pluginID string - generation int64 - }{ - {name: "core", consumerKind: orderedConsumerCore, wireID: "wire-replay"}, - { - name: "plugin", - consumerKind: orderedConsumerPlugin, - consumerID: "consumer-replay", - pluginID: "plugin-replay", - generation: 7, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - service := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - for sequence := 1; sequence <= 12; sequence++ { - _, err := service.SessionEvents().Append( - "session-replay", - nil, - "message.deleted", - json.RawMessage(`{"type":"message.deleted","session_id":"session-replay","message_id":"message"}`), - ) - if err != nil { - t.Fatalf("append event %d: %v", sequence, err) - } - } - - client := newTestClient("replay-client-" + test.name) - client.hub = newTestHub(t) - client.hub.SetPluginConversationService(service) - client.controlSend = make(chan []byte, 16) - client.orderedSessionSubscriptions = make(map[string]map[string]plugins.SessionDeliveryCursorKey) - userID := client.ownUserTopic() - key := plugins.SessionDeliveryCursorKey{ - SessionID: "session-replay", ConsumerKind: test.consumerKind, - ConsumerID: test.consumerID, WireID: test.wireID, PluginID: test.pluginID, - Generation: test.generation, UserID: userID, - } - lastSeen := uint64(10) - req := SessionSubscribeRequest{ - SessionID: "session-replay", ConsumerKind: test.consumerKind, - PluginID: test.pluginID, Generation: test.generation, - ConsumerID: test.consumerID, WireID: test.wireID, - LastSeenSequence: &lastSeen, - } - _, initialResume, _, err := service.MintSessionStreamGrant( - test.pluginID, - userID, - test.generation, - "session-replay", - test.consumerID, - test.wireID, - 10, - 10, - ) - if err != nil { - t.Fatalf("mint initial resume token: %v", err) - } - req.ResumeToken = initialResume - if err := service.SessionEvents().RegisterCursor(key, 10); err != nil { - t.Fatalf("register replay cursor: %v", err) - } - replay := resolveOrderedSessionReplay(service, req, key) - if replay.result != "replay" || replay.watermark != 12 || replay.cursorSequence != 10 { - t.Fatalf("replay = %+v, want cursor 10 and watermark 12", replay) - } - msg := &ws.Message{ID: "subscribe-replay", Type: ws.MessageTypeRequest, Action: "session.subscribe"} - if !client.acceptOrderedSessionSubscription(msg, req, key, userID, replay) { - t.Fatal("replay subscription was rejected") - } - var subscribeResponse struct { - Payload struct { - SnapshotCutoff uint64 `json:"snapshot_cutoff"` - ResumeToken string `json:"resume_token"` - } `json:"payload"` - } - if err := json.Unmarshal(<-client.controlSend, &subscribeResponse); err != nil { - t.Fatalf("decode replay response: %v", err) - } - if subscribeResponse.Payload.SnapshotCutoff != 12 { - t.Fatalf("snapshot cutoff = %d, want 12", subscribeResponse.Payload.SnapshotCutoff) - } - - ack := func(sequence uint64, resumeToken string) string { - raw, err := json.Marshal(SessionAckRequest{ - SessionID: "session-replay", ConsumerKind: test.consumerKind, - PluginID: test.pluginID, Generation: test.generation, - Sequence: sequence, ResumeToken: resumeToken, - ConsumerID: test.consumerID, WireID: test.wireID, - }) - if err != nil { - t.Fatalf("marshal ACK %d: %v", sequence, err) - } - client.handleSessionAck(&ws.Message{ - ID: "ack-replay", Type: ws.MessageTypeRequest, Action: "session.ack", Payload: raw, - }) - var response struct { - Type ws.MessageType `json:"type"` - Payload struct { - Success bool `json:"success"` - ResumeToken string `json:"resume_token"` - Acknowledged uint64 `json:"acknowledged_sequence"` - ErrorCode string `json:"code"` - } `json:"payload"` - } - if err := json.Unmarshal(<-client.controlSend, &response); err != nil { - t.Fatalf("decode ACK %d response: %v", sequence, err) - } - if !response.Payload.Success { - t.Fatalf("ACK %d failed with %s", sequence, response.Payload.ErrorCode) - } - if response.Payload.Acknowledged != sequence { - t.Fatalf("ACK %d acknowledged %d", sequence, response.Payload.Acknowledged) - } - return response.Payload.ResumeToken - } - - resumeToken := subscribeResponse.Payload.ResumeToken - resumeToken = ack(11, resumeToken) - resumeToken = ack(12, resumeToken) - if _, err := service.SessionEvents().Append( - "session-replay", - nil, - "message.deleted", - json.RawMessage(`{"type":"message.deleted","session_id":"session-replay","message_id":"message-13"}`), - ); err != nil { - t.Fatalf("append live event: %v", err) - } - _ = ack(13, resumeToken) - }) - } -} - -func TestOrderedReplayGrantRejectsInvalidAck(t *testing.T) { - service := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - for sequence := 1; sequence <= 2; sequence++ { - if _, err := service.SessionEvents().Append( - "session-invalid-ack", - nil, - "message.deleted", - json.RawMessage(`{"type":"message.deleted","session_id":"session-invalid-ack","message_id":"message"}`), - ); err != nil { - t.Fatalf("append event %d: %v", sequence, err) - } - } - - client := newTestClient("invalid-ack-client") - client.hub = newTestHub(t) - client.hub.SetPluginConversationService(service) - client.controlSend = make(chan []byte, 8) - client.orderedSessionSubscriptions = make(map[string]map[string]plugins.SessionDeliveryCursorKey) - userID := client.ownUserTopic() - key := plugins.SessionDeliveryCursorKey{ - SessionID: "session-invalid-ack", ConsumerKind: orderedConsumerCore, - WireID: "wire-invalid-ack", UserID: userID, - } - if err := service.SessionEvents().RegisterCursor(key, 1); err != nil { - t.Fatalf("register ACK cursor: %v", err) - } - _, resumeToken, _, err := service.MintSessionStreamGrant( - "", userID, 0, key.SessionID, "", key.WireID, 1, 1, - ) - if err != nil { - t.Fatalf("mint resume token: %v", err) - } - client.orderedSessionSubscriptions[key.SessionID] = map[string]plugins.SessionDeliveryCursorKey{ - key.WireID: key, - } - - ack := func(sequence uint64, token string) (success bool, code string) { - raw, err := json.Marshal(SessionAckRequest{ - SessionID: key.SessionID, ConsumerKind: orderedConsumerCore, - Sequence: sequence, ResumeToken: token, WireID: key.WireID, - }) - if err != nil { - t.Fatalf("marshal ACK %d: %v", sequence, err) - } - client.handleSessionAck(&ws.Message{ - ID: "ack-invalid", Type: ws.MessageTypeRequest, Action: "session.ack", Payload: raw, - }) - var response struct { - Payload struct { - Success bool `json:"success"` - Error struct { - Code string `json:"code"` - } `json:"error"` - } `json:"payload"` - } - if err := json.Unmarshal(<-client.controlSend, &response); err != nil { - t.Fatalf("decode ACK %d response: %v", sequence, err) - } - return response.Payload.Success, response.Payload.Error.Code - } - - if success, code := ack(3, resumeToken); success || code != "forward_gap" { - t.Fatalf("forward-gap ACK result = success %v, code %q; want forward_gap", success, code) - } - if success, code := ack(2, resumeToken+"tampered"); success || code != "invalid_binding" { - t.Fatalf("tampered ACK result = success %v, code %q; want invalid_binding", success, code) - } - if success, code := ack(2, resumeToken); !success || code != "" { - t.Fatalf("valid ACK after rejected requests = success %v, code %q; want success", success, code) - } -} - -func TestOrderedReplayRebindsWhenPoisonDeliveryIsAlreadyClaimed(t *testing.T) { - service := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - poison, err := service.SessionEvents().Append( - "session-poison-replay", - nil, - "unknown.event", - json.RawMessage(`{"type":"unknown.event","session_id":"session-poison-replay"}`), - ) - if err != nil { - t.Fatalf("append poison event: %v", err) - } - claim, err := service.SessionDelivery().Claim( - poison.SessionID, - poison.ID, - time.Now().UTC(), - ) - if err != nil || claim.Disposition != plugins.SessionDeliveryClaimed { - t.Fatalf("initial poison claim = %q, %v", claim.Disposition, err) - } - zero := uint64(0) - req := SessionSubscribeRequest{ - SessionID: "session-poison-replay", ConsumerKind: orderedConsumerCore, - WireID: "wire-poison", LastSeenSequence: &zero, - } - key := plugins.SessionDeliveryCursorKey{ - SessionID: "session-poison-replay", ConsumerKind: orderedConsumerCore, WireID: "wire-poison", - } - replay := resolveOrderedSessionReplay(service, req, key) - hub := newTestHub(t) - hub.SetPluginConversationService(service) - client := newTestClient("poison-replay-client") - client.hub = hub - client.controlSend = make(chan []byte, 1) - client.orderedSessionSubscriptions = make(map[string]map[string]plugins.SessionDeliveryCursorKey) - msg := &ws.Message{ID: "subscribe-poison", Type: ws.MessageTypeRequest, Action: "session.subscribe"} - - if !client.acceptOrderedSessionSubscription(msg, req, key, "", replay) { - t.Fatal("poison replay replacement subscription was rejected") - } - - var response struct { - Payload struct { - Result string `json:"result"` - } `json:"payload"` - } - if err := json.Unmarshal(<-client.controlSend, &response); err != nil { - t.Fatalf("decode subscription response: %v", err) - } - if response.Payload.Result != "invalid_resume" { - t.Fatalf("poison replay result = %q, want invalid_resume", response.Payload.Result) - } - if err := service.SessionEvents().Acknowledge(key, poison.Sequence); err != nil { - t.Fatalf("replacement cursor did not advance past poison: %v", err) - } -} - -// TestOrderedReplayRebindsOnRetainedGap pins that a replay whose first -// retained row starts past lastSeen+1 (intermediate rows aged out on both -// sides) returns the replacement-cursor result instead of masquerading as a -// contiguous replay that would hide the lost history. -func TestOrderedReplayRebindsOnRetainedGap(t *testing.T) { - service := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - // A fresh durable-log partition with only a high sequence present mirrors - // the state after both sides pruned the intermediate rows. - appended, err := service.SessionEvents().AppendCommitted(plugins.SessionEvent{ - SessionID: "session-gap", ID: "session-gap:10", Sequence: 10, - EventType: "message.deleted", - Payload: json.RawMessage(`{"type":"message.deleted","session_id":"session-gap","message_id":"message-1"}`), - }) - if err != nil { - t.Fatalf("mirror event: %v", err) - } - if !appended { - t.Fatal("expected the gap heal to accept the retained row") - } - key := plugins.SessionDeliveryCursorKey{ - SessionID: "session-gap", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", UserID: "user-1", - } - lastSeen := uint64(5) - replay := resolveOrderedSessionReplay(service, SessionSubscribeRequest{ - SessionID: "session-gap", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", - LastSeenSequence: &lastSeen, - }, key) - if replay.result != "invalid_resume" || len(replay.events) != 0 || replay.cursorSequence != replay.watermark { - t.Fatalf("retained-gap result = %+v, want replacement cursor at watermark without replay", replay) - } -} - -// TestOrderedReplayRebindsWhenPartitionWatermarkAheadWithNoRows pins the -// fully-pruned case: the partition watermark is ahead of the cursor but every -// retained event row aged out, so the reconnect must rebind rather than treat -// the empty replay as a fresh success. -func TestOrderedReplayRebindsWhenPartitionWatermarkAheadWithNoRows(t *testing.T) { - service := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - base := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - appended, err := service.SessionEvents().AppendCommitted(plugins.SessionEvent{ - SessionID: "session-pruned", ID: "session-pruned:10", Sequence: 10, - EventType: "message.added", ProtocolVersion: 1, - Payload: json.RawMessage(`{"type":"message.added","session_id":"session-pruned","message_id":"m1","author_type":"user","content":"x","created_at":"2026-09-07T12:00:00Z"}`), - CreatedAt: base, - }) - if err != nil || !appended { - t.Fatalf("mirror event appended=%v err=%v", appended, err) - } - // Age the only retained row out: the partition keeps its watermark. - if err := service.SessionEvents().CollectExpired(base.Add(plugins.SessionEventRetention + time.Hour)); err != nil { - t.Fatalf("collect expired: %v", err) - } - key := plugins.SessionDeliveryCursorKey{ - SessionID: "session-pruned", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", UserID: "user-1", - } - lastSeen := uint64(5) - replay := resolveOrderedSessionReplay(service, SessionSubscribeRequest{ - SessionID: "session-pruned", ConsumerKind: orderedConsumerPlugin, - PluginID: "plugin-1", Generation: 7, ConsumerID: "consumer-1", - LastSeenSequence: &lastSeen, - }, key) - if replay.result != "invalid_resume" || len(replay.events) != 0 || replay.cursorSequence != replay.watermark { - t.Fatalf("pruned-gap result = %+v, want replacement cursor at watermark without replay", replay) - } - if replay.watermark != 10 { - t.Fatalf("watermark = %d, want 10", replay.watermark) - } -} diff --git a/apps/backend/internal/gateway/websocket/conversation_checks.go b/apps/backend/internal/gateway/websocket/conversation_checks.go new file mode 100644 index 0000000000..dd879a31b1 --- /dev/null +++ b/apps/backend/internal/gateway/websocket/conversation_checks.go @@ -0,0 +1,140 @@ +package websocket + +import ( + "context" + "encoding/json" + "time" + + "github.com/kandev/kandev/internal/plugins" + "github.com/kandev/kandev/internal/task/models" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func (h *Hub) runConversationChecks(ctx context.Context) { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + h.checkConversationSubscriptions(ctx) + } + } +} + +func (h *Hub) checkConversationSubscriptions(ctx context.Context) { + h.mu.RLock() + reader := h.conversationSourceReader + clients := make([]*Client, 0, len(h.clients)) + for client := range h.clients { + clients = append(clients, client) + } + h.mu.RUnlock() + if reader == nil { + return + } + // Read each subscribed session once per pass, regardless of panel count. + revisions := make(map[string]models.ConversationRevision) + for _, client := range clients { + client.mu.RLock() + subscriptions := make([]conversationSubscription, 0, len(client.conversationSubscriptions)) + for _, subscription := range client.conversationSubscriptions { + subscriptions = append(subscriptions, subscription) + } + client.mu.RUnlock() + for _, subscription := range subscriptions { + if ctx.Err() != nil { + return + } + if !h.conversationAuthorized(client, subscription) { + h.terminateConversation(client, subscription) + continue + } + revision, ok := revisions[subscription.SessionID] + if !ok { + var err error + revision, err = reader.ReadConversationRevision(ctx, subscription.SessionID) + if err != nil { + continue + } + revisions[subscription.SessionID] = revision + } + if !revision.Exists { + h.terminateConversation(client, subscription) + continue + } + h.sendConversationPayload(client, subscription, conversationChangedPayload{ + ProtocolVersion: 2, ScopeID: subscription.ScopeID, SessionID: subscription.SessionID, + Epoch: subscription.Epoch, BaseRevision: formatRevision(revision.Revision), Revision: formatRevision(revision.Revision), + Check: true, Operations: []conversationChangedOperation{}, + }) + } + } +} + +func (h *Hub) conversationAuthorized(client *Client, subscription conversationSubscription) bool { + h.mu.RLock() + policy, service := h.authPolicy, h.pluginConversationService + h.mu.RUnlock() + ctx := client.dispatchContext() + // Synthetic and identity-less clients are the unscoped single-user mode. + // They do not have an account to revalidate, so an account-backed active + // user hook must not terminate their conversation subscriptions. + if identityIsScoped(client.identity) && policy.ActiveUser != nil && !policy.ActiveUser(ctx, subscription.UserID) { + return false + } + if policy.Subscriptions.Session != nil && policy.Subscriptions.Session(ctx, subscription.SessionID) != nil { + return false + } + if subscription.ConsumerKind != conversationConsumerPlugin { + return true + } + if service == nil { + return false + } + record, err := service.Get(subscription.PluginID) + return err == nil && record.Status == plugins.StatusActive && record.Capabilities.CanRead("messages") && + record.InstalledAt.UnixMicro() == subscription.Generation +} + +func (h *Hub) terminateConversation(client *Client, subscription conversationSubscription) { + client.mu.Lock() + current, ok := client.conversationSubscriptions[subscription.ScopeID] + if !ok || current.instance != subscription.instance { + client.mu.Unlock() + return + } + delete(client.conversationSubscriptions, subscription.ScopeID) + client.mu.Unlock() + payload := conversationChangedPayload{ + ProtocolVersion: 2, ScopeID: subscription.ScopeID, SessionID: subscription.SessionID, Epoch: subscription.Epoch, + BaseRevision: "0", Revision: "0", Terminal: true, Operations: []conversationChangedOperation{}, + } + message, _ := ws.NewNotification(ws.ActionSessionConversationChanged, payload) + encoded, err := json.Marshal(message) + if err == nil { + client.sendNotification(encoded, ws.ActionSessionConversationChanged) + } +} + +func (h *Hub) sendConversationPayload(client *Client, subscription conversationSubscription, payload conversationChangedPayload) { + if !h.conversationAuthorized(client, subscription) { + h.terminateConversation(client, subscription) + return + } + client.mu.RLock() + current, ok := client.conversationSubscriptions[subscription.ScopeID] + client.mu.RUnlock() + if !ok || current.instance != subscription.instance { + return + } + message, err := ws.NewNotification(ws.ActionSessionConversationChanged, payload) + if err != nil { + return + } + encoded, err := json.Marshal(message) + if err == nil { + client.sendNotification(encoded, ws.ActionSessionConversationChanged) + } +} diff --git a/apps/backend/internal/gateway/websocket/conversation_delivery.go b/apps/backend/internal/gateway/websocket/conversation_delivery.go new file mode 100644 index 0000000000..5c4bad6ba2 --- /dev/null +++ b/apps/backend/internal/gateway/websocket/conversation_delivery.go @@ -0,0 +1,455 @@ +package websocket + +import ( + "encoding/json" + "errors" + "github.com/google/uuid" + "strconv" + "strings" + "time" + + "github.com/kandev/kandev/internal/plugins" + "github.com/kandev/kandev/internal/sysprompt" + "github.com/kandev/kandev/internal/task/models" + ws "github.com/kandev/kandev/pkg/websocket" +) + +const ( + conversationConsumerPlugin = "plugin" + conversationConsumerCore = "core" +) + +type conversationSubscription struct { + instance string + ScopeID string + SessionID string + ConsumerKind string + PluginID string + Generation int64 + UserID string + TaskID *string + Authors []string + Sort string + Epoch string +} + +type conversationSubscribeRequest struct { + SessionID string `json:"session_id"` + ScopeID string `json:"scope_id"` + ConsumerKind string `json:"consumer_kind"` + PluginID string `json:"plugin_id,omitempty"` + Generation int64 `json:"generation,omitempty"` + BindingToken string `json:"binding_token,omitempty"` + TaskID *string `json:"task_id,omitempty"` + Authors []string `json:"authors,omitempty"` + Sort string `json:"sort,omitempty"` +} + +type conversationChangedPayload struct { + ProtocolVersion int `json:"protocol_version"` + ScopeID string `json:"scope_id"` + SessionID string `json:"session_id"` + Epoch string `json:"epoch"` + BaseRevision string `json:"base_revision"` + Revision string `json:"revision"` + Check bool `json:"check,omitempty"` + Terminal bool `json:"terminal,omitempty"` + Reset bool `json:"reset,omitempty"` + Operations []conversationChangedOperation `json:"operations"` +} + +type conversationChangedOperation struct { + Kind models.ConversationMutationKind `json:"kind"` + Entity models.ConversationEntityKind `json:"entity"` + ID string `json:"id"` + Message any `json:"message,omitempty"` + Turn any `json:"turn,omitempty"` +} + +func (c *Client) handleConversationSubscribe(msg *ws.Message) { + var req conversationSubscribeRequest + if err := msg.ParsePayload(&req); err != nil { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "invalid conversation subscription", false) + return + } + if req.SessionID == "" || req.ScopeID == "" { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "session_id and scope_id are required", false) + return + } + if req.ConsumerKind != conversationConsumerPlugin && req.ConsumerKind != conversationConsumerCore { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "unsupported conversation consumer", false) + return + } + if !c.maySubscribeSession(msg, req.SessionID) { + return + } + c.hub.mu.RLock() + service := c.hub.pluginConversationService + c.hub.mu.RUnlock() + userID := c.ownUserTopic() + if !c.authorizeConversationIdentity(msg, req, service, userID) { + return + } + revision, ok := c.readConversationSubscription(msg, req) + if !ok { + return + } + h := c.hub + + sortOrder := strings.ToLower(req.Sort) + if sortOrder == "" { + sortOrder = "asc" + } + if sortOrder != "asc" && sortOrder != "desc" { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "sort must be asc or desc", false) + return + } + epoch := h.conversationEpoch + if req.ConsumerKind == conversationConsumerPlugin { + epoch = service.ConversationEpoch() + } + subscription := conversationSubscription{ + instance: uuid.NewString(), + ScopeID: req.ScopeID, SessionID: req.SessionID, ConsumerKind: req.ConsumerKind, + PluginID: req.PluginID, Generation: req.Generation, UserID: userID, + TaskID: cloneOptionalString(req.TaskID), Authors: append([]string(nil), req.Authors...), + Sort: sortOrder, Epoch: epoch, + } + c.mu.Lock() + c.conversationSubscriptions[req.ScopeID] = subscription + c.mu.Unlock() + response, _ := ws.NewResponse(msg.ID, msg.Action, map[string]any{ + "success": true, "protocol_version": 2, "scope_id": req.ScopeID, + "session_id": req.SessionID, "epoch": epoch, "revision": formatRevision(revision.Revision), + }) + c.sendMessage(response) +} + +func (c *Client) authorizeConversationIdentity(msg *ws.Message, req conversationSubscribeRequest, service *plugins.Service, userID string) bool { + if req.ConsumerKind == conversationConsumerCore { + if req.PluginID != "" || req.Generation != 0 || req.BindingToken != "" { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "invalid core conversation identity", false) + return false + } + return true + } + if service == nil || req.PluginID == "" || req.Generation == 0 || req.BindingToken == "" { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "invalid plugin conversation identity", false) + return false + } + if err := service.AuthorizeConversationConsumer(req.PluginID, userID, req.Generation, req.BindingToken); err != nil { + code, retryable := "invalid_binding", false + if errors.Is(err, plugins.ErrConversationGenerationSuperseded) { + code, retryable = "generation_superseded", true + } + c.sendConversationFailure(msg, req.SessionID, code, "plugin conversation binding rejected", retryable) + return false + } + return true +} + +func (c *Client) readConversationSubscription(msg *ws.Message, req conversationSubscribeRequest) (models.ConversationRevision, bool) { + h := c.hub + h.mu.RLock() + reader := h.conversationSourceReader + h.mu.RUnlock() + if reader == nil { + c.sendConversationFailure(msg, req.SessionID, "upstream_failure", "conversation source unavailable", true) + return models.ConversationRevision{}, false + } + session, err := reader.GetTaskSession(c.dispatchContext(), req.SessionID) + if err != nil || session == nil || session.ID != req.SessionID { + c.sendConversationFailure(msg, req.SessionID, "not_found", "task session not found", false) + return models.ConversationRevision{}, false + } + if req.TaskID != nil && *req.TaskID != session.TaskID { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "task_id does not match the task session", false) + return models.ConversationRevision{}, false + } + revision, err := reader.ReadConversationRevision(c.dispatchContext(), req.SessionID) + if err != nil { + c.sendConversationFailure(msg, req.SessionID, "upstream_failure", "conversation source unavailable", true) + return models.ConversationRevision{}, false + } + if !revision.Exists { + c.sendConversationFailure(msg, req.SessionID, "not_found", "task session not found", false) + return models.ConversationRevision{}, false + } + return revision, true +} + +func (c *Client) handleConversationUnsubscribe(msg *ws.Message) { + var req conversationSubscribeRequest + if err := msg.ParsePayload(&req); err != nil || req.ScopeID == "" { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "scope_id is required", false) + return + } + c.mu.Lock() + subscription, ok := c.conversationSubscriptions[req.ScopeID] + if ok && (req.SessionID == "" || subscription.SessionID == req.SessionID) { + delete(c.conversationSubscriptions, req.ScopeID) + } else { + ok = false + } + c.mu.Unlock() + if !ok { + c.sendConversationFailure(msg, req.SessionID, "invalid_request", "conversation subscription not found", false) + return + } + response, _ := ws.NewResponse(msg.ID, msg.Action, map[string]any{ + "success": true, "protocol_version": 2, "scope_id": req.ScopeID, + "session_id": subscription.SessionID, + }) + c.sendMessage(response) +} + +func (c *Client) sendConversationFailure(msg *ws.Message, sessionID, code, message string, retryable bool) { + payload := map[string]any{ + "success": false, + "error": map[string]any{"code": code, "message": message, "retryable": retryable}, + } + if sessionID != "" { + payload["session_id"] = sessionID + } + response, _ := ws.NewResponse(msg.ID, msg.Action, payload) + c.sendMessage(response) +} + +func cloneOptionalString(value *string) *string { + if value == nil { + return nil + } + copy := *value + return © +} + +// BroadcastConversationMutation projects a committed source receipt to every +// matching Host-only subscription. It accepts the bus event data so NATS and +// memory buses can both carry either a typed receipt or its JSON map form. +func (h *Hub) BroadcastConversationMutation(data any) { + receipt, hasReceipt := conversationReceiptFromData(data) + sessionID := "" + if receipt != nil { + sessionID = receipt.SessionID + } else { + sessionID = extractSessionID(data) + } + if sessionID == "" { + return + } + for _, client := range h.conversationRecipients(sessionID) { + client.mu.RLock() + subscriptions := make([]conversationSubscription, 0, len(client.conversationSubscriptions)) + for _, subscription := range client.conversationSubscriptions { + if subscription.SessionID == sessionID { + subscriptions = append(subscriptions, subscription) + } + } + client.mu.RUnlock() + for _, subscription := range subscriptions { + payload := h.conversationChanged(subscription, receipt, hasReceipt) + h.sendConversationPayload(client, subscription, payload) + } + } +} + +func (h *Hub) conversationRecipients(sessionID string) []*Client { + h.mu.RLock() + defer h.mu.RUnlock() + clients := make([]*Client, 0, len(h.clients)) + for client := range h.clients { + client.mu.RLock() + matched := false + for _, subscription := range client.conversationSubscriptions { + if subscription.SessionID == sessionID { + matched = true + break + } + } + client.mu.RUnlock() + if matched { + clients = append(clients, client) + } + } + return clients +} + +func (h *Hub) conversationChanged( + subscription conversationSubscription, + receipt *models.ConversationMutationReceipt, + hasReceipt bool, +) conversationChangedPayload { + payload := conversationChangedPayload{ + ProtocolVersion: 2, ScopeID: subscription.ScopeID, SessionID: subscription.SessionID, + Epoch: subscription.Epoch, Operations: []conversationChangedOperation{}, + } + if !hasReceipt || receipt == nil || !receipt.Complete { + payload.Reset = true + if receipt != nil { + payload.BaseRevision = formatRevision(receipt.BaseRevision) + payload.Revision = formatRevision(receipt.Revision) + } + return payload + } + payload.BaseRevision = formatRevision(receipt.BaseRevision) + payload.Revision = formatRevision(receipt.Revision) + for _, operation := range receipt.Operations { + if !conversationOperationSelected(subscription, operation) { + continue + } + projected, ok := projectConversationOperation(subscription, operation) + if !ok { + payload.Reset = true + payload.Operations = []conversationChangedOperation{} + return payload + } + payload.Operations = append(payload.Operations, projected) + } + return payload +} + +func formatRevision(value int64) string { + return strconv.FormatInt(value, 10) +} + +func conversationOperationSelected(subscription conversationSubscription, operation models.ConversationMutationOperation) bool { + if operation.SessionID != subscription.SessionID { + return false + } + if subscription.TaskID != nil && *subscription.TaskID != operation.TaskID { + return false + } + if operation.Entity == models.ConversationEntityMessage && len(subscription.Authors) > 0 { + for _, author := range subscription.Authors { + if author == operation.AuthorType { + return true + } + } + return false + } + return operation.Entity == models.ConversationEntityMessage || operation.Entity == models.ConversationEntityTurn +} + +func projectConversationOperation(subscription conversationSubscription, operation models.ConversationMutationOperation) (conversationChangedOperation, bool) { + result := conversationChangedOperation{Kind: operation.Kind, Entity: operation.Entity, ID: operation.ID} + if operation.Kind == models.ConversationMutationRemove { + return result, true + } + if operation.Kind != models.ConversationMutationUpsert { + return result, false + } + switch operation.Entity { + case models.ConversationEntityMessage: + if operation.Message == nil { + return result, false + } + if subscription.ConsumerKind == conversationConsumerPlugin { + result.Message = safePluginConversationMessage(operation.Message) + } else { + result.Message = operation.Message.ToAPI() + } + case models.ConversationEntityTurn: + if operation.Turn == nil { + return result, false + } + if subscription.ConsumerKind == conversationConsumerPlugin { + result.Turn = safePluginConversationTurn(operation.Turn) + } else { + result.Turn = safeCoreConversationTurn(operation.Turn, operation.HadOutput) + } + default: + return result, false + } + return result, true +} + +func safePluginConversationMessage(message *models.Message) map[string]any { + payload := map[string]any{ + "id": message.ID, "sessionId": message.TaskSessionID, "authorType": string(message.AuthorType), + "type": string(message.Type), "content": sysprompt.StripSystemContent(message.Content), + "createdAt": message.CreatedAt.UTC().Format(time.RFC3339Nano), + "updatedAt": message.UpdatedAt.UTC().Format(time.RFC3339Nano), + } + if message.TaskID != "" { + payload["taskId"] = message.TaskID + } + if message.TurnID != "" { + payload["turnId"] = message.TurnID + } + if message.PromptIndex > 0 { + payload["promptIndex"] = message.PromptIndex + } + if senderTaskID, ok := message.Metadata["sender_task_id"].(string); ok && senderTaskID != "" { + payload["senderTaskId"] = senderTaskID + } + return payload +} + +func safePluginConversationTurn(turn *models.Turn) map[string]any { + payload := map[string]any{ + "id": turn.ID, "sessionId": turn.TaskSessionID, "startedAt": turn.StartedAt.UTC().Format(time.RFC3339Nano), + "updatedAt": turn.UpdatedAt.UTC().Format(time.RFC3339Nano), + } + if turn.TaskID != "" { + payload["taskId"] = turn.TaskID + } + if turn.CompletedAt != nil { + payload["completedAt"] = turn.CompletedAt.UTC().Format(time.RFC3339Nano) + } + return payload +} + +func safeCoreConversationTurn(turn *models.Turn, hadOutput *bool) map[string]any { + payload := map[string]any{ + "id": turn.ID, "session_id": turn.TaskSessionID, "task_id": turn.TaskID, + "started_at": turn.StartedAt, "updated_at": turn.UpdatedAt, + } + if metadata := models.ProjectTurnMetadata(turn.Metadata); len(metadata) > 0 { + payload["metadata"] = metadata + } + if turn.ExecutionProfileID != "" { + payload["execution_profile_id"] = turn.ExecutionProfileID + } + if turn.RouteGeneration != 0 { + payload["route_generation"] = turn.RouteGeneration + } + if turn.CompletedAt != nil { + payload["completed_at"] = turn.CompletedAt + } + if hadOutput != nil { + payload["had_output"] = *hadOutput + } + return payload +} + +func conversationReceiptFromData(data any) (*models.ConversationMutationReceipt, bool) { + if receipt, ok := data.(*models.ConversationMutationReceipt); ok { + return receipt, true + } + if receipt, ok := data.(models.ConversationMutationReceipt); ok { + return &receipt, true + } + if payload, ok := data.(map[string]any); ok { + value, exists := payload["conversation_receipt"] + if !exists { + return nil, false + } + return conversationReceiptFromData(value) + } + encoded, err := json.Marshal(data) + if err != nil { + return nil, false + } + var envelope struct { + ConversationReceipt json.RawMessage `json:"conversation_receipt"` + } + if err := json.Unmarshal(encoded, &envelope); err == nil && len(envelope.ConversationReceipt) > 0 { + return conversationReceiptFromData(envelope.ConversationReceipt) + } + var receipt models.ConversationMutationReceipt + if err := json.Unmarshal(encoded, &receipt); err != nil || receipt.SessionID == "" || + (receipt.BaseRevision == 0 && receipt.Revision == 0 && receipt.Operations == nil && !receipt.Complete) { + return nil, false + } + return &receipt, true +} diff --git a/apps/backend/internal/gateway/websocket/conversation_delivery_test.go b/apps/backend/internal/gateway/websocket/conversation_delivery_test.go new file mode 100644 index 0000000000..2fdd55f9c7 --- /dev/null +++ b/apps/backend/internal/gateway/websocket/conversation_delivery_test.go @@ -0,0 +1,387 @@ +package websocket + +import ( + "context" + "encoding/json" + "errors" + "github.com/kandev/kandev/internal/plugins" + "github.com/kandev/kandev/internal/plugins/manifest" + "github.com/kandev/kandev/internal/plugins/store" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/kandev/kandev/internal/auth/authn" + "github.com/kandev/kandev/internal/task/models" + ws "github.com/kandev/kandev/pkg/websocket" +) + +type conversationSourceFixture struct { + reads *atomic.Int64 + session *models.TaskSession + revision models.ConversationRevision +} + +func (f conversationSourceFixture) GetTaskSession(_ context.Context, id string) (*models.TaskSession, error) { + if f.session == nil || f.session.ID != id { + return nil, models.ErrTaskSessionNotFound + } + return f.session, nil +} + +func (f conversationSourceFixture) ReadConversationRevision(context.Context, string) (models.ConversationRevision, error) { + if f.reads != nil { + f.reads.Add(1) + } + return f.revision, nil +} + +func TestConversationDeliveryProjectsSelectedUpsertAndCoverage(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + client := NewClient("conversation-client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + hub.clients[client] = true + registry := plugins.NewRegistry() + installedAt := time.Now() + registry.Add(&store.Record{Manifest: manifest.Manifest{ID: "plugin-1", Capabilities: manifest.Capabilities{APIRead: []string{"messages"}}}, Status: plugins.StatusActive, InstalledAt: installedAt}) + hub.SetPluginConversationService(plugins.NewService(nil, registry, nil, testLogger())) + taskID := "task-selected" + client.conversationSubscriptions["scope-1"] = conversationSubscription{ + ScopeID: "scope-1", SessionID: "session-1", ConsumerKind: conversationConsumerPlugin, + PluginID: "plugin-1", Generation: installedAt.UnixMicro(), UserID: "user-1", TaskID: &taskID, Sort: "asc", Epoch: "epoch-1", + } + + message := &models.Message{ + ID: "message-1", TaskSessionID: "session-1", TaskID: taskID, + AuthorType: models.MessageAuthorUser, Content: "prompt", + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + UpdatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 4, Revision: 5, Complete: true, + Operations: []models.ConversationMutationOperation{{ + Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityMessage, + ID: message.ID, SessionID: message.TaskSessionID, TaskID: taskID, + AuthorType: string(message.AuthorType), Message: message, + }}, + }) + + var frame ws.Message + if err := json.Unmarshal(<-client.send, &frame); err != nil { + t.Fatalf("decode changed frame: %v", err) + } + if frame.Action != ws.ActionSessionConversationChanged { + t.Fatalf("action = %q", frame.Action) + } + var payload conversationChangedPayload + if err := json.Unmarshal(frame.Payload, &payload); err != nil { + t.Fatalf("decode changed payload: %v", err) + } + if payload.BaseRevision != "4" || payload.Revision != "5" || len(payload.Operations) != 1 || payload.Operations[0].ID != message.ID { + t.Fatalf("changed payload = %+v", payload) + } +} + +func TestConversationDeliveryProjectsCoreEntitiesAtTheAPIBoundary(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + client := NewClient("conversation-client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + hub.clients[client] = true + client.conversationSubscriptions["scope-1"] = conversationSubscription{ + ScopeID: "scope-1", SessionID: "session-1", ConsumerKind: conversationConsumerCore, + UserID: "user-1", Epoch: "epoch-1", + } + message := &models.Message{ + ID: "message-core", TaskSessionID: "session-1", TaskID: "task-1", + AuthorType: models.MessageAuthorAgent, + Content: "private promptvisible", + Metadata: map[string]any{"normalized": map[string]any{"shell_exec": map[string]any{"output": map[string]any{"stdout": "secret shell body"}}}}, + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + UpdatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 1, Revision: 2, Complete: true, + Operations: []models.ConversationMutationOperation{{ + Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityMessage, + ID: message.ID, SessionID: message.TaskSessionID, TaskID: message.TaskID, Message: message, + }}, + }) + + var frame ws.Message + if err := json.Unmarshal(<-client.send, &frame); err != nil { + t.Fatalf("decode changed frame: %v", err) + } + var payload map[string]any + if err := json.Unmarshal(frame.Payload, &payload); err != nil { + t.Fatalf("decode changed payload: %v", err) + } + operation := payload["operations"].([]any)[0].(map[string]any) + projected := operation["message"].(map[string]any) + if projected["content"] != "visible" { + t.Fatalf("core content = %#v, want visible content", projected["content"]) + } + encoded, _ := json.Marshal(projected) + if strings.Contains(string(encoded), "secret shell body") { + t.Fatal("core message leaked shell metadata at the transport boundary") + } + if message.Content == "visible" { + t.Fatal("core projection mutated the source message") + } +} + +func TestConversationDeliveryProjectsSafeCoreTurnMetadataAndCompletion(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + client := NewClient("conversation-client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + hub.clients[client] = true + client.conversationSubscriptions["scope-1"] = conversationSubscription{ + ScopeID: "scope-1", SessionID: "session-1", ConsumerKind: conversationConsumerCore, + UserID: "user-1", Epoch: "epoch-1", + } + hadOutput := false + turn := &models.Turn{ + ID: "turn-core", TaskSessionID: "session-1", TaskID: "task-1", + Metadata: map[string]any{ + models.TurnMetaKeyRuntimeConfigSnapshot: models.TurnRuntimeConfigSnapshot{Model: "mock-fast"}, + models.TurnMetaKeyPromptDispatchPending: true, + "internal": "private turn state", + }, + } + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 1, Revision: 2, Complete: true, + Operations: []models.ConversationMutationOperation{{ + Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityTurn, + ID: turn.ID, SessionID: turn.TaskSessionID, TaskID: turn.TaskID, + Turn: turn, HadOutput: &hadOutput, + }}, + }) + + var frame ws.Message + if err := json.Unmarshal(<-client.send, &frame); err != nil { + t.Fatalf("decode changed frame: %v", err) + } + var payload map[string]any + if err := json.Unmarshal(frame.Payload, &payload); err != nil { + t.Fatalf("decode changed payload: %v", err) + } + operation := payload["operations"].([]any)[0].(map[string]any) + projected := operation["turn"].(map[string]any) + if projected["had_output"] != false { + t.Fatalf("core turn had_output = %#v, want false", projected["had_output"]) + } + metadata := projected["metadata"].(map[string]any) + if metadata[models.TurnMetaKeyRuntimeConfigSnapshot] == nil { + t.Fatal("core turn dropped runtime configuration metadata") + } + if _, ok := metadata[models.TurnMetaKeyPromptDispatchPending]; ok { + t.Fatal("core turn exposed prompt-dispatch metadata") + } + if _, ok := metadata["internal"]; ok { + t.Fatal("core turn exposed private metadata") + } +} + +func TestConversationDeliverySendsCoverageOnlyForIrrelevantMutation(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + client := NewClient("conversation-client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + hub.clients[client] = true + registry := plugins.NewRegistry() + installedAt := time.Now() + registry.Add(&store.Record{Manifest: manifest.Manifest{ID: "plugin-1", Capabilities: manifest.Capabilities{APIRead: []string{"messages"}}}, Status: plugins.StatusActive, InstalledAt: installedAt}) + hub.SetPluginConversationService(plugins.NewService(nil, registry, nil, testLogger())) + taskID := "task-selected" + client.conversationSubscriptions["scope-1"] = conversationSubscription{ + ScopeID: "scope-1", SessionID: "session-1", ConsumerKind: conversationConsumerPlugin, + PluginID: "plugin-1", Generation: installedAt.UnixMicro(), UserID: "user-1", TaskID: &taskID, Epoch: "epoch-1", + } + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 8, Revision: 9, Complete: true, + Operations: []models.ConversationMutationOperation{{ + Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityMessage, + ID: "message-other", SessionID: "session-1", TaskID: "task-other", + AuthorType: string(models.MessageAuthorAgent), + }}, + }) + + var frame ws.Message + if err := json.Unmarshal(<-client.send, &frame); err != nil { + t.Fatalf("decode coverage frame: %v", err) + } + var payload conversationChangedPayload + if err := json.Unmarshal(frame.Payload, &payload); err != nil { + t.Fatalf("decode coverage payload: %v", err) + } + if len(payload.Operations) != 0 || payload.BaseRevision != "8" || payload.Revision != "9" { + t.Fatalf("coverage payload = %+v", payload) + } +} + +func TestConversationDeliveryResetsIncompleteReceipt(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + client := NewClient("conversation-client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + hub.clients[client] = true + client.conversationSubscriptions["scope-1"] = conversationSubscription{ + ScopeID: "scope-1", SessionID: "session-1", ConsumerKind: conversationConsumerCore, + UserID: "user-1", Epoch: "epoch-1", + } + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 2, Revision: 4, + Complete: false, + }) + + var frame ws.Message + if err := json.Unmarshal(<-client.send, &frame); err != nil { + t.Fatalf("decode reset frame: %v", err) + } + var payload conversationChangedPayload + if err := json.Unmarshal(frame.Payload, &payload); err != nil { + t.Fatalf("decode reset payload: %v", err) + } + if !payload.Reset || len(payload.Operations) != 0 { + t.Fatalf("reset payload = %+v", payload) + } +} + +func TestConversationSubscribeReturnsSourceRevisionForCore(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + hub.SetConversationSourceReader(conversationSourceFixture{ + session: &models.TaskSession{ID: "session-1", TaskID: "task-1"}, + revision: models.ConversationRevision{Exists: true, Revision: 17}, + }) + client := NewClient("conversation-client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + request, err := ws.NewRequest("request-1", ws.ActionSessionConversationSubscribe, map[string]any{ + "scope_id": "core:web:scope-1", "session_id": "session-1", "consumer_kind": "core", + }) + if err != nil { + t.Fatalf("request: %v", err) + } + client.handleConversationSubscribe(request) + var response ws.Message + if err := json.Unmarshal(<-client.controlSend, &response); err != nil { + t.Fatalf("decode response: %v", err) + } + var payload map[string]any + if err := json.Unmarshal(response.Payload, &payload); err != nil { + t.Fatalf("decode payload: %v", err) + } + if payload["revision"] != "17" || payload["epoch"] == "" || payload["success"] != true { + t.Fatalf("subscribe payload = %+v", payload) + } +} + +func TestConversationDeliveryStopsAfterRevocation(t *testing.T) { + for _, reason := range []string{"disabled", "capability", "generation", "session"} { + t.Run(reason, func(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + registry := plugins.NewRegistry() + record := &store.Record{Manifest: manifest.Manifest{ID: "plugin-1", Capabilities: manifest.Capabilities{APIRead: []string{"messages"}}}, Status: plugins.StatusActive, InstalledAt: time.Now()} + registry.Add(record) + hub.SetPluginConversationService(plugins.NewService(nil, registry, nil, testLogger())) + client := NewClient("client", authn.Identity{UserID: "user-1"}, nil, hub, testLogger()) + hub.clients[client] = true + client.conversationSubscriptions["scope"] = conversationSubscription{ScopeID: "scope", SessionID: "session-1", ConsumerKind: conversationConsumerPlugin, PluginID: "plugin-1", Generation: record.InstalledAt.UnixMicro(), UserID: "user-1"} + switch reason { + case "disabled": + record.Status = plugins.StatusDisabled + case "capability": + record.Capabilities.APIRead = nil + case "generation": + record.InstalledAt = record.InstalledAt.Add(time.Second) + case "session": + hub.authPolicy.Subscriptions.Session = func(context.Context, string) error { return errors.New("revoked") } + } + registry.Add(record) + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{SessionID: "session-1", BaseRevision: 0, Revision: 1, Complete: true}) + if len(client.conversationSubscriptions) != 0 { + t.Fatal("revoked subscription remains active") + } + select { + case frame := <-client.send: + var message ws.Message + if err := json.Unmarshal(frame, &message); err != nil { + t.Fatal(err) + } + var payload map[string]any + if err := json.Unmarshal(message.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload["terminal"] != true { + t.Fatalf("expected terminal notification, got %s", frame) + } + default: + t.Fatal("missing terminal notification") + } + }) + } +} + +func TestConversationDeliveryKeepsSyntheticSubscriptionWhenActiveUserCheckFails(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + hub.setAuthPolicy(AuthPolicy{ActiveUser: func(context.Context, string) bool { return false }}) + client := NewClient("synthetic-client", authn.Identity{UserID: "default-user", Synthetic: true}, nil, hub, testLogger()) + hub.clients[client] = true + client.conversationSubscriptions["scope-1"] = conversationSubscription{ + ScopeID: "scope-1", SessionID: "session-1", ConsumerKind: conversationConsumerCore, Epoch: "epoch-1", + } + + hub.BroadcastConversationMutation(&models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 1, Revision: 2, Complete: true, + }) + + var frame ws.Message + if err := json.Unmarshal(<-client.send, &frame); err != nil { + t.Fatalf("decode changed frame: %v", err) + } + if frame.Action != ws.ActionSessionConversationChanged { + t.Fatalf("action = %q", frame.Action) + } + var payload conversationChangedPayload + if err := json.Unmarshal(frame.Payload, &payload); err != nil { + t.Fatalf("decode changed payload: %v", err) + } + if payload.Terminal || payload.Revision != "2" { + t.Fatalf("changed payload = %+v", payload) + } +} + +func TestConversationRevisionWorkerChecksIdleSubscriptions(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + hub := NewHub(ws.NewDispatcher(), testLogger()) + reads := &atomic.Int64{} + hub.SetConversationSourceReader(conversationSourceFixture{reads: reads, revision: models.ConversationRevision{Exists: true, Revision: 7}}) + client := NewClient("client", authn.Identity{}, nil, hub, testLogger()) + hub.clients[client] = true + client.conversationSubscriptions["scope"] = conversationSubscription{ScopeID: "scope", SessionID: "session-1", ConsumerKind: conversationConsumerCore, Epoch: "epoch"} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go hub.Run(ctx) + synctest.Wait() + time.Sleep(5 * time.Second) + synctest.Wait() + select { + case frame := <-client.send: + var message ws.Message + if err := json.Unmarshal(frame, &message); err != nil { + t.Fatal(err) + } + var payload map[string]any + if err := json.Unmarshal(message.Payload, &payload); err != nil { + t.Fatal(err) + } + if payload["check"] != true || payload["revision"] != "7" { + t.Fatalf("unexpected revision check: %s", frame) + } + default: + t.Fatal("idle subscriber never receives source revision") + } + client.mu.Lock() + delete(client.conversationSubscriptions, "scope") + client.mu.Unlock() + time.Sleep(5 * time.Second) + synctest.Wait() + if reads.Load() != 1 { + t.Fatalf("reads after unsubscribe = %d", reads.Load()) + } + cancel() + synctest.Wait() + }) +} diff --git a/apps/backend/internal/gateway/websocket/hub.go b/apps/backend/internal/gateway/websocket/hub.go index dd197af06b..defa6c9271 100644 --- a/apps/backend/internal/gateway/websocket/hub.go +++ b/apps/backend/internal/gateway/websocket/hub.go @@ -6,8 +6,10 @@ import ( "encoding/json" "sync" + "github.com/google/uuid" "github.com/kandev/kandev/internal/common/logger" "github.com/kandev/kandev/internal/plugins" + "github.com/kandev/kandev/internal/task/models" ws "github.com/kandev/kandev/pkg/websocket" "go.uber.org/zap" ) @@ -55,6 +57,8 @@ type Hub struct { sessionGitDataProvider SessionGitDataProvider userSubscriptionListeners []func(userID string) pluginConversationService *plugins.Service + conversationSourceReader ConversationSourceReader + conversationEpoch string // clientDisconnectListener releases connection-bound resources after a // client is removed from the hub. It runs asynchronously so durable cleanup @@ -77,11 +81,8 @@ type Hub struct { // value = unscoped, today's behavior. See access.go. authPolicy AuthPolicy - // orderedSessionMu closes the replay/register/append race for the ordered - // session stream. Never acquire it while holding h.mu or a client mutex. - orderedSessionMu sync.Mutex - mu sync.RWMutex - logger *logger.Logger + mu sync.RWMutex + logger *logger.Logger } // NewHub creates a new WebSocket hub @@ -100,6 +101,7 @@ func NewHub(dispatcher *ws.Dispatcher, log *logger.Logger) *Hub { dispatcher: dispatcher, sessionMode: newSessionModeTracker(), logger: log.WithFields(zap.String("component", "ws_hub")), + conversationEpoch: uuid.NewString(), } } @@ -107,13 +109,19 @@ func (h *Hub) SetPluginConversationService(service *plugins.Service) { h.mu.Lock() h.pluginConversationService = service h.mu.Unlock() - if service != nil { - service.SetSessionEventSink(func(event plugins.SessionEvent) { - h.orderedSessionMu.Lock() - defer h.orderedSessionMu.Unlock() - h.broadcastCommittedOrderedSessionEvent(service, event) - }) - } +} + +// SetConversationSourceReader wires the authorized task source used to +// establish the initial revision for Host-only conversation subscriptions. +func (h *Hub) SetConversationSourceReader(reader ConversationSourceReader) { + h.mu.Lock() + h.conversationSourceReader = reader + h.mu.Unlock() +} + +type ConversationSourceReader interface { + GetTaskSession(context.Context, string) (*models.TaskSession, error) + ReadConversationRevision(context.Context, string) (models.ConversationRevision, error) } type SystemMetricsInterestTracker interface { @@ -132,6 +140,9 @@ func (h *Hub) Run(ctx context.Context) { h.mu.Lock() h.dispatchCtx = ctx h.mu.Unlock() + checksDone := make(chan struct{}) + go func() { defer close(checksDone); h.runConversationChecks(ctx) }() + defer func() { <-checksDone }() for { select { @@ -533,22 +544,12 @@ func (h *Hub) getSessionRecipientsLocked(sessionID string) []*Client { // BroadcastToSession sends a notification to clients subscribed to OR focused on // a specific session. See getSessionRecipientsLocked for why focus is included. func (h *Hub) BroadcastToSession(sessionID string, msg *ws.Message) { - h.appendAndBroadcastOrderedSessionEvent(sessionID, msg) data, err := json.Marshal(msg) if err != nil { h.logger.Error("Failed to marshal message", zap.Error(err)) return } clients := h.authorizedSessionRecipients(sessionID) - if _, ordered := orderedEventTypeByAction[msg.Action]; ordered { - legacyClients := clients[:0] - for _, client := range clients { - if !client.hasOrderedSessionSubscription(sessionID) { - legacyClients = append(legacyClients, client) - } - } - clients = legacyClients - } h.logger.Debug("BroadcastToSession", zap.String("session_id", sessionID), zap.String("action", msg.Action), diff --git a/apps/backend/internal/gateway/websocket/hub_broadcast_test.go b/apps/backend/internal/gateway/websocket/hub_broadcast_test.go index 0c01313195..c66d34fdf1 100644 --- a/apps/backend/internal/gateway/websocket/hub_broadcast_test.go +++ b/apps/backend/internal/gateway/websocket/hub_broadcast_test.go @@ -4,13 +4,9 @@ import ( "testing" "github.com/kandev/kandev/internal/auth/authn" - "github.com/kandev/kandev/internal/plugins" ws "github.com/kandev/kandev/pkg/websocket" ) -// registerTestClient adds a client to the hub's connected set so broadcasts can -// reach it (BroadcastToSession only iterates subscriber/focus maps, but the -// client must be a live connection in real usage). func registerTestClient(h *Hub, c *Client) { h.mu.Lock() h.clients[c] = true @@ -26,18 +22,16 @@ func clientReceived(c *Client) bool { } } -// TestBroadcastToSession_ReachesFocusedClient guards the resume regression: -// a client that is focused on a session but whose ref-counted session.subscribe -// was dropped (subscriber_count 0) must still receive session-scoped broadcasts -// such as session.message.updated. Focus is the stable "actively viewing" signal; -// the subscribe ref-count churns to 0 during task-switch/resume. -func TestBroadcastToSession_ReachesFocusedClient(t *testing.T) { +func TestBroadcastToSessionReachesFocusedAndSubscribedClients(t *testing.T) { h := newTestHub(t) - c := newTestClient("c1") - registerTestClient(h, c) - - // Focused but NOT subscribed — mirrors the dropped-subscription resume race. - h.FocusSession(c, "sess-1") + focused := newTestClient("focused") + subscribed := newTestClient("subscribed") + other := newTestClient("other") + registerTestClient(h, focused) + registerTestClient(h, subscribed) + registerTestClient(h, other) + h.FocusSession(focused, "sess-1") + h.SubscribeToSession(subscribed, "sess-1") msg, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{"session_id": "sess-1"}) if err != nil { @@ -45,19 +39,20 @@ func TestBroadcastToSession_ReachesFocusedClient(t *testing.T) { } h.BroadcastToSession("sess-1", msg) - if !clientReceived(c) { - t.Fatal("focused-but-unsubscribed client did not receive session broadcast") + if !clientReceived(focused) || !clientReceived(subscribed) { + t.Fatal("focused and subscribed clients must receive the broadcast") + } + if clientReceived(other) { + t.Fatal("uninterested client received the broadcast") } } -// TestBroadcastToSession_ReachesSubscribedClient is the baseline: a subscribed -// (sidebar/background) client still receives broadcasts after the union change. -func TestBroadcastToSession_ReachesSubscribedClient(t *testing.T) { +func TestBroadcastToSessionDeduplicatesFocusedSubscription(t *testing.T) { h := newTestHub(t) c := newTestClient("c1") registerTestClient(h, c) - h.SubscribeToSession(c, "sess-1") + h.FocusSession(c, "sess-1") msg, err := ws.NewNotification(ws.ActionSessionMessageAdded, map[string]any{"session_id": "sess-1"}) if err != nil { @@ -65,139 +60,29 @@ func TestBroadcastToSession_ReachesSubscribedClient(t *testing.T) { } h.BroadcastToSession("sess-1", msg) - if !clientReceived(c) { - t.Fatal("subscribed client did not receive session broadcast") + if !clientReceived(c) || clientReceived(c) { + t.Fatal("client must receive exactly one broadcast") } } -// TestBroadcastToSession_NotDeliveredToUninterestedClient ensures the union does -// not over-deliver: a client neither subscribed nor focused gets nothing. -func TestBroadcastToSession_NotDeliveredToUninterestedClient(t *testing.T) { - h := newTestHub(t) - focused := newTestClient("focused") - other := newTestClient("other") - registerTestClient(h, focused) - registerTestClient(h, other) - - h.FocusSession(focused, "sess-1") - - msg, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{"session_id": "sess-1"}) - if err != nil { - t.Fatalf("notification: %v", err) - } - h.BroadcastToSession("sess-1", msg) - - if !clientReceived(focused) { - t.Fatal("focused client did not receive broadcast") - } - if clientReceived(other) { - t.Fatal("uninterested client should not receive broadcast") - } -} - -// TestBroadcastToSession_DeliversOnceToSubscribedAndFocusedClient ensures a -// client that is BOTH subscribed and focused receives exactly one copy (the -// union must dedupe). -func TestBroadcastToSession_DeliversOnceToSubscribedAndFocusedClient(t *testing.T) { - h := newTestHub(t) - c := newTestClient("c1") - registerTestClient(h, c) - - h.SubscribeToSession(c, "sess-1") - h.FocusSession(c, "sess-1") - - msg, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{"session_id": "sess-1"}) - if err != nil { - t.Fatalf("notification: %v", err) - } - h.BroadcastToSession("sess-1", msg) - - if !clientReceived(c) { - t.Fatal("subscribed+focused client did not receive broadcast") - } - if clientReceived(c) { - t.Fatal("subscribed+focused client received a duplicate broadcast") - } -} - -func TestBroadcastToSession_SuppressesLegacyCopyForOrderedCoreConsumer(t *testing.T) { - h := newTestHub(t) - c := newTestClient("c1") - registerTestClient(h, c) - h.SubscribeToSession(c, "sess-1") - c.orderedSessionSubscriptions = map[string]map[string]plugins.SessionDeliveryCursorKey{ - "sess-1": { - "core-1": {SessionID: "sess-1", ConsumerKind: "core", WireID: "core-1"}, - }, - } - - msg, err := ws.NewNotification( - ws.ActionSessionMessageAdded, - map[string]any{"session_id": "sess-1"}, - ) - if err != nil { - t.Fatalf("notification: %v", err) - } - h.BroadcastToSession("sess-1", msg) - - if clientReceived(c) { - t.Fatal("ordered core consumer received a duplicate legacy notification") - } -} - -func TestBroadcastToUserReportsWhetherAFrameWasQueued(t *testing.T) { +func TestBroadcastToUserReportsQueueResult(t *testing.T) { h := newTestHub(t) msg, err := ws.NewNotification("system.update_available", map[string]any{"occurrence_id": "v1.2.3"}) if err != nil { t.Fatalf("notification: %v", err) } if h.BroadcastToUser("user-1", msg) { - t.Fatal("broadcast without a user subscriber must report no queued recipient") + t.Fatal("broadcast without subscribers must report false") } - c := newTestClient("c1") registerTestClient(h, c) h.SubscribeToUser(c, "user-1") if !h.BroadcastToUser("user-1", msg) { - t.Fatal("broadcast with a user subscriber must report a queued recipient") + t.Fatal("broadcast with a subscriber must report true") } } -func TestBroadcastToUserReportsFalseWhenSubscriberBufferIsFull(t *testing.T) { - h := newTestHub(t) - c := newTestClient("c1") - registerTestClient(h, c) - h.SubscribeToUser(c, "user-1") - for index := 0; index < cap(c.send); index++ { - c.send <- []byte("queued") - } - - msg, err := ws.NewNotification("system.update_available", map[string]any{"occurrence_id": "v1.2.3"}) - if err != nil { - t.Fatalf("notification: %v", err) - } - if h.BroadcastToUser("user-1", msg) { - t.Fatal("broadcast with a full subscriber buffer must report no queued recipient") - } -} - -func TestBroadcastToUserReportsFalseWhenSubscriberIsClosed(t *testing.T) { - h := newTestHub(t) - c := newTestClient("c1") - registerTestClient(h, c) - h.SubscribeToUser(c, "user-1") - c.closeSend() - - msg, err := ws.NewNotification("system.update_available", map[string]any{"occurrence_id": "v1.2.3"}) - if err != nil { - t.Fatalf("notification: %v", err) - } - if h.BroadcastToUser("user-1", msg) { - t.Fatal("broadcast to a closed subscriber must report no queued recipient") - } -} - -func TestSendToIdentityTargetsConnectedClientsWithoutSubscription(t *testing.T) { +func TestSendToIdentityTargetsMatchingClients(t *testing.T) { h := newTestHub(t) first := newTestClient("first") first.identity = authn.Identity{UserID: "user-1"} @@ -216,117 +101,7 @@ func TestSendToIdentityTargetsConnectedClientsWithoutSubscription(t *testing.T) if got := h.SendToIdentity("user-1", msg); got != 2 { t.Fatalf("queued recipients = %d, want 2", got) } - if !clientReceived(first) || !clientReceived(second) { - t.Fatal("both matching identity clients must receive the notification") - } - if clientReceived(other) { - t.Fatal("a different identity must not receive the notification") + if !clientReceived(first) || !clientReceived(second) || clientReceived(other) { + t.Fatal("identity broadcast reached the wrong clients") } } - -// TestBroadcastCommittedPoisonFrameDeliversAndCountsAttempt pins the unified -// poison contract: the mirrored frame IS delivered to live subscribers (their -// recovery depends on seeing it) and the delivery attempt is counted only when -// a recipient exists. -func TestBroadcastCommittedPoisonFrameDeliversAndCountsAttempt(t *testing.T) { - h := newTestHub(t) - svc := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - h.SetPluginConversationService(svc) - - // message.added missing author/content -> projection poison. - event, err := svc.SessionEvents().Append( - "sess-poison", newStringPointer("task-poison"), "message.added", - []byte(`{"type":"message.added","session_id":"sess-poison","task_id":"task-poison"}`), - ) - if err != nil { - t.Fatalf("append poison event: %v", err) - } - if _, ok := svc.SessionEvents().Poison("sess-poison", event.ID); !ok { - t.Fatal("append did not poison the malformed event") - } - - c := newTestClient("c-poison") - registerTestClient(h, c) - c.orderedSessionSubscriptions = map[string]map[string]plugins.SessionDeliveryCursorKey{ - "sess-poison": { - "core-1": {SessionID: "sess-poison", ConsumerKind: "core", WireID: "core-1"}, - }, - } - - h.broadcastCommittedOrderedSessionEvent(svc, event) - - if !clientReceived(c) { - t.Fatal("live subscriber did not receive the poison frame") - } - record, ok := svc.SessionEvents().Poison("sess-poison", event.ID) - if !ok { - t.Fatal("poison record missing after broadcast") - } - if record.Attempts != 1 { - t.Fatalf("poison attempts = %d, want 1 (one delivery with a recipient)", record.Attempts) - } -} - -// TestBroadcastCommittedPoisonFrameWithoutRecipientsSkipsAttempt pins that a -// mirrored poison with nobody subscribed does not burn a delivery attempt. -func TestBroadcastCommittedPoisonFrameWithoutRecipientsSkipsAttempt(t *testing.T) { - h := newTestHub(t) - svc := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - h.SetPluginConversationService(svc) - - event, err := svc.SessionEvents().Append( - "sess-poison-idle", newStringPointer("task-poison"), "message.added", - []byte(`{"type":"message.added","session_id":"sess-poison-idle","task_id":"task-poison"}`), - ) - if err != nil { - t.Fatalf("append poison event: %v", err) - } - - h.broadcastCommittedOrderedSessionEvent(svc, event) - - record, ok := svc.SessionEvents().Poison("sess-poison-idle", event.ID) - if !ok { - t.Fatal("poison record missing after broadcast") - } - if record.Attempts != 0 { - t.Fatalf("poison attempts = %d, want 0 with no recipients", record.Attempts) - } -} - -func TestBroadcastCommittedPoisonQueueFailureDoesNotCountAttempt(t *testing.T) { - h := newTestHub(t) - svc := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - h.SetPluginConversationService(svc) - event, err := svc.SessionEvents().Append( - "sess-poison-full", newStringPointer("task-poison"), "message.added", - []byte(`{"type":"message.added","session_id":"sess-poison-full","task_id":"task-poison"}`), - ) - if err != nil { - t.Fatalf("append poison event: %v", err) - } - c := newTestClient("c-poison-full") - for range cap(c.send) { - c.send <- []byte("occupied") - } - registerTestClient(h, c) - c.orderedSessionSubscriptions = map[string]map[string]plugins.SessionDeliveryCursorKey{ - "sess-poison-full": { - "core-1": {SessionID: "sess-poison-full", ConsumerKind: "core", WireID: "core-1"}, - }, - } - - h.broadcastCommittedOrderedSessionEvent(svc, event) - - record, ok := svc.SessionEvents().Poison("sess-poison-full", event.ID) - if !ok { - t.Fatal("poison record missing after queue failure") - } - if record.Attempts != 0 || record.State != plugins.SessionPoisonPending { - t.Fatalf("poison after queue failure = attempts %d state %q, want pending at zero", record.Attempts, record.State) - } - if !c.closed { - t.Fatal("ordered queue failure must close the client for replay") - } -} - -func newStringPointer(value string) *string { return &value } diff --git a/apps/backend/internal/gateway/websocket/ordered_session_events.go b/apps/backend/internal/gateway/websocket/ordered_session_events.go deleted file mode 100644 index 8a23ca34b0..0000000000 --- a/apps/backend/internal/gateway/websocket/ordered_session_events.go +++ /dev/null @@ -1,214 +0,0 @@ -package websocket - -import ( - "context" - "encoding/json" - "time" - - "github.com/kandev/kandev/internal/plugins" - "github.com/kandev/kandev/internal/sysprompt" - ws "github.com/kandev/kandev/pkg/websocket" - "go.uber.org/zap" -) - -const ( - orderedMessageAddedEvent = "message.added" - orderedMessageUpdatedEvent = "message.updated" - orderedMessageDeletedEvent = "message.deleted" - orderedTurnStartedEvent = "session.turn.started" - orderedTurnCompletedEvent = "session.turn.completed" - orderedTurnRemovedEvent = "session.turn.removed" - orderedSessionRemovedEvent = "session.removed" -) - -var orderedEventTypeByAction = map[string]string{ - ws.ActionSessionMessageAdded: orderedMessageAddedEvent, - ws.ActionSessionMessageUpdated: orderedMessageUpdatedEvent, - ws.ActionSessionMessageDeleted: orderedMessageDeletedEvent, - ws.ActionSessionTurnStarted: orderedTurnStartedEvent, - ws.ActionSessionTurnCompleted: orderedTurnCompletedEvent, - ws.ActionSessionTurnRemoved: orderedTurnRemovedEvent, - ws.ActionSessionRemoved: orderedSessionRemovedEvent, -} - -const orderedContentPayloadKey = "content" - -//nolint:cyclop // This boundary coordinates journal sync, poison recording, and live fan-out. -func (h *Hub) appendAndBroadcastOrderedSessionEvent(sessionID string, message *ws.Message) { - h.orderedSessionMu.Lock() - defer h.orderedSessionMu.Unlock() - service := h.pluginConversationService - eventType := orderedEventTypeByAction[message.Action] - if service == nil || eventType == "" { - return - } - if service.HasConversationJournal() { - events, err := service.SyncCommittedSessionEvents(context.Background(), sessionID) - if err != nil { - if h.logger != nil { - h.logger.Error("mirror committed ordered session events", zap.Error(err)) - } - return - } - for _, event := range events { - h.broadcastCommittedOrderedSessionEvent(service, event) - } - return - } - var source map[string]any - if err := json.Unmarshal(message.Payload, &source); err != nil { - source = map[string]any{"raw": message.Payload} - } - payload := sanitizedOrderedSessionPayload(eventType, source) - encoded, err := json.Marshal(payload) - if err != nil { - return - } - var taskID *string - if value, ok := payload["task_id"].(string); ok && value != "" { - taskID = &value - } - event, err := service.SessionEvents().Append(sessionID, taskID, eventType, encoded) - if err != nil { - if h.logger != nil { - h.logger.Error("append ordered session event", zap.Error(err)) - } - return - } - h.deliverOrderedSessionEvent(service, event, h.orderedSessionRecipients(sessionID)) -} - -// broadcastCommittedOrderedSessionEvent fans a mirrored primary-journal event -// out through the same atomic delivery claim as locally appended events. -func (h *Hub) broadcastCommittedOrderedSessionEvent(service *plugins.Service, event plugins.SessionEvent) { - h.deliverOrderedSessionEvent(service, event, h.orderedSessionRecipients(event.SessionID)) -} - -func (h *Hub) deliverOrderedSessionEvent( - service *plugins.Service, - event plugins.SessionEvent, - recipients []*Client, -) { - if len(recipients) == 0 { - return - } - claim, err := service.SessionDelivery().Claim(event.SessionID, event.ID, time.Now().UTC()) - if err != nil { - if h.logger != nil { - h.logger.Error("claim ordered session poison", zap.String("event_id", event.ID), zap.Error(err)) - } - return - } - if claim.Disposition != plugins.SessionDeliveryUntracked && - claim.Disposition != plugins.SessionDeliveryClaimed { - return - } - queued := false - for _, client := range recipients { - queued = client.sendOrderedSessionEvent(event) || queued - } - if claim.Disposition == plugins.SessionDeliveryClaimed { - if err := service.SessionDelivery().Complete(claim, queued, time.Now().UTC()); err != nil && h.logger != nil { - h.logger.Error("complete ordered session poison", zap.String("event_id", event.ID), zap.Error(err)) - } - } -} - -func sanitizedOrderedSessionPayload(eventType string, source map[string]any) map[string]any { - payload := map[string]any{eventTypePayloadKey: eventType} - keys := []string{sessionIDPayloadKey, taskIDPayloadKey} - switch eventType { - case orderedMessageAddedEvent, orderedMessageUpdatedEvent: - keys = append(keys, - "message_id", "turn_id", "author_type", orderedContentPayloadKey, - "created_at", "updated_at", "prompt_index", "requests_input", - ) - if messageType, exists := source[eventTypePayloadKey]; exists { - payload["message_type"] = messageType - } - if metadata, ok := source["metadata"].(map[string]any); ok { - if sanitized := plugins.SanitizeConversationMessageMetadata(metadata); len(sanitized) > 0 { - payload["metadata"] = sanitized - } - } - if senderTaskID, ok := source["sender_task_id"].(string); ok && senderTaskID != "" { - payload["sender_task_id"] = senderTaskID - if metadata, ok := payload["metadata"].(map[string]any); !ok { - payload["metadata"] = map[string]any{"sender_task_id": senderTaskID} - } else { - metadata["sender_task_id"] = senderTaskID - } - } - case orderedMessageDeletedEvent: - keys = append(keys, "message_id") - case orderedTurnStartedEvent, orderedTurnCompletedEvent, orderedTurnRemovedEvent: - keys = append(keys, "id", "started_at", "completed_at", "updated_at", "had_output") - } - for _, key := range keys { - if value, exists := source[key]; exists { - if key == orderedContentPayloadKey { - if content, ok := value.(string); ok { - value = sysprompt.StripSystemContent(content) - } - } - payload[key] = value - } - } - return payload -} - -func (h *Hub) orderedSessionRecipients(sessionID string) []*Client { - h.mu.RLock() - clients := make([]*Client, 0) - for client := range h.clients { - if client.hasOrderedSessionSubscription(sessionID) { - clients = append(clients, client) - } - } - h.mu.RUnlock() - // Re-authorize at fanout time, mirroring the legacy session recipients: a - // client whose workspace/session membership was revoked after subscribing - // must not keep receiving ordered frames. Denied clients lose the ordered - // subscription itself AND its durable cursors (same revocation semantics - // as the legacy path), so the partition stops being pinned by them and - // retention can reclaim it. - allowed, denied := h.partitionAuthorized(clients, sessionID, h.authPolicy.Subscriptions.Session) - service := h.pluginConversationService - for _, client := range denied { - client.mu.Lock() - revoked := make([]plugins.SessionDeliveryCursorKey, 0, len(client.orderedSessionSubscriptions[sessionID])) - for _, key := range client.orderedSessionSubscriptions[sessionID] { - revoked = append(revoked, key) - } - delete(client.orderedSessionSubscriptions, sessionID) - client.mu.Unlock() - if service == nil { - continue - } - for _, key := range revoked { - if err := service.SessionEvents().ReleaseCursor(key); err != nil && h.logger != nil { - h.logger.Warn( - "release revoked ordered session cursor", - zap.String("session_id", sessionID), - zap.Error(err), - ) - } - } - } - return allowed -} - -func (c *Client) hasOrderedSessionSubscription(sessionID string) bool { - c.mu.RLock() - defer c.mu.RUnlock() - return len(c.orderedSessionSubscriptions[sessionID]) > 0 -} - -func sessionEventFrame(event plugins.SessionEvent) ([]byte, error) { - return json.Marshal(map[string]any{ - eventTypePayloadKey: "session.event", "protocol_version": event.ProtocolVersion, - "event_type": event.EventType, sessionIDPayloadKey: event.SessionID, - taskIDPayloadKey: event.TaskID, eventSequencePayloadKey: event.Sequence, - "event_id": event.ID, "payload": event.Payload, - }) -} diff --git a/apps/backend/internal/gateway/websocket/ordered_session_events_test.go b/apps/backend/internal/gateway/websocket/ordered_session_events_test.go deleted file mode 100644 index 30012fc9c3..0000000000 --- a/apps/backend/internal/gateway/websocket/ordered_session_events_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package websocket - -import ( - "context" - "errors" - "testing" - - "github.com/kandev/kandev/internal/auth/authn" - "github.com/kandev/kandev/internal/plugins" - ws "github.com/kandev/kandev/pkg/websocket" - "github.com/stretchr/testify/require" -) - -func TestSanitizedOrderedSessionPayloadStripsSystemContent(t *testing.T) { - payload := sanitizedOrderedSessionPayload("message.added", map[string]any{ - "session_id": "session-1", - "task_id": "task-1", - "message_id": "message-1", - orderedContentPayloadKey: "visible secret", - }) - - require.Equal(t, "visible", payload["content"]) -} -func TestTurnRemovedUsesOrderedSessionEventMapping(t *testing.T) { - require.Equal(t, "session.turn.removed", orderedEventTypeByAction[ws.ActionSessionTurnRemoved]) - payload := sanitizedOrderedSessionPayload(orderedTurnRemovedEvent, map[string]any{ - "id": "turn-1", - "session_id": "session-1", - "task_id": "task-1", - }) - require.Equal(t, "turn-1", payload["id"]) -} - -// TestOrderedRecipientsRevocationReleasesDurableCursor pins the round-9 fix: -// when authorization denies a previously-subscribed client at fanout time, the -// denied client is removed and its durable cursor is released so retention can -// reclaim the partition. -func TestOrderedRecipientsRevocationReleasesDurableCursor(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - h := NewHub(ws.NewDispatcher(), testLogger()) - go h.Run(ctx) - svc := plugins.NewService(nil, plugins.NewRegistry(), nil, testLogger()) - h.SetPluginConversationService(svc) - key := plugins.SessionDeliveryCursorKey{ - SessionID: "sess-revoked", ConsumerKind: "core", WireID: "wire-revoked", UserID: "user-1", - } - require.NoError(t, svc.SessionEvents().RegisterCursor(key, 0)) - - c := NewClient("c-revoked", authn.Identity{}, nil, h, testLogger()) - registerTestClient(h, c) - c.orderedSessionSubscriptions = map[string]map[string]plugins.SessionDeliveryCursorKey{ - "sess-revoked": {"wire-revoked": key}, - } - deny := func(context.Context, string) error { return errors.New("membership revoked") } - h.authPolicy.Subscriptions.Session = deny - - recipients := h.orderedSessionRecipients("sess-revoked") - require.Len(t, recipients, 0) - require.False(t, c.hasOrderedSessionSubscription("sess-revoked")) - require.False(t, svc.SessionEvents().HasCursor(key)) -} diff --git a/apps/backend/internal/gateway/websocket/task_notifications.go b/apps/backend/internal/gateway/websocket/task_notifications.go index b0db471487..3c99609679 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications.go +++ b/apps/backend/internal/gateway/websocket/task_notifications.go @@ -298,12 +298,16 @@ func (b *TaskEventBroadcaster) routeBroadcast( case ws.ActionSessionMessageAdded, ws.ActionSessionMessageUpdated, ws.ActionSessionMessageDeleted, ws.ActionSessionTurnStarted, ws.ActionSessionTurnCompleted, ws.ActionSessionTurnRemoved: if sessionID != "" { + b.hub.BroadcastConversationMutation(data) b.hub.BroadcastToSession(sessionID, msg) return nil } case ws.ActionSessionRemoved: if sessionID != "" { - b.hub.appendAndBroadcastOrderedSessionEvent(sessionID, msg) + if _, hasReceipt := conversationReceiptFromData(data); hasReceipt { + b.hub.BroadcastConversationMutation(data) + } + b.hub.BroadcastToSession(sessionID, msg) return nil } case ws.ActionSessionWorkspaceSourcesUpdated: diff --git a/apps/backend/internal/gateway/websocket/task_notifications_test.go b/apps/backend/internal/gateway/websocket/task_notifications_test.go index c8ac31cdf5..5dbb0a739c 100644 --- a/apps/backend/internal/gateway/websocket/task_notifications_test.go +++ b/apps/backend/internal/gateway/websocket/task_notifications_test.go @@ -337,6 +337,79 @@ func TestTaskEventBroadcaster_PreservesAllFields(t *testing.T) { } } +func TestTaskEventBroadcaster_ProjectsConversationReceiptFromEventData(t *testing.T) { + log := testLogger() + hub := NewHub(nil, log) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go hub.Run(ctx) + client := newTestClient("conversation-client") + client.hub = hub + client.conversationSubscriptions = map[string]conversationSubscription{ + "core-scope": { + ScopeID: "core-scope", SessionID: "session-1", ConsumerKind: conversationConsumerCore, + Epoch: "epoch-1", + }, + } + registerTestClient(hub, client) + broadcaster := &TaskEventBroadcaster{hub: hub, logger: log} + + receipt := &models.ConversationMutationReceipt{ + SessionID: "session-1", BaseRevision: 4, Revision: 5, Complete: true, + Operations: []models.ConversationMutationOperation{{ + Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityMessage, + ID: "message-1", SessionID: "session-1", AuthorType: string(models.MessageAuthorUser), + Message: &models.Message{ID: "message-1", TaskSessionID: "session-1", AuthorType: models.MessageAuthorUser, Content: "prompt"}, + }}, + } + + require.NoError(t, broadcaster.broadcastEvent(context.Background(), bus.NewEvent( + events.MessageAdded, + "task-service", + map[string]interface{}{ + "session_id": "session-1", + "conversation_receipt": receipt, + }, + ), ws.ActionSessionMessageAdded)) + + frame := <-client.send + var message ws.Message + require.NoError(t, json.Unmarshal(frame, &message)) + if message.Action != ws.ActionSessionConversationChanged { + t.Fatalf("action = %q, want %q", message.Action, ws.ActionSessionConversationChanged) + } +} + +func TestTaskEventBroadcasterDoesNotResetConversationOnReceiptlessSessionRemoval(t *testing.T) { + hub := newTestHub(t) + client := newTestClient("conversation-client") + client.hub = hub + registerTestClient(hub, client) + client.conversationSubscriptions = map[string]conversationSubscription{ + "core-scope": { + ScopeID: "core-scope", SessionID: "session-1", ConsumerKind: conversationConsumerCore, + Epoch: "epoch-1", + }, + } + hub.SubscribeToSession(client, "session-1") + broadcaster := &TaskEventBroadcaster{hub: hub, logger: testLogger()} + msg, err := ws.NewNotification(ws.ActionSessionRemoved, map[string]any{"session_id": "session-1"}) + require.NoError(t, err) + require.NoError(t, broadcaster.routeBroadcast(ws.ActionSessionRemoved, msg.Payload, "session-1", "", msg)) + + frame := <-client.send + var received ws.Message + require.NoError(t, json.Unmarshal(frame, &received)) + if received.Action != ws.ActionSessionRemoved { + t.Fatalf("first action = %q, want session removal", received.Action) + } + select { + case extra := <-client.send: + t.Fatalf("receiptless removal emitted an extra conversation frame: %s", extra) + default: + } +} + func TestTaskEventBroadcaster_CancellationIsSessionScoped(t *testing.T) { h := newTestHub(t) first := newTestClient("first") diff --git a/apps/backend/internal/office/repository/sqlite/migrations_priority_test.go b/apps/backend/internal/office/repository/sqlite/migrations_priority_test.go index 25e2d55326..9104dbe246 100644 --- a/apps/backend/internal/office/repository/sqlite/migrations_priority_test.go +++ b/apps/backend/internal/office/repository/sqlite/migrations_priority_test.go @@ -130,8 +130,10 @@ func TestMigrate_PriorityIdempotent(t *testing.T) { CREATE TABLE tasks ( id TEXT PRIMARY KEY, title TEXT NOT NULL, + description TEXT DEFAULT '', priority TEXT NOT NULL DEFAULT 'medium' CHECK (priority IN ('critical','high','medium','low')), + identifier TEXT, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/apps/backend/internal/office/testharness/routes.go b/apps/backend/internal/office/testharness/routes.go index 2688a57257..e1d7e2c8a4 100644 --- a/apps/backend/internal/office/testharness/routes.go +++ b/apps/backend/internal/office/testharness/routes.go @@ -830,7 +830,8 @@ func seedMessageHandler( if req.CreatedAt != nil { msg.CreatedAt = req.CreatedAt.UTC() } - if err := repo.CreateMessage(ctx, msg); err != nil { + receipt, err := repo.CreateMessageWithConversationReceipt(ctx, msg) + if err != nil { log.Error("test harness: create message failed", zap.Error(err)) c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return @@ -848,12 +849,12 @@ func seedMessageHandler( } if taskSvc != nil { - if err := taskSvc.PublishMessageEvent(ctx, events.MessageAdded, msg); err != nil { + if err := taskSvc.PublishMessageEvent(ctx, events.MessageAdded, msg, receipt); err != nil { log.Warn("test harness: publish message added failed; using fallback", zap.Error(err)) - publishMessageAddedFallback(ctx, eventBus, msg, log) + publishMessageAddedFallback(ctx, eventBus, msg, receipt, log) } } else { - publishMessageAddedFallback(ctx, eventBus, msg, log) + publishMessageAddedFallback(ctx, eventBus, msg, receipt, log) } c.JSON(http.StatusOK, gin.H{testMessageIDKey: msg.ID, testTurnIDKey: msg.TurnID}) } @@ -864,7 +865,13 @@ func seedMessageHandler( // carries the raw message fields (prompt_index, RFC3339Nano created_at) but // none of the session-scoped pending_action projection taskSvc.PublishMessageEvent // computes — callers exercising AC-34/AC-51 must supply a real taskSvc instead. -func publishMessageAddedFallback(ctx context.Context, eventBus bus.EventBus, msg *models.Message, log *logger.Logger) { +func publishMessageAddedFallback( + ctx context.Context, + eventBus bus.EventBus, + msg *models.Message, + receipt *models.ConversationMutationReceipt, + log *logger.Logger, +) { if eventBus == nil { return } @@ -889,6 +896,9 @@ func publishMessageAddedFallback(ctx context.Context, eventBus bus.EventBus, msg if msg.Metadata != nil { data["metadata"] = msg.Metadata } + if receipt != nil { + data["conversation_receipt"] = receipt + } publishMessageEvent(ctx, eventBus, events.MessageAdded, data, log) } @@ -927,11 +937,13 @@ func updateMessageHandler( } message.Content = request.Content message.UpdatedAt = time.Now().UTC() - if err := repo.UpdateMessage(c.Request.Context(), message); err != nil { + receipt, err := repo.UpdateMessageWithConversationReceipt(c.Request.Context(), message) + if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } data := messageEventData(message) + data["conversation_receipt"] = receipt publishMessageEvent(c.Request.Context(), eventBus, events.MessageUpdated, data, log) c.JSON(http.StatusOK, gin.H{"message_id": message.ID, "updated_at": data["updated_at"]}) } @@ -948,11 +960,13 @@ func deleteMessageHandler( c.JSON(http.StatusNotFound, gin.H{"error": "message not found"}) return } - if err := repo.DeleteMessage(c.Request.Context(), message.ID); err != nil { + receipt, err := repo.DeleteMessageWithConversationReceipt(c.Request.Context(), message.ID) + if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } data := messageEventData(message) + data["conversation_receipt"] = receipt publishMessageEvent(c.Request.Context(), eventBus, events.MessageDeleted, data, log) c.JSON(http.StatusOK, gin.H{testMessageIDKey: message.ID}) } @@ -983,7 +997,8 @@ func completeTurnHandler( ) gin.HandlerFunc { return func(c *gin.Context) { turnID := c.Param("id") - if err := repo.CompleteTurn(c.Request.Context(), turnID); err != nil { + receipt, err := repo.CompleteTurnWithConversationReceipt(c.Request.Context(), turnID) + if err != nil { c.JSON(http.StatusNotFound, gin.H{"error": "turn not found"}) return } @@ -1001,6 +1016,7 @@ func completeTurnHandler( if turn.CompletedAt != nil { data[testCompletedAtKey] = turn.CompletedAt.UTC().Format(time.RFC3339Nano) } + data["conversation_receipt"] = receipt if eventBus != nil { if err := eventBus.Publish( c.Request.Context(), diff --git a/apps/backend/internal/office/testharness/routes_test.go b/apps/backend/internal/office/testharness/routes_test.go index ed358aca9e..a0b2795bb5 100644 --- a/apps/backend/internal/office/testharness/routes_test.go +++ b/apps/backend/internal/office/testharness/routes_test.go @@ -671,6 +671,13 @@ func TestSeedMessageUserAuthorPersistsPromptIndex(t *testing.T) { if created, ok := data["created_at"].(string); !ok || !strings.Contains(created, ".") { t.Fatalf("event created_at = %#v, want RFC3339Nano fractional precision", data["created_at"]) } + receipt, ok := data["conversation_receipt"].(*models.ConversationMutationReceipt) + if !ok { + t.Fatalf("conversation_receipt = %T, want *models.ConversationMutationReceipt", data["conversation_receipt"]) + } + if !receipt.Complete || receipt.Revision != receipt.BaseRevision+1 { + t.Fatalf("conversation_receipt = %+v, want one complete revision", receipt) + } } // TestSeedMessageDefaultsToAgentWithoutPromptIndex verifies that non-user seeded messages carry no prompt index. diff --git a/apps/backend/internal/plugins/conversation_auth.go b/apps/backend/internal/plugins/conversation_auth.go new file mode 100644 index 0000000000..cc80061c09 --- /dev/null +++ b/apps/backend/internal/plugins/conversation_auth.go @@ -0,0 +1,28 @@ +package plugins + +import "errors" + +var ( + ErrConversationBindingInvalid = errors.New("invalid conversation binding") + ErrConversationGenerationSuperseded = errors.New("conversation generation superseded") +) + +// AuthorizeConversationConsumer validates the short-lived plugin binding used +// by the live source subscription. +func (s *Service) AuthorizeConversationConsumer( + pluginID, userID string, generation int64, bindingToken string, +) error { + record, err := s.Get(pluginID) + if err != nil || record.Status != StatusActive || !record.Capabilities.CanRead("messages") { + return ErrConversationBindingInvalid + } + if conversationGeneration(record.InstalledAt) != generation { + return ErrConversationGenerationSuperseded + } + claims, err := s.conversationTokens.parse(bindingToken) + if err != nil || claims.Kind != tokenKindBinding || claims.PluginID != pluginID || + claims.UserID != userID || claims.Generation != generation { + return ErrConversationBindingInvalid + } + return nil +} diff --git a/apps/backend/internal/plugins/conversation_handlers.go b/apps/backend/internal/plugins/conversation_handlers.go index 5170bb1b59..c285a140ed 100644 --- a/apps/backend/internal/plugins/conversation_handlers.go +++ b/apps/backend/internal/plugins/conversation_handlers.go @@ -2,6 +2,7 @@ package plugins import ( "context" + "errors" "net/http" "strconv" "time" @@ -29,6 +30,12 @@ type ConversationReader interface { ListTurnsBySession(context.Context, string) ([]*taskmodels.Turn, error) } +type conversationSourceReader interface { + ReadConversationRevision(context.Context, string) (taskmodels.ConversationRevision, error) + ReadConversationMessagesPage(context.Context, taskmodels.ConversationMessagePageRequest) (taskmodels.ConversationMessagePage, error) + ReadConversationTurnsPage(context.Context, taskmodels.ConversationTurnPageRequest) (taskmodels.ConversationTurnPage, error) +} + type conversationError struct { Code string `json:"code"` Message string `json:"message"` @@ -99,6 +106,27 @@ type conversationTurnsResponse struct { Turns []conversationTurnDTO `json:"turns"` } +type conversationSourceMessagesResponse struct { + Messages []conversationMessageDTO `json:"messages"` + HasMore bool `json:"hasMore"` + Cursor *string `json:"cursor"` + Epoch string `json:"epoch"` + Revision int64 `json:"revision,string"` +} + +type conversationSourceTurnsResponse struct { + Turns []conversationTurnDTO `json:"turns"` + HasMore bool `json:"hasMore"` + Cursor *string `json:"cursor"` + Epoch string `json:"epoch"` + Revision int64 `json:"revision,string"` +} + +type conversationSourceRevisionResponse struct { + Epoch string `json:"epoch"` + Revision int64 `json:"revision,string"` +} + type conversationContinuationRenewRequest struct { Cursor string `json:"cursor"` SnapshotToken string `json:"snapshot_token"` @@ -122,11 +150,218 @@ type conversationMessageQuery struct { func registerConversationRoutes(api *gin.RouterGroup, ctrl *Controller) { api.GET("/:id/conversation/binding", ctrl.conversationBinding) + api.GET("/:id/conversation/v2/task-sessions/:sessionId/messages", ctrl.conversationSourceMessages) + api.GET("/:id/conversation/v2/task-sessions/:sessionId/turns", ctrl.conversationSourceTurns) + api.GET("/:id/conversation/v2/task-sessions/:sessionId/revision", ctrl.conversationSourceRevision) api.GET("/:id/conversation/task-sessions/:sessionId/messages", ctrl.conversationMessages) api.GET("/:id/conversation/task-sessions/:sessionId/turns", ctrl.conversationTurns) api.POST("/:id/conversation/continuation/renew", ctrl.conversationContinuationRenew) } +func (c *Controller) conversationSourceMessages(ctx *gin.Context) { + ctx.Header("Cache-Control", conversationNoStore) + record, identity, ok := c.authorizeConversationRequest(ctx) + if !ok || !c.validBinding(ctx, record, identity.UserID) { + return + } + reader, ok := c.conversationReader.(conversationSourceReader) + if !ok { + writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) + return + } + query, expected, cursorID, ok := c.parseSourceRequest(ctx, record, identity.UserID, true) + if !ok { + return + } + sessionID := ctx.Param("sessionId") + page, err := reader.ReadConversationMessagesPage(ctx.Request.Context(), taskmodels.ConversationMessagePageRequest{ + SessionID: sessionID, + TaskID: query.taskID, + Authors: query.authors, + Sort: query.sort, + CursorID: cursorID, + Limit: query.limit, + }) + if !c.writeConversationSourceReadError(ctx, err) { + return + } + if expected != nil && page.Revision != *expected { + writeConversationError(ctx, http.StatusConflict, "reconciliation_required", "conversation revision changed", true) + return + } + response := conversationSourceMessagesResponse{ + Messages: make([]conversationMessageDTO, 0, len(page.Messages)), + HasMore: page.HasMore, + Epoch: c.svc.conversationEpoch, + Revision: page.Revision, + } + for _, message := range page.Messages { + response.Messages = append(response.Messages, conversationMessageModelToDTO(message)) + } + if page.HasMore { + cursor, mintErr := c.conversationTokens.mintSourceCursor( + record.ID, identity.UserID, conversationGeneration(record.InstalledAt), sessionID, + query.taskID, query.sort, query.authors, page.CursorID, c.svc.conversationEpoch, query.limit, + ) + if mintErr != nil { + writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) + return + } + response.Cursor = &cursor + } + ctx.JSON(http.StatusOK, response) +} + +func (c *Controller) conversationSourceTurns(ctx *gin.Context) { + ctx.Header("Cache-Control", conversationNoStore) + record, identity, ok := c.authorizeConversationRequest(ctx) + if !ok || !c.validBinding(ctx, record, identity.UserID) { + return + } + reader, ok := c.conversationReader.(conversationSourceReader) + if !ok { + writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) + return + } + query, expected, cursorID, ok := c.parseSourceRequest(ctx, record, identity.UserID, false) + if !ok { + return + } + sessionID := ctx.Param("sessionId") + page, err := reader.ReadConversationTurnsPage(ctx.Request.Context(), taskmodels.ConversationTurnPageRequest{ + SessionID: sessionID, + TaskID: query.taskID, + Sort: query.sort, + CursorID: cursorID, + Limit: query.limit, + }) + if !c.writeConversationSourceReadError(ctx, err) { + return + } + if expected != nil && page.Revision != *expected { + writeConversationError(ctx, http.StatusConflict, "reconciliation_required", "conversation revision changed", true) + return + } + response := conversationSourceTurnsResponse{ + Turns: make([]conversationTurnDTO, 0, len(page.Turns)), + HasMore: page.HasMore, + Epoch: c.svc.conversationEpoch, + Revision: page.Revision, + } + for _, turn := range page.Turns { + response.Turns = append(response.Turns, conversationTurnModelToDTO(turn)) + } + if page.HasMore { + cursor, mintErr := c.conversationTokens.mintSourceCursor( + record.ID, identity.UserID, conversationGeneration(record.InstalledAt), sessionID, + query.taskID, query.sort, nil, page.CursorID, c.svc.conversationEpoch, query.limit, + ) + if mintErr != nil { + writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) + return + } + response.Cursor = &cursor + } + ctx.JSON(http.StatusOK, response) +} + +func (c *Controller) conversationSourceRevision(ctx *gin.Context) { + ctx.Header("Cache-Control", conversationNoStore) + record, identity, ok := c.authorizeConversationRequest(ctx) + if !ok || !c.validBinding(ctx, record, identity.UserID) { + return + } + reader, ok := c.conversationReader.(conversationSourceReader) + if !ok { + writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) + return + } + revision, err := reader.ReadConversationRevision(ctx.Request.Context(), ctx.Param("sessionId")) + if !c.writeConversationSourceReadError(ctx, err) { + return + } + if !revision.Exists { + writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + return + } + ctx.JSON(http.StatusOK, conversationSourceRevisionResponse{Epoch: c.svc.conversationEpoch, Revision: revision.Revision}) +} + +func (c *Controller) parseSourceRequest(ctx *gin.Context, record *store.Record, userID string, allowAuthors bool) (conversationMessageQuery, *int64, string, bool) { + query, ok := parseConversationMessageQuery(ctx) + if !ok { + return conversationMessageQuery{}, nil, "", false + } + if !allowAuthors && len(query.authors) > 0 { + writeConversationError(ctx, http.StatusBadRequest, "invalid_query", "author_type is not supported for turns", false) + return conversationMessageQuery{}, nil, "", false + } + var expected *int64 + if raw, exists := ctx.GetQuery("expected_revision"); exists { + revision, err := strconv.ParseInt(raw, 10, 64) + if err != nil || revision < 0 { + writeConversationError(ctx, http.StatusBadRequest, "invalid_query", "expected_revision must be a nonnegative integer", false) + return conversationMessageQuery{}, nil, "", false + } + expected = &revision + } + sessionID := ctx.Param("sessionId") + if query.taskID != nil && c.conversationReader != nil && !c.validateSourceTaskQuery(ctx, sessionID, *query.taskID) { + return conversationMessageQuery{}, nil, "", false + } + if query.cursor == "" { + return query, expected, "", true + } + lastID, ok := c.parseSourceCursor(ctx, record, userID, query) + return query, expected, lastID, ok +} + +func (c *Controller) validateSourceTaskQuery(ctx *gin.Context, sessionID, taskID string) bool { + session, err := c.conversationReader.GetTaskSession(ctx.Request.Context(), sessionID) + if err != nil { + return c.writeConversationSourceReadError(ctx, err) + } + if session == nil || session.ID != sessionID { + writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + return false + } + if taskID != session.TaskID { + writeConversationError(ctx, http.StatusBadRequest, "invalid_query", "task_id does not match the task session", false) + return false + } + return true +} + +func (c *Controller) parseSourceCursor(ctx *gin.Context, record *store.Record, userID string, query conversationMessageQuery) (string, bool) { + sessionID := ctx.Param("sessionId") + claims, err := c.conversationTokens.parse(query.cursor) + if err != nil || claims.Kind != tokenKindCursor || claims.PluginID != record.ID || + claims.UserID != userID || claims.Generation != conversationGeneration(record.InstalledAt) || + claims.SessionID != sessionID || !equalOptionalString(claims.TaskID, query.taskID) || + claims.Sort != query.sort || !equalStrings(claims.Authors, query.authors) || + claims.Epoch != c.svc.conversationEpoch || claims.PageSize != query.limit || claims.LastID == "" { + writeConversationError(ctx, http.StatusBadRequest, "invalid_query", "invalid cursor", false) + return "", false + } + return claims.LastID, true +} + +func (c *Controller) writeConversationSourceReadError(ctx *gin.Context, err error) bool { + if err == nil { + return true + } + if errors.Is(err, taskmodels.ErrTaskSessionNotFound) { + writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + return false + } + if errors.Is(err, taskmodels.ErrConversationCursorStale) { + writeConversationError(ctx, http.StatusConflict, "reconciliation_required", "conversation cursor is stale", true) + return false + } + writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) + return false +} + func (c *Controller) conversationBinding(ctx *gin.Context) { ctx.Header("Cache-Control", conversationNoStore) identity, ok := authn.IdentityFromContext(ctx.Request.Context()) @@ -182,11 +417,14 @@ func (c *Controller) conversationMessages(ctx *gin.Context) { return } session, err := c.conversationReader.GetTaskSession(ctx.Request.Context(), sessionID) - if err != nil || session == nil || session.ID != sessionID { - if !c.svc.isSessionRemoved(sessionID) { - writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + if err != nil { + if !c.writeConversationSourceReadError(ctx, err) { return } + } + if session == nil || session.ID != sessionID { + writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + return } else if query.taskID != nil && *query.taskID != session.TaskID { writeConversationError(ctx, http.StatusBadRequest, "invalid_query", "task_id does not match the task session", false) return @@ -204,20 +442,7 @@ func (c *Controller) conversationMessages(ctx *gin.Context) { if !c.applyConversationMessageCursor(ctx, record, identity.UserID, sessionID, query, snapshot, &request) { return } - var messages []*taskmodels.Message - var hasMore bool - if c.svc.HasConversationJournal() { - cursorID := request.After - if query.sort == "desc" { - cursorID = request.Before - } - messages, hasMore, err = c.svc.conversationMessagesAt( - ctx.Request.Context(), sessionID, uint64(snapshot.Cutoff), - query.taskID, query.authors, query.sort, cursorID, query.limit, - ) - } else { - messages, hasMore, err = c.conversationReader.ListMessagesPaginated(ctx.Request.Context(), request) - } + messages, hasMore, err := c.conversationReader.ListMessagesPaginated(ctx.Request.Context(), request) if err != nil { writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) return @@ -339,7 +564,7 @@ func (c *Controller) conversationTurns(ctx *gin.Context) { return } sessionID := ctx.Param("sessionId") - snapshot, ok := c.validSnapshot(ctx, record, identity.UserID, sessionID) + _, ok = c.validSnapshot(ctx, record, identity.UserID, sessionID) if !ok { return } @@ -352,12 +577,15 @@ func (c *Controller) conversationTurns(ctx *gin.Context) { taskID = &rawTaskID } session, err := c.conversationReader.GetTaskSession(ctx.Request.Context(), sessionID) - sessionFound := err == nil && session != nil && session.ID == sessionID - if !sessionFound { - if !c.svc.isSessionRemoved(sessionID) { - writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + if err != nil { + if !c.writeConversationSourceReadError(ctx, err) { return } + } + sessionFound := session != nil && session.ID == sessionID + if !sessionFound { + writeConversationError(ctx, http.StatusNotFound, "not_found", "task session not found", false) + return } else if taskID != nil && *taskID != session.TaskID { writeConversationError(ctx, http.StatusBadRequest, "invalid_query", "task_id does not match the task session", false) return @@ -368,15 +596,10 @@ func (c *Controller) conversationTurns(ctx *gin.Context) { } else if sessionFound { scopedTaskID = &session.TaskID } - var turns []*taskmodels.Turn - if c.svc.HasConversationJournal() { - turns, err = c.svc.conversationTurnsAt(ctx.Request.Context(), sessionID, uint64(snapshot.Cutoff), scopedTaskID) - } else { - // The no-journal fallback must honor the requested/inherited task - // scope like the journal branch: a session that ever accumulated - // mixed-task turn rows must not leak turns outside the validated task. - turns, err = loadFallbackTurns(ctx.Request.Context(), c.conversationReader, sessionID, *scopedTaskID) - } + // The source repository remains authoritative. The task scope is already + // validated against the session above, so filter the source result before + // mapping it to the public DTO. + turns, err := loadFallbackTurns(ctx.Request.Context(), c.conversationReader, sessionID, optionalStringValue(scopedTaskID)) if err != nil { writeConversationError(ctx, http.StatusInternalServerError, "upstream_failure", "conversation service unavailable", true) return diff --git a/apps/backend/internal/plugins/conversation_handlers_test.go b/apps/backend/internal/plugins/conversation_handlers_test.go index 6930603684..a0884f1562 100644 --- a/apps/backend/internal/plugins/conversation_handlers_test.go +++ b/apps/backend/internal/plugins/conversation_handlers_test.go @@ -3,6 +3,7 @@ package plugins import ( "context" "encoding/json" + "errors" "github.com/gin-gonic/gin" "net/http" "testing" @@ -113,7 +114,6 @@ func TestConversationReadsRequireAuthenticationBeforeLookup(t *testing.T) { // @covers AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 func TestConversationReadsRejectJournalFallbackForUnauthorizedLiveSession(t *testing.T) { _, service := newTestRouter(t) - service.SetConversationJournalDB(newTestPool(t).Writer()) service.registry.Add(conversationPluginRecord("kandev-plugin-history", time.Now().UTC())) router := registerPluginRoutesWithIdentity( t, @@ -134,18 +134,47 @@ func TestConversationReadsRejectJournalFallbackForUnauthorizedLiveSession(t *tes } type fakeConversationReader struct { - session *taskmodels.TaskSession - messages []*taskmodels.Message - turns []*taskmodels.Turn - hasMore bool - request taskservice.ListMessagesRequest + session *taskmodels.TaskSession + sessionErr error + messages []*taskmodels.Message + turns []*taskmodels.Turn + hasMore bool + request taskservice.ListMessagesRequest + sourceMessages taskmodels.ConversationMessagePage + sourceTurns taskmodels.ConversationTurnPage + sourceRevision taskmodels.ConversationRevision + sourceMessageRequest taskmodels.ConversationMessagePageRequest + sourceTurnRequest taskmodels.ConversationTurnPageRequest } func (f *fakeConversationReader) GetTaskSession( _ context.Context, _ string, ) (*taskmodels.TaskSession, error) { - return f.session, nil + return f.session, f.sessionErr +} + +func TestConversationReadsClassifySessionLookupFailuresAsRetryable(t *testing.T) { + _, service := newTestRouter(t) + service.registry.Add(conversationPluginRecord("kandev-plugin-history", time.Now().UTC())) + router := registerPluginRoutesWithIdentity( + t, + service, + authn.Identity{UserID: "user_1", Role: authn.RoleMember}, + &fakeConversationReader{sessionErr: errors.New("database unavailable")}, + ) + headers := conversationReadHeaders(t, service, router, "kandev-plugin-history", "session-1") + paths := []string{ + "/api/plugins/kandev-plugin-history/conversation/task-sessions/session-1/messages", + "/api/plugins/kandev-plugin-history/conversation/task-sessions/session-1/turns", + "/api/plugins/kandev-plugin-history/conversation/v2/task-sessions/session-1/messages?task_id=task-1", + "/api/plugins/kandev-plugin-history/conversation/v2/task-sessions/session-1/turns?task_id=task-1", + } + for _, path := range paths { + response := doAuthedRequest(router, http.MethodGet, path, "", headers) + require.Equal(t, http.StatusInternalServerError, response.Code, path) + require.JSONEq(t, `{"error":{"code":"upstream_failure","message":"conversation service unavailable","retryable":true}}`, response.Body.String(), path) + } } func (f *fakeConversationReader) ListMessagesPaginated( @@ -163,6 +192,148 @@ func (f *fakeConversationReader) ListTurnsBySession( return f.turns, nil } +func (f *fakeConversationReader) ReadConversationRevision( + _ context.Context, + _ string, +) (taskmodels.ConversationRevision, error) { + return f.sourceRevision, nil +} + +func (f *fakeConversationReader) ReadConversationMessagesPage( + _ context.Context, + request taskmodels.ConversationMessagePageRequest, +) (taskmodels.ConversationMessagePage, error) { + f.sourceMessageRequest = request + return f.sourceMessages, nil +} + +func (f *fakeConversationReader) ReadConversationTurnsPage( + _ context.Context, + request taskmodels.ConversationTurnPageRequest, +) (taskmodels.ConversationTurnPage, error) { + f.sourceTurnRequest = request + return f.sourceTurns, nil +} + +func TestConversationSourceMessagesReturnRevisionBoundDTOs(t *testing.T) { + created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + reader := &fakeConversationReader{ + session: &taskmodels.TaskSession{ID: "session-source-http", TaskID: "task-source-http"}, + sourceMessages: taskmodels.ConversationMessagePage{ + Messages: []*taskmodels.Message{{ + ID: "message-source-http", + TaskSessionID: "session-source-http", + TaskID: "task-source-http", + AuthorType: taskmodels.MessageAuthorUser, + Content: "hiddenprompt", + CreatedAt: created, + UpdatedAt: created, + }}, + Revision: 7, + }, + } + _, service := newTestRouter(t) + service.registry.Add(conversationPluginRecord("kandev-plugin-history", created)) + router := registerPluginRoutesWithIdentity( + t, + service, + authn.Identity{UserID: "user_1", Role: authn.RoleMember}, + reader, + ) + headers := conversationReadHeaders(t, service, router, "kandev-plugin-history", "session-source-http") + + response := doAuthedRequest( + router, + http.MethodGet, + "/api/plugins/kandev-plugin-history/conversation/v2/task-sessions/session-source-http/messages?task_id=task-source-http&author_type=user&limit=25&expected_revision=7", + "", + headers, + ) + + require.Equal(t, http.StatusOK, response.Code, response.Body.String()) + require.JSONEq(t, `{"messages":[{"id":"message-source-http","taskId":"task-source-http","sessionId":"session-source-http","authorType":"user","type":"message","content":"prompt","createdAt":"2026-09-16T12:00:00Z","updatedAt":"2026-09-16T12:00:00Z"}],"hasMore":false,"cursor":null,"epoch":"`+service.conversationEpoch+`","revision":"7"}`, response.Body.String()) + require.Equal(t, "session-source-http", reader.sourceMessageRequest.SessionID) + require.Equal(t, []string{"user"}, reader.sourceMessageRequest.Authors) +} + +func TestConversationSourceReadsRequireMatchingRevision(t *testing.T) { + created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + reader := &fakeConversationReader{ + session: &taskmodels.TaskSession{ID: "session-source-conflict", TaskID: "task-source-conflict"}, + sourceMessages: taskmodels.ConversationMessagePage{Revision: 9}, + } + _, service := newTestRouter(t) + service.registry.Add(conversationPluginRecord("kandev-plugin-history", created)) + router := registerPluginRoutesWithIdentity( + t, + service, + authn.Identity{UserID: "user_1", Role: authn.RoleMember}, + reader, + ) + headers := conversationReadHeaders(t, service, router, "kandev-plugin-history", "session-source-conflict") + response := doAuthedRequest( + router, + http.MethodGet, + "/api/plugins/kandev-plugin-history/conversation/v2/task-sessions/session-source-conflict/messages?expected_revision=8", + "", + headers, + ) + + require.Equal(t, http.StatusConflict, response.Code) + require.JSONEq(t, `{"error":{"code":"reconciliation_required","message":"conversation revision changed","retryable":true}}`, response.Body.String()) +} + +func TestConversationSourceRevisionUsesDecimalString(t *testing.T) { + created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + reader := &fakeConversationReader{ + session: &taskmodels.TaskSession{ID: "session-source-revision-http", TaskID: "task-source-revision-http"}, + sourceRevision: taskmodels.ConversationRevision{SessionID: "session-source-revision-http", Exists: true, Revision: 11}, + } + _, service := newTestRouter(t) + service.registry.Add(conversationPluginRecord("kandev-plugin-history", created)) + router := registerPluginRoutesWithIdentity( + t, + service, + authn.Identity{UserID: "user_1", Role: authn.RoleMember}, + reader, + ) + headers := conversationReadHeaders(t, service, router, "kandev-plugin-history", "session-source-revision-http") + response := doAuthedRequest( + router, + http.MethodGet, + "/api/plugins/kandev-plugin-history/conversation/v2/task-sessions/session-source-revision-http/revision", + "", + headers, + ) + + require.Equal(t, http.StatusOK, response.Code) + require.JSONEq(t, `{"epoch":"`+service.conversationEpoch+`","revision":"11"}`, response.Body.String()) +} + +func TestConversationSourceTurnsRejectMessageOnlyAuthorFilter(t *testing.T) { + created := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + reader := &fakeConversationReader{session: &taskmodels.TaskSession{ID: "session-source-turn-http", TaskID: "task-source-turn-http"}} + _, service := newTestRouter(t) + service.registry.Add(conversationPluginRecord("kandev-plugin-history", created)) + router := registerPluginRoutesWithIdentity( + t, + service, + authn.Identity{UserID: "user_1", Role: authn.RoleMember}, + reader, + ) + headers := conversationReadHeaders(t, service, router, "kandev-plugin-history", "session-source-turn-http") + response := doAuthedRequest( + router, + http.MethodGet, + "/api/plugins/kandev-plugin-history/conversation/v2/task-sessions/session-source-turn-http/turns?author_type=user", + "", + headers, + ) + + require.Equal(t, http.StatusBadRequest, response.Code) + require.JSONEq(t, `{"error":{"code":"invalid_query","message":"author_type is not supported for turns","retryable":false}}`, response.Body.String()) +} + // @covers AC-PLUGINS-PROMPT-HISTORY-HOST-002.1 // @covers AC-PLUGINS-PROMPT-HISTORY-HOST-002.2 // @covers AC-PLUGINS-PROMPT-HISTORY-HOST-002.3 @@ -315,15 +486,16 @@ func TestConversationStreamGrantFirstPageRenewsQueryBoundCursor(t *testing.T) { reader, ) binding := conversationBindingToken(t, router, "kandev-plugin-history") - snapshot, _, _, err := service.MintSessionStreamGrant( + snapshot, err := service.conversationTokens.mintSnapshot( "kandev-plugin-history", "user_1", conversationGeneration(installedAt), "session-1", - "consumer-1", + nil, "", + nil, 0, - 0, + "consumer-1", ) require.NoError(t, err) headers := map[string]string{ @@ -573,15 +745,16 @@ func conversationReadHeaders( binding := conversationBindingToken(t, router, pluginID) record, err := service.Get(pluginID) require.NoError(t, err) - snapshot, _, _, err := service.MintSessionStreamGrant( + snapshot, err := service.conversationTokens.mintSnapshot( pluginID, "user_1", conversationGeneration(record.InstalledAt), sessionID, - "test-consumer", + nil, "", - service.SessionEvents().Watermark(sessionID), - service.SessionEvents().Watermark(sessionID), + nil, + 0, + "test-consumer", ) require.NoError(t, err) return map[string]string{ @@ -590,6 +763,8 @@ func conversationReadHeaders( } } +func stringPtr(value string) *string { return &value } + func TestFilterTurnsByTaskNarrowsMixedTaskSession(t *testing.T) { turns := []*taskmodels.Turn{ {ID: "turn-a", TaskID: "task-session"}, diff --git a/apps/backend/internal/plugins/conversation_journal.go b/apps/backend/internal/plugins/conversation_journal.go deleted file mode 100644 index b26c3c9dcb..0000000000 --- a/apps/backend/internal/plugins/conversation_journal.go +++ /dev/null @@ -1,721 +0,0 @@ -//nolint:revive // The journal projection and sanitization contract is intentionally co-located. -package plugins - -import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "time" - - "github.com/jmoiron/sqlx" - "github.com/kandev/kandev/internal/db" - "github.com/kandev/kandev/internal/events" - "github.com/kandev/kandev/internal/sysprompt" - taskmodels "github.com/kandev/kandev/internal/task/models" - "go.uber.org/zap" -) - -// errConversationCursorGone reports a page cursor whose message was -// deleted/tombstoned before the walk reached it; the page would otherwise -// silently truncate. -var errConversationCursorGone = errors.New("conversation page cursor message no longer exists") - -const ( - conversationContentKey = "content" - conversationTypeKey = "type" - conversationSessionIDKey = "session_id" - conversationTaskIDKey = "task_id" - conversationMessageIDKey = "message_id" - conversationAuthorTypeKey = "author_type" - conversationTurnCompleted = "session.turn.completed" - conversationTurnRemoved = "session.turn.removed" - conversationTurnStarted = "session.turn.started" -) - -//nolint:goconst // These keys are an explicit privacy allowlist. -var conversationMessageMetadataKeys = map[string]struct{}{ - "action_details": {}, "action_type": {}, "action_visibility": {}, - "actions": {}, "agent_disconnected": {}, - "attempt": {}, "attachments": {}, "auth_methods": {}, "auto_start": {}, - "base_branch": {}, "context": {}, "context_files": {}, "decision_id": {}, - "effective_model": {}, "entity_references": {}, "error_output": {}, - "failure_code": {}, "failure_details": {}, "failure_kind": {}, - "fallback_model": {}, "has_hidden_prompts": {}, "has_resume_token": {}, - "has_review_comments": {}, "is_auth_error": {}, "kind": {}, "max_attempts": {}, - "message": {}, "missing_branch": {}, "model_id": {}, "new_branch": {}, - "original_branch": {}, "options": {}, "pending_id": {}, "plan_mode": {}, "progress": {}, - "provider_name": {}, "question": {}, "question_id": {}, "question_index": {}, - "question_total": {}, "recovery_actions": {}, "remediation": {}, - "remediation_url": {}, "requested_model": {}, "request_id": {}, - "requests_input": {}, "response": {}, "reset_at": {}, "retry_at": {}, - "retry_in_seconds": {}, - "retrying": {}, "sender_session_id": {}, "sender_session_name": {}, - "sender_task_id": {}, "sender_task_title": {}, "stage": {}, "status": {}, - "script_type": {}, "agent_name": {}, "command": {}, "exit_code": {}, - "is_resuming": {}, "started_at": {}, "completed_at": {}, "error": {}, - "task_id": {}, "text": {}, "tool_call_id": {}, "variant": {}, "workflow_message": {}, - "workflow_step_color": {}, "workflow_step_id": {}, "workflow_step_name": {}, -} - -// SetConversationJournalDB configures the primary database that owns source -// mutations and their immutable conversation journal rows. -func (s *Service) SetConversationJournalDB(database *sqlx.DB) { - s.conversationJournal = database -} - -func (s *Service) HasConversationJournal() bool { - return s.conversationJournal != nil -} - -func (s *Service) conversationJournalTablesExist(ctx context.Context) (bool, error) { - return s.conversationTableExists(ctx, "conversation_message_versions") -} - -func toAnySlice(values []string) []any { - out := make([]any, len(values)) - for index, value := range values { - out[index] = value - } - return out -} - -func (s *Service) conversationTableExists(ctx context.Context, table string) (bool, error) { - present, err := db.TableExistsContext(ctx, s.conversationJournal, table) - if err != nil { - return false, fmt.Errorf("check conversation journal schema (%s): %w", table, err) - } - return present, nil -} - -// SyncCommittedSessionEvents mirrors primary-journal rows into the durable -// delivery log without allocating a second identity or sequence. -func (s *Service) SyncCommittedSessionEvents(ctx context.Context, sessionID string) ([]SessionEvent, error) { - if s.conversationJournal == nil || s.sessionEvents == nil { - return nil, nil - } - watermark := s.sessionEvents.Watermark(sessionID) - query := s.conversationJournal.Rebind(` - SELECT session_id, event_id, sequence, protocol_version, event_type, task_id, payload, created_at - FROM conversation_session_events - WHERE session_id = ? AND sequence > ? - ORDER BY sequence ASC`) - rows, err := s.conversationJournal.QueryxContext(ctx, query, sessionID, watermark) - if err != nil { - return nil, fmt.Errorf("query committed conversation events: %w", err) - } - defer func() { _ = rows.Close() }() - - mirrored := make([]SessionEvent, 0) - expected := watermark - for rows.Next() { - var event SessionEvent - var taskID sql.NullString - var payload string - if err := rows.Scan(&event.SessionID, &event.ID, &event.Sequence, &event.ProtocolVersion, &event.EventType, &taskID, &payload, &event.CreatedAt); err != nil { - return nil, fmt.Errorf("scan committed conversation event: %w", err) - } - event.Payload = sanitizeConversationEventPayload(event.EventType, json.RawMessage(payload)) - if taskID.Valid { - event.TaskID = &taskID.String - } - appended, err := s.sessionEvents.AppendCommitted(event) - if err != nil { - return nil, fmt.Errorf("mirror committed conversation event: %w", err) - } - if !appended { - continue - } - mirrored = append(mirrored, event) - if event.Sequence > expected+1 && s.log != nil { - s.log.Warn("journal mirror healed a retention gap", - zap.String("session_id", sessionID), - zap.Uint64("expected_sequence", expected+1), - zap.Uint64("sequence", event.Sequence), - ) - } - expected = event.Sequence - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate committed conversation events: %w", err) - } - return mirrored, nil -} - -// emptyConversationPayload is the durable payload persisted for a -// primary-journal row whose JSON is malformed. It carries no raw bytes yet is -// non-nil, so the BLOB NOT NULL session_events column accepts the write and -// the mirrored row is retained as a poison record instead of failing the -// append (the projection rejects it because no type/identity fields exist). -var emptyConversationPayload = json.RawMessage("{}") - -func sanitizeConversationEventPayload(eventType string, raw json.RawMessage) json.RawMessage { - var source map[string]any - if err := json.Unmarshal(raw, &source); err != nil { - return emptyConversationPayload - } - payload := map[string]any{conversationTypeKey: eventType} - copyConversationPayloadFields(payload, source, conversationSessionIDKey, conversationTaskIDKey) - switch eventType { - case events.MessageAdded, events.MessageUpdated: - copyConversationPayloadFields(payload, source, - conversationMessageIDKey, "turn_id", conversationAuthorTypeKey, - "created_at", "updated_at", "prompt_index", "requests_input", - "message_type") - sanitizeConversationMessagePayload(payload, source) - case events.MessageDeleted: - copyConversationPayloadFields(payload, source, conversationMessageIDKey) - case conversationTurnStarted, conversationTurnCompleted, conversationTurnRemoved: - copyConversationPayloadFields(payload, source, "id", "started_at", "completed_at", "updated_at", "had_output", "metadata") - case events.SessionRemoved: - // The common identity fields above are the complete public removal DTO. - } - sanitized, err := json.Marshal(payload) - if err != nil { - return nil - } - return sanitized -} - -func copyConversationPayloadFields(target, source map[string]any, keys ...string) { - for _, key := range keys { - if value, exists := source[key]; exists { - target[key] = value - } - } -} - -// SanitizeConversationMessageMetadata returns the presentation metadata that -// the web client needs for a conversation message. Internal agent/tool -// metadata is deliberately excluded from ordered session events. -func SanitizeConversationMessageMetadata(source map[string]any) map[string]any { - if len(source) == 0 { - return nil - } - target := make(map[string]any) - for key := range conversationMessageMetadataKeys { - value, exists := source[key] - if !exists || value == nil { - continue - } - target[key] = sanitizeConversationMetadataValue(value) - } - return target -} - -func sanitizeConversationMetadataValue(value any) any { - switch typed := value.(type) { - case string: - return sysprompt.StripSystemContent(typed) - case map[string]any: - target := make(map[string]any, len(typed)) - for key, nested := range typed { - target[key] = sanitizeConversationMetadataValue(nested) - } - return target - case []any: - target := make([]any, len(typed)) - for index, nested := range typed { - target[index] = sanitizeConversationMetadataValue(nested) - } - return target - default: - return value - } -} - -func sanitizeConversationMessagePayload(target, source map[string]any) { - if content, ok := source[conversationContentKey].(string); ok { - target[conversationContentKey] = sysprompt.StripSystemContent(content) - } - metadata, _ := source["metadata"].(map[string]any) - sanitizedMetadata := SanitizeConversationMessageMetadata(metadata) - if senderTaskID, ok := source["sender_task_id"].(string); ok && senderTaskID != "" { - target["sender_task_id"] = senderTaskID - if sanitizedMetadata == nil { - sanitizedMetadata = map[string]any{} - } - sanitizedMetadata["sender_task_id"] = senderTaskID - } - if len(sanitizedMetadata) > 0 { - target["metadata"] = sanitizedMetadata - if senderTaskID, ok := sanitizedMetadata["sender_task_id"].(string); ok && senderTaskID != "" { - target["sender_task_id"] = senderTaskID - } - } -} - -func (s *Service) syncAllCommittedSessionEvents(ctx context.Context) ([]SessionEvent, error) { - if s.conversationJournal == nil || s.sessionEvents == nil { - return nil, nil - } - rows, err := s.conversationJournal.QueryxContext(ctx, `SELECT session_id FROM conversation_session_streams ORDER BY session_id`) - if err != nil { - return nil, fmt.Errorf("query conversation stream partitions: %w", err) - } - defer func() { _ = rows.Close() }() - var sessionIDs []string - for rows.Next() { - var sessionID string - if err := rows.Scan(&sessionID); err != nil { - return nil, fmt.Errorf("scan conversation stream partition: %w", err) - } - sessionIDs = append(sessionIDs, sessionID) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate conversation stream partitions: %w", err) - } - var mirrored []SessionEvent - var failures []string - for _, sessionID := range sessionIDs { - events, err := s.SyncCommittedSessionEvents(ctx, sessionID) - if err != nil { - // Isolate one unhealthy partition: it must not freeze retention - // for every other session. The combined error is still returned - // so the failure stays loud (it is only reachable through state - // tampering, since the maintenance tick prunes both sides). - failures = append(failures, sessionID+": "+err.Error()) - continue - } - mirrored = append(mirrored, events...) - } - if len(failures) > 0 { - return mirrored, fmt.Errorf("mirror %d session partition(s): %s", len(failures), strings.Join(failures, "; ")) - } - return mirrored, nil -} - -func (s *Service) SetSessionEventSink(sink func(SessionEvent)) { - s.mu.Lock() - defer s.mu.Unlock() - - s.sessionEventSink = sink -} - -//nolint:cyclop // Retention orchestration: schema gates plus four prune passes over three tables. -func (s *Service) pruneConversationJournal( - ctx context.Context, - cutoff time.Time, - retained map[string]struct{}, -) error { - if s.conversationJournal == nil { - return nil - } - exists, err := s.conversationJournalTablesExist(ctx) - if err != nil { - return err - } - if !exists { - return nil - } - purgeable, err := s.deadSessionPurgeEnabled(ctx) - if err != nil { - return err - } - ids := make([]string, 0, len(retained)) - for sessionID := range retained { - ids = append(ids, sessionID) - } - sort.Strings(ids) - exclusion, extraArgs := retainedSessionExclusion(ids) - args := append([]any{cutoff.UTC()}, extraArgs...) - tx, err := s.conversationJournal.BeginTxx(ctx, nil) - if err != nil { - return fmt.Errorf("begin conversation journal retention: %w", err) - } - rollback := func(cause error) error { - _ = tx.Rollback() - return cause - } - eventQuery := s.conversationJournal.Rebind( - `DELETE FROM conversation_session_events - WHERE created_at < ? AND event_type <> 'session.removed' AND ` + exclusion, - ) - if _, err := tx.ExecContext(ctx, eventQuery, args...); err != nil { - return rollback(fmt.Errorf("prune conversation events: %w", err)) - } - messageQuery := s.conversationJournal.Rebind( - `DELETE FROM conversation_message_versions AS current_version - WHERE current_version.created_at < ? AND ` + exclusion + ` - AND EXISTS ( - SELECT 1 FROM conversation_message_versions AS newer - WHERE newer.session_id = current_version.session_id - AND newer.message_id = current_version.message_id - AND newer.row_sequence > current_version.row_sequence - )`, - ) - if _, err := tx.ExecContext(ctx, messageQuery, args...); err != nil { - return rollback(fmt.Errorf("prune conversation message versions: %w", err)) - } - turnQuery := s.conversationJournal.Rebind( - `DELETE FROM conversation_turn_versions AS current_version - WHERE current_version.started_at < ? AND ` + exclusion + ` - AND EXISTS ( - SELECT 1 FROM conversation_turn_versions AS newer - WHERE newer.session_id = current_version.session_id - AND newer.turn_id = current_version.turn_id - AND newer.row_sequence > current_version.row_sequence - )`, - ) - if _, err := tx.ExecContext(ctx, turnQuery, args...); err != nil { - return rollback(fmt.Errorf("prune conversation turn versions: %w", err)) - } - if purgeable { - if err := s.purgeDeadSessionPartitions(ctx, tx, exclusion, args); err != nil { - return rollback(err) - } - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit conversation journal retention: %w", err) - } - return nil -} - -// deadSessionPurgeEnabled reports whether the retention pass can purge whole -// dead-session partitions: both the journal stream table and the source -// session table must be present. Checked before the retention tx opens so a -// single-connection pool (in-memory test DBs) cannot deadlock on the catalog -// lookups. -func (s *Service) deadSessionPurgeEnabled(ctx context.Context) (bool, error) { - streams, err := s.conversationTableExists(ctx, "conversation_session_streams") - if err != nil { - return false, err - } - if !streams { - return false, nil - } - return s.conversationTableExists(ctx, "task_sessions") -} - -// retainedSessionExclusion builds the SQL exclusion clause and its bound -// arguments for the retained session ids; with none retained it returns the -// no-op "1 = 1" clause so every row is eligible for age-based pruning. -func retainedSessionExclusion(ids []string) (string, []any) { - if len(ids) == 0 { - return "1 = 1", nil - } - placeholders := make([]string, len(ids)) - extra := make([]any, len(ids)) - for index, sessionID := range ids { - placeholders[index] = "?" - extra[index] = sessionID - } - return "session_id NOT IN (" + strings.Join(placeholders, ",") + ")", extra -} - -// purgeDeadSessionPartitions deletes every journal row (message and turn -// versions, events, and the stream row) for sessions whose source task_sessions -// row no longer exists and whose partition is older than the retention cutoff -// (args[0]) and not protected by a live cursor/poison (remaining args). The -// dead ids are materialized first because a CTE whose DELETE targets the same -// table the CTE reads can stall SQLite. -// -// The session.removed terminal marker intentionally lives exactly as long as -// the rest of the partition: this pass fires only after updated_at has aged -// past SessionEventRetention, so a subscriber arriving within that window -// still receives session_removed (the durable-log mirror keeps the terminal -// tombstone through the same window and only then drops it). After both -// stores forget the partition, the session is gone for good and later -// snapshot reads correctly 404. -func (s *Service) purgeDeadSessionPartitions(ctx context.Context, tx *sqlx.Tx, exclusion string, args []any) error { - deadQuery := s.conversationJournal.Rebind(` - SELECT st.session_id - FROM conversation_session_streams st - WHERE st.updated_at < ? AND ` + exclusion + ` - AND NOT EXISTS ( - SELECT 1 FROM task_sessions ts WHERE ts.id = st.session_id - )`) - var deadIDs []string - if err := tx.SelectContext(ctx, &deadIDs, deadQuery, args...); err != nil { - return fmt.Errorf("select dead session journal partitions: %w", err) - } - if len(deadIDs) == 0 { - return nil - } - placeholders := make([]string, len(deadIDs)) - for index := range deadIDs { - placeholders[index] = "?" - } - inClause := strings.Join(placeholders, ",") - for _, child := range []string{"conversation_message_versions", "conversation_turn_versions", "conversation_session_events"} { - query := s.conversationJournal.Rebind( - `DELETE FROM ` + child + ` WHERE session_id IN (` + inClause + `)`) - if _, err := tx.ExecContext(ctx, query, toAnySlice(deadIDs)...); err != nil { - return fmt.Errorf("purge dead session journal rows (%s): %w", child, err) - } - } - streamQuery := s.conversationJournal.Rebind( - `DELETE FROM conversation_session_streams WHERE session_id IN (` + inClause + `)`) - if _, err := tx.ExecContext(ctx, streamQuery, toAnySlice(deadIDs)...); err != nil { - return fmt.Errorf("purge dead session stream rows: %w", err) - } - return nil -} - -type journalMessagePayload struct { - MessageID string `json:"message_id"` - TurnID string `json:"turn_id"` - TaskID string `json:"task_id"` - AuthorType string `json:"author_type"` - Content string `json:"content"` - MessageType string `json:"message_type"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - PromptIndex int `json:"prompt_index"` - RequestsInput any `json:"requests_input"` - SenderTaskID string `json:"sender_task_id"` - Metadata map[string]any `json:"metadata"` -} - -//nolint:goconst // SQL projection values are protocol literals. -func (s *Service) conversationMessagesAt( - ctx context.Context, - sessionID string, - cutoff uint64, - taskID *string, - authors []string, - sortOrder string, - cursorID string, - limit int, -) ([]*taskmodels.Message, bool, error) { - if s.conversationJournal == nil { - return nil, false, nil - } - args := []any{sessionID, cutoff} - where := []string{"version_rank = 1", "tombstone = FALSE"} - if taskID != nil { - where = append(where, "task_id = ?") - args = append(args, *taskID) - } - if len(authors) > 0 { - placeholders := make([]string, len(authors)) - for index, author := range authors { - placeholders[index] = "?" - args = append(args, author) - } - where = append(where, "author_type IN ("+strings.Join(placeholders, ",")+")") - } - orderDirection := "ASC" - comparison := ">" - if sortOrder == "desc" { - orderDirection = "DESC" - comparison = "<" - } - if cursorID != "" { - // A cursor whose message was deleted/tombstoned mid-walk is gone from - // the live projection; the cursor predicate would otherwise match - // nothing and the client would read a silent empty terminal page. - // Surface an error (parity with the source-table fallback) instead. - live, err := s.liveConversationMessageAtCutoff(ctx, sessionID, cutoff, cursorID) - if err != nil { - return nil, false, err - } - if !live { - return nil, false, fmt.Errorf("%w: %s", errConversationCursorGone, cursorID) - } - where = append(where, fmt.Sprintf("(created_at %s (SELECT created_at FROM live WHERE message_id = ?) OR (created_at = (SELECT created_at FROM live WHERE message_id = ?) AND message_id %s ?))", comparison, comparison)) - args = append(args, cursorID, cursorID, cursorID) - } - args = append(args, limit+1) - query := fmt.Sprintf(` - WITH ranked AS ( - SELECT message_id, task_id, author_type, created_at, tombstone, payload, - ROW_NUMBER() OVER (PARTITION BY message_id ORDER BY row_sequence DESC) AS version_rank - FROM conversation_message_versions - WHERE session_id = ? AND row_sequence <= ? - ), live AS ( - SELECT * FROM ranked WHERE version_rank = 1 AND tombstone = FALSE - ) - SELECT payload FROM live - WHERE %s - ORDER BY created_at %s, message_id %s - LIMIT ?`, strings.Join(where, " AND "), orderDirection, orderDirection) - query = s.conversationJournal.Rebind(query) - rows, err := s.conversationJournal.QueryxContext(ctx, query, args...) - if err != nil { - return nil, false, fmt.Errorf("query conversation message snapshot: %w", err) - } - defer func() { _ = rows.Close() }() - return decodeConversationSnapshotRows(rows, sessionID, limit) -} - -type conversationRowScanner interface { - Next() bool - Scan(dest ...any) error - Err() error -} - -// decodeConversationSnapshotRows scans message-version payload rows into -// messages, applying the limit+1 page bound, and reports whether another page -// remains. -func decodeConversationSnapshotRows(rows conversationRowScanner, sessionID string, limit int) ([]*taskmodels.Message, bool, error) { - messages := make([]*taskmodels.Message, 0, limit+1) - for rows.Next() { - var raw []byte - if err := rows.Scan(&raw); err != nil { - return nil, false, fmt.Errorf("scan conversation message snapshot: %w", err) - } - message, err := decodeJournalMessage(raw, sessionID) - if err != nil { - return nil, false, err - } - messages = append(messages, message) - } - if err := rows.Err(); err != nil { - return nil, false, fmt.Errorf("iterate conversation message snapshot: %w", err) - } - hasMore := len(messages) > limit - if hasMore { - messages = messages[:limit] - } - return messages, hasMore, nil -} - -// liveConversationMessageAtCutoff reports whether the message cursor still has -// a live (non-tombstoned) version at or before the snapshot cutoff. -func (s *Service) liveConversationMessageAtCutoff(ctx context.Context, sessionID string, cutoff uint64, cursorID string) (bool, error) { - query := s.conversationJournal.Rebind(` - WITH ranked AS ( - SELECT message_id, tombstone, - ROW_NUMBER() OVER (PARTITION BY message_id ORDER BY row_sequence DESC) AS version_rank - FROM conversation_message_versions - WHERE session_id = ? AND row_sequence <= ? - ) - SELECT COUNT(*) FROM ranked WHERE message_id = ? AND version_rank = 1 AND tombstone = FALSE`) - var live int - if err := s.conversationJournal.GetContext(ctx, &live, query, sessionID, cutoff, cursorID); err != nil { - return false, fmt.Errorf("resolve conversation page cursor: %w", err) - } - return live > 0, nil -} - -func conversationRequestsInput(value any) bool { - switch typed := value.(type) { - case bool: - return typed - case float64: - return typed != 0 - default: - return false - } -} - -func decodeJournalMessage(raw []byte, sessionID string) (*taskmodels.Message, error) { - var payload journalMessagePayload - if err := json.Unmarshal(raw, &payload); err != nil { - return nil, fmt.Errorf("decode conversation message version: %w", err) - } - createdAt, err := parseJournalTime(payload.CreatedAt) - if err != nil { - return nil, fmt.Errorf("decode conversation message created_at: %w", err) - } - updatedAt, err := parseJournalTime(payload.UpdatedAt) - if err != nil { - return nil, fmt.Errorf("decode conversation message updated_at: %w", err) - } - metadata := payload.Metadata - if metadata == nil { - metadata = map[string]any{} - } - if payload.SenderTaskID != "" { - metadata["sender_task_id"] = payload.SenderTaskID - } - return &taskmodels.Message{ - ID: payload.MessageID, TaskSessionID: sessionID, TurnID: payload.TurnID, TaskID: payload.TaskID, - AuthorType: taskmodels.MessageAuthorType(payload.AuthorType), - Content: payload.Content, Type: taskmodels.MessageType(payload.MessageType), - RequestsInput: conversationRequestsInput(payload.RequestsInput), - CreatedAt: createdAt, UpdatedAt: updatedAt, PromptIndex: payload.PromptIndex, Metadata: metadata, - }, nil -} - -type journalTurnPayload struct { - TurnID string `json:"id"` - TaskID string `json:"task_id"` - StartedAt string `json:"started_at"` - CompletedAt *string `json:"completed_at"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - Metadata map[string]any `json:"metadata"` -} - -func (s *Service) conversationTurnsAt(ctx context.Context, sessionID string, cutoff uint64, taskID *string) ([]*taskmodels.Turn, error) { - if s.conversationJournal == nil { - return nil, nil - } - args := []any{sessionID, cutoff} - where := "version_rank = 1 AND tombstone = FALSE" - if taskID != nil { - where += " AND task_id = ?" - args = append(args, *taskID) - } - query := fmt.Sprintf(` - WITH ranked AS ( - SELECT turn_id, task_id, started_at, tombstone, payload, - ROW_NUMBER() OVER (PARTITION BY turn_id ORDER BY row_sequence DESC) AS version_rank - FROM conversation_turn_versions - WHERE session_id = ? AND row_sequence <= ? - ) - SELECT payload FROM ranked WHERE %s ORDER BY started_at ASC, turn_id ASC`, where) - query = s.conversationJournal.Rebind(query) - rows, err := s.conversationJournal.QueryxContext(ctx, query, args...) - if err != nil { - return nil, fmt.Errorf("query conversation turn snapshot: %w", err) - } - defer func() { _ = rows.Close() }() - var turns []*taskmodels.Turn - for rows.Next() { - var raw []byte - if err := rows.Scan(&raw); err != nil { - return nil, fmt.Errorf("scan conversation turn snapshot: %w", err) - } - var payload journalTurnPayload - if err := json.Unmarshal(raw, &payload); err != nil { - return nil, fmt.Errorf("decode conversation turn version: %w", err) - } - startedAt, err := parseJournalTime(payload.StartedAt) - if err != nil { - return nil, fmt.Errorf("decode conversation turn started_at: %w", err) - } - createdAt, err := parseJournalTime(payload.CreatedAt) - if err != nil { - return nil, fmt.Errorf("decode conversation turn created_at: %w", err) - } - updatedAt, err := parseJournalTime(payload.UpdatedAt) - if err != nil { - return nil, fmt.Errorf("decode conversation turn updated_at: %w", err) - } - metadata := payload.Metadata - if metadata == nil { - metadata = map[string]any{} - } - turn := &taskmodels.Turn{ - ID: payload.TurnID, TaskSessionID: sessionID, TaskID: payload.TaskID, - StartedAt: startedAt, Metadata: metadata, CreatedAt: createdAt, UpdatedAt: updatedAt, - } - if payload.CompletedAt != nil { - completedAt, err := parseJournalTime(*payload.CompletedAt) - if err != nil { - return nil, fmt.Errorf("decode conversation turn completed_at: %w", err) - } - turn.CompletedAt = &completedAt - } - turns = append(turns, turn) - } - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterate conversation turn snapshot: %w", err) - } - return turns, nil -} - -func parseJournalTime(value string) (time.Time, error) { - for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05.999999999Z07:00", "2006-01-02 15:04:05Z07:00"} { - if parsed, err := time.Parse(layout, value); err == nil { - return parsed, nil - } - } - return time.Time{}, fmt.Errorf("unsupported timestamp %q", value) -} diff --git a/apps/backend/internal/plugins/conversation_journal_postgres_test.go b/apps/backend/internal/plugins/conversation_journal_postgres_test.go deleted file mode 100644 index b973cffbeb..0000000000 --- a/apps/backend/internal/plugins/conversation_journal_postgres_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package plugins - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/jmoiron/sqlx" - "github.com/kandev/kandev/internal/task/models" - tasksqlite "github.com/kandev/kandev/internal/task/repository/sqlite" - "github.com/kandev/kandev/internal/testutil" - "github.com/stretchr/testify/require" -) - -// TestPostgresConversationJournalProjectsRFC3339Times covers the real -// PostgreSQL trigger output at both consumers: immutable snapshot decoding and -// ordered ProjectSessionEvent validation. It skips when KANDEV_TEST_POSTGRES_DSN -// is unavailable. -func TestPostgresConversationJournalProjectsRFC3339Times(t *testing.T) { - database := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) - repository, err := tasksqlite.NewWithDB(database, database, nil) - require.NoError(t, err) - _, err = database.Exec(`SET TIME ZONE 'America/New_York'`) - require.NoError(t, err) - - const ( - taskID = "task-pg-journal-projection" - sessionID = "session-pg-journal-projection" - turnID = "turn-pg-journal-projection" - ) - startedAt := time.Date(2026, 9, 7, 12, 0, 0, 123456000, time.UTC) - insertPostgresConversationTurn(t, database, taskID, sessionID, turnID, startedAt) - messageAt := time.Date(2026, 9, 7, 12, 1, 0, 654321000, time.UTC) - require.NoError(t, repository.CreateMessage(context.Background(), &models.Message{ - ID: "message-pg-journal-projection", - TaskSessionID: sessionID, - TaskID: taskID, - TurnID: turnID, - AuthorType: models.MessageAuthorAgent, - Type: models.MessageTypeMessage, - Content: "visible", - CreatedAt: messageAt, - UpdatedAt: messageAt, - })) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - var cutoff uint64 - require.NoError(t, database.Get(&cutoff, - `SELECT watermark FROM conversation_session_streams WHERE session_id = $1`, sessionID)) - - turns, err := service.conversationTurnsAt(context.Background(), sessionID, cutoff, nil) - require.NoError(t, err) - require.Len(t, turns, 1) - require.True(t, turns[0].StartedAt.Equal(startedAt), "started_at = %s", turns[0].StartedAt) - require.True(t, turns[0].UpdatedAt.Equal(startedAt), "updated_at = %s", turns[0].UpdatedAt) - messages, more, err := service.conversationMessagesAt( - context.Background(), sessionID, cutoff, nil, nil, "asc", "", 20, - ) - require.NoError(t, err) - require.False(t, more) - require.Len(t, messages, 1) - require.True(t, messages[0].CreatedAt.Equal(messageAt), "created_at = %s", messages[0].CreatedAt) - require.True(t, messages[0].UpdatedAt.Equal(messageAt), "updated_at = %s", messages[0].UpdatedAt) - - events, err := service.SyncCommittedSessionEvents(context.Background(), sessionID) - require.NoError(t, err) - require.Len(t, events, 2) - projectedTimes := make(map[string]map[string]any, len(events)) - for _, event := range events { - projected, projectionErr := ProjectSessionEvent(event) - require.NoError(t, projectionErr) - var payload map[string]any - require.NoError(t, json.Unmarshal(projected, &payload)) - projectedTimes[event.EventType] = payload - } - require.Equal( - t, - startedAt.Format(time.RFC3339Nano), - projectedTimes["session.turn.started"]["started_at"], - ) - require.Equal( - t, - startedAt.Format(time.RFC3339Nano), - projectedTimes["session.turn.started"]["updated_at"], - ) - require.Equal( - t, - messageAt.Format(time.RFC3339Nano), - projectedTimes["message.added"]["created_at"], - ) - require.Equal( - t, - messageAt.Format(time.RFC3339Nano), - projectedTimes["message.added"]["updated_at"], - ) -} -func insertPostgresConversationTurn( - t *testing.T, - database *sqlx.DB, - taskID string, - sessionID string, - turnID string, - startedAt time.Time, -) { - t.Helper() - _, err := database.Exec( - `INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) - VALUES ($1, '', 'test task', $2, $2)`, - taskID, - startedAt, - ) - require.NoError(t, err) - _, err = database.Exec( - `INSERT INTO task_sessions (id, task_id, started_at, updated_at) - VALUES ($1, $2, $3, $3)`, - sessionID, - taskID, - startedAt, - ) - require.NoError(t, err) - _, err = database.Exec( - `INSERT INTO task_session_turns ( - id, task_session_id, task_id, started_at, created_at, updated_at - ) VALUES ($1, $2, $3, $4, $4, $4)`, - turnID, - sessionID, - taskID, - startedAt, - ) - require.NoError(t, err) -} diff --git a/apps/backend/internal/plugins/conversation_journal_test.go b/apps/backend/internal/plugins/conversation_journal_test.go deleted file mode 100644 index a984d1dd3b..0000000000 --- a/apps/backend/internal/plugins/conversation_journal_test.go +++ /dev/null @@ -1,484 +0,0 @@ -package plugins - -import ( - "context" - "encoding/json" - "path/filepath" - "testing" - "time" - - "github.com/jmoiron/sqlx" - "github.com/kandev/kandev/internal/events" - _ "github.com/mattn/go-sqlite3" - "github.com/stretchr/testify/require" -) - -func TestConversationMessagesAtUsesImmutableCutoff(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_message_versions ( - session_id TEXT NOT NULL, message_id TEXT NOT NULL, row_sequence INTEGER NOT NULL, - task_id TEXT, author_type TEXT, created_at TIMESTAMP NOT NULL, - tombstone BOOLEAN NOT NULL, payload TEXT NOT NULL - );`) - require.NoError(t, err) - - createdAt := "2026-09-07T12:00:00Z" - insertJournalMessageVersion(t, database, 1, false, map[string]any{ - "message_id": "message-1", "turn_id": "turn-1", "task_id": "task-1", - "author_type": "agent", "author_id": "agent-1", "content": "before", - "requests_input": 0, "message_type": "message", "created_at": createdAt, - "updated_at": createdAt, "prompt_index": 0, "metadata_json": `{}`, - }) - insertJournalMessageVersion(t, database, 2, false, map[string]any{ - "message_id": "message-1", "turn_id": "turn-1", "task_id": "task-1", - "author_type": "agent", "author_id": "agent-1", "content": "after", - "requests_input": 0, "message_type": "message", "created_at": createdAt, - "updated_at": "2026-09-07T12:01:00Z", "prompt_index": 0, "metadata_json": `{}`, - }) - insertJournalMessageVersion(t, database, 3, true, map[string]any{"message_id": "message-1"}) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - - atInsert, more, err := service.conversationMessagesAt(context.Background(), "session-1", 1, nil, nil, "asc", "", 20) - require.NoError(t, err) - require.False(t, more) - require.Len(t, atInsert, 1) - require.Equal(t, "before", atInsert[0].Content) - require.Equal(t, "session-1", atInsert[0].TaskSessionID) - - atUpdate, _, err := service.conversationMessagesAt(context.Background(), "session-1", 2, nil, nil, "asc", "", 20) - require.NoError(t, err) - require.Len(t, atUpdate, 1) - require.Equal(t, "after", atUpdate[0].Content) - - afterDelete, _, err := service.conversationMessagesAt(context.Background(), "session-1", 3, nil, nil, "asc", "", 20) - require.NoError(t, err) - require.Empty(t, afterDelete) -} - -func TestSessionEventMaintenanceDispatchesUnsignaledCommittedRowsOnce(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_session_streams ( - session_id TEXT PRIMARY KEY, watermark INTEGER NOT NULL - ); - CREATE TABLE conversation_session_events ( - session_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_id TEXT NOT NULL, - protocol_version INTEGER NOT NULL, event_type TEXT NOT NULL, task_id TEXT, - payload TEXT NOT NULL, created_at TIMESTAMP NOT NULL - ); - INSERT INTO conversation_session_streams(session_id, watermark) VALUES ('session-1', 1); - INSERT INTO conversation_session_events( - session_id, sequence, event_id, protocol_version, event_type, task_id, payload, created_at - ) VALUES ( - 'session-1', 1, 'session-1:1', 1, 'message.deleted', 'task-1', - '{"type":"message.deleted","session_id":"session-1","task_id":"task-1","message_id":"message-1"}', - '2026-09-07T12:00:00Z' - );`) - require.NoError(t, err) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - var dispatched []SessionEvent - service.SetSessionEventSink(func(event SessionEvent) { - dispatched = append(dispatched, event) - }) - - require.NoError(t, service.maintainSessionEvents(context.Background(), time.Now().UTC())) - require.Len(t, dispatched, 1) - require.Equal(t, uint64(1), dispatched[0].Sequence) - require.NoError(t, service.maintainSessionEvents(context.Background(), time.Now().UTC())) - require.Len(t, dispatched, 1) -} - -func TestPruneConversationJournalKeepsLatestRowsAndRecentEvents(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_message_versions ( - session_id TEXT NOT NULL, message_id TEXT NOT NULL, row_sequence INTEGER NOT NULL, - task_id TEXT, author_type TEXT, created_at TIMESTAMP NOT NULL, - tombstone BOOLEAN NOT NULL, payload TEXT NOT NULL - ); - CREATE TABLE conversation_turn_versions ( - session_id TEXT NOT NULL, turn_id TEXT NOT NULL, row_sequence INTEGER NOT NULL, - task_id TEXT, started_at TIMESTAMP NOT NULL, tombstone BOOLEAN NOT NULL, payload TEXT NOT NULL - ); - CREATE TABLE conversation_session_events ( - session_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_id TEXT NOT NULL, - protocol_version INTEGER NOT NULL, event_type TEXT NOT NULL, task_id TEXT, - payload TEXT NOT NULL, created_at TIMESTAMP NOT NULL - );`) - require.NoError(t, err) - old := time.Now().UTC().Add(-2 * SessionEventRetention) - recent := time.Now().UTC() - _, err = database.Exec(` - INSERT INTO conversation_message_versions VALUES - ('session-1', 'message-1', 1, 'task-1', 'agent', ?, FALSE, '{}'), - ('session-1', 'message-1', 2, 'task-1', 'agent', ?, FALSE, '{}'); - INSERT INTO conversation_turn_versions VALUES - ('session-1', 'turn-1', 1, 'task-1', ?, FALSE, '{}'), - ('session-1', 'turn-1', 2, 'task-1', ?, FALSE, '{}'); - INSERT INTO conversation_session_events VALUES - ('session-1', 1, 'session-1:1', 1, 'message.added', 'task-1', '{}', ?), - ('session-1', 2, 'session-1:2', 1, 'message.updated', 'task-1', '{}', ?);`, - old, recent, old, recent, old, recent) - require.NoError(t, err) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - require.NoError(t, service.pruneConversationJournal(context.Background(), time.Now().UTC().Add(-SessionEventRetention), nil)) - - var count int - require.NoError(t, database.Get(&count, "SELECT COUNT(*) FROM conversation_message_versions")) - require.Equal(t, 1, count) - require.NoError(t, database.Get(&count, "SELECT COUNT(*) FROM conversation_turn_versions")) - require.Equal(t, 1, count) - require.NoError(t, database.Get(&count, "SELECT COUNT(*) FROM conversation_session_events")) - require.Equal(t, 1, count) -} -func TestPruneConversationJournalPurgestDeadSessionPartitions(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - database.SetMaxOpenConns(1) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_session_streams ( - session_id TEXT PRIMARY KEY, watermark INTEGER NOT NULL DEFAULT 0, - terminal BOOLEAN NOT NULL DEFAULT FALSE, updated_at TIMESTAMP NOT NULL - ); - CREATE TABLE conversation_message_versions ( - session_id TEXT NOT NULL, message_id TEXT NOT NULL, row_sequence INTEGER NOT NULL, - task_id TEXT, author_type TEXT, created_at TIMESTAMP NOT NULL, - tombstone BOOLEAN NOT NULL, payload TEXT NOT NULL - ); - CREATE TABLE conversation_turn_versions ( - session_id TEXT NOT NULL, turn_id TEXT NOT NULL, row_sequence INTEGER NOT NULL, - task_id TEXT, started_at TIMESTAMP NOT NULL, tombstone BOOLEAN NOT NULL, payload TEXT NOT NULL - ); - CREATE TABLE conversation_session_events ( - session_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_id TEXT NOT NULL, - protocol_version INTEGER NOT NULL, event_type TEXT NOT NULL, task_id TEXT, - payload TEXT NOT NULL, created_at TIMESTAMP NOT NULL - ); - CREATE TABLE task_sessions ( - id TEXT PRIMARY KEY, task_id TEXT NOT NULL, started_at TIMESTAMP NOT NULL - );`) - require.NoError(t, err) - old := time.Now().UTC().Add(-2 * SessionEventRetention) - recent := time.Now().UTC() - for _, seed := range []struct { - sql string - args []any - }{ - {`INSERT INTO conversation_session_streams VALUES ('session-dead', 5, TRUE, ?)`, []any{old}}, - {`INSERT INTO conversation_session_streams VALUES ('session-live', 5, TRUE, ?)`, []any{recent}}, - {`INSERT INTO conversation_session_streams VALUES ('session-retained', 5, TRUE, ?)`, []any{recent}}, - {`INSERT INTO conversation_message_versions VALUES ('session-dead', 'm-1', 1, 'task-1', 'agent', ?, FALSE, '{}')`, []any{old}}, - {`INSERT INTO conversation_message_versions VALUES ('session-dead', 'm-1', 2, 'task-1', 'agent', ?, FALSE, '{}')`, []any{recent}}, - {`INSERT INTO conversation_turn_versions VALUES ('session-dead', 'turn-1', 3, 'task-1', ?, FALSE, '{}')`, []any{old}}, - {`INSERT INTO conversation_session_events VALUES ('session-dead', 1, 'session-dead:1', 1, 'message.added', 'task-1', '{}', ?)`, []any{old}}, - {`INSERT INTO conversation_session_events VALUES ('session-dead', 5, 'session-dead:5', 1, 'session.removed', 'task-1', '{}', ?)`, []any{recent}}, - {`INSERT INTO task_sessions VALUES ('session-live', 'task-1', ?)`, []any{recent}}, - } { - if _, err := database.Exec(seed.sql, seed.args...); err != nil { - t.Fatalf("seed: %v\n%s", err, seed.sql) - } - } - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - retained := map[string]struct{}{"session-retained": {}} - require.NoError(t, service.pruneConversationJournal(context.Background(), time.Now().UTC(), retained)) - - var streams int - require.NoError(t, database.Get(&streams, "SELECT COUNT(*) FROM conversation_session_streams")) - require.Equal(t, 2, streams) // live + retained survive; dead purged - var versions int - require.NoError(t, database.Get(&versions, "SELECT COUNT(*) FROM conversation_message_versions")) - require.Equal(t, 0, versions) - require.NoError(t, database.Get(&versions, "SELECT COUNT(*) FROM conversation_turn_versions")) - require.Equal(t, 0, versions) - require.NoError(t, database.Get(&versions, "SELECT COUNT(*) FROM conversation_session_events")) - require.Equal(t, 0, versions) - require.NoError(t, database.Get(&streams, "SELECT COUNT(*) FROM conversation_session_streams WHERE session_id = 'session-dead'")) - require.Equal(t, 0, streams) -} - -func insertJournalMessageVersion(t *testing.T, database *sqlx.DB, sequence int, tombstone bool, payload map[string]any) { - - t.Helper() - raw, err := json.Marshal(payload) - require.NoError(t, err) - _, err = database.Exec(` - INSERT INTO conversation_message_versions( - session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - "session-1", "message-1", sequence, "task-1", "agent", time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC), tombstone, string(raw), - ) - require.NoError(t, err) -} -func TestSanitizeConversationTurnEventPreservesPublicMetadata(t *testing.T) { - raw := json.RawMessage(`{ - "type": "session.turn.completed", - "session_id": "session-1", - "task_id": "task-1", - "id": "turn-1", - "started_at": "2026-09-07T12:00:00Z", - "completed_at": "2026-09-07T12:01:00Z", - "metadata": { - "runtime_config_snapshot": { - "model": "mock-fast", - "config_baseline": {"effort": "medium"} - } - }, - "private": "must-not-be-copied" - }`) - - sanitized := sanitizeConversationEventPayload(conversationTurnCompleted, raw) - var payload map[string]any - require.NoError(t, json.Unmarshal(sanitized, &payload)) - require.Equal(t, map[string]any{ - "runtime_config_snapshot": map[string]any{ - "model": "mock-fast", - "config_baseline": map[string]any{"effort": "medium"}, - }, - }, payload["metadata"]) - require.NotContains(t, payload, "private") -} -func TestSanitizeConversationMessageEventPreservesPresentationMetadata(t *testing.T) { - raw := json.RawMessage(`{ - "type": "message.added", - "session_id": "session-1", - "task_id": "task-1", - "message_id": "message-1", - "author_type": "agent", - "message_type": "clarification_request", - "requests_input": true, - "metadata": { - "request_id": "request-1", - "tool_call_id": "tool-1", - "action_type": "execute", - "action_details": {"command": "ls"}, - "options": [ - {"option_id": "allow", "name": "Allow", "kind": "allow_once"}, - {"option_id": "reject", "name": "Reject", "kind": "reject_once"} - ], - "pending_id": "pending-1", - "question": {"id": "question-1", "prompt": "Choose hidden"}, - "secret": "must-not-be-copied", - "raw_content": "must-not-be-copied" - }, - "content": "Question" - }`) - - sanitized := sanitizeConversationEventPayload(events.MessageAdded, raw) - var payload map[string]any - require.NoError(t, json.Unmarshal(sanitized, &payload)) - require.Equal(t, "clarification_request", payload["message_type"]) - require.Equal(t, true, payload["requests_input"]) - require.Equal(t, map[string]any{ - "action_details": map[string]any{"command": "ls"}, - "action_type": "execute", - "options": []any{ - map[string]any{"option_id": "allow", "name": "Allow", "kind": "allow_once"}, - map[string]any{"option_id": "reject", "name": "Reject", "kind": "reject_once"}, - }, - "pending_id": "pending-1", - "question": map[string]any{"id": "question-1", "prompt": "Choose"}, - "request_id": "request-1", - "tool_call_id": "tool-1", - }, payload["metadata"]) -} - -func TestSanitizeConversationMessageEventPreservesAgentBootMetadata(t *testing.T) { - raw := json.RawMessage(`{ - "type": "message.updated", - "session_id": "session-1", - "task_id": "task-1", - "message_id": "message-1", - "author_type": "agent", - "message_type": "script_execution", - "metadata": { - "script_type": "agent_boot", - "agent_name": "Mock", - "command": "/usr/local/bin/mock-agent", - "status": "exited", - "exit_code": 0, - "is_resuming": true, - "started_at": "2026-09-15T09:45:58.245720115Z", - "completed_at": "2026-09-15T09:45:59.774503743Z", - "error": "must-not-be-dropped", - "private": "must-not-be-copied" - }, - "content": "boot output" - }`) - - sanitized := sanitizeConversationEventPayload(events.MessageUpdated, raw) - var payload map[string]any - require.NoError(t, json.Unmarshal(sanitized, &payload)) - require.Equal(t, map[string]any{ - "agent_name": "Mock", - "command": "/usr/local/bin/mock-agent", - "completed_at": "2026-09-15T09:45:59.774503743Z", - "error": "must-not-be-dropped", - "exit_code": float64(0), - "is_resuming": true, - "script_type": "agent_boot", - "started_at": "2026-09-15T09:45:58.245720115Z", - "status": "exited", - }, payload["metadata"]) - require.NotContains(t, payload["metadata"], "private") -} - -func TestSyncCommittedSessionEventsStripsSystemContent(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_session_streams ( - session_id TEXT PRIMARY KEY, watermark INTEGER NOT NULL - ); - CREATE TABLE conversation_session_events ( - session_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_id TEXT NOT NULL, - protocol_version INTEGER NOT NULL, event_type TEXT NOT NULL, task_id TEXT, - payload TEXT NOT NULL, created_at TIMESTAMP NOT NULL - ); - INSERT INTO conversation_session_streams(session_id, watermark) VALUES ('session-1', 1); - INSERT INTO conversation_session_events VALUES ( - 'session-1', 1, 'session-1:1', 1, 'message.added', 'task-1', - '{"type":"message.added","session_id":"session-1","task_id":"task-1","message_id":"message-1","author_type":"user","author_id":"agent-1","metadata":{"sender_task_id":"sender-1","secret":"no"},"content":"visible secret"}', - '2026-09-07T12:00:00Z' - );`) - require.NoError(t, err) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - var payload map[string]any - events, err := service.SyncCommittedSessionEvents(context.Background(), "session-1") - require.NoError(t, err) - require.NoError(t, json.Unmarshal(events[0].Payload, &payload)) - require.Equal(t, "visible", payload["content"]) - require.Equal(t, "sender-1", payload["sender_task_id"]) - require.NotContains(t, payload, "author_id") - require.Equal(t, map[string]any{"sender_task_id": "sender-1"}, payload["metadata"]) -} - -func TestSyncCommittedSessionEventsPoisonsMalformedPayloadWithoutRetainingRawBytes(t *testing.T) { - const sessionID = "malformed-journal-session" - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_session_streams ( - session_id TEXT PRIMARY KEY, watermark INTEGER NOT NULL - ); - CREATE TABLE conversation_session_events ( - session_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_id TEXT NOT NULL, - protocol_version INTEGER NOT NULL, event_type TEXT NOT NULL, task_id TEXT, - payload TEXT NOT NULL, created_at TIMESTAMP NOT NULL - ); - INSERT INTO conversation_session_streams(session_id, watermark) VALUES ('` + sessionID + `', 1); - INSERT INTO conversation_session_events VALUES ( - '` + sessionID + `', 1, '` + sessionID + `:1', 1, 'message.added', 'task-1', - '{"content":"private secret"', - '2026-09-07T12:00:00Z' - );`) - require.NoError(t, err) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - events, err := service.SyncCommittedSessionEvents(context.Background(), sessionID) - require.NoError(t, err) - require.Len(t, events, 1) - require.Equal(t, "{}", string(events[0].Payload)) - require.NotContains(t, string(events[0].Payload), "private") - require.NotContains(t, string(events[0].Payload), "kandev-system") - _, projectionErr := ProjectSessionEvent(events[0]) - require.Error(t, projectionErr) -} - -func TestConversationMessagesAtErrorsWhenPageCursorWasTombstoned(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_message_versions ( - session_id TEXT NOT NULL, message_id TEXT NOT NULL, row_sequence INTEGER NOT NULL, - task_id TEXT, author_type TEXT, created_at TIMESTAMP NOT NULL, - tombstone BOOLEAN NOT NULL, payload TEXT NOT NULL - );`) - require.NoError(t, err) - createdAt := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - insertJournalMessageVersion(t, database, 1, false, map[string]any{ - "message_id": "message-1", "turn_id": "turn-1", "task_id": "task-1", - "author_type": "agent", "content": "first", "message_type": "message", - "created_at": createdAt.Format(time.RFC3339Nano), - "updated_at": createdAt.Format(time.RFC3339Nano), - }) - insertJournalMessageVersion(t, database, 2, true, map[string]any{"message_id": "message-1"}) - - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - service.SetConversationJournalDB(database) - - _, _, err = service.conversationMessagesAt( - context.Background(), "session-1", 2, nil, nil, "asc", "message-1", 20, - ) - require.ErrorIs(t, err, errConversationCursorGone) -} - -// TestSyncCommittedSessionEventsPoisonsMalformedPayloadOnDurableLog pins that -// a malformed primary-journal row is mirrored into the FILE-backed durable log -// as a poison record with a non-nil, non-raw payload (the session_events -// payload column is BLOB NOT NULL, so nil would fail the append). -func TestSyncCommittedSessionEventsPoisonsMalformedPayloadOnDurableLog(t *testing.T) { - database, err := sqlx.Open("sqlite3", ":memory:") - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, database.Close()) }) - _, err = database.Exec(` - CREATE TABLE conversation_session_streams ( - session_id TEXT PRIMARY KEY, watermark INTEGER NOT NULL - ); - CREATE TABLE conversation_session_events ( - session_id TEXT NOT NULL, sequence INTEGER NOT NULL, event_id TEXT NOT NULL, - protocol_version INTEGER NOT NULL, event_type TEXT NOT NULL, task_id TEXT, - payload TEXT NOT NULL, created_at TIMESTAMP NOT NULL - ); - INSERT INTO conversation_session_streams(session_id, watermark) VALUES ('sess-durable-poison', 1); - INSERT INTO conversation_session_events VALUES ( - 'sess-durable-poison', 1, 'sess-durable-poison:1', 1, 'message.added', 'task-1', - '{"content":"private secret"', - '2026-09-07T12:00:00Z' - );`) - require.NoError(t, err) - - dir := t.TempDir() - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - require.NoError(t, service.SetPluginsDir(dir)) - t.Cleanup(func() { _ = service.Close() }) - service.SetConversationJournalDB(database) - - events, err := service.SyncCommittedSessionEvents(context.Background(), "sess-durable-poison") - require.NoError(t, err) - require.Len(t, events, 1) - require.Equal(t, "{}", string(events[0].Payload)) - log := service.SessionEvents() - record, ok := log.Poison("sess-durable-poison", events[0].ID) - require.True(t, ok) - require.NotEmpty(t, record.LastError) - - // A restart of the durable log keeps the poison and the non-raw payload. - reopened, err := NewSessionEventLog(filepath.Join(dir, ".host", "session-events.sqlite")) - require.NoError(t, err) - t.Cleanup(func() { _ = reopened.Close() }) - restored, ok := reopened.Poison("sess-durable-poison", events[0].ID) - require.True(t, ok) - require.Equal(t, record.Attempts, restored.Attempts) -} diff --git a/apps/backend/internal/plugins/conversation_sanitization.go b/apps/backend/internal/plugins/conversation_sanitization.go new file mode 100644 index 0000000000..ac354d12b2 --- /dev/null +++ b/apps/backend/internal/plugins/conversation_sanitization.go @@ -0,0 +1,68 @@ +package plugins + +import ( + "github.com/kandev/kandev/internal/sysprompt" +) + +//nolint:goconst // These keys are an explicit privacy allowlist. +var conversationMessageMetadataKeys = map[string]struct{}{ + "action_details": {}, "action_type": {}, "action_visibility": {}, + "actions": {}, "agent_disconnected": {}, + "attempt": {}, "attachments": {}, "auth_methods": {}, "auto_start": {}, + "base_branch": {}, "context": {}, "context_files": {}, "decision_id": {}, + "effective_model": {}, "entity_references": {}, "error_output": {}, + "failure_code": {}, "failure_details": {}, "failure_kind": {}, + "fallback_model": {}, "has_hidden_prompts": {}, "has_resume_token": {}, + "has_review_comments": {}, "is_auth_error": {}, "kind": {}, "max_attempts": {}, + "message": {}, "missing_branch": {}, "model_id": {}, "new_branch": {}, + "original_branch": {}, "options": {}, "pending_id": {}, "plan_mode": {}, "progress": {}, + "provider_name": {}, "question": {}, "question_id": {}, "question_index": {}, + "question_total": {}, "recovery_actions": {}, "remediation": {}, + "remediation_url": {}, "requested_model": {}, "request_id": {}, + "requests_input": {}, "response": {}, "reset_at": {}, "retry_at": {}, + "retry_in_seconds": {}, "retrying": {}, "sender_session_id": {}, "sender_session_name": {}, + "sender_task_id": {}, "sender_task_title": {}, "stage": {}, "status": {}, + "script_type": {}, "agent_name": {}, "command": {}, "exit_code": {}, + "is_resuming": {}, "started_at": {}, "completed_at": {}, "error": {}, + "task_id": {}, "text": {}, "tool_call_id": {}, "variant": {}, "workflow_message": {}, + "workflow_step_color": {}, "workflow_step_id": {}, "workflow_step_name": {}, +} + +// SanitizeConversationMessageMetadata returns the presentation metadata that +// the plugin conversation stream may expose. Internal agent and tool metadata +// is excluded by the allowlist. +func SanitizeConversationMessageMetadata(source map[string]any) map[string]any { + if len(source) == 0 { + return nil + } + target := make(map[string]any) + for key := range conversationMessageMetadataKeys { + value, exists := source[key] + if !exists || value == nil { + continue + } + target[key] = sanitizeConversationMetadataValue(value) + } + return target +} + +func sanitizeConversationMetadataValue(value any) any { + switch typed := value.(type) { + case string: + return sysprompt.StripSystemContent(typed) + case map[string]any: + target := make(map[string]any, len(typed)) + for key, nested := range typed { + target[key] = sanitizeConversationMetadataValue(nested) + } + return target + case []any: + target := make([]any, len(typed)) + for index, nested := range typed { + target[index] = sanitizeConversationMetadataValue(nested) + } + return target + default: + return value + } +} diff --git a/apps/backend/internal/plugins/conversation_stream.go b/apps/backend/internal/plugins/conversation_stream.go deleted file mode 100644 index ae8eae3f82..0000000000 --- a/apps/backend/internal/plugins/conversation_stream.go +++ /dev/null @@ -1,1354 +0,0 @@ -//nolint:revive // This file groups the durable stream state machine and its protocol validation. -package plugins - -import ( - "database/sql" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "time" - - "github.com/google/uuid" - "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" -) - -const ( - SessionEventProtocolVersion = 1 - SessionPoisonPending = "pending" - SessionPoisonLeased = "leased" - SessionPoisonExhausted = "exhausted" - SessionPoisonRequeued = "requeued" - SessionPoisonMaxAttempts = 5 - SessionPoisonLease = 30 * time.Second - SessionReconnectGrace = 10 * time.Minute - SessionEventRetention = conversationTokenTTL + SessionReconnectGrace - sessionRemovedEventType = "session.removed" -) - -var ( - ErrSessionRemoved = errors.New("session removed") - ErrPoisonEvent = errors.New("poison session event") - ErrForwardGap = errors.New("session event forward gap") - ErrStaleOwnerEpoch = errors.New("stale poison owner epoch") -) - -// SessionDeliveryDisposition is the atomic poison-delivery claim outcome. -type SessionDeliveryDisposition string - -const ( - SessionDeliveryUntracked SessionDeliveryDisposition = "untracked" - SessionDeliveryClaimed SessionDeliveryDisposition = "claimed" - SessionDeliveryLeased SessionDeliveryDisposition = "leased" - SessionDeliveryBackoff SessionDeliveryDisposition = "backoff" - SessionDeliveryExhausted SessionDeliveryDisposition = "exhausted" -) - -// SessionDeliveryClaim is an opaque reservation. Complete must receive the -// exact claim so a stale sender cannot mutate a newer lease or owner epoch. -type SessionDeliveryClaim struct { - Disposition SessionDeliveryDisposition - sessionID string - eventID string - ownerEpoch uint64 - leaseUntil time.Time -} - -// SessionEvent is the immutable Host-only ordered stream envelope. -type SessionEvent struct { - ProtocolVersion int `json:"protocol_version"` - EventType string `json:"event_type"` - SessionID string `json:"session_id"` - TaskID *string `json:"task_id"` - Sequence uint64 `json:"sequence"` - ID string `json:"event_id"` - Payload json.RawMessage `json:"payload"` - CreatedAt time.Time `json:"created_at"` -} - -// SessionDeliveryCursorKey is the complete durable consumer binding. -type SessionDeliveryCursorKey struct { - SessionID string `json:"session_id"` - ConsumerKind string `json:"consumer_kind"` - ConsumerID string `json:"consumer_id"` - WireID string `json:"wire_id"` - PluginID string `json:"plugin_id"` - Generation int64 `json:"generation"` - UserID string `json:"user_id"` -} - -type SessionDeliveryCursor struct { - Key SessionDeliveryCursorKey `json:"key"` - AcknowledgedSequence uint64 `json:"acknowledged_sequence"` - OwnerEpoch uint64 `json:"owner_epoch"` - LeaseUntil time.Time `json:"lease_until,omitempty"` - RetryAt time.Time `json:"retry_at,omitempty"` - UpdatedAt time.Time `json:"updated_at"` -} - -type SessionPoisonRecord struct { - SessionID string `json:"session_id"` - EventID string `json:"event_id"` - Sequence uint64 `json:"sequence"` - ProtocolVersion int `json:"protocol_version"` - LastError string `json:"last_error"` - State string `json:"state"` - Attempts int `json:"attempts"` - OwnerEpoch uint64 `json:"owner_epoch"` - LeaseUntil time.Time `json:"lease_until,omitempty"` - NextRetry time.Time `json:"next_retry,omitempty"` - LastActor string `json:"last_actor,omitempty"` - UpdatedAt time.Time `json:"updated_at"` -} - -type SessionPoisonAudit struct { - SessionID string `json:"session_id"` - EventID string `json:"event_id"` - Actor string `json:"actor"` - PriorState string `json:"prior_state"` - Attempts int `json:"attempts"` - At time.Time `json:"at"` -} - -type sessionEventPartition struct { - Events []SessionEvent `json:"events"` - Watermark uint64 `json:"watermark"` - Terminal bool `json:"terminal"` -} - -type sessionEventLogState struct { - Sessions map[string]*sessionEventPartition `json:"sessions"` - Cursors map[string]*SessionDeliveryCursor `json:"cursors"` - Poison map[string]*SessionPoisonRecord `json:"poison"` - Audits []SessionPoisonAudit `json:"audits"` -} - -func cloneSessionEventLogState(state sessionEventLogState) (sessionEventLogState, error) { - encoded, err := json.Marshal(state) - if err != nil { - return sessionEventLogState{}, fmt.Errorf("clone session event log state: %w", err) - } - var clone sessionEventLogState - if err := json.Unmarshal(encoded, &clone); err != nil { - return sessionEventLogState{}, fmt.Errorf("clone session event log state: %w", err) - } - return clone, nil -} - -// SessionEventLog serializes append, versioning, poison, terminal tombstones, -// and consumer cursor advancement behind one persistence boundary. -type SessionEventLog struct { - mu sync.Mutex - path string - db *sql.DB - state sessionEventLogState - now func() time.Time -} - -// SessionDeliveryDispatcher owns poison delivery transitions and recovery. -type SessionDeliveryDispatcher struct { - events *SessionEventLog -} - -func NewSessionDeliveryDispatcher(events *SessionEventLog) *SessionDeliveryDispatcher { - return &SessionDeliveryDispatcher{events: events} -} - -// Claim atomically reserves a poison delivery. Untracked events need no -// completion; claimed poison must be completed after the queue attempt. -func (d *SessionDeliveryDispatcher) Claim(sessionID, eventID string, now time.Time) (SessionDeliveryClaim, error) { - return d.events.claimPoisonDelivery(sessionID, eventID, now) -} - -// Complete records whether a claimed poison frame reached any client queue. -func (d *SessionDeliveryDispatcher) Complete(claim SessionDeliveryClaim, queued bool, now time.Time) error { - return d.events.completePoisonDelivery(claim, queued, now) -} - -func (d *SessionDeliveryDispatcher) Requeue( - sessionID string, - eventID string, - expectedOwnerEpoch uint64, - actor string, - now time.Time, -) error { - return d.events.requeuePoison(sessionID, eventID, expectedOwnerEpoch, actor, now) -} - -func (d *SessionDeliveryDispatcher) ReclaimExpiredLeases(now time.Time) error { - return d.events.reclaimExpiredLeases(now) -} - -func NewSessionEventLog(path string) (*SessionEventLog, error) { - log := &SessionEventLog{ - state: sessionEventLogState{ - Sessions: make(map[string]*sessionEventPartition), - Cursors: make(map[string]*SessionDeliveryCursor), - Poison: make(map[string]*SessionPoisonRecord), - }, - now: time.Now, - } - if path == "" { - return log, nil - } - log.path = path - if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { - return nil, fmt.Errorf("create session event log directory: %w", err) - } - db, err := sql.Open("sqlite3", path) - if err != nil { - return nil, fmt.Errorf("open session event log: %w", err) - } - db.SetMaxOpenConns(1) - log.db = db - if err := log.initializeDatabase(); err != nil { - _ = db.Close() - return nil, err - } - if err := log.loadDatabase(); err != nil { - _ = db.Close() - return nil, err - } - return log, nil -} - -// Close releases the durable event log database. -func (l *SessionEventLog) Close() error { - l.mu.Lock() - defer l.mu.Unlock() - if l.db == nil { - return nil - } - err := l.db.Close() - l.db = nil - return err -} -func mustSessionEventLog() *SessionEventLog { - log, err := NewSessionEventLog("") - if err != nil { - panic(err) - } - return log -} - -func (l *SessionEventLog) Append( - sessionID string, - taskID *string, - eventType string, - payload json.RawMessage, -) (SessionEvent, error) { - if !json.Valid(payload) { - return SessionEvent{}, fmt.Errorf("%w: malformed payload", ErrPoisonEvent) - } - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[sessionID] - newPartition := partition == nil - if newPartition { - partition = &sessionEventPartition{} - l.state.Sessions[sessionID] = partition - } - previousTerminal := partition.Terminal - previousWatermark := partition.Watermark - if partition.Terminal { - return SessionEvent{}, ErrSessionRemoved - } - event := SessionEvent{ - ProtocolVersion: SessionEventProtocolVersion, - EventType: eventType, - SessionID: sessionID, - TaskID: taskID, - Sequence: partition.Watermark + 1, - ID: uuid.NewString(), - Payload: append(json.RawMessage(nil), payload...), - CreatedAt: l.now().UTC(), - } - partition.Events = append(partition.Events, event) - partition.Watermark = event.Sequence - if eventType == sessionRemovedEventType { - partition.Terminal = true - } - var poison *SessionPoisonRecord - if _, projectionErr := ProjectSessionEvent(event); projectionErr != nil { - poison = &SessionPoisonRecord{ - SessionID: sessionID, EventID: event.ID, Sequence: event.Sequence, - ProtocolVersion: event.ProtocolVersion, LastError: projectionErr.Error(), - State: SessionPoisonPending, OwnerEpoch: 1, UpdatedAt: event.CreatedAt, - } - l.state.Poison[poisonKey(sessionID, event.ID)] = poison - } - if err := l.persistAppendLocked(event, poison); err != nil { - partition.Events = partition.Events[:len(partition.Events)-1] - partition.Terminal = previousTerminal - partition.Watermark = previousWatermark - delete(l.state.Poison, poisonKey(sessionID, event.ID)) - if newPartition { - delete(l.state.Sessions, sessionID) - } - return SessionEvent{}, err - } - return event, nil -} - -// AppendCommitted mirrors an event whose sequence and identity were allocated -// in the source mutation's primary-database transaction. -func (l *SessionEventLog) AppendCommitted(event SessionEvent) (bool, error) { - if !json.Valid(event.Payload) { - return false, fmt.Errorf("%w: malformed payload", ErrPoisonEvent) - } - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[event.SessionID] - newPartition := partition == nil - if newPartition { - partition = &sessionEventPartition{} - l.state.Sessions[event.SessionID] = partition - } - previousWatermark := partition.Watermark - if event.Sequence <= partition.Watermark { - for _, existing := range partition.Events { - if existing.Sequence == event.Sequence { - if existing.ID != event.ID || existing.EventType != event.EventType { - return false, fmt.Errorf("%w: sequence identity mismatch", ErrPoisonEvent) - } - return false, nil - } - } - // The local replay row may already have aged out while its monotonic - // watermark remains. The primary journal is authoritative. - return false, nil - } - if partition.Terminal { - return false, ErrSessionRemoved - } - if event.Sequence != partition.Watermark+1 { - if !mirrorGapHealable(newPartition, partition) { - return false, ErrForwardGap - } - // Collection can drop a partition (terminal, after retention) or - // truncate its rows (idle, non-terminal) once no cursor or poison - // protects the session. The primary journal then prunes the same - // aged rows, so the first surviving row may start past the local - // watermark. The primary is authoritative for mirrored rows: jump - // the replay boundary and accept the row. - partition.Watermark = event.Sequence - 1 - } - previousTerminal := partition.Terminal - partition.Events = append(partition.Events, event) - partition.Watermark = event.Sequence - if event.EventType == sessionRemovedEventType { - partition.Terminal = true - } - var poison *SessionPoisonRecord - if _, projectionErr := ProjectSessionEvent(event); projectionErr != nil { - poison = &SessionPoisonRecord{ - SessionID: event.SessionID, EventID: event.ID, Sequence: event.Sequence, - ProtocolVersion: event.ProtocolVersion, LastError: projectionErr.Error(), - State: SessionPoisonPending, OwnerEpoch: 1, UpdatedAt: event.CreatedAt, - } - l.state.Poison[poisonKey(event.SessionID, event.ID)] = poison - } - if err := l.persistAppendLocked(event, poison); err != nil { - partition.Events = partition.Events[:len(partition.Events)-1] - partition.Watermark = previousWatermark - partition.Terminal = previousTerminal - delete(l.state.Poison, poisonKey(event.SessionID, event.ID)) - if newPartition { - delete(l.state.Sessions, event.SessionID) - } - return false, err - } - return true, nil -} - -// mirrorGapHealable reports whether a mirror row that starts past the local -// watermark can be accepted by jumping the replay boundary. Cursor/poison- -// protected sessions always keep their rows, so a gap into a non-empty (or -// still-terminal) partition means real history is missing and must stay an -// error; an empty partition can only mean both sides already discarded the -// intermediate rows. -func mirrorGapHealable(newPartition bool, partition *sessionEventPartition) bool { - if newPartition { - return true - } - return len(partition.Events) == 0 && !partition.Terminal -} - -func (l *SessionEventLog) EventsAfter(sessionID string, sequence uint64) ([]SessionEvent, bool) { - events, _, terminal := l.ReplayState(sessionID, sequence) - return events, terminal -} - -// ReplayState captures replay rows, the current watermark, and terminal state -// under one lock so subscription registration cannot construct a torn cutoff. -func (l *SessionEventLog) ReplayState( - sessionID string, - sequence uint64, -) ([]SessionEvent, uint64, bool) { - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[sessionID] - if partition == nil { - return nil, 0, false - } - start := len(partition.Events) - for index := range partition.Events { - if partition.Events[index].Sequence > sequence { - start = index - break - } - } - result := append([]SessionEvent(nil), partition.Events[start:]...) - return result, partition.Watermark, partition.Terminal -} - -func (l *SessionEventLog) Watermark(sessionID string) uint64 { - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[sessionID] - if partition == nil { - return 0 - } - return partition.Watermark -} - -func (l *SessionEventLog) RegisterCursor(key SessionDeliveryCursorKey, sequence uint64) error { - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[key.SessionID] - if partition != nil && sequence > partition.Watermark { - return ErrForwardGap - } - encodedKey := deliveryCursorKey(key) - previous := l.state.Cursors[encodedKey] - if previous != nil { - previousCopy := *previous - previous.UpdatedAt = l.now().UTC() - if err := l.persistLocked(); err != nil { - *previous = previousCopy - return err - } - return nil - } - l.state.Cursors[encodedKey] = &SessionDeliveryCursor{ - Key: key, AcknowledgedSequence: sequence, OwnerEpoch: 1, UpdatedAt: l.now().UTC(), - } - if err := l.persistLocked(); err != nil { - delete(l.state.Cursors, encodedKey) - return err - } - return nil -} - -func (l *SessionEventLog) ReleaseCursor(key SessionDeliveryCursorKey) error { - l.mu.Lock() - defer l.mu.Unlock() - encodedKey := deliveryCursorKey(key) - cursor := l.state.Cursors[encodedKey] - if cursor == nil { - return nil - } - delete(l.state.Cursors, encodedKey) - if err := l.persistLocked(); err != nil { - l.state.Cursors[encodedKey] = cursor - return err - } - return nil -} - -// ReplaceCursor atomically advances a consumer to a new replay boundary. -// Rebinds use this after invalid resume or poison recovery; RegisterCursor -// intentionally preserves an existing acknowledgement for ordinary reconnects. -func (l *SessionEventLog) ReplaceCursor(key SessionDeliveryCursorKey, sequence uint64) error { - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[key.SessionID] - if partition != nil && sequence > partition.Watermark { - return ErrForwardGap - } - encodedKey := deliveryCursorKey(key) - previous, err := cloneSessionEventLogState(l.state) - if err != nil { - return err - } - cursor := l.state.Cursors[encodedKey] - if cursor == nil { - cursor = &SessionDeliveryCursor{Key: key} - l.state.Cursors[encodedKey] = cursor - } - cursor.AcknowledgedSequence = sequence - cursor.OwnerEpoch++ - cursor.UpdatedAt = l.now().UTC() - if err := l.persistLocked(); err != nil { - l.state = previous - return err - } - return nil -} -func (l *SessionEventLog) HasCursor(key SessionDeliveryCursorKey) bool { - l.mu.Lock() - defer l.mu.Unlock() - _, exists := l.state.Cursors[deliveryCursorKey(key)] - return exists -} - -func (l *SessionEventLog) Acknowledge(key SessionDeliveryCursorKey, sequence uint64) error { - l.mu.Lock() - defer l.mu.Unlock() - partition := l.state.Sessions[key.SessionID] - watermark := uint64(0) - if partition != nil { - watermark = partition.Watermark - } - if sequence > watermark { - return ErrForwardGap - } - encodedKey := deliveryCursorKey(key) - cursor := l.state.Cursors[encodedKey] - if cursor == nil { - cursor = &SessionDeliveryCursor{Key: key} - l.state.Cursors[encodedKey] = cursor - } - if sequence <= cursor.AcknowledgedSequence { - return nil - } - for _, event := range partition.Events { - if event.Sequence <= cursor.AcknowledgedSequence || event.Sequence > sequence { - continue - } - if _, poisoned := l.state.Poison[poisonKey(key.SessionID, event.ID)]; poisoned { - return ErrPoisonEvent - } - } - previous := cursor.AcknowledgedSequence - previousUpdatedAt := cursor.UpdatedAt - cursor.AcknowledgedSequence = sequence - cursor.UpdatedAt = l.now().UTC() - // ACKs are per contiguous event, so persist only the advanced cursor row - // (one upsert) instead of rewriting every retained partition, event, - // cursor, poison record, and audit with the full-state transaction. - if err := l.persistCursorRowLocked(encodedKey, cursor); err != nil { - cursor.AcknowledgedSequence = previous - cursor.UpdatedAt = previousUpdatedAt - return err - } - return nil -} - -// persistCursorRowLocked writes exactly one session_delivery_cursors row, -// leaving every other row untouched. Called on the hot ACK path; full-state -// rewrites remain for structural changes (collection, poison transitions, -// registration of the first cursor). -func (l *SessionEventLog) persistCursorRowLocked(encodedKey string, cursor *SessionDeliveryCursor) error { - if l.path == "" { - return nil - } - if l.db == nil { - return errors.New("session event log database is closed") - } - tx, err := l.db.Begin() - if err != nil { - return fmt.Errorf("begin session event cursor transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - binding, err := json.Marshal(cursor.Key) - if err != nil { - return fmt.Errorf("encode delivery cursor: %w", err) - } - query := sqlx.Rebind(sqlx.QUESTION, ` - INSERT INTO session_delivery_cursors(cursor_key, binding, acknowledged_sequence, owner_epoch, lease_until, retry_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(cursor_key) DO UPDATE SET - binding = excluded.binding, - acknowledged_sequence = excluded.acknowledged_sequence, - owner_epoch = excluded.owner_epoch, - lease_until = excluded.lease_until, - retry_at = excluded.retry_at, - updated_at = excluded.updated_at`) - if _, err := tx.Exec(query, - encodedKey, binding, cursor.AcknowledgedSequence, cursor.OwnerEpoch, - nullableTime(cursor.LeaseUntil), nullableTime(cursor.RetryAt), - cursor.UpdatedAt.Format(time.RFC3339Nano)); err != nil { - return fmt.Errorf("persist delivery cursor row: %w", err) - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit session event cursor transaction: %w", err) - } - return nil -} - -// CollectExpired removes replay state only after both the binding-token and -// reconnect-grace windows have elapsed. A live cursor retains its complete -// partition so reconnect never observes a partially collected history. -// -//nolint:cyclop // Collection coordinates cursor, poison, tombstone, and partition retention. -func (l *SessionEventLog) CollectExpired(now time.Time) error { - l.mu.Lock() - defer l.mu.Unlock() - previous, err := cloneSessionEventLogState(l.state) - if err != nil { - return err - } - cutoff := now.UTC().Add(-SessionEventRetention) - activeSessions := make(map[string]struct{}) - for encodedKey, cursor := range l.state.Cursors { - if cursor.UpdatedAt.After(cutoff) { - activeSessions[cursor.Key.SessionID] = struct{}{} - continue - } - delete(l.state.Cursors, encodedKey) - } - for key, record := range l.state.Poison { - if !record.UpdatedAt.After(cutoff) { - delete(l.state.Poison, key) - } - } - keptAudits := l.state.Audits[:0] - for _, audit := range l.state.Audits { - if audit.At.After(cutoff) { - keptAudits = append(keptAudits, audit) - } - } - l.state.Audits = keptAudits - for sessionID, partition := range l.state.Sessions { - if _, active := activeSessions[sessionID]; active || l.sessionHasPoisonLocked(sessionID) { - continue - } - kept := partition.Events[:0] - for _, event := range partition.Events { - if event.CreatedAt.After(cutoff) { - kept = append(kept, event) - } - } - partition.Events = kept - if partition.Terminal && len(partition.Events) == 0 { - delete(l.state.Sessions, sessionID) - } - } - if err := l.persistLocked(); err != nil { - l.state = previous - return err - } - return nil -} - -func (l *SessionEventLog) sessionHasPoisonLocked(sessionID string) bool { - for _, record := range l.state.Poison { - if record.SessionID == sessionID { - return true - } - } - return false -} - -// RetainedSessionIDs returns sessions whose sidecar state still protects -// reconnect or poison delivery. Primary journal history may be pruned only -// for sessions outside this set. -func (l *SessionEventLog) RetainedSessionIDs(now time.Time) map[string]struct{} { - l.mu.Lock() - defer l.mu.Unlock() - cutoff := now.UTC().Add(-SessionEventRetention) - retained := make(map[string]struct{}) - for _, cursor := range l.state.Cursors { - if cursor.UpdatedAt.After(cutoff) { - retained[cursor.Key.SessionID] = struct{}{} - } - } - for _, record := range l.state.Poison { - if record.UpdatedAt.After(cutoff) { - retained[record.SessionID] = struct{}{} - } - } - return retained -} - -func (l *SessionEventLog) Poison(sessionID, eventID string) (SessionPoisonRecord, bool) { - l.mu.Lock() - defer l.mu.Unlock() - record, ok := l.state.Poison[poisonKey(sessionID, eventID)] - if !ok { - return SessionPoisonRecord{}, false - } - return *record, true -} - -func (l *SessionEventLog) claimPoisonDelivery( - sessionID string, - eventID string, - now time.Time, -) (SessionDeliveryClaim, error) { - l.mu.Lock() - defer l.mu.Unlock() - record := l.state.Poison[poisonKey(sessionID, eventID)] - if record == nil { - return SessionDeliveryClaim{Disposition: SessionDeliveryUntracked}, nil - } - now = now.UTC() - disposition := SessionDeliveryClaimed - switch { - case record.State == SessionPoisonExhausted: - disposition = SessionDeliveryExhausted - case record.State == SessionPoisonLeased && record.LeaseUntil.After(now): - disposition = SessionDeliveryLeased - case record.NextRetry.After(now): - disposition = SessionDeliveryBackoff - } - if disposition != SessionDeliveryClaimed { - return SessionDeliveryClaim{Disposition: disposition}, nil - } - previous, err := cloneSessionEventLogState(l.state) - if err != nil { - return SessionDeliveryClaim{}, err - } - record.State = SessionPoisonLeased - record.LeaseUntil = now.Add(SessionPoisonLease) - record.UpdatedAt = now - claim := SessionDeliveryClaim{ - Disposition: SessionDeliveryClaimed, - sessionID: sessionID, eventID: eventID, - ownerEpoch: record.OwnerEpoch, leaseUntil: record.LeaseUntil, - } - if err := l.persistLocked(); err != nil { - l.state = previous - return SessionDeliveryClaim{}, err - } - return claim, nil -} - -func (l *SessionEventLog) completePoisonDelivery( - claim SessionDeliveryClaim, - queued bool, - now time.Time, -) error { - if claim.Disposition != SessionDeliveryClaimed { - return nil - } - l.mu.Lock() - defer l.mu.Unlock() - record := l.state.Poison[poisonKey(claim.sessionID, claim.eventID)] - if record == nil { - return ErrPoisonEvent - } - if record.State != SessionPoisonLeased || - record.OwnerEpoch != claim.ownerEpoch || - !record.LeaseUntil.Equal(claim.leaseUntil) { - return ErrStaleOwnerEpoch - } - previous, err := cloneSessionEventLogState(l.state) - if err != nil { - return err - } - now = now.UTC() - if !queued { - record.State = SessionPoisonPending - record.LeaseUntil = time.Time{} - record.NextRetry = time.Time{} - record.UpdatedAt = now - } else { - record.Attempts++ - backoff := time.Second << min(record.Attempts-1, 5) - if backoff > SessionPoisonLease { - backoff = SessionPoisonLease - } - record.NextRetry = record.LeaseUntil.Add(backoff) - if record.Attempts >= SessionPoisonMaxAttempts { - record.State = SessionPoisonExhausted - record.LeaseUntil = time.Time{} - record.NextRetry = time.Time{} - } - record.UpdatedAt = now - } - if err := l.persistLocked(); err != nil { - l.state = previous - return err - } - return nil -} - -func (l *SessionEventLog) PendingPoison(now time.Time) []SessionPoisonRecord { - l.mu.Lock() - defer l.mu.Unlock() - now = now.UTC() - result := make([]SessionPoisonRecord, 0) - for _, record := range l.state.Poison { - if record.State == SessionPoisonPending && !record.NextRetry.After(now) { - result = append(result, *record) - } - } - return result -} - -func (l *SessionEventLog) requeuePoison( - sessionID string, - eventID string, - expectedOwnerEpoch uint64, - actor string, - now time.Time, -) error { - l.mu.Lock() - defer l.mu.Unlock() - record := l.state.Poison[poisonKey(sessionID, eventID)] - if record == nil { - return ErrPoisonEvent - } - if record.State == SessionPoisonPending && record.OwnerEpoch == expectedOwnerEpoch+1 { - return nil - } - if record.OwnerEpoch != expectedOwnerEpoch { - return ErrStaleOwnerEpoch - } - previous, err := cloneSessionEventLogState(l.state) - if err != nil { - return err - } - priorState := record.State - priorAttempts := record.Attempts - record.State = SessionPoisonRequeued - record.OwnerEpoch++ - record.Attempts = 0 - record.LeaseUntil = time.Time{} - record.NextRetry = time.Time{} - record.LastActor = actor - record.UpdatedAt = now.UTC() - l.state.Audits = append(l.state.Audits, SessionPoisonAudit{ - SessionID: sessionID, EventID: eventID, Actor: actor, - PriorState: priorState, Attempts: priorAttempts, At: now.UTC(), - }) - record.State = SessionPoisonPending - if err := l.persistLocked(); err != nil { - l.state = previous - return err - } - return nil -} - -func (l *SessionEventLog) reclaimExpiredLeases(now time.Time) error { - l.mu.Lock() - defer l.mu.Unlock() - previous, err := cloneSessionEventLogState(l.state) - if err != nil { - return err - } - changed := false - for _, record := range l.state.Poison { - if record.State == SessionPoisonLeased && !record.LeaseUntil.After(now) { - record.State = SessionPoisonPending - record.LeaseUntil = time.Time{} - record.UpdatedAt = now.UTC() - changed = true - } - } - if !changed { - return nil - } - if err := l.persistLocked(); err != nil { - l.state = previous - return err - } - return nil -} - -//nolint:goconst // Event names are protocol literals. -func ProjectSessionEvent(event SessionEvent) (json.RawMessage, error) { - if event.ProtocolVersion != SessionEventProtocolVersion { - return nil, fmt.Errorf("%w: unsupported protocol version", ErrPoisonEvent) - } - switch event.EventType { - case "message.added", "message.updated", "message.deleted", - "session.turn.started", "session.turn.completed", "session.turn.removed", "session.removed": - default: - return nil, fmt.Errorf("%w: unsupported event type", ErrPoisonEvent) - } - var payload map[string]any - if err := json.Unmarshal(event.Payload, &payload); err != nil { - return nil, fmt.Errorf("%w: malformed payload", ErrPoisonEvent) - } - if payload["type"] != event.EventType { - return nil, fmt.Errorf("%w: payload event type mismatch", ErrPoisonEvent) - } - if sessionID, ok := payload["session_id"].(string); !ok || sessionID == "" || sessionID != event.SessionID { - return nil, fmt.Errorf("%w: payload session identity mismatch", ErrPoisonEvent) - } - if err := validateSessionEventTask(event, payload); err != nil { - return nil, err - } - if err := validateSessionEventPayload(event.EventType, payload); err != nil { - return nil, err - } - return append(json.RawMessage(nil), event.Payload...), nil -} - -func validateSessionEventTask(event SessionEvent, payload map[string]any) error { - taskID, present := payload["task_id"] - if event.TaskID == nil { - if !present || taskID == nil { - return nil - } - return fmt.Errorf("%w: payload task identity mismatch", ErrPoisonEvent) - } - value, ok := taskID.(string) - if !ok || value == "" || value != *event.TaskID { - return fmt.Errorf("%w: payload task identity mismatch", ErrPoisonEvent) - } - return nil -} - -//nolint:cyclop,goconst // Event validation is intentionally explicit per protocol event type. -func validateSessionEventPayload(eventType string, payload map[string]any) error { - requiredString := func(field string) (string, error) { - value, ok := payload[field].(string) - if !ok || value == "" { - return "", fmt.Errorf("%w: missing %s", ErrPoisonEvent, field) - } - return value, nil - } - requiredTime := func(field string) error { - value, err := requiredString(field) - if err != nil { - return err - } - if _, err := time.Parse(time.RFC3339Nano, value); err != nil { - return fmt.Errorf("%w: invalid %s", ErrPoisonEvent, field) - } - return nil - } - switch eventType { - case "message.added", "message.updated": - if _, err := requiredString("message_id"); err != nil { - return err - } - author, err := requiredString("author_type") - if err != nil { - return err - } - if author != "user" && author != "agent" { - return fmt.Errorf("%w: invalid author_type", ErrPoisonEvent) - } - if _, ok := payload["content"].(string); !ok { - return fmt.Errorf("%w: missing content", ErrPoisonEvent) - } - if err := requiredTime("created_at"); err != nil { - return err - } - if eventType == "message.updated" { - return requiredTime("updated_at") - } - case "message.deleted": - _, err := requiredString("message_id") - return err - case "session.turn.removed": - _, err := requiredString("id") - return err - case "session.turn.started", "session.turn.completed": - if _, err := requiredString("id"); err != nil { - return err - } - if err := requiredTime("started_at"); err != nil { - return err - } - if eventType == "session.turn.completed" { - if err := requiredTime("completed_at"); err != nil { - return err - } - return requiredTime("updated_at") - } - case "session.removed": - return nil - } - return nil -} - -func (l *SessionEventLog) persistAppendLocked(event SessionEvent, poison *SessionPoisonRecord) error { - if l.path == "" { - return nil - } - if l.db == nil { - return errors.New("session event log database is closed") - } - tx, err := l.db.Begin() - if err != nil { - return fmt.Errorf("begin session event append: %w", err) - } - defer func() { _ = tx.Rollback() }() - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_event_partitions(session_id, watermark, terminal) VALUES (?, ?, ?) ON CONFLICT(session_id) DO UPDATE SET watermark = excluded.watermark, terminal = excluded.terminal`), - event.SessionID, event.Sequence, event.EventType == sessionRemovedEventType); err != nil { - return fmt.Errorf("persist session partition: %w", err) - } - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_events(session_id, sequence, event_id, protocol_version, event_type, task_id, payload, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`), - event.SessionID, event.Sequence, event.ID, event.ProtocolVersion, event.EventType, nullableString(event.TaskID), []byte(event.Payload), event.CreatedAt.Format(time.RFC3339Nano)); err != nil { - return fmt.Errorf("persist session event: %w", err) - } - if poison != nil { - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_poison(poison_key, record) VALUES (?, ?)`), poisonKey(event.SessionID, event.ID), mustJSON(poison)); err != nil { - return fmt.Errorf("persist poison record: %w", err) - } - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit session event append: %w", err) - } - return nil -} - -func (l *SessionEventLog) persistLocked() error { - if l.path == "" { - return nil - } - if l.db == nil { - return errors.New("session event log database is closed") - } - tx, err := l.db.Begin() - if err != nil { - return fmt.Errorf("begin session event transaction: %w", err) - } - defer func() { _ = tx.Rollback() }() - if err := resetSessionEventTables(tx); err != nil { - return err - } - if err := l.persistSessions(tx); err != nil { - return err - } - if err := l.persistConsumerState(tx); err != nil { - return err - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit session event transaction: %w", err) - } - return nil -} - -func resetSessionEventTables(tx *sql.Tx) error { - for _, table := range []string{"session_poison_audits", "session_poison", "session_delivery_cursors", "session_events", "session_event_partitions"} { - if _, err := tx.Exec("DELETE FROM " + table); err != nil { - return fmt.Errorf("reset %s: %w", table, err) - } - } - return nil -} - -func (l *SessionEventLog) persistSessions(tx *sql.Tx) error { - for sessionID, partition := range l.state.Sessions { - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_event_partitions(session_id, watermark, terminal) VALUES (?, ?, ?)`), sessionID, partition.Watermark, partition.Terminal); err != nil { - return fmt.Errorf("persist session partition: %w", err) - } - for _, event := range partition.Events { - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_events(session_id, sequence, event_id, protocol_version, event_type, task_id, payload, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`), - event.SessionID, event.Sequence, event.ID, event.ProtocolVersion, event.EventType, nullableString(event.TaskID), []byte(event.Payload), event.CreatedAt.Format(time.RFC3339Nano)); err != nil { - return fmt.Errorf("persist session event: %w", err) - } - } - } - return nil -} - -func (l *SessionEventLog) persistConsumerState(tx *sql.Tx) error { - for encodedKey, cursor := range l.state.Cursors { - key, err := json.Marshal(cursor.Key) - if err != nil { - return fmt.Errorf("encode delivery cursor: %w", err) - } - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_delivery_cursors(cursor_key, binding, acknowledged_sequence, owner_epoch, lease_until, retry_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)`), - encodedKey, key, cursor.AcknowledgedSequence, cursor.OwnerEpoch, nullableTime(cursor.LeaseUntil), nullableTime(cursor.RetryAt), cursor.UpdatedAt.Format(time.RFC3339Nano)); err != nil { - return fmt.Errorf("persist delivery cursor: %w", err) - } - } - for encodedKey, record := range l.state.Poison { - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_poison(poison_key, record) VALUES (?, ?)`), encodedKey, mustJSON(record)); err != nil { - return fmt.Errorf("persist poison record: %w", err) - } - } - for _, audit := range l.state.Audits { - if _, err := tx.Exec(sqlx.Rebind(sqlx.QUESTION, `INSERT INTO session_poison_audits(record) VALUES (?)`), mustJSON(audit)); err != nil { - return fmt.Errorf("persist poison audit: %w", err) - } - } - return nil -} - -func (l *SessionEventLog) initializeDatabase() error { - const schema = ` -CREATE TABLE IF NOT EXISTS session_event_partitions ( - session_id TEXT PRIMARY KEY, - watermark INTEGER NOT NULL DEFAULT 0, - terminal INTEGER NOT NULL -); -CREATE TABLE IF NOT EXISTS session_events ( - session_id TEXT NOT NULL, - sequence INTEGER NOT NULL, - event_id TEXT NOT NULL UNIQUE, - protocol_version INTEGER NOT NULL, - event_type TEXT NOT NULL, - task_id TEXT, - payload BLOB NOT NULL, - created_at TEXT NOT NULL, - PRIMARY KEY(session_id, sequence) -); -CREATE TABLE IF NOT EXISTS session_delivery_cursors ( - cursor_key TEXT PRIMARY KEY, - binding BLOB NOT NULL, - acknowledged_sequence INTEGER NOT NULL, - owner_epoch INTEGER NOT NULL DEFAULT 1, - lease_until TEXT NOT NULL DEFAULT '', - retry_at TEXT NOT NULL DEFAULT '', - updated_at TEXT NOT NULL -); -CREATE TABLE IF NOT EXISTS session_poison ( - poison_key TEXT PRIMARY KEY, - record BLOB NOT NULL -); -CREATE TABLE IF NOT EXISTS session_poison_audits ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - record BLOB NOT NULL -);` - if _, err := l.db.Exec(schema); err != nil { - return fmt.Errorf("initialize session event log: %w", err) - } - if _, err := l.db.Exec(`ALTER TABLE session_event_partitions ADD COLUMN watermark INTEGER NOT NULL DEFAULT 0`); err != nil && - !strings.Contains(strings.ToLower(err.Error()), "duplicate column") { - return fmt.Errorf("migrate session event partition watermark: %w", err) - } - for _, migration := range []struct { - column string - definition string - }{ - {"owner_epoch", "INTEGER NOT NULL DEFAULT 1"}, - {"lease_until", "TEXT NOT NULL DEFAULT ''"}, - {"retry_at", "TEXT NOT NULL DEFAULT ''"}, - } { - query := fmt.Sprintf("ALTER TABLE session_delivery_cursors ADD COLUMN %s %s", migration.column, migration.definition) - if _, err := l.db.Exec(query); err != nil && - !strings.Contains(strings.ToLower(err.Error()), "duplicate column") { - return fmt.Errorf("migrate session delivery cursor %s: %w", migration.column, err) - } - } - return nil -} - -func (l *SessionEventLog) loadDatabase() error { - if err := l.loadPartitions(); err != nil { - return err - } - if err := l.loadEvents(); err != nil { - return err - } - if err := l.loadCursors(); err != nil { - return err - } - if err := l.loadPoisonState(); err != nil { - return err - } - return nil -} - -func (l *SessionEventLog) loadPartitions() error { - rows, err := l.db.Query(`SELECT session_id, watermark, terminal FROM session_event_partitions`) - if err != nil { - return fmt.Errorf("load session partitions: %w", err) - } - for rows.Next() { - var sessionID string - var watermark uint64 - var terminal bool - if err := rows.Scan(&sessionID, &watermark, &terminal); err != nil { - _ = rows.Close() - return fmt.Errorf("scan session partition: %w", err) - } - l.state.Sessions[sessionID] = &sessionEventPartition{Watermark: watermark, Terminal: terminal} - } - if err := rows.Err(); err != nil { - _ = rows.Close() - return fmt.Errorf("iterate session partitions: %w", err) - } - if err := rows.Close(); err != nil { - return fmt.Errorf("close session partitions: %w", err) - } - return nil -} - -func (l *SessionEventLog) loadEvents() error { - rows, err := l.db.Query(`SELECT session_id, sequence, event_id, protocol_version, event_type, task_id, payload, created_at FROM session_events ORDER BY session_id, sequence`) - if err != nil { - return fmt.Errorf("load session events: %w", err) - } - for rows.Next() { - event, err := scanSessionEvent(rows) - if err != nil { - _ = rows.Close() - return err - } - partition := l.state.Sessions[event.SessionID] - if partition == nil { - partition = &sessionEventPartition{} - l.state.Sessions[event.SessionID] = partition - } - partition.Events = append(partition.Events, event) - if event.Sequence > partition.Watermark { - partition.Watermark = event.Sequence - } - } - if err := rows.Err(); err != nil { - _ = rows.Close() - return fmt.Errorf("iterate session events: %w", err) - } - if err := rows.Close(); err != nil { - return fmt.Errorf("close session events: %w", err) - } - return nil -} - -func scanSessionEvent(rows *sql.Rows) (SessionEvent, error) { - var event SessionEvent - var taskID sql.NullString - var createdAt string - if err := rows.Scan(&event.SessionID, &event.Sequence, &event.ID, &event.ProtocolVersion, &event.EventType, &taskID, &event.Payload, &createdAt); err != nil { - return SessionEvent{}, fmt.Errorf("scan session event: %w", err) - } - if taskID.Valid { - event.TaskID = &taskID.String - } - parsedCreatedAt, err := time.Parse(time.RFC3339Nano, createdAt) - if err != nil { - return SessionEvent{}, fmt.Errorf("parse session event time: %w", err) - } - event.CreatedAt = parsedCreatedAt - return event, nil -} - -func (l *SessionEventLog) loadCursors() error { - err := l.loadJSONRows(`SELECT cursor_key, binding, acknowledged_sequence, owner_epoch, lease_until, retry_at, updated_at FROM session_delivery_cursors`, func(columns []any) error { - encodedKey := columns[0].(string) - var key SessionDeliveryCursorKey - if err := json.Unmarshal(columns[1].([]byte), &key); err != nil { - return err - } - leaseUntil, err := parseOptionalTime(columns[4]) - if err != nil { - return err - } - retryAt, err := parseOptionalTime(columns[5]) - if err != nil { - return err - } - updatedAt, err := time.Parse(time.RFC3339Nano, columns[6].(string)) - if err != nil { - return err - } - l.state.Cursors[encodedKey] = &SessionDeliveryCursor{ - Key: key, AcknowledgedSequence: uint64(columns[2].(int64)), OwnerEpoch: uint64(columns[3].(int64)), - LeaseUntil: leaseUntil, RetryAt: retryAt, UpdatedAt: updatedAt, - } - return nil - }) - if err != nil { - return fmt.Errorf("load delivery cursors: %w", err) - } - return nil -} - -func (l *SessionEventLog) loadPoisonState() error { - if err := l.loadJSONRows(`SELECT poison_key, record FROM session_poison`, func(columns []any) error { - var record SessionPoisonRecord - if err := json.Unmarshal(columns[1].([]byte), &record); err != nil { - return err - } - l.state.Poison[columns[0].(string)] = &record - return nil - }); err != nil { - return fmt.Errorf("load poison records: %w", err) - } - if err := l.loadJSONRows(`SELECT record FROM session_poison_audits ORDER BY id`, func(columns []any) error { - var audit SessionPoisonAudit - if err := json.Unmarshal(columns[0].([]byte), &audit); err != nil { - return err - } - l.state.Audits = append(l.state.Audits, audit) - return nil - }); err != nil { - return fmt.Errorf("load poison audits: %w", err) - } - return nil -} - -func (l *SessionEventLog) loadJSONRows(query string, consume func([]any) error) error { - rows, err := l.db.Query(query) - if err != nil { - return err - } - defer func() { _ = rows.Close() }() - columnNames, err := rows.Columns() - if err != nil { - return err - } - for rows.Next() { - values := make([]any, len(columnNames)) - destinations := make([]any, len(columnNames)) - for index := range values { - destinations[index] = &values[index] - } - if err := rows.Scan(destinations...); err != nil { - return err - } - if err := consume(values); err != nil { - return err - } - } - return rows.Err() -} -func nullableTime(value time.Time) any { - if value.IsZero() { - return "" - } - return value.UTC().Format(time.RFC3339Nano) -} - -func parseOptionalTime(value any) (time.Time, error) { - var raw string - switch typed := value.(type) { - case string: - raw = typed - case []byte: - raw = string(typed) - case nil: - return time.Time{}, nil - default: - return time.Time{}, fmt.Errorf("unexpected optional time type %T", value) - } - if raw == "" { - return time.Time{}, nil - } - return time.Parse(time.RFC3339Nano, raw) -} - -func nullableString(value *string) any { - if value == nil { - return nil - } - return *value -} - -func mustJSON(value any) []byte { - encoded, err := json.Marshal(value) - if err != nil { - panic(err) - } - return encoded -} - -func poisonKey(sessionID, eventID string) string { - return sessionID + "\x00" + eventID -} - -func deliveryCursorKey(key SessionDeliveryCursorKey) string { - return strings.Join([]string{ - key.SessionID, - key.ConsumerKind, - key.ConsumerID, - key.WireID, - key.PluginID, - fmt.Sprintf("%d", key.Generation), - key.UserID, - }, "\x00") -} diff --git a/apps/backend/internal/plugins/conversation_stream_service.go b/apps/backend/internal/plugins/conversation_stream_service.go deleted file mode 100644 index cd4cdeceff..0000000000 --- a/apps/backend/internal/plugins/conversation_stream_service.go +++ /dev/null @@ -1,125 +0,0 @@ -package plugins - -import ( - "errors" - "time" -) - -var ( - ErrConversationBindingInvalid = errors.New("invalid conversation binding") - ErrConversationGenerationSuperseded = errors.New("conversation generation superseded") -) - -// SessionEvents returns the service-owned durable stream store used by the -// production WebSocket gateway. -func (s *Service) SessionEvents() *SessionEventLog { - return s.sessionEvents -} - -// isSessionRemoved reports whether the durable stream has a terminal removal -// marker for sessionID. It is the only journal fallback authorization after -// the task-service session lookup no longer succeeds. -func (s *Service) isSessionRemoved(sessionID string) bool { - if s.sessionEvents == nil { - return false - } - _, _, terminal := s.sessionEvents.ReplayState(sessionID, 0) - return terminal -} - -// SessionDelivery returns the service-owned poison delivery dispatcher. -func (s *Service) SessionDelivery() *SessionDeliveryDispatcher { - return s.sessionDelivery -} - -// AuthorizeConversationConsumer validates an active capability declaration and -// the loader-issued user/plugin/generation binding without exposing token data. -func (s *Service) AuthorizeConversationConsumer( - pluginID string, - userID string, - generation int64, - bindingToken string, -) error { - record, err := s.Get(pluginID) - if err != nil || record.Status != StatusActive || !record.Capabilities.CanRead("messages") { - return ErrConversationBindingInvalid - } - if conversationGeneration(record.InstalledAt) != generation { - return ErrConversationGenerationSuperseded - } - claims, err := s.conversationTokens.parse(bindingToken) - if err != nil || - claims.Kind != tokenKindBinding || - claims.PluginID != pluginID || - claims.UserID != userID || - claims.Generation != generation { - return ErrConversationBindingInvalid - } - return nil -} - -func (s *Service) MintSessionStreamGrant( - pluginID string, - userID string, - generation int64, - sessionID string, - consumerID string, - wireID string, - snapshotCutoff uint64, - resumeCursor uint64, -) (snapshotToken string, resumeToken string, expiresAt time.Time, err error) { - fingerprint := consumerID - if fingerprint == "" { - fingerprint = wireID - } - snapshotToken, err = s.conversationTokens.mintSnapshot( - pluginID, - userID, - generation, - sessionID, - nil, - "", - nil, - int64(snapshotCutoff), - fingerprint, - ) - if err != nil { - return "", "", time.Time{}, err - } - resumeToken, err = s.conversationTokens.mintResume( - pluginID, - userID, - generation, - sessionID, - consumerID, - wireID, - resumeCursor, - ) - if err != nil { - return "", "", time.Time{}, err - } - return snapshotToken, resumeToken, time.Now().UTC().Add(conversationTokenTTL), nil -} - -func (s *Service) ValidateSessionResume( - token string, - key SessionDeliveryCursorKey, - sequence uint64, -) error { - if token == "" { - return ErrConversationBindingInvalid - } - claims, err := s.conversationTokens.parse(token) - if err != nil || - claims.Kind != tokenKindResume || - claims.PluginID != key.PluginID || - claims.UserID != key.UserID || - claims.Generation != key.Generation || - claims.SessionID != key.SessionID || - claims.ConsumerID != key.ConsumerID || - claims.WireID != key.WireID || - claims.Sequence > sequence { - return ErrConversationBindingInvalid - } - return nil -} diff --git a/apps/backend/internal/plugins/conversation_stream_test.go b/apps/backend/internal/plugins/conversation_stream_test.go deleted file mode 100644 index e7f3f1e578..0000000000 --- a/apps/backend/internal/plugins/conversation_stream_test.go +++ /dev/null @@ -1,439 +0,0 @@ -package plugins - -import ( - "context" - "encoding/json" - "path/filepath" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/require" -) - -func TestSessionEventLogPersistsSequenceAndTerminalRemoval(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - first, err := log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - removed, err := log.Append("session-1", stringPtr("task-1"), "session.removed", validSessionRemovedPayload()) - require.NoError(t, err) - require.Equal(t, uint64(1), first.Sequence) - require.Equal(t, uint64(2), removed.Sequence) - require.NoError(t, log.Close()) - - reopened, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, reopened.Close()) }) - events, terminal := reopened.EventsAfter("session-1", 0) - require.True(t, terminal) - require.Equal(t, []uint64{1, 2}, []uint64{events[0].Sequence, events[1].Sequence}) - _, err = reopened.Append("session-1", stringPtr("task-1"), "message.updated", json.RawMessage(`{"type":"message.updated","id":"m1"}`)) - require.ErrorIs(t, err, ErrSessionRemoved) -} - -func TestSessionEventLogPoisonBlocksAckUntilAuditedRequeue(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - dispatcher := NewSessionDeliveryDispatcher(log) - _, err = log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - poison, err := log.Append("session-1", stringPtr("task-1"), "unknown.event", json.RawMessage(`{"type":"unknown.event","session_id":"session-1","task_id":"task-1"}`)) - require.NoError(t, err) - cursor := testCursor() - require.NoError(t, log.RegisterCursor(cursor, 0)) - require.NoError(t, log.Acknowledge(cursor, 1)) - require.ErrorIs(t, log.Acknowledge(cursor, 2), ErrPoisonEvent) - base := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - for attempt := range SessionPoisonMaxAttempts { - claimAndCompletePoison(t, dispatcher, "session-1", poison.ID, base.Add(time.Duration(attempt)*time.Minute), true) - } - record, ok := log.Poison("session-1", poison.ID) - require.True(t, ok) - require.Equal(t, SessionPoisonExhausted, record.State) - require.Equal(t, SessionPoisonMaxAttempts, record.Attempts) - require.ErrorIs(t, dispatcher.Requeue("session-1", poison.ID, record.OwnerEpoch+1, "admin-1", base), ErrStaleOwnerEpoch) - require.NoError(t, dispatcher.Requeue("session-1", poison.ID, record.OwnerEpoch, "admin-1", base)) -} - -func TestSessionEventPoisonDeliveryClaimControlsLeaseBackoffExhaustionAndRequeue(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - dispatcher := NewSessionDeliveryDispatcher(log) - plain, err := log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("plain")) - require.NoError(t, err) - plainClaim, err := dispatcher.Claim("session-1", plain.ID, time.Now()) - require.NoError(t, err) - require.Equal(t, SessionDeliveryUntracked, plainClaim.Disposition) - - poison, err := log.Append("session-1", stringPtr("task-1"), "unknown.event", json.RawMessage(`{"type":"unknown.event","session_id":"session-1","task_id":"task-1"}`)) - require.NoError(t, err) - base := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - first := claimAndCompletePoison(t, dispatcher, "session-1", poison.ID, base, true) - require.Equal(t, SessionDeliveryClaimed, first.Disposition) - leased, err := dispatcher.Claim("session-1", poison.ID, base.Add(time.Second)) - require.NoError(t, err) - require.Equal(t, SessionDeliveryLeased, leased.Disposition) - backoff, err := dispatcher.Claim("session-1", poison.ID, base.Add(SessionPoisonLease)) - require.NoError(t, err) - require.Equal(t, SessionDeliveryBackoff, backoff.Disposition) - - for attempt := 1; attempt < SessionPoisonMaxAttempts; attempt++ { - claimAndCompletePoison(t, dispatcher, "session-1", poison.ID, base.Add(time.Duration(attempt)*time.Minute), true) - } - exhausted, err := dispatcher.Claim("session-1", poison.ID, base.Add(10*time.Minute)) - require.NoError(t, err) - require.Equal(t, SessionDeliveryExhausted, exhausted.Disposition) - record, ok := log.Poison("session-1", poison.ID) - require.True(t, ok) - require.Equal(t, SessionPoisonMaxAttempts, record.Attempts) - - require.NoError(t, dispatcher.Requeue("session-1", poison.ID, record.OwnerEpoch, "admin-1", base.Add(11*time.Minute))) - requeued := claimAndCompletePoison(t, dispatcher, "session-1", poison.ID, base.Add(11*time.Minute), true) - require.Equal(t, SessionDeliveryClaimed, requeued.Disposition) - record, ok = log.Poison("session-1", poison.ID) - require.True(t, ok) - require.Equal(t, 1, record.Attempts) -} - -func claimAndCompletePoison( - t *testing.T, - dispatcher *SessionDeliveryDispatcher, - sessionID string, - eventID string, - at time.Time, - queued bool, -) SessionDeliveryClaim { - t.Helper() - claim, err := dispatcher.Claim(sessionID, eventID, at) - require.NoError(t, err) - require.Equal(t, SessionDeliveryClaimed, claim.Disposition) - require.NoError(t, dispatcher.Complete(claim, queued, at)) - return claim -} - -func TestSessionEventRetentionKeepsTerminalTombstoneThroughBothBounds(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - log.now = func() time.Time { return start } - _, err = log.Append("session-1", stringPtr("task-1"), "session.removed", validSessionRemovedPayload()) - require.NoError(t, err) - require.NoError(t, log.CollectExpired(start.Add(SessionEventRetention-time.Second))) - _, terminal := log.EventsAfter("session-1", 0) - require.True(t, terminal) - require.NoError(t, log.CollectExpired(start.Add(SessionEventRetention))) - events, terminal := log.EventsAfter("session-1", 0) - require.False(t, terminal) - require.Empty(t, events) -} - -func TestSessionEventResyncAfterTerminalCollectionHealsForwardGap(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - log.now = func() time.Time { return start } - removed, err := log.AppendCommitted(SessionEvent{ - SessionID: "session-1", TaskID: stringPtr("task-1"), Sequence: 3, - ID: "session-1:3", ProtocolVersion: SessionEventProtocolVersion, EventType: "session.removed", - Payload: validSessionRemovedPayload(), CreatedAt: start, - }) - require.NoError(t, err) - require.True(t, removed) - - // After retention both bounds lapse the local partition is dropped: the - // terminal tombstone is gone from the durable log. - require.NoError(t, log.CollectExpired(start.Add(SessionEventRetention+time.Hour))) - _, terminal := log.EventsAfter("session-1", 0) - require.False(t, terminal) - - // The primary journal keeps only its session.removed row (all other rows - // are pruned), which is not at sequence 1. Re-mirroring it must heal the - // gap instead of failing forever. - removed, err = log.AppendCommitted(SessionEvent{ - SessionID: "session-1", TaskID: stringPtr("task-1"), Sequence: 3, - ID: "session-1:3", ProtocolVersion: SessionEventProtocolVersion, EventType: "session.removed", - Payload: validSessionRemovedPayload(), CreatedAt: start.Add(time.Hour), - }) - require.NoError(t, err) - require.True(t, removed) - events, terminal := log.EventsAfter("session-1", 0) - require.True(t, terminal) - require.Len(t, events, 1) - require.Equal(t, uint64(3), events[0].Sequence) -} - -func TestSessionEventResyncAfterTruncationHealsForwardGap(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - log.now = func() time.Time { return start } - // seq 1 and 2 mirror fine; then the idle partition's rows age out and are - // truncated while the primary keeps a newer row at seq 4 (seq 3 pruned). - for _, sequence := range []uint64{1, 2} { - _, err := log.AppendCommitted(SessionEvent{ - SessionID: "session-1", TaskID: stringPtr("task-1"), Sequence: sequence, - ID: "session-1:" + strconv.FormatUint(sequence, 10), ProtocolVersion: SessionEventProtocolVersion, EventType: "message.added", - Payload: validMessageAddedPayload("m" + strconv.FormatUint(sequence, 10)), CreatedAt: start, - }) - require.NoError(t, err) - } - require.NoError(t, log.CollectExpired(start.Add(SessionEventRetention+time.Hour))) - _, terminal := log.EventsAfter("session-1", 0) - require.False(t, terminal) - require.Equal(t, uint64(2), log.Watermark("session-1")) - - appended, err := log.AppendCommitted(SessionEvent{ - SessionID: "session-1", TaskID: stringPtr("task-1"), Sequence: 4, - ID: "session-1:4", ProtocolVersion: SessionEventProtocolVersion, EventType: "message.added", - Payload: validMessageAddedPayload("m4"), CreatedAt: start.Add(time.Hour), - }) - require.NoError(t, err) - require.True(t, appended) - require.Equal(t, uint64(4), log.Watermark("session-1")) -} - -func TestSessionEventRetentionExpiresExhaustedPoisonState(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - log.now = func() time.Time { return start } - event, err := log.Append("session-1", stringPtr("task-1"), "unknown.event", json.RawMessage(`{"type":"unknown.event","session_id":"session-1","task_id":"task-1"}`)) - require.NoError(t, err) - dispatcher := NewSessionDeliveryDispatcher(log) - for attempt := range SessionPoisonMaxAttempts { - claimAndCompletePoison(t, dispatcher, "session-1", event.ID, start.Add(time.Duration(attempt)*time.Minute), true) - } - require.NoError(t, log.CollectExpired(start.Add(SessionEventRetention+5*time.Minute))) - _, ok := log.Poison("session-1", event.ID) - require.False(t, ok) - events, terminal := log.EventsAfter("session-1", 0) - require.False(t, terminal) - require.Empty(t, events) -} - -func TestSessionEventCursorRegistrationAndExplicitRelease(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - event, err := log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - key := testCursor() - require.NoError(t, log.RegisterCursor(key, event.Sequence)) - require.Contains(t, log.state.Cursors, deliveryCursorKey(key)) - require.NoError(t, log.ReleaseCursor(key)) - require.NotContains(t, log.state.Cursors, deliveryCursorKey(key)) -} - -func TestSessionEventCursorReplacementAdvancesExistingCursor(t *testing.T) { - log, err := NewSessionEventLog("") - require.NoError(t, err) - event, err := log.Append("session-1", stringPtr("task-1"), "unknown.event", json.RawMessage(`{"type":"unknown.event","session_id":"session-1","task_id":"task-1"}`)) - require.NoError(t, err) - key := testCursor() - require.NoError(t, log.RegisterCursor(key, 0)) - require.NoError(t, log.ReplaceCursor(key, event.Sequence)) - require.NoError(t, log.Acknowledge(key, event.Sequence)) -} - -func TestSessionEventCursorLifecycleFieldsSurviveRestart(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - key := testCursor() - key.Generation = 4 - require.NoError(t, log.RegisterCursor(key, 2)) - leaseUntil := time.Date(2026, 9, 7, 12, 5, 0, 0, time.UTC) - retryAt := time.Date(2026, 9, 7, 12, 6, 0, 0, time.UTC) - log.mu.Lock() - cursor := log.state.Cursors[deliveryCursorKey(key)] - cursor.LeaseUntil = leaseUntil - cursor.RetryAt = retryAt - cursor.OwnerEpoch = 9 - err = log.persistLocked() - log.mu.Unlock() - require.NoError(t, err) - - require.NoError(t, log.Close()) - - reopened, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, reopened.Close()) }) - restored := reopened.state.Cursors[deliveryCursorKey(key)] - require.Equal(t, uint64(9), restored.OwnerEpoch) - require.Equal(t, leaseUntil, restored.LeaseUntil) - require.Equal(t, retryAt, restored.RetryAt) -} - -func TestSessionEventMaintenanceDoesNotCountUndeliveredPoisonAttempts(t *testing.T) { - service := NewService(nil, NewRegistry(), nil, testLogger(t)) - start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - poison, err := service.SessionEvents().Append("session-1", stringPtr("task-1"), "unknown.event", json.RawMessage(`{"type":"unknown.event","session_id":"session-1","task_id":"task-1"}`)) - require.NoError(t, err) - // Maintenance runs many times; without any actual delivery attempt the - // poison must stay pending at zero attempts instead of being exhausted. - for _, at := range []time.Time{start, start.Add(SessionPoisonLease), start.Add(2 * SessionPoisonLease)} { - require.NoError(t, service.maintainSessionEvents(context.Background(), at)) - record, ok := service.SessionEvents().Poison("session-1", poison.ID) - require.True(t, ok) - require.Equal(t, 0, record.Attempts) - require.Equal(t, SessionPoisonPending, record.State) - } - // A real queued delivery advances the attempt bookkeeping. - claimAndCompletePoison(t, service.SessionDelivery(), "session-1", poison.ID, start.Add(3*SessionPoisonLease), true) - record, ok := service.SessionEvents().Poison("session-1", poison.ID) - require.True(t, ok) - require.Equal(t, 1, record.Attempts) - require.Equal(t, SessionPoisonLeased, record.State) -} - -func TestSessionEventCursorReplacementRestoresMemoryOnPersistFailure(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - key := testCursor() - require.NoError(t, log.RegisterCursor(key, 0)) - event, err := log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - // Close the held database before redirecting the path so persistence - // failures exercise the closed-log guard. - require.NoError(t, log.Close()) - log.path = filepath.Join(t.TempDir(), "missing", "session-events.db") - - require.Error(t, log.ReplaceCursor(key, event.Sequence)) - - // The in-memory mutation rolled back, so a restart sees the committed - // acknowledgement, not the failed replacement. - restored := log.state.Cursors[deliveryCursorKey(key)] - require.Equal(t, uint64(0), restored.AcknowledgedSequence) - require.Equal(t, uint64(1), restored.OwnerEpoch) - require.NoError(t, log.Close()) - - reopened, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, reopened.Close()) }) - require.Equal(t, uint64(0), reopened.state.Cursors[deliveryCursorKey(key)].AcknowledgedSequence) -} - -func TestSessionEventPoisonAttemptRestoresMemoryOnPersistFailure(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - dispatcher := NewSessionDeliveryDispatcher(log) - poison, err := log.Append("session-1", stringPtr("task-1"), "unknown.event", json.RawMessage(`{"type":"unknown.event","session_id":"session-1","task_id":"task-1"}`)) - require.NoError(t, err) - require.NoError(t, log.Close()) - log.path = filepath.Join(t.TempDir(), "missing", "session-events.db") - - base := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - _, err = dispatcher.Claim("session-1", poison.ID, base) - require.Error(t, err) - - record, ok := log.Poison("session-1", poison.ID) - require.True(t, ok) - require.Equal(t, 0, record.Attempts) - require.Equal(t, SessionPoisonPending, record.State) -} - -func TestSessionEventCollectionRestoresMemoryOnPersistFailure(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - start := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - log.now = func() time.Time { return start } - _, err = log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - _, err = log.Append("session-1", stringPtr("task-1"), "session.removed", validSessionRemovedPayload()) - require.NoError(t, err) - require.NoError(t, log.RegisterCursor(testCursor(), 0)) - require.NoError(t, log.Close()) - log.path = filepath.Join(t.TempDir(), "missing", "session-events.db") - - require.Error(t, log.CollectExpired(start.Add(SessionEventRetention+time.Hour))) - - // Nothing was collected in memory: the cursor, poison, and partition - // survive until persistence actually commits. - require.True(t, log.HasCursor(testCursor())) - events, terminal := log.EventsAfter("session-1", 0) - require.True(t, terminal) - require.Len(t, events, 2) -} - -func testCursor() SessionDeliveryCursorKey { - return SessionDeliveryCursorKey{SessionID: "session-1", ConsumerKind: "plugin", ConsumerID: "consumer-1", PluginID: "plugin-1", Generation: 1, UserID: "user-1"} -} - -func stringPtr(value string) *string { return &value } - -func validMessageAddedPayload(messageID string) json.RawMessage { - return json.RawMessage(`{"type":"message.added","session_id":"session-1","task_id":"task-1","message_id":"` + messageID + `","author_type":"user","content":"hello","message_type":"message","created_at":"2026-09-07T12:00:00Z"}`) -} - -func validSessionRemovedPayload() json.RawMessage { - return json.RawMessage(`{"type":"session.removed","session_id":"session-1","task_id":"task-1"}`) -} - -func TestSessionEventAcknowledgePersistsOnlyTheCursorRow(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - event, err := log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - _, err = log.AppendCommitted(SessionEvent{ - SessionID: "session-1", TaskID: stringPtr("task-1"), Sequence: event.Sequence + 1, - ID: "session-1:2", EventType: "message.added", ProtocolVersion: 1, - Payload: validMessageAddedPayload("m2"), CreatedAt: event.CreatedAt, - }) - require.NoError(t, err) - cursorA := testCursor() - cursorA.ConsumerID = "consumer-a" - cursorB := testCursor() - cursorB.ConsumerID = "consumer-b" - require.NoError(t, log.RegisterCursor(cursorA, 0)) - require.NoError(t, log.RegisterCursor(cursorB, 0)) - require.NoError(t, log.Acknowledge(cursorA, 2)) - - require.NoError(t, log.Close()) - reopened, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, reopened.Close()) }) - restoredA := reopened.state.Cursors[deliveryCursorKey(cursorA)] - require.Equal(t, uint64(2), restoredA.AcknowledgedSequence) - restoredB := reopened.state.Cursors[deliveryCursorKey(cursorB)] - require.Equal(t, uint64(0), restoredB.AcknowledgedSequence) - // The partition and both event rows survive the delta write untouched. - events, terminal := reopened.EventsAfter("session-1", 0) - require.False(t, terminal) - require.Len(t, events, 2) - require.Equal(t, uint64(2), reopened.Watermark("session-1")) -} - -func TestSessionEventAcknowledgeRestoresTimestampOnPersistFailure(t *testing.T) { - path := filepath.Join(t.TempDir(), "session-events.db") - log, err := NewSessionEventLog(path) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, log.Close()) }) - base := time.Date(2026, 9, 7, 12, 0, 0, 0, time.UTC) - log.now = func() time.Time { return base } - event, err := log.Append("session-1", stringPtr("task-1"), "message.added", validMessageAddedPayload("m1")) - require.NoError(t, err) - key := testCursor() - require.NoError(t, log.RegisterCursor(key, 0)) - previous := log.state.Cursors[deliveryCursorKey(key)].UpdatedAt - log.now = func() time.Time { return base.Add(time.Minute) } - - require.NoError(t, log.Close()) - require.Error(t, log.Acknowledge(key, event.Sequence)) - - restored := log.state.Cursors[deliveryCursorKey(key)] - require.Equal(t, uint64(0), restored.AcknowledgedSequence) - require.Equal(t, previous, restored.UpdatedAt) -} diff --git a/apps/backend/internal/plugins/conversation_tokens.go b/apps/backend/internal/plugins/conversation_tokens.go index 1b4ccc1ee1..0cac86c989 100644 --- a/apps/backend/internal/plugins/conversation_tokens.go +++ b/apps/backend/internal/plugins/conversation_tokens.go @@ -45,6 +45,8 @@ type conversationTokenClaims struct { ConsumerID string `json:"consumer_id,omitempty"` WireID string `json:"wire_id,omitempty"` Sequence uint64 `json:"sequence,omitempty"` + Epoch string `json:"epoch,omitempty"` + PageSize int `json:"page_size,omitempty"` Nonce string `json:"nonce"` } @@ -161,6 +163,24 @@ func (m *conversationTokenManager) mintSnapshot( }) } +func (m *conversationTokenManager) mintSourceCursor( + pluginID, userID string, + generation int64, + sessionID string, + taskID *string, + sortOrder string, + authors []string, + lastID, epoch string, + pageSize int, +) (string, error) { + return m.seal(conversationTokenClaims{ + Version: 1, Kind: tokenKindCursor, PluginID: pluginID, UserID: userID, + Generation: generation, ExpiresAt: m.now().UTC().Add(conversationTokenTTL).Unix(), + SessionID: sessionID, TaskID: taskID, Sort: sortOrder, Authors: authors, + LastID: lastID, Epoch: epoch, PageSize: pageSize, + }) +} + func (m *conversationTokenManager) renew(claims conversationTokenClaims) (string, time.Time, error) { expiresAt := m.now().UTC().Add(conversationTokenTTL) claims.ExpiresAt = expiresAt.Unix() diff --git a/apps/backend/internal/plugins/provider.go b/apps/backend/internal/plugins/provider.go index a810f902b5..8f9aeacd25 100644 --- a/apps/backend/internal/plugins/provider.go +++ b/apps/backend/internal/plugins/provider.go @@ -118,13 +118,8 @@ func ProvideWithStoreErrors(cfg *config.Config, dbPool *db.Pool, secrets SecretV configureWebAppStorage(cfg, svc, instanceStore, instanceState, log) svc.SetSecrets(secrets) - if dbPool != nil && dbPool.Writer() != nil { - svc.SetConversationJournalDB(dbPool.Writer()) - } if err := svc.SetPluginsDir(dir); err != nil { - warnProvider(log, "Plugins durable conversation state initialization failed; continuing with degraded conversation capabilities", err) - } else if _, err := svc.syncAllCommittedSessionEvents(context.Background()); err != nil { - warnProvider(log, "Plugins committed conversation journal synchronization failed; continuing with degraded conversation capabilities", err) + warnProvider(log, "Plugins conversation binding key initialization failed; continuing with degraded conversation capabilities", err) } seedMarketplace(svc, sourceStore, storeErrors, log) @@ -138,11 +133,8 @@ func ProvideWithStoreErrors(cfg *config.Config, dbPool *db.Pool, secrets SecretV if cfg.Features.Canvases { stopArtifactCleanup = svc.StartWebAppArtifactCleanupWorker(context.Background()) } - stopSessionEventMaintenance := svc.StartSessionEventMaintenanceWorker(context.Background()) - cleanup := func() error { stopArtifactCleanup() - stopSessionEventMaintenance() svc.closeWebAppEvents() return svc.Close() } diff --git a/apps/backend/internal/plugins/provider_test.go b/apps/backend/internal/plugins/provider_test.go index bf80c0e6cc..ddc8d68821 100644 --- a/apps/backend/internal/plugins/provider_test.go +++ b/apps/backend/internal/plugins/provider_test.go @@ -82,6 +82,62 @@ func TestSetPluginsDirKeepsInstallRootWhenConversationStateFails(t *testing.T) { } } +func TestSetPluginsDirRemovesLegacyConversationFiles(t *testing.T) { + dir := t.TempDir() + hostDir := filepath.Join(dir, ".host") + if err := os.MkdirAll(hostDir, 0o700); err != nil { + t.Fatalf("create host directory: %v", err) + } + for _, name := range []string{"session-events.sqlite", "session-events.sqlite-wal", "session-events.sqlite-shm"} { + if err := os.WriteFile(filepath.Join(hostDir, name), []byte("legacy"), 0o600); err != nil { + t.Fatalf("write legacy file %s: %v", name, err) + } + } + + svc := NewService(store.NewFSStore(filepath.Join(dir, "store")), NewRegistry(), nil, testLogger(t)) + t.Cleanup(func() { _ = svc.Close() }) + if err := svc.SetPluginsDir(dir); err != nil { + t.Fatalf("SetPluginsDir() unexpected error: %v", err) + } + for _, name := range []string{"session-events.sqlite", "session-events.sqlite-wal", "session-events.sqlite-shm"} { + if _, err := os.Stat(filepath.Join(hostDir, name)); !os.IsNotExist(err) { + t.Fatalf("legacy file %s still exists: %v", name, err) + } + } +} + +func TestSetPluginsDirRejectsLegacyConversationSymlink(t *testing.T) { + dir := t.TempDir() + hostDir := filepath.Join(dir, ".host") + if err := os.MkdirAll(hostDir, 0o700); err != nil { + t.Fatalf("create host directory: %v", err) + } + target := filepath.Join(dir, "outside.sqlite") + if err := os.WriteFile(target, []byte("keep"), 0o600); err != nil { + t.Fatalf("write target: %v", err) + } + legacy := filepath.Join(hostDir, "session-events.sqlite") + if err := os.Symlink(target, legacy); err != nil { + t.Fatalf("create legacy symlink: %v", err) + } + + svc := NewService(store.NewFSStore(filepath.Join(dir, "store")), NewRegistry(), nil, testLogger(t)) + t.Cleanup(func() { _ = svc.Close() }) + if err := svc.SetPluginsDir(dir); err == nil { + t.Fatal("SetPluginsDir() accepted a legacy conversation symlink") + } + if _, err := os.Lstat(legacy); err != nil { + t.Fatalf("legacy symlink was removed: %v", err) + } + contents, err := os.ReadFile(target) + if err != nil { + t.Fatalf("read symlink target: %v", err) + } + if string(contents) != "keep" { + t.Fatalf("symlink target changed to %q", contents) + } +} + func TestProvideLoadsExistingInstallationsFromDisk(t *testing.T) { homeDir := t.TempDir() cfg := &config.Config{HomeDir: homeDir} diff --git a/apps/backend/internal/plugins/service.go b/apps/backend/internal/plugins/service.go index 21238ed0cc..2906dbe0fa 100644 --- a/apps/backend/internal/plugins/service.go +++ b/apps/backend/internal/plugins/service.go @@ -5,12 +5,11 @@ import ( "errors" "fmt" "net/http" + "os" "path/filepath" "strings" "sync" - "time" - "github.com/jmoiron/sqlx" "go.uber.org/zap" "github.com/google/uuid" @@ -94,27 +93,24 @@ type Service struct { // directory belongs to an install still waiting for it. extractingPaths map[string]int - pluginsDir string - store store.Store - approvals *approvalLedger - registry *Registry - state *state.Store - userState *state.UserStore - instances *instances.Store - instanceState *state.InstanceStore - webArtifacts *webapp.ArtifactStore - webRuntime *webapp.Runtime - eventHub *webapp.EventHub - eventSubscription bus.Subscription - userStateCleanup userStateCleanupStore - agentConvs AgentConversationService - eventBus bus.EventBus - conversationTokens *conversationTokenManager - sessionEvents *SessionEventLog - sessionDelivery *SessionDeliveryDispatcher - conversationJournal *sqlx.DB - sessionEventSink func(SessionEvent) - log *logger.Logger + pluginsDir string + store store.Store + approvals *approvalLedger + registry *Registry + state *state.Store + userState *state.UserStore + instances *instances.Store + instanceState *state.InstanceStore + webArtifacts *webapp.ArtifactStore + webRuntime *webapp.Runtime + eventHub *webapp.EventHub + eventSubscription bus.Subscription + userStateCleanup userStateCleanupStore + agentConvs AgentConversationService + eventBus bus.EventBus + conversationTokens *conversationTokenManager + conversationEpoch string + log *logger.Logger deliverer Deliverer agentToolCatalogListener AgentToolCatalogListener @@ -195,8 +191,6 @@ type Service struct { reservedReferenceProviderKinds map[string]struct{} } -const sessionEventMaintenanceInterval = time.Minute - // ReferenceIdentity reserves a host-owned composer source and its canonical // provider/kind pair so a plugin cannot shadow a built-in integration. type ReferenceIdentity struct { @@ -209,7 +203,6 @@ type ReferenceIdentity struct { // Provide is the usual entry point in production; NewService is exposed // directly for tests that want a fake store.Store/PluginRuntime. func NewService(pluginStore store.Store, registry *Registry, eventBus bus.EventBus, log *logger.Logger) *Service { - sessionEvents := mustSessionEventLog() service := &Service{ store: pluginStore, registry: registry, @@ -221,12 +214,18 @@ func NewService(pluginStore store.Store, registry *Registry, eventBus bus.EventB agentToolGeneration: uuid.NewString(), eventHub: webapp.NewEventHub(), conversationTokens: newConversationTokenManager(), - sessionEvents: sessionEvents, + conversationEpoch: uuid.NewString(), } - service.sessionDelivery = NewSessionDeliveryDispatcher(sessionEvents) return service } +// ConversationEpoch identifies this backend process for Host-only source +// reconciliation. A restart starts a new epoch, so a client cannot treat a +// cursor or live batch from the previous process as current. +func (s *Service) ConversationEpoch() string { + return s.conversationEpoch +} + // SetGitCredentialLeaseRevoker wires immediate provider-lease revocation for // plugin lifecycle changes. The callback receives manifest-declared provider // IDs, never a plugin ID, because broker leases are scoped by provider. @@ -703,116 +702,54 @@ func (s *Service) Shutdown() { } } -// Close releases durable plugin conversation state after workers stop. +// Close stops plugin runtimes after workers stop. func (s *Service) Close() error { s.Shutdown() - if s.sessionEvents == nil { - return nil - } - return s.sessionEvents.Close() + return nil } // SetPluginsDir wires the root directory pkgtar.Install/pkgtar.Remove -// operate under and initializes mandatory durable conversation state. +// operate under and initializes the signing key for conversation bindings. func (s *Service) SetPluginsDir(dir string) error { - // Keep package installation rooted correctly even when durable conversation - // state initialization fails and the caller continues in degraded mode. + // Keep package installation rooted correctly even when binding setup fails + // and the caller continues in degraded mode. s.pluginsDir = dir s.approvals = newApprovalLedger(dir) hostDir := filepath.Join(dir, ".host") + if err := removeLegacyConversationFiles(hostDir); err != nil { + return err + } conversationTokens, err := loadOrCreateConversationTokenManager( filepath.Join(hostDir, "conversation-token.key"), ) if err != nil { return err } - sessionEvents, err := NewSessionEventLog(filepath.Join(hostDir, "session-events.sqlite")) - if err != nil { - return err - } - sessionDelivery := NewSessionDeliveryDispatcher(sessionEvents) - now := time.Now().UTC() - if err := sessionDelivery.ReclaimExpiredLeases(now); err != nil { - _ = sessionEvents.Close() - return fmt.Errorf("reclaim plugin session event leases: %w", err) - } - if err := sessionEvents.CollectExpired(now); err != nil { - _ = sessionEvents.Close() - return fmt.Errorf("collect expired plugin session events: %w", err) - } - previousSessionEvents := s.sessionEvents s.pluginsDir = dir s.conversationTokens = conversationTokens - s.sessionEvents = sessionEvents - s.sessionDelivery = sessionDelivery - if previousSessionEvents != nil { - _ = previousSessionEvents.Close() - } return nil } -func (s *Service) maintainSessionEvents(ctx context.Context, now time.Time) error { - // A failing partition must not freeze the rest of maintenance: mirror - // errors are isolated per session (syncAll collects them), and the - // remaining passes still run so healthy sessions are collected and the - // primary journal is pruned every tick. - events, syncErr := s.syncAllCommittedSessionEvents(ctx) - s.mu.Lock() - sink := s.sessionEventSink - s.mu.Unlock() - if sink != nil { - for _, event := range events { - sink(event) +func removeLegacyConversationFiles(hostDir string) error { + for _, name := range []string{"session-events.sqlite", "session-events.sqlite-wal", "session-events.sqlite-shm"} { + path := filepath.Join(hostDir, name) + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return fmt.Errorf("inspect legacy conversation file %s: %w", path, err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return fmt.Errorf("refusing to remove non-regular legacy conversation file %s", path) + } + if err := os.Remove(path); err != nil { + return fmt.Errorf("remove legacy conversation file %s: %w", path, err) } - } - if err := s.sessionDelivery.ReclaimExpiredLeases(now); err != nil { - return fmt.Errorf("reclaim plugin session event leases: %w", err) - } - // Poison attempts are counted only when delivery is actually observed - // (a hub fanout or replay attempt), never by the maintenance ticker. - // A poison no subscriber ever attempted to consume stays pending until - // retention reaps it instead of being exhaustively cycled blind. - if err := s.sessionEvents.CollectExpired(now); err != nil { - return fmt.Errorf("collect expired plugin session events: %w", err) - } - retained := s.sessionEvents.RetainedSessionIDs(now) - if err := s.pruneConversationJournal( - ctx, now.UTC().Add(-SessionEventRetention), retained, - ); err != nil { - return fmt.Errorf("prune primary conversation journal: %w", err) - } - if syncErr != nil { - return fmt.Errorf("synchronize committed conversation journal: %w", syncErr) } return nil } -// StartSessionEventMaintenanceWorker keeps cursor retention and expired poison -// leases bounded for the lifetime of the backend, not only during startup. -func (s *Service) StartSessionEventMaintenanceWorker(ctx context.Context) func() { - workerContext, cancel := context.WithCancel(ctx) - done := make(chan struct{}) - go func() { - defer close(done) - ticker := time.NewTicker(sessionEventMaintenanceInterval) - defer ticker.Stop() - for { - select { - case <-workerContext.Done(): - return - case now := <-ticker.C: - if err := s.maintainSessionEvents(workerContext, now.UTC()); err != nil && s.log != nil { - s.log.Error("plugins: maintain session event stream", zap.Error(err)) - } - } - } - }() - return func() { - cancel() - <-done - } -} - // RevealSecret resolves the cleartext value of the secret reference ref via // the shared secret vault. Returns an error if no vault was wired (e.g. a // test Service constructed via NewService directly) or if ref does not diff --git a/apps/backend/internal/task/models/conversation_source.go b/apps/backend/internal/task/models/conversation_source.go new file mode 100644 index 0000000000..0f33b41f94 --- /dev/null +++ b/apps/backend/internal/task/models/conversation_source.go @@ -0,0 +1,100 @@ +package models + +import "errors" + +// ErrConversationCursorStale indicates that a page boundary no longer exists +// in the current source state and the caller must restart reconciliation. +var ErrConversationCursorStale = errors.New("conversation cursor is stale") + +// ConversationMutationKind identifies the source operation represented by a +// transient mutation receipt. +type ConversationMutationKind string + +const ( + ConversationMutationUpsert ConversationMutationKind = "upsert" + ConversationMutationRemove ConversationMutationKind = "remove" +) + +// ConversationEntityKind identifies which source table an operation changes. +type ConversationEntityKind string + +const ( + ConversationEntityMessage ConversationEntityKind = "message" + ConversationEntityTurn ConversationEntityKind = "turn" +) + +// ConversationMutationOperation is the in-memory projection of one source +// mutation. PreviousSessionID is set for a move so a subscriber can remove +// the old selection before applying the new one. +type ConversationMutationOperation struct { + Kind ConversationMutationKind `json:"kind"` + Entity ConversationEntityKind `json:"entity"` + ID string `json:"id"` + SessionID string `json:"session_id"` + TaskID string `json:"task_id,omitempty"` + AuthorType string `json:"author_type,omitempty"` + PreviousSessionID string `json:"previous_session_id,omitempty"` + Message *Message `json:"message,omitempty"` + Turn *Turn `json:"turn,omitempty"` + // HadOutput is populated only on a turn-completion operation. It keeps the + // live core projection equivalent to the ordered turn.completed event + // without persisting a transient UI-only fact in the turn row. + HadOutput *bool `json:"had_output,omitempty"` +} + +// ConversationMutationReceipt describes a complete revision interval for +// one session. Receipts are transient and are published only after the source +// transaction commits. Complete is false when a caller changed more rows +// than it could describe, in which case delivery must request reconciliation. +type ConversationMutationReceipt struct { + SessionID string `json:"session_id"` + BaseRevision int64 `json:"base_revision"` + Revision int64 `json:"revision"` + Operations []ConversationMutationOperation `json:"operations"` + Complete bool `json:"complete"` +} + +// ConversationMessagePageRequest describes one bounded source-backed message +// read. CursorID is an opaque boundary to callers; the repository uses it only +// to apply the requested keyset comparison inside the read transaction. +type ConversationMessagePageRequest struct { + SessionID string + TaskID *string + Authors []string + Sort string + CursorID string + Limit int +} + +// ConversationMessagePage is a source snapshot and its consistency revision. +type ConversationMessagePage struct { + Messages []*Message + HasMore bool + Revision int64 + CursorID string +} + +// ConversationTurnPageRequest describes one bounded source-backed turn read. +type ConversationTurnPageRequest struct { + SessionID string + TaskID *string + Sort string + CursorID string + Limit int +} + +// ConversationTurnPage is a source snapshot and its consistency revision. +type ConversationTurnPage struct { + Turns []*Turn + HasMore bool + Revision int64 + CursorID string +} + +// ConversationRevision reports whether a session exists and its current +// source revision. Missing revision rows are revision zero. +type ConversationRevision struct { + SessionID string + Revision int64 + Exists bool +} diff --git a/apps/backend/internal/task/models/message_shell_output.go b/apps/backend/internal/task/models/message_shell_output.go index 150e2abdaf..e6fd4c7111 100644 --- a/apps/backend/internal/task/models/message_shell_output.go +++ b/apps/backend/internal/task/models/message_shell_output.go @@ -70,6 +70,9 @@ func shellOutputFromMetadata(metadata map[string]any) (ShellExecOutputSnapshot, if !shellOK { return ShellExecOutputSnapshot{}, false } + if output, outputOK := shell["output"].(map[string]any); outputOK && isProjectedShellOutput(output) { + return ShellExecOutputSnapshot{}, false + } return shellOutputFromMap(shell["output"]), true default: return ShellExecOutputSnapshot{}, false @@ -104,6 +107,19 @@ func shellOutputFromMap(raw any) ShellExecOutputSnapshot { return result } +func isProjectedShellOutput(output map[string]any) bool { + if _, hasStdout := output["stdout"]; hasStdout { + return false + } + if _, hasStderr := output["stderr"]; hasStderr { + return false + } + _, hasOutput := output["has_output"] + _, hasStdoutBytes := output["stdout_bytes"] + _, hasStderrBytes := output["stderr_bytes"] + return hasOutput || hasStdoutBytes || hasStderrBytes +} + func shellOutputSummary(output ShellExecOutputSnapshot) map[string]any { summary := map[string]any{ "has_output": output.Stdout != "" || output.Stderr != "", diff --git a/apps/backend/internal/task/models/message_shell_output_test.go b/apps/backend/internal/task/models/message_shell_output_test.go index d56d623dbd..318d9ab3d6 100644 --- a/apps/backend/internal/task/models/message_shell_output_test.go +++ b/apps/backend/internal/task/models/message_shell_output_test.go @@ -34,6 +34,10 @@ func TestMessageToAPIProjectsTypedShellOutputWithoutMutation(t *testing.T) { require.Equal(t, float64(len("warning\n")), output["stderr_bytes"]) require.Equal(t, float64(exitCode), output["exit_code"]) require.Equal(t, true, output["truncated"]) + projectedAgain := jsonMetadata(t, ProjectMessageMetadata(projected)) + outputAgain := projectedAgain["normalized"].(map[string]any)["shell_exec"].(map[string]any)["output"].(map[string]any) + require.Equal(t, true, outputAgain["has_output"]) + require.Equal(t, float64(len("pass ✓\n")), outputAgain["stdout_bytes"]) require.Equal(t, "pass ✓\n", normalized.ShellExec().Output.Stdout) require.Equal(t, "warning\n", normalized.ShellExec().Output.Stderr) } diff --git a/apps/backend/internal/task/models/models.go b/apps/backend/internal/task/models/models.go index 274ab0d6dd..8e16755088 100644 --- a/apps/backend/internal/task/models/models.go +++ b/apps/backend/internal/task/models/models.go @@ -527,6 +527,41 @@ func ClearPromptDispatchMetadata(metadata map[string]interface{}) { } } +// publicTurnMetadataKeys is the allowlist for first-party live turn +// projections. Prompt-dispatch recovery state and arbitrary repository-owned +// fields stay inside the service and are not sent through the shared event +// bus. The REST turn DTO has its own compatibility contract; this narrower +// projection is used only for the v2 Host conversation transport. +var publicTurnMetadataKeys = [...]string{ + TurnMetaKeyRuntimeConfigSnapshot, + TurnMetaKeyWorkflowStepIDAtStart, + TurnMetaKeyLifecycleOnly, + "prompt_usage", + "model", + "agent_id", + "agent_type", + "usage_multiplier", +} + +// ProjectTurnMetadata returns the first-party-safe metadata needed by live +// core conversation consumers without exposing arbitrary turn state. +func ProjectTurnMetadata(metadata map[string]interface{}) map[string]interface{} { + if len(metadata) == 0 { + return nil + } + projected := make(map[string]interface{}, len(publicTurnMetadataKeys)) + for _, key := range publicTurnMetadataKeys { + if value, ok := metadata[key]; ok { + projected[key] = value + } + } + ClearPromptDispatchMetadata(projected) + if len(projected) == 0 { + return nil + } + return projected +} + // PromptDispatchRecovery identifies the exact clarification claim that an // unpublished successor reservation must restore after a backend restart. type PromptDispatchRecovery struct { diff --git a/apps/backend/internal/task/repository/interface.go b/apps/backend/internal/task/repository/interface.go index cdb44ad6f8..b5345ff87d 100644 --- a/apps/backend/internal/task/repository/interface.go +++ b/apps/backend/internal/task/repository/interface.go @@ -323,6 +323,32 @@ type MessageRepository interface { DeleteMessage(ctx context.Context, id string) error } +// ConversationSourceRepository provides the current-state conversation reads +// used by the Host-only v2 transport. It is separate from MessageRepository +// so existing repository fakes can continue to model the broader message +// surface without also implementing the consistency boundary. +type ConversationSourceRepository interface { + ReadConversationRevision(context.Context, string) (models.ConversationRevision, error) + ReadConversationMessagesPage(context.Context, models.ConversationMessagePageRequest) (models.ConversationMessagePage, error) + ReadConversationTurnsPage(context.Context, models.ConversationTurnPageRequest) (models.ConversationTurnPage, error) +} + +// ConversationMutationWriter returns a transient receipt from the same +// transaction as a source mutation. It is optional during the migration so +// existing repository fakes and exceptional bulk writers remain operational. +type ConversationMutationWriter interface { + CreateMessageWithConversationReceipt(context.Context, *models.Message) (*models.ConversationMutationReceipt, error) + UpdateMessageWithConversationReceipt(context.Context, *models.Message) (*models.ConversationMutationReceipt, error) + DeleteMessageWithConversationReceipt(context.Context, string) (*models.ConversationMutationReceipt, error) + CreateTurnWithConversationReceipt(context.Context, *models.Turn) (*models.ConversationMutationReceipt, error) + UpdateTurnWithConversationReceipt(context.Context, *models.Turn) (*models.ConversationMutationReceipt, error) + CompleteTurnWithConversationReceipt(context.Context, string) (*models.ConversationMutationReceipt, error) +} + +type ConversationTurnStampWriter interface { + CreateTurnWithStepStampConversationReceipt(context.Context, *models.Turn) (bool, *models.ConversationMutationReceipt, error) +} + // AttachmentRepository stores file-backed prompt attachment descriptors. // Implementations must keep ownership and aggregate-claim checks in the same // transaction as state transitions so a retry cannot partially claim a batch. diff --git a/apps/backend/internal/task/repository/sqlite/base.go b/apps/backend/internal/task/repository/sqlite/base.go index 3f2b61e1be..63a206c005 100644 --- a/apps/backend/internal/task/repository/sqlite/base.go +++ b/apps/backend/internal/task/repository/sqlite/base.go @@ -45,6 +45,11 @@ type Repository struct { // ownership cutover. It is separate from the worktree failpoint because the // two migrations can be exercised independently in the same repository. failGitSnapshotCutoverAfter string + // failConversationJournalCleanupAfter is a test-only failpoint for the + // atomic removal of the retired conversation journal. It lets migration + // tests prove that a failed cleanup leaves the source tables and legacy + // tables unchanged for a retry. + failConversationJournalCleanupAfter string // failUsageEventAttempts/failUsageEventErr are a test-only failpoint for // CreateTaskUsageEvent's AC-32 transient-retry loop: while // failUsageEventAttempts > 0, insertUsageEventAndRollup returns diff --git a/apps/backend/internal/task/repository/sqlite/base_schema.go b/apps/backend/internal/task/repository/sqlite/base_schema.go index 99a02d2ffa..83879615b5 100644 --- a/apps/backend/internal/task/repository/sqlite/base_schema.go +++ b/apps/backend/internal/task/repository/sqlite/base_schema.go @@ -61,7 +61,8 @@ func (r *Repository) initSchemaContext(ctx context.Context) error { r.ensureWorkspaceIndexes, r.ensureMessageMetadataIndexes, r.ensurePromptOrderIndex, - r.initConversationJournalSchema, + r.initConversationSourceSchema, + r.cleanupLegacyConversationJournal, } // Every boundary is checked before and after its step. The task repository // passes the same context to startup SQL through migrationContext, so a diff --git a/apps/backend/internal/task/repository/sqlite/conversation_journal.go b/apps/backend/internal/task/repository/sqlite/conversation_journal.go deleted file mode 100644 index 869837c592..0000000000 --- a/apps/backend/internal/task/repository/sqlite/conversation_journal.go +++ /dev/null @@ -1,945 +0,0 @@ -//nolint:revive // This file keeps the dialect-specific journal trigger lifecycle atomic. -package sqlite - -import ( - "database/sql" - "encoding/json" - "fmt" - "strconv" - "strings" - "time" - - "github.com/jmoiron/sqlx" - "github.com/kandev/kandev/internal/db/dialect" - "github.com/kandev/kandev/internal/sysprompt" -) - -const conversationJournalTables = ` -CREATE TABLE IF NOT EXISTS conversation_session_streams ( - session_id TEXT PRIMARY KEY, - watermark BIGINT NOT NULL DEFAULT 0, - terminal BOOLEAN NOT NULL DEFAULT FALSE, - updated_at TIMESTAMP NOT NULL -); -CREATE TABLE IF NOT EXISTS conversation_session_events ( - session_id TEXT NOT NULL, - sequence BIGINT NOT NULL, - event_id TEXT NOT NULL UNIQUE, - protocol_version INTEGER NOT NULL DEFAULT 1, - event_type TEXT NOT NULL, - task_id TEXT, - payload TEXT NOT NULL, - created_at TIMESTAMP NOT NULL, - PRIMARY KEY (session_id, sequence) -); -CREATE INDEX IF NOT EXISTS idx_conversation_events_created - ON conversation_session_events(created_at); -CREATE TABLE IF NOT EXISTS conversation_message_versions ( - session_id TEXT NOT NULL, - message_id TEXT NOT NULL, - row_sequence BIGINT NOT NULL, - task_id TEXT, - author_type TEXT, - created_at TIMESTAMP NOT NULL, - tombstone BOOLEAN NOT NULL DEFAULT FALSE, - payload TEXT NOT NULL, - PRIMARY KEY (session_id, message_id, row_sequence) -); -CREATE INDEX IF NOT EXISTS idx_conversation_message_snapshot - ON conversation_message_versions(session_id, row_sequence, created_at, message_id); -CREATE TABLE IF NOT EXISTS conversation_turn_versions ( - session_id TEXT NOT NULL, - turn_id TEXT NOT NULL, - row_sequence BIGINT NOT NULL, - task_id TEXT, - started_at TIMESTAMP NOT NULL, - tombstone BOOLEAN NOT NULL DEFAULT FALSE, - payload TEXT NOT NULL, - PRIMARY KEY (session_id, turn_id, row_sequence) -); -CREATE INDEX IF NOT EXISTS idx_conversation_turn_snapshot - ON conversation_turn_versions(session_id, row_sequence, started_at, turn_id); -CREATE TABLE IF NOT EXISTS conversation_journal_meta ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL -); -` - -// journalSchemaVersion is the current conversation journal schema version. -// Boot runs the trigger (re)creation plus its one-time sanitize migration only -// while the stored version is below this constant, then records it, so the -// per-boot full-corpus payload rewrite never repeats. -const journalSchemaVersion = 4 - -// journalBackfillKey records that the one-time backfill of pre-trigger source -// rows completed, so boot does not re-scan the whole message/turn corpus. -const ( - // Repeated payload/column keys kept as constants for goconst. - jKeySessionID = "session_id" - jKeyTaskID = "task_id" -) - -const journalBackfillKey = "backfill.complete" - -func (r *Repository) initConversationJournalSchema() error { - if _, err := r.db.Exec(conversationJournalTables); err != nil { - return fmt.Errorf("create conversation journal tables: %w", err) - } - version, err := r.readConversationJournalMetaInt("schema.version") - if err != nil { - return err - } - if version < journalSchemaVersion { - var triggerErr error - if dialect.IsPostgres(r.db.DriverName()) { - triggerErr = r.initPostgresConversationJournalTriggers() - } else { - triggerErr = r.initSQLiteConversationJournalTriggers() - } - if triggerErr != nil { - return triggerErr - } - if err := r.writeConversationJournalMeta("schema.version", journalSchemaVersion); err != nil { - return err - } - } - backfilled, err := r.readConversationJournalMetaFlag(journalBackfillKey) - if err != nil { - return err - } - if backfilled { - return nil - } - if err := r.backfillConversationJournal(); err != nil { - return err - } - return r.writeConversationJournalMeta(journalBackfillKey, 1) -} - -func (r *Repository) readConversationJournalMetaInt(key string) (int, error) { - var value string - err := r.db.Get(&value, r.db.Rebind(`SELECT value FROM conversation_journal_meta WHERE key = ?`), key) - if err == sql.ErrNoRows { - return 0, nil - } - if err != nil { - return 0, fmt.Errorf("read conversation journal meta %s: %w", key, err) - } - parsed, err := strconv.Atoi(value) - if err != nil { - return 0, fmt.Errorf("parse conversation journal meta %s: %w", key, err) - } - return parsed, nil -} - -func (r *Repository) readConversationJournalMetaFlag(key string) (bool, error) { - value, err := r.readConversationJournalMetaInt(key) - if err != nil { - return false, err - } - return value > 0, nil -} - -func (r *Repository) writeConversationJournalMeta(key string, value int) error { - _, err := r.db.Exec(r.db.Rebind(` - INSERT INTO conversation_journal_meta(key, value) VALUES (?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value`), key, strconv.Itoa(value)) - if err != nil { - return fmt.Errorf("write conversation journal meta %s: %w", key, err) - } - return nil -} - -type conversationJournalMessageSeed struct { - ID string `db:"id"` - TaskSessionID string `db:"task_session_id"` - TaskID string `db:"task_id"` - TurnID string `db:"turn_id"` - AuthorType string `db:"author_type"` - AuthorID string `db:"author_id"` - Content string `db:"content"` - RequestsInput bool `db:"requests_input"` - MessageType string `db:"message_type"` - Metadata string `db:"metadata"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - PromptIndex int `db:"prompt_index"` -} - -type conversationJournalTurnSeed struct { - ID string `db:"id"` - TaskSessionID string `db:"task_session_id"` - TaskID string `db:"task_id"` - ExecutionProfileID string `db:"execution_profile_id"` - RouteGeneration int64 `db:"route_generation"` - Metadata string `db:"metadata"` - StartedAt time.Time `db:"started_at"` - CompletedAt *time.Time `db:"completed_at"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` -} - -const conversationJournalBackfillBatchSize = 100 - -//nolint:cyclop,dupl,gocognit,funlen // Turn and message backfills intentionally share bounded keyset control flow. -func (r *Repository) backfillConversationJournal() error { - var lastSessionID string - var lastStartedAt time.Time - var lastTurnID string - //nolint:dupl // Turn and message batches use the same transaction boundaries. - for { - tx, err := r.db.Beginx() - if err != nil { - return fmt.Errorf("begin conversation turn journal backfill: %w", err) - } - var turns []conversationJournalTurnSeed - query := ` - SELECT t.id, t.task_session_id, t.task_id, t.execution_profile_id, t.route_generation, - t.metadata, t.started_at, t.completed_at, t.created_at, t.updated_at - FROM task_session_turns t - WHERE NOT EXISTS ( - SELECT 1 FROM conversation_turn_versions v - WHERE v.session_id = t.task_session_id AND v.turn_id = t.id - )` - args := []any{} - if lastTurnID != "" { - query += ` - AND (t.task_session_id > ? OR - (t.task_session_id = ? AND - (t.started_at > ? OR (t.started_at = ? AND t.id > ?))))` - args = append(args, lastSessionID, lastSessionID, lastStartedAt, lastStartedAt, lastTurnID) - } - query += ` ORDER BY t.task_session_id, t.started_at, t.id LIMIT ?` - args = append(args, conversationJournalBackfillBatchSize) - if err := tx.Select(&turns, r.db.Rebind(query), args...); err != nil { - _ = tx.Rollback() - return fmt.Errorf("list conversation turn backfill: %w", err) - } - if len(turns) == 0 { - _ = tx.Rollback() - break - } - for _, turn := range turns { - if err := r.backfillConversationTurn(tx, turn); err != nil { - _ = tx.Rollback() - return err - } - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit conversation turn journal backfill: %w", err) - } - lastTurn := turns[len(turns)-1] - lastSessionID, lastStartedAt, lastTurnID = lastTurn.TaskSessionID, lastTurn.StartedAt, lastTurn.ID - } - - lastSessionID = "" - var lastCreatedAt time.Time - var lastMessageID string - for { - tx, err := r.db.Beginx() - if err != nil { - return fmt.Errorf("begin conversation message journal backfill: %w", err) - } - var messages []conversationJournalMessageSeed - query := ` - SELECT m.id, m.task_session_id, m.task_id, m.turn_id, m.author_type, m.author_id, - m.content, m.requests_input, m.type AS message_type, m.metadata, - m.created_at, m.updated_at, m.prompt_seq AS prompt_index - FROM task_session_messages m - WHERE NOT EXISTS ( - SELECT 1 FROM conversation_message_versions v - WHERE v.session_id = m.task_session_id AND v.message_id = m.id - )` - args := []any{} - if lastMessageID != "" { - query += ` - AND (m.task_session_id > ? OR - (m.task_session_id = ? AND - (m.created_at > ? OR (m.created_at = ? AND m.id > ?))))` - args = append(args, lastSessionID, lastSessionID, lastCreatedAt, lastCreatedAt, lastMessageID) - } - query += ` ORDER BY m.task_session_id, m.created_at, m.id LIMIT ?` - args = append(args, conversationJournalBackfillBatchSize) - if err := tx.Select(&messages, r.db.Rebind(query), args...); err != nil { - _ = tx.Rollback() - return fmt.Errorf("list conversation message backfill: %w", err) - } - if len(messages) == 0 { - _ = tx.Rollback() - break - } - for _, message := range messages { - if err := r.backfillConversationMessage(tx, message); err != nil { - _ = tx.Rollback() - return err - } - } - if err := tx.Commit(); err != nil { - return fmt.Errorf("commit conversation message journal backfill: %w", err) - } - lastMessage := messages[len(messages)-1] - lastSessionID, lastCreatedAt, lastMessageID = lastMessage.TaskSessionID, lastMessage.CreatedAt, lastMessage.ID - } - return nil -} - -func (r *Repository) nextConversationJournalSequence(tx *sqlx.Tx, sessionID string) (uint64, error) { - var sequence uint64 - query := r.db.Rebind(`UPDATE conversation_session_streams - SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP - WHERE session_id = ? RETURNING watermark`) - err := tx.Get(&sequence, query, sessionID) - if err == nil { - return sequence, nil - } - if err != sql.ErrNoRows { - return 0, fmt.Errorf("advance conversation journal sequence: %w", err) - } - query = r.db.Rebind(`INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (?, 1, FALSE, CURRENT_TIMESTAMP)`) - if _, err := tx.Exec(query, sessionID); err != nil { - return 0, fmt.Errorf("initialize conversation journal sequence: %w", err) - } - return 1, nil -} - -// sameTurnSourceImage reports whether the freshly read source row still -// matches the backfill seed for every payload-bound field. -// -//nolint:goconst // Backfill payload values mirror the public event contract. -func sameTurnSourceImage(current struct { - TaskID string `db:"task_id"` - ExecutionProfileID string `db:"execution_profile_id"` - RouteGeneration int64 `db:"route_generation"` - Metadata string `db:"metadata"` - StartedAt time.Time `db:"started_at"` - CompletedAt *time.Time `db:"completed_at"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` -}, turn conversationJournalTurnSeed) bool { - completedAtEqual := (current.CompletedAt == nil) == (turn.CompletedAt == nil) - if current.CompletedAt != nil && turn.CompletedAt != nil && !current.CompletedAt.Equal(*turn.CompletedAt) { - return false - } - return completedAtEqual && - current.TaskID == turn.TaskID && - current.ExecutionProfileID == turn.ExecutionProfileID && - current.RouteGeneration == turn.RouteGeneration && - current.Metadata == turn.Metadata && - current.StartedAt.Equal(turn.StartedAt) && - current.CreatedAt.Equal(turn.CreatedAt) && - current.UpdatedAt.Equal(turn.UpdatedAt) -} - -func (r *Repository) backfillConversationTurn(tx *sqlx.Tx, turn conversationJournalTurnSeed) error { - // Optimistic re-check of the complete source image: if any payload-bound - // field changed since the backfill SELECT, its trigger already journaled - // the current image, and a stale pre-update image must not win the - // snapshot with a later sequence. Comparing completion state alone would - // let a same-state update (metadata, profile, timestamps) pass. - var current struct { - TaskID string `db:"task_id"` - ExecutionProfileID string `db:"execution_profile_id"` - RouteGeneration int64 `db:"route_generation"` - Metadata string `db:"metadata"` - StartedAt time.Time `db:"started_at"` - CompletedAt *time.Time `db:"completed_at"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - } - err := tx.Get(¤t, r.db.Rebind(` - SELECT task_id, execution_profile_id, route_generation, metadata, - started_at, completed_at, created_at, updated_at - FROM task_session_turns WHERE id = ?`), turn.ID) - if err == sql.ErrNoRows { - return nil // deleted concurrently; its history is gone - } - if err != nil { - return fmt.Errorf("re-check conversation turn backfill %s: %w", turn.ID, err) - } - if !sameTurnSourceImage(current, turn) { - return nil // source image changed concurrently; trigger journaled it - } - sequence, err := r.nextConversationJournalSequence(tx, turn.TaskSessionID) - if err != nil { - return err - } - eventType := "session.turn.started" - if turn.CompletedAt != nil { - eventType = "session.turn.completed" - } - payload, err := json.Marshal(map[string]any{ - "type": eventType, jKeySessionID: turn.TaskSessionID, jKeyTaskID: journalTaskID(turn.TaskID), - "id": turn.ID, "execution_profile_id": turn.ExecutionProfileID, "route_generation": turn.RouteGeneration, - "started_at": turn.StartedAt.UTC().Format(time.RFC3339Nano), "completed_at": journalTime(turn.CompletedAt), - "metadata_json": turn.Metadata, columnCreatedAt: turn.CreatedAt.UTC().Format(time.RFC3339Nano), - columnUpdatedAt: turn.UpdatedAt.UTC().Format(time.RFC3339Nano), - }) - if err != nil { - return fmt.Errorf("encode conversation turn backfill: %w", err) - } - query := r.db.Rebind(`INSERT INTO conversation_turn_versions( - session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload - ) VALUES (?, ?, ?, ?, ?, FALSE, ?)`) - if _, err := tx.Exec(query, turn.TaskSessionID, turn.ID, sequence, journalTaskID(turn.TaskID), turn.StartedAt, payload); err != nil { - return fmt.Errorf("insert conversation turn backfill version: %w", err) - } - return r.insertConversationBackfillEvent(tx, turn.TaskSessionID, turn.TaskID, sequence, eventType, payload) -} - -//nolint:goconst // Backfill payload values mirror the public event contract. -func (r *Repository) backfillConversationMessage(tx *sqlx.Tx, message conversationJournalMessageSeed) error { - // Optimistic re-check of the complete source image: if any payload-bound - // field changed since the backfill SELECT, its trigger already journaled - // the current image, and a stale pre-update image must not win the - // snapshot with a later sequence. Comparing content alone would let a - // same-content update (metadata, task/turn/type/author, timestamps) pass. - var current struct { - TaskID string `db:"task_id"` - TurnID string `db:"turn_id"` - AuthorType string `db:"author_type"` - Content string `db:"content"` - Type string `db:"type"` - Metadata string `db:"metadata"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - } - err := tx.Get(¤t, r.db.Rebind(` - SELECT task_id, turn_id, author_type, content, type, metadata, created_at, updated_at - FROM task_session_messages WHERE id = ?`), message.ID) - if err == sql.ErrNoRows { - return nil // deleted concurrently; its tombstone was journaled - } - if err != nil { - return fmt.Errorf("re-check conversation message backfill %s: %w", message.ID, err) - } - if current.TaskID != message.TaskID || - current.TurnID != message.TurnID || - current.AuthorType != message.AuthorType || - current.Content != message.Content || - current.Type != message.MessageType || - current.Metadata != message.Metadata || - !current.CreatedAt.Equal(message.CreatedAt) || - !current.UpdatedAt.Equal(message.UpdatedAt) { - return nil // source image changed concurrently; trigger journaled it - } - sequence, err := r.nextConversationJournalSequence(tx, message.TaskSessionID) - if err != nil { - return err - } - var metadata map[string]any - _ = json.Unmarshal([]byte(message.Metadata), &metadata) - senderTaskID, _ := metadata["sender_task_id"].(string) - payload, err := json.Marshal(map[string]any{ - "type": "message.added", jKeySessionID: message.TaskSessionID, jKeyTaskID: journalTaskID(message.TaskID), - "message_id": message.ID, "turn_id": message.TurnID, "author_type": message.AuthorType, - "content": sysprompt.StripSystemContent(message.Content), - "message_type": message.MessageType, columnCreatedAt: message.CreatedAt.UTC().Format(time.RFC3339Nano), - columnUpdatedAt: message.UpdatedAt.UTC().Format(time.RFC3339Nano), "prompt_index": message.PromptIndex, - "sender_task_id": senderTaskID, - }) - if err != nil { - return fmt.Errorf("encode conversation message backfill: %w", err) - } - query := r.db.Rebind(`INSERT INTO conversation_message_versions( - session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload - ) VALUES (?, ?, ?, ?, ?, ?, FALSE, ?)`) - if _, err := tx.Exec(query, message.TaskSessionID, message.ID, sequence, journalTaskID(message.TaskID), message.AuthorType, message.CreatedAt, payload); err != nil { - return fmt.Errorf("insert conversation message backfill version: %w", err) - } - return r.insertConversationBackfillEvent(tx, message.TaskSessionID, message.TaskID, sequence, "message.added", payload) -} - -func (r *Repository) insertConversationBackfillEvent( - tx *sqlx.Tx, - sessionID string, - taskID string, - sequence uint64, - eventType string, - payload []byte, -) error { - query := r.db.Rebind(`INSERT INTO conversation_session_events( - session_id, sequence, event_id, protocol_version, event_type, task_id, payload, created_at - ) VALUES (?, ?, ?, 1, ?, ?, ?, CURRENT_TIMESTAMP)`) - if _, err := tx.Exec(query, sessionID, sequence, sessionID+":"+strconv.FormatUint(sequence, 10), eventType, journalTaskID(taskID), payload); err != nil { - return fmt.Errorf("insert conversation journal backfill event: %w", err) - } - return nil -} - -// sqliteStripSystemContentExpr returns a scalar subquery that removes every -// well-formed ... block from expr - including -// multi-line blocks, multiple blocks, and nested evasion - and trims the -// result, mirroring sysprompt.StripSystemContent. SQLite has no scalar regex -// replace, so a bounded recursive CTE reapplies first-block removal until no -// well-formed block remains. Depth 64 caps adversarial input. If another full -// block remains at that bound, the expression keeps only the prefix before it; -// dropping the unexamined suffix fails closed rather than persisting system -// content. The final row is always non-NULL for text input. -func sqliteStripSystemContentExpr(expr string) string { - // Go's helper also consumes the whitespace run after each closing tag - // (`\s*` in the pattern); ltrim the tail so junction spacing matches. - whitespace := "char(32) || char(9) || char(10) || char(13)" - return `(SELECT trim(x) FROM ( - WITH RECURSIVE strip(x, depth) AS ( - SELECT ` + expr + `, 0 - UNION ALL - SELECT substr(x, 1, instr(x, '') - 1) - || ltrim(substr(x, - instr(x, '') + length('') - + instr(substr(x, instr(x, '') + length('')), '') - 1 - + length('')), - ` + whitespace + `), depth + 1 - FROM strip - WHERE depth < 64 - AND instr(x, '') > 0 - AND instr(substr(x, instr(x, '') + length('')), '') > 0 - ) - SELECT CASE WHEN depth = 64 AND instr(x, '') > 0 AND instr(substr(x, instr(x, '') + length('')), '') > 0 THEN substr(x, 1, instr(x, '') - 1) ELSE x END AS x FROM strip ORDER BY depth DESC LIMIT 1 - ))` -} - -// sqliteStripSystemToken marks where a trigger or migration must insert the -// strip expression; the raw SQL literal keeps strftime %-formats untouched. -const sqliteStripSystemToken = "__SQLITE_STRIP_SYSTEM__" -const sqliteConversationMetadataToken = "__SQLITE_CONVERSATION_METADATA__" -const postgresConversationMetadataToken = "__POSTGRES_CONVERSATION_METADATA__" - -//nolint:goconst // These keys are an explicit privacy allowlist. -var conversationMessageMetadataKeys = []string{ - "action_details", "action_type", "action_visibility", "actions", "agent_disconnected", "attempt", "attachments", - "auth_methods", "auto_start", "base_branch", "context", "context_files", - "decision_id", "effective_model", "entity_references", "error_output", - "failure_code", "failure_details", "failure_kind", "fallback_model", - "has_hidden_prompts", "has_resume_token", "has_review_comments", "is_auth_error", - "kind", "max_attempts", "message", "missing_branch", "model_id", "new_branch", - "original_branch", "options", "pending_id", "plan_mode", "progress", "provider_name", - "question", "question_id", "question_index", "question_total", "recovery_actions", - "remediation", "remediation_url", "requested_model", "request_id", "response", "reset_at", - "retry_at", "retry_in_seconds", "retrying", "sender_session_id", - "sender_session_name", "sender_task_id", "sender_task_title", "stage", "status", - "script_type", "agent_name", "command", "exit_code", "is_resuming", "started_at", - "completed_at", "error", - "task_id", "text", "tool_call_id", "variant", "workflow_message", "workflow_step_color", - "workflow_step_id", "workflow_step_name", -} - -func sqliteConversationMetadataExpr(metadataExpr string) string { - arguments := make([]string, 0, len(conversationMessageMetadataKeys)*2) - for _, key := range conversationMessageMetadataKeys { - arguments = append(arguments, "'"+key+"'", sqliteConversationMetadataValueExpr(metadataExpr, key)) - } - return "CASE WHEN json_valid(" + metadataExpr + ") THEN json_object(" + strings.Join(arguments, ",") + ") ELSE json_object() END" -} - -func sqliteConversationMetadataValueExpr(metadataExpr, key string) string { - path := "'$." + key + "'" - value := "json_extract(" + metadataExpr + ", " + path + ")" - return "CASE json_type(" + metadataExpr + ", " + path + ") " + - "WHEN 'true' THEN json('true') " + - "WHEN 'false' THEN json('false') " + - "ELSE " + value + " END" -} - -func postgresConversationMetadataExpr(metadataExpr string) string { - source := "conversation_safe_jsonb(" + metadataExpr + ")" - const maxKeysPerJSONBObject = 40 // PostgreSQL limits function calls to 100 arguments. - objects := make([]string, 0, (len(conversationMessageMetadataKeys)+maxKeysPerJSONBObject-1)/maxKeysPerJSONBObject) - for start := 0; start < len(conversationMessageMetadataKeys); start += maxKeysPerJSONBObject { - end := min(start+maxKeysPerJSONBObject, len(conversationMessageMetadataKeys)) - arguments := make([]string, 0, (end-start)*2) - for _, key := range conversationMessageMetadataKeys[start:end] { - arguments = append(arguments, "'"+key+"'", source+" -> '"+key+"'") - } - objects = append(objects, "jsonb_build_object("+strings.Join(arguments, ",")+")") - } - return "jsonb_strip_nulls(" + strings.Join(objects, " || ") + ")" -} - -func journalTaskID(taskID string) any { - if taskID == "" { - return nil - } - return taskID -} -func journalTime(value *time.Time) any { - if value == nil { - return nil - } - return value.UTC().Format(time.RFC3339Nano) -} - -//nolint:funlen // Trigger definitions are kept together so schema initialization is atomic. -func (r *Repository) initSQLiteConversationJournalTriggers() error { - triggerSQL := strings.ReplaceAll(` -DROP TRIGGER IF EXISTS conversation_message_insert; -CREATE TRIGGER IF NOT EXISTS conversation_message_insert -AFTER INSERT ON task_session_messages -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_message_versions(session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload) - SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.author_type, NEW.created_at, FALSE, - json_object('type','message.added','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), - 'message_id',NEW.id,'turn_id',NULLIF(NEW.turn_id,''),'author_type',NEW.author_type, - 'content',__SQLITE_STRIP_SYSTEM__,'message_type',NEW.type,'requests_input',NEW.requests_input, - 'created_at',__SQLITE_RFC3339_MILLIS__(NEW.created_at), - 'updated_at',__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.created_at)),'prompt_index',NEW.prompt_seq, - 'metadata',__SQLITE_CONVERSATION_METADATA__(NEW.metadata), - 'sender_task_id',CASE WHEN json_valid(NEW.metadata) AND typeof(json_extract(NEW.metadata,'$.sender_task_id')) = 'text' THEN json_extract(NEW.metadata,'$.sender_task_id') END) - FROM conversation_session_streams WHERE session_id = NEW.task_session_id; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'message.added', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP - FROM conversation_session_streams JOIN conversation_message_versions - ON conversation_message_versions.session_id = conversation_session_streams.session_id - AND conversation_message_versions.message_id = NEW.id - AND conversation_message_versions.row_sequence = conversation_session_streams.watermark - WHERE conversation_session_streams.session_id = NEW.task_session_id; -END; - -DROP TRIGGER IF EXISTS conversation_message_update; -CREATE TRIGGER IF NOT EXISTS conversation_message_update -AFTER UPDATE ON task_session_messages -WHEN OLD.id IS NOT NEW.id - OR OLD.task_session_id IS NOT NEW.task_session_id - OR OLD.task_id IS NOT NEW.task_id - OR OLD.turn_id IS NOT NEW.turn_id - OR OLD.author_type IS NOT NEW.author_type - OR OLD.author_id IS NOT NEW.author_id - OR OLD.content IS NOT NEW.content - OR OLD.requests_input IS NOT NEW.requests_input - OR OLD.type IS NOT NEW.type - OR OLD.metadata IS NOT NEW.metadata - OR OLD.created_at IS NOT NEW.created_at - OR OLD.updated_at IS NOT NEW.updated_at - OR OLD.prompt_seq IS NOT NEW.prompt_seq -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_message_versions(session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload) - SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.author_type, NEW.created_at, FALSE, - json_object('type','message.updated','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), - 'message_id',NEW.id,'turn_id',NULLIF(NEW.turn_id,''),'author_type',NEW.author_type, - 'content',__SQLITE_STRIP_SYSTEM__,'message_type',NEW.type,'requests_input',NEW.requests_input, - 'created_at',__SQLITE_RFC3339_MILLIS__(NEW.created_at), - 'updated_at',__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.created_at)),'prompt_index',NEW.prompt_seq, - 'metadata',__SQLITE_CONVERSATION_METADATA__(NEW.metadata), - 'sender_task_id',CASE WHEN json_valid(NEW.metadata) AND typeof(json_extract(NEW.metadata,'$.sender_task_id')) = 'text' THEN json_extract(NEW.metadata,'$.sender_task_id') END) - FROM conversation_session_streams WHERE session_id = NEW.task_session_id; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'message.updated', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP - FROM conversation_session_streams JOIN conversation_message_versions - ON conversation_message_versions.session_id = conversation_session_streams.session_id - AND conversation_message_versions.message_id = NEW.id - AND conversation_message_versions.row_sequence = conversation_session_streams.watermark - WHERE conversation_session_streams.session_id = NEW.task_session_id; -END; -DROP TRIGGER IF EXISTS conversation_message_delete; - -CREATE TRIGGER IF NOT EXISTS conversation_message_delete -BEFORE DELETE ON task_session_messages -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (OLD.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_message_versions(session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload) - SELECT OLD.task_session_id, OLD.id, watermark, NULLIF(OLD.task_id, ''), OLD.author_type, OLD.created_at, TRUE, - json_object('type','message.deleted','session_id',OLD.task_session_id,'task_id',NULLIF(OLD.task_id,''),'message_id',OLD.id) - FROM conversation_session_streams WHERE session_id = OLD.task_session_id; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT OLD.task_session_id, watermark, OLD.task_session_id || ':' || watermark, 'message.deleted', NULLIF(OLD.task_id,''), payload, CURRENT_TIMESTAMP - FROM conversation_session_streams JOIN conversation_message_versions - ON conversation_message_versions.session_id = conversation_session_streams.session_id - AND conversation_message_versions.message_id = OLD.id - AND conversation_message_versions.row_sequence = conversation_session_streams.watermark - WHERE conversation_session_streams.session_id = OLD.task_session_id; -END; -DROP TRIGGER IF EXISTS conversation_turn_insert; - -CREATE TRIGGER IF NOT EXISTS conversation_turn_insert -AFTER INSERT ON task_session_turns -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_turn_versions(session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload) - SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.started_at, FALSE, - json_object('type','session.turn.started','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), - 'id',NEW.id,'started_at',__SQLITE_RFC3339_MILLIS__(NEW.started_at), - 'completed_at',__SQLITE_RFC3339_MILLIS__(NEW.completed_at), - 'created_at',__SQLITE_RFC3339_MILLIS__(NEW.created_at), - 'updated_at',__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.started_at))) - FROM conversation_session_streams WHERE session_id = NEW.task_session_id; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'session.turn.started', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP - FROM conversation_session_streams JOIN conversation_turn_versions - ON conversation_turn_versions.session_id = conversation_session_streams.session_id - AND conversation_turn_versions.turn_id = NEW.id - AND conversation_turn_versions.row_sequence = conversation_session_streams.watermark - WHERE conversation_session_streams.session_id = NEW.task_session_id; -END; -DROP TRIGGER IF EXISTS conversation_turn_complete; - -CREATE TRIGGER IF NOT EXISTS conversation_turn_complete -AFTER UPDATE OF completed_at ON task_session_turns -WHEN OLD.completed_at IS NULL AND NEW.completed_at IS NOT NULL -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_turn_versions(session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload) - SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.started_at, FALSE, - json_object('type','session.turn.completed','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), - 'id',NEW.id,'started_at',__SQLITE_RFC3339_MILLIS__(NEW.started_at), - 'completed_at',__SQLITE_RFC3339_MILLIS__(NEW.completed_at), - 'created_at',__SQLITE_RFC3339_MILLIS__(NEW.created_at), - 'updated_at',__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.completed_at,NEW.started_at)), - 'metadata',CASE WHEN json_valid(NEW.metadata) THEN json(NEW.metadata) ELSE json('{}') END, - 'had_output',json(CASE WHEN EXISTS ( - SELECT 1 FROM task_session_messages output - WHERE output.turn_id = NEW.id AND output.author_type = 'agent' - AND ( - output.type IN ('tool_call','tool_edit','tool_read','tool_search','tool_execute', - 'agent_plan','todo','permission_request','clarification_request') - OR (output.type IN ('message','content','') AND trim(output.content) <> '') - )) THEN 'true' ELSE 'false' END)) - FROM conversation_session_streams WHERE session_id = NEW.task_session_id; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'session.turn.completed', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP - FROM conversation_session_streams JOIN conversation_turn_versions - ON conversation_turn_versions.session_id = conversation_session_streams.session_id - AND conversation_turn_versions.turn_id = NEW.id - AND conversation_turn_versions.row_sequence = conversation_session_streams.watermark - WHERE conversation_session_streams.session_id = NEW.task_session_id; -END; - -DROP TRIGGER IF EXISTS conversation_turn_delete; -CREATE TRIGGER IF NOT EXISTS conversation_turn_delete -AFTER DELETE ON task_session_turns -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (OLD.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_turn_versions(session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload) - SELECT OLD.task_session_id, OLD.id, watermark, NULLIF(OLD.task_id, ''), OLD.started_at, TRUE, - json_object('type','session.turn.removed','session_id',OLD.task_session_id,'task_id',NULLIF(OLD.task_id,''),'id',OLD.id) - FROM conversation_session_streams WHERE session_id = OLD.task_session_id; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT OLD.task_session_id, watermark, OLD.task_session_id || ':' || watermark, 'session.turn.removed', NULLIF(OLD.task_id,''), - payload, CURRENT_TIMESTAMP - FROM conversation_session_streams JOIN conversation_turn_versions - ON conversation_turn_versions.session_id = conversation_session_streams.session_id - AND conversation_turn_versions.turn_id = OLD.id - AND conversation_turn_versions.row_sequence = conversation_session_streams.watermark - WHERE conversation_session_streams.session_id = OLD.task_session_id; -END; - -DROP TRIGGER IF EXISTS conversation_session_delete; -CREATE TRIGGER IF NOT EXISTS conversation_session_delete -AFTER DELETE ON task_sessions -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (OLD.id, 1, TRUE, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, terminal = TRUE, updated_at = CURRENT_TIMESTAMP; - INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) - SELECT OLD.id, watermark, OLD.id || ':' || watermark, 'session.removed', NULLIF(OLD.task_id,''), - json_object('type','session.removed','session_id',OLD.id,'task_id',NULLIF(OLD.task_id,'')), CURRENT_TIMESTAMP - FROM conversation_session_streams WHERE session_id = OLD.id; -END; -`, sqliteStripSystemToken, sqliteStripSystemContentExpr("NEW.content")) - triggerSQL = strings.ReplaceAll(triggerSQL, - sqliteConversationMetadataToken+"(NEW.metadata)", sqliteConversationMetadataExpr("NEW.metadata")) - triggerSQL = strings.NewReplacer( - "__SQLITE_RFC3339_MILLIS__(NEW.created_at)", dialect.RFC3339Millis(dialect.SQLite3, "NEW.created_at"), - "__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.created_at))", dialect.RFC3339Millis(dialect.SQLite3, "COALESCE(NEW.updated_at,NEW.created_at)"), - "__SQLITE_RFC3339_MILLIS__(NEW.started_at)", dialect.RFC3339Millis(dialect.SQLite3, "NEW.started_at"), - "__SQLITE_RFC3339_MILLIS__(NEW.completed_at)", dialect.RFC3339Millis(dialect.SQLite3, "NEW.completed_at"), - "__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.started_at))", dialect.RFC3339Millis(dialect.SQLite3, "COALESCE(NEW.updated_at,NEW.started_at)"), - "__SQLITE_RFC3339_MILLIS__(COALESCE(NEW.updated_at,NEW.completed_at,NEW.started_at))", dialect.RFC3339Millis(dialect.SQLite3, "COALESCE(NEW.updated_at,NEW.completed_at,NEW.started_at)"), - ).Replace(triggerSQL) - if _, err := r.db.Exec(triggerSQL); err != nil { - return fmt.Errorf("create SQLite conversation journal triggers: %w", err) - } - messageMigrate := ` - UPDATE conversation_message_versions - SET payload = json_set(payload, '$.content', __SQLITE_STRIP_SYSTEM__) - WHERE json_valid(payload) - AND json_extract(payload, '$.content') LIKE '%%' - ` - contentExpr := sqliteStripSystemContentExpr("json_extract(payload, '$.content')") - if _, err := r.db.Exec(strings.ReplaceAll(messageMigrate, sqliteStripSystemToken, contentExpr)); err != nil { - return fmt.Errorf("sanitize SQLite conversation message journal: %w", err) - } - eventMigrate := ` - UPDATE conversation_session_events - SET payload = json_set(payload, '$.content', __SQLITE_STRIP_SYSTEM__) - WHERE json_valid(payload) - AND json_extract(payload, '$.content') LIKE '%%' - ` - if _, err := r.db.Exec(strings.ReplaceAll(eventMigrate, sqliteStripSystemToken, contentExpr)); err != nil { - return fmt.Errorf("sanitize SQLite conversation event journal: %w", err) - } - return nil -} - -//nolint:funlen // Trigger definitions are kept together so schema initialization is atomic. -func (r *Repository) initPostgresConversationJournalTriggers() error { - triggerSQL := strings.ReplaceAll(` -CREATE OR REPLACE FUNCTION conversation_next_sequence(p_session_id TEXT, p_terminal BOOLEAN DEFAULT FALSE) -RETURNS BIGINT AS $$ -DECLARE next_value BIGINT; -BEGIN - INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) - VALUES (p_session_id, 1, p_terminal, CURRENT_TIMESTAMP) - ON CONFLICT(session_id) DO UPDATE SET - watermark = conversation_session_streams.watermark + 1, - terminal = conversation_session_streams.terminal OR EXCLUDED.terminal, - updated_at = CURRENT_TIMESTAMP - RETURNING watermark INTO next_value; - RETURN next_value; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION conversation_safe_jsonb(value TEXT) RETURNS JSONB AS $$ -BEGIN - RETURN COALESCE(NULLIF(value, ''), '{}')::jsonb; -EXCEPTION WHEN others THEN - RETURN '{}'::jsonb; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION conversation_visible_content(value TEXT) RETURNS TEXT AS $$ -DECLARE - result TEXT := value; - open_pos INTEGER; - close_offset INTEGER; - close_pos INTEGER; -BEGIN - LOOP - open_pos := strpos(result, ''); - IF open_pos = 0 THEN - EXIT; - END IF; - close_offset := strpos( - substr(result, open_pos + length('')), - '' - ); - IF close_offset = 0 THEN - EXIT; - END IF; - close_pos := open_pos + length('') + close_offset - 1; - result := substr(result, 1, open_pos - 1) - || ltrim(substr(result, close_pos + length('')), E' \t\n\r'); - END LOOP; - RETURN btrim(result); -END; -$$ LANGUAGE plpgsql; -CREATE OR REPLACE FUNCTION conversation_message_journal() RETURNS TRIGGER AS $$ -DECLARE seq BIGINT; event_name TEXT; source_row task_session_messages%ROWTYPE; deleted BOOLEAN; -BEGIN - IF TG_OP = 'UPDATE' AND OLD IS NOT DISTINCT FROM NEW THEN RETURN NEW; END IF; - deleted := TG_OP = 'DELETE'; - IF deleted THEN source_row := OLD; event_name := 'message.deleted'; - ELSIF TG_OP = 'INSERT' THEN source_row := NEW; event_name := 'message.added'; - ELSE source_row := NEW; event_name := 'message.updated'; END IF; - seq := conversation_next_sequence(source_row.task_session_id); - INSERT INTO conversation_message_versions(session_id,message_id,row_sequence,task_id,author_type,created_at,tombstone,payload) - VALUES (source_row.task_session_id,source_row.id,seq,NULLIF(source_row.task_id,''),source_row.author_type,source_row.created_at,deleted, - CASE WHEN deleted THEN json_build_object('type',event_name,'session_id',source_row.task_session_id,'task_id',NULLIF(source_row.task_id,''),'message_id',source_row.id)::text - ELSE json_build_object('type',event_name,'session_id',source_row.task_session_id,'task_id',NULLIF(source_row.task_id,''), - 'message_id',source_row.id,'turn_id',NULLIF(source_row.turn_id,''),'author_type',source_row.author_type, - 'content',conversation_visible_content(source_row.content),'message_type',source_row.type, - 'requests_input',source_row.requests_input, - 'created_at',to_char(source_row.created_at,'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), - 'updated_at',to_char(COALESCE(source_row.updated_at,source_row.created_at),'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'),'prompt_index',source_row.prompt_seq, - 'metadata',__POSTGRES_CONVERSATION_METADATA__(source_row.metadata), - 'sender_task_id',CASE WHEN jsonb_typeof(conversation_safe_jsonb(source_row.metadata) -> 'sender_task_id') = 'string' THEN conversation_safe_jsonb(source_row.metadata) ->> 'sender_task_id' END)::text END); - INSERT INTO conversation_session_events(session_id,sequence,event_id,event_type,task_id,payload,created_at) - SELECT source_row.task_session_id,seq,source_row.task_session_id || ':' || seq,event_name,NULLIF(source_row.task_id,''),payload,CURRENT_TIMESTAMP - FROM conversation_message_versions WHERE session_id=source_row.task_session_id AND message_id=source_row.id AND row_sequence=seq; - RETURN NULL; -END; -$$ LANGUAGE plpgsql; -DROP TRIGGER IF EXISTS conversation_message_journal_trigger ON task_session_messages; -CREATE TRIGGER conversation_message_journal_trigger AFTER INSERT OR UPDATE OR DELETE ON task_session_messages -FOR EACH ROW EXECUTE FUNCTION conversation_message_journal(); - -CREATE OR REPLACE FUNCTION conversation_turn_journal() RETURNS TRIGGER AS $$ -DECLARE seq BIGINT; event_name TEXT; source_row task_session_turns%ROWTYPE; deleted BOOLEAN; -BEGIN - deleted := TG_OP = 'DELETE'; - IF deleted THEN source_row := OLD; event_name := 'session.turn.removed'; - ELSIF TG_OP = 'UPDATE' AND NOT (OLD.completed_at IS NULL AND NEW.completed_at IS NOT NULL) THEN RETURN NEW; - ELSIF TG_OP = 'INSERT' THEN source_row := NEW; event_name := 'session.turn.started'; - ELSE source_row := NEW; event_name := 'session.turn.completed'; END IF; - seq := conversation_next_sequence(source_row.task_session_id); - INSERT INTO conversation_turn_versions(session_id,turn_id,row_sequence,task_id,started_at,tombstone,payload) - VALUES (source_row.task_session_id,source_row.id,seq,NULLIF(source_row.task_id,''),source_row.started_at,deleted, - CASE WHEN deleted THEN - json_build_object('type',event_name,'session_id',source_row.task_session_id,'task_id',NULLIF(source_row.task_id,''),'id',source_row.id)::text - ELSE - json_build_object('type',event_name,'session_id',source_row.task_session_id,'task_id',NULLIF(source_row.task_id,''), - 'id',source_row.id,'started_at',to_char(source_row.started_at,'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), - 'completed_at',CASE WHEN source_row.completed_at IS NULL THEN NULL ELSE to_char(source_row.completed_at,'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') END, - 'created_at',to_char(source_row.created_at,'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), - 'updated_at',to_char(COALESCE(source_row.updated_at,source_row.completed_at,source_row.started_at),'YYYY-MM-DD"T"HH24:MI:SS.US"Z"'), - 'metadata',CASE WHEN source_row.completed_at IS NULL THEN NULL ELSE conversation_safe_jsonb(source_row.metadata) END, - 'had_output',CASE WHEN source_row.completed_at IS NULL THEN NULL ELSE EXISTS ( - SELECT 1 FROM task_session_messages output - WHERE output.turn_id = source_row.id AND output.author_type = 'agent' - AND ( - output.type IN ('tool_call','tool_edit','tool_read','tool_search','tool_execute', - 'agent_plan','todo','permission_request','clarification_request') - OR (output.type IN ('message','content','') AND btrim(output.content) <> '') - )) END)::text - END); - INSERT INTO conversation_session_events(session_id,sequence,event_id,event_type,task_id,payload,created_at) - SELECT source_row.task_session_id,seq,source_row.task_session_id || ':' || seq,event_name,NULLIF(source_row.task_id,''),payload,CURRENT_TIMESTAMP - FROM conversation_turn_versions - WHERE session_id=source_row.task_session_id AND turn_id=source_row.id AND row_sequence=seq; - IF deleted THEN RETURN OLD; END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; -DROP TRIGGER IF EXISTS conversation_turn_journal_trigger ON task_session_turns; -CREATE TRIGGER conversation_turn_journal_trigger AFTER INSERT OR UPDATE OR DELETE ON task_session_turns -FOR EACH ROW EXECUTE FUNCTION conversation_turn_journal(); - -CREATE OR REPLACE FUNCTION conversation_session_delete_journal() RETURNS TRIGGER AS $$ -DECLARE seq BIGINT; -BEGIN - seq := conversation_next_sequence(OLD.id, TRUE); - INSERT INTO conversation_session_events(session_id,sequence,event_id,event_type,task_id,payload,created_at) - VALUES (OLD.id,seq,OLD.id || ':' || seq,'session.removed',NULLIF(OLD.task_id,''), - json_build_object('type','session.removed','session_id',OLD.id,'task_id',NULLIF(OLD.task_id,''))::text,CURRENT_TIMESTAMP); - RETURN OLD; -END; -$$ LANGUAGE plpgsql; -DROP TRIGGER IF EXISTS conversation_session_delete_trigger ON task_sessions; -CREATE TRIGGER conversation_session_delete_trigger AFTER DELETE ON task_sessions -FOR EACH ROW EXECUTE FUNCTION conversation_session_delete_journal(); - `, postgresConversationMetadataToken+"(source_row.metadata)", postgresConversationMetadataExpr("source_row.metadata")) - _, err := r.db.Exec(r.db.Rebind(triggerSQL)) - if err != nil { - return fmt.Errorf("create PostgreSQL conversation journal triggers: %w", err) - } - if _, err := r.db.Exec(` - UPDATE conversation_message_versions - SET payload = jsonb_set(payload::jsonb, '{content}', - to_jsonb(conversation_visible_content(payload::jsonb ->> 'content')))::text - WHERE conversation_safe_jsonb(payload) ->> 'content' LIKE '%%' - `); err != nil { - return fmt.Errorf("sanitize PostgreSQL conversation message journal: %w", err) - } - if _, err := r.db.Exec(` - UPDATE conversation_session_events - SET payload = jsonb_set(payload::jsonb, '{content}', - to_jsonb(conversation_visible_content(payload::jsonb ->> 'content')))::text - WHERE conversation_safe_jsonb(payload) ->> 'content' LIKE '%%' - `); err != nil { - return fmt.Errorf("sanitize PostgreSQL conversation event journal: %w", err) - } - return nil -} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup.go b/apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup.go new file mode 100644 index 0000000000..7f03555196 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup.go @@ -0,0 +1,74 @@ +package sqlite + +import ( + "fmt" + + "github.com/kandev/kandev/internal/db/dialect" +) + +// cleanupLegacyConversationJournal removes the payload journal after the +// source revision schema is ready. All DDL runs in one transaction so an +// interrupted upgrade leaves the old journal available for a later retry. +func (r *Repository) cleanupLegacyConversationJournal() error { + tx, err := r.db.BeginTxx(r.migrationContext(), nil) + if err != nil { + return fmt.Errorf("begin conversation journal cleanup: %w", err) + } + rollback := func(cause error) error { + _ = tx.Rollback() + return cause + } + + statements := legacyConversationJournalCleanupSQL(r.db.DriverName()) + for index, statement := range statements { + if r.failConversationJournalCleanupAfter != "" && + r.failConversationJournalCleanupAfter == fmt.Sprintf("statement-%d", index) { + return rollback(fmt.Errorf("conversation journal cleanup failpoint at statement-%d", index)) + } + if _, err := tx.ExecContext(r.migrationContext(), statement); err != nil { + return rollback(fmt.Errorf("conversation journal cleanup statement %d: %w", index, err)) + } + } + if r.failConversationJournalCleanupAfter == "commit" { + return rollback(fmt.Errorf("conversation journal cleanup failpoint before commit")) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit conversation journal cleanup: %w", err) + } + return nil +} + +func legacyConversationJournalCleanupSQL(driver string) []string { + if dialect.IsPostgres(driver) { + return []string{ + `DROP TRIGGER IF EXISTS conversation_message_journal_trigger ON task_session_messages`, + `DROP TRIGGER IF EXISTS conversation_turn_journal_trigger ON task_session_turns`, + `DROP TRIGGER IF EXISTS conversation_session_delete_trigger ON task_sessions`, + `DROP FUNCTION IF EXISTS conversation_message_journal()`, + `DROP FUNCTION IF EXISTS conversation_turn_journal()`, + `DROP FUNCTION IF EXISTS conversation_session_delete_journal()`, + `DROP FUNCTION IF EXISTS conversation_next_sequence(TEXT, BOOLEAN)`, + `DROP FUNCTION IF EXISTS conversation_safe_jsonb(TEXT)`, + `DROP FUNCTION IF EXISTS conversation_visible_content(TEXT)`, + `DROP TABLE IF EXISTS conversation_session_events`, + `DROP TABLE IF EXISTS conversation_message_versions`, + `DROP TABLE IF EXISTS conversation_turn_versions`, + `DROP TABLE IF EXISTS conversation_session_streams`, + `DROP TABLE IF EXISTS conversation_journal_meta`, + } + } + return []string{ + `DROP TRIGGER IF EXISTS conversation_message_insert`, + `DROP TRIGGER IF EXISTS conversation_message_update`, + `DROP TRIGGER IF EXISTS conversation_message_delete`, + `DROP TRIGGER IF EXISTS conversation_turn_insert`, + `DROP TRIGGER IF EXISTS conversation_turn_complete`, + `DROP TRIGGER IF EXISTS conversation_turn_delete`, + `DROP TRIGGER IF EXISTS conversation_session_delete`, + `DROP TABLE IF EXISTS conversation_session_events`, + `DROP TABLE IF EXISTS conversation_message_versions`, + `DROP TABLE IF EXISTS conversation_turn_versions`, + `DROP TABLE IF EXISTS conversation_session_streams`, + `DROP TABLE IF EXISTS conversation_journal_meta`, + } +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup_test.go b/apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup_test.go new file mode 100644 index 0000000000..b08e901a8c --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup_test.go @@ -0,0 +1,106 @@ +package sqlite + +import ( + "context" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +func TestConversationJournalCleanupRollsBackAndCanRetry(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-conversation-cleanup" + sessionID = "session-conversation-cleanup" + turnID = "turn-conversation-cleanup" + ) + seedForMsgTest(t, repo, taskID, sessionID, turnID) + message := &models.Message{ + ID: "message-conversation-cleanup", + TaskSessionID: sessionID, + TaskID: taskID, + TurnID: turnID, + AuthorType: models.MessageAuthorUser, + Type: models.MessageTypeMessage, + Content: "source payload survives journal cleanup", + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + if err := repo.CreateMessage(ctx, message); err != nil { + t.Fatalf("create source message: %v", err) + } + + for _, statement := range []string{ + `CREATE TABLE conversation_session_events (id TEXT)`, + `CREATE TABLE conversation_message_versions (id TEXT)`, + `CREATE TABLE conversation_turn_versions (id TEXT)`, + `CREATE TABLE conversation_session_streams (id TEXT)`, + `CREATE TABLE conversation_journal_meta (id TEXT)`, + `CREATE TRIGGER conversation_message_insert AFTER INSERT ON task_session_messages BEGIN INSERT INTO conversation_message_versions(id) VALUES (NEW.id); END`, + } { + if _, err := repo.db.Exec(statement); err != nil { + t.Fatalf("create legacy fixture with %q: %v", statement, err) + } + } + + repo.failConversationJournalCleanupAfter = "statement-1" + if err := repo.cleanupLegacyConversationJournal(); err == nil { + t.Fatal("cleanup with failpoint succeeded") + } + assertConversationJournalObjectsExist(t, repo) + got, err := repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("read source message after rollback: %v", err) + } + if got.Content != message.Content { + t.Fatalf("source content after rollback = %q, want %q", got.Content, message.Content) + } + + repo.failConversationJournalCleanupAfter = "" + if err := repo.cleanupLegacyConversationJournal(); err != nil { + t.Fatalf("retry cleanup: %v", err) + } + assertConversationJournalObjectsAbsent(t, repo) + got, err = repo.GetMessage(ctx, message.ID) + if err != nil { + t.Fatalf("read source message after cleanup: %v", err) + } + if got.Content != message.Content { + t.Fatalf("source content after cleanup = %q, want %q", got.Content, message.Content) + } + if err := repo.cleanupLegacyConversationJournal(); err != nil { + t.Fatalf("repeated cleanup: %v", err) + } +} + +func assertConversationJournalObjectsExist(t *testing.T, repo *Repository) { + t.Helper() + assertConversationJournalObjects(t, repo, true) +} + +func assertConversationJournalObjectsAbsent(t *testing.T, repo *Repository) { + t.Helper() + assertConversationJournalObjects(t, repo, false) +} + +func assertConversationJournalObjects(t *testing.T, repo *Repository, wantPresent bool) { + t.Helper() + for _, name := range []string{ + "conversation_session_events", + "conversation_message_versions", + "conversation_turn_versions", + "conversation_session_streams", + "conversation_journal_meta", + "conversation_message_insert", + } { + var count int + if err := repo.db.Get(&count, `SELECT COUNT(*) FROM sqlite_master WHERE name = ?`, name); err != nil { + t.Fatalf("inspect legacy object %q: %v", name, err) + } + present := count != 0 + if present != wantPresent { + t.Fatalf("legacy object %q present = %v, want %v", name, present, wantPresent) + } + } +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_journal_postgres_test.go b/apps/backend/internal/task/repository/sqlite/conversation_journal_postgres_test.go deleted file mode 100644 index 16a1d9d396..0000000000 --- a/apps/backend/internal/task/repository/sqlite/conversation_journal_postgres_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package sqlite - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/kandev/kandev/internal/sysprompt" - "github.com/kandev/kandev/internal/task/models" - "github.com/kandev/kandev/internal/testutil" -) - -// TestPostgresConversationJournalRoundTrip exercises the PostgreSQL journal -// triggers end to end: message insert/update sanitization through -// conversation_visible_content, the meta schema-version marker, and the -// version-guarded one-time migration. Skips unless KANDEV_TEST_POSTGRES_DSN is -// set (CI runs it in postgres-boot). -func TestPostgresConversationJournalRoundTrip(t *testing.T) { - db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) - repo, err := NewWithDB(db, db, nil) - if err != nil { - t.Fatalf("new repo: %v", err) - } - ctx := context.Background() - seedPostgresConversationJournal(t, repo, "task-pg-journal", "session-pg-journal", "turn-pg-journal") - - content := "pre hidden one\nline two mid two post" - message := &models.Message{ - ID: "message-pg-journal", TaskSessionID: "session-pg-journal", TaskID: "task-pg-journal", - TurnID: "turn-pg-journal", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: content, - Metadata: map[string]interface{}{ - "pending_id": "pending-pg", "status": "pending", "raw_content": "secret", - }, - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - want := sysprompt.StripSystemContent(content) - - var eventPayload string - if err := repo.db.Get(&eventPayload, `SELECT payload FROM conversation_session_events WHERE session_id = 'session-pg-journal' AND event_type = 'message.added'`); err != nil { - t.Fatalf("read event payload: %v", err) - } - var payload map[string]any - if err := json.Unmarshal([]byte(eventPayload), &payload); err != nil { - t.Fatalf("decode event payload: %v", err) - } - metadata, ok := payload["metadata"].(map[string]interface{}) - if !ok || metadata["pending_id"] != "pending-pg" || metadata["status"] != "pending" { - t.Fatalf("postgres journal metadata = %#v, want allowlisted presentation metadata", payload["metadata"]) - } - if _, ok := metadata["raw_content"]; ok { - t.Fatalf("postgres journal retained internal raw_content metadata: %#v", metadata) - } - got, ok := payload["content"].(string) - if !ok || got != want || strings.Contains(got, "hidden") { - t.Fatalf("postgres journal content = %q, want %q", got, want) - } - - // The schema marker must be recorded so the one-time migration is not - // re-run (and the full-corpus rewrite never repeats on later boots). - version, err := repo.readConversationJournalMetaInt("schema.version") - if err != nil { - t.Fatalf("read schema version: %v", err) - } - if version < journalSchemaVersion { - t.Fatalf("schema version = %d, want >= %d", version, journalSchemaVersion) - } - backfilled, err := repo.readConversationJournalMetaFlag(journalBackfillKey) - if err != nil { - t.Fatalf("read backfill flag: %v", err) - } - if !backfilled { - t.Fatal("backfill flag not recorded after init") - } -} - -func TestPostgresConversationJournalStripsPastSQLiteDepthLimit(t *testing.T) { - db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) - repo, err := NewWithDB(db, db, nil) - if err != nil { - t.Fatalf("new repo: %v", err) - } - seedPostgresConversationJournal(t, repo, "task-pg-strip-deep", "session-pg-strip-deep", "turn-pg-strip-deep") - content := strings.Repeat("secret", 65) + "visible" - message := &models.Message{ - ID: "message-pg-strip-deep", TaskSessionID: "session-pg-strip-deep", TaskID: "task-pg-strip-deep", - TurnID: "turn-pg-strip-deep", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: content, - } - if err := repo.CreateMessage(context.Background(), message); err != nil { - t.Fatalf("create message: %v", err) - } - - for _, table := range []string{"conversation_message_versions", "conversation_session_events"} { - var payload string - orderColumn := "row_sequence" - if table == "conversation_session_events" { - orderColumn = "sequence" - } - query := "SELECT payload FROM " + table + " WHERE session_id = $1 ORDER BY " + orderColumn + " DESC LIMIT 1" - if err := repo.db.Get(&payload, query, "session-pg-strip-deep"); err != nil { - t.Fatalf("read %s payload: %v", table, err) - } - if strings.Contains(payload, "secret") || strings.Contains(payload, "") { - t.Fatalf("%s persisted system content: %s", table, payload) - } - } -} - -func TestPostgresConversationJournalTurnDeletePreservesCutoffHistory(t *testing.T) { - db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) - repo, err := NewWithDB(db, db, nil) - if err != nil { - t.Fatalf("new repo: %v", err) - } - const ( - taskID = "task-pg-turn-delete" - sessionID = "session-pg-turn-delete" - turnID = "turn-pg-turn-delete" - ) - seedPostgresConversationJournal(t, repo, taskID, sessionID, turnID) - now := time.Now().UTC() - if _, err := repo.db.Exec(repo.db.Rebind(`UPDATE task_session_turns SET completed_at = ?, updated_at = ? WHERE id = ?`), now, now, turnID); err != nil { - t.Fatalf("complete turn: %v", err) - } - var cutoffBefore int64 - if err := repo.db.Get(&cutoffBefore, repo.db.Rebind(`SELECT watermark FROM conversation_session_streams WHERE session_id = ?`), sessionID); err != nil { - t.Fatalf("read pre-delete cutoff: %v", err) - } - if _, err := repo.db.Exec(repo.db.Rebind(`DELETE FROM task_session_turns WHERE id = ?`), turnID); err != nil { - t.Fatalf("delete turn: %v", err) - } - var cutoffAfter int64 - if err := repo.db.Get(&cutoffAfter, repo.db.Rebind(`SELECT watermark FROM conversation_session_streams WHERE session_id = ?`), sessionID); err != nil { - t.Fatalf("read deletion cutoff: %v", err) - } - assertTurnLiveAtCutoff(t, repo, sessionID, turnID, cutoffBefore, true) - assertTurnLiveAtCutoff(t, repo, sessionID, turnID, cutoffAfter, false) -} - -func seedPostgresConversationJournal(t *testing.T, repo *Repository, taskID, sessionID, turnID string) { - t.Helper() - now := time.Now().UTC() - if _, err := repo.db.Exec(repo.db.Rebind(` - INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) - VALUES (?, '', 'test task', ?, ?) - `), taskID, now, now); err != nil { - t.Fatalf("seed task: %v", err) - } - if _, err := repo.db.Exec(repo.db.Rebind(` - INSERT INTO task_sessions (id, task_id, started_at, updated_at) - VALUES (?, ?, ?, ?) - `), sessionID, taskID, now, now); err != nil { - t.Fatalf("seed session: %v", err) - } - if _, err := repo.db.Exec(repo.db.Rebind(` - INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?) - `), turnID, sessionID, taskID, now, now, now); err != nil { - t.Fatalf("seed turn: %v", err) - } -} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_journal_test.go b/apps/backend/internal/task/repository/sqlite/conversation_journal_test.go deleted file mode 100644 index 5afb07ca5c..0000000000 --- a/apps/backend/internal/task/repository/sqlite/conversation_journal_test.go +++ /dev/null @@ -1,652 +0,0 @@ -package sqlite - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/kandev/kandev/internal/sysprompt" - "github.com/kandev/kandev/internal/task/models" -) - -func TestConversationJournalVersionsMutationsAndTerminalDeletion(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal", "session-journal", "turn-journal") - message := &models.Message{ - ID: "message-journal", TaskSessionID: "session-journal", TaskID: "task-journal", - TurnID: "turn-journal", AuthorType: models.MessageAuthorAgent, - Type: models.MessageTypeMessage, Content: "hiddenfirst", - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - message.Content = "hidden-updateupdated" - if err := repo.UpdateMessage(ctx, message); err != nil { - t.Fatalf("update message: %v", err) - } - if err := repo.DeleteMessage(ctx, message.ID); err != nil { - t.Fatalf("delete message: %v", err) - } - session, err := repo.GetTaskSession(ctx, "session-journal") - if err != nil { - t.Fatalf("load session: %v", err) - } - if err := repo.DeleteTaskSession(ctx, session); err != nil { - t.Fatalf("delete session: %v", err) - } - - var terminal bool - var watermark int64 - if err := repo.db.QueryRow(`SELECT watermark, terminal FROM conversation_session_streams WHERE session_id = 'session-journal'`).Scan(&watermark, &terminal); err != nil { - t.Fatalf("read stream: %v", err) - } - if !terminal || watermark != 6 { - t.Fatalf("stream = watermark %d terminal %v, want 6 true", watermark, terminal) - } - rows, err := repo.db.Query(`SELECT event_type FROM conversation_session_events WHERE session_id = 'session-journal' ORDER BY sequence`) - if err != nil { - t.Fatalf("list events: %v", err) - } - defer func() { _ = rows.Close() }() - var eventTypes []string - for rows.Next() { - var eventType string - if err := rows.Scan(&eventType); err != nil { - t.Fatal(err) - } - eventTypes = append(eventTypes, eventType) - } - want := []string{"session.turn.started", "message.added", "message.updated", "message.deleted", "session.turn.removed", "session.removed"} - if len(eventTypes) != len(want) { - t.Fatalf("event types = %v, want %v", eventTypes, want) - } - for index := range want { - if eventTypes[index] != want[index] { - t.Fatalf("event types = %v, want %v", eventTypes, want) - } - } - var payloadBytes []byte - if err := repo.db.Get(&payloadBytes, `SELECT payload FROM conversation_session_events WHERE session_id = 'session-journal' AND event_type = 'message.added'`); err != nil { - t.Fatalf("read message event payload: %v", err) - } - var payload map[string]any - if err := json.Unmarshal(payloadBytes, &payload); err != nil { - t.Fatalf("decode message event payload: %v", err) - } - if payload["type"] != "message.added" || payload["session_id"] != "session-journal" { - t.Fatalf("message event identity = %#v", payload) - } - if content, ok := payload["content"].(string); !ok || content != "first" || strings.Contains(content, "hidden") { - t.Fatalf("journal leaked system content: %#v", payload["content"]) - } - createdAt, ok := payload["created_at"].(string) - if !ok { - t.Fatalf("message created_at = %#v", payload["created_at"]) - } - if _, err := time.Parse(time.RFC3339Nano, createdAt); err != nil { - t.Fatalf("message created_at is not RFC3339: %q: %v", createdAt, err) - } - var tombstone bool - if err := repo.db.QueryRow(`SELECT tombstone FROM conversation_message_versions WHERE session_id = 'session-journal' AND message_id = 'message-journal' ORDER BY row_sequence DESC LIMIT 1`).Scan(&tombstone); err != nil { - t.Fatalf("read message tombstone: %v", err) - } - if !tombstone { - t.Fatal("latest message version is not a tombstone") - } -} - -func TestConversationJournalRollsBackWithSourceMutation(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-rollback", "session-journal-rollback", "turn-journal-rollback") - tx, err := repo.db.BeginTxx(ctx, nil) - if err != nil { - t.Fatal(err) - } - now := time.Now().UTC() - _, err = tx.ExecContext(ctx, repo.db.Rebind(` - INSERT INTO task_session_messages - (id, task_session_id, task_id, turn_id, author_type, content, type, metadata, created_at, updated_at) - VALUES (?, ?, ?, ?, 'agent', 'rolled back', 'message', '{}', ?, ?) - `), "message-rollback", "session-journal-rollback", "task-journal-rollback", "turn-journal-rollback", now, now) - if err != nil { - _ = tx.Rollback() - t.Fatalf("insert source row: %v", err) - } - if err := tx.Rollback(); err != nil { - t.Fatalf("rollback: %v", err) - } - - var count int - if err := repo.db.Get(&count, `SELECT COUNT(*) FROM conversation_session_events WHERE session_id = 'session-journal-rollback' AND event_type = 'message.added'`); err != nil { - t.Fatalf("count rolled-back event: %v", err) - } - if count != 0 { - t.Fatalf("rolled-back mutation left %d event rows", count) - } -} - -func TestConversationJournalStripsMultiLineAndMultiBlockSystemContent(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-strip", "session-journal-strip", "turn-journal-strip") - content := "pre hidden one\nline two mid\nhidden2 post" - message := &models.Message{ - ID: "message-strip", TaskSessionID: "session-journal-strip", TaskID: "task-journal-strip", - TurnID: "turn-journal-strip", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: content, - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - want := sysprompt.StripSystemContent(content) - - var versionContent string - if err := repo.db.Get(&versionContent, `SELECT json_extract(payload, '$.content') FROM conversation_message_versions WHERE message_id = 'message-strip'`); err != nil { - t.Fatalf("read message version content: %v", err) - } - if versionContent != want { - t.Fatalf("version content = %q, want %q", versionContent, want) - } - var eventContent string - if err := repo.db.Get(&eventContent, `SELECT json_extract(payload, '$.content') FROM conversation_session_events WHERE session_id = 'session-journal-strip' AND event_type = 'message.added'`); err != nil { - t.Fatalf("read message event content: %v", err) - } - if eventContent != want { - t.Fatalf("event content = %q, want %q", eventContent, want) - } - - // The update trigger strips the same way. - message.Content = "a\nupdated bc end" - if err := repo.UpdateMessage(ctx, message); err != nil { - t.Fatalf("update message: %v", err) - } - wantUpdated := sysprompt.StripSystemContent(message.Content) - if err := repo.db.Get(&versionContent, `SELECT json_extract(payload, '$.content') FROM conversation_message_versions WHERE message_id = 'message-strip' ORDER BY row_sequence DESC LIMIT 1`); err != nil { - t.Fatalf("read updated message version content: %v", err) - } - if versionContent != wantUpdated { - t.Fatalf("updated version content = %q, want %q", versionContent, wantUpdated) - } - if err := repo.db.Get(&eventContent, `SELECT json_extract(payload, '$.content') FROM conversation_session_events WHERE session_id = 'session-journal-strip' AND event_type = 'message.updated'`); err != nil { - t.Fatalf("read updated message event content: %v", err) - } - if eventContent != wantUpdated { - t.Fatalf("updated event content = %q, want %q", eventContent, wantUpdated) - } -} - -func TestConversationJournalStripIgnoresClosingTagWithoutOpeningTag(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-closing-tag", "session-journal-closing-tag", "turn-journal-closing-tag") - content := "visibletail" - message := &models.Message{ - ID: "message-closing-tag", TaskSessionID: "session-journal-closing-tag", - TaskID: "task-journal-closing-tag", TurnID: "turn-journal-closing-tag", - AuthorType: models.MessageAuthorUser, Type: models.MessageTypeMessage, Content: content, - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - var eventContent string - if err := repo.db.Get(&eventContent, `SELECT json_extract(payload, '$.content') FROM conversation_session_events WHERE session_id = 'session-journal-closing-tag' AND event_type = 'message.added'`); err != nil { - t.Fatalf("read message event content: %v", err) - } - if eventContent != content { - t.Fatalf("event content = %q, want %q", eventContent, content) - } -} - -func TestConversationJournalStripFailsClosedPastDepthLimit(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-strip-deep", "session-journal-strip-deep", "turn-journal-strip-deep") - content := strings.Repeat("secret", 65) + "visible" - message := &models.Message{ - ID: "message-strip-deep", TaskSessionID: "session-journal-strip-deep", TaskID: "task-journal-strip-deep", - TurnID: "turn-journal-strip-deep", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: content, - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - - for _, table := range []string{"conversation_message_versions", "conversation_session_events"} { - var payload string - query := "SELECT payload FROM " + table + " WHERE session_id = ? ORDER BY row_sequence DESC LIMIT 1" - if table == "conversation_session_events" { - query = "SELECT payload FROM " + table + " WHERE session_id = ? AND event_type = 'message.added' ORDER BY sequence DESC LIMIT 1" - } - if err := repo.db.Get(&payload, repo.db.Rebind(query), "session-journal-strip-deep"); err != nil { - t.Fatalf("read %s payload: %v", table, err) - } - if strings.Contains(payload, "secret") || strings.Contains(payload, "") { - t.Fatalf("%s persisted system content past sanitizer depth: %s", table, payload) - } - } -} -func TestConversationJournalPreservesPresentationMetadata(t *testing.T) { - repo := newRepoForSessionTests(t) - seedForMsgTest(t, repo, "task-presentation-metadata", "session-presentation-metadata", "turn-presentation-metadata") - message := &models.Message{ - ID: "message-presentation-metadata", TaskSessionID: "session-presentation-metadata", - TaskID: "task-presentation-metadata", TurnID: "turn-presentation-metadata", - AuthorType: models.MessageAuthorAgent, Type: models.MessageTypeClarificationRequest, - Content: "Question", RequestsInput: true, - Metadata: map[string]any{ - "agent_disconnected": true, - "agent_name": "Mock", - "command": "/usr/local/bin/mock-agent", - "completed_at": "2026-09-15T09:45:59.774503743Z", - "error": "must-not-be-dropped", - "exit_code": 0, - "has_hidden_prompts": false, - "is_resuming": true, - "pending_id": "pending-1", - "question": map[string]any{"id": "question-1", "prompt": "Choose"}, - "raw_content": "must-not-be-copied", - "script_type": "agent_boot", - "started_at": "2026-09-15T09:45:58.245720115Z", - }, - } - if err := repo.CreateMessage(context.Background(), message); err != nil { - t.Fatalf("create message: %v", err) - } - - var payload string - if err := repo.db.Get(&payload, `SELECT payload FROM conversation_session_events WHERE session_id = ? AND event_type = 'message.added'`, message.TaskSessionID); err != nil { - t.Fatalf("read event payload: %v", err) - } - var event map[string]any - if err := json.Unmarshal([]byte(payload), &event); err != nil { - t.Fatalf("decode event payload: %v", err) - } - if event["message_type"] != string(models.MessageTypeClarificationRequest) || event["requests_input"] != float64(1) { - t.Fatalf("message presentation fields = %#v", event) - } - metadata, ok := event["metadata"].(map[string]any) - if !ok || metadata["pending_id"] != "pending-1" || metadata["agent_disconnected"] != true || metadata["has_hidden_prompts"] != false { - t.Fatalf("message metadata = %#v", event["metadata"]) - } - for key, want := range map[string]any{ - "agent_name": "Mock", "command": "/usr/local/bin/mock-agent", "completed_at": "2026-09-15T09:45:59.774503743Z", - "error": "must-not-be-dropped", "exit_code": float64(0), "is_resuming": true, "script_type": "agent_boot", - "started_at": "2026-09-15T09:45:58.245720115Z", - } { - if metadata[key] != want { - t.Fatalf("message metadata[%q] = %#v, want %#v", key, metadata[key], want) - } - } - if _, exists := metadata["raw_content"]; exists { - t.Fatalf("private message metadata leaked: %#v", metadata) - } -} - -func TestConversationJournalStripKeepsRawTextWhenNoWellFormedBlock(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-strip-raw", "session-journal-strip-raw", "turn-journal-strip-raw") - // Shapes where the Go helper has no full match: an unterminated opening - // tag and a stray closing tag before the first opening tag. The SQLite - // strip must keep the raw text (never NULL), byte-identical to Go. - cases := map[string]string{ - "unterminated-open": "visible never closed", - "stray-close": "before closed after", - "deep-nesting": "" + strings.Repeat("", 70) + "x" + strings.Repeat("", 70) + "tail", - } - for name, content := range cases { - t.Run(name, func(t *testing.T) { - id := "message-strip-raw-" + name - message := &models.Message{ - ID: id, TaskSessionID: "session-journal-strip-raw", TaskID: "task-journal-strip-raw", - TurnID: "turn-journal-strip-raw", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: content, - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - var versionContent *string - if err := repo.db.Get(&versionContent, `SELECT json_extract(payload, '$.content') FROM conversation_message_versions WHERE message_id = ?`, id); err != nil { - t.Fatalf("read message version content: %v", err) - } - if versionContent == nil { - t.Fatalf("strip returned NULL for %q", content) - } - want := sysprompt.StripSystemContent(content) - if name != "deep-nesting" && *versionContent != want { - t.Fatalf("version content = %q, want %q", *versionContent, want) - } - if strings.Contains(*versionContent, "") && name == "unterminated-open" { - // Unterminated opener text is preserved verbatim. - if *versionContent != content { - t.Fatalf("unterminated content = %q, want raw %q", *versionContent, content) - } - } - }) - } -} - -func TestConversationJournalSanitizeMigrationRewritesLegacyRows(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-migrate", "session-journal-migrate", "turn-journal-migrate") - legacy := "old\nsecretkept second text" - message := &models.Message{ - ID: "message-legacy", TaskSessionID: "session-journal-migrate", TaskID: "task-journal-migrate", - TurnID: "turn-journal-migrate", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: "placeholder", - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - // Simulate rows written before the sanitize migration existed. - if _, err := repo.db.Exec(` - UPDATE conversation_message_versions SET payload = json_set(payload, '$.content', ?) WHERE message_id = 'message-legacy'; - `, legacy); err != nil { - t.Fatalf("seed legacy version payload: %v", err) - } - if _, err := repo.db.Exec(` - UPDATE conversation_session_events SET payload = json_set(payload, '$.content', ?) WHERE session_id = 'session-journal-migrate' AND event_type = 'message.added'; - `, legacy); err != nil { - t.Fatalf("seed legacy event payload: %v", err) - } - - if err := repo.initSQLiteConversationJournalTriggers(); err != nil { - t.Fatalf("re-init triggers (runs sanitize migration): %v", err) - } - want := sysprompt.StripSystemContent(legacy) - var versionContent string - if err := repo.db.Get(&versionContent, `SELECT json_extract(payload, '$.content') FROM conversation_message_versions WHERE message_id = 'message-legacy'`); err != nil { - t.Fatalf("read migrated version content: %v", err) - } - if versionContent != want { - t.Fatalf("migrated version content = %q, want %q", versionContent, want) - } - var eventContent string - if err := repo.db.Get(&eventContent, `SELECT json_extract(payload, '$.content') FROM conversation_session_events WHERE session_id = 'session-journal-migrate' AND event_type = 'message.added'`); err != nil { - t.Fatalf("read migrated event content: %v", err) - } - if eventContent != want { - t.Fatalf("migrated event content = %q, want %q", eventContent, want) - } -} - -func TestConversationJournalTurnDeletePreservesHistoryWithTombstone(t *testing.T) { - repo := newRepoForSessionTests(t) - seedForMsgTest(t, repo, "task-journal-turn-del", "session-journal-turn-del", "turn-journal-turn-del") - - now := time.Now().UTC() - if _, err := repo.db.Exec(repo.db.Rebind(`UPDATE task_session_turns SET completed_at = ?, updated_at = ? WHERE id = ?`), now, now, "turn-journal-turn-del"); err != nil { - t.Fatalf("complete turn: %v", err) - } - var cutoffBefore int64 - if err := repo.db.Get(&cutoffBefore, `SELECT watermark FROM conversation_session_streams WHERE session_id = 'session-journal-turn-del'`); err != nil { - t.Fatalf("read pre-delete cutoff: %v", err) - } - - if _, err := repo.db.Exec(repo.db.Rebind(`DELETE FROM task_session_turns WHERE id = ?`), "turn-journal-turn-del"); err != nil { - t.Fatalf("delete turn: %v", err) - } - var cutoffAfter int64 - if err := repo.db.Get(&cutoffAfter, `SELECT watermark FROM conversation_session_streams WHERE session_id = 'session-journal-turn-del'`); err != nil { - t.Fatalf("read deletion cutoff: %v", err) - } - if cutoffAfter != cutoffBefore+1 { - t.Fatalf("turn delete watermark = %d, want %d", cutoffAfter, cutoffBefore+1) - } - - var versions int - if err := repo.db.Get(&versions, `SELECT COUNT(*) FROM conversation_turn_versions WHERE turn_id = 'turn-journal-turn-del'`); err != nil { - t.Fatalf("count turn versions: %v", err) - } - if versions != 3 { - t.Fatalf("turn version count = %d, want two live versions plus tombstone", versions) - } - var tombstoneSequence int64 - if err := repo.db.Get(&tombstoneSequence, ` - SELECT row_sequence FROM conversation_turn_versions - WHERE turn_id = 'turn-journal-turn-del' AND tombstone = TRUE - `); err != nil { - t.Fatalf("read deletion tombstone: %v", err) - } - if tombstoneSequence != cutoffAfter { - t.Fatalf("tombstone sequence = %d, want deletion cutoff %d", tombstoneSequence, cutoffAfter) - } - - assertTurnLiveAtCutoff(t, repo, "session-journal-turn-del", "turn-journal-turn-del", cutoffBefore, true) - assertTurnLiveAtCutoff(t, repo, "session-journal-turn-del", "turn-journal-turn-del", cutoffAfter, false) - - var removalEvent string - if err := repo.db.Get(&removalEvent, `SELECT event_type FROM conversation_session_events WHERE session_id = 'session-journal-turn-del' ORDER BY sequence DESC LIMIT 1`); err != nil { - t.Fatalf("read last event type: %v", err) - } - if removalEvent != "session.turn.removed" { - t.Fatalf("last event type = %q, want session.turn.removed", removalEvent) - } -} - -func assertTurnLiveAtCutoff(t *testing.T, repo *Repository, sessionID, turnID string, cutoff int64, want bool) { - t.Helper() - var count int - if err := repo.db.Get(&count, repo.db.Rebind(` - WITH ranked AS ( - SELECT tombstone, - ROW_NUMBER() OVER (PARTITION BY turn_id ORDER BY row_sequence DESC) AS version_rank - FROM conversation_turn_versions - WHERE session_id = ? AND turn_id = ? AND row_sequence <= ? - ) - SELECT COUNT(*) FROM ranked WHERE version_rank = 1 AND tombstone = FALSE - `), sessionID, turnID, cutoff); err != nil { - t.Fatalf("query turn at cutoff %d: %v", cutoff, err) - } - if got := count == 1; got != want { - t.Fatalf("turn live at cutoff %d = %v, want %v", cutoff, got, want) - } -} - -func TestConversationJournalBackfillsExistingRowsIdempotently(t *testing.T) { - repo := newRepoForSessionTests(t) - seedForMsgTest(t, repo, "task-backfill", "session-backfill", "turn-backfill") - message := &models.Message{ - ID: "message-backfill", TaskSessionID: "session-backfill", TaskID: "task-backfill", - TurnID: "turn-backfill", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: "hiddenexisting", - } - if err := repo.CreateMessage(context.Background(), message); err != nil { - t.Fatalf("create source message: %v", err) - } - if _, err := repo.db.Exec(` - DELETE FROM conversation_session_events; - DELETE FROM conversation_message_versions; - DELETE FROM conversation_turn_versions; - DELETE FROM conversation_session_streams; - `); err != nil { - t.Fatalf("clear journal: %v", err) - } - - if err := repo.backfillConversationJournal(); err != nil { - t.Fatalf("backfill conversation journal: %v", err) - } - if err := repo.backfillConversationJournal(); err != nil { - t.Fatalf("repeat conversation journal backfill: %v", err) - } - var eventCount int - if err := repo.db.Get(&eventCount, `SELECT COUNT(*) FROM conversation_session_events WHERE session_id = 'session-backfill'`); err != nil { - t.Fatalf("count backfill events: %v", err) - } - if eventCount != 2 { - t.Fatalf("backfill event count = %d, want 2", eventCount) - } - var content string - if err := repo.db.Get(&content, `SELECT json_extract(payload, '$.content') FROM conversation_message_versions WHERE message_id = 'message-backfill'`); err != nil { - t.Fatalf("read backfill message content: %v", err) - } - if content != "existing" { - t.Fatalf("backfill content = %q, want existing", content) - } -} - -func TestConversationJournalSenderTaskIDRequiresString(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-sender", "session-journal-sender", "turn-journal-sender") - if err := repo.CreateMessage(ctx, &models.Message{ - ID: "message-sender-numeric", TaskSessionID: "session-journal-sender", TaskID: "task-journal-sender", - TurnID: "turn-journal-sender", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: "numeric", - Metadata: map[string]any{"sender_task_id": 42}, - }); err != nil { - t.Fatalf("create numeric-sender message: %v", err) - } - if err := repo.CreateMessage(ctx, &models.Message{ - ID: "message-sender-string", TaskSessionID: "session-journal-sender", TaskID: "task-journal-sender", - TurnID: "turn-journal-sender", AuthorType: models.MessageAuthorAgent, - Type: models.MessageTypeMessage, Content: "string", - Metadata: map[string]any{"sender_task_id": "task-sender-1"}, - }); err != nil { - t.Fatalf("create string-sender message: %v", err) - } - - var numericSender any - if err := repo.db.Get(&numericSender, `SELECT json_extract(payload, '$.sender_task_id') FROM conversation_message_versions WHERE message_id = 'message-sender-numeric'`); err != nil { - t.Fatalf("read numeric-sender payload: %v", err) - } - if numericSender != nil { - t.Fatalf("numeric sender_task_id stored as %v, want NULL", numericSender) - } - var stringSender string - if err := repo.db.Get(&stringSender, `SELECT json_extract(payload, '$.sender_task_id') FROM conversation_message_versions WHERE message_id = 'message-sender-string'`); err != nil { - t.Fatalf("read string-sender payload: %v", err) - } - if stringSender != "task-sender-1" { - t.Fatalf("string sender_task_id = %q, want task-sender-1", stringSender) - } -} - -// TestBackfillSkipsStaleMessageImageWithSameContent pins the round-9 fix: a -// concurrent update that only touches metadata (same content) must not let the -// stale pre-update backfill image win a later version. -func TestBackfillSkipsStaleMessageImageWithSameContent(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-backfill-stale", "session-backfill-stale", "turn-backfill-stale") - now := time.Now().UTC().Truncate(time.Microsecond) - message := &models.Message{ - ID: "message-stale", TaskSessionID: "session-backfill-stale", TaskID: "task-backfill-stale", - TurnID: "turn-backfill-stale", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: "same-content", - Metadata: map[string]any{"sender_task_id": "sender-1"}, CreatedAt: now, UpdatedAt: now, - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - // A concurrent metadata-only update journals its own version. - message.Metadata = map[string]any{"sender_task_id": "sender-2"} - message.UpdatedAt = now.Add(time.Minute) - if err := repo.UpdateMessage(ctx, message); err != nil { - t.Fatalf("update message: %v", err) - } - var versionsAfterUpdate int - if err := repo.db.Get(&versionsAfterUpdate, `SELECT COUNT(*) FROM conversation_message_versions WHERE message_id = 'message-stale'`); err != nil { - t.Fatalf("count versions: %v", err) - } - - tx, err := repo.db.Beginx() - if err != nil { - t.Fatal(err) - } - // The backfill seed reflects the pre-update image (same content, older - // timestamp): it must be skipped entirely. - stale := conversationJournalMessageSeed{ - ID: "message-stale", TaskSessionID: "session-backfill-stale", TaskID: "task-backfill-stale", - TurnID: "turn-backfill-stale", AuthorType: string(models.MessageAuthorUser), - Content: "same-content", MessageType: string(models.MessageTypeMessage), - Metadata: `{"sender_task_id":"sender-1"}`, CreatedAt: now, UpdatedAt: now, PromptIndex: message.PromptIndex, - } - if err := repo.backfillConversationMessage(tx, stale); err != nil { - _ = tx.Rollback() - t.Fatalf("backfill stale image: %v", err) - } - if err := tx.Commit(); err != nil { - t.Fatal(err) - } - var versions int - if err := repo.db.Get(&versions, `SELECT COUNT(*) FROM conversation_message_versions WHERE message_id = 'message-stale'`); err != nil { - t.Fatalf("count versions: %v", err) - } - if versions != versionsAfterUpdate { - t.Fatalf("stale backfill added a version: %d -> %d", versionsAfterUpdate, versions) - } -} - -// TestConversationJournalSanitizeMigrationSkipsMalformedRows pins the -// round-9 fix: a pre-existing malformed payload must not abort the boot-time -// sanitize migration; the row is preserved verbatim so the mirror path can -// turn it into a durable poison record. -func TestConversationJournalSanitizeMigrationSkipsMalformedRows(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-malformed", "session-journal-malformed", "turn-journal-malformed") - message := &models.Message{ - ID: "message-malformed", TaskSessionID: "session-journal-malformed", TaskID: "task-journal-malformed", - TurnID: "turn-journal-malformed", AuthorType: models.MessageAuthorUser, - Type: models.MessageTypeMessage, Content: "placeholder", - } - if err := repo.CreateMessage(ctx, message); err != nil { - t.Fatalf("create message: %v", err) - } - malformed := `{"content":"not valid json` - if _, err := repo.db.Exec(` - UPDATE conversation_message_versions SET payload = ? WHERE message_id = 'message-malformed'; - UPDATE conversation_session_events SET payload = ? WHERE session_id = 'session-journal-malformed' AND event_type = 'message.added'; - `, malformed, malformed); err != nil { - t.Fatalf("seed malformed rows: %v", err) - } - - if err := repo.initSQLiteConversationJournalTriggers(); err != nil { - t.Fatalf("re-init triggers (runs sanitize migration): %v", err) - } - var versionPayload string - if err := repo.db.Get(&versionPayload, `SELECT payload FROM conversation_message_versions WHERE message_id = 'message-malformed'`); err != nil { - t.Fatalf("read version payload: %v", err) - } - if versionPayload != malformed { - t.Fatalf("malformed version payload was rewritten to %q", versionPayload) - } - var eventPayload string - if err := repo.db.Get(&eventPayload, `SELECT payload FROM conversation_session_events WHERE session_id = 'session-journal-malformed' AND event_type = 'message.added'`); err != nil { - t.Fatalf("read event payload: %v", err) - } - if eventPayload != malformed { - t.Fatalf("malformed event payload was rewritten to %q", eventPayload) - } -} - -func TestConversationJournalTurnCompletionIncludesHadOutput(t *testing.T) { - repo := newRepoForSessionTests(t) - ctx := context.Background() - seedForMsgTest(t, repo, "task-journal-empty", "session-journal-empty", "turn-journal-empty") - - if err := repo.CompleteTurn(ctx, "turn-journal-empty"); err != nil { - t.Fatalf("complete turn: %v", err) - } - - var raw string - if err := repo.db.Get(&raw, ` - SELECT payload FROM conversation_session_events - WHERE session_id = 'session-journal-empty' AND event_type = 'session.turn.completed' - `); err != nil { - t.Fatalf("read completed turn event: %v", err) - } - var payload map[string]any - if err := json.Unmarshal([]byte(raw), &payload); err != nil { - t.Fatalf("decode completed turn event: %v", err) - } - if hadOutput, ok := payload["had_output"].(bool); !ok || hadOutput { - t.Fatalf("had_output = %#v, want false", payload["had_output"]) - } -} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_receipts.go b/apps/backend/internal/task/repository/sqlite/conversation_receipts.go new file mode 100644 index 0000000000..865990b703 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_receipts.go @@ -0,0 +1,464 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/db/dialect" + "github.com/kandev/kandev/internal/task/models" +) + +func (r *Repository) beginConversationMutation(ctx context.Context, sessionID string) (*sqlx.Tx, int64, error) { + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return nil, 0, fmt.Errorf("begin conversation mutation: %w", err) + } + rollback := func(err error) (*sqlx.Tx, int64, error) { + _ = tx.Rollback() + return nil, 0, err + } + base, err := r.prepareConversationMutation(ctx, tx, sessionID) + if err != nil { + return rollback(err) + } + return tx, base, nil +} + +func (r *Repository) prepareConversationMutation(ctx context.Context, tx *sqlx.Tx, sessionID string) (int64, error) { + if err := lockSessionTurnWrites(ctx, tx, r.db.DriverName(), sessionID); err != nil { + return 0, err + } + return r.ensureConversationRevisionTx(ctx, tx, sessionID) +} + +func (r *Repository) completePendingToolCallsForTurnTx(ctx context.Context, tx *sqlx.Tx, turnID string) (int64, error) { + drv := r.db.DriverName() + query := fmt.Sprintf(` + UPDATE task_session_messages + SET metadata = %s, updated_at = CURRENT_TIMESTAMP + WHERE turn_id = ? + AND type != 'permission_request' + AND %s NOT IN ('complete', 'error') + AND %s + `, dialect.JSONSet(drv, "metadata", "status", "complete"), + dialect.JSONExtract(drv, "metadata", "status"), + dialect.JSONExtractIsNotNull(drv, "metadata", "tool_call_id")) + result, err := tx.ExecContext(ctx, tx.Rebind(query), turnID) + if err != nil { + return 0, fmt.Errorf("failed to complete pending tool calls for turn %s: %w", turnID, err) + } + rows, err := result.RowsAffected() + if err != nil { + return 0, err + } + return rows, nil +} + +func (r *Repository) ensureConversationRevisionTx(ctx context.Context, tx *sqlx.Tx, sessionID string) (int64, error) { + result, err := tx.ExecContext(ctx, tx.Rebind(` + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT ?, 0 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = ?) + ON CONFLICT(session_id) DO NOTHING + `), sessionID, sessionID) + if err != nil { + return 0, fmt.Errorf("ensure conversation revision: %w", err) + } + if affected, err := result.RowsAffected(); err == nil && affected == 0 { + var exists int + if scanErr := tx.QueryRowxContext(ctx, tx.Rebind(`SELECT 1 FROM task_sessions WHERE id = ?`), sessionID).Scan(&exists); scanErr != nil { + if scanErr == sql.ErrNoRows { + return 0, fmt.Errorf("task session %s not found", sessionID) + } + return 0, scanErr + } + } + query := `SELECT revision FROM conversation_session_revisions WHERE session_id = ?` + if dialect.IsPostgres(r.db.DriverName()) { + query += forUpdateClause + } + var revision int64 + if err := tx.QueryRowxContext(ctx, tx.Rebind(query), sessionID).Scan(&revision); err != nil { + return 0, fmt.Errorf("read base conversation revision: %w", err) + } + return revision, nil +} + +func (r *Repository) conversationMutationReceipt( + ctx context.Context, + tx *sqlx.Tx, + sessionID string, + base int64, + operation models.ConversationMutationOperation, +) (*models.ConversationMutationReceipt, error) { + var revision int64 + if err := tx.QueryRowxContext(ctx, tx.Rebind( + `SELECT revision FROM conversation_session_revisions WHERE session_id = ?`, + ), sessionID).Scan(&revision); err != nil { + return nil, fmt.Errorf("read committed conversation revision: %w", err) + } + return &models.ConversationMutationReceipt{ + SessionID: sessionID, + BaseRevision: base, + Revision: revision, + Operations: []models.ConversationMutationOperation{operation}, + Complete: revision == base+1, + }, nil +} + +func (r *Repository) finishConversationMutation(tx *sqlx.Tx) error { + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit conversation mutation: %w", err) + } + return nil +} + +func (r *Repository) populateConversationMessageReceipt( + ctx context.Context, + tx *sqlx.Tx, + receipt *models.ConversationMutationReceipt, + base int64, + message *models.Message, + kind models.ConversationMutationKind, +) error { + result := message + if kind == models.ConversationMutationUpsert { + read, err := r.readConversationMessageTx(ctx, tx, message.ID) + if err != nil { + return err + } + result = read + } + operation := models.ConversationMutationOperation{ + Kind: kind, + Entity: models.ConversationEntityMessage, + ID: message.ID, + SessionID: message.TaskSessionID, + TaskID: message.TaskID, + AuthorType: string(message.AuthorType), + } + if kind == models.ConversationMutationUpsert { + operation.Message = result + } + updated, err := r.conversationMutationReceipt(ctx, tx, message.TaskSessionID, base, operation) + if err != nil { + return err + } + *receipt = *updated + return nil +} + +func (r *Repository) readConversationMessageTx(ctx context.Context, tx *sqlx.Tx, messageID string) (*models.Message, error) { + message := &models.Message{} + var requestsInput int + var messageType string + var metadataJSON string + err := tx.QueryRowxContext(ctx, tx.Rebind(` + SELECT id, task_session_id, task_id, turn_id, author_type, author_id, content, + requests_input, type, metadata, created_at, updated_at, + CASE WHEN author_type = 'user' THEN prompt_seq ELSE 0 END + FROM task_session_messages WHERE id = ? + `), messageID).Scan( + &message.ID, &message.TaskSessionID, &message.TaskID, &message.TurnID, + &message.AuthorType, &message.AuthorID, &message.Content, &requestsInput, + &messageType, &metadataJSON, &message.CreatedAt, &message.UpdatedAt, + &message.PromptIndex, + ) + if err != nil { + return nil, err + } + message.RequestsInput = requestsInput == 1 + message.Type = models.MessageType(messageType) + if metadataJSON != "" && metadataJSON != "{}" { + if err := json.Unmarshal([]byte(metadataJSON), &message.Metadata); err != nil { + return nil, fmt.Errorf("deserialize conversation message: %w", err) + } + } + return message, nil +} + +func (r *Repository) populateConversationTurnReceipt( + ctx context.Context, + tx *sqlx.Tx, + receipt *models.ConversationMutationReceipt, + base int64, + turn *models.Turn, + kind models.ConversationMutationKind, +) error { + result := turn + if kind == models.ConversationMutationUpsert { + read, err := r.readConversationTurnTx(ctx, tx, turn.ID) + if err != nil { + return err + } + result = read + } + operation := models.ConversationMutationOperation{ + Kind: kind, + Entity: models.ConversationEntityTurn, + ID: turn.ID, + SessionID: turn.TaskSessionID, + TaskID: turn.TaskID, + } + if kind == models.ConversationMutationUpsert { + operation.Turn = result + } + updated, err := r.conversationMutationReceipt(ctx, tx, turn.TaskSessionID, base, operation) + if err != nil { + return err + } + *receipt = *updated + return nil +} + +func (r *Repository) readConversationTurnTx(ctx context.Context, tx *sqlx.Tx, turnID string) (*models.Turn, error) { + return scanTurn(tx.QueryRowxContext(ctx, tx.Rebind(` + SELECT id, task_session_id, task_id, execution_profile_id, route_generation, + started_at, completed_at, metadata, created_at, updated_at + FROM task_session_turns WHERE id = ? + `), turnID)) +} + +func (r *Repository) CreateMessageWithConversationReceipt(ctx context.Context, message *models.Message) (*models.ConversationMutationReceipt, error) { + if message.ID == "" { + message.ID = uuid.NewString() + } + if message.AuthorType == "" { + message.AuthorType = models.MessageAuthorUser + } + requestsInput := 0 + if message.RequestsInput { + requestsInput = 1 + } + messageType := string(message.Type) + if messageType == "" { + messageType = string(models.MessageTypeMessage) + } + metadataJSON := "{}" + if message.Metadata != nil { + encoded, err := json.Marshal(message.Metadata) + if err != nil { + return nil, fmt.Errorf("serialize message metadata: %w", err) + } + metadataJSON = string(encoded) + } + if message.AuthorType == models.MessageAuthorUser { + return r.createUserMessageWithBoundaryReceipt(ctx, message, requestsInput, messageType, metadataJSON) + } + if message.CreatedAt.IsZero() { + message.CreatedAt = time.Now().UTC() + } + if message.UpdatedAt.IsZero() { + message.UpdatedAt = message.CreatedAt + } + tx, base, err := r.beginConversationMutation(ctx, message.TaskSessionID) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + if err := r.insertMessageWithPayloadGuardTx(ctx, tx, message, requestsInput, messageType, metadataJSON); err != nil { + return nil, err + } + receipt := &models.ConversationMutationReceipt{} + if err := r.populateConversationMessageReceipt(ctx, tx, receipt, base, message, models.ConversationMutationUpsert); err != nil { + return nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return nil, err + } + return receipt, nil +} + +func (r *Repository) UpdateMessageWithConversationReceipt(ctx context.Context, message *models.Message) (*models.ConversationMutationReceipt, error) { + metadataJSON, err := json.Marshal(message.Metadata) + if err != nil { + return nil, fmt.Errorf("serialize message metadata: %w", err) + } + requestsInput := 0 + if message.RequestsInput { + requestsInput = 1 + } + message.UpdatedAt = time.Now().UTC() + tx, base, err := r.beginConversationMutation(ctx, message.TaskSessionID) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + if err := r.updateMessageWithPayloadGuardTx(ctx, tx, message, metadataJSON, requestsInput); err != nil { + return nil, err + } + receipt := &models.ConversationMutationReceipt{} + if err := r.populateConversationMessageReceipt(ctx, tx, receipt, base, message, models.ConversationMutationUpsert); err != nil { + return nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return nil, err + } + return receipt, nil +} + +func (r *Repository) DeleteMessageWithConversationReceipt(ctx context.Context, messageID string) (*models.ConversationMutationReceipt, error) { + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin message deletion: %w", err) + } + defer func() { _ = tx.Rollback() }() + message, err := r.readConversationMessageTx(ctx, tx, messageID) + if err != nil { + return nil, err + } + base, err := r.prepareConversationMutation(ctx, tx, message.TaskSessionID) + if err != nil { + return nil, err + } + result, err := tx.ExecContext(ctx, tx.Rebind(`DELETE FROM task_session_messages WHERE id = ?`), messageID) + if err != nil { + return nil, err + } + if affected, err := result.RowsAffected(); err != nil || affected == 0 { + if err != nil { + return nil, err + } + return nil, fmt.Errorf("message not found: %s", messageID) + } + receipt, err := r.conversationMutationReceipt(ctx, tx, message.TaskSessionID, base, models.ConversationMutationOperation{ + Kind: models.ConversationMutationRemove, Entity: models.ConversationEntityMessage, + ID: messageID, SessionID: message.TaskSessionID, TaskID: message.TaskID, + AuthorType: string(message.AuthorType), + }) + if err != nil { + return nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return nil, err + } + return receipt, nil +} + +func (r *Repository) CreateTurnWithConversationReceipt(ctx context.Context, turn *models.Turn) (*models.ConversationMutationReceipt, error) { + stampTurnDefaults(turn) + tx, base, err := r.beginConversationMutation(ctx, turn.TaskSessionID) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + if err := r.insertTurnRow(ctx, tx, turn); err != nil { + return nil, err + } + receipt := &models.ConversationMutationReceipt{} + if err := r.populateConversationTurnReceipt(ctx, tx, receipt, base, turn, models.ConversationMutationUpsert); err != nil { + return nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return nil, err + } + return receipt, nil +} + +func (r *Repository) CreateTurnWithStepStampConversationReceipt(ctx context.Context, turn *models.Turn) (bool, *models.ConversationMutationReceipt, error) { + stampTurnDefaults(turn) + tx, base, err := r.beginConversationMutation(ctx, turn.TaskSessionID) + if err != nil { + return false, nil, err + } + defer func() { _ = tx.Rollback() }() + _, stepID, found, stepErr := r.readTaskStepInTx(ctx, tx, turn.TaskID) + stamped := false + if stepErr == nil && found && stepID != "" { + if turn.Metadata == nil { + turn.Metadata = map[string]interface{}{} + } + turn.Metadata[models.TurnMetaKeyWorkflowStepIDAtStart] = stepID + stamped = true + } + if err := r.insertTurnRow(ctx, tx, turn); err != nil { + return false, nil, err + } + receipt := &models.ConversationMutationReceipt{} + if err := r.populateConversationTurnReceipt(ctx, tx, receipt, base, turn, models.ConversationMutationUpsert); err != nil { + return false, nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return false, nil, err + } + return stamped, receipt, nil +} + +func (r *Repository) UpdateTurnWithConversationReceipt(ctx context.Context, turn *models.Turn) (*models.ConversationMutationReceipt, error) { + metadataJSON, err := serializeTurnMetadata(turn.Metadata) + if err != nil { + return nil, err + } + tx, base, err := r.beginConversationMutation(ctx, turn.TaskSessionID) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback() }() + updatedAt := r.nowUTC() + result, err := tx.ExecContext(ctx, tx.Rebind(` + UPDATE task_session_turns + SET completed_at = ?, metadata = ?, execution_profile_id = ?, route_generation = ?, updated_at = ? + WHERE id = ? AND task_session_id = ? AND updated_at = ? + `), turn.CompletedAt, metadataJSON, turn.ExecutionProfileID, turn.RouteGeneration, updatedAt, turn.ID, turn.TaskSessionID, turn.UpdatedAt) + if err != nil { + return nil, err + } + if affected, err := result.RowsAffected(); err != nil || affected != 1 { + if err != nil { + return nil, err + } + return nil, fmt.Errorf("update turn %s: stale metadata snapshot", turn.ID) + } + turn.UpdatedAt = updatedAt + receipt := &models.ConversationMutationReceipt{} + if err := r.populateConversationTurnReceipt(ctx, tx, receipt, base, turn, models.ConversationMutationUpsert); err != nil { + return nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return nil, err + } + return receipt, nil +} + +func (r *Repository) CompleteTurnWithConversationReceipt(ctx context.Context, turnID string) (*models.ConversationMutationReceipt, error) { + tx, err := r.db.BeginTxx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin turn completion: %w", err) + } + defer func() { _ = tx.Rollback() }() + turn, err := r.readConversationTurnTx(ctx, tx, turnID) + if err != nil { + return nil, err + } + base, err := r.prepareConversationMutation(ctx, tx, turn.TaskSessionID) + if err != nil { + return nil, err + } + if _, err := r.completePendingToolCallsForTurnTx(ctx, tx, turnID); err != nil { + return nil, err + } + now := time.Now().UTC() + if _, err := tx.ExecContext(ctx, tx.Rebind(` + UPDATE task_session_turns SET completed_at = ?, updated_at = ? WHERE id = ? + `), now, now, turnID); err != nil { + return nil, err + } + turn, err = r.readConversationTurnTx(ctx, tx, turnID) + if err != nil { + return nil, err + } + receipt := &models.ConversationMutationReceipt{} + if err := r.populateConversationTurnReceipt(ctx, tx, receipt, base, turn, models.ConversationMutationUpsert); err != nil { + return nil, err + } + if err := r.finishConversationMutation(tx); err != nil { + return nil, err + } + return receipt, nil +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_source.go b/apps/backend/internal/task/repository/sqlite/conversation_source.go new file mode 100644 index 0000000000..82fb8ce064 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_source.go @@ -0,0 +1,468 @@ +package sqlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "time" + + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/db/dialect" + "github.com/kandev/kandev/internal/task/models" +) + +const ( + conversationTimestampParameter = "CAST(? AS timestamp)" + conversationAscending = "ASC" + conversationDescending = "DESC" +) + +// initConversationSourceSchema installs the bounded revision metadata used by +// current-state conversation readers. Source rows remain authoritative; this +// table never stores message or turn payloads. +func (r *Repository) initConversationSourceSchema() error { + if _, err := r.db.ExecContext(r.migrationContext(), ` + CREATE TABLE IF NOT EXISTS conversation_session_revisions ( + session_id TEXT PRIMARY KEY, + revision BIGINT NOT NULL DEFAULT 0 CHECK (revision >= 0), + FOREIGN KEY (session_id) REFERENCES task_sessions(id) ON DELETE CASCADE + )`); err != nil { + return fmt.Errorf("create conversation source revisions: %w", err) + } + if dialect.IsPostgres(r.db.DriverName()) { + return r.initPostgresConversationSourceTriggers() + } + return r.initSQLiteConversationSourceTriggers() +} + +const sqliteConversationSourceTriggerSQL = ` +CREATE TRIGGER IF NOT EXISTS conversation_source_message_insert +AFTER INSERT ON task_session_messages +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT NEW.task_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = NEW.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; +END; + +CREATE TRIGGER IF NOT EXISTS conversation_source_message_update +AFTER UPDATE ON task_session_messages +WHEN OLD.task_session_id IS NOT NEW.task_session_id + OR OLD.task_id IS NOT NEW.task_id + OR OLD.turn_id IS NOT NEW.turn_id + OR OLD.author_type IS NOT NEW.author_type + OR OLD.author_id IS NOT NEW.author_id + OR OLD.content IS NOT NEW.content + OR OLD.requests_input IS NOT NEW.requests_input + OR OLD.type IS NOT NEW.type + OR OLD.metadata IS NOT NEW.metadata + OR OLD.created_at IS NOT NEW.created_at + OR OLD.updated_at IS NOT NEW.updated_at + OR OLD.prompt_seq IS NOT NEW.prompt_seq +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT OLD.task_session_id, 1 + WHERE OLD.task_session_id != NEW.task_session_id + AND EXISTS (SELECT 1 FROM task_sessions WHERE id = OLD.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT NEW.task_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = NEW.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; +END; + +CREATE TRIGGER IF NOT EXISTS conversation_source_message_delete +AFTER DELETE ON task_session_messages +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT OLD.task_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = OLD.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; +END; + +CREATE TRIGGER IF NOT EXISTS conversation_source_turn_insert +AFTER INSERT ON task_session_turns +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT NEW.task_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = NEW.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; +END; + +CREATE TRIGGER IF NOT EXISTS conversation_source_turn_update +AFTER UPDATE ON task_session_turns +WHEN OLD.task_session_id IS NOT NEW.task_session_id + OR OLD.task_id IS NOT NEW.task_id + OR OLD.execution_profile_id IS NOT NEW.execution_profile_id + OR OLD.route_generation IS NOT NEW.route_generation + OR OLD.started_at IS NOT NEW.started_at + OR OLD.completed_at IS NOT NEW.completed_at + OR OLD.metadata IS NOT NEW.metadata + OR OLD.created_at IS NOT NEW.created_at + OR OLD.updated_at IS NOT NEW.updated_at +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT OLD.task_session_id, 1 + WHERE OLD.task_session_id != NEW.task_session_id + AND EXISTS (SELECT 1 FROM task_sessions WHERE id = OLD.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT NEW.task_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = NEW.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; +END; + +CREATE TRIGGER IF NOT EXISTS conversation_source_turn_delete +AFTER DELETE ON task_session_turns +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT OLD.task_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = OLD.task_session_id) + ON CONFLICT(session_id) DO UPDATE SET revision = conversation_session_revisions.revision + 1; +END;` + +func (r *Repository) initSQLiteConversationSourceTriggers() error { + if _, err := r.db.ExecContext(r.migrationContext(), sqliteConversationSourceTriggerSQL); err != nil { + return fmt.Errorf("create SQLite conversation source triggers: %w", err) + } + return nil +} + +func (r *Repository) initPostgresConversationSourceTriggers() error { + const triggerSQL = ` +CREATE OR REPLACE FUNCTION conversation_source_bump_revision(p_session_id TEXT) +RETURNS VOID AS $$ +BEGIN + INSERT INTO conversation_session_revisions(session_id, revision) + SELECT p_session_id, 1 + WHERE EXISTS (SELECT 1 FROM task_sessions WHERE id = p_session_id) + ON CONFLICT(session_id) DO UPDATE + SET revision = conversation_session_revisions.revision + 1; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION conversation_source_message_revision() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM conversation_source_bump_revision(OLD.task_session_id); + RETURN OLD; + END IF; + IF TG_OP = 'UPDATE' AND OLD.task_session_id IS DISTINCT FROM NEW.task_session_id THEN + PERFORM conversation_source_bump_revision(OLD.task_session_id); + END IF; + PERFORM conversation_source_bump_revision(NEW.task_session_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS conversation_source_message_revision_trigger ON task_session_messages; +CREATE TRIGGER conversation_source_message_revision_trigger +AFTER INSERT OR UPDATE OR DELETE ON task_session_messages +FOR EACH ROW EXECUTE FUNCTION conversation_source_message_revision(); + +CREATE OR REPLACE FUNCTION conversation_source_turn_revision() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + PERFORM conversation_source_bump_revision(OLD.task_session_id); + RETURN OLD; + END IF; + IF TG_OP = 'UPDATE' AND OLD.task_session_id IS DISTINCT FROM NEW.task_session_id THEN + PERFORM conversation_source_bump_revision(OLD.task_session_id); + END IF; + PERFORM conversation_source_bump_revision(NEW.task_session_id); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +DROP TRIGGER IF EXISTS conversation_source_turn_revision_trigger ON task_session_turns; +CREATE TRIGGER conversation_source_turn_revision_trigger +AFTER INSERT OR UPDATE OR DELETE ON task_session_turns +FOR EACH ROW EXECUTE FUNCTION conversation_source_turn_revision();` + if _, err := r.db.ExecContext(r.migrationContext(), r.db.Rebind(triggerSQL)); err != nil { + return fmt.Errorf("create PostgreSQL conversation source triggers: %w", err) + } + return nil +} + +// ReadConversationRevision reads session existence and its current revision in +// one short read transaction. A revision row is optional because revision zero +// is the correct value for an existing session that has not changed since the +// source schema was installed. +func (r *Repository) ReadConversationRevision(ctx context.Context, sessionID string) (models.ConversationRevision, error) { + tx, err := r.beginConversationSourceRead(ctx) + if err != nil { + return models.ConversationRevision{}, err + } + defer func() { _ = tx.Rollback() }() + state, err := readConversationState(ctx, tx, r.db.DriverName(), sessionID) + if err != nil { + return models.ConversationRevision{}, err + } + if err := tx.Commit(); err != nil { + return models.ConversationRevision{}, fmt.Errorf("commit conversation revision read: %w", err) + } + return state, nil +} + +// ReadConversationMessagesPage reads source messages and their revision from a +// single consistent transaction. The result is a current-state page; it is +// never a read from the historical journal tables. +func (r *Repository) ReadConversationMessagesPage( + ctx context.Context, + req models.ConversationMessagePageRequest, +) (models.ConversationMessagePage, error) { + tx, err := r.beginConversationSourceRead(ctx) + if err != nil { + return models.ConversationMessagePage{}, err + } + defer func() { _ = tx.Rollback() }() + state, err := readConversationState(ctx, tx, r.db.DriverName(), req.SessionID) + if err != nil { + return models.ConversationMessagePage{}, err + } + if !state.Exists { + return models.ConversationMessagePage{}, models.ErrTaskSessionNotFound + } + limit := conversationSourcePageLimit(req.Limit) + cursorKey, cursorID, err := readConversationMessageCursor(ctx, tx, r.db.DriverName(), req.SessionID, req.CursorID) + if err != nil { + return models.ConversationMessagePage{}, err + } + query, args := buildConversationMessagePageQuery(r.db.DriverName(), req, cursorKey, cursorID, limit) + rows, err := tx.QueryxContext(ctx, tx.Rebind(query), args...) + if err != nil { + return models.ConversationMessagePage{}, fmt.Errorf("read conversation message page: %w", err) + } + messages, hasMore, scanErr := scanPromptIndexedMessageRows(rows, limit) + _ = rows.Close() + if scanErr != nil { + return models.ConversationMessagePage{}, scanErr + } + page := models.ConversationMessagePage{Messages: messages, HasMore: hasMore, Revision: state.Revision} + if hasMore && len(messages) > 0 { + page.CursorID = messages[len(messages)-1].ID + } + if err := tx.Commit(); err != nil { + return models.ConversationMessagePage{}, fmt.Errorf("commit conversation message page: %w", err) + } + return page, nil +} + +// ReadConversationTurnsPage is the turn counterpart to +// ReadConversationMessagesPage. Turn metadata remains available to first-party +// callers, while the plugin layer maps it to its safe DTO. +func (r *Repository) ReadConversationTurnsPage( + ctx context.Context, + req models.ConversationTurnPageRequest, +) (models.ConversationTurnPage, error) { + tx, err := r.beginConversationSourceRead(ctx) + if err != nil { + return models.ConversationTurnPage{}, err + } + defer func() { _ = tx.Rollback() }() + state, err := readConversationState(ctx, tx, r.db.DriverName(), req.SessionID) + if err != nil { + return models.ConversationTurnPage{}, err + } + if !state.Exists { + return models.ConversationTurnPage{}, models.ErrTaskSessionNotFound + } + limit := conversationSourcePageLimit(req.Limit) + cursorStarted, cursorID, err := readConversationTurnCursor(ctx, tx, req.SessionID, req.CursorID) + if err != nil { + return models.ConversationTurnPage{}, err + } + query, args := buildConversationTurnPageQuery(r.db.DriverName(), req, cursorStarted, cursorID, limit) + rows, err := tx.QueryxContext(ctx, tx.Rebind(query), args...) + if err != nil { + return models.ConversationTurnPage{}, fmt.Errorf("read conversation turn page: %w", err) + } + defer func() { _ = rows.Close() }() + turns := make([]*models.Turn, 0, limit) + for rows.Next() { + turn, scanErr := scanTurn(rows) + if scanErr != nil { + return models.ConversationTurnPage{}, scanErr + } + turns = append(turns, turn) + } + if err := rows.Err(); err != nil { + return models.ConversationTurnPage{}, err + } + hasMore := len(turns) > limit + if hasMore { + turns = turns[:limit] + } + page := models.ConversationTurnPage{Turns: turns, HasMore: hasMore, Revision: state.Revision} + if hasMore && len(turns) > 0 { + page.CursorID = turns[len(turns)-1].ID + } + if err := tx.Commit(); err != nil { + return models.ConversationTurnPage{}, fmt.Errorf("commit conversation turn page: %w", err) + } + return page, nil +} + +func (r *Repository) beginConversationSourceRead(ctx context.Context) (*sqlx.Tx, error) { + options := &sql.TxOptions{ReadOnly: true} + if dialect.IsPostgres(r.db.DriverName()) { + options.Isolation = sql.LevelRepeatableRead + } + return r.ro.BeginTxx(ctx, options) +} + +func readConversationState(ctx context.Context, tx *sqlx.Tx, _ string, sessionID string) (models.ConversationRevision, error) { + var exists int + if err := tx.QueryRowxContext(ctx, tx.Rebind(`SELECT 1 FROM task_sessions WHERE id = ?`), sessionID).Scan(&exists); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return models.ConversationRevision{SessionID: sessionID}, nil + } + return models.ConversationRevision{}, fmt.Errorf("read conversation session: %w", err) + } + var revision int64 + if err := tx.QueryRowxContext(ctx, tx.Rebind(`SELECT revision FROM conversation_session_revisions WHERE session_id = ?`), sessionID).Scan(&revision); err != nil && !errors.Is(err, sql.ErrNoRows) { + return models.ConversationRevision{}, fmt.Errorf("read conversation revision: %w", err) + } + return models.ConversationRevision{SessionID: sessionID, Revision: revision, Exists: exists == 1}, nil +} + +func conversationSourcePageLimit(limit int) int { + if limit <= 0 { + return 20 + } + if limit > 100 { + return 100 + } + return limit +} + +func readConversationMessageCursor(ctx context.Context, tx *sqlx.Tx, driver, sessionID, cursorID string) (string, string, error) { + if cursorID == "" { + return "", "", nil + } + var cursorSession string + var cursorCreated time.Time + query := `SELECT task_session_id, created_at FROM task_session_messages WHERE id = ?` + if err := tx.QueryRowxContext(ctx, tx.Rebind(query), cursorID).Scan(&cursorSession, &cursorCreated); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", "", fmt.Errorf("%w: message cursor %s", models.ErrConversationCursorStale, cursorID) + } + return "", "", err + } + if cursorSession != sessionID { + return "", "", fmt.Errorf("%w: message cursor %s", models.ErrConversationCursorStale, cursorID) + } + if dialect.IsPostgres(driver) { + return formatPromptKey(cursorCreated), cursorID, nil + } + var key string + if err := tx.QueryRowxContext(ctx, tx.Rebind(fmt.Sprintf( + `SELECT %s FROM task_session_messages WHERE id = ?`, + dialect.NormalizedMicrosecond(driver, "created_at"), + )), cursorID).Scan(&key); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", "", fmt.Errorf("%w: message cursor %s", models.ErrConversationCursorStale, cursorID) + } + return "", "", err + } + return key, cursorID, nil +} + +func readConversationTurnCursor(ctx context.Context, tx *sqlx.Tx, sessionID, cursorID string) (time.Time, string, error) { + if cursorID == "" { + return time.Time{}, "", nil + } + var cursorSession string + var cursorStarted time.Time + if err := tx.QueryRowxContext(ctx, tx.Rebind( + `SELECT task_session_id, started_at FROM task_session_turns WHERE id = ?`, + ), cursorID).Scan(&cursorSession, &cursorStarted); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return time.Time{}, "", fmt.Errorf("%w: turn cursor %s", models.ErrConversationCursorStale, cursorID) + } + return time.Time{}, "", err + } + if cursorSession != sessionID { + return time.Time{}, "", fmt.Errorf("%w: turn cursor %s", models.ErrConversationCursorStale, cursorID) + } + return cursorStarted, cursorID, nil +} + +func buildConversationMessagePageQuery( + driver string, + req models.ConversationMessagePageRequest, + cursorKey, cursorID string, + limit int, +) (string, []interface{}) { + normalized := dialect.NormalizedMicrosecond(driver, "created_at") + bound := "?" + if dialect.IsPostgres(driver) { + bound = conversationTimestampParameter + } + query := ` + SELECT id, task_session_id, task_id, turn_id, author_type, author_id, content, + requests_input, type, metadata, created_at, updated_at, + CASE WHEN author_type = 'user' THEN prompt_seq ELSE 0 END AS prompt_index + FROM task_session_messages + WHERE task_session_id = ?` + args := []interface{}{req.SessionID} + if req.TaskID != nil { + query += " AND task_id = ?" + args = append(args, *req.TaskID) + } + if len(req.Authors) > 0 { + placeholders := make([]string, len(req.Authors)) + for index, author := range req.Authors { + placeholders[index] = "?" + args = append(args, author) + } + query += " AND author_type IN (" + strings.Join(placeholders, ",") + ")" + } + sortOrder := conversationAscending + compare := ">" + if strings.EqualFold(req.Sort, "desc") { + sortOrder = conversationDescending + compare = "<" + } + if cursorID != "" { + query += fmt.Sprintf(" AND (%s %s %s OR (%s = %s AND id %s ?))", normalized, compare, bound, normalized, bound, compare) + args = append(args, cursorKey, cursorKey, cursorID) + } + query += fmt.Sprintf(" ORDER BY %s %s, id %s", normalized, sortOrder, sortOrder) + query += sqlLimitClause + args = append(args, limit+1) + return query, args +} + +func buildConversationTurnPageQuery(driver string, req models.ConversationTurnPageRequest, cursorStarted time.Time, cursorID string, limit int) (string, []interface{}) { + query := ` + SELECT id, task_session_id, task_id, execution_profile_id, route_generation, + started_at, completed_at, metadata, created_at, updated_at + FROM task_session_turns turn_row + WHERE turn_row.task_session_id = ? AND ` + turnHistoryPredicate(driver, "turn_row") + args := []interface{}{req.SessionID} + if req.TaskID != nil { + query += " AND turn_row.task_id = ?" + args = append(args, *req.TaskID) + } + sortOrder := conversationAscending + if strings.EqualFold(req.Sort, "desc") { + sortOrder = conversationDescending + } + if cursorID != "" { + compare := ">" + if sortOrder == conversationDescending { + compare = "<" + } + query += fmt.Sprintf(" AND (turn_row.started_at %s ? OR (turn_row.started_at = ? AND turn_row.id %s ?))", compare, compare) + args = append(args, cursorStarted, cursorStarted, cursorID) + } + query += " ORDER BY turn_row.started_at " + sortOrder + ", turn_row.id " + sortOrder + sqlLimitClause + args = append(args, limit+1) + return query, args +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_source_bench_test.go b/apps/backend/internal/task/repository/sqlite/conversation_source_bench_test.go new file mode 100644 index 0000000000..4c965ad94f --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_source_bench_test.go @@ -0,0 +1,100 @@ +package sqlite + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/db" +) + +// BenchmarkConversationSourceStorage provides disposable 1 KiB source-write +// and restart measurements for the storage replacement. Run it once when +// recording upgrade notes, for example with -benchtime=1x. +func BenchmarkConversationSourceStorage(b *testing.B) { + for _, messageCount := range []int{10_000, 100_000} { + b.Run(fmt.Sprintf("messages_%d", messageCount), func(b *testing.B) { + dbPath := filepath.Join(b.TempDir(), "conversation-source.db") + dbConn, err := db.OpenSQLite(dbPath) + if err != nil { + b.Fatal(err) + } + sqlxDB := sqlx.NewDb(dbConn, "sqlite3") + repo, err := NewWithDB(sqlxDB, sqlxDB, nil) + if err != nil { + b.Fatal(err) + } + seedConversationSourceBenchmark(b, repo) + payload := strings.Repeat("x", 1024) + insert := sqlxDB.Rebind(` + INSERT INTO task_session_messages + (id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at) + VALUES (?, ?, ?, ?, 'agent', '', ?, 0, 'message', '{}', ?) + `) + + b.ResetTimer() + for run := 0; run < b.N; run++ { + for index := 0; index < messageCount; index++ { + id := fmt.Sprintf("benchmark-message-%d-%d", run, index) + if _, err := sqlxDB.Exec(insert, id, "conversation-source-benchmark-session", "conversation-source-benchmark-task", "conversation-source-benchmark-turn", payload, time.Now().UTC()); err != nil { + b.Fatal(err) + } + } + } + b.StopTimer() + + stat, err := fileSize(dbPath) + if err != nil { + b.Fatal(err) + } + b.ReportMetric(float64(stat), "db-bytes") + + if err := sqlxDB.Close(); err != nil { + b.Fatal(err) + } + start := time.Now() + startupPath := dbPath + reopen, err := db.OpenSQLite(startupPath) + if err != nil { + b.Fatal(err) + } + reopenDB := sqlx.NewDb(reopen, "sqlite3") + if _, err := NewWithDB(reopenDB, reopenDB, nil); err != nil { + b.Fatal(err) + } + b.ReportMetric(float64(time.Since(start).Nanoseconds()), "startup-ns") + if err := reopenDB.Close(); err != nil { + b.Fatal(err) + } + }) + } +} + +func seedConversationSourceBenchmark(b *testing.B, repo *Repository) { + b.Helper() + now := time.Now().UTC() + if _, err := repo.db.Exec(repo.db.Rebind(`INSERT INTO workspaces (id, name, created_at, updated_at) VALUES ('conversation-source-benchmark-workspace', 'benchmark', ?, ?)`), now, now); err != nil { + b.Fatal(err) + } + if _, err := repo.db.Exec(repo.db.Rebind(`INSERT INTO tasks (id, workspace_id, title, created_at, updated_at) VALUES ('conversation-source-benchmark-task', 'conversation-source-benchmark-workspace', 'benchmark', ?, ?)`), now, now); err != nil { + b.Fatal(err) + } + if _, err := repo.db.Exec(repo.db.Rebind(`INSERT INTO task_sessions (id, task_id, queue_incarnation_id, started_at, updated_at) VALUES ('conversation-source-benchmark-session', 'conversation-source-benchmark-task', 'benchmark-incarnation', ?, ?)`), now, now); err != nil { + b.Fatal(err) + } + if _, err := repo.db.Exec(repo.db.Rebind(`INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at) VALUES ('conversation-source-benchmark-turn', 'conversation-source-benchmark-session', 'conversation-source-benchmark-task', ?, ?, ?)`), now, now, now); err != nil { + b.Fatal(err) + } +} + +func fileSize(path string) (int64, error) { + stat, err := os.Stat(path) + if err != nil { + return 0, err + } + return stat.Size(), nil +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_source_postgres_test.go b/apps/backend/internal/task/repository/sqlite/conversation_source_postgres_test.go new file mode 100644 index 0000000000..630e7ce227 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_source_postgres_test.go @@ -0,0 +1,803 @@ +package sqlite + +import ( + "context" + "fmt" + "reflect" + "sort" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" + "github.com/kandev/kandev/internal/testutil" +) + +func TestConversationPostgresSource(t *testing.T) { + dsn := testutil.PostgresDSNFromEnv(t) + db := testutil.OpenIsolatedPostgres(t, dsn) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + const ( + taskID = "task-conversation-source-pg" + sessionID = "session-conversation-source-pg" + turnID = "turn-conversation-source-pg" + ) + seedPostgresConversationSource(t, repo, taskID, sessionID, turnID) + taskFilter := taskID + + if got := readPostgresConversationRevision(t, repo, sessionID); got != 1 { + t.Fatalf("seed revision = %d, want 1", got) + } + firstAt := time.Date(2026, 9, 16, 12, 0, 0, 123456, time.UTC) + insertPostgresConversationMessage(t, repo, "message-source-pg-a", sessionID, taskID, turnID, "user", "first", firstAt, 1) + insertPostgresConversationMessage(t, repo, "message-source-pg-b", sessionID, taskID, turnID, "agent", "second", firstAt, 0) + if got := readPostgresConversationRevision(t, repo, sessionID); got != 3 { + t.Fatalf("insert revision = %d, want 3", got) + } + if _, err := repo.db.Exec(repo.db.Rebind(` + UPDATE task_session_turns SET completed_at = ?, updated_at = ? WHERE id = ? + `), firstAt, firstAt, turnID); err != nil { + t.Fatalf("complete turn: %v", err) + } + if got := readPostgresConversationRevision(t, repo, sessionID); got != 4 { + t.Fatalf("turn completion revision = %d, want 4", got) + } + + if _, err := repo.db.Exec(repo.db.Rebind(` + UPDATE task_session_messages SET content = ?, updated_at = ? WHERE id = ? + `), "updated", firstAt.Add(time.Second), "message-source-pg-a"); err != nil { + t.Fatalf("direct message update: %v", err) + } + if got := readPostgresConversationRevision(t, repo, sessionID); got != 5 { + t.Fatalf("update revision = %d, want 5", got) + } + + page, err := repo.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: sessionID, + TaskID: &taskFilter, + Authors: []string{"user"}, + Sort: "asc", + Limit: 1, + }) + if err != nil { + t.Fatalf("read filtered message page: %v", err) + } + if page.Revision != 5 || len(page.Messages) != 1 || page.Messages[0].ID != "message-source-pg-a" || page.Messages[0].Content != "updated" { + t.Fatalf("filtered page = %+v, want updated user message at revision 5", page) + } + if page.HasMore || page.CursorID != "" { + t.Fatalf("filtered page continuation = hasMore %v cursor %q, want no continuation", page.HasMore, page.CursorID) + } + + allPage, err := repo.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: sessionID, + Sort: "asc", + Limit: 1, + }) + if err != nil { + t.Fatalf("read first message page: %v", err) + } + if len(allPage.Messages) != 1 || allPage.Messages[0].ID != "message-source-pg-a" || !allPage.HasMore || allPage.CursorID != "message-source-pg-a" { + t.Fatalf("first page = %+v, want first message with continuation", allPage) + } + allPage, err = repo.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: sessionID, + Sort: "asc", + CursorID: allPage.CursorID, + Limit: 1, + }) + if err != nil { + t.Fatalf("read second message page: %v", err) + } + if len(allPage.Messages) != 1 || allPage.Messages[0].ID != "message-source-pg-b" || allPage.HasMore { + t.Fatalf("second page = %+v, want final message", allPage) + } + + if _, err := repo.db.Exec(repo.db.Rebind(`DELETE FROM task_session_messages WHERE id = ?`), "message-source-pg-b"); err != nil { + t.Fatalf("direct message delete: %v", err) + } + if got := readPostgresConversationRevision(t, repo, sessionID); got != 6 { + t.Fatalf("delete revision = %d, want 6", got) + } + + const movedSessionID = "session-conversation-source-pg-moved" + const movedTurnID = "turn-conversation-source-pg-moved" + seedPostgresConversationSource(t, repo, taskID+"-moved", movedSessionID, movedTurnID) + if _, err := repo.db.Exec(repo.db.Rebind(` + UPDATE task_session_messages SET task_session_id = ?, task_id = ? WHERE id = ? + `), movedSessionID, taskID+"-moved", "message-source-pg-a"); err != nil { + t.Fatalf("move message between sessions: %v", err) + } + if got := readPostgresConversationRevision(t, repo, sessionID); got != 7 { + t.Fatalf("source-session move revision = %d, want 7", got) + } + if got := readPostgresConversationRevision(t, repo, movedSessionID); got != 2 { + t.Fatalf("destination-session move revision = %d, want 2", got) + } + + if _, err := repo.db.Exec(repo.db.Rebind(` + UPDATE task_session_turns SET task_session_id = ? WHERE id = ? + `), movedSessionID, turnID); err != nil { + t.Fatalf("move turn between sessions: %v", err) + } + if got := readPostgresConversationRevision(t, repo, sessionID); got != 8 { + t.Fatalf("source-session turn move revision = %d, want 8", got) + } + if got := readPostgresConversationRevision(t, repo, movedSessionID); got != 3 { + t.Fatalf("destination-session turn move revision = %d, want 3", got) + } + + const ( + snapshotTaskID = "task-conversation-source-pg-snapshot" + snapshotSessionID = "session-conversation-source-pg-snapshot" + snapshotTurnID = "turn-conversation-source-pg-snapshot" + ) + seedPostgresConversationSource(t, repo, snapshotTaskID, snapshotSessionID, snapshotTurnID) + snapshotAt := firstAt.Add(2 * time.Minute) + insertPostgresConversationMessage(t, repo, "message-source-pg-snapshot", snapshotSessionID, snapshotTaskID, snapshotTurnID, "agent", "before snapshot", snapshotAt, 0) + secondDB := openSecondPostgresConnection(t, dsn, db) + readTx, err := repo.beginConversationSourceRead(ctx) + if err != nil { + t.Fatalf("begin repeatable-read source transaction: %v", err) + } + snapshotState, err := readConversationState(ctx, readTx, repo.db.DriverName(), snapshotSessionID) + if err != nil { + _ = readTx.Rollback() + t.Fatalf("read repeatable-read source state: %v", err) + } + var snapshotContent string + if err := readTx.QueryRowxContext(ctx, readTx.Rebind(`SELECT content FROM task_session_messages WHERE id = ?`), "message-source-pg-snapshot").Scan(&snapshotContent); err != nil { + _ = readTx.Rollback() + t.Fatalf("read repeatable-read source row: %v", err) + } + if _, err := secondDB.Exec(secondDB.Rebind(`UPDATE task_session_messages SET content = ? WHERE id = ?`), "after snapshot", "message-source-pg-snapshot"); err != nil { + _ = readTx.Rollback() + t.Fatalf("update source row after snapshot: %v", err) + } + currentState, err := readConversationState(ctx, readTx, repo.db.DriverName(), snapshotSessionID) + if err != nil { + _ = readTx.Rollback() + t.Fatalf("read repeatable-read state after concurrent commit: %v", err) + } + if snapshotState.Revision != 2 || currentState.Revision != snapshotState.Revision || snapshotContent != "before snapshot" { + _ = readTx.Rollback() + t.Fatalf("repeatable-read snapshot = state before %+v, state after %+v, content %q", snapshotState, currentState, snapshotContent) + } + if err := readTx.Commit(); err != nil { + t.Fatalf("commit repeatable-read source transaction: %v", err) + } + if got := readPostgresConversationRevision(t, repo, snapshotSessionID); got != 3 { + t.Fatalf("post-snapshot revision = %d, want 3", got) + } + + rollbackTx, err := repo.db.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("begin rollback transaction: %v", err) + } + if _, err := rollbackTx.Exec(repo.db.Rebind(` + INSERT INTO task_session_messages + (id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at, prompt_seq) + VALUES (?, ?, ?, ?, 'agent', '', ?, 0, 'message', '{}', ?, ?, 0) + `), "message-source-pg-rolled-back", movedSessionID, taskID+"-moved", movedTurnID, "rolled back", firstAt, firstAt); err != nil { + t.Fatalf("insert rollback message: %v", err) + } + if err := rollbackTx.Rollback(); err != nil { + t.Fatalf("rollback source mutation: %v", err) + } + if got := readPostgresConversationRevision(t, repo, movedSessionID); got != 3 { + t.Fatalf("rolled-back revision = %d, want 3", got) + } + var rolledBackCount int + if err := repo.db.Get(&rolledBackCount, repo.db.Rebind(`SELECT COUNT(*) FROM task_session_messages WHERE id = ?`), "message-source-pg-rolled-back"); err != nil { + t.Fatalf("count rolled-back message: %v", err) + } + if rolledBackCount != 0 { + t.Fatalf("rolled-back message count = %d, want 0", rolledBackCount) + } + + if _, err := repo.db.Exec(repo.db.Rebind(`DELETE FROM task_sessions WHERE id = ?`), movedSessionID); err != nil { + t.Fatalf("delete session cascade: %v", err) + } + for _, table := range []string{"task_session_messages", "task_session_turns", "conversation_session_revisions"} { + var count int + if err := repo.db.Get(&count, repo.db.Rebind(`SELECT COUNT(*) FROM `+table+` WHERE `+map[string]string{ + "task_session_messages": "task_session_id", + "task_session_turns": "task_session_id", + "conversation_session_revisions": "session_id", + }[table]+` = ?`), movedSessionID); err != nil { + t.Fatalf("count %s after session cascade: %v", table, err) + } + if count != 0 { + t.Fatalf("%s rows after session cascade = %d, want 0", table, count) + } + } + + var revisionRows, sessionCount int + if err := repo.db.Get(&revisionRows, `SELECT COUNT(*) FROM conversation_session_revisions`); err != nil { + t.Fatalf("count revision rows: %v", err) + } + if err := repo.db.Get(&sessionCount, `SELECT COUNT(*) FROM task_sessions`); err != nil { + t.Fatalf("count sessions: %v", err) + } + if revisionRows > sessionCount { + t.Fatalf("revision rows = %d, session rows = %d, revision table is not bounded by sessions", revisionRows, sessionCount) + } + + turnPage, err := repo.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{ + SessionID: sessionID, + Sort: "asc", + Limit: 10, + }) + if err != nil { + t.Fatalf("read moved turn page: %v", err) + } + if len(turnPage.Turns) != 0 { + t.Fatalf("old session turn page after move = %+v, want empty", turnPage.Turns) + } +} + +func TestConversationPostgresReceipts(t *testing.T) { + dsn := testutil.PostgresDSNFromEnv(t) + db := testutil.OpenIsolatedPostgres(t, dsn) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + const ( + taskID = "task-conversation-receipts-pg" + sessionID = "session-conversation-receipts-pg" + turnID = "turn-conversation-receipts-pg" + ) + seedPostgresConversationSource(t, repo, taskID, sessionID, turnID) + + message := &models.Message{ + ID: "message-conversation-receipts-pg", + TaskSessionID: sessionID, + TaskID: taskID, + TurnID: turnID, + AuthorType: models.MessageAuthorAgent, + Type: models.MessageTypeMessage, + Content: "receipt one", + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 123456, time.UTC), + } + receipt, err := repo.CreateMessageWithConversationReceipt(ctx, message) + if err != nil { + t.Fatalf("create message receipt: %v", err) + } + assertPostgresConversationReceipt(t, receipt, sessionID, 1, 2, true, models.ConversationMutationUpsert, models.ConversationEntityMessage, message.ID) + if receipt.Operations[0].Message == nil || receipt.Operations[0].Message.Content != message.Content { + t.Fatalf("create receipt payload = %+v, want source content", receipt.Operations[0]) + } + + message.Content = "receipt two" + receipt, err = repo.UpdateMessageWithConversationReceipt(ctx, message) + if err != nil { + t.Fatalf("update message receipt: %v", err) + } + assertPostgresConversationReceipt(t, receipt, sessionID, 2, 3, true, models.ConversationMutationUpsert, models.ConversationEntityMessage, message.ID) + if receipt.Operations[0].Message == nil || receipt.Operations[0].Message.Content != "receipt two" { + t.Fatalf("update receipt payload = %+v, want updated source content", receipt.Operations[0]) + } + + receipt, err = repo.DeleteMessageWithConversationReceipt(ctx, message.ID) + if err != nil { + t.Fatalf("delete message receipt: %v", err) + } + assertPostgresConversationReceipt(t, receipt, sessionID, 3, 4, true, models.ConversationMutationRemove, models.ConversationEntityMessage, message.ID) + if receipt.Operations[0].Message != nil { + t.Fatalf("delete receipt unexpectedly includes message payload: %+v", receipt.Operations[0]) + } + + newTurn := &models.Turn{ + ID: "turn-conversation-receipts-pg-created", + TaskSessionID: sessionID, + TaskID: taskID, + StartedAt: time.Date(2026, 9, 16, 12, 1, 0, 0, time.UTC), + CreatedAt: time.Date(2026, 9, 16, 12, 1, 0, 0, time.UTC), + } + receipt, err = repo.CreateTurnWithConversationReceipt(ctx, newTurn) + if err != nil { + t.Fatalf("create turn receipt: %v", err) + } + assertPostgresConversationReceipt(t, receipt, sessionID, 4, 5, true, models.ConversationMutationUpsert, models.ConversationEntityTurn, newTurn.ID) + + receipt, err = repo.CompleteTurnWithConversationReceipt(ctx, newTurn.ID) + if err != nil { + t.Fatalf("complete turn receipt: %v", err) + } + assertPostgresConversationReceipt(t, receipt, sessionID, 5, 6, true, models.ConversationMutationUpsert, models.ConversationEntityTurn, newTurn.ID) + if receipt.Operations[0].Turn == nil || receipt.Operations[0].Turn.CompletedAt == nil { + t.Fatalf("completed turn receipt = %+v, want completed source turn", receipt.Operations[0]) + } + + badMessage := &models.Message{ + ID: "message-conversation-receipts-pg-rollback", + TaskSessionID: sessionID, + TaskID: taskID, + TurnID: "turn-conversation-receipts-pg-missing", + AuthorType: models.MessageAuthorAgent, + Content: "must roll back", + } + if _, err := repo.CreateMessageWithConversationReceipt(ctx, badMessage); err == nil { + t.Fatal("message receipt with missing turn succeeded") + } + if got := readPostgresConversationRevision(t, repo, sessionID); got != 6 { + t.Fatalf("failed receipt revision = %d, want 6", got) + } + var failedMessageCount int + if err := repo.db.Get(&failedMessageCount, repo.db.Rebind(`SELECT COUNT(*) FROM task_session_messages WHERE id = ?`), badMessage.ID); err != nil { + t.Fatalf("count rolled-back receipt message: %v", err) + } + if failedMessageCount != 0 { + t.Fatalf("rolled-back receipt message count = %d, want 0", failedMessageCount) + } + + secondDB := openSecondPostgresConnection(t, dsn, db) + secondRepo, err := NewWithDB(secondDB, secondDB, nil) + if err != nil { + t.Fatalf("init second postgres repository: %v", err) + } + start := make(chan struct{}) + results := make(chan postgresReceiptResult, 2) + for index, writer := range []*Repository{repo, secondRepo} { + index, writer := index, writer + go func() { + <-start + concurrentMessage := &models.Message{ + ID: fmt.Sprintf("message-conversation-receipts-pg-concurrent-%d", index), + TaskSessionID: sessionID, + TaskID: taskID, + TurnID: turnID, + AuthorType: models.MessageAuthorAgent, + Content: fmt.Sprintf("concurrent receipt %d", index), + } + concurrentReceipt, receiptErr := writer.CreateMessageWithConversationReceipt(context.Background(), concurrentMessage) + results <- postgresReceiptResult{receipt: concurrentReceipt, err: receiptErr} + }() + } + close(start) + concurrentCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + concurrentReceipts := make([]*models.ConversationMutationReceipt, 0, 2) + for len(concurrentReceipts) < 2 { + select { + case result := <-results: + if result.err != nil { + t.Fatalf("concurrent receipt write: %v", result.err) + } + concurrentReceipts = append(concurrentReceipts, result.receipt) + case <-concurrentCtx.Done(): + t.Fatal("timed out waiting for concurrent receipt writes") + } + } + sort.Slice(concurrentReceipts, func(left, right int) bool { + return concurrentReceipts[left].BaseRevision < concurrentReceipts[right].BaseRevision + }) + if concurrentReceipts[0].BaseRevision != 6 || concurrentReceipts[0].Revision != 7 || !concurrentReceipts[0].Complete { + t.Fatalf("first concurrent receipt = %+v, want interval 6..7", concurrentReceipts[0]) + } + if concurrentReceipts[1].BaseRevision != 7 || concurrentReceipts[1].Revision != 8 || !concurrentReceipts[1].Complete { + t.Fatalf("second concurrent receipt = %+v, want interval 7..8", concurrentReceipts[1]) + } + + // A single receipt cannot describe two direct source changes. The receipt + // must expose the larger transaction interval so the subscriber requests a + // source reconciliation instead of applying an incomplete delta. + intervalTx, err := repo.db.BeginTxx(ctx, nil) + if err != nil { + t.Fatalf("begin multi-row receipt transaction: %v", err) + } + base, err := repo.ensureConversationRevisionTx(ctx, intervalTx, sessionID) + if err != nil { + _ = intervalTx.Rollback() + t.Fatalf("lock multi-row receipt transaction: %v", err) + } + for index := 0; index < 2; index++ { + id := fmt.Sprintf("message-conversation-receipts-pg-multi-%d", index) + if _, err := intervalTx.Exec(repo.db.Rebind(` + INSERT INTO task_session_messages + (id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at, prompt_seq) + VALUES (?, ?, ?, ?, 'agent', '', ?, 0, 'message', '{}', ?, ?, 0) + `), id, sessionID, taskID, turnID, "multi-row", time.Now().UTC(), time.Now().UTC()); err != nil { + _ = intervalTx.Rollback() + t.Fatalf("insert multi-row receipt message %d: %v", index, err) + } + } + intervalReceipt := &models.ConversationMutationReceipt{} + if err := repo.populateConversationMessageReceipt(ctx, intervalTx, intervalReceipt, base, &models.Message{ + ID: "message-conversation-receipts-pg-multi-0", TaskSessionID: sessionID, TaskID: taskID, + TurnID: turnID, AuthorType: models.MessageAuthorAgent, + }, models.ConversationMutationUpsert); err != nil { + _ = intervalTx.Rollback() + t.Fatalf("build incomplete receipt: %v", err) + } + if intervalReceipt.BaseRevision != 8 || intervalReceipt.Revision != 10 || intervalReceipt.Complete { + _ = intervalTx.Rollback() + t.Fatalf("incomplete receipt = %+v, want interval 8..10 with complete=false", intervalReceipt) + } + if err := intervalTx.Commit(); err != nil { + t.Fatalf("commit multi-row receipt transaction: %v", err) + } +} + +type postgresReceiptResult struct { + receipt *models.ConversationMutationReceipt + err error +} + +func assertPostgresConversationReceipt( + t *testing.T, + receipt *models.ConversationMutationReceipt, + sessionID string, + base, revision int64, + complete bool, + kind models.ConversationMutationKind, + entity models.ConversationEntityKind, + id string, +) { + t.Helper() + if receipt == nil { + t.Fatal("conversation receipt is nil") + } + if receipt.SessionID != sessionID || receipt.BaseRevision != base || receipt.Revision != revision || receipt.Complete != complete || len(receipt.Operations) != 1 { + t.Fatalf("conversation receipt = %+v, want session %s interval %d..%d complete=%v", receipt, sessionID, base, revision, complete) + } + operation := receipt.Operations[0] + if operation.Kind != kind || operation.Entity != entity || operation.ID != id || operation.SessionID != sessionID { + t.Fatalf("conversation receipt operation = %+v, want %s/%s/%s", operation, kind, entity, id) + } +} + +func TestConversationPostgresCleanup(t *testing.T) { + db := testutil.OpenIsolatedPostgres(t, testutil.PostgresDSNFromEnv(t)) + repo, err := NewWithDB(db, db, nil) + if err != nil { + t.Fatalf("init postgres schema: %v", err) + } + ctx := context.Background() + const ( + taskID = "task-conversation-cleanup-pg" + sessionID = "session-conversation-cleanup-pg" + turnID = "turn-conversation-cleanup-pg" + messageID = "message-conversation-cleanup-pg" + ) + seedPostgresConversationSource(t, repo, taskID, sessionID, turnID) + sourceAt := time.Date(2026, 9, 16, 12, 0, 0, 123456, time.UTC) + insertPostgresConversationMessage(t, repo, messageID, sessionID, taskID, turnID, "agent", "source bytes survive cleanup", sourceAt, 0) + wantSource := readPostgresConversationSentinel(t, repo, messageID) + wantIndexes := readPostgresConversationSourceIndexes(t, repo) + + if _, err := repo.db.ExecContext(ctx, postgresLegacyConversationJournalDDL); err != nil { + t.Fatalf("create comparison-base legacy postgres fixture: %v", err) + } + if _, err := repo.db.ExecContext(ctx, postgresUnrelatedCleanupObjectDDL); err != nil { + t.Fatalf("create unrelated postgres cleanup fixture: %v", err) + } + assertPostgresLegacyConversationObjects(t, repo, true) + assertPostgresUnrelatedCleanupObject(t, repo, true) + + repo.failConversationJournalCleanupAfter = "statement-6" + if err := repo.cleanupLegacyConversationJournal(); err == nil { + t.Fatal("cleanup failure injection succeeded") + } + assertPostgresLegacyConversationObjects(t, repo, true) + assertPostgresUnrelatedCleanupObject(t, repo, true) + assertPostgresConversationSentinel(t, repo, messageID, wantSource) + if got := readPostgresConversationSourceIndexes(t, repo); !reflect.DeepEqual(got, wantIndexes) { + t.Fatalf("source indexes after rolled-back cleanup = %#v, want %#v", got, wantIndexes) + } + + repo.failConversationJournalCleanupAfter = "" + if err := repo.cleanupLegacyConversationJournal(); err != nil { + t.Fatalf("retry comparison-base cleanup: %v", err) + } + assertPostgresLegacyConversationObjects(t, repo, false) + assertPostgresUnrelatedCleanupObject(t, repo, true) + assertPostgresConversationSentinel(t, repo, messageID, wantSource) + if got := readPostgresConversationSourceIndexes(t, repo); !reflect.DeepEqual(got, wantIndexes) { + t.Fatalf("source indexes after cleanup = %#v, want %#v", got, wantIndexes) + } + if err := repo.cleanupLegacyConversationJournal(); err != nil { + t.Fatalf("repeated fresh-schema cleanup: %v", err) + } +} + +type postgresConversationSentinel struct { + Content string + Metadata string + CreatedAt time.Time + UpdatedAt time.Time +} + +func readPostgresConversationSentinel(t *testing.T, repo *Repository, messageID string) postgresConversationSentinel { + t.Helper() + var sentinel postgresConversationSentinel + if err := repo.db.QueryRowx(repo.db.Rebind(` + SELECT content, metadata, created_at, updated_at + FROM task_session_messages WHERE id = ? + `), messageID).Scan(&sentinel.Content, &sentinel.Metadata, &sentinel.CreatedAt, &sentinel.UpdatedAt); err != nil { + t.Fatalf("read source sentinel %s: %v", messageID, err) + } + return sentinel +} + +func assertPostgresConversationSentinel(t *testing.T, repo *Repository, messageID string, want postgresConversationSentinel) { + t.Helper() + got := readPostgresConversationSentinel(t, repo, messageID) + if !reflect.DeepEqual(got, want) { + t.Fatalf("source sentinel after cleanup = %#v, want %#v", got, want) + } +} + +func readPostgresConversationSourceIndexes(t *testing.T, repo *Repository) map[string]string { + t.Helper() + rows, err := repo.db.Queryx(` + SELECT tablename, indexname, indexdef + FROM pg_indexes + WHERE schemaname = current_schema() + AND tablename IN ('task_session_messages', 'task_session_turns', 'conversation_session_revisions') + ORDER BY tablename, indexname + `) + if err != nil { + t.Fatalf("read conversation source indexes: %v", err) + } + defer func() { _ = rows.Close() }() + indexes := make(map[string]string) + for rows.Next() { + var table, name, definition string + if err := rows.Scan(&table, &name, &definition); err != nil { + t.Fatalf("scan conversation source index: %v", err) + } + indexes[table+":"+name] = definition + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate conversation source indexes: %v", err) + } + return indexes +} + +func assertPostgresLegacyConversationObjects(t *testing.T, repo *Repository, wantPresent bool) { + t.Helper() + for _, table := range []string{ + "conversation_session_streams", + "conversation_session_events", + "conversation_message_versions", + "conversation_turn_versions", + "conversation_journal_meta", + } { + assertPostgresCleanupObject(t, repo, "table", table, wantPresent) + } + for _, function := range []string{ + "conversation_message_journal", + "conversation_turn_journal", + "conversation_session_delete_journal", + } { + assertPostgresCleanupObject(t, repo, "function", function, wantPresent) + } + assertPostgresCleanupFunction(t, repo, "conversation_next_sequence", 2, wantPresent) + assertPostgresCleanupFunction(t, repo, "conversation_safe_jsonb", 1, wantPresent) + assertPostgresCleanupFunction(t, repo, "conversation_visible_content", 1, wantPresent) + for _, trigger := range []string{ + "conversation_message_journal_trigger", + "conversation_turn_journal_trigger", + "conversation_session_delete_trigger", + } { + assertPostgresCleanupObject(t, repo, "trigger", trigger, wantPresent) + } +} + +func assertPostgresUnrelatedCleanupObject(t *testing.T, repo *Repository, wantPresent bool) { + t.Helper() + assertPostgresCleanupObject(t, repo, "table", "conversation_cleanup_unrelated", wantPresent) + assertPostgresCleanupObject(t, repo, "function", "conversation_cleanup_unrelated", wantPresent) + assertPostgresCleanupObject(t, repo, "trigger", "conversation_cleanup_unrelated_trigger", wantPresent) +} + +func assertPostgresCleanupObject(t *testing.T, repo *Repository, kind, name string, wantPresent bool) { + t.Helper() + var present bool + var err error + switch kind { + case "table": + err = repo.db.Get(&present, ` + SELECT EXISTS ( + SELECT 1 FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() + AND c.relname = $1 AND c.relkind IN ('r', 'p') + )`, name) + case "function": + err = repo.db.Get(&present, ` + SELECT EXISTS ( + SELECT 1 FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = current_schema() AND p.proname = $1 AND p.pronargs = 0 + )`, name) + case "trigger": + err = repo.db.Get(&present, ` + SELECT EXISTS ( + SELECT 1 + FROM pg_trigger tr + JOIN pg_class c ON c.oid = tr.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() AND tr.tgname = $1 AND NOT tr.tgisinternal + )`, name) + default: + t.Fatalf("unsupported postgres cleanup object kind %q", kind) + } + if err != nil { + t.Fatalf("inspect postgres %s %q: %v", kind, name, err) + } + if present != wantPresent { + t.Fatalf("postgres %s %q present = %v, want %v", kind, name, present, wantPresent) + } +} + +func assertPostgresCleanupFunction(t *testing.T, repo *Repository, name string, argumentCount int, wantPresent bool) { + t.Helper() + var present bool + if err := repo.db.Get(&present, ` + SELECT EXISTS ( + SELECT 1 FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = current_schema() AND p.proname = $1 AND p.pronargs = $2 + )`, name, argumentCount); err != nil { + t.Fatalf("inspect postgres function %q: %v", name, err) + } + if present != wantPresent { + t.Fatalf("postgres function %q(%d args) present = %v, want %v", name, argumentCount, present, wantPresent) + } +} + +// postgresLegacyConversationJournalDDL is copied from the comparison base's +// conversationJournalTables declaration. The fixture intentionally retains +// the original PostgreSQL types and indexes so cleanup is tested against the +// upgrade shape, not a SQLite approximation. +const postgresLegacyConversationJournalDDL = ` +CREATE TABLE IF NOT EXISTS conversation_session_streams ( + session_id TEXT PRIMARY KEY, + watermark BIGINT NOT NULL DEFAULT 0, + terminal BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMP NOT NULL +); +CREATE TABLE IF NOT EXISTS conversation_session_events ( + session_id TEXT NOT NULL, + sequence BIGINT NOT NULL, + event_id TEXT NOT NULL UNIQUE, + protocol_version INTEGER NOT NULL DEFAULT 1, + event_type TEXT NOT NULL, + task_id TEXT, + payload TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (session_id, sequence) +); +CREATE INDEX IF NOT EXISTS idx_conversation_events_created + ON conversation_session_events(created_at); +CREATE TABLE IF NOT EXISTS conversation_message_versions ( + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + row_sequence BIGINT NOT NULL, + task_id TEXT, + author_type TEXT, + created_at TIMESTAMP NOT NULL, + tombstone BOOLEAN NOT NULL DEFAULT FALSE, + payload TEXT NOT NULL, + PRIMARY KEY (session_id, message_id, row_sequence) +); +CREATE INDEX IF NOT EXISTS idx_conversation_message_snapshot + ON conversation_message_versions(session_id, row_sequence, created_at, message_id); +CREATE TABLE IF NOT EXISTS conversation_turn_versions ( + session_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + row_sequence BIGINT NOT NULL, + task_id TEXT, + started_at TIMESTAMP NOT NULL, + tombstone BOOLEAN NOT NULL DEFAULT FALSE, + payload TEXT NOT NULL, + PRIMARY KEY (session_id, turn_id, row_sequence) +); +CREATE INDEX IF NOT EXISTS idx_conversation_turn_snapshot + ON conversation_turn_versions(session_id, row_sequence, started_at, turn_id); +CREATE TABLE IF NOT EXISTS conversation_journal_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +CREATE OR REPLACE FUNCTION conversation_message_journal() RETURNS TRIGGER AS $$ +BEGIN + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION conversation_turn_journal() RETURNS TRIGGER AS $$ +BEGIN + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION conversation_session_delete_journal() RETURNS TRIGGER AS $$ +BEGIN + RETURN OLD; +END; +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION conversation_next_sequence(p_session_id TEXT, p_terminal BOOLEAN DEFAULT FALSE) +RETURNS BIGINT AS $$ +BEGIN + RETURN 1; +END; +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION conversation_safe_jsonb(value TEXT) +RETURNS JSONB AS $$ +BEGIN + RETURN '{}'::jsonb; +END; +$$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION conversation_visible_content(value TEXT) +RETURNS TEXT AS $$ +BEGIN + RETURN value; +END; +$$ LANGUAGE plpgsql; +DROP TRIGGER IF EXISTS conversation_message_journal_trigger ON task_session_messages; +CREATE TRIGGER conversation_message_journal_trigger AFTER INSERT OR UPDATE OR DELETE ON task_session_messages +FOR EACH ROW EXECUTE FUNCTION conversation_message_journal(); +DROP TRIGGER IF EXISTS conversation_turn_journal_trigger ON task_session_turns; +CREATE TRIGGER conversation_turn_journal_trigger AFTER INSERT OR UPDATE OR DELETE ON task_session_turns +FOR EACH ROW EXECUTE FUNCTION conversation_turn_journal(); +DROP TRIGGER IF EXISTS conversation_session_delete_trigger ON task_sessions; +CREATE TRIGGER conversation_session_delete_trigger AFTER DELETE ON task_sessions +FOR EACH ROW EXECUTE FUNCTION conversation_session_delete_journal(); +` + +const postgresUnrelatedCleanupObjectDDL = ` +CREATE TABLE conversation_cleanup_unrelated (id TEXT PRIMARY KEY); +CREATE OR REPLACE FUNCTION conversation_cleanup_unrelated() RETURNS TRIGGER AS $$ +BEGIN + RETURN NEW; +END; +$$ LANGUAGE plpgsql; +CREATE TRIGGER conversation_cleanup_unrelated_trigger AFTER INSERT ON task_sessions +FOR EACH ROW EXECUTE FUNCTION conversation_cleanup_unrelated(); +` + +func seedPostgresConversationSource(t *testing.T, repo *Repository, taskID, sessionID, turnID string) { + t.Helper() + seedPostgresTask(t, repo, taskID) + now := time.Date(2026, 9, 16, 11, 0, 0, 0, time.UTC) + if _, err := repo.db.Exec(repo.db.Rebind(` + INSERT INTO task_sessions (id, task_id, started_at, updated_at) + VALUES (?, ?, ?, ?) + `), sessionID, taskID, now, now); err != nil { + t.Fatalf("seed session %s: %v", sessionID, err) + } + if _, err := repo.db.Exec(repo.db.Rebind(` + INSERT INTO task_session_turns (id, task_session_id, task_id, started_at, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?) + `), turnID, sessionID, taskID, now, now, now); err != nil { + t.Fatalf("seed turn %s: %v", turnID, err) + } +} + +func insertPostgresConversationMessage(t *testing.T, repo *Repository, id, sessionID, taskID, turnID, author, content string, createdAt time.Time, promptSeq int) { + t.Helper() + if _, err := repo.db.Exec(repo.db.Rebind(` + INSERT INTO task_session_messages + (id, task_session_id, task_id, turn_id, author_type, author_id, content, requests_input, type, metadata, created_at, updated_at, prompt_seq) + VALUES (?, ?, ?, ?, ?, '', ?, 0, 'message', '{}', ?, ?, ?) + `), id, sessionID, taskID, turnID, author, content, createdAt, createdAt, promptSeq); err != nil { + t.Fatalf("insert message %s: %v", id, err) + } +} + +func readPostgresConversationRevision(t *testing.T, repo *Repository, sessionID string) int64 { + t.Helper() + revision, err := repo.ReadConversationRevision(context.Background(), sessionID) + if err != nil { + t.Fatalf("read revision for %s: %v", sessionID, err) + } + if !revision.Exists { + t.Fatalf("session %s unexpectedly missing", sessionID) + } + return revision.Revision +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_source_test.go b/apps/backend/internal/task/repository/sqlite/conversation_source_test.go new file mode 100644 index 0000000000..18feb180f1 --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_source_test.go @@ -0,0 +1,341 @@ +package sqlite + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kandev/kandev/internal/task/models" +) + +func TestConversationSourceRevisionTracksMessageLifecycle(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const sessionID = "session-source-revision" + seedForMsgTest(t, repo, "task-source-revision", sessionID, "turn-source-revision") + + // seedForMsgTest creates the turn that owns the message, so the source + // revision already includes that persisted turn. + assertConversationRevision(t, repo, sessionID, 1) + + message := &models.Message{ + ID: "message-source-revision", + TaskSessionID: sessionID, + TaskID: "task-source-revision", + TurnID: "turn-source-revision", + AuthorType: models.MessageAuthorAgent, + Type: models.MessageTypeMessage, + Content: "first", + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + if err := repo.CreateMessage(ctx, message); err != nil { + t.Fatalf("create message: %v", err) + } + assertConversationRevision(t, repo, sessionID, 2) + + message.Content = "updated" + if err := repo.UpdateMessage(ctx, message); err != nil { + t.Fatalf("update message: %v", err) + } + assertConversationRevision(t, repo, sessionID, 3) + + if err := repo.DeleteMessage(ctx, message.ID); err != nil { + t.Fatalf("delete message: %v", err) + } + assertConversationRevision(t, repo, sessionID, 4) +} + +func TestConversationSourceMessagePageReturnsConsistentRevision(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-source-page" + sessionID = "session-source-page" + turnID = "turn-source-page" + ) + seedForMsgTest(t, repo, taskID, sessionID, turnID) + insertAgentMsg(t, repo, "message-source-page-a", sessionID, turnID, "user", "a", time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)) + insertAgentMsg(t, repo, "message-source-page-b", sessionID, turnID, "agent", "b", time.Date(2026, 9, 16, 12, 0, 1, 0, time.UTC)) + + reader, ok := any(repo).(interface { + ReadConversationMessagesPage(context.Context, models.ConversationMessagePageRequest) (models.ConversationMessagePage, error) + }) + if !ok { + t.Fatal("repository does not implement source-backed message pages") + } + page, err := reader.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: sessionID, + Sort: "asc", + Limit: 1, + }) + if err != nil { + t.Fatalf("read message page: %v", err) + } + if page.Revision != 3 { + t.Fatalf("page revision = %d, want 3", page.Revision) + } + if len(page.Messages) != 1 || page.Messages[0].ID != "message-source-page-a" { + t.Fatalf("page messages = %#v", page.Messages) + } + if !page.HasMore || page.CursorID != "message-source-page-a" { + t.Fatalf("page continuation = hasMore %v cursor %q", page.HasMore, page.CursorID) + } +} + +func TestConversationSourceTurnPageUsesKeysetCursor(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-source-turn-page" + sessionID = "session-source-turn-page" + ) + seedForMsgTest(t, repo, taskID, sessionID, "turn-source-turn-page-a") + secondStarted := time.Now().UTC().Add(time.Second) + for _, turn := range []*models.Turn{ + {ID: "turn-source-turn-page-b", TaskSessionID: sessionID, TaskID: taskID, StartedAt: secondStarted, CreatedAt: secondStarted}, + } { + // The seed turn already owns the first ID. Only insert the second row. + if turn.ID == "turn-source-turn-page-b" { + if err := repo.CreateTurn(ctx, turn); err != nil { + t.Fatalf("create second turn: %v", err) + } + } + } + + page, err := repo.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{ + SessionID: sessionID, + Sort: "asc", + Limit: 1, + }) + if err != nil { + t.Fatalf("read first turn page: %v", err) + } + if len(page.Turns) != 1 || page.Turns[0].ID != "turn-source-turn-page-a" { + t.Fatalf("first turn page = %#v (id=%q)", page.Turns, page.Turns[0].ID) + } + if !page.HasMore || page.CursorID != "turn-source-turn-page-a" { + t.Fatalf("first turn continuation = hasMore %v cursor %q", page.HasMore, page.CursorID) + } + + page, err = repo.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{ + SessionID: sessionID, + Sort: "asc", + CursorID: page.CursorID, + Limit: 1, + }) + if err != nil { + t.Fatalf("read second turn page: %v", err) + } + if len(page.Turns) != 1 || page.Turns[0].ID != "turn-source-turn-page-b" || page.HasMore { + t.Fatalf("second turn page = %#v, hasMore=%v", page.Turns, page.HasMore) + } +} + +func TestConversationSourceReadsRejectCursorFromAnotherSession(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + seedForMsgTest(t, repo, "task-source-cursor-a", "session-source-cursor-a", "turn-source-cursor-a") + seedForMsgTest(t, repo, "task-source-cursor-b", "session-source-cursor-b", "turn-source-cursor-b") + insertAgentMsg(t, repo, "message-source-cursor-b", "session-source-cursor-b", "turn-source-cursor-b", "agent", "b", time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)) + + _, err := repo.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: "session-source-cursor-a", + Sort: "asc", + CursorID: "message-source-cursor-b", + Limit: 1, + }) + if err == nil { + t.Fatal("message cursor from another session was accepted") + } + + _, err = repo.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{ + SessionID: "session-source-cursor-a", + Sort: "asc", + CursorID: "turn-source-cursor-b", + Limit: 1, + }) + if err == nil { + t.Fatal("turn cursor from another session was accepted") + } +} + +func TestConversationSourceReadsReturnStaleCursorAfterBoundaryDeletion(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const ( + taskID = "task-source-stale-cursor" + sessionID = "session-source-stale-cursor" + turnID = "turn-source-stale-cursor" + ) + seedForMsgTest(t, repo, taskID, sessionID, turnID) + insertAgentMsg(t, repo, "message-source-stale-a", sessionID, turnID, "agent", "a", time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC)) + insertAgentMsg(t, repo, "message-source-stale-b", sessionID, turnID, "agent", "b", time.Date(2026, 9, 16, 12, 0, 1, 0, time.UTC)) + messagePage, err := repo.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: sessionID, Sort: "asc", Limit: 1, + }) + if err != nil { + t.Fatalf("read message cursor page: %v", err) + } + if err := repo.DeleteMessage(ctx, messagePage.CursorID); err != nil { + t.Fatalf("delete message cursor boundary: %v", err) + } + _, err = repo.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{ + SessionID: sessionID, Sort: "asc", CursorID: messagePage.CursorID, Limit: 1, + }) + if !errors.Is(err, models.ErrConversationCursorStale) { + t.Fatalf("deleted message cursor error = %v, want ErrConversationCursorStale", err) + } + + const ( + turnTaskID = "task-source-stale-turn-cursor" + turnSessionID = "session-source-stale-turn-cursor" + ) + seedForMsgTest(t, repo, turnTaskID, turnSessionID, "turn-source-stale-turn-a") + secondTurnAt := time.Date(2026, 9, 16, 12, 1, 0, 0, time.UTC) + if err := repo.CreateTurn(ctx, &models.Turn{ + ID: "turn-source-stale-turn-b", TaskSessionID: turnSessionID, TaskID: turnTaskID, + StartedAt: secondTurnAt, CreatedAt: secondTurnAt, + }); err != nil { + t.Fatalf("create second turn: %v", err) + } + turnPage, err := repo.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{ + SessionID: turnSessionID, Sort: "asc", Limit: 1, + }) + if err != nil { + t.Fatalf("read turn cursor page: %v", err) + } + if _, err := repo.db.ExecContext(ctx, repo.db.Rebind(`DELETE FROM task_session_turns WHERE id = ?`), turnPage.CursorID); err != nil { + t.Fatalf("delete turn cursor boundary: %v", err) + } + _, err = repo.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{ + SessionID: turnSessionID, Sort: "asc", CursorID: turnPage.CursorID, Limit: 1, + }) + if !errors.Is(err, models.ErrConversationCursorStale) { + t.Fatalf("deleted turn cursor error = %v, want ErrConversationCursorStale", err) + } +} + +func TestConversationSourceRevisionMissingRowMeansZero(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const sessionID = "session-source-zero" + seedForMsgTest(t, repo, "task-source-zero", sessionID, "turn-source-zero") + if _, err := repo.db.Exec(`DELETE FROM conversation_session_revisions WHERE session_id = ?`, sessionID); err != nil { + t.Fatalf("delete revision row: %v", err) + } + + revision, err := repo.ReadConversationRevision(ctx, sessionID) + if err != nil { + t.Fatalf("read zero revision: %v", err) + } + if !revision.Exists || revision.Revision != 0 { + t.Fatalf("revision = %+v, want existing revision zero", revision) + } +} + +func TestConversationSourceRevisionRowCascadesWithSession(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const sessionID = "session-source-cascade" + seedForMsgTest(t, repo, "task-source-cascade", sessionID, "turn-source-cascade") + if _, err := repo.db.ExecContext(ctx, `DELETE FROM task_sessions WHERE id = ?`, sessionID); err != nil { + t.Fatalf("delete session: %v", err) + } + + var count int + if err := repo.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM conversation_session_revisions WHERE session_id = ?`, sessionID).Scan(&count); err != nil { + t.Fatalf("count revision rows: %v", err) + } + if count != 0 { + t.Fatalf("revision row count = %d, want 0", count) + } +} + +func TestConversationMutationReceiptCapturesMessageTransaction(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const sessionID = "session-source-receipt-message" + seedForMsgTest(t, repo, "task-source-receipt-message", sessionID, "turn-source-receipt-message") + message := &models.Message{ + ID: "message-source-receipt", + TaskSessionID: sessionID, + TaskID: "task-source-receipt-message", + TurnID: "turn-source-receipt-message", + AuthorType: models.MessageAuthorAgent, + Content: "receipt", + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + + receipt, err := repo.CreateMessageWithConversationReceipt(ctx, message) + if err != nil { + t.Fatalf("create message with receipt: %v", err) + } + if receipt.BaseRevision != 1 || receipt.Revision != 2 || !receipt.Complete || len(receipt.Operations) != 1 { + t.Fatalf("create receipt = %+v", receipt) + } + operation := receipt.Operations[0] + if operation.Kind != models.ConversationMutationUpsert || operation.Entity != models.ConversationEntityMessage || operation.ID != message.ID || operation.Message == nil { + t.Fatalf("create operation = %+v", operation) + } + + message.Content = "updated receipt" + receipt, err = repo.UpdateMessageWithConversationReceipt(ctx, message) + if err != nil { + t.Fatalf("update message with receipt: %v", err) + } + if receipt.BaseRevision != 2 || receipt.Revision != 3 || receipt.Operations[0].Message == nil || receipt.Operations[0].Message.Content != "updated receipt" { + t.Fatalf("update receipt = %+v", receipt) + } + + receipt, err = repo.DeleteMessageWithConversationReceipt(ctx, message.ID) + if err != nil { + t.Fatalf("delete message with receipt: %v", err) + } + if receipt.BaseRevision != 3 || receipt.Revision != 4 || receipt.Operations[0].Kind != models.ConversationMutationRemove || receipt.Operations[0].Message != nil { + t.Fatalf("delete receipt = %+v", receipt) + } +} + +func TestConversationMutationReceiptCapturesTurnTransaction(t *testing.T) { + repo := newRepoForSessionTests(t) + ctx := context.Background() + const sessionID = "session-source-receipt-turn" + seedForMsgTest(t, repo, "task-source-receipt-turn", sessionID, "turn-source-receipt-turn-seed") + turn := &models.Turn{ + ID: "turn-source-receipt-turn", + TaskSessionID: sessionID, + TaskID: "task-source-receipt-turn", + StartedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + CreatedAt: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + receipt, err := repo.CreateTurnWithConversationReceipt(ctx, turn) + if err != nil { + t.Fatalf("create turn with receipt: %v", err) + } + if receipt.BaseRevision != 1 || receipt.Revision != 2 || receipt.Operations[0].Turn == nil { + t.Fatalf("create turn receipt = %+v", receipt) + } + + receipt, err = repo.CompleteTurnWithConversationReceipt(ctx, turn.ID) + if err != nil { + t.Fatalf("complete turn with receipt: %v", err) + } + if receipt.BaseRevision != 2 || receipt.Revision != 3 || receipt.Operations[0].Turn == nil || receipt.Operations[0].Turn.CompletedAt == nil { + t.Fatalf("complete turn receipt = %+v", receipt) + } +} + +func assertConversationRevision(t *testing.T, repo *Repository, sessionID string, want int64) { + t.Helper() + var got int64 + if err := repo.db.QueryRow( + `SELECT revision FROM conversation_session_revisions WHERE session_id = ?`, + sessionID, + ).Scan(&got); err != nil { + t.Fatalf("read conversation revision: %v", err) + } + if got != want { + t.Fatalf("conversation revision = %d, want %d", got, want) + } +} diff --git a/apps/backend/internal/task/repository/sqlite/conversation_upgrade_large_test.go b/apps/backend/internal/task/repository/sqlite/conversation_upgrade_large_test.go new file mode 100644 index 0000000000..4a077b01ce --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/conversation_upgrade_large_test.go @@ -0,0 +1,241 @@ +package sqlite + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jmoiron/sqlx" + "github.com/kandev/kandev/internal/common/config" + "github.com/kandev/kandev/internal/common/logger" + "github.com/kandev/kandev/internal/db" + "github.com/kandev/kandev/internal/persistence" + "github.com/kandev/kandev/internal/persistence/requiredstores" + "github.com/kandev/kandev/internal/task/models" +) + +// Explicit opt-in: this fixture needs several GiB and retains its DB and backup +// in an operator-selected empty directory for subsequent isolated startup checks. +func TestConversationLargeLegacyUpgrade(t *testing.T) { + root := os.Getenv("KANDEV_LARGE_UPGRADE_DIR") + if root == "" { + t.Skip("set KANDEV_LARGE_UPGRADE_DIR to an empty disposable directory") + } + path := filepath.Join(root, "data", "kandev.db") + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatal("fixture database must not already exist") + } + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + conn, err := db.OpenSQLite(path) + if err != nil { + t.Fatal(err) + } + writer := sqlx.NewDb(conn, "sqlite3") + t.Cleanup(func() { _ = writer.Close() }) + repo, err := NewWithDB(writer, writer, nil) + if err != nil { + t.Fatal(err) + } + seedLargeLegacy(t, repo) + stat, _ := os.Stat(path) + t.Logf("legacy_db_bytes=%d", stat.Size()) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + cfg := &config.Config{HomeDir: root} + cfg.Database.Path = path + log, err := logger.NewLogger(logger.LoggingConfig{Level: "info", Format: "json", OutputPath: filepath.Join(root, "persistence.log")}) + if err != nil { + t.Fatal(err) + } + started := time.Now() + pool, closePool, err := persistence.Provide(cfg, log, "large-upgrade-verification") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = closePool() }) + t.Logf("open_and_backup_ms=%.3f", float64(time.Since(started).Microseconds())/1000) + repo = &Repository{db: pool.Writer(), ro: pool.Reader()} + started = time.Now() + if err := repo.cleanupLegacyConversationJournal(); err != nil { + t.Fatal(err) + } + t.Logf("cleanup_ms=%.3f", float64(time.Since(started).Microseconds())/1000) + started = time.Now() + repo, err = NewWithDB(pool.Writer(), pool.Reader(), nil) + if err != nil { + t.Fatal(err) + } + t.Logf("task_repository_startup_ms=%.3f", float64(time.Since(started).Microseconds())/1000) + var remaining int + err = pool.Reader().Get(&remaining, `SELECT count(*) FROM sqlite_master WHERE name IN ('conversation_session_events','conversation_message_versions','conversation_turn_versions','conversation_session_streams','conversation_journal_meta') OR (type='trigger' AND (sql LIKE '%INSERT INTO conversation_message_versions%' OR sql LIKE '%INSERT INTO conversation_session_events%' OR sql LIKE '%INSERT INTO conversation_turn_versions%'))`) + if err != nil || remaining != 0 { + t.Fatalf("legacy objects remaining=%d error=%v", remaining, err) + } + var count int + if err = pool.Reader().Get(&count, `SELECT count(*) FROM task_session_messages`); err != nil || count != 50000 { + t.Fatalf("source count=%d error=%v", count, err) + } + var pages, free int64 + if err = pool.Reader().Get(&pages, "PRAGMA page_count"); err != nil { + t.Fatal(err) + } + if err = pool.Reader().Get(&free, "PRAGMA freelist_count"); err != nil { + t.Fatal(err) + } + t.Logf("post_cleanup_pages=%d free_pages=%d", pages, free) + runLargeUpgradeWriterLoad(t, repo, pool) +} + +func seedLargeLegacy(t *testing.T, repo *Repository) { + t.Helper() + const stamp = "2026-09-16T12:00:00Z" + for _, query := range []string{ + `INSERT INTO workspaces(id,name,created_at,updated_at) VALUES('large-workspace','Large fixture',?,?)`, + `INSERT INTO tasks(id,workspace_id,title,created_at,updated_at) VALUES('large-task','large-workspace','large legacy',?,?)`, + `INSERT INTO task_sessions(id,task_id,queue_incarnation_id,started_at,updated_at) VALUES('large-session','large-task','large-incarnation',?,?)`, + } { + if _, err := repo.db.Exec(query, stamp, stamp); err != nil { + t.Fatal(err) + } + } + if _, err := repo.db.Exec(`INSERT INTO task_session_turns(id,task_session_id,task_id,started_at,created_at,updated_at) VALUES('large-turn','large-session','large-task',?,?,?)`, stamp, stamp, stamp); err != nil { + t.Fatal(err) + } + var triggers []string + if err := repo.db.Select(&triggers, `SELECT name FROM sqlite_master WHERE type='trigger' AND name LIKE 'conversation_source_%'`); err != nil { + t.Fatal(err) + } + for _, name := range triggers { + if _, err := repo.db.Exec(`DROP TRIGGER "` + name + `"`); err != nil { + t.Fatal(err) + } + } + if _, err := repo.db.Exec(`DROP TABLE conversation_session_revisions`); err != nil { + t.Fatal(err) + } + fixture, err := os.ReadFile("testdata/conversation-journal-88c6c0fe.sql") + if err != nil { + t.Fatal(err) + } + if _, err := repo.db.Exec(string(fixture)); err != nil { + t.Fatal(err) + } + payload := strings.Repeat("synthetic-data-", 585) + for batch := 0; batch < 100; batch++ { + tx, err := repo.db.Beginx() + if err != nil { + t.Fatal(err) + } + for index := 0; index < 500; index++ { + id := fmt.Sprintf("large-%d", batch*500+index) + if _, err = tx.Exec(`INSERT INTO task_session_messages(id,task_session_id,task_id,turn_id,author_type,author_id,content,requests_input,type,metadata,created_at) VALUES(?,'large-session','large-task','large-turn','agent','',?,0,'message','{}',?)`, id, payload, stamp); err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + if _, err = tx.Exec(`UPDATE task_session_messages SET content=content || 'final' WHERE id=?`, id); err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + } + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if batch%20 == 0 { + t.Logf("seeded_messages=%d", (batch+1)*500) + } + } + if _, err = repo.db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil { + t.Fatal(err) + } +} + +func runLargeUpgradeWriterLoad(t *testing.T, repo *Repository, pool *db.Pool) { + t.Helper() + tracker, err := requiredstores.NewTracker([]requiredstores.Descriptor{{ID: "task", OwnerPackage: "internal/task", RequiredTables: []string{"tasks", "task_sessions", "task_session_messages", "task_session_turns", "conversation_session_revisions"}}}) + if err != nil { + t.Fatal(err) + } + if err = tracker.RecordSuccess("task"); err != nil { + t.Fatal(err) + } + if err = tracker.Complete(); err != nil { + t.Fatal(err) + } + health := requiredstores.NewHealth(tracker, pool, nil) + if err = health.Check(context.Background()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + var updates, failures atomic.Int64 + var workers sync.WaitGroup + for worker := 0; worker < 8; worker++ { + workers.Add(1) + go func(id int) { + defer workers.Done() + msg, readErr := repo.GetMessage(ctx, fmt.Sprintf("large-%d", id)) + if readErr != nil { + failures.Add(1) + return + } + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + msg.Content = strings.Repeat("stream-", 1170) + fmt.Sprint(updates.Load()) + msg.Type = models.MessageTypeMessage + if _, writeErr := repo.UpdateMessageWithConversationReceipt(ctx, msg); writeErr != nil { + if ctx.Err() == nil { + failures.Add(1) + } + } else { + updates.Add(1) + } + } + } + }(worker) + } + probes, probeFailures := 0, 0 + var worst time.Duration + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for ctx.Err() == nil { + select { + case <-ctx.Done(): + case <-ticker.C: + probeCtx, stop := context.WithTimeout(ctx, 2*time.Second) + start := time.Now() + probeErr := health.Check(probeCtx) + elapsed := time.Since(start) + stop() + if ctx.Err() != nil { + break + } + probes++ + if elapsed > worst { + worst = elapsed + } + if probeErr != nil { + probeFailures++ + t.Logf("probe_failure=%v", probeErr) + } + } + } + workers.Wait() + t.Logf("stream_seconds=60 workers=8 target_updates_per_second=800 updates=%d write_failures=%d probes=%d probe_failures=%d worst_probe_ms=%.3f", updates.Load(), failures.Load(), probes, probeFailures, float64(worst.Microseconds())/1000) + if failures.Load() != 0 || probeFailures != 0 { + t.Fatal("post-readiness streaming load failed") + } +} diff --git a/apps/backend/internal/task/repository/sqlite/message.go b/apps/backend/internal/task/repository/sqlite/message.go index eef49820b5..6a8f9a13e7 100644 --- a/apps/backend/internal/task/repository/sqlite/message.go +++ b/apps/backend/internal/task/repository/sqlite/message.go @@ -939,23 +939,18 @@ func (r *Repository) FindMessageByPendingIDAndQuestion(ctx context.Context, sess // `status` is the user's approve/reject decision, not the tool call state. Forcing them to // "complete" wipes "approved"/"rejected" and re-shows the prompt buttons in the UI. func (r *Repository) CompletePendingToolCallsForTurn(ctx context.Context, turnID string) (int64, error) { - drv := r.db.DriverName() - query := fmt.Sprintf(` - UPDATE task_session_messages - SET metadata = %s, updated_at = CURRENT_TIMESTAMP - WHERE turn_id = ? - AND type != 'permission_request' - AND %s NOT IN ('complete', 'error') - AND %s - `, dialect.JSONSet(drv, "metadata", "status", "complete"), - dialect.JSONExtract(drv, "metadata", "status"), - dialect.JSONExtractIsNotNull(drv, "metadata", "tool_call_id")) - result, err := r.db.ExecContext(ctx, r.db.Rebind(query), turnID) + tx, err := r.db.BeginTxx(ctx, nil) if err != nil { - return 0, fmt.Errorf("failed to complete pending tool calls for turn %s: %w", turnID, err) + return 0, err + } + defer func() { _ = tx.Rollback() }() + rows, err := r.completePendingToolCallsForTurnTx(ctx, tx, turnID) + if err != nil { + return 0, err + } + if err := tx.Commit(); err != nil { + return 0, err } - - rows, _ := result.RowsAffected() return rows, nil } diff --git a/apps/backend/internal/task/repository/sqlite/message_payload_replay.go b/apps/backend/internal/task/repository/sqlite/message_payload_replay.go index a7acad9fbb..2c8771db15 100644 --- a/apps/backend/internal/task/repository/sqlite/message_payload_replay.go +++ b/apps/backend/internal/task/repository/sqlite/message_payload_replay.go @@ -3,37 +3,61 @@ package sqlite import ( "bytes" "context" + "database/sql" "encoding/json" + "errors" "fmt" "time" + "github.com/jmoiron/sqlx" + + "github.com/kandev/kandev/internal/db/dialect" "github.com/kandev/kandev/internal/task/models" ) -// updateMessageWithPayloadGuard takes the writer before reading the tombstone. -// A stale replacement may update display status, but cannot change the retained -// tool identity, normalized summary, or removal receipt. +// updateMessageWithPayloadGuard serializes the metadata read with a concurrent +// replacement. A stale replacement may update display status, but cannot change +// the retained tool identity, normalized summary, or removal receipt. func (r *Repository) updateMessageWithPayloadGuard(ctx context.Context, message *models.Message, metadataJSON []byte, requestsInput int) error { tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return err } defer func() { _ = tx.Rollback() }() - result, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE task_session_messages SET id = id WHERE id = ?`), message.ID) - if err != nil { + if err := r.updateMessageWithPayloadGuardTx(ctx, tx, message, metadataJSON, requestsInput); err != nil { return err } - rows, err := result.RowsAffected() - if err != nil { - return err + return tx.Commit() +} + +func (r *Repository) updateMessageWithPayloadGuardTx(ctx context.Context, tx *sqlx.Tx, message *models.Message, metadataJSON []byte, requestsInput int) error { + if !dialect.IsPostgres(r.db.DriverName()) { + // SQLite has a database-level writer lock, and this no-op update takes it + // before the retained metadata is read. PostgreSQL uses FOR UPDATE below; + // an UPDATE here would fire the conversation revision trigger. + result, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE task_session_messages SET id = id WHERE id = ?`), message.ID) + if err != nil { + return err + } + rows, err := result.RowsAffected() + if err != nil { + return err + } + if rows == 0 { + return fmt.Errorf("message not found: %s", message.ID) + } } - if rows == 0 { - return fmt.Errorf("message not found: %s", message.ID) + query := `SELECT metadata,type,updated_at FROM task_session_messages WHERE id = ?` + if dialect.IsPostgres(r.db.DriverName()) { + query += forUpdateClause } var raw string var storedType models.MessageType var storedUpdatedAt time.Time - if err := tx.QueryRowContext(ctx, tx.Rebind(`SELECT metadata,type,updated_at FROM task_session_messages WHERE id = ?`), message.ID).Scan(&raw, &storedType, &storedUpdatedAt); err != nil { + if err := tx.QueryRowContext(ctx, tx.Rebind(query), message.ID).Scan(&raw, &storedType, &storedUpdatedAt); err != nil { + if errors.Is(err, sql.ErrNoRows) && dialect.IsPostgres(r.db.DriverName()) { + return fmt.Errorf("message not found: %s", message.ID) + } return err } metadataJSON, sanitized, err := mergeRetainedMessageMetadata([]byte(raw), metadataJSON) @@ -52,9 +76,6 @@ func (r *Repository) updateMessageWithPayloadGuard(ctx context.Context, message if err != nil { return err } - if err := tx.Commit(); err != nil { - return err - } if sanitized != nil { message.Metadata = sanitized message.Type = nextType @@ -66,20 +87,32 @@ func (r *Repository) updateMessageWithPayloadGuard(ctx context.Context, message // insertMessageWithPayloadGuard prevents fallback-create and repeated start // frames from giving a removed tool call a fresh row identity. func (r *Repository) insertMessageWithPayloadGuard(ctx context.Context, message *models.Message, requestsInput int, messageType, metadataJSON string) error { - toolCallID, _ := message.Metadata["tool_call_id"].(string) - if toolCallID == "" || messageType == string(models.MessageTypePermissionRequest) { - return r.insertMessageRow(ctx, r.db, message, requestsInput, messageType, metadataJSON) - } tx, err := r.db.BeginTxx(ctx, nil) if err != nil { return err } defer func() { _ = tx.Rollback() }() + if err := r.insertMessageWithPayloadGuardTx(ctx, tx, message, requestsInput, messageType, metadataJSON); err != nil { + return err + } + return tx.Commit() +} + +func (r *Repository) insertMessageWithPayloadGuardTx(ctx context.Context, tx *sqlx.Tx, message *models.Message, requestsInput int, messageType, metadataJSON string) error { + toolCallID, _ := message.Metadata["tool_call_id"].(string) + if toolCallID == "" || messageType == string(models.MessageTypePermissionRequest) { + return r.insertMessageRow(ctx, tx, message, requestsInput, messageType, metadataJSON) + } if _, err := tx.ExecContext(ctx, tx.Rebind(`UPDATE task_sessions SET id = id WHERE id = ?`), message.TaskSessionID); err != nil { return err } var removed bool - err = tx.QueryRowContext(ctx, tx.Rebind(`SELECT EXISTS(SELECT 1 FROM task_session_messages WHERE task_session_id = ? AND json_extract(metadata, '$.tool_call_id') = ? AND type != 'permission_request' AND json_type(metadata, '$.payload_retention') IS NOT NULL)`), message.TaskSessionID, toolCallID).Scan(&removed) + driver := r.db.DriverName() + query := fmt.Sprintf(`SELECT EXISTS(SELECT 1 FROM task_session_messages WHERE task_session_id = ? AND %s = ? AND type != 'permission_request' AND %s)`, + dialect.JSONExtract(driver, "metadata", "tool_call_id"), + dialect.JSONExtractIsNotNull(driver, "metadata", "payload_retention"), + ) + err := tx.QueryRowContext(ctx, tx.Rebind(query), message.TaskSessionID, toolCallID).Scan(&removed) if err != nil { return err } @@ -89,7 +122,7 @@ func (r *Repository) insertMessageWithPayloadGuard(ctx context.Context, message if err := r.insertMessageRow(ctx, tx, message, requestsInput, messageType, metadataJSON); err != nil { return err } - return tx.Commit() + return nil } func mergeRetainedMessageMetadata(stored, incoming []byte) ([]byte, map[string]any, error) { diff --git a/apps/backend/internal/task/repository/sqlite/message_prompt_index.go b/apps/backend/internal/task/repository/sqlite/message_prompt_index.go index a9b8871dfc..69ba3cf3f9 100644 --- a/apps/backend/internal/task/repository/sqlite/message_prompt_index.go +++ b/apps/backend/internal/task/repository/sqlite/message_prompt_index.go @@ -218,6 +218,21 @@ func (r *Repository) createUserMessageWithBoundary( ) } +func (r *Repository) createUserMessageWithBoundaryReceipt( + ctx context.Context, + message *models.Message, + requestsInput int, + messageType, metadataJSON string, +) (*models.ConversationMutationReceipt, error) { + receipt := &models.ConversationMutationReceipt{} + driver := r.db.DriverName() + nm := dialect.NormalizedMicrosecond(driver, "created_at") + if err := r.executeBoundaryTransaction(ctx, message, requestsInput, messageType, metadataJSON, driver, nm, nil, receipt); err != nil { + return nil, err + } + return receipt, nil +} + func (r *Repository) createUserMessageWithBoundaryAndInitialTaskBrief( ctx context.Context, message *models.Message, @@ -227,7 +242,7 @@ func (r *Repository) createUserMessageWithBoundaryAndInitialTaskBrief( ) error { driver := r.db.DriverName() nm := dialect.NormalizedMicrosecond(driver, "created_at") - return r.executeBoundaryTransaction(ctx, message, requestsInput, messageType, metadataJSON, driver, nm, candidate) + return r.executeBoundaryTransaction(ctx, message, requestsInput, messageType, metadataJSON, driver, nm, candidate, nil) } // executeBoundaryTransaction runs one per-session write boundary: begin a @@ -241,6 +256,7 @@ func (r *Repository) executeBoundaryTransaction( requestsInput int, messageType, metadataJSON, driver, nm string, candidate *admission.InitialTaskBriefCandidate, + receipt *models.ConversationMutationReceipt, ) (err error) { origCreatedAt := message.CreatedAt origUpdatedAt := message.UpdatedAt @@ -281,6 +297,13 @@ func (r *Repository) executeBoundaryTransaction( if err := lockSessionTurnWrites(ctx, tx, driver, message.TaskSessionID); err != nil { return err } + var baseRevision int64 + if receipt != nil { + baseRevision, err = r.ensureConversationRevisionTx(ctx, tx, message.TaskSessionID) + if err != nil { + return err + } + } if candidate != nil { if err := r.selectInitialTaskBriefCandidate(ctx, tx, message.TaskSessionID, message, candidate); err != nil { return err @@ -295,6 +318,11 @@ func (r *Repository) executeBoundaryTransaction( if err := r.insertMessageRow(ctx, tx, message, requestsInput, messageType, metadataJSON); err != nil { return err } + if receipt != nil { + if err := r.populateConversationMessageReceipt(ctx, tx, receipt, baseRevision, message, models.ConversationMutationUpsert); err != nil { + return err + } + } if err := tx.Commit(); err != nil { return fmt.Errorf("commit user message creation: %w", err) } diff --git a/apps/backend/internal/task/repository/sqlite/testdata/conversation-journal-88c6c0fe.README.md b/apps/backend/internal/task/repository/sqlite/testdata/conversation-journal-88c6c0fe.README.md new file mode 100644 index 0000000000..19ad4c8b4b --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/testdata/conversation-journal-88c6c0fe.README.md @@ -0,0 +1,17 @@ +# Legacy conversation journal fixture + +`conversation-journal-88c6c0fe.sql` contains the legacy SQLite journal tables, +indexes, and seven payload-copy triggers from commit +`88c6c0fe0a6ae5d25332070d603b99f7d9241645`. + +The SQL was exported from `conversationJournalTables` and +`initSQLiteConversationJournalTriggers` in the historical +`conversation_journal.go`. The historical Go helpers expanded the system-content, +metadata, and timestamp expressions. The export excludes the old post-install +sanitization UPDATE statements. The load fixture contains synthetic text only. + +This is test data, not an installation or migration script. Production startup +must not execute it. `TestConversationLargeLegacyUpgrade` loads it only when +`KANDEV_LARGE_UPGRADE_DIR` names an empty disposable directory. The test retains +that directory for an isolated backend startup check; remove it after collecting +results. Allow at least 10 GiB free for the database, WAL, backup, and headroom. diff --git a/apps/backend/internal/task/repository/sqlite/testdata/conversation-journal-88c6c0fe.sql b/apps/backend/internal/task/repository/sqlite/testdata/conversation-journal-88c6c0fe.sql new file mode 100644 index 0000000000..778e4ead3e --- /dev/null +++ b/apps/backend/internal/task/repository/sqlite/testdata/conversation-journal-88c6c0fe.sql @@ -0,0 +1,258 @@ + +CREATE TABLE IF NOT EXISTS conversation_session_streams ( + session_id TEXT PRIMARY KEY, + watermark BIGINT NOT NULL DEFAULT 0, + terminal BOOLEAN NOT NULL DEFAULT FALSE, + updated_at TIMESTAMP NOT NULL +); +CREATE TABLE IF NOT EXISTS conversation_session_events ( + session_id TEXT NOT NULL, + sequence BIGINT NOT NULL, + event_id TEXT NOT NULL UNIQUE, + protocol_version INTEGER NOT NULL DEFAULT 1, + event_type TEXT NOT NULL, + task_id TEXT, + payload TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (session_id, sequence) +); +CREATE INDEX IF NOT EXISTS idx_conversation_events_created + ON conversation_session_events(created_at); +CREATE TABLE IF NOT EXISTS conversation_message_versions ( + session_id TEXT NOT NULL, + message_id TEXT NOT NULL, + row_sequence BIGINT NOT NULL, + task_id TEXT, + author_type TEXT, + created_at TIMESTAMP NOT NULL, + tombstone BOOLEAN NOT NULL DEFAULT FALSE, + payload TEXT NOT NULL, + PRIMARY KEY (session_id, message_id, row_sequence) +); +CREATE INDEX IF NOT EXISTS idx_conversation_message_snapshot + ON conversation_message_versions(session_id, row_sequence, created_at, message_id); +CREATE TABLE IF NOT EXISTS conversation_turn_versions ( + session_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + row_sequence BIGINT NOT NULL, + task_id TEXT, + started_at TIMESTAMP NOT NULL, + tombstone BOOLEAN NOT NULL DEFAULT FALSE, + payload TEXT NOT NULL, + PRIMARY KEY (session_id, turn_id, row_sequence) +); +CREATE INDEX IF NOT EXISTS idx_conversation_turn_snapshot + ON conversation_turn_versions(session_id, row_sequence, started_at, turn_id); +CREATE TABLE IF NOT EXISTS conversation_journal_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +DROP TRIGGER IF EXISTS conversation_message_insert; +CREATE TRIGGER IF NOT EXISTS conversation_message_insert +AFTER INSERT ON task_session_messages +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_message_versions(session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload) + SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.author_type, NEW.created_at, FALSE, + json_object('type','message.added','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), + 'message_id',NEW.id,'turn_id',NULLIF(NEW.turn_id,''),'author_type',NEW.author_type, + 'content',(SELECT trim(x) FROM ( + WITH RECURSIVE strip(x, depth) AS ( + SELECT NEW.content, 0 + UNION ALL + SELECT substr(x, 1, instr(x, '') - 1) + || ltrim(substr(x, + instr(x, '') + length('') + + instr(substr(x, instr(x, '') + length('')), '') - 1 + + length('')), + char(32) || char(9) || char(10) || char(13)), depth + 1 + FROM strip + WHERE depth < 64 + AND instr(x, '') > 0 + AND instr(substr(x, instr(x, '') + length('')), '') > 0 + ) + SELECT CASE WHEN depth = 64 AND instr(x, '') > 0 AND instr(substr(x, instr(x, '') + length('')), '') > 0 THEN substr(x, 1, instr(x, '') - 1) ELSE x END AS x FROM strip ORDER BY depth DESC LIMIT 1 + )),'message_type',NEW.type,'requests_input',NEW.requests_input, + 'created_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.created_at), + 'updated_at',strftime('%Y-%m-%dT%H:%M:%fZ', COALESCE(NEW.updated_at,NEW.created_at)),'prompt_index',NEW.prompt_seq, + 'metadata',CASE WHEN json_valid(NEW.metadata) THEN json_object('action_details',CASE json_type(NEW.metadata, '$.action_details') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.action_details') END,'action_type',CASE json_type(NEW.metadata, '$.action_type') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.action_type') END,'action_visibility',CASE json_type(NEW.metadata, '$.action_visibility') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.action_visibility') END,'actions',CASE json_type(NEW.metadata, '$.actions') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.actions') END,'agent_disconnected',CASE json_type(NEW.metadata, '$.agent_disconnected') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.agent_disconnected') END,'attempt',CASE json_type(NEW.metadata, '$.attempt') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.attempt') END,'attachments',CASE json_type(NEW.metadata, '$.attachments') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.attachments') END,'auth_methods',CASE json_type(NEW.metadata, '$.auth_methods') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.auth_methods') END,'auto_start',CASE json_type(NEW.metadata, '$.auto_start') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.auto_start') END,'base_branch',CASE json_type(NEW.metadata, '$.base_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.base_branch') END,'context',CASE json_type(NEW.metadata, '$.context') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.context') END,'context_files',CASE json_type(NEW.metadata, '$.context_files') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.context_files') END,'decision_id',CASE json_type(NEW.metadata, '$.decision_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.decision_id') END,'effective_model',CASE json_type(NEW.metadata, '$.effective_model') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.effective_model') END,'entity_references',CASE json_type(NEW.metadata, '$.entity_references') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.entity_references') END,'error_output',CASE json_type(NEW.metadata, '$.error_output') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.error_output') END,'failure_code',CASE json_type(NEW.metadata, '$.failure_code') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.failure_code') END,'failure_details',CASE json_type(NEW.metadata, '$.failure_details') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.failure_details') END,'failure_kind',CASE json_type(NEW.metadata, '$.failure_kind') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.failure_kind') END,'fallback_model',CASE json_type(NEW.metadata, '$.fallback_model') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.fallback_model') END,'has_hidden_prompts',CASE json_type(NEW.metadata, '$.has_hidden_prompts') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.has_hidden_prompts') END,'has_resume_token',CASE json_type(NEW.metadata, '$.has_resume_token') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.has_resume_token') END,'has_review_comments',CASE json_type(NEW.metadata, '$.has_review_comments') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.has_review_comments') END,'is_auth_error',CASE json_type(NEW.metadata, '$.is_auth_error') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.is_auth_error') END,'kind',CASE json_type(NEW.metadata, '$.kind') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.kind') END,'max_attempts',CASE json_type(NEW.metadata, '$.max_attempts') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.max_attempts') END,'message',CASE json_type(NEW.metadata, '$.message') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.message') END,'missing_branch',CASE json_type(NEW.metadata, '$.missing_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.missing_branch') END,'model_id',CASE json_type(NEW.metadata, '$.model_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.model_id') END,'new_branch',CASE json_type(NEW.metadata, '$.new_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.new_branch') END,'original_branch',CASE json_type(NEW.metadata, '$.original_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.original_branch') END,'options',CASE json_type(NEW.metadata, '$.options') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.options') END,'pending_id',CASE json_type(NEW.metadata, '$.pending_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.pending_id') END,'plan_mode',CASE json_type(NEW.metadata, '$.plan_mode') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.plan_mode') END,'progress',CASE json_type(NEW.metadata, '$.progress') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.progress') END,'provider_name',CASE json_type(NEW.metadata, '$.provider_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.provider_name') END,'question',CASE json_type(NEW.metadata, '$.question') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question') END,'question_id',CASE json_type(NEW.metadata, '$.question_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question_id') END,'question_index',CASE json_type(NEW.metadata, '$.question_index') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question_index') END,'question_total',CASE json_type(NEW.metadata, '$.question_total') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question_total') END,'recovery_actions',CASE json_type(NEW.metadata, '$.recovery_actions') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.recovery_actions') END,'remediation',CASE json_type(NEW.metadata, '$.remediation') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.remediation') END,'remediation_url',CASE json_type(NEW.metadata, '$.remediation_url') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.remediation_url') END,'requested_model',CASE json_type(NEW.metadata, '$.requested_model') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.requested_model') END,'request_id',CASE json_type(NEW.metadata, '$.request_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.request_id') END,'response',CASE json_type(NEW.metadata, '$.response') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.response') END,'reset_at',CASE json_type(NEW.metadata, '$.reset_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.reset_at') END,'retry_at',CASE json_type(NEW.metadata, '$.retry_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.retry_at') END,'retry_in_seconds',CASE json_type(NEW.metadata, '$.retry_in_seconds') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.retry_in_seconds') END,'retrying',CASE json_type(NEW.metadata, '$.retrying') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.retrying') END,'sender_session_id',CASE json_type(NEW.metadata, '$.sender_session_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_session_id') END,'sender_session_name',CASE json_type(NEW.metadata, '$.sender_session_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_session_name') END,'sender_task_id',CASE json_type(NEW.metadata, '$.sender_task_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_task_id') END,'sender_task_title',CASE json_type(NEW.metadata, '$.sender_task_title') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_task_title') END,'stage',CASE json_type(NEW.metadata, '$.stage') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.stage') END,'status',CASE json_type(NEW.metadata, '$.status') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.status') END,'script_type',CASE json_type(NEW.metadata, '$.script_type') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.script_type') END,'agent_name',CASE json_type(NEW.metadata, '$.agent_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.agent_name') END,'command',CASE json_type(NEW.metadata, '$.command') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.command') END,'exit_code',CASE json_type(NEW.metadata, '$.exit_code') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.exit_code') END,'is_resuming',CASE json_type(NEW.metadata, '$.is_resuming') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.is_resuming') END,'started_at',CASE json_type(NEW.metadata, '$.started_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.started_at') END,'completed_at',CASE json_type(NEW.metadata, '$.completed_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.completed_at') END,'error',CASE json_type(NEW.metadata, '$.error') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.error') END,'task_id',CASE json_type(NEW.metadata, '$.task_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.task_id') END,'text',CASE json_type(NEW.metadata, '$.text') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.text') END,'tool_call_id',CASE json_type(NEW.metadata, '$.tool_call_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.tool_call_id') END,'variant',CASE json_type(NEW.metadata, '$.variant') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.variant') END,'workflow_message',CASE json_type(NEW.metadata, '$.workflow_message') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_message') END,'workflow_step_color',CASE json_type(NEW.metadata, '$.workflow_step_color') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_step_color') END,'workflow_step_id',CASE json_type(NEW.metadata, '$.workflow_step_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_step_id') END,'workflow_step_name',CASE json_type(NEW.metadata, '$.workflow_step_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_step_name') END) ELSE json_object() END, + 'sender_task_id',CASE WHEN json_valid(NEW.metadata) AND typeof(json_extract(NEW.metadata,'$.sender_task_id')) = 'text' THEN json_extract(NEW.metadata,'$.sender_task_id') END) + FROM conversation_session_streams WHERE session_id = NEW.task_session_id; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'message.added', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP + FROM conversation_session_streams JOIN conversation_message_versions + ON conversation_message_versions.session_id = conversation_session_streams.session_id + AND conversation_message_versions.message_id = NEW.id + AND conversation_message_versions.row_sequence = conversation_session_streams.watermark + WHERE conversation_session_streams.session_id = NEW.task_session_id; +END; + +DROP TRIGGER IF EXISTS conversation_message_update; +CREATE TRIGGER IF NOT EXISTS conversation_message_update +AFTER UPDATE ON task_session_messages +WHEN OLD.id IS NOT NEW.id + OR OLD.task_session_id IS NOT NEW.task_session_id + OR OLD.task_id IS NOT NEW.task_id + OR OLD.turn_id IS NOT NEW.turn_id + OR OLD.author_type IS NOT NEW.author_type + OR OLD.author_id IS NOT NEW.author_id + OR OLD.content IS NOT NEW.content + OR OLD.requests_input IS NOT NEW.requests_input + OR OLD.type IS NOT NEW.type + OR OLD.metadata IS NOT NEW.metadata + OR OLD.created_at IS NOT NEW.created_at + OR OLD.updated_at IS NOT NEW.updated_at + OR OLD.prompt_seq IS NOT NEW.prompt_seq +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_message_versions(session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload) + SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.author_type, NEW.created_at, FALSE, + json_object('type','message.updated','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), + 'message_id',NEW.id,'turn_id',NULLIF(NEW.turn_id,''),'author_type',NEW.author_type, + 'content',(SELECT trim(x) FROM ( + WITH RECURSIVE strip(x, depth) AS ( + SELECT NEW.content, 0 + UNION ALL + SELECT substr(x, 1, instr(x, '') - 1) + || ltrim(substr(x, + instr(x, '') + length('') + + instr(substr(x, instr(x, '') + length('')), '') - 1 + + length('')), + char(32) || char(9) || char(10) || char(13)), depth + 1 + FROM strip + WHERE depth < 64 + AND instr(x, '') > 0 + AND instr(substr(x, instr(x, '') + length('')), '') > 0 + ) + SELECT CASE WHEN depth = 64 AND instr(x, '') > 0 AND instr(substr(x, instr(x, '') + length('')), '') > 0 THEN substr(x, 1, instr(x, '') - 1) ELSE x END AS x FROM strip ORDER BY depth DESC LIMIT 1 + )),'message_type',NEW.type,'requests_input',NEW.requests_input, + 'created_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.created_at), + 'updated_at',strftime('%Y-%m-%dT%H:%M:%fZ', COALESCE(NEW.updated_at,NEW.created_at)),'prompt_index',NEW.prompt_seq, + 'metadata',CASE WHEN json_valid(NEW.metadata) THEN json_object('action_details',CASE json_type(NEW.metadata, '$.action_details') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.action_details') END,'action_type',CASE json_type(NEW.metadata, '$.action_type') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.action_type') END,'action_visibility',CASE json_type(NEW.metadata, '$.action_visibility') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.action_visibility') END,'actions',CASE json_type(NEW.metadata, '$.actions') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.actions') END,'agent_disconnected',CASE json_type(NEW.metadata, '$.agent_disconnected') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.agent_disconnected') END,'attempt',CASE json_type(NEW.metadata, '$.attempt') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.attempt') END,'attachments',CASE json_type(NEW.metadata, '$.attachments') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.attachments') END,'auth_methods',CASE json_type(NEW.metadata, '$.auth_methods') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.auth_methods') END,'auto_start',CASE json_type(NEW.metadata, '$.auto_start') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.auto_start') END,'base_branch',CASE json_type(NEW.metadata, '$.base_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.base_branch') END,'context',CASE json_type(NEW.metadata, '$.context') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.context') END,'context_files',CASE json_type(NEW.metadata, '$.context_files') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.context_files') END,'decision_id',CASE json_type(NEW.metadata, '$.decision_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.decision_id') END,'effective_model',CASE json_type(NEW.metadata, '$.effective_model') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.effective_model') END,'entity_references',CASE json_type(NEW.metadata, '$.entity_references') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.entity_references') END,'error_output',CASE json_type(NEW.metadata, '$.error_output') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.error_output') END,'failure_code',CASE json_type(NEW.metadata, '$.failure_code') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.failure_code') END,'failure_details',CASE json_type(NEW.metadata, '$.failure_details') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.failure_details') END,'failure_kind',CASE json_type(NEW.metadata, '$.failure_kind') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.failure_kind') END,'fallback_model',CASE json_type(NEW.metadata, '$.fallback_model') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.fallback_model') END,'has_hidden_prompts',CASE json_type(NEW.metadata, '$.has_hidden_prompts') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.has_hidden_prompts') END,'has_resume_token',CASE json_type(NEW.metadata, '$.has_resume_token') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.has_resume_token') END,'has_review_comments',CASE json_type(NEW.metadata, '$.has_review_comments') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.has_review_comments') END,'is_auth_error',CASE json_type(NEW.metadata, '$.is_auth_error') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.is_auth_error') END,'kind',CASE json_type(NEW.metadata, '$.kind') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.kind') END,'max_attempts',CASE json_type(NEW.metadata, '$.max_attempts') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.max_attempts') END,'message',CASE json_type(NEW.metadata, '$.message') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.message') END,'missing_branch',CASE json_type(NEW.metadata, '$.missing_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.missing_branch') END,'model_id',CASE json_type(NEW.metadata, '$.model_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.model_id') END,'new_branch',CASE json_type(NEW.metadata, '$.new_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.new_branch') END,'original_branch',CASE json_type(NEW.metadata, '$.original_branch') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.original_branch') END,'options',CASE json_type(NEW.metadata, '$.options') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.options') END,'pending_id',CASE json_type(NEW.metadata, '$.pending_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.pending_id') END,'plan_mode',CASE json_type(NEW.metadata, '$.plan_mode') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.plan_mode') END,'progress',CASE json_type(NEW.metadata, '$.progress') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.progress') END,'provider_name',CASE json_type(NEW.metadata, '$.provider_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.provider_name') END,'question',CASE json_type(NEW.metadata, '$.question') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question') END,'question_id',CASE json_type(NEW.metadata, '$.question_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question_id') END,'question_index',CASE json_type(NEW.metadata, '$.question_index') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question_index') END,'question_total',CASE json_type(NEW.metadata, '$.question_total') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.question_total') END,'recovery_actions',CASE json_type(NEW.metadata, '$.recovery_actions') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.recovery_actions') END,'remediation',CASE json_type(NEW.metadata, '$.remediation') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.remediation') END,'remediation_url',CASE json_type(NEW.metadata, '$.remediation_url') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.remediation_url') END,'requested_model',CASE json_type(NEW.metadata, '$.requested_model') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.requested_model') END,'request_id',CASE json_type(NEW.metadata, '$.request_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.request_id') END,'response',CASE json_type(NEW.metadata, '$.response') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.response') END,'reset_at',CASE json_type(NEW.metadata, '$.reset_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.reset_at') END,'retry_at',CASE json_type(NEW.metadata, '$.retry_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.retry_at') END,'retry_in_seconds',CASE json_type(NEW.metadata, '$.retry_in_seconds') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.retry_in_seconds') END,'retrying',CASE json_type(NEW.metadata, '$.retrying') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.retrying') END,'sender_session_id',CASE json_type(NEW.metadata, '$.sender_session_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_session_id') END,'sender_session_name',CASE json_type(NEW.metadata, '$.sender_session_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_session_name') END,'sender_task_id',CASE json_type(NEW.metadata, '$.sender_task_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_task_id') END,'sender_task_title',CASE json_type(NEW.metadata, '$.sender_task_title') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.sender_task_title') END,'stage',CASE json_type(NEW.metadata, '$.stage') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.stage') END,'status',CASE json_type(NEW.metadata, '$.status') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.status') END,'script_type',CASE json_type(NEW.metadata, '$.script_type') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.script_type') END,'agent_name',CASE json_type(NEW.metadata, '$.agent_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.agent_name') END,'command',CASE json_type(NEW.metadata, '$.command') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.command') END,'exit_code',CASE json_type(NEW.metadata, '$.exit_code') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.exit_code') END,'is_resuming',CASE json_type(NEW.metadata, '$.is_resuming') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.is_resuming') END,'started_at',CASE json_type(NEW.metadata, '$.started_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.started_at') END,'completed_at',CASE json_type(NEW.metadata, '$.completed_at') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.completed_at') END,'error',CASE json_type(NEW.metadata, '$.error') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.error') END,'task_id',CASE json_type(NEW.metadata, '$.task_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.task_id') END,'text',CASE json_type(NEW.metadata, '$.text') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.text') END,'tool_call_id',CASE json_type(NEW.metadata, '$.tool_call_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.tool_call_id') END,'variant',CASE json_type(NEW.metadata, '$.variant') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.variant') END,'workflow_message',CASE json_type(NEW.metadata, '$.workflow_message') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_message') END,'workflow_step_color',CASE json_type(NEW.metadata, '$.workflow_step_color') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_step_color') END,'workflow_step_id',CASE json_type(NEW.metadata, '$.workflow_step_id') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_step_id') END,'workflow_step_name',CASE json_type(NEW.metadata, '$.workflow_step_name') WHEN 'true' THEN json('true') WHEN 'false' THEN json('false') ELSE json_extract(NEW.metadata, '$.workflow_step_name') END) ELSE json_object() END, + 'sender_task_id',CASE WHEN json_valid(NEW.metadata) AND typeof(json_extract(NEW.metadata,'$.sender_task_id')) = 'text' THEN json_extract(NEW.metadata,'$.sender_task_id') END) + FROM conversation_session_streams WHERE session_id = NEW.task_session_id; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'message.updated', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP + FROM conversation_session_streams JOIN conversation_message_versions + ON conversation_message_versions.session_id = conversation_session_streams.session_id + AND conversation_message_versions.message_id = NEW.id + AND conversation_message_versions.row_sequence = conversation_session_streams.watermark + WHERE conversation_session_streams.session_id = NEW.task_session_id; +END; +DROP TRIGGER IF EXISTS conversation_message_delete; + +CREATE TRIGGER IF NOT EXISTS conversation_message_delete +BEFORE DELETE ON task_session_messages +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (OLD.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_message_versions(session_id, message_id, row_sequence, task_id, author_type, created_at, tombstone, payload) + SELECT OLD.task_session_id, OLD.id, watermark, NULLIF(OLD.task_id, ''), OLD.author_type, OLD.created_at, TRUE, + json_object('type','message.deleted','session_id',OLD.task_session_id,'task_id',NULLIF(OLD.task_id,''),'message_id',OLD.id) + FROM conversation_session_streams WHERE session_id = OLD.task_session_id; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT OLD.task_session_id, watermark, OLD.task_session_id || ':' || watermark, 'message.deleted', NULLIF(OLD.task_id,''), payload, CURRENT_TIMESTAMP + FROM conversation_session_streams JOIN conversation_message_versions + ON conversation_message_versions.session_id = conversation_session_streams.session_id + AND conversation_message_versions.message_id = OLD.id + AND conversation_message_versions.row_sequence = conversation_session_streams.watermark + WHERE conversation_session_streams.session_id = OLD.task_session_id; +END; +DROP TRIGGER IF EXISTS conversation_turn_insert; + +CREATE TRIGGER IF NOT EXISTS conversation_turn_insert +AFTER INSERT ON task_session_turns +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_turn_versions(session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload) + SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.started_at, FALSE, + json_object('type','session.turn.started','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), + 'id',NEW.id,'started_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.started_at), + 'completed_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.completed_at), + 'created_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.created_at), + 'updated_at',strftime('%Y-%m-%dT%H:%M:%fZ', COALESCE(NEW.updated_at,NEW.started_at))) + FROM conversation_session_streams WHERE session_id = NEW.task_session_id; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'session.turn.started', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP + FROM conversation_session_streams JOIN conversation_turn_versions + ON conversation_turn_versions.session_id = conversation_session_streams.session_id + AND conversation_turn_versions.turn_id = NEW.id + AND conversation_turn_versions.row_sequence = conversation_session_streams.watermark + WHERE conversation_session_streams.session_id = NEW.task_session_id; +END; +DROP TRIGGER IF EXISTS conversation_turn_complete; + +CREATE TRIGGER IF NOT EXISTS conversation_turn_complete +AFTER UPDATE OF completed_at ON task_session_turns +WHEN OLD.completed_at IS NULL AND NEW.completed_at IS NOT NULL +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (NEW.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_turn_versions(session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload) + SELECT NEW.task_session_id, NEW.id, watermark, NULLIF(NEW.task_id, ''), NEW.started_at, FALSE, + json_object('type','session.turn.completed','session_id',NEW.task_session_id,'task_id',NULLIF(NEW.task_id,''), + 'id',NEW.id,'started_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.started_at), + 'completed_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.completed_at), + 'created_at',strftime('%Y-%m-%dT%H:%M:%fZ', NEW.created_at), + 'updated_at',strftime('%Y-%m-%dT%H:%M:%fZ', COALESCE(NEW.updated_at,NEW.completed_at,NEW.started_at)), + 'metadata',CASE WHEN json_valid(NEW.metadata) THEN json(NEW.metadata) ELSE json('{}') END, + 'had_output',json(CASE WHEN EXISTS ( + SELECT 1 FROM task_session_messages output + WHERE output.turn_id = NEW.id AND output.author_type = 'agent' + AND ( + output.type IN ('tool_call','tool_edit','tool_read','tool_search','tool_execute', + 'agent_plan','todo','permission_request','clarification_request') + OR (output.type IN ('message','content','') AND trim(output.content) <> '') + )) THEN 'true' ELSE 'false' END)) + FROM conversation_session_streams WHERE session_id = NEW.task_session_id; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT NEW.task_session_id, watermark, NEW.task_session_id || ':' || watermark, 'session.turn.completed', NULLIF(NEW.task_id,''), payload, CURRENT_TIMESTAMP + FROM conversation_session_streams JOIN conversation_turn_versions + ON conversation_turn_versions.session_id = conversation_session_streams.session_id + AND conversation_turn_versions.turn_id = NEW.id + AND conversation_turn_versions.row_sequence = conversation_session_streams.watermark + WHERE conversation_session_streams.session_id = NEW.task_session_id; +END; + +DROP TRIGGER IF EXISTS conversation_turn_delete; +CREATE TRIGGER IF NOT EXISTS conversation_turn_delete +AFTER DELETE ON task_session_turns +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (OLD.task_session_id, 1, FALSE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_turn_versions(session_id, turn_id, row_sequence, task_id, started_at, tombstone, payload) + SELECT OLD.task_session_id, OLD.id, watermark, NULLIF(OLD.task_id, ''), OLD.started_at, TRUE, + json_object('type','session.turn.removed','session_id',OLD.task_session_id,'task_id',NULLIF(OLD.task_id,''),'id',OLD.id) + FROM conversation_session_streams WHERE session_id = OLD.task_session_id; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT OLD.task_session_id, watermark, OLD.task_session_id || ':' || watermark, 'session.turn.removed', NULLIF(OLD.task_id,''), + payload, CURRENT_TIMESTAMP + FROM conversation_session_streams JOIN conversation_turn_versions + ON conversation_turn_versions.session_id = conversation_session_streams.session_id + AND conversation_turn_versions.turn_id = OLD.id + AND conversation_turn_versions.row_sequence = conversation_session_streams.watermark + WHERE conversation_session_streams.session_id = OLD.task_session_id; +END; + +DROP TRIGGER IF EXISTS conversation_session_delete; +CREATE TRIGGER IF NOT EXISTS conversation_session_delete +AFTER DELETE ON task_sessions +BEGIN + INSERT INTO conversation_session_streams(session_id, watermark, terminal, updated_at) + VALUES (OLD.id, 1, TRUE, CURRENT_TIMESTAMP) + ON CONFLICT(session_id) DO UPDATE SET watermark = watermark + 1, terminal = TRUE, updated_at = CURRENT_TIMESTAMP; + INSERT INTO conversation_session_events(session_id, sequence, event_id, event_type, task_id, payload, created_at) + SELECT OLD.id, watermark, OLD.id || ':' || watermark, 'session.removed', NULLIF(OLD.task_id,''), + json_object('type','session.removed','session_id',OLD.id,'task_id',NULLIF(OLD.task_id,'')), CURRENT_TIMESTAMP + FROM conversation_session_streams WHERE session_id = OLD.id; +END; diff --git a/apps/backend/internal/task/service/service_conversation_source.go b/apps/backend/internal/task/service/service_conversation_source.go new file mode 100644 index 0000000000..4c5bc80808 --- /dev/null +++ b/apps/backend/internal/task/service/service_conversation_source.go @@ -0,0 +1,50 @@ +package service + +import ( + "context" + "errors" + + "github.com/kandev/kandev/internal/task/models" + taskrepo "github.com/kandev/kandev/internal/task/repository" +) + +var errConversationSourceUnavailable = errors.New("conversation source repository is unavailable") + +// ReadConversationRevision returns the authorized current-state revision for a +// session. The repository performs the existence and revision read together. +func (s *Service) ReadConversationRevision(ctx context.Context, sessionID string) (models.ConversationRevision, error) { + if err := s.AuthorizeSessionAccess(ctx, sessionID); err != nil { + return models.ConversationRevision{}, err + } + reader, ok := s.messages.(taskrepo.ConversationSourceRepository) + if !ok { + return models.ConversationRevision{}, errConversationSourceUnavailable + } + return reader.ReadConversationRevision(ctx, sessionID) +} + +// ReadConversationMessagesPage returns one authorized, revision-bound source +// page without touching the first-party session message cache. +func (s *Service) ReadConversationMessagesPage(ctx context.Context, req models.ConversationMessagePageRequest) (models.ConversationMessagePage, error) { + if err := s.AuthorizeSessionAccess(ctx, req.SessionID); err != nil { + return models.ConversationMessagePage{}, err + } + reader, ok := s.messages.(taskrepo.ConversationSourceRepository) + if !ok { + return models.ConversationMessagePage{}, errConversationSourceUnavailable + } + return reader.ReadConversationMessagesPage(ctx, req) +} + +// ReadConversationTurnsPage returns one authorized, revision-bound source +// page without mutating the first-party turn cache. +func (s *Service) ReadConversationTurnsPage(ctx context.Context, req models.ConversationTurnPageRequest) (models.ConversationTurnPage, error) { + if err := s.AuthorizeSessionAccess(ctx, req.SessionID); err != nil { + return models.ConversationTurnPage{}, err + } + reader, ok := s.messages.(taskrepo.ConversationSourceRepository) + if !ok { + return models.ConversationTurnPage{}, errConversationSourceUnavailable + } + return reader.ReadConversationTurnsPage(ctx, req) +} diff --git a/apps/backend/internal/task/service/service_conversation_source_test.go b/apps/backend/internal/task/service/service_conversation_source_test.go new file mode 100644 index 0000000000..79e63b62af --- /dev/null +++ b/apps/backend/internal/task/service/service_conversation_source_test.go @@ -0,0 +1,136 @@ +package service + +import ( + "context" + "errors" + "testing" + + "github.com/kandev/kandev/internal/task/models" + taskrepo "github.com/kandev/kandev/internal/task/repository" + "github.com/kandev/kandev/internal/task/repository/repoerrors" +) + +type conversationSourceMessageRepository struct { + taskrepo.MessageRepository + taskrepo.ConversationSourceRepository + turnCalls int +} + +func (r *conversationSourceMessageRepository) ReadConversationTurnsPage( + ctx context.Context, + req models.ConversationTurnPageRequest, +) (models.ConversationTurnPage, error) { + r.turnCalls++ + return r.ConversationSourceRepository.ReadConversationTurnsPage(ctx, req) +} + +type conversationMessageOnlyRepository struct { + taskrepo.MessageRepository +} + +func TestConversationSourceServiceUsesMessageRepositoryForEveryRead(t *testing.T) { + source := &conversationSourceMessageRepository{ + ConversationSourceRepository: conversationSourceStub{ + revision: models.ConversationRevision{SessionID: "session-1", Exists: true, Revision: 7}, + messages: models.ConversationMessagePage{Revision: 7}, + turns: models.ConversationTurnPage{Revision: 7}, + }, + } + svc, _, _ := createTestService(t) + svc.messages = source + + ctx := context.Background() + if _, err := svc.ReadConversationRevision(ctx, "session-1"); err != nil { + t.Fatalf("ReadConversationRevision: %v", err) + } + if _, err := svc.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{SessionID: "session-1"}); err != nil { + t.Fatalf("ReadConversationMessagesPage: %v", err) + } + if _, err := svc.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{SessionID: "session-1"}); err != nil { + t.Fatalf("ReadConversationTurnsPage: %v", err) + } + if source.turnCalls != 1 { + t.Fatalf("turn source calls = %d, want 1", source.turnCalls) + } +} + +func TestConversationSourceServiceAuthorizesAllReadsBeforeDispatch(t *testing.T) { + svc, _, repo := createTestService(t) + seedScopedWorkspaces(t, repo) + source := &conversationSourceMessageRepository{ + ConversationSourceRepository: conversationSourceStub{}, + } + svc.messages = source + ctx := ctxAs("user-a") + + checks := []struct { + name string + call func() error + }{ + { + name: "revision", + call: func() error { + _, err := svc.ReadConversationRevision(ctx, "sess-b") + return err + }, + }, + { + name: "messages", + call: func() error { + _, err := svc.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{SessionID: "sess-b"}) + return err + }, + }, + { + name: "turns", + call: func() error { + _, err := svc.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{SessionID: "sess-b"}) + return err + }, + }, + } + for _, check := range checks { + t.Run(check.name, func(t *testing.T) { + if err := check.call(); !errors.Is(err, repoerrors.ErrTaskNotFound) { + t.Fatalf("error = %v, want ErrTaskNotFound", err) + } + }) + } + if source.turnCalls != 0 { + t.Fatalf("source turns calls = %d after denied reads, want 0", source.turnCalls) + } +} + +func TestConversationSourceServiceReportsUnavailableMessageRepository(t *testing.T) { + svc, _, _ := createTestService(t) + svc.messages = conversationMessageOnlyRepository{} + ctx := context.Background() + + if _, err := svc.ReadConversationRevision(ctx, "session-1"); !errors.Is(err, errConversationSourceUnavailable) { + t.Fatalf("revision error = %v, want source unavailable", err) + } + if _, err := svc.ReadConversationMessagesPage(ctx, models.ConversationMessagePageRequest{SessionID: "session-1"}); !errors.Is(err, errConversationSourceUnavailable) { + t.Fatalf("message error = %v, want source unavailable", err) + } + if _, err := svc.ReadConversationTurnsPage(ctx, models.ConversationTurnPageRequest{SessionID: "session-1"}); !errors.Is(err, errConversationSourceUnavailable) { + t.Fatalf("turn error = %v, want source unavailable", err) + } +} + +type conversationSourceStub struct { + revision models.ConversationRevision + messages models.ConversationMessagePage + turns models.ConversationTurnPage +} + +func (s conversationSourceStub) ReadConversationRevision(context.Context, string) (models.ConversationRevision, error) { + return s.revision, nil +} + +func (s conversationSourceStub) ReadConversationMessagesPage(context.Context, models.ConversationMessagePageRequest) (models.ConversationMessagePage, error) { + return s.messages, nil +} + +func (s conversationSourceStub) ReadConversationTurnsPage(context.Context, models.ConversationTurnPageRequest) (models.ConversationTurnPage, error) { + return s.turns, nil +} diff --git a/apps/backend/internal/task/service/service_events.go b/apps/backend/internal/task/service/service_events.go index 7631ed3dd0..f5137dd6c7 100644 --- a/apps/backend/internal/task/service/service_events.go +++ b/apps/backend/internal/task/service/service_events.go @@ -1010,8 +1010,13 @@ func (s *Service) publishEnvironmentEvent(ctx context.Context, eventType string, // straight into the repository so specs can script clarification/permission // states deterministically, and without this it never triggers the // pending_action recompute a real agent turn would. -func (s *Service) PublishMessageEvent(ctx context.Context, eventType string, message *models.Message) error { - return s.publishMessageEvent(ctx, eventType, message) +func (s *Service) PublishMessageEvent( + ctx context.Context, + eventType string, + message *models.Message, + receipts ...*models.ConversationMutationReceipt, +) error { + return s.publishMessageEvent(ctx, eventType, message, receipts...) } // publishMessageEvent publishes message events to the event bus. @@ -1020,12 +1025,17 @@ func (s *Service) PublishMessageEvent(ctx context.Context, eventType string, mes // Ordinary persistence callers intentionally treat delivery as best effort // after their durable write succeeds. Synchronization-sensitive callers, such // as clarification bundle convergence, check and propagate the returned error. -func (s *Service) publishMessageEvent(ctx context.Context, eventType string, message *models.Message) error { +func (s *Service) publishMessageEvent(ctx context.Context, eventType string, message *models.Message, receipts ...*models.ConversationMutationReceipt) error { if s.eventBus == nil { s.logger.Warn("publishMessageEvent: eventBus is nil, skipping") return errors.New("event bus is unavailable") } event := newMessageEvent(eventType, message) + if len(receipts) > 0 && receipts[0] != nil { + if data, ok := event.Data.(map[string]interface{}); ok { + data["conversation_receipt"] = projectConversationReceipt(receipts[0]) + } + } pendingProjection := s.addMessagePendingAction(ctx, eventType, message, event) if err := s.eventBus.Publish(ctx, eventType, event); err != nil { s.logger.Error("failed to publish message event", @@ -1040,6 +1050,42 @@ func (s *Service) publishMessageEvent(ctx context.Context, eventType string, mes return nil } +// projectConversationReceipt keeps the transient source receipt useful to the +// live conversation transport without exposing repository-owned metadata or +// system-injected message content through the event bus. +func projectConversationReceipt(receipt *models.ConversationMutationReceipt) *models.ConversationMutationReceipt { + if receipt == nil { + return nil + } + projected := &models.ConversationMutationReceipt{ + SessionID: receipt.SessionID, + BaseRevision: receipt.BaseRevision, + Revision: receipt.Revision, + Complete: receipt.Complete, + Operations: make([]models.ConversationMutationOperation, 0, len(receipt.Operations)), + } + for _, operation := range receipt.Operations { + copyOperation := operation + if operation.Message != nil { + message := *operation.Message + message.Content = sysprompt.StripSystemContent(message.Content) + message.Metadata = models.ProjectMessageMetadata(message.Metadata) + copyOperation.Message = &message + } + if operation.Turn != nil { + turn := *operation.Turn + turn.Metadata = models.ProjectTurnMetadata(turn.Metadata) + copyOperation.Turn = &turn + } + if operation.HadOutput != nil { + hadOutput := *operation.HadOutput + copyOperation.HadOutput = &hadOutput + } + projected.Operations = append(projected.Operations, copyOperation) + } + return projected +} + type pendingActionProjection struct { action models.TaskPendingAction revision models.PendingActionRevision diff --git a/apps/backend/internal/task/service/service_events_conversation_receipt_test.go b/apps/backend/internal/task/service/service_events_conversation_receipt_test.go new file mode 100644 index 0000000000..72605c6a0b --- /dev/null +++ b/apps/backend/internal/task/service/service_events_conversation_receipt_test.go @@ -0,0 +1,66 @@ +package service + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/kandev/kandev/internal/task/models" +) + +func TestProjectConversationReceiptDoesNotExposeOrMutateSourceEntities(t *testing.T) { + message := &models.Message{ + ID: "message-1", + Content: "private promptvisible", + Metadata: map[string]any{ + "normalized": map[string]any{ + "shell_exec": map[string]any{ + "output": map[string]any{"stdout": "private shell output"}, + }, + }, + }, + } + turn := &models.Turn{ID: "turn-1", Metadata: map[string]any{ + models.TurnMetaKeyRuntimeConfigSnapshot: models.TurnRuntimeConfigSnapshot{Model: "mock-fast"}, + models.TurnMetaKeyPromptDispatchPending: true, + "internal": "private turn state", + }} + receipt := &models.ConversationMutationReceipt{ + SessionID: "session-1", + Operations: []models.ConversationMutationOperation{ + {Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityMessage, Message: message}, + {Kind: models.ConversationMutationUpsert, Entity: models.ConversationEntityTurn, Turn: turn}, + }, + } + + projected := projectConversationReceipt(receipt) + if projected == receipt || projected.Operations[0].Message == message || projected.Operations[1].Turn == turn { + t.Fatal("receipt projection reused source-owned entities") + } + if got := projected.Operations[0].Message.Content; got != "visible" { + t.Fatalf("projected message content = %q, want visible content", got) + } + projectedMetadata := projected.Operations[1].Turn.Metadata + if projectedMetadata[models.TurnMetaKeyRuntimeConfigSnapshot] == nil { + t.Fatal("projected turn dropped public runtime metadata") + } + if _, ok := projectedMetadata[models.TurnMetaKeyPromptDispatchPending]; ok { + t.Fatal("projected turn retained prompt-dispatch metadata") + } + if _, ok := projectedMetadata["internal"]; ok { + t.Fatal("projected turn retained private metadata") + } + encoded, err := json.Marshal(projected) + if err != nil { + t.Fatalf("marshal projected receipt: %v", err) + } + if strings.Contains(string(encoded), "private prompt") || strings.Contains(string(encoded), "private shell output") { + t.Fatalf("projected receipt contains private source data: %s", encoded) + } + if message.Content != "private promptvisible" { + t.Fatal("receipt projection mutated source message content") + } + if turn.Metadata["internal"] != "private turn state" || turn.Metadata[models.TurnMetaKeyPromptDispatchPending] != true { + t.Fatal("receipt projection mutated source turn metadata") + } +} diff --git a/apps/backend/internal/task/service/service_messages.go b/apps/backend/internal/task/service/service_messages.go index 61e101b05c..0825629651 100644 --- a/apps/backend/internal/task/service/service_messages.go +++ b/apps/backend/internal/task/service/service_messages.go @@ -84,6 +84,12 @@ type initialTaskBriefQueuedPlanCommentMessageWriter interface { ) (*models.TaskPlanCommentSnapshot, error) } +type conversationMessageReceiptWriter interface { + CreateMessageWithConversationReceipt(context.Context, *models.Message) (*models.ConversationMutationReceipt, error) + UpdateMessageWithConversationReceipt(context.Context, *models.Message) (*models.ConversationMutationReceipt, error) + DeleteMessageWithConversationReceipt(context.Context, string) (*models.ConversationMutationReceipt, error) +} + type planCommentAdmissionLocker interface { AcquirePlanCommentAdmission(context.Context, string) (context.Context, func(), error) } @@ -246,14 +252,14 @@ func (s *Service) CreateMessage(ctx context.Context, req *CreateMessageRequest) // as an explicit import and reject same-microsecond creates. } - snapshot, err := s.persistMessage(ctx, message, req) + snapshot, receipt, err := s.persistMessage(ctx, message, req) if err != nil { s.logger.Error("failed to create message", zap.Error(err)) return nil, err } // Publish message.added event - s.publishMessageEvent(ctx, events.MessageAdded, message) + _ = s.publishMessageEvent(ctx, events.MessageAdded, message, receipt) s.publishMessagePlanCommentSnapshot(ctx, snapshot) s.logger.Info("message created", @@ -504,7 +510,7 @@ func (s *Service) CreateMessageWithID(ctx context.Context, id string, req *Creat return nil, err } - snapshot, err := s.createMessageWithRequestRetry( + snapshot, receipt, err := s.createMessageWithRequestRetry( ctx, message, req, messageCreateMaxRetries, messageCreateRetryDelay, ) if err != nil { @@ -512,7 +518,7 @@ func (s *Service) CreateMessageWithID(ctx context.Context, id string, req *Creat } // Publish message.added event - s.publishMessageEvent(ctx, events.MessageAdded, message) + _ = s.publishMessageEvent(ctx, events.MessageAdded, message, receipt) s.publishMessagePlanCommentSnapshot(ctx, snapshot) s.logger.Info("message created with ID", @@ -645,19 +651,20 @@ func (s *Service) createMessageWithRequestRetry( req *CreateMessageRequest, maxRetries int, retryDelay time.Duration, -) (*models.TaskPlanCommentSnapshot, error) { +) (*models.TaskPlanCommentSnapshot, *models.ConversationMutationReceipt, error) { var err error for attempt := 0; attempt < maxRetries; attempt++ { var snapshot *models.TaskPlanCommentSnapshot - snapshot, err = s.persistMessage(ctx, message, req) + var receipt *models.ConversationMutationReceipt + snapshot, receipt, err = s.persistMessage(ctx, message, req) if err == nil { - return snapshot, nil + return snapshot, receipt, nil } if ctx.Err() != nil { - return nil, ctx.Err() + return nil, nil, ctx.Err() } if errors.Is(err, repoerrors.ErrInitialTaskBriefStale) { - return nil, err + return nil, nil, err } if attempt < maxRetries-1 { s.logger.Debug("failed to create message, retrying", @@ -672,41 +679,47 @@ func (s *Service) createMessageWithRequestRetry( zap.String("id", message.ID), zap.Int("retries", maxRetries), zap.Error(err)) - return nil, err + return nil, nil, err } func (s *Service) persistMessage( ctx context.Context, message *models.Message, req *CreateMessageRequest, -) (*models.TaskPlanCommentSnapshot, error) { +) (*models.TaskPlanCommentSnapshot, *models.ConversationMutationReceipt, error) { if len(req.PlanCommentRefs) == 0 { if req.InitialTaskBrief != nil { writer, ok := s.messages.(initialTaskBriefMessageWriter) if !ok { - return nil, errors.New("initial task brief admission is unavailable") + return nil, nil, errors.New("initial task brief admission is unavailable") } - return nil, writer.CreateMessageWithInitialTaskBrief(ctx, message, req.InitialTaskBrief) + return nil, nil, writer.CreateMessageWithInitialTaskBrief(ctx, message, req.InitialTaskBrief) } - return nil, s.messages.CreateMessage(ctx, message) + if writer, ok := s.messages.(conversationMessageReceiptWriter); ok { + receipt, err := writer.CreateMessageWithConversationReceipt(ctx, message) + return nil, receipt, err + } + return nil, nil, s.messages.CreateMessage(ctx, message) } if req.InitialTaskBrief != nil { writer, ok := s.messages.(initialTaskBriefPlanCommentMessageWriter) if !ok { - return nil, errors.New("plan comment initial task brief admission is unavailable") + return nil, nil, errors.New("plan comment initial task brief admission is unavailable") } - return writer.CreateMessageWithPlanCommentsWithInitialTaskBrief( + snapshot, err := writer.CreateMessageWithPlanCommentsWithInitialTaskBrief( ctx, message, req.InitialTaskBrief, req.PlanCommentRefs, req.RequirePrimarySession, req.ExpectedSessionState, req.AttachmentClaim, ) + return snapshot, nil, err } writer, ok := s.messages.(planCommentMessageWriter) if !ok { - return nil, errors.New("plan comment message admission is unavailable") + return nil, nil, errors.New("plan comment message admission is unavailable") } - return writer.CreateMessageWithPlanComments( + snapshot, err := writer.CreateMessageWithPlanComments( ctx, message, req.PlanCommentRefs, req.RequirePrimarySession, req.ExpectedSessionState, req.AttachmentClaim, ) + return snapshot, nil, err } type planCommentMessageReplayIdentity struct { @@ -866,13 +879,20 @@ func (s *Service) SearchMessages(ctx context.Context, sessionID, query string, l // DeleteMessage deletes a message func (s *Service) DeleteMessage(ctx context.Context, id string) error { message, getErr := s.messages.GetMessage(ctx, id) - if err := s.messages.DeleteMessage(ctx, id); err != nil { + var receipt *models.ConversationMutationReceipt + var err error + if writer, ok := s.messages.(conversationMessageReceiptWriter); ok { + receipt, err = writer.DeleteMessageWithConversationReceipt(ctx, id) + } else { + err = s.messages.DeleteMessage(ctx, id) + } + if err != nil { s.logger.Error("failed to delete message", zap.String("message_id", id), zap.Error(err)) return err } if getErr == nil && message != nil { - s.publishMessageEvent(ctx, events.MessageDeleted, message) + _ = s.publishMessageEvent(ctx, events.MessageDeleted, message, receipt) } s.logger.Info("message deleted", zap.String("message_id", id)) return nil @@ -880,7 +900,8 @@ func (s *Service) DeleteMessage(ctx context.Context, id string) error { // UpdateMessage updates an existing message and publishes an event. func (s *Service) UpdateMessage(ctx context.Context, message *models.Message) error { - if err := s.messages.UpdateMessage(ctx, message); err != nil { + receipt, err := s.updateMessageWithReceipt(ctx, message) + if err != nil { s.logger.Error("failed to update message", zap.String("message_id", message.ID), zap.Error(err)) @@ -904,11 +925,18 @@ func (s *Service) UpdateMessage(ctx context.Context, message *models.Message) er // Publish message.updated event for real-time streaming. Delivery is best // effort after the durable write succeeded (see publishMessageEvent). - _ = s.publishMessageEvent(ctx, events.MessageUpdated, published) + _ = s.publishMessageEvent(ctx, events.MessageUpdated, published, receipt) return nil } +func (s *Service) updateMessageWithReceipt(ctx context.Context, message *models.Message) (*models.ConversationMutationReceipt, error) { + if writer, ok := s.messages.(conversationMessageReceiptWriter); ok { + return writer.UpdateMessageWithConversationReceipt(ctx, message) + } + return nil, s.messages.UpdateMessage(ctx, message) +} + // AppendMessageContent appends additional content to an existing message. // This is used for streaming agent responses where content arrives incrementally. func (s *Service) AppendMessageContent(ctx context.Context, messageID, additionalContent string) error { @@ -923,7 +951,8 @@ func (s *Service) AppendMessageContent(ctx context.Context, messageID, additiona // Append the new content message.Content += additionalContent - if err := s.messages.UpdateMessage(ctx, message); err != nil { + receipt, err := s.updateMessageWithReceipt(ctx, message) + if err != nil { s.logger.Error("failed to append message content", zap.String("message_id", messageID), zap.Error(err)) @@ -931,7 +960,7 @@ func (s *Service) AppendMessageContent(ctx context.Context, messageID, additiona } // Publish message.updated event for real-time streaming - s.publishMessageEvent(ctx, events.MessageUpdated, message) + _ = s.publishMessageEvent(ctx, events.MessageUpdated, message, receipt) s.logger.Debug("message content appended", zap.String("message_id", messageID), @@ -964,7 +993,8 @@ func (s *Service) AppendThinkingContent(ctx context.Context, messageID, addition } message.Metadata["thinking"] = existingThinking + additionalContent - if err := s.messages.UpdateMessage(ctx, message); err != nil { + receipt, err := s.updateMessageWithReceipt(ctx, message) + if err != nil { s.logger.Error("failed to append thinking content", zap.String("message_id", messageID), zap.Error(err)) @@ -972,7 +1002,7 @@ func (s *Service) AppendThinkingContent(ctx context.Context, messageID, addition } // Publish message.updated event for real-time streaming - s.publishMessageEvent(ctx, events.MessageUpdated, message) + _ = s.publishMessageEvent(ctx, events.MessageUpdated, message, receipt) s.logger.Debug("thinking content appended", zap.String("message_id", messageID), @@ -1015,7 +1045,8 @@ func (s *Service) UpdateToolCallMessageWithCreate(ctx context.Context, sessionID s.applyToolCallMessageUpdate(message, status, result, title, normalized) - if err := s.messages.UpdateMessage(ctx, message); err != nil { + receipt, err := s.updateMessageWithReceipt(ctx, message) + if err != nil { s.logger.Error("failed to update tool call message", zap.String("message_id", message.ID), zap.String("tool_call_id", toolCallID), @@ -1024,7 +1055,7 @@ func (s *Service) UpdateToolCallMessageWithCreate(ctx context.Context, sessionID } // Publish message.updated event - s.publishMessageEvent(ctx, events.MessageUpdated, message) + _ = s.publishMessageEvent(ctx, events.MessageUpdated, message, receipt) s.logger.Info("tool call message updated", zap.String("message_id", message.ID), @@ -1200,7 +1231,8 @@ func (s *Service) UpdatePermissionMessage(ctx context.Context, taskID, sessionID } message.Metadata["status"] = string(status) - if err := s.messages.UpdateMessage(ctx, message); err != nil { + receipt, err := s.updateMessageWithReceipt(ctx, message) + if err != nil { s.logger.Error("failed to update permission message", zap.String("message_id", message.ID), zap.String("pending_id", pendingID), @@ -1209,7 +1241,7 @@ func (s *Service) UpdatePermissionMessage(ctx context.Context, taskID, sessionID } // Publish message.updated event - s.publishMessageEvent(ctx, events.MessageUpdated, message) + _ = s.publishMessageEvent(ctx, events.MessageUpdated, message, receipt) // When a permission expires, also mark the related tool call as cancelled // so the UI no longer shows a loading spinner on the tool call. @@ -1325,7 +1357,8 @@ func (s *Service) UpdateClarificationMessageForQuestion(ctx context.Context, ses message.Metadata["response"] = answer } - if err := s.messages.UpdateMessage(ctx, message); err != nil { + receipt, err := s.updateMessageWithReceipt(ctx, message) + if err != nil { s.logger.Error("failed to update clarification message", zap.String("message_id", message.ID), zap.String("pending_id", pendingID), @@ -1334,7 +1367,7 @@ func (s *Service) UpdateClarificationMessageForQuestion(ctx context.Context, ses return err } - s.publishMessageEvent(ctx, events.MessageUpdated, message) + _ = s.publishMessageEvent(ctx, events.MessageUpdated, message, receipt) s.logger.Info("clarification message updated", zap.String("message_id", message.ID), diff --git a/apps/backend/internal/task/service/service_turns.go b/apps/backend/internal/task/service/service_turns.go index 16fe1d88be..841cdcaee1 100644 --- a/apps/backend/internal/task/service/service_turns.go +++ b/apps/backend/internal/task/service/service_turns.go @@ -86,7 +86,15 @@ func (s *Service) createTurn( UpdatedAt: time.Now().UTC(), } - stamped, err := s.turns.CreateTurnWithStepStamp(ctx, turn) + var ( + stamped bool + receipt *models.ConversationMutationReceipt + ) + if writer, ok := s.turns.(taskrepo.ConversationTurnStampWriter); ok { + stamped, receipt, err = writer.CreateTurnWithStepStampConversationReceipt(ctx, turn) + } else { + stamped, err = s.turns.CreateTurnWithStepStamp(ctx, turn) + } if err != nil { s.logger.Error("failed to create turn", zap.Error(err)) return nil, err @@ -97,7 +105,7 @@ func (s *Service) createTurn( if publishStarted { // had_output is only meaningful on turn.completed; omit it from turn.started. - _ = s.publishTurnEvent(events.TurnStarted, turn, nil) + _ = s.publishTurnEvent(events.TurnStarted, turn, nil, receipt) } s.logger.Debug("started turn", @@ -447,20 +455,12 @@ func (s *Service) CompleteTurn(ctx context.Context, turnID string) error { return nil // No active turn to complete } - if err := s.turns.CompleteTurn(ctx, turnID); err != nil { + receipt, err := s.completeTurnMutation(ctx, turnID) + if err != nil { s.logger.Error("failed to complete turn", zap.String("turn_id", turnID), zap.Error(err)) return err } - // Safety net: mark any tool calls still in a non-terminal state as "complete" - if affected, err := s.turns.CompletePendingToolCallsForTurn(ctx, turnID); err != nil { - s.logger.Warn("failed to complete pending tool calls for turn", zap.String("turn_id", turnID), zap.Error(err)) - } else if affected > 0 { - s.logger.Info("completed stale pending tool calls on turn end", - zap.String("turn_id", turnID), - zap.Int64("affected", affected)) - } - // Fetch the completed turn to get the completed_at timestamp turn, err := s.turns.GetTurn(ctx, turnID) if err != nil { @@ -470,7 +470,7 @@ func (s *Service) CompleteTurn(ctx context.Context, turnID string) error { } hadOutput := s.turnHadOutput(ctx, turn) - _ = s.publishTurnEvent(events.TurnCompleted, turn, &hadOutput) + _ = s.publishTurnEvent(events.TurnCompleted, turn, &hadOutput, receipt) s.logger.Debug("completed turn", zap.String("turn_id", turnID), @@ -480,6 +480,20 @@ func (s *Service) CompleteTurn(ctx context.Context, turnID string) error { return nil } +func (s *Service) completeTurnMutation(ctx context.Context, turnID string) (*models.ConversationMutationReceipt, error) { + if writer, ok := s.turns.(taskrepo.ConversationMutationWriter); ok { + return writer.CompleteTurnWithConversationReceipt(ctx, turnID) + } + if affected, err := s.turns.CompletePendingToolCallsForTurn(ctx, turnID); err != nil { + s.logger.Warn("failed to complete pending tool calls for turn", zap.String("turn_id", turnID), zap.Error(err)) + } else if affected > 0 { + s.logger.Info("completed stale pending tool calls on turn end", + zap.String("turn_id", turnID), + zap.Int64("affected", affected)) + } + return nil, s.turns.CompleteTurn(ctx, turnID) +} + // GetActiveTurn returns the currently active (non-completed) turn for a session. // Returns (nil, nil) when no turn is active. Other errors (DB failures) are returned as-is. func (s *Service) GetActiveTurn(ctx context.Context, sessionID string) (*models.Turn, error) { @@ -625,7 +639,7 @@ func (s *Service) PublishTurnStarted(ctx context.Context, turn *models.Turn) err // turn.completed events (the frontend uses it to surface an "empty turn" // notice). Pass nil for turn.started so the field is omitted entirely rather // than carrying a misleading "false" on a turn that has not completed. -func (s *Service) publishTurnEvent(eventType string, turn *models.Turn, hadOutput *bool) error { +func (s *Service) publishTurnEvent(eventType string, turn *models.Turn, hadOutput *bool, receipts ...*models.ConversationMutationReceipt) error { if s.eventBus == nil { return errors.New("turn event bus is unavailable") } @@ -654,6 +668,21 @@ func (s *Service) publishTurnEvent(eventType string, turn *models.Turn, hadOutpu if hadOutput != nil { payload["had_output"] = *hadOutput } + if len(receipts) > 0 && receipts[0] != nil { + projectedReceipt := projectConversationReceipt(receipts[0]) + if hadOutput != nil { + for index := range projectedReceipt.Operations { + operation := &projectedReceipt.Operations[index] + if operation.Entity != models.ConversationEntityTurn || operation.Turn == nil || operation.Turn.ID != turn.ID { + continue + } + output := *hadOutput + operation.HadOutput = &output + break + } + } + payload["conversation_receipt"] = projectedReceipt + } if err := s.eventBus.Publish(context.Background(), eventType, bus.NewEvent(eventType, "task-service", payload)); err != nil { s.logger.Error("failed to publish turn event", zap.String("event_type", eventType), diff --git a/apps/backend/internal/task/service/service_turns_output_test.go b/apps/backend/internal/task/service/service_turns_output_test.go index b67d4a0554..c1b16890a9 100644 --- a/apps/backend/internal/task/service/service_turns_output_test.go +++ b/apps/backend/internal/task/service/service_turns_output_test.go @@ -33,6 +33,33 @@ func lastTurnCompletedHadOutput(t *testing.T, eventBus *MockEventBus) (bool, boo return false, false } +func lastTurnCompletedReceiptHadOutput(t *testing.T, eventBus *MockEventBus) (bool, bool) { + t.Helper() + published := eventBus.GetPublishedEvents() + for i := len(published) - 1; i >= 0; i-- { + ev := published[i] + if ev.Type != events.TurnCompleted { + continue + } + data, ok := ev.Data.(map[string]interface{}) + if !ok { + t.Fatalf("turn.completed event data is not a map: %T", ev.Data) + } + receipt, ok := data["conversation_receipt"].(*models.ConversationMutationReceipt) + if !ok || receipt == nil { + return false, false + } + for _, operation := range receipt.Operations { + if operation.Entity != models.ConversationEntityTurn || operation.HadOutput == nil { + continue + } + return *operation.HadOutput, true + } + return false, false + } + return false, false +} + func TestCompleteTurn_PublishesHadOutput(t *testing.T) { tests := []struct { name string @@ -95,6 +122,13 @@ func TestCompleteTurn_PublishesHadOutput(t *testing.T) { if got != tt.want { t.Errorf("had_output = %v, want %v", got, tt.want) } + receiptGot, receiptFound := lastTurnCompletedReceiptHadOutput(t, eventBus) + if !receiptFound { + t.Fatal("conversation receipt missing had_output") + } + if receiptGot != tt.want { + t.Errorf("conversation receipt had_output = %v, want %v", receiptGot, tt.want) + } }) } } diff --git a/apps/backend/pkg/websocket/actions.go b/apps/backend/pkg/websocket/actions.go index a09fb39762..e7b4bc5ad9 100644 --- a/apps/backend/pkg/websocket/actions.go +++ b/apps/backend/pkg/websocket/actions.go @@ -169,12 +169,15 @@ const ( ActionWorkflowHistoryList = "workflow.history.list" // Subscription actions - ActionTaskSubscribe = "task.subscribe" - ActionTaskUnsubscribe = "task.unsubscribe" - ActionSessionSubscribe = "session.subscribe" - ActionSessionUnsubscribe = "session.unsubscribe" - ActionSessionAck = "session.ack" - ActionSessionPoisonRequeue = "session.event.poison.requeue" + ActionTaskSubscribe = "task.subscribe" + ActionTaskUnsubscribe = "task.unsubscribe" + ActionSessionSubscribe = "session.subscribe" + ActionSessionUnsubscribe = "session.unsubscribe" + // Host-only v2 conversation delivery. These actions use transient + // revision-covered change batches and never expose the legacy replay log. + ActionSessionConversationSubscribe = "session.conversation.subscribe" + ActionSessionConversationUnsubscribe = "session.conversation.unsubscribe" + ActionSessionConversationChanged = "session.conversation.changed" // Focus signals are layered on top of subscriptions to indicate which // session the user is actively viewing (task details page or task panel), // vs merely subscribed (sidebar diff badges). Drives backend polling tier. diff --git a/apps/web/e2e/fixtures/plugins/prompt-history-plugin/bundle.js b/apps/web/e2e/fixtures/plugins/prompt-history-plugin/bundle.js index 4aa8cad07c..de2bf211bd 100644 --- a/apps/web/e2e/fixtures/plugins/prompt-history-plugin/bundle.js +++ b/apps/web/e2e/fixtures/plugins/prompt-history-plugin/bundle.js @@ -34,7 +34,7 @@ * a tiny listener set, so it survives across route navigations (the page * component unmounts/remounts as the user navigates away and back). */ -/* eslint-disable max-lines, max-lines-per-function, sonarjs/no-duplicate-string -- The fixture is the literal self-contained, dependency-free browser bundle exercised and hashed by E2E. */ +/* eslint-disable max-lines, max-lines-per-function -- The fixture is the literal self-contained, dependency-free browser bundle exercised and hashed by E2E. */ (function () { var moduleCount = 0; var listeners = new Set(); @@ -70,6 +70,9 @@ return count; } + var FIXTURE_REPOSITORY_ID = "fixture-repository"; + var FIXTURE_HELLO_PATH = "/plugins/e2e-hello"; + var FIXTURE_SIDEBAR_SECTION = "sidebar-footer"; var PROVIDER_ID = "fixture-source-control"; var PULL_REQUEST_URL = "https://bitbucket.example.test/projects/TEAM/repos/fixture/pull-requests/42"; @@ -77,8 +80,8 @@ function fixtureRepository() { return { - id: "fixture-repository", - repositoryId: "fixture-repository", + id: FIXTURE_REPOSITORY_ID, + repositoryId: FIXTURE_REPOSITORY_ID, owner: "TEAM", ownerOrProject: "TEAM", name: "fixture", @@ -731,14 +734,14 @@ registry.registerNavItem({ id: "e2e-hello", label: "Hello E2E", - path: "/plugins/e2e-hello", + path: FIXTURE_HELLO_PATH, section: "main", }); registry.registerNavItem({ id: "e2e-insights-tools", label: "E2E Insights Tools", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); // Three more sidebar-footer items so this one plugin install alone // produces P = 4 (budget MAX_INLINE_PLUGIN_FOOTER_ITEMS = 3, plus one @@ -754,22 +757,22 @@ registry.registerNavItem({ id: "e2e-insights-tools-2", label: "E2E Overflow Item 2", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); registry.registerNavItem({ id: "e2e-insights-tools-3", label: "E2E Overflow Item 3", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); registry.registerNavItem({ id: "e2e-insights-tools-4", label: "E2E Overflow Item 4", - path: "/plugins/e2e-hello", - section: "sidebar-footer", + path: FIXTURE_HELLO_PATH, + section: FIXTURE_SIDEBAR_SECTION, }); - registry.registerRoute("/plugins/e2e-hello", PluginPage); + registry.registerRoute(FIXTURE_HELLO_PATH, PluginPage); registry.registerComponent("task-sidebar", SidebarSlot); registry.registerComponent("main-top-bar", MainTopBarSlot); registry.registerComponent("app-status-bar-left", StatusSlot); @@ -801,7 +804,7 @@ providerId: PROVIDER_ID, providerHost: "bitbucket.example.test", ownerOrProject: "TEAM", - repositoryId: "fixture-repository", + repositoryId: FIXTURE_REPOSITORY_ID, repositoryName: "fixture", cloneUrl: REPOSITORY_URL, defaultBranch: "main", @@ -835,7 +838,7 @@ title: "Bitbucket Pull Request #42", url: PULL_REQUEST_URL, connectionScope: "https://bitbucket.example.test", - repositoryId: "fixture-repository", + repositoryId: FIXTURE_REPOSITORY_ID, changeRequestNumber: 42, state: "OPEN", statusBadge: { label: "Open" }, @@ -847,7 +850,7 @@ title: "Bitbucket Pull Request #43", url: "https://bitbucket.example.test/projects/TEAM/repos/fixture/pull-requests/43", connectionScope: "https://bitbucket.example.test", - repositoryId: "fixture-repository", + repositoryId: FIXTURE_REPOSITORY_ID, changeRequestNumber: 43, state: "OPEN", statusBadge: { label: "Open" }, diff --git a/apps/web/e2e/helpers/ws-drop.ts b/apps/web/e2e/helpers/ws-drop.ts index 2aeee69ad8..703943feed 100644 --- a/apps/web/e2e/helpers/ws-drop.ts +++ b/apps/web/e2e/helpers/ws-drop.ts @@ -16,9 +16,9 @@ type MessageAddResponseDropController = { droppedCount: () => number; }; -type ExpiredPluginSnapshotController = { - expireNextPluginSnapshot: () => void; - modifiedCount: () => number; +type ConversationChangeDropController = { + dropChange: (content: string) => void; + droppedCount: () => number; pluginSubscribeCount: () => number; }; @@ -36,9 +36,9 @@ function asRecord(value: unknown): Record | null { } function parseJSONFrames(message: string | Buffer): Array> { - if (typeof message !== "string") return []; + const text = typeof message === "string" ? message : message.toString("utf8"); const frames: Array> = []; - for (const part of message.split("\n")) { + for (const part of text.split("\n")) { if (!part.trim()) continue; try { const parsed = asRecord(JSON.parse(part)); @@ -102,6 +102,11 @@ function filterServerFrame( dropped.push({ action: targetAction(parsed), content: parsed.payload.content }); continue; } + if (hasConversationChangeContent(parsed, prompt)) { + didDrop = true; + dropped.push({ action: "session.conversation.changed", content: prompt }); + continue; + } } catch { // Preserve non-JSON frames; the app's WS client handles parse failures too. } @@ -408,17 +413,57 @@ export async function routeMainWebSocketWithQueueAdmissionDrops( }; } +function hasConversationChangeContent(message: unknown, content: string): boolean { + const envelope = asRecord(message); + if (envelope?.action !== "session.conversation.changed") { + return false; + } + return JSON.stringify(envelope).includes(content); +} + +function filterConversationChange( + message: string | Buffer, + content: string | null, + state: { value: number }, +): string | Buffer { + if (content === null) return message; + const text = typeof message === "string" ? message : message.toString("utf8"); + const kept: string[] = []; + let didDrop = false; + for (const part of text.split("\n")) { + const trimmed = part.trim(); + if (!trimmed) { + kept.push(part); + continue; + } + let frame: unknown; + try { + frame = JSON.parse(trimmed); + } catch { + kept.push(part); + continue; + } + if (hasConversationChangeContent(frame, content)) { + state.value += 1; + didDrop = true; + continue; + } + kept.push(part); + } + if (!didDrop) return message; + const filtered = kept.join("\n"); + return typeof message === "string" ? filtered : Buffer.from(filtered, "utf8"); +} + /** - * Rewrites one plugin subscription expiry in the browser transport. The - * signed token remains server-valid, so the panel must exercise its normal - * fresh rebind path instead of relying on a relaxed backend validation rule. + * Drops one durable plugin conversation change while preserving the socket. + * The following change creates a revision gap and exercises source recovery. */ -export async function routeMainWebSocketWithExpiredPluginSnapshot( +export async function routeMainWebSocketWithConversationChangeDrop( page: Page, -): Promise { - const requestIDs = new Set(); - let armed = false; - let modified = 0; +): Promise { + let contentToDrop: string | null = null; + const dropped = { value: 0 }; let pluginSubscribeRequests = 0; await page.routeWebSocket(/\/ws$/, (ws) => { @@ -428,64 +473,27 @@ export async function routeMainWebSocketWithExpiredPluginSnapshot( const payload = asRecord(frame.payload); if ( frame.type === "request" && - frame.action === "session.subscribe" && - typeof frame.id === "string" && + frame.action === "session.conversation.subscribe" && payload?.consumer_kind === "plugin" ) { pluginSubscribeRequests += 1; - if (armed) requestIDs.add(frame.id); } } server.send(message); }); server.onMessage((message) => { - if (typeof message !== "string") { - ws.send(message); - return; - } - const rewritten: string[] = []; - let didRewrite = false; - for (const part of message.split("\n")) { - const trimmed = part.trim(); - if (!trimmed) { - rewritten.push(part); - continue; - } - let frame: Record | null = null; - try { - frame = asRecord(JSON.parse(trimmed)); - } catch { - // Preserve non-JSON frames. - } - if ( - armed && - frame?.type === "response" && - frame.action === "session.subscribe" && - typeof frame.id === "string" && - requestIDs.delete(frame.id) - ) { - const payload = asRecord(frame.payload); - if (payload?.success === true) { - frame.payload = { ...payload, expires_at: "2000-01-01T00:00:00Z" }; - rewritten.push(JSON.stringify(frame)); - armed = false; - modified += 1; - didRewrite = true; - continue; - } - } - rewritten.push(part); - } - ws.send(didRewrite ? rewritten.join("\n") : message); + const filtered = filterConversationChange(message, contentToDrop, dropped); + if (dropped.value > 0 && filtered !== message) contentToDrop = null; + ws.send(filtered); }); }); return { - expireNextPluginSnapshot: () => { - requestIDs.clear(); - armed = true; + dropChange: (content: string) => { + contentToDrop = content; + dropped.value = 0; }, - modifiedCount: () => modified, + droppedCount: () => dropped.value, pluginSubscribeCount: () => pluginSubscribeRequests, }; } diff --git a/apps/web/e2e/tests/plugins/conversation-recovery.spec.ts b/apps/web/e2e/tests/plugins/conversation-recovery.spec.ts index 24137f6c1f..f2602780cf 100644 --- a/apps/web/e2e/tests/plugins/conversation-recovery.spec.ts +++ b/apps/web/e2e/tests/plugins/conversation-recovery.spec.ts @@ -2,7 +2,7 @@ import { expect, test } from "../../fixtures/test-base"; import type { SeedData } from "../../fixtures/test-base"; import type { ApiClient } from "../../helpers/api-client"; import { - routeMainWebSocketWithExpiredPluginSnapshot, + routeMainWebSocketWithConversationChangeDrop, routeMainWebSocketWithPromptDrop, } from "../../helpers/ws-drop"; import { installFixturePlugin, PLUGIN_ID } from "../../helpers/plugin-fixture"; @@ -116,25 +116,19 @@ test.describe("Conversation recovery", () => { expect(gatewayCloseCount, "core recovery must keep the gateway socket connected").toBe(0); }); - test("rebinds an expired plugin continuation and keeps paging without remount", async ({ + test("repairs a missed plugin change and keeps paging without remount", async ({ testPage, apiClient, seedData, }) => { test.setTimeout(90_000); await installFixturePlugin(testPage); - const expiry = await routeMainWebSocketWithExpiredPluginSnapshot(testPage); - const renewalRequests: string[] = []; - testPage.on("request", (request) => { - if (request.url().includes("/conversation/continuation/renew")) { - renewalRequests.push(request.url()); - } - }); + const wsDrop = await routeMainWebSocketWithConversationChangeDrop(testPage); const { taskId, sessionId } = await createSeededConversation( apiClient, seedData, - "Plugin continuation recovery", + "Plugin source recovery", ); for (const [index, content] of [ "plugin recovery oldest prompt", @@ -154,7 +148,6 @@ test.describe("Conversation recovery", () => { await testPage.goto(`/t/${taskId}`); const session = new SessionPage(testPage); await session.waitForLoad(); - expiry.expireNextPluginSnapshot(); await session.addPanelButton().click(); const panelMenuItem = session.addPanelPluginItem(PLUGIN_ID, PANEL_KEY); await expect(panelMenuItem).toBeVisible(); @@ -163,28 +156,45 @@ test.describe("Conversation recovery", () => { const panel = testPage.getByTestId("fixture-prompt-history-panel"); await expect(panel).toBeVisible({ timeout: 15_000 }); await expect(panel.getByTestId("fixture-prompt-history-row")).toHaveCount(2); - await expect.poll(expiry.modifiedCount).toBe(1); - await panel.getByTestId("fixture-prompt-history-load-more").click(); + const skippedPrompt = "plugin recovery skipped prompt"; + wsDrop.dropChange(skippedPrompt); + const skipped = await apiClient.seedSessionMessage(sessionId, { + type: "message", + content: skippedPrompt, + authorType: "user", + newTurn: true, + turnStartedAt: "2026-09-14T12:03:00Z", + turnCompletedAt: "2026-09-14T12:03:01Z", + }); await expect - .poll(expiry.pluginSubscribeCount, { - timeout: 15_000, - message: "expected a fresh plugin subscription after expiry", + .poll(wsDrop.droppedCount, { + timeout: 10_000, + message: "expected one plugin conversation change to be dropped", }) - .toBeGreaterThan(1); - await expect(panel.getByTestId("fixture-prompt-history-row")).toHaveCount(3); - expect(renewalRequests).toHaveLength(0); + .toBe(1); const laterLive = await apiClient.seedSessionMessage(sessionId, { type: "message", content: "plugin recovery later live prompt", authorType: "user", newTurn: true, - turnStartedAt: "2026-09-14T12:03:00Z", - turnCompletedAt: "2026-09-14T12:03:01Z", + turnStartedAt: "2026-09-14T12:04:00Z", + turnCompletedAt: "2026-09-14T12:04:01Z", + }); + await expect + .poll(wsDrop.pluginSubscribeCount, { + timeout: 15_000, + message: "expected a fresh plugin subscription after the source gap", + }) + .toBeGreaterThan(1); + await expect(panel.locator(`[data-message-id="${skipped.messageId}"]`)).toBeVisible({ + timeout: 15_000, }); await expect(panel.locator(`[data-message-id="${laterLive.messageId}"]`)).toBeVisible({ timeout: 15_000, }); + await panel.getByTestId("fixture-prompt-history-load-more").click(); + await expect(panel.getByTestId("fixture-prompt-history-row")).toHaveCount(4); }); }); diff --git a/apps/web/lib/plugins/conversation-host.tsx b/apps/web/lib/plugins/conversation-host.tsx index 0dbbe8dcbe..53e3dfee0a 100644 --- a/apps/web/lib/plugins/conversation-host.tsx +++ b/apps/web/lib/plugins/conversation-host.tsx @@ -24,6 +24,7 @@ import { parseConversationResponse, pluginConversationUrl, type ConversationScope, + type OrderedReady, } from "./conversation-scope"; export { PluginConversationScopeProvider } from "./conversation-scope"; @@ -31,9 +32,17 @@ type MessagePage = { messages: PluginConversationMessage[]; hasMore: boolean; cursor: string | null; + epoch?: string; + revision?: string; }; -type TurnsPage = { turns: PluginConversationTurn[] }; +type TurnsPage = { + turns: PluginConversationTurn[]; + hasMore?: boolean; + cursor?: string | null; + epoch?: string; + revision?: string; +}; const EMPTY_MESSAGES: PluginSessionMessagesState = { messages: [], @@ -322,7 +331,7 @@ async function fetchMessagePage({ sort: "asc" | "desc"; limit: number; cursor: string | null; - binding: { bindingToken: string; snapshotToken: string }; + binding: Pick; }): Promise { const params = new URLSearchParams({ sort, limit: String(limit) }); if (taskId !== null) params.set("task_id", taskId); @@ -330,24 +339,90 @@ async function fetchMessagePage({ params.append("author_type", author); } if (cursor) params.set("cursor", cursor); - const response = await fetch( - pluginConversationUrl( - scope.pluginId, - `/conversation/task-sessions/${encodeURIComponent(sessionId)}/messages?${params}`, - ), - { - credentials: "include", - cache: "no-store", - headers: { - "X-Kandev-Plugin-Binding": binding.bindingToken, - "X-Kandev-Snapshot-Token": binding.snapshotToken, - }, - signal: scope.signal, - }, - ); + if (binding.source && binding.revision !== undefined) { + params.set("expected_revision", binding.revision); + } + const path = binding.source + ? `/conversation/v2/task-sessions/${encodeURIComponent(sessionId)}/messages?${params}` + : `/conversation/task-sessions/${encodeURIComponent(sessionId)}/messages?${params}`; + const headers: Record = { + "X-Kandev-Plugin-Binding": binding.bindingToken, + }; + if (binding.snapshotToken) headers["X-Kandev-Snapshot-Token"] = binding.snapshotToken; + const response = await fetch(pluginConversationUrl(scope.pluginId, path), { + credentials: "include", + cache: "no-store", + headers, + signal: scope.signal, + }); return parseConversationResponse(response); } +async function fetchTurnPage({ + scope, + sessionId, + taskId, + binding, + cursor, +}: { + cursor?: string; + scope: ConversationScope; + sessionId: string; + taskId: string | null; + binding: Pick; +}): Promise { + const params = new URLSearchParams({ limit: "100" }); + if (cursor) params.set("cursor", cursor); + if (taskId !== null) params.set("task_id", taskId); + if (binding.source && binding.revision !== undefined) { + params.set("expected_revision", binding.revision); + } + const queryString = params.size ? `?${params}` : ""; + const path = binding.source + ? `/conversation/v2/task-sessions/${encodeURIComponent(sessionId)}/turns${queryString}` + : `/conversation/task-sessions/${encodeURIComponent(sessionId)}/turns${queryString}`; + const headers: Record = { + "X-Kandev-Plugin-Binding": binding.bindingToken, + }; + if (binding.snapshotToken) headers["X-Kandev-Snapshot-Token"] = binding.snapshotToken; + const response = await fetch(pluginConversationUrl(scope.pluginId, path), { + credentials: "include", + cache: "no-store", + headers, + signal: scope.signal, + }); + return parseConversationResponse(response); +} + +async function fetchAllTurns(args: Parameters[0]): Promise { + const first = await fetchTurnPage(args); + const turns = [...first.turns]; + const cursors = new Set(); + let page = first; + while (page.hasMore) { + if (!page.cursor || cursors.has(page.cursor)) { + // i18n-exempt: internal protocol diagnostic rendered through structured API error handling. + throw new Error("Invalid conversation turn cursor"); + } + cursors.add(page.cursor); + const binding = await args.scope.ready(); + page = await fetchTurnPage({ + ...args, + cursor: page.cursor, + binding: { ...binding, revision: first.revision }, + }); + if (page.epoch !== first.epoch || page.revision !== first.revision) { + // i18n-exempt: internal protocol diagnostic rendered through structured API error handling. + throw Object.assign(new Error("Conversation changed during pagination"), { + code: "reconciliation_required", + retryable: true, + }); + } + turns.push(...page.turns); + } + return { ...first, turns, hasMore: false, cursor: null }; +} + function mergeMessagePage( current: readonly PluginConversationMessage[], page: readonly PluginConversationMessage[], @@ -446,7 +521,8 @@ function useMessagePageLoader({ error: null, hasMore: page.hasMore, })); - scope.commitSnapshot("messages", snapshotKey); + scope.setSourceSnapshot?.(page.epoch, page.revision); + scope.commitSnapshot("messages", snapshotKey, page.epoch, page.revision); cursorRef.current = page.cursor; return additionCount; } catch (cause) { @@ -797,25 +873,12 @@ function useSessionTurns( ); try { const binding = await scope.ready(); - const params = new URLSearchParams(); - if (resolved.taskId !== null) params.set("task_id", resolved.taskId); - const queryString = params.size ? `?${params}` : ""; - const response = await fetch( - pluginConversationUrl( - scope.pluginId, - `/conversation/task-sessions/${encodeURIComponent(sessionId)}/turns${queryString}`, - ), - { - credentials: "include", - cache: "no-store", - headers: { - "X-Kandev-Plugin-Binding": binding.bindingToken, - "X-Kandev-Snapshot-Token": binding.snapshotToken, - }, - signal: scope.signal, - }, - ); - const page = await parseConversationResponse(response); + const page = await fetchAllTurns({ + scope, + sessionId, + taskId: resolved.taskId, + binding, + }); if (turnsRequestRef.current !== requestId || scope.signal.aborted || scope.isTerminal()) { return; } @@ -826,7 +889,8 @@ function useSessionTurns( hydrated: true, error: null, })); - scope.commitSnapshot("turns", snapshotKey); + scope.setSourceSnapshot?.(page.epoch, page.revision); + scope.commitSnapshot("turns", snapshotKey, page.epoch, page.revision); } catch (cause) { if (turnsRequestRef.current !== requestId || scope.signal.aborted || scope.isTerminal()) { return; diff --git a/apps/web/lib/plugins/conversation-reconciliation.test.ts b/apps/web/lib/plugins/conversation-reconciliation.test.ts new file mode 100644 index 0000000000..4437e3b09c --- /dev/null +++ b/apps/web/lib/plugins/conversation-reconciliation.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; +import { + reconcileConversationChange, + type ConversationChange, + type ConversationReconciliationState, +} from "./conversation-reconciliation"; + +const state: ConversationReconciliationState = { appliedRevision: "10", epoch: "epoch-1" }; + +function change(overrides: Partial = {}): ConversationChange { + return { + protocol_version: 2, + scope_id: "scope-1", + session_id: "session-1", + epoch: "epoch-1", + base_revision: "10", + revision: "11", + operations: [], + ...overrides, + }; +} + +describe("conversation revision reconciliation", () => { + it("applies a contiguous coverage batch and advances the revision", () => { + const result = reconcileConversationChange(state, change({ revision: "11" })); + + expect(result.kind).toBe("applied"); + expect(result.state).toEqual({ appliedRevision: "11", epoch: "epoch-1" }); + }); + + it("requests repair when a revision interval starts after the applied revision", () => { + const result = reconcileConversationChange( + state, + change({ base_revision: "11", revision: "12" }), + ); + + expect(result.kind).toBe("recover"); + expect(result.state).toBe(state); + }); + + it("treats an already covered batch as a duplicate", () => { + const result = reconcileConversationChange( + { appliedRevision: "12", epoch: "epoch-1" }, + change({ base_revision: "10", revision: "12" }), + ); + + expect(result.kind).toBe("duplicate"); + expect(result.state.appliedRevision).toBe("12"); + }); + + it("requests repair for a changed epoch, reset, malformed revision, or oversized batch", () => { + expect(reconcileConversationChange(state, change({ epoch: "epoch-2" })).kind).toBe("recover"); + expect(reconcileConversationChange(state, change({ reset: true })).kind).toBe("recover"); + expect(reconcileConversationChange(state, change({ revision: "not-a-revision" })).kind).toBe( + "recover", + ); + expect( + reconcileConversationChange( + state, + change({ + operations: Array.from({ length: 257 }, (_, index) => ({ + kind: "remove", + entity: "message", + id: `message-${index}`, + })), + }), + ).kind, + ).toBe("recover"); + }); +}); diff --git a/apps/web/lib/plugins/conversation-reconciliation.ts b/apps/web/lib/plugins/conversation-reconciliation.ts new file mode 100644 index 0000000000..aad56d9e79 --- /dev/null +++ b/apps/web/lib/plugins/conversation-reconciliation.ts @@ -0,0 +1,102 @@ +import type { PluginConversationMessage, PluginConversationTurn } from "./types"; + +export type ConversationChangeOperation = { + kind: "upsert" | "remove"; + entity: "message" | "turn"; + id: string; + message?: PluginConversationMessage; + turn?: PluginConversationTurn; +}; + +export type ConversationChange = { + protocol_version: number; + scope_id: string; + session_id: string; + epoch: string; + base_revision: string; + revision: string; + check?: boolean; + terminal?: boolean; + reset?: boolean; + operations: ConversationChangeOperation[]; +}; + +export type ConversationReconciliationState = { + appliedRevision: string; + epoch: string; +}; + +export type ConversationReconciliationResult = + | { + kind: "applied"; + state: ConversationReconciliationState; + operations: ConversationChangeOperation[]; + } + | { kind: "duplicate" | "recover"; state: ConversationReconciliationState }; + +const MAX_CONVERSATION_OPERATIONS = 256; + +function parseRevision(value: unknown): bigint | null { + if (typeof value !== "string" || !/^(0|[1-9][0-9]*)$/.test(value)) return null; + try { + return BigInt(value); + } catch { + return null; + } +} + +function validOperation(value: unknown): value is ConversationChangeOperation { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const operation = value as Partial; + if ( + (operation.kind !== "upsert" && operation.kind !== "remove") || + (operation.entity !== "message" && operation.entity !== "turn") || + typeof operation.id !== "string" || + operation.id === "" + ) { + return false; + } + if (operation.kind === "remove") return true; + return operation.entity === "message" ? Boolean(operation.message) : Boolean(operation.turn); +} + +function validChangeMetadata( + change: Partial, + state: ConversationReconciliationState, +): change is ConversationChange { + return ( + change.protocol_version === 2 && + typeof change.epoch === "string" && + change.epoch === state.epoch && + change.reset !== true && + Array.isArray(change.operations) && + change.operations.length <= MAX_CONVERSATION_OPERATIONS && + change.operations.every(validOperation) + ); +} + +export function reconcileConversationChange( + state: ConversationReconciliationState, + value: unknown, +): ConversationReconciliationResult { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return { kind: "recover", state }; + } + const change = value as Partial; + const applied = parseRevision(state.appliedRevision); + const base = parseRevision(change.base_revision); + const revision = parseRevision(change.revision); + if (applied === null || base === null || revision === null) { + return { kind: "recover", state }; + } + if (!validChangeMetadata(change, state) || revision <= base) { + return { kind: "recover", state }; + } + if (revision <= applied) return { kind: "duplicate", state }; + if (base !== applied) return { kind: "recover", state }; + return { + kind: "applied", + state: { ...state, appliedRevision: change.revision as string }, + operations: change.operations, + }; +} diff --git a/apps/web/lib/plugins/conversation-scope.tsx b/apps/web/lib/plugins/conversation-scope.tsx index 7e0b2e78ae..d20298bdd3 100644 --- a/apps/web/lib/plugins/conversation-scope.tsx +++ b/apps/web/lib/plugins/conversation-scope.tsx @@ -9,6 +9,7 @@ import { isRawSessionEvent } from "@/lib/ws/ordered-session-events"; import { getWebSocketClient } from "@/lib/ws/connection"; import type { RawSessionEvent } from "@/lib/ws/client"; import type { PluginConversationError } from "./types"; +import { SourceConversationScope } from "./conversation-source-scope"; export type OrderedReady = { bindingToken: string; @@ -19,10 +20,13 @@ export type OrderedReady = { resumeToken: string; consumerId: string; watermark: number; + source?: boolean; + epoch?: string; + revision?: string; }; -type Binding = Pick; -type SnapshotKind = "messages" | "turns"; +export type Binding = Pick; +export type SnapshotKind = "messages" | "turns"; type ConversationEventListener = (event: RawSessionEvent) => boolean; type SnapshotKey = string; type OrderedSubscribeAck = @@ -65,7 +69,16 @@ export type ConversationScope = { ): () => void; subscribeRebind(listener: RebindListener): () => void; isTerminal(): boolean; - commitSnapshot(kind: SnapshotKind, snapshotKey?: SnapshotKey): void; + commitSnapshot( + kind: SnapshotKind, + snapshotKey?: SnapshotKey, + epoch?: string, + revision?: string, + ): void; + setSourceSnapshot?(epoch: string | undefined, revision: string | undefined): void; + acceptChange?(value: unknown): void; + acceptTerminalRemoval?(): void; + readonly source?: boolean; invalidateSnapshot(kind: SnapshotKind, snapshotKey?: SnapshotKey): void; reconnect(): void; accept(event: RawSessionEvent): void; @@ -92,7 +105,10 @@ export async function parseConversationResponse(response: Response): Promise< ); } -async function fetchBinding(pluginId: string, signal: AbortSignal): Promise { +export async function fetchConversationBinding( + pluginId: string, + signal: AbortSignal, +): Promise { const response = await fetch(pluginConversationUrl(pluginId, "/conversation/binding"), { credentials: "include", cache: "no-store", @@ -425,7 +441,7 @@ class OrderedConversationScope implements ConversationScope { } private getBinding() { - const pending = (this.bindingPromise ??= fetchBinding(this.pluginId, this.signal)); + const pending = (this.bindingPromise ??= fetchConversationBinding(this.pluginId, this.signal)); return pending.catch((error: unknown) => { if (this.bindingPromise === pending) this.bindingPromise = null; throw error; @@ -481,7 +497,7 @@ class OrderedConversationScope implements ConversationScope { return Promise.resolve(current); } if (this.bindingRefreshPromise) return this.bindingRefreshPromise; - this.bindingPromise = fetchBinding(this.pluginId, this.signal); + this.bindingPromise = fetchConversationBinding(this.pluginId, this.signal); const pending = this.bindingPromise .then(async (binding) => { if (binding.generation !== current.generation) { @@ -749,14 +765,35 @@ export function PluginConversationScopeProvider({ () => new AbortController(), [generation, pluginId, presentation, sessionId, taskId], ); + const sourceTransport = typeof getWebSocketClient()?.on === "function"; const scope = React.useMemo( - () => new OrderedConversationScope(pluginId, taskId, sessionId, controller), - [controller, pluginId, sessionId, taskId], + () => + sourceTransport + ? new SourceConversationScope(pluginId, taskId, sessionId, controller) + : new OrderedConversationScope(pluginId, taskId, sessionId, controller), + [controller, pluginId, sessionId, sourceTransport, taskId], ); React.useLayoutEffect(() => { const client = getWebSocketClient(); let sawDisconnect = false; - const removeListener = client?.onRawSessionEvent((event) => scope.accept(event)); + const removeListener = scope.source + ? client?.on("session.conversation.changed", (message) => { + scope.acceptChange?.(message.payload); + }) + : client?.onRawSessionEvent((event) => scope.accept(event)); + const removeTerminalListener = scope.source + ? client?.on("session.removed", (message) => { + const payload = message.payload; + if ( + payload && + typeof payload === "object" && + "session_id" in payload && + payload.session_id === sessionId + ) { + scope.acceptTerminalRemoval?.(); + } + }) + : undefined; const removeStatusListener = client?.onConnectionStatus((status) => { if (status !== "connected") { sawDisconnect = true; @@ -769,6 +806,7 @@ export function PluginConversationScopeProvider({ }); return () => { removeListener?.(); + removeTerminalListener?.(); removeStatusListener?.(); scope.close(); controller.abort(); diff --git a/apps/web/lib/plugins/conversation-source-scope.test.tsx b/apps/web/lib/plugins/conversation-source-scope.test.tsx new file mode 100644 index 0000000000..033a93d09a --- /dev/null +++ b/apps/web/lib/plugins/conversation-source-scope.test.tsx @@ -0,0 +1,593 @@ +/* eslint-disable max-lines-per-function -- the fixture owns one complete source-scope lifecycle. */ +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { SourceConversationScope } from "./conversation-source-scope"; +import type { ConversationChangedPayload } from "@/lib/types/session-events"; +import { PluginConversationScopeProvider, pluginConversationApi } from "./conversation-host"; + +const transport = vi.hoisted(() => ({ + request: vi.fn(), + on: vi.fn(), + onConnectionStatus: vi.fn(() => () => undefined), + changeListener: null as ((message: { payload: ConversationChangedPayload }) => void) | null, + removalListener: null as ((message: { payload: { session_id: string } }) => void) | null, +})); +const SESSION_ID = "session-1"; +const CREATED_AT = "2026-09-16T12:00:00Z"; +const PLUGIN_ID = "plugin-history"; +const CONVERSATION_SUBSCRIBE_ACTION = "session.conversation.subscribe"; + +vi.mock("@/lib/config", () => ({ getBackendConfig: () => ({ apiBaseUrl: "http://host" }) })); +vi.mock("@/lib/ws/connection", () => ({ + getWebSocketClient: () => transport, +})); + +function response(body: unknown): Response { + return new Response(JSON.stringify(body), { status: 200 }); +} + +function Harness() { + const state = pluginConversationApi.useSessionMessages({ sessionId: SESSION_ID, sort: "asc" }); + return ( + <> + {state.messages.map((item) => item.content).join("|")} + {String(state.removed)} + + ); +} + +describe("source conversation scope", () => { + beforeEach(() => { + let pageRevision = "0"; + transport.changeListener = null; + transport.removalListener = null; + transport.on.mockReset(); + transport.on.mockImplementation((action: string, listener: typeof transport.changeListener) => { + if (action === "session.conversation.changed") transport.changeListener = listener; + if (action === "session.removed") { + transport.removalListener = listener as typeof transport.removalListener; + } + return () => { + if (action === "session.conversation.changed") transport.changeListener = null; + if (action === "session.removed") transport.removalListener = null; + }; + }); + transport.request.mockReset(); + transport.request.mockImplementation((action: string, payload: { scope_id?: string }) => { + if (action === CONVERSATION_SUBSCRIBE_ACTION) { + return Promise.resolve({ + success: true, + protocol_version: 2, + scope_id: payload.scope_id, + session_id: SESSION_ID, + epoch: "epoch-1", + revision: pageRevision, + }); + } + return Promise.resolve({ success: true }); + }); + vi.stubGlobal( + "fetch", + vi.fn((input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/conversation/binding")) { + return Promise.resolve( + response({ + bindingToken: "binding-1", + generation: 7, + expiresAt: "2099-01-01T00:00:00Z", + }), + ); + } + return Promise.resolve( + response({ + messages: [], + hasMore: false, + cursor: null, + epoch: "epoch-1", + revision: pageRevision, + }), + ); + }), + ); + Object.defineProperty(transport, "setPageRevision", { + configurable: true, + value: (revision: string) => { + pageRevision = revision; + }, + }); + }); + + afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + }); + + it("uses source reads and applies a matching ID update without rereading", async () => { + const fetchMock = globalThis.fetch as unknown as ReturnType; + render( + + + , + ); + await waitFor(() => expect(screen.getByTestId("messages").textContent).toBe("")); + expect(transport.request).toHaveBeenCalledWith( + CONVERSATION_SUBSCRIBE_ACTION, + expect.objectContaining({ consumer_kind: "plugin", session_id: SESSION_ID }), + ); + expect( + transport.request.mock.calls.find( + ([action]) => action === CONVERSATION_SUBSCRIBE_ACTION, + )?.[1], + ).not.toHaveProperty("task_id"); + expect(fetchMock).toHaveBeenCalledWith( + expect.stringContaining(`/conversation/v2/task-sessions/${SESSION_ID}/messages`), + expect.anything(), + ); + + const subscribePayload = transport.request.mock.calls.find( + ([action]) => action === CONVERSATION_SUBSCRIBE_ACTION, + )?.[1] as { scope_id: string }; + act(() => { + transport.changeListener?.({ + payload: { + protocol_version: 2, + scope_id: subscribePayload.scope_id, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: "0", + revision: "1", + operations: [ + { + kind: "upsert", + entity: "message", + id: "message-1", + message: { + id: "message-1", + taskId: "task-1", + sessionId: SESSION_ID, + authorType: "user", + type: "message", + content: "from source", + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + }, + }, + ], + }, + }); + }); + await waitFor(() => expect(screen.getByTestId("messages").textContent).toBe("from source")); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("delivers live updates from every task to a complete-session query", async () => { + function CompleteSessionHarness() { + const state = pluginConversationApi.useSessionMessages({ + sessionId: SESSION_ID, + taskId: null, + sort: "asc", + }); + return ( + + {state.messages.map((item) => item.content).join("|")} + + ); + } + + render( + + + , + ); + await waitFor(() => expect(transport.changeListener).not.toBeNull()); + const scopeID = transport.request.mock.calls.find( + ([action]) => action === CONVERSATION_SUBSCRIBE_ACTION, + )?.[1].scope_id as string; + act(() => { + transport.changeListener?.({ + payload: { + protocol_version: 2, + scope_id: scopeID, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: "0", + revision: "1", + operations: [ + { + kind: "upsert", + entity: "message", + id: "message-other-task", + message: { + id: "message-other-task", + taskId: "task-2", + sessionId: SESSION_ID, + authorType: "agent", + type: "message", + content: "other task", + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + }, + }, + ], + }, + }); + }); + await waitFor(() => + expect(screen.getByTestId("complete-session-messages").textContent).toBe("other task"), + ); + }); + + it("repairs a revision gap through a fresh source page", async () => { + const setPageRevision = ( + transport as typeof transport & { setPageRevision: (value: string) => void } + ).setPageRevision; + render( + + + , + ); + await waitFor(() => expect(transport.changeListener).not.toBeNull()); + const scopeID = transport.request.mock.calls.find( + ([action]) => action === CONVERSATION_SUBSCRIBE_ACTION, + )?.[1].scope_id as string; + setPageRevision("3"); + act(() => { + transport.changeListener?.({ + payload: { + protocol_version: 2, + scope_id: scopeID, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: "2", + revision: "3", + operations: [], + }, + }); + }); + await waitFor(() => + expect((globalThis.fetch as unknown as ReturnType).mock.calls.length).toBe(3), + ); + expect( + transport.request.mock.calls.filter(([action]) => action === CONVERSATION_SUBSCRIBE_ACTION), + ).toHaveLength(2); + }); + + it("projects terminal session removal from the core notification", async () => { + render( + + + , + ); + await waitFor(() => expect(screen.getByTestId("removed").textContent).toBe("false")); + + act(() => { + transport.removalListener?.({ payload: { session_id: SESSION_ID } }); + }); + + await waitFor(() => expect(screen.getByTestId("removed").textContent).toBe("true")); + }); + it("renews an expired binding for reads and reconnects", async () => { + const now = Date.now(); + const clock = vi.spyOn(Date, "now").mockReturnValue(now); + const fetchMock = vi.mocked(fetch); + fetchMock.mockImplementation(async () => + response({ + bindingToken: Date.now() === now ? "old" : "renewed", + generation: 7, + expiresAt: new Date(Date.now() + 600_000).toISOString(), + }), + ); + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + try { + expect((await scope.ready()).bindingToken).toBe("old"); + clock.mockReturnValue(now + 601_000); + const [first, second] = await Promise.all([scope.ready(), scope.renewContinuation("cursor")]); + expect(first.bindingToken).toBe("renewed"); + expect(second.binding.bindingToken).toBe("renewed"); + scope.reconnect(); + await waitFor(() => + expect(transport.request).toHaveBeenLastCalledWith( + CONVERSATION_SUBSCRIBE_ACTION, + expect.objectContaining({ binding_token: "renewed" }), + ), + ); + expect(fetchMock).toHaveBeenCalledTimes(2); + } finally { + scope.close(); + clock.mockRestore(); + } + }); + + it("loads every turn page before publishing hydrated history", async () => { + function TurnsHarness() { + const state = pluginConversationApi.useSessionTurns(SESSION_ID); + return {state.turns.map((turn) => turn.id).join("|")}; + } + const original = vi.mocked(fetch).getMockImplementation()!; + vi.mocked(fetch).mockImplementation(async (input, init) => { + const url = new URL(String(input)); + if (!url.pathname.endsWith("/turns")) return original(input, init); + const second = url.searchParams.has("cursor"); + const turns = Array.from({ length: second ? 1 : 100 }, (_, index) => ({ + id: `turn-${second ? 100 : index}`, + sessionId: SESSION_ID, + taskId: "task-1", + startedAt: CREATED_AT, + completedAt: "2026-09-16T12:01:00Z", + })); + return response({ + turns, + hasMore: !second, + cursor: second ? null : "next", + epoch: "epoch-1", + revision: "0", + }); + }); + render( + + + , + ); + await waitFor(() => + expect(screen.getByTestId("turns").textContent?.split("|")).toHaveLength(101), + ); + expect(screen.getByTestId("turns").textContent).toContain("turn-100"); + }); + + it("checks observed revisions without certifying missed updates", async () => { + vi.useFakeTimers(); + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + try { + await scope.ready(); + const scopeID = transport.request.mock.calls[0][1].scope_id; + const recover = vi.fn(); + scope.subscribeRebind(recover); + const check = (revision: string) => + scope.acceptChange({ + protocol_version: 2, + scope_id: scopeID, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: revision, + revision, + check: true, + operations: [], + }); + check("0"); + await vi.advanceTimersByTimeAsync(1000); + expect(recover).not.toHaveBeenCalled(); + check("1"); + await vi.advanceTimersByTimeAsync(999); + expect(recover).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + expect(recover).toHaveBeenCalledOnce(); + } finally { + scope.close(); + vi.useRealTimers(); + } + }); + + it("retries a failed revision recovery instead of wedging the scope", async () => { + vi.useFakeTimers(); + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + try { + await scope.ready(); + const scopeID = transport.request.mock.calls[0][1].scope_id; + const recover = vi.fn(); + scope.subscribeRebind(recover); + transport.request.mockRejectedValueOnce(new Error("temporary outage")); + scope.acceptChange({ + protocol_version: 2, + scope_id: scopeID, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: "1", + revision: "2", + operations: [], + }); + await Promise.resolve(); + await Promise.resolve(); + expect(recover).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1000); + await vi.advanceTimersByTimeAsync(0); + expect(recover).toHaveBeenCalledOnce(); + } finally { + scope.close(); + vi.useRealTimers(); + } + }); + + it("collapses an oversized pending buffer into one recovery", async () => { + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + const applied: string[] = []; + try { + await scope.ready(); + const scopeID = transport.request.mock.calls[0][1].scope_id; + scope.subscribe( + (event) => { + if (event.event_type === "message.added") { + const payload = event.payload as { content?: unknown }; + applied.push(String(payload.content)); + } + return true; + }, + "messages", + "all", + ); + const recover = vi.fn(() => { + scope.setSourceSnapshot("epoch-1", "257"); + scope.commitSnapshot("messages", "all"); + }); + scope.subscribeRebind(recover); + for (let revision = 1; revision <= 257; revision += 1) { + scope.acceptChange({ + protocol_version: 2, + scope_id: scopeID, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: String(revision - 1), + revision: String(revision), + operations: [ + { + kind: "upsert", + entity: "message", + id: `message-${revision}`, + message: { content: `message-${revision}`, taskId: "task-1" }, + }, + ], + }); + } + await waitFor(() => expect(recover).toHaveBeenCalledOnce()); + expect(applied).toEqual([]); + scope.acceptChange({ + protocol_version: 2, + scope_id: scopeID, + session_id: SESSION_ID, + epoch: "epoch-1", + base_revision: "257", + revision: "258", + operations: [ + { + kind: "upsert", + entity: "message", + id: "message-current", + message: { content: "current", taskId: "task-1" }, + }, + ], + }); + expect(applied).toEqual(["current"]); + } finally { + scope.close(); + } + }); + + it("terminates the plugin scope on server authorization removal", async () => { + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + await scope.ready(); + scope.acceptChange({ + scope_id: transport.request.mock.calls[0][1].scope_id, + session_id: SESSION_ID, + terminal: true, + }); + expect(scope.isTerminal()).toBe(true); + scope.close(); + }); + + it("retries binding acquisition after a failed request", async () => { + vi.mocked(fetch).mockRejectedValueOnce(new Error("offline")); + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + try { + await expect(scope.ready()).rejects.toThrow("offline"); + await expect(scope.ready()).resolves.toMatchObject({ bindingToken: "binding-1" }); + } finally { + scope.close(); + } + }); + + it.each(["delivered", "closed"])( + "does not recover a revision check after %s", + async (outcome) => { + vi.useFakeTimers(); + const scope = new SourceConversationScope( + PLUGIN_ID, + "task-1", + SESSION_ID, + new AbortController(), + ); + try { + await scope.ready(); + const recover = vi.fn(); + scope.subscribeRebind(recover); + const frame = { + protocol_version: 2, + scope_id: transport.request.mock.calls[0][1].scope_id, + session_id: SESSION_ID, + epoch: "epoch-1", + revision: "1", + operations: [], + }; + scope.acceptChange({ ...frame, base_revision: "1", check: true }); + if (outcome === "closed") scope.close(); + else scope.acceptChange({ ...frame, base_revision: "0" }); + await vi.advanceTimersByTimeAsync(1000); + expect(recover).not.toHaveBeenCalled(); + } finally { + scope.close(); + vi.useRealTimers(); + } + }, + ); + it("does not publish partial turns when a continuation fails", async () => { + function FailedTurnsHarness() { + const state = pluginConversationApi.useSessionTurns(SESSION_ID); + return ( + {`${state.turns.length}:${state.hydrated}:${state.error?.code ?? "none"}`} + ); + } + const original = vi.mocked(fetch).getMockImplementation()!; + vi.mocked(fetch).mockImplementation(async (input, init) => { + const url = new URL(String(input)); + if (!url.pathname.endsWith("/turns")) return original(input, init); + if (url.searchParams.has("cursor")) + return new Response( + JSON.stringify({ + error: { code: "upstream_failure", retryable: true }, + }), + { status: 500 }, + ); + return response({ + turns: [ + { + id: "turn-1", + sessionId: SESSION_ID, + taskId: "task-1", + startedAt: CREATED_AT, + }, + ], + hasMore: true, + cursor: "next", + epoch: "epoch-1", + revision: "0", + }); + }); + render( + + + , + ); + await waitFor(() => + expect(screen.getByTestId("turn-result").textContent).toBe("0:false:upstream_failure"), + ); + }); +}); diff --git a/apps/web/lib/plugins/conversation-source-scope.ts b/apps/web/lib/plugins/conversation-source-scope.ts new file mode 100644 index 0000000000..549628f5f9 --- /dev/null +++ b/apps/web/lib/plugins/conversation-source-scope.ts @@ -0,0 +1,671 @@ +/* eslint-disable max-lines -- this class owns one ordered source lifecycle */ +import { generateUUID } from "@/lib/uuid"; +import { getWebSocketClient } from "@/lib/ws/connection"; +import type { RawSessionEvent } from "@/lib/ws/client"; +import { + fetchConversationBinding, + pluginConversationUrl, + type Binding, + type ConversationScope, + type OrderedReady, + type SnapshotKind, +} from "./conversation-scope"; +import { + reconcileConversationChange, + type ConversationChange, + type ConversationChangeOperation, + type ConversationReconciliationState, +} from "./conversation-reconciliation"; + +type Listener = (event: RawSessionEvent) => boolean; +type RebindListener = () => void | Promise; + +type SourceSubscribeResponse = { + success: true; + protocol_version: 2; + scope_id: string; + session_id: string; + epoch: string; + revision: string; +}; + +type SourceFailure = { + success: false; + error?: { code?: string; message?: string; retryable?: boolean }; +}; + +const MAX_PENDING_CONVERSATION_OPERATIONS = 256; +const MAX_PENDING_CONVERSATION_BYTES = 1 << 20; +const MAX_PENDING_CONVERSATION_AGE_MS = 1000; +const RECOVERY_RETRY_DELAY_MS = 1000; + +function sourceError(response: SourceFailure): Error { + // i18n-exempt: transport/API diagnostic. The caller renders the structured + // error code through translated UI copy. + const error = new Error(response.error?.message ?? "Conversation subscription failed"); + Object.assign(error, { + code: response.error?.code ?? "upstream_failure", + retryable: response.error?.retryable ?? true, + }); + return error; +} + +function validRevision(value: unknown): value is string { + return typeof value === "string" && /^(0|[1-9][0-9]*)$/.test(value); +} + +function compareRevision(left: string, right: string): number { + const leftValue = BigInt(left); + const rightValue = BigInt(right); + if (leftValue < rightValue) return -1; + if (leftValue > rightValue) return 1; + return 0; +} + +function snapshotKindForOperation(operation: ConversationChangeOperation): SnapshotKind { + return operation.entity === "message" ? "messages" : "turns"; +} + +function operationTaskID(operation: ConversationChangeOperation, fallback: string): string { + const value = operation.entity === "message" ? operation.message?.taskId : operation.turn?.taskId; + return typeof value === "string" && value !== "" ? value : fallback; +} + +function operationEventType(operation: ConversationChangeOperation): string { + if (operation.entity === "message") { + return operation.kind === "remove" ? "message.deleted" : "message.added"; + } + return operation.kind === "remove" ? "session.turn.removed" : "session.turn.started"; +} + +function pendingChangeMetrics(value: unknown): { operations: number; bytes: number } | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const operations = (value as Partial).operations; + if (!Array.isArray(operations) || operations.length === 0) return null; + let encoded: string | undefined; + try { + encoded = JSON.stringify(value); + } catch { + return null; + } + if (encoded === undefined) return null; + return { operations: operations.length, bytes: new TextEncoder().encode(encoded).byteLength }; +} + +function sourceOperationEvent( + operation: ConversationChangeOperation, + sessionID: string, + taskID: string, + revision: string, + index: number, +): RawSessionEvent { + const eventType = operationEventType(operation); + const operationTaskId = operationTaskID(operation, taskID); + const payload: Record = { + type: eventType, + session_id: sessionID, + task_id: operationTaskId, + }; + if (operation.entity === "message") { + payload.message_id = operation.id; + if (operation.kind === "upsert" && operation.message) { + const message = operation.message; + payload.author_type = message.authorType; + payload.content = message.content; + payload.message_type = message.type; + payload.created_at = message.createdAt; + payload.updated_at = message.updatedAt; + payload.turn_id = message.turnId; + payload.prompt_index = message.promptIndex; + payload.sender_task_id = message.senderTaskId; + } + } else { + payload.id = operation.id; + if (operation.kind === "upsert" && operation.turn) { + const turn = operation.turn; + payload.started_at = turn.startedAt; + payload.completed_at = turn.completedAt; + payload.updated_at = turn.updatedAt; + } + } + const numericRevision = Number(revision); + return { + type: "session.event", + protocol_version: 1, + event_type: eventType, + session_id: sessionID, + task_id: operationTaskId, + sequence: Number.isSafeInteger(numericRevision) ? numericRevision : index + 1, + event_id: `conversation:${sessionID}:${revision}:${operation.entity}:${operation.id}`, + payload, + }; +} + +export class SourceConversationScope implements ConversationScope { + readonly source = true; + readonly signal: AbortSignal; + private readonly scopeID = `plugin-conversation:${generateUUID()}`; + private readonly listeners = new Map(); + private readonly rebindListeners = new Set(); + private readonly committedSnapshots = new Set(); + private readonly pendingChanges: unknown[] = []; + private bindingPromise: Promise | null = null; + private readyPromise: Promise | null = null; + private recoveryPromise: Promise | null = null; + private recovering = false; + private recoveryNeeded = false; + private recoveryRetryTimer: ReturnType | undefined; + private pendingOperationCount = 0; + private pendingPayloadBytes = 0; + private pendingSince: number | undefined; + private pendingBufferTimer: ReturnType | undefined; + private pendingRecoveryMarker = false; + private terminal = false; + private revisionCheckTimer: ReturnType | undefined; + private closed = false; + private state: ConversationReconciliationState = { appliedRevision: "0", epoch: "" }; + + constructor( + readonly pluginId: string, + readonly taskId: string, + readonly sessionId: string | null, + private readonly controller: AbortController, + ) { + this.signal = controller.signal; + } + + ready(): Promise { + const pending = (this.readyPromise ??= this.initializeReady()); + return pending + .then(async (current) => { + const binding = await this.getBinding(); + if (binding.bindingToken === current.bindingToken) return current; + if (binding.generation !== current.generation) return this.subscribeSource(binding); + const next = { ...current, ...binding }; + this.readyPromise = Promise.resolve(next); + return next; + }) + .catch((error: unknown) => { + if (this.readyPromise === pending) this.readyPromise = null; + throw error; + }); + } + + subscribe(listener: Listener, kind: SnapshotKind, snapshotKey: string): () => void { + this.listeners.set(listener, { kind, key: snapshotKey }); + return () => { + this.listeners.delete(listener); + this.drainPending(); + }; + } + + async renewContinuation(cursor: string): Promise<{ + cursor: string; + binding: OrderedReady; + }> { + return { cursor, binding: await this.ready() }; + } + + subscribeRebind(listener: RebindListener): () => void { + this.rebindListeners.add(listener); + return () => this.rebindListeners.delete(listener); + } + + isTerminal(): boolean { + return this.terminal; + } + + invalidateSnapshot(kind: SnapshotKind, snapshotKey = "") { + this.committedSnapshots.delete(`${kind}:${snapshotKey}`); + } + + commitSnapshot(kind: SnapshotKind, snapshotKey = "") { + if (this.closed) return; + this.committedSnapshots.add(`${kind}:${snapshotKey}`); + this.drainPending(); + } + + setSourceSnapshot(epoch: string | undefined, revision: string | undefined) { + if (this.closed || !validRevision(revision)) return; + const nextEpoch = epoch ?? this.state.epoch; + if (nextEpoch === "") return; + if (nextEpoch !== this.state.epoch) { + this.clearPendingBuffer(); + this.recovering = false; + this.state = { epoch: nextEpoch, appliedRevision: revision }; + } else if (compareRevision(revision, this.state.appliedRevision) > 0) { + this.state = { ...this.state, appliedRevision: revision }; + this.recovering = false; + } + this.pendingRecoveryMarker = false; + this.recoveryNeeded = false; + this.clearRecoveryRetry(); + this.discardCoveredChanges(this.state.appliedRevision); + this.updateReadyState(); + this.drainPending(); + } + + acceptChange(value: unknown) { + if (this.closed || this.terminal || !this.sessionId) return; + const change = value as Partial; + if (change.session_id !== this.sessionId || change.scope_id !== this.scopeID) return; + if (change.terminal) { + this.acceptTerminalRemoval(); + return; + } + if (change.check) { + this.checkRevision(change); + return; + } + if (this.recovering || this.shouldBuffer(change)) { + this.bufferPendingChange(value); + return; + } + this.applyChange(value); + } + + accept(_event: RawSessionEvent): void { + // Source scopes consume v2 changes and terminal removal notifications separately. + } + + acceptTerminalRemoval(): void { + if (this.closed || this.terminal || !this.sessionId) return; + this.terminal = true; + clearTimeout(this.revisionCheckTimer); + this.clearPendingBuffer(); + this.clearRecoveryRetry(); + const event: RawSessionEvent = { + type: "session.event", + protocol_version: 1, + event_type: "session.removed", + session_id: this.sessionId, + task_id: null, + sequence: 0, + event_id: `conversation:removed:${this.sessionId}`, + payload: { + type: "session.removed", + session_id: this.sessionId, + }, + }; + for (const listener of this.listeners.keys()) { + try { + listener(event); + } catch { + // A terminal notification must not prevent other consumers from closing. + } + } + } + + reconnect() { + if (this.closed || this.terminal || !this.sessionId || !this.readyPromise) return; + this.beginRecovery(); + } + + close() { + this.closed = true; + clearTimeout(this.revisionCheckTimer); + this.clearPendingBuffer(); + this.clearRecoveryRetry(); + this.listeners.clear(); + this.rebindListeners.clear(); + const ready = this.readyPromise; + if (!ready || !this.sessionId) return; + void ready + .then((current) => + getWebSocketClient()?.request("session.conversation.unsubscribe", { + scope_id: this.scopeID, + session_id: this.sessionId, + consumer_kind: "plugin", + plugin_id: this.pluginId, + generation: current.generation, + }), + ) + .catch(() => undefined); + } + + private checkRevision(change: Partial) { + if ( + !validRevision(change.revision) || + change.epoch !== this.state.epoch || + compareRevision(change.revision, this.state.appliedRevision) < 0 + ) { + this.beginRecovery(); + return; + } + if (change.revision === this.state.appliedRevision || this.revisionCheckTimer !== undefined) + return; + const observed = change.revision; + this.revisionCheckTimer = setTimeout(() => { + this.revisionCheckTimer = undefined; + if ( + !this.closed && + !this.terminal && + compareRevision(observed, this.state.appliedRevision) > 0 + ) { + this.beginRecovery(); + } + }, 1000); + } + + private async initializeReady(): Promise { + const binding = await this.getBinding(); + if (!this.sessionId) { + return { + ...binding, + source: true, + snapshotToken: "", + snapshotExpiresAt: binding.expiresAt, + resumeToken: "", + consumerId: this.scopeID, + watermark: 0, + }; + } + return this.subscribeSource(binding); + } + + private async getBinding(): Promise { + let pending = (this.bindingPromise ??= fetchConversationBinding(this.pluginId, this.signal)); + try { + const current = await pending; + if (new Date(current.expiresAt).getTime() - Date.now() > 120_000) return current; + if (this.bindingPromise === pending) { + this.bindingPromise = fetchConversationBinding(this.pluginId, this.signal); + } + pending = this.bindingPromise!; + return await pending; + } catch (error) { + if (this.bindingPromise === pending) this.bindingPromise = null; + throw error; + } + } + + private async subscribeSource(binding: Binding, notifyOnChange = true): Promise { + const client = getWebSocketClient(); + // i18n-exempt: transport/API diagnostic. The caller renders the + // structured error code through translated UI copy. + if (!client) throw new Error("WebSocket unavailable"); + const response = await client.request( + "session.conversation.subscribe", + { + scope_id: this.scopeID, + session_id: this.sessionId, + consumer_kind: "plugin", + plugin_id: this.pluginId, + generation: binding.generation, + binding_token: binding.bindingToken, + }, + ); + if (!response.success) throw sourceError(response); + if ( + response.protocol_version !== 2 || + response.session_id !== this.sessionId || + response.scope_id !== this.scopeID || + !response.epoch || + !validRevision(response.revision) + ) { + // i18n-exempt: transport/API diagnostic. The caller renders the + // structured error code through translated UI copy. + throw new Error("Invalid conversation subscription response"); + } + const changed = + this.state.epoch !== "" && + (this.state.epoch !== response.epoch || + compareRevision(response.revision, this.state.appliedRevision) > 0); + this.state = { epoch: response.epoch, appliedRevision: response.revision }; + this.recovering = false; + const next: OrderedReady = { + ...binding, + source: true, + snapshotToken: "", + snapshotExpiresAt: binding.expiresAt, + resumeToken: "", + consumerId: this.scopeID, + watermark: 0, + epoch: response.epoch, + revision: response.revision, + }; + this.readyPromise = Promise.resolve(next); + if (changed && notifyOnChange) await this.notifyRebindListeners(); + return next; + } + + private async resubscribe(notifyOnChange = true) { + const binding = await this.getBinding(); + await this.subscribeSource(binding, notifyOnChange); + } + + private updateReadyState() { + if (!this.readyPromise) return; + void this.readyPromise.then((ready) => { + if (this.readyPromise) { + this.readyPromise = Promise.resolve({ + ...ready, + epoch: this.state.epoch, + revision: this.state.appliedRevision, + }); + } + }); + } + + private shouldBuffer(value: Partial): boolean { + if (!Array.isArray(value.operations) || value.operations.length === 0) return false; + return value.operations.some((operation) => { + if (!operation || (operation.kind !== "upsert" && operation.kind !== "remove")) return true; + const kind = snapshotKindForOperation(operation as ConversationChangeOperation); + return [...this.listeners.values()] + .filter((listener) => listener.kind === kind) + .some((listener) => !this.committedSnapshots.has(`${kind}:${listener.key}`)); + }); + } + + private applyChange(value: unknown) { + const result = reconcileConversationChange(this.state, value); + if (result.kind !== "applied") { + if (result.kind === "recover") { + this.bufferPendingChange(value); + this.beginRecovery(); + } + return; + } + this.state = result.state; + this.updateReadyState(); + const change = value as ConversationChange; + result.operations.forEach((operation, index) => { + const event = sourceOperationEvent( + operation, + change.session_id, + this.taskId, + change.revision, + index, + ); + for (const listener of this.listeners.keys()) { + try { + if (!listener(event)) { + this.beginRecovery(); + return; + } + } catch { + this.beginRecovery(); + return; + } + } + }); + } + + private discardCoveredChanges(revision: string) { + this.pendingChanges.splice( + 0, + this.pendingChanges.length, + ...this.pendingChanges.filter((value) => { + const change = value as Partial; + return validRevision(change.revision) && compareRevision(change.revision, revision) > 0; + }), + ); + this.recalculatePendingMetrics(); + } + + private drainPending() { + if (this.closed || this.recovering) return; + if (this.pendingRecoveryMarker) { + this.beginRecovery(); + return; + } + if (this.pendingChanges.length === 0) return; + const pending = this.pendingChanges.splice(0); + this.resetPendingMetrics(); + for (const change of pending) { + if (this.shouldBuffer(change as Partial)) { + this.bufferPendingChange(change); + continue; + } + this.applyChange(change); + if (this.recovering || this.pendingRecoveryMarker) return; + } + } + + private beginRecovery() { + if (this.closed || this.terminal) return; + this.recoveryNeeded = true; + if (this.recovering || this.recoveryPromise) return; + this.clearRecoveryRetry(); + this.recovering = true; + const recovery = this.resubscribe(false) + .then(() => this.notifyRebindListeners()) + .then(() => { + this.recovering = false; + this.recoveryNeeded = false; + }) + .catch((error: unknown) => { + this.recovering = false; + if (this.isRetryableRecoveryError(error)) this.scheduleRecoveryRetry(); + else this.recoveryNeeded = false; + }) + .finally(() => { + if (this.recoveryPromise === recovery) this.recoveryPromise = null; + }); + this.recoveryPromise = recovery; + } + + private isRetryableRecoveryError(error: unknown): boolean { + return !( + error && + typeof error === "object" && + "retryable" in error && + (error as { retryable?: unknown }).retryable === false + ); + } + + private scheduleRecoveryRetry() { + if ( + this.recoveryRetryTimer !== undefined || + this.closed || + this.terminal || + !this.recoveryNeeded + ) { + return; + } + this.recoveryRetryTimer = setTimeout(() => { + this.recoveryRetryTimer = undefined; + if (this.recoveryNeeded) this.beginRecovery(); + }, RECOVERY_RETRY_DELAY_MS); + } + + private clearRecoveryRetry() { + clearTimeout(this.recoveryRetryTimer); + this.recoveryRetryTimer = undefined; + } + + private bufferPendingChange(value: unknown) { + if (this.closed || this.pendingRecoveryMarker) { + if (!this.closed) this.beginRecovery(); + return; + } + const metrics = pendingChangeMetrics(value); + if ( + !metrics || + metrics.operations > MAX_PENDING_CONVERSATION_OPERATIONS || + metrics.bytes > MAX_PENDING_CONVERSATION_BYTES || + this.pendingOperationCount + metrics.operations > MAX_PENDING_CONVERSATION_OPERATIONS || + this.pendingPayloadBytes + metrics.bytes > MAX_PENDING_CONVERSATION_BYTES || + (this.pendingSince !== undefined && + Date.now() - this.pendingSince >= MAX_PENDING_CONVERSATION_AGE_MS) + ) { + this.pendingChanges.length = 0; + this.resetPendingMetrics(); + this.pendingRecoveryMarker = true; + this.beginRecovery(); + return; + } + if (this.pendingSince === undefined) { + this.pendingSince = Date.now(); + this.pendingBufferTimer = setTimeout(() => { + this.pendingBufferTimer = undefined; + this.pendingChanges.length = 0; + this.resetPendingMetrics(); + this.pendingRecoveryMarker = true; + this.beginRecovery(); + }, MAX_PENDING_CONVERSATION_AGE_MS); + } + this.pendingChanges.push(value); + this.pendingOperationCount += metrics.operations; + this.pendingPayloadBytes += metrics.bytes; + } + + private resetPendingMetrics() { + this.pendingOperationCount = 0; + this.pendingPayloadBytes = 0; + this.pendingSince = undefined; + clearTimeout(this.pendingBufferTimer); + this.pendingBufferTimer = undefined; + } + + private clearPendingBuffer() { + this.pendingChanges.length = 0; + this.resetPendingMetrics(); + this.pendingRecoveryMarker = false; + } + + private recalculatePendingMetrics() { + this.resetPendingMetrics(); + if (this.pendingChanges.length === 0) return; + let operations = 0; + let bytes = 0; + for (const change of this.pendingChanges) { + const metrics = pendingChangeMetrics(change); + if (!metrics) { + this.pendingChanges.length = 0; + this.pendingRecoveryMarker = true; + return; + } + operations += metrics.operations; + bytes += metrics.bytes; + } + this.pendingOperationCount = operations; + this.pendingPayloadBytes = bytes; + this.pendingSince = Date.now(); + this.pendingBufferTimer = setTimeout(() => { + this.pendingBufferTimer = undefined; + this.pendingChanges.length = 0; + this.resetPendingMetrics(); + this.pendingRecoveryMarker = true; + this.beginRecovery(); + }, MAX_PENDING_CONVERSATION_AGE_MS); + } + + private async notifyRebindListeners() { + await Promise.all( + [...this.rebindListeners].map((listener) => Promise.resolve().then(() => listener())), + ); + } +} + +export function sourceConversationMessageUrl( + pluginId: string, + sessionId: string, + query: URLSearchParams, +): string { + return pluginConversationUrl( + pluginId, + `/conversation/v2/task-sessions/${encodeURIComponent(sessionId)}/messages?${query}`, + ); +} diff --git a/apps/web/lib/types/session-events.ts b/apps/web/lib/types/session-events.ts index 2a4fef902d..d7f1148a66 100644 --- a/apps/web/lib/types/session-events.ts +++ b/apps/web/lib/types/session-events.ts @@ -220,6 +220,27 @@ export type QueueStatusChangedPayload = { auto_merge_revision?: number; }; +export type ConversationChangedOperation = { + kind: "upsert" | "remove"; + entity: "message" | "turn"; + id: string; + message?: Record; + turn?: Record; +}; + +export type ConversationChangedPayload = { + protocol_version: 2; + scope_id: string; + session_id: string; + epoch: string; + base_revision: string; + revision: string; + check?: boolean; + terminal?: boolean; + reset?: boolean; + operations: ConversationChangedOperation[]; +}; + export type AvailableCommandPayload = { name: string; description?: string; @@ -230,6 +251,7 @@ export type SessionBackendMessageMap = { "session.message.added": BackendMessage<"session.message.added", MessageAddedPayload>; "session.message.updated": BackendMessage<"session.message.updated", MessageAddedPayload>; "session.message.deleted": BackendMessage<"session.message.deleted", MessageAddedPayload>; + "session.removed": BackendMessage<"session.removed", { session_id: string; task_id?: string }>; "session.state_changed": BackendMessage<"session.state_changed", TaskSessionStateChangedPayload>; "session.turn_finished": BackendMessage<"session.turn_finished", TaskSessionNotificationPayload>; "session.activity_changed": BackendMessage< @@ -269,6 +291,10 @@ export type SessionBackendMessageMap = { "session.turn.removed", { id: string; session_id: string; task_id: string } >; + "session.conversation.changed": BackendMessage< + "session.conversation.changed", + ConversationChangedPayload + >; "session.available_commands": BackendMessage< "session.available_commands", AvailableCommandsPayload diff --git a/apps/web/lib/ws/client.test.ts b/apps/web/lib/ws/client.test.ts index d37be6f3b6..c045bd35f6 100644 --- a/apps/web/lib/ws/client.test.ts +++ b/apps/web/lib/ws/client.test.ts @@ -58,6 +58,7 @@ class FakeWebSocket { function connectClient(options?: ConstructorParameters[2]) { const client = new WebSocketClient("ws://test", undefined, { enabled: false, + conversationProtocol: "v1", ...options, }); client.connect(); @@ -900,4 +901,220 @@ describe("ordered core session validation", () => { unregister(); subscription.unsubscribe(); }); + + it("projects source conversation batches and ignores legacy duplicates", async () => { + const { client, socket } = connectClient({ conversationProtocol: "v2" }); + const projected = vi.fn(); + const changed = vi.fn(); + const completed = vi.fn(); + client.on("session.message.added", projected); + client.on("session.conversation.changed", changed); + client.on("session.turn.completed", completed); + const subscription = client.subscribeSessionWithReady("sess-1"); + const v2Request = socket.sent.find( + (message) => message.action === "session.conversation.subscribe", + ); + if (!v2Request) throw new Error("No source conversation subscribe request was sent"); + const scopeID = (v2Request.payload as { scope_id: string }).scope_id; + socket.receive({ + id: v2Request.id, + type: "response", + payload: { + success: true, + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + revision: "0", + }, + }); + const legacyRequest = socket.sent.find((message) => message.action === "session.subscribe"); + if (!legacyRequest) throw new Error("No session subscribe request was sent"); + socket.receive({ id: legacyRequest.id, type: "response", payload: { success: true } }); + await subscription.ready; + + socket.receive({ + type: "notification", + action: "session.conversation.changed", + payload: { + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + base_revision: "0", + revision: "1", + operations: [ + { + kind: "upsert", + entity: "message", + id: "message-1", + message: { + task_id: "task-1", + author_type: "user", + content: "source", + type: "message", + created_at: "2026-09-16T12:00:00Z", + updated_at: "2026-09-16T12:00:00Z", + }, + }, + { + kind: "upsert", + entity: "turn", + id: "turn-1", + turn: { + task_id: "task-1", + started_at: "2026-09-16T11:59:00Z", + completed_at: "2026-09-16T12:00:01Z", + updated_at: "2026-09-16T12:00:01Z", + execution_profile_id: "profile-1", + route_generation: 3, + metadata: { runtime_config_snapshot: { model: "mock-fast" } }, + had_output: false, + }, + }, + ], + }, + }); + socket.receive({ + type: "notification", + action: "session.message.added", + payload: { session_id: "sess-1", message_id: "message-1" }, + }); + expect(projected).toHaveBeenCalledTimes(1); + expect(changed).toHaveBeenCalledTimes(1); + expect(completed).toHaveBeenCalledWith( + expect.objectContaining({ + payload: expect.objectContaining({ + id: "turn-1", + execution_profile_id: "profile-1", + route_generation: 3, + metadata: { runtime_config_snapshot: { model: "mock-fast" } }, + had_output: false, + }), + }), + ); + subscription.unsubscribe(); + }); + + it("repairs malformed source operations before advancing the applied revision", async () => { + const { client, socket } = connectClient({ conversationProtocol: "v2" }); + const projected = vi.fn(); + const recover = vi.fn(async () => true); + client.on("session.message.added", projected); + const subscription = client.subscribeSessionWithReady("sess-1"); + const v2Request = socket.sent.find( + (message) => message.action === "session.conversation.subscribe", + ); + if (!v2Request) throw new Error("No source conversation subscribe request was sent"); + const scopeID = (v2Request.payload as { scope_id: string }).scope_id; + socket.receive({ + id: v2Request.id, + type: "response", + payload: { + success: true, + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + revision: "0", + }, + }); + const legacyRequest = socket.sent.find((message) => message.action === "session.subscribe"); + if (!legacyRequest) throw new Error("No session subscribe request was sent"); + socket.receive({ id: legacyRequest.id, type: "response", payload: { success: true } }); + await subscription.ready; + client.registerCoreSessionRecovery("sess-1", recover); + + socket.receive({ + type: "notification", + action: "session.conversation.changed", + payload: { + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + base_revision: "0", + revision: "1", + operations: [{ kind: "upsert", entity: "message", id: "message-1" }], + }, + }); + + await vi.waitFor(() => expect(recover).toHaveBeenCalledTimes(1)); + expect(projected).not.toHaveBeenCalled(); + subscription.unsubscribe(); + }); + it.each(["missed", "delivered", "closed"])( + "checks idle source revisions with %s updates", + async (outcome) => { + vi.useFakeTimers(); + const { client, socket } = connectClient({ conversationProtocol: "v2" }); + const projected = vi.fn(); + const recover = vi.fn(async () => true); + client.on("session.message.added", projected); + const subscription = client.subscribeSessionWithReady("sess-1"); + const v2Request = socket.sent.find( + (message) => message.action === "session.conversation.subscribe", + ); + if (!v2Request) throw new Error("No source conversation subscribe request was sent"); + const scopeID = (v2Request.payload as { scope_id: string }).scope_id; + socket.receive({ + id: v2Request.id, + type: "response", + payload: { + success: true, + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + revision: "0", + }, + }); + const legacyRequest = socket.sent.find((message) => message.action === "session.subscribe"); + if (!legacyRequest) throw new Error("No session subscribe request was sent"); + socket.receive({ id: legacyRequest.id, type: "response", payload: { success: true } }); + await subscription.ready; + client.registerCoreSessionRecovery("sess-1", recover); + + const check = (revision: string) => + socket.receive({ + type: "notification", + action: "session.conversation.changed", + payload: { + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + base_revision: revision, + revision, + check: true, + operations: [], + }, + }); + check("0"); + await vi.advanceTimersByTimeAsync(1000); + expect(recover).not.toHaveBeenCalled(); + check("1"); + await vi.advanceTimersByTimeAsync(999); + expect(recover).not.toHaveBeenCalled(); + if (outcome === "closed") subscription.unsubscribe(); + if (outcome === "delivered") + socket.receive({ + type: "notification", + action: "session.conversation.changed", + payload: { + protocol_version: 2, + scope_id: scopeID, + session_id: "sess-1", + epoch: "epoch-1", + base_revision: "0", + revision: "1", + operations: [], + }, + }); + await vi.advanceTimersByTimeAsync(1); + expect(recover).toHaveBeenCalledTimes(outcome === "missed" ? 1 : 0); + + subscription.unsubscribe(); + }, + ); }); diff --git a/apps/web/lib/ws/client.ts b/apps/web/lib/ws/client.ts index 312365c5ca..ff0128a140 100644 --- a/apps/web/lib/ws/client.ts +++ b/apps/web/lib/ws/client.ts @@ -1,6 +1,7 @@ /* eslint-disable max-lines -- WebSocketClient intentionally owns one connection's complete request, subscription, and reconnect lifecycle. */ import type { BackendMessageMap, BackendMessageType } from "@/lib/types/backend"; +import type { ConversationChangedPayload } from "@/lib/types/session-events"; import type { ConnectionStatus } from "@/lib/types/connection"; import { generateUUID } from "@/lib/utils"; import { createDebugLogger, isDebug } from "@/lib/debug/log"; @@ -50,6 +51,7 @@ export interface ReconnectOptions { initialDelay?: number; maxDelay?: number; backoffMultiplier?: number; + conversationProtocol?: "v1" | "v2"; } export interface SessionSubscriptionHandle { @@ -59,6 +61,177 @@ export interface SessionSubscriptionHandle { export type CoreSessionRecoveryHandler = () => Promise; +const CORE_CONVERSATION_ACTIONS = new Set([ + "session.message.added", + "session.message.updated", + "session.message.deleted", + "session.turn.started", + "session.turn.completed", + "session.turn.removed", +]); +const SESSION_SUBSCRIBE_ACTION = "session.subscribe"; +const SESSION_UNSUBSCRIBE_ACTION = "session.unsubscribe"; +const SESSION_CONVERSATION_CHANGED_ACTION = "session.conversation.changed" as const; + +function isConversationNotificationAction(action: BackendMessageType): boolean { + return CORE_CONVERSATION_ACTIONS.has(action); +} + +function isCoreConversationOperation(value: unknown): boolean { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const operation = value as { + kind?: unknown; + entity?: unknown; + id?: unknown; + message?: unknown; + turn?: unknown; + }; + if ( + (operation.kind !== "upsert" && operation.kind !== "remove") || + (operation.entity !== "message" && operation.entity !== "turn") || + typeof operation.id !== "string" || + operation.id === "" + ) { + return false; + } + if (operation.kind === "remove") return true; + const payload = operation.entity === "message" ? operation.message : operation.turn; + return Boolean(payload && typeof payload === "object" && !Array.isArray(payload)); +} + +type ParsedCoreConversationChange = { + sessionId: string; + payload: Partial & { operations?: unknown[] }; + valid: boolean; +}; + +function isDecimalRevision(value: unknown): value is string { + return typeof value === "string" && /^(0|[1-9][0-9]*)$/.test(value); +} + +function parseCoreConversationChange(value: unknown): ParsedCoreConversationChange | null { + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const payload = value as Partial & { operations?: unknown[] }; + if (typeof payload.session_id !== "string") return null; + const valid = + payload.protocol_version === 2 && + typeof payload.scope_id === "string" && + typeof payload.epoch === "string" && + isDecimalRevision(payload.base_revision) && + isDecimalRevision(payload.revision) && + Array.isArray(payload.operations) && + payload.operations.length <= 256 && + payload.operations.every(isCoreConversationOperation) && + BigInt(payload.revision) > BigInt(payload.base_revision); + return { sessionId: payload.session_id, payload, valid }; +} + +function needsCoreConversationRecovery( + parsed: ParsedCoreConversationChange, + stream: CoreSessionStream, +): boolean { + return !parsed.valid || parsed.payload.scope_id !== stream.conversationScopeId; +} + +function hasCoreConversationRevisionGap( + payload: ConversationChangedPayload, + stream: CoreSessionStream, + appliedRevision: string, +): boolean { + return ( + payload.reset === true || + payload.epoch !== stream.conversationEpoch || + payload.base_revision !== appliedRevision + ); +} + +type CoreConversationOperation = { + kind: "upsert" | "remove"; + entity: "message" | "turn"; + id: string; + message?: Record; + turn?: Record; +}; + +function coreTurnAction(operation: CoreConversationOperation): BackendMessageType { + if (operation.kind === "remove") return "session.turn.removed"; + return operation.turn?.completed_at ? "session.turn.completed" : "session.turn.started"; +} + +function projectCoreMessageOperation( + sessionId: string, + operation: CoreConversationOperation & { entity: "message" }, +): { action: BackendMessageType; payload: Record } { + const message = operation.message; + const payload: Record = { + type: operation.kind === "remove" ? "message.deleted" : (message?.type ?? "message"), + session_id: sessionId, + task_id: message?.task_id ?? null, + message_id: operation.id, + }; + if (operation.kind === "upsert" && message) { + Object.assign(payload, { + author_type: message.author_type, + author_id: message.author_id, + content: message.content, + raw_content: message.raw_content, + message_type: message.type, + turn_id: message.turn_id, + metadata: message.metadata, + requests_input: message.requests_input, + created_at: message.created_at, + updated_at: message.updated_at ?? message.created_at, + prompt_index: message.prompt_index, + }); + } + return { + action: operation.kind === "remove" ? "session.message.deleted" : "session.message.added", + payload, + }; +} + +function projectCoreTurnOperation( + sessionId: string, + operation: CoreConversationOperation & { entity: "turn" }, +): { action: BackendMessageType; payload: Record } { + const turn = operation.turn; + const payload: Record = { + type: operation.kind === "remove" ? "session.turn.removed" : "session.turn.started", + session_id: sessionId, + task_id: turn?.task_id ?? null, + id: operation.id, + }; + if (operation.kind === "upsert" && turn) { + Object.assign(payload, { + started_at: turn.started_at, + completed_at: turn.completed_at, + updated_at: turn.updated_at ?? turn.started_at, + created_at: turn.created_at ?? turn.started_at, + execution_profile_id: turn.execution_profile_id, + route_generation: turn.route_generation, + metadata: turn.metadata, + had_output: turn.had_output, + }); + } + return { action: coreTurnAction(operation), payload }; +} + +function projectCoreConversationOperation( + sessionId: string, + operation: CoreConversationOperation, +): { action: BackendMessageType; payload: Record } { + if (operation.entity === "message") { + return projectCoreMessageOperation( + sessionId, + operation as CoreConversationOperation & { entity: "message" }, + ); + } + return projectCoreTurnOperation( + sessionId, + operation as CoreConversationOperation & { entity: "turn" }, + ); +} + type SessionSubscriptionReadiness = { promise: Promise; resolve: () => void; @@ -167,6 +340,7 @@ const DEFAULT_RECONNECT_OPTIONS: Required = { initialDelay: 1000, maxDelay: 30000, backoffMultiplier: 1.5, + conversationProtocol: "v2", }; // i18n-exempt: transport/API diagnostic. Callers branch on the error code and // render translated copy; this text only ever appears in a console or as an @@ -197,6 +371,7 @@ export class WebSocketClient { private sessionSubscriptionReadiness = new Map(); private coreSessionStreams = new Map(); private coreSessionRecoveryHandlers = new Map>(); + private coreConversationSessions = new Set(); // Ref-counted focus signals: a session can be focused by both the task panel // and the task details page if both are mounted. Backend wakes its workspace // tracker into fast-poll mode while any client has focus, falling back to @@ -459,11 +634,15 @@ export class WebSocketClient { this.sessionSubscriptions.delete(sessionId); this.cancelSessionSubscriptionReadiness(sessionId); const stream = this.coreSessionStreams.get(sessionId); - if (this.status === "connected" && stream) { + if ( + this.status === "connected" && + stream && + this.reconnectOptions.conversationProtocol === "v1" + ) { this.send({ id: generateUUID(), type: "request", - action: "session.unsubscribe", + action: SESSION_UNSUBSCRIBE_ACTION, payload: { session_id: sessionId, consumer_kind: "core", @@ -471,13 +650,31 @@ export class WebSocketClient { }, }); } + if ( + this.status === "connected" && + stream && + this.reconnectOptions.conversationProtocol === "v2" + ) { + this.send({ + id: generateUUID(), + type: "request", + action: "session.conversation.unsubscribe", + payload: { + scope_id: stream.conversationScopeId, + session_id: sessionId, + consumer_kind: "core", + }, + }); + } + clearTimeout(stream?.conversationCheckTimer); this.coreSessionStreams.delete(sessionId); + this.coreConversationSessions.delete(sessionId); this.coreSessionRecoveryHandlers.delete(sessionId); if (this.status === "connected") { this.send({ id: generateUUID(), type: "request", - action: "session.unsubscribe", + action: SESSION_UNSUBSCRIBE_ACTION, payload: { session_id: sessionId }, }); } @@ -631,14 +828,19 @@ export class WebSocketClient { this.rawSessionEventHandlers.forEach((handler) => handler(value)); return; } - if (this.handleMalformedSessionEnvelope(value)) return; - if (!value || typeof value !== "object" || !("type" in value)) return; - const rawMessage = value as RawWebSocketMessage; - if (this.handleRequestResult(rawMessage)) return; - const message = value as BackendMessageMap[BackendMessageType]; - if (message.type !== "notification") return; + if (this.handleControlFrame(value)) return; + const message = this.notificationMessage(value); + if (!message) return; const action = message.action; - if (!action) return; + if (action === SESSION_CONVERSATION_CHANGED_ACTION) { + this.dispatchConversationChanged(message); + return; + } + const payloadSessionID = + message.payload && typeof message.payload === "object" + ? (message.payload as { session_id?: unknown }).session_id + : undefined; + if (this.shouldSuppressCoreConversationNotification(action, payloadSessionID)) return; const handlers = this.handlers.get(action); this.debugNotification(action, message.payload, handlers?.size ?? 0); if (handlers) { @@ -646,6 +848,185 @@ export class WebSocketClient { } dispatchToPluginWsHandlers(action, message.payload); } + + private handleControlFrame(value: unknown): boolean { + if (this.handleMalformedSessionEnvelope(value)) return true; + if (!value || typeof value !== "object" || !("type" in value)) return true; + return this.handleRequestResult(value as RawWebSocketMessage); + } + + private notificationMessage(value: unknown): BackendMessageMap[BackendMessageType] | null { + if (!value || typeof value !== "object" || !("type" in value)) return null; + const message = value as BackendMessageMap[BackendMessageType]; + if (message.type !== "notification" || !message.action) return null; + return message; + } + + private dispatchConversationChanged( + message: BackendMessageMap[typeof SESSION_CONVERSATION_CHANGED_ACTION], + ) { + this.handleCoreConversationChanged(message.payload); + const handlers = this.handlers.get(SESSION_CONVERSATION_CHANGED_ACTION); + this.debugNotification( + SESSION_CONVERSATION_CHANGED_ACTION, + message.payload, + handlers?.size ?? 0, + ); + handlers?.forEach((handler) => handler(message)); + dispatchToPluginWsHandlers(SESSION_CONVERSATION_CHANGED_ACTION, message.payload); + } + + private shouldSuppressCoreConversationNotification( + action: BackendMessageType, + sessionId: unknown, + ): boolean { + return ( + this.reconnectOptions.conversationProtocol === "v2" && + typeof sessionId === "string" && + this.coreConversationSessions.has(sessionId) && + isConversationNotificationAction(action) + ); + } + + private handleCoreConversationChanged(value: unknown) { + const parsed = parseCoreConversationChange(value); + if (!parsed) return; + const { sessionId } = parsed; + const stream = this.coreSessionStreams.get(sessionId); + if (!stream || this.reconnectOptions.conversationProtocol !== "v2") return; + if (this.handleCoreConversationControl(parsed, stream)) return; + if (needsCoreConversationRecovery(parsed, stream)) { + this.beginCoreConversationRecovery( + sessionId, + stream, + stream.conversationAppliedRevision ?? "0", + ); + return; + } + if (!parsed.valid) return; + const validPayload = parsed.payload as ConversationChangedPayload; + if (!stream.ready || stream.needsHydration || stream.projectionPaused) { + stream.conversationPendingChanges.push(validPayload); + this.trimCoreConversationPendingChanges(sessionId, stream, validPayload.revision); + return; + } + const applied = stream.conversationAppliedRevision ?? "0"; + const revision = validPayload.revision; + if (BigInt(revision) <= BigInt(applied)) return; + if (hasCoreConversationRevisionGap(validPayload, stream, applied)) { + stream.conversationPendingChanges.push(validPayload); + this.beginCoreConversationRecovery(sessionId, stream, revision); + return; + } + stream.conversationAppliedRevision = revision; + this.projectCoreConversationOperations(sessionId, validPayload.operations ?? []); + } + + private handleCoreConversationControl( + parsed: ParsedCoreConversationChange, + stream: CoreSessionStream, + ): boolean { + const { sessionId } = parsed; + if (parsed.payload.scope_id !== stream.conversationScopeId) return true; + if (parsed.payload.terminal) { + clearTimeout(stream.conversationCheckTimer); + stream.projectionPaused = true; + stream.conversationPendingChanges.length = 0; + this.handlers.get("session.removed")?.forEach((handler) => + handler({ + type: "notification", + action: "session.removed", + payload: { session_id: sessionId }, + }), + ); + return true; + } + if (parsed.payload.check) { + this.checkCoreConversationRevision(sessionId, stream, parsed.payload); + return true; + } + return false; + } + + private checkCoreConversationRevision( + sessionId: string, + stream: CoreSessionStream, + payload: Partial, + ) { + const applied = stream.conversationAppliedRevision ?? "0"; + if ( + !isDecimalRevision(payload.revision) || + payload.epoch !== stream.conversationEpoch || + BigInt(payload.revision) < BigInt(applied) + ) { + this.beginCoreConversationRecovery(sessionId, stream, applied); + return; + } + if (payload.revision === applied || stream.conversationCheckTimer !== undefined) return; + const observed = payload.revision; + stream.conversationCheckTimer = setTimeout(() => { + stream.conversationCheckTimer = undefined; + if (this.coreSessionStreams.get(sessionId) !== stream || this.status !== "connected") return; + if (BigInt(observed) > BigInt(stream.conversationAppliedRevision ?? "0")) { + this.beginCoreConversationRecovery(sessionId, stream, observed); + } + }, 1000); + } + + private trimCoreConversationPendingChanges( + sessionId: string, + stream: CoreSessionStream, + recoveryRevision: string, + ) { + if (stream.conversationPendingChanges.length <= 256) return; + stream.conversationPendingChanges.splice(0, stream.conversationPendingChanges.length - 256); + this.beginCoreConversationRecovery(sessionId, stream, recoveryRevision); + } + + private drainCoreConversationChanges(sessionId: string, stream: CoreSessionStream) { + if (!stream.ready || stream.needsHydration || stream.projectionPaused) return; + const pending = stream.conversationPendingChanges.splice(0); + for (const change of pending) { + this.handleCoreConversationChanged(change); + if (stream.needsHydration || stream.projectionPaused) return; + } + if (this.coreSessionStreams.get(sessionId) !== stream) return; + } + + private projectCoreConversationOperations(sessionId: string, operations: unknown[]) { + for (const value of operations) { + if (!value || typeof value !== "object" || Array.isArray(value)) continue; + const operation = value as CoreConversationOperation; + if (!isCoreConversationOperation(operation)) continue; + const projected = projectCoreConversationOperation(sessionId, operation); + this.dispatchConversationNotification(projected.action, projected.payload); + } + } + + private dispatchConversationNotification(action: BackendMessageType, payload: unknown) { + const handlers = this.handlers.get(action); + this.debugNotification(action, payload, handlers?.size ?? 0); + handlers?.forEach((handler) => handler({ type: "notification", action, payload } as never)); + dispatchToPluginWsHandlers(action, payload); + } + + private beginCoreConversationRecovery( + sessionId: string, + stream: CoreSessionStream, + revision: string, + ) { + stream.projectionPaused = true; + stream.needsHydration = true; + if ( + !stream.conversationRecoveryRevision || + BigInt(revision) > BigInt(stream.conversationRecoveryRevision) + ) { + stream.conversationRecoveryRevision = revision; + } + stream.recoveryGeneration += 1; + void this.continueCoreSessionRecovery(sessionId, stream); + } + /** * A frame that is session.event-shaped but fails the strict envelope check * (bad protocol version, missing identity fields, malformed payload) can @@ -844,15 +1225,19 @@ export class WebSocketClient { readiness.requestStarted = true; readiness.attempt += 1; const stream = this.getOrCreateCoreSessionStream(sessionId); + if (this.reconnectOptions.conversationProtocol === "v2") { + this.startSourceSessionSubscription(sessionId, readiness, stream); + return; + } stream.ready = false; stream.pendingEvents.length = 0; const legacySubscription = this.request( - "session.subscribe", + SESSION_SUBSCRIBE_ACTION, { session_id: sessionId }, SESSION_ENTRY_REQUEST_TIMEOUT_MS, ); const orderedSubscription = this.request( - "session.subscribe", + SESSION_SUBSCRIBE_ACTION, { session_id: sessionId, consumer_kind: "core", @@ -912,17 +1297,105 @@ export class WebSocketClient { }); } + private startSourceSessionSubscription( + sessionId: string, + readiness: SessionSubscriptionReadiness, + stream: CoreSessionStream, + ) { + stream.ready = false; + stream.conversationPendingChanges.length = 0; + const legacySubscription = this.request( + SESSION_SUBSCRIBE_ACTION, + { session_id: sessionId }, + SESSION_ENTRY_REQUEST_TIMEOUT_MS, + ); + const conversationSubscription = this.request( + "session.conversation.subscribe", + { + scope_id: stream.conversationScopeId, + session_id: sessionId, + consumer_kind: "core", + }, + SESSION_ENTRY_REQUEST_TIMEOUT_MS, + ).then((response) => { + const payload = response as Partial & { + success?: boolean; + error?: unknown; + }; + if ( + payload.success !== true || + payload.protocol_version !== 2 || + payload.scope_id !== stream.conversationScopeId || + payload.session_id !== sessionId || + typeof payload.epoch !== "string" || + typeof payload.revision !== "string" || + !/^(0|[1-9][0-9]*)$/.test(payload.revision) + ) { + // i18n-exempt: transport/API diagnostic. The caller renders the + // structured error code through translated UI copy. + throw new Error("Invalid conversation subscription response"); + } + const hadConversationState = stream.conversationEpoch !== undefined; + const missedConversationState = + hadConversationState && + (stream.conversationEpoch !== payload.epoch || + BigInt(payload.revision) > BigInt(stream.conversationAppliedRevision ?? "0")); + stream.conversationEpoch = payload.epoch; + if (!missedConversationState) stream.conversationAppliedRevision = payload.revision; + if (missedConversationState) { + stream.needsHydration = true; + stream.projectionPaused = true; + stream.conversationRecoveryRevision = payload.revision; + stream.recoveryGeneration += 1; + } + stream.ready = true; + this.coreConversationSessions.add(sessionId); + this.drainCoreConversationChanges(sessionId, stream); + }); + void Promise.all([legacySubscription, conversationSubscription]) + .then(() => { + if (this.sessionSubscriptionReadiness.get(sessionId) !== readiness) return; + readiness.retryTimer = null; + readiness.settled = true; + readiness.resolve(); + if (stream.needsHydration) void this.continueCoreSessionRecovery(sessionId, stream); + }) + .catch((error: unknown) => { + stream.ready = false; + stream.conversationPendingChanges.length = 0; + if (this.sessionSubscriptionReadiness.get(sessionId) !== readiness) return; + readiness.requestStarted = false; + if ( + isWebSocketRequestTimeoutError(error) && + readiness.attempt < MAX_SESSION_SUBSCRIPTION_ATTEMPTS && + this.status === "connected" && + this.socket + ) { + readiness.retryTimer = setTimeout(() => { + readiness.retryTimer = null; + this.startSessionSubscription(sessionId, readiness); + }, SESSION_ENTRY_RETRY_DELAY_MS); + return; + } + this.sessionSubscriptionReadiness.delete(sessionId); + readiness.settled = true; + readiness.reject(error); + }); + } + private getOrCreateCoreSessionStream(sessionId: string): CoreSessionStream { const current = this.coreSessionStreams.get(sessionId); if (current) return current; const stream = { wireId: `core:web:${generateUUID()}`, + conversationScopeId: `core:web:${generateUUID()}`, lastSeenSequence: 0, ready: false, pendingEvents: [], projectionPaused: false, needsHydration: false, recoveryGeneration: 0, + conversationPendingChanges: [], }; this.coreSessionStreams.set(sessionId, stream); return stream; @@ -1019,7 +1492,7 @@ export class WebSocketClient { stream.projectionPaused = true; stream.recoveryGeneration += 1; stream.recoveryWatermark = undefined; - stream.poisonRecovery = this.request("session.subscribe", { + stream.poisonRecovery = this.request(SESSION_SUBSCRIBE_ACTION, { session_id: sessionId, consumer_kind: "core", wire_id: stream.wireId, @@ -1045,7 +1518,13 @@ export class WebSocketClient { sessionId: string, stream: CoreSessionStream, ): Promise { - if (!stream.needsHydration || stream.recoveryWatermark === undefined) { + if ( + !stream.needsHydration || + (this.reconnectOptions.conversationProtocol === "v1" && + stream.recoveryWatermark === undefined) || + (this.reconnectOptions.conversationProtocol === "v2" && + stream.conversationRecoveryRevision === undefined) + ) { return Promise.resolve(false); } const existing = stream.recoveryHydration; @@ -1062,15 +1541,31 @@ export class WebSocketClient { ) { return false; } - const watermark = stream.recoveryWatermark; - if (watermark === undefined) return false; - stream.lastSeenSequence = Math.max(stream.lastSeenSequence, watermark); - stream.pendingEvents = stream.pendingEvents.filter( - (event) => event.sequence > stream.lastSeenSequence, - ); + if (this.reconnectOptions.conversationProtocol === "v2") { + const revision = stream.conversationRecoveryRevision; + if (revision === undefined) return false; + stream.conversationAppliedRevision = revision; + stream.conversationPendingChanges = stream.conversationPendingChanges.filter((change) => { + if (!change || typeof change !== "object") return false; + const candidate = change as { revision?: unknown }; + return ( + typeof candidate.revision === "string" && + BigInt(candidate.revision) > BigInt(revision) + ); + }); + stream.conversationRecoveryRevision = undefined; + } else { + const watermark = stream.recoveryWatermark; + if (watermark === undefined) return false; + stream.lastSeenSequence = Math.max(stream.lastSeenSequence, watermark); + stream.pendingEvents = stream.pendingEvents.filter( + (event) => event.sequence > stream.lastSeenSequence, + ); + } stream.needsHydration = false; stream.projectionPaused = false; stream.recoveryWatermark = undefined; + this.drainCoreConversationChanges(sessionId, stream); this.drainCoreSessionEvents(sessionId, stream); return true; }) diff --git a/apps/web/lib/ws/ordered-session-events.ts b/apps/web/lib/ws/ordered-session-events.ts index 0d70492fcd..51e9d2ada5 100644 --- a/apps/web/lib/ws/ordered-session-events.ts +++ b/apps/web/lib/ws/ordered-session-events.ts @@ -24,6 +24,12 @@ export type CoreSessionStream = { recoveryGeneration: number; recoveryWatermark?: number; recoveryHydration?: Promise; + conversationScopeId: string; + conversationEpoch?: string; + conversationAppliedRevision?: string; + conversationRecoveryRevision?: string; + conversationPendingChanges: unknown[]; + conversationCheckTimer?: ReturnType; }; const ORDERED_EVENT_ACTIONS: Readonly> = { diff --git a/apps/web/lib/ws/use-websocket.tsx b/apps/web/lib/ws/use-websocket.tsx index 6823962a96..9e51412cb0 100644 --- a/apps/web/lib/ws/use-websocket.tsx +++ b/apps/web/lib/ws/use-websocket.tsx @@ -43,6 +43,7 @@ export function useWebSocket(store: StoreApi, url: string) { initialDelay: 1000, maxDelay: 30000, backoffMultiplier: 1.5, + conversationProtocol: "v2", }, ); client.subscribeUser(); diff --git a/docs/decisions/2026-09-06-browser-plugin-conversation-facade.md b/docs/decisions/2026-09-06-browser-plugin-conversation-facade.md index 9944979cfa..d6cbd0ab68 100644 --- a/docs/decisions/2026-09-06-browser-plugin-conversation-facade.md +++ b/docs/decisions/2026-09-06-browser-plugin-conversation-facade.md @@ -36,3 +36,9 @@ Add a typed browser `host.conversation` facade and scoped task-panel navigation - **Require a plugin backend to proxy Host gRPC reads.** Rejected: it adds a process and network hop to a browser-only panel, still does not solve native navigation or browser-local favorites, and prevents a UI-only package. - **Add prompt-history-specific REST and React components as one monolithic Host widget.** Rejected: it would move the feature into a differently named core component rather than establish reusable conversation and panel boundaries. - **Move mobile scroll-target state into global Zustand.** Rejected: the current mobile layout owns its active full-height surface and exact scroll target locally. A scoped injected adapter preserves that ownership and prevents stale global navigation. + +## Storage decision, 2026-09-16 + +The [source reconciliation decision](2026-09-16-conversation-source-reconciliation.md) retains the browser facade boundary. +It selects source reads and revision checks instead of mandatory conversation payload history. +The replacement implementation remains pending in its linked plan. diff --git a/docs/decisions/2026-09-16-conversation-source-reconciliation.md b/docs/decisions/2026-09-16-conversation-source-reconciliation.md new file mode 100644 index 0000000000..f7cf5ba6e1 --- /dev/null +++ b/docs/decisions/2026-09-16-conversation-source-reconciliation.md @@ -0,0 +1,73 @@ +# ADR-2026-09-16-conversation-source-reconciliation: Read original conversation records + +**Status:** accepted +**Date:** 2026-09-16 +**Area:** backend, frontend, protocol + +## Context + +PR #3588 adds sanitized message and turn versions, a primary event journal, and a separate Host event database. +Startup copies existing history. Database triggers copy subsequent mutations. +Retention removes old events but preserves the latest version of each record. +Every installation bears this cost, including installations without a conversation plugin. + +The user requested a replacement design package after review of this cost. +The existing browser facade remains useful. Durable replay is not necessary for a current-state history panel. +The earlier [maintainer direction](https://github.com/kdlbs/kandev/issues/3567#issuecomment-5619730030) also requested justification for durable replay. + +## Decision + +Original task messages and turns remain the only authoritative conversation payloads. +The Host exposes authorized, sanitized reads and manages current-state reconciliation. +A small per-session revision supports consistency checks without payload history. +Revision changes commit with source mutations. Transient receipts bind complete change intervals to their persisted results. +Subscriptions deliver updates by ID and coverage-only batches for irrelevant changes. +Applied revisions advance only through complete contiguous batches or source snapshots. +Observed revisions cannot establish cache correctness by themselves. +Hashes are excluded. They add maintenance and write cost without establishing delivery completeness. +Periodic checks cover lost notifications and writes outside the event bus. + +Browser hook signatures, task-panel capabilities, and native navigation remain compatible. +The plugin owns filtering, derived entries, durations, and presentation. +An optional plugin cache stores only data necessary for its feature under existing storage permissions. +A plugin does not need a full transcript copy to display Prompt History. + +The Host does not preserve intermediate revisions or exactly-once event delivery for this facade. +A future durable replay requirement needs a separate decision and measured storage costs. + +## Consequences + +Startup no longer copies history for plugin access. Normal writes update only a small revision record for this capability. +Normal live changes update individual IDs without rereading loaded history. +Recovery can read the loaded range again after missing delivery or reconnect. +The design bounds transient batching and checks revisions without reading payloads. +Mutation writers must capture complete receipts inside their transactions. Exceptional untracked writes trigger repair. +Live history is not an immutable snapshot across page requests. + +Cleanup must remove existing triggers, tables, and the specific Host event database. +A source revert alone leaves database triggers active. +SQLite space becomes reusable after cleanup but physical shrink requires the existing operator compaction workflow. +Downgrade after cleanup requires backup restoration. Mixed old and new backend binaries are unsupported during cutover. + +## Refinement, 2026-09-16 + +The user selected incremental updates without hashes after discussion of broad refresh costs. +This refinement replaces the initial invalidation-only plan. Database cleanup remains unchanged. + +## Alternatives Considered + +- Hash each message: rejected because hashes do not prove delivery coverage and introduce content-hash maintenance. +- Refresh on every revision: rejected because unrelated streaming repeatedly rereads unchanged prompts. +- Keep the journal and tune retention: rejected because latest versions still duplicate content and startup still copies history. +- Move the full journal into a plugin: rejected because the panel needs current records, not a second transcript history. +- Rely only on the existing event bus: rejected because missed events and direct database mutations need a repair path. +- Hold database snapshots across browser requests: rejected because slow or suspended clients prolong database transactions. +- Revert all of PR #3588: rejected because the useful API, navigation, and display boundaries can remain. + +## Related artifacts + +- [Host requirements](../specs/plugins/requirements/prompt-history-extraction-host.md) +- [Replacement design](../specs/plugins/system-design/conversation-source-reconciliation.md) +- [Implementation plan](../plans/conversation-storage-replacement/plan.md) +- [Browser facade decision](2026-09-06-browser-plugin-conversation-facade.md) +- [Remaining validation and remediation](../plans/conversation-storage-follow-up/plan.md) diff --git a/docs/plans/conversation-storage-follow-up/implementation-commit-receipt.md b/docs/plans/conversation-storage-follow-up/implementation-commit-receipt.md new file mode 100644 index 0000000000..c9e10856eb --- /dev/null +++ b/docs/plans/conversation-storage-follow-up/implementation-commit-receipt.md @@ -0,0 +1,53 @@ +# Implementation commit receipt + +```text +parent_sha: 88c6c0fe0a6ae5d25332070d603b99f7d9241645 +commit_sha: c0a048bc128f7ef9a1051caf95ed442627faf9df +pre_commit_hook: active +commit_msg_hook: active +hook_results (pre-commit): harness-lint=passed, docs-catalog=passed, architecture-lint=passed, specification-lint=passed, gofmt=passed, go-lint=passed, prettier-format=passed, web-lint=passed, i18n-new-code=passed, e2e-sleep-ratchet=passed, public-copy-em-dash=passed +hook_results (commit-msg): commitlint=passed, harness-lint=passed, docs-catalog=skipped, architecture-lint=passed, specification-lint=skipped, gofmt=skipped, go-lint=skipped, prettier-format=skipped, web-lint=skipped, i18n-new-code=skipped, e2e-sleep-ratchet=skipped, public-copy-em-dash=passed +bypass: false +commit_result: pass +worktree: clean (immediately after implementation commit, before this plan) +``` + +The first commit attempt failed Go lint, fixture ESLint, and formatting hooks. +The successful retry followed mechanical fixes and formatter restaging. +No hook was bypassed. Existing best-effort publication behavior remains unchanged. + +Additional focused checks passed: + +- `go test -race ./internal/task/repository/sqlite -run '^TestConversation' -count=1` +- `go test ./internal/plugins -run 'Conversation' -count=1` +- `go test ./internal/task/service -run 'TestPublishMessageEvent|TestPermissionResolutionServicePublishesOnlySuccessfulWrites|TestPublishClarificationBundleUpdates' -count=1` + +These checks do not replace the pending PostgreSQL, full package, or recovery E2E gates. + +## Captured hook output + +```text +Harness file lint........................................................Passed +Documentation catalog validation.........................................Passed +Architecture lint and compatibility expiry...............................Passed +Specification structure and size lint....................................Passed +gofmt (backend)..........................................................Passed +Go lint (PR changed code)................................................Passed +Prettier format (web/cli/packages).......................................Passed +Web lint (changed files only)............................................Passed +i18n guard (new code only)...............................................Passed +E2E sleep guard (new code only)..........................................Passed +Public copy em-dash guard................................................Passed +Validate commit message (Conventional Commits)...........................Passed +Harness file lint........................................................Passed +Documentation catalog validation.....................(no files to check)Skipped +Architecture lint and compatibility expiry...............................Passed +Specification structure and size lint................(no files to check)Skipped +gofmt (backend)......................................(no files to check)Skipped +Go lint (PR changed code)............................(no files to check)Skipped +Prettier format (web/cli/packages)...................(no files to check)Skipped +Web lint (changed files only)........................(no files to check)Skipped +i18n guard (new code only)...........................(no files to check)Skipped +E2E sleep guard (new code only)......................(no files to check)Skipped +Public copy em-dash guard................................................Passed +``` diff --git a/docs/plans/conversation-storage-follow-up/plan.md b/docs/plans/conversation-storage-follow-up/plan.md new file mode 100644 index 0000000000..467e032236 --- /dev/null +++ b/docs/plans/conversation-storage-follow-up/plan.md @@ -0,0 +1,129 @@ +--- +created: 2026-09-16 +status: complete +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-005 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +system_design: + - ../../specs/plugins/system-design/conversation-source-reconciliation.md +legacy_specs: [] +--- + +# Implementation Plan: Conversation storage remaining gates + +## Overview + +Close the remaining delivery gates for the committed conversation storage replacement. +The user requested this handoff for another agent to implement later. +This package reuses approved requirements and design; it introduces no new architecture. +Implementation starts from `c0a048bc128f7ef9a1051caf95ed442627faf9df`. +The comparison base before replacement is `88c6c0fe0a6ae5d25332070d603b99f7d9241645`. + +## Scope + +### In scope + +- Add and execute real PostgreSQL source, receipt, and cleanup coverage. +- Reproduce and resolve the Office migration and task-service package failures. +- Prove recovery on desktop and mobile after all remediation. +- Keep test evidence and companion plan statuses accurate. + +### Out of scope + +- Automatic startup VACUUM, content hashes, transcript copies, or a new replay system. +- Plugin extraction, publication, UI redesign, and production database operations. +- Repeating completed large SQLite benchmarks without a related code change. +- Creating Kandev tasks, starting agents, pushing, or creating a PR during planning. + +## Completed baseline + +The [large SQLite report](../conversation-storage-replacement/verification/large-sqlite-upgrade.md) +records separate backup, cleanup, startup, streaming, and manual compaction evidence. +A 2.95 GB synthetic legacy database retained its file size after logical cleanup. +Manual compaction reduced it to 477 MB. Source records remained intact. +Both streaming runs passed without write or health-probe failures. +The SQLite catalog and runtime worker audit found no active legacy payload copies or journal workers. +The follow-up PostgreSQL tests now provide direct source, receipt, and cleanup +evidence against PostgreSQL 16.15. + +Implementation commit hooks passed, including Go lint and zero-warning web lint. +The former lint findings are resolved; they are not another pending work order. +See [the implementation hook receipt](implementation-commit-receipt.md). +Earlier desktop 6/6 and mobile 2/2 results predate the final remediation. +They are historical evidence, not proof of this final implementation. + +## Technical approach + +Task 01 adds PostgreSQL coverage using `internal/testutil.OpenIsolatedPostgres`. +Merely setting a DSN while running SQLite-only tests does not prove PostgreSQL behavior. +Task 02 isolates `TestMigrate_PriorityIdempotent` and its missing `description` column. +Task 03 identifies the exact task-service failure, which the previous summary did not retain. +Compare both failures against the recorded base before calling either pre-existing. +Fix the smallest demonstrated cause, without weakening assertions or skipping tests. +Task 04 closes the browser recovery matrix after all backend fixes. + +Keep source reads authoritative, revision checks payload-free, and receipts transient. +Keep compatibility, authorization, and loaded-range reconciliation from the existing design. +No rendered UI changes are proposed. Desktop panel and native mobile navigation remain required. + +## Tests + +Task 01 maps criteria 006.1-2, 006.5, 006.10-11 and 007.1-3 to real PostgreSQL tests. +It covers transactions, concurrent writers, pagination, cleanup rollback, and source preservation. +Task 02 owns the Office migration fixture and repeat-initialization evidence for the backend upgrade gate. +Task 03 owns the task-service package failure and its focused regression evidence. +These two gates do not establish new Office or task-service product requirements. +Each work order contains exact commands and file ownership. + +## E2E tests + +Task 04 maps criteria 005.2-3 and 006.3-10, 006.12 to desktop and mobile recovery. +Use the named plugin, prompt-history, and stream-isolation Playwright files in that order. +Test lost final delivery, reconnect, authorization loss, binding renewal, complete turn hydration, +and zero history rereads for irrelevant streaming or equal revisions. +Use deterministic fixture controls and existing unit coverage where a browser cannot inject a condition. +Record the boundary used for each scenario; do not report a unit test as browser evidence. + +## Work orders + +Recommended execution: 01, 02, 03, then 04. Only Task 04 depends on all preceding results. +A wave does not authorize delegation. The user will arrange implementation separately. + +- [x] [Task 01: PostgreSQL source and cleanup coverage](task-01-postgres-coverage.md) +- [x] [Task 02: Office migration failure](task-02-office-migration.md) +- [x] [Task 03: Task-service failure](task-03-task-service.md) +- [x] [Task 04: Desktop and mobile recovery](task-04-recovery-e2e.md) + +## Verification results + +Follow-up implementation gates completed on 2026-09-17 at code revision +`213492517315abb38697b765e91e6fe0ea5388c7`: + +- Task 01 passed all three named PostgreSQL tests on disposable PostgreSQL 16.15, + the conversation SQLite and PostgreSQL conformance checks, and SQL guard. +- Task 02 classified the Office failure as a stale `fts5` fixture and passed the + corrected targeted and full Office repository race tests. The comparison base + reproduces the missing `description` column failure. +- Task 03 passed the full task-service race package three times on the current + tree and once on the comparison base. No task-service production fix was + needed. +- Task 04 passed the final desktop 5/5 and mobile 5/5 guarded Playwright + matrices, the 41 frontend reconciliation tests, TypeScript, gateway race + tests, specification validation, and whitespace checks. +- The broad `make test` target was attempted. It still reports unrelated + `agentctl/server/process/probe`, `common/config`, and `launcher` failures in + this workspace, including home-config discovery against the existing + `/root/.kandev/config.yaml`. The Office failure seen during that run was fixed + by the fixture correction above. These unrelated failures are recorded and + are not hidden as passed follow-up gates. +- No PostgreSQL test was skipped, no production database was changed, and the + disposable PostgreSQL service was removed after verification. +- Final plan validators and diff checks were rerun after this synchronization. + +## Risks + +- PostgreSQL evidence depends on a disposable service and must never use operator data. +- The broad backend audit still has unrelated process-probe, home-config, and launcher failures in this workspace. +- Physical SQLite space is reclaimed only by explicit maintenance after backup. diff --git a/docs/plans/conversation-storage-follow-up/task-01-postgres-coverage.md b/docs/plans/conversation-storage-follow-up/task-01-postgres-coverage.md new file mode 100644 index 0000000000..88850d126e --- /dev/null +++ b/docs/plans/conversation-storage-follow-up/task-01-postgres-coverage.md @@ -0,0 +1,105 @@ +--- +id: "01-postgres-coverage" +title: "Prove PostgreSQL source and cleanup behavior" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.11 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.3 +system_design: + - ../../specs/plugins/system-design/conversation-source-reconciliation.md +--- + +# Task 01: Prove PostgreSQL source and cleanup behavior + +## Summary + +Add actual PostgreSQL tests for the replacement and run them on a disposable database. +Resolve demonstrated dialect defects while preserving the approved source and receipt contract. + +## In scope + +- Own repository PostgreSQL test coverage and any resulting source, receipt, or cleanup fixes. +- Reuse `testutil.PostgresDSNFromEnv` and `testutil.OpenIsolatedPostgres`; never target user data. +- Add named tests `TestConversationPostgresSource`, `TestConversationPostgresReceipts`, and + `TestConversationPostgresCleanup` in `conversation_source_postgres_test.go` (new). +- Cover fresh and legacy schemas, direct SQL changes, inserts/updates/deletes, cross-session moves, + turn completion, session/task cascades, atomic rollback, and revision-row bounds. +- Exercise real concurrent connections, deterministic revision locks, and complete receipt intervals. +- Cover repeatable-read pages, keyset boundaries, task/author filters, and expected-revision mismatch. +- Build legacy PostgreSQL fixtures from the comparison base's DDL, not translated SQLite SQL. +- Verify exact old tables, triggers, and functions disappear without unrelated object removal. + Inject cleanup failure, prove rollback and successful retry, and preserve source bytes and indexes. +- Run existing previous-stable upgrade conformance. Record which dialect each subtest actually uses. + +## Out of scope + +SQLite performance reruns, production upgrades, automatic compaction, and new durable storage. + +## Acceptance + +1. All three named PostgreSQL tests run without skipping and cover the listed matrix. +2. Fresh, repeated, interrupted, and legacy upgrades preserve source data and remove legacy copies. +3. Concurrent writes and consistent reads meet the existing revision/receipt contract; evidence includes + PostgreSQL version, connection concurrency, counts, command output, and tested commit without credentials. + +## Verification + +Run from the repository root after adding the named tests. Missing tests or skips fail this gate. +Provision a disposable PostgreSQL service through available local tooling, or report the missing DSN. + +```bash +: "${KANDEV_TEST_POSTGRES_DSN:?Set a disposable PostgreSQL test DSN}" +(cd apps/backend && go test -race ./internal/task/repository/sqlite -run '^TestConversationPostgres' -count=1 -v) +(cd apps/backend && go test -race ./internal/task/repository/sqlite -run '^TestConversation' -count=1) +(cd apps/backend && go test -race ./internal/persistence/storeconformance -run 'PreviousStableUpgrade|UpgradeFixtureManifest' -count=1 -v) +(cd apps/backend && go run ./cmd/sqlguard ./internal) +``` + +## Files likely touched + +- `apps/backend/internal/task/repository/sqlite/conversation_source_postgres_test.go` (new) +- `apps/backend/internal/task/repository/sqlite/conversation_source.go` +- `apps/backend/internal/task/repository/sqlite/conversation_receipts.go` +- `apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup.go` +- `apps/backend/internal/persistence/storeconformance/upgrade_test.go` +- Reuse helpers from `workspace_orphan_postgres_test.go` and `internal/testutil`. + +## Risks + +An existing PostgreSQL initialization failure can block the matrix. Isolate it against the base; +fix narrow prerequisites with regression evidence, and document any larger required design change. + +## Dependencies + +None. Run sequentially in the recommended order. + +## Parallelism + +`sequential` + +## Inputs + +- [Host requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and [source reconciliation design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [Architecture decision](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Implementation commit `c0a048bc128f7ef9a1051caf95ed442627faf9df`; comparison base `88c6c0fe0a6ae5d25332070d603b99f7d9241645`. +- [Original package](../conversation-storage-replacement/plan.md) and its recorded evidence. + +## Results + +- Added `conversation_source_postgres_test.go` with the three required named tests. They cover source-page filters and keyset paging, repeatable-read consistency, direct source updates/deletes and session moves, turn completion, cascades, transaction rollback, revision bounds, transaction-bound receipts, concurrent writers, incomplete intervals, cleanup rollback/retry, source sentinels, index preservation, and unrelated PostgreSQL objects. +- On disposable PostgreSQL 16.15, `KANDEV_TEST_POSTGRES_DSN= go test -race ./internal/task/repository/sqlite -run '^TestConversationPostgres' -count=1 -v` passed all 3/3 named tests in 5.721s. The receipt test used two repository connections for concurrent writes. +- `go test -race ./internal/task/repository/sqlite -run '^TestConversation' -count=1` passed. `go test -race ./internal/persistence/storeconformance -run 'PreviousStableUpgrade|UpgradeFixtureManifest' -count=1 -v` passed with the SQLite and PostgreSQL dynamic conformance routes. `go run ./cmd/sqlguard ./internal` passed. +- No PostgreSQL test was skipped. The database was disposable and was removed after verification. No production source or cleanup fix was required. +- Tested code revision: `213492517315abb38697b765e91e6fe0ea5388c7`. diff --git a/docs/plans/conversation-storage-follow-up/task-02-office-migration.md b/docs/plans/conversation-storage-follow-up/task-02-office-migration.md new file mode 100644 index 0000000000..c0dc1f6897 --- /dev/null +++ b/docs/plans/conversation-storage-follow-up/task-02-office-migration.md @@ -0,0 +1,84 @@ +--- +id: "02-office-migration" +title: "Resolve the Office migration test failure" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.2 +system_design: + - ../../specs/plugins/system-design/conversation-source-reconciliation.md +--- + +# Task 02: Resolve the Office migration test failure + +## Summary + +Reproduce `TestMigrate_PriorityIdempotent`, previously reported as missing the `description` column. +Determine whether the fixture is stale or the replacement changes upgrade behavior, then fix the cause. + +## In scope + +- Own the Office migration test and the smallest required migration/fixture correction. +- Capture the exact failure on the implementation head and comparison base using the same command. + Use a temporary detached worktree for the base, leaving the active checkout intact. +- Preserve the test's repeated-initialization and priority assertions. Match the intended historical + schema if fixing the fixture; do not add arbitrary columns merely to silence each failure. +- Add a failing regression first if production migration logic must change. + +## Out of scope + +Office behavior redesign, unrelated migration cleanup, task-service fixes, and test suppression. + +## Acceptance + +1. Evidence classifies the failure using both SHAs, or records why the base cannot reproduce it. +2. The targeted test and entire Office SQLite repository package pass with the race detector. +3. Initialization stays idempotent and original task values remain intact. + +## Verification + +Run this block at both the current head and comparison worktree root, retaining the failing baseline. +After remediation, run it again at the final implementation head. + +```bash +(cd apps/backend && go test -tags fts5 -race ./internal/office/repository/sqlite -run '^TestMigrate_PriorityIdempotent$' -count=1 -v) +(cd apps/backend && go test -tags fts5 -race ./internal/office/repository/sqlite -count=1) +``` + +## Files likely touched + +- `apps/backend/internal/office/repository/sqlite/migrations_priority_test.go` +- Corresponding Office migration implementation only if reproduction proves a production defect. +- `apps/backend/internal/task/repository/sqlite/base_schema.go` only if the failure originates there. + +## Risks + +Changing a historical fixture can conceal an actual upgrade regression. Preserve the intended source +schema and distinguish test setup failure from migration failure in the results. + +## Dependencies + +None. Run sequentially in the recommended order. + +## Parallelism + +`sequential` + +## Inputs + +- [Host requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and [source reconciliation design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [Architecture decision](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Implementation commit `c0a048bc128f7ef9a1051caf95ed442627faf9df`; comparison base `88c6c0fe0a6ae5d25332070d603b99f7d9241645`. +- [Original package](../conversation-storage-replacement/plan.md) and its recorded evidence. + +## Results + +- The comparison base `88c6c0fe0a6ae5d25332070d603b99f7d9241645` reproduced the failure with the tagged command: `backfill tasks FTS: no such column: description`. +- The idempotence fixture was stale for the production `fts5` migration. It defined only the priority-era columns, while the migration backfills `tasks.description` and `tasks.identifier`. Added those two historical columns to the fixture; no production migration behavior changed. +- On revision `213492517315abb38697b765e91e6fe0ea5388c7`, both exact commands passed with `-tags fts5 -race`: the targeted test and the full Office SQLite repository package. +- The idempotence test still runs both initializations and verifies that the existing `high` priority remains unchanged. diff --git a/docs/plans/conversation-storage-follow-up/task-03-task-service.md b/docs/plans/conversation-storage-follow-up/task-03-task-service.md new file mode 100644 index 0000000000..52c48170a3 --- /dev/null +++ b/docs/plans/conversation-storage-follow-up/task-03-task-service.md @@ -0,0 +1,88 @@ +--- +id: "03-task-service" +title: "Identify and resolve the task-service failure" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 +system_design: + - ../../specs/plugins/system-design/conversation-source-reconciliation.md +--- + +# Task 03: Identify and resolve the task-service failure + +## Summary + +Identify the exact failing task-service test and resolve its demonstrated cause. +The earlier full backend run recorded only the package failure, so its cause is currently unknown. + +## In scope + +- Own failure isolation and the smallest task-service correction with regression coverage. +- Capture package output and identify each failing test before editing production code. +- Reproduce the same test on the comparison base. Do not assume it is unrelated or pre-existing. +- For conversation failures, preserve persisted source values, post-commit publication, and complete + receipt intervals. Publication failures must not turn a successful durable write into a false rollback. +- If reproduction finds an unrelated defect, document the actual contract and add its existing + requirement/design references before implementing a behavior change. + +## Out of scope + +A task-service rewrite, new conversation contracts, unrelated cleanup, and weakening tests. + +## Acceptance + +1. The report names every failing test and includes head/base evidence or an explicit reproduction limit. +2. The narrow regression and full task-service package pass under the race detector. +3. Tests prove the corrected behavior; no failure is hidden through skips or reduced assertions. + +## Verification + +Run at current and comparison worktree roots. Isolate reported test names with `-run` before fixing. +Record that exact command once the failure is known, then rerun the package after remediation. + +```bash +(cd apps/backend && go test -race ./internal/task/service -count=1 -v) +``` + +If the failure cannot be reproduced, run the same command three times and record all results and +relevant environment differences. Do not claim a fix without a demonstrated change. + +## Files likely touched + +- `apps/backend/internal/task/service/*_test.go` (exact test determined by reproduction) +- `apps/backend/internal/task/service/service_messages.go` +- `apps/backend/internal/task/service/service_events.go` +- Other task-service files only as required by the isolated cause. + +## Risks + +The unknown failure may depend on timing or environment. Preserve full diagnostics and avoid speculative fixes. + +## Dependencies + +None. Run sequentially in the recommended order. + +## Parallelism + +`sequential` + +## Inputs + +- [Host requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and [source reconciliation design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [Architecture decision](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Implementation commit `c0a048bc128f7ef9a1051caf95ed442627faf9df`; comparison base `88c6c0fe0a6ae5d25332070d603b99f7d9241645`. +- [Original package](../conversation-storage-replacement/plan.md) and its recorded evidence. + +## Results + +- The current implementation head did not reproduce a task-service defect after isolating the package. `go test -race ./internal/task/service -count=1` passed in three consecutive runs (268.522s, 289.599s, and 272.722s in the recorded runs). +- The comparison base `88c6c0fe0a6ae5d25332070d603b99f7d9241645` also passed the same command (312.033s). No task-service production change was required, and no failure was hidden with a skip or reduced assertion. +- The package preserves durable-write success and post-commit publication behavior covered by its existing tests. +- Tested code revision: `213492517315abb38697b765e91e6fe0ea5388c7`. diff --git a/docs/plans/conversation-storage-follow-up/task-04-recovery-e2e.md b/docs/plans/conversation-storage-follow-up/task-04-recovery-e2e.md new file mode 100644 index 0000000000..1b53abad51 --- /dev/null +++ b/docs/plans/conversation-storage-follow-up/task-04-recovery-e2e.md @@ -0,0 +1,125 @@ +--- +id: "04-recovery-e2e" +title: "Close desktop and mobile recovery coverage" +status: done +wave: 2 +depends_on: + - "01-postgres-coverage" + - "02-office-migration" + - "03-task-service" +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-005 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-005.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-005.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.6 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.8 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.12 +system_design: + - ../../specs/plugins/system-design/conversation-source-reconciliation.md +--- + +# Task 04: Close desktop and mobile recovery coverage + +## Summary + +Prove the existing core and plugin recovery behavior after the backend work orders finish. +Extend missing deterministic scenarios and fix only defects exposed by this recovery matrix. + +## In scope + +- Own recovery E2E tests, fixture controls, and any narrowly required client/gateway fixes. +- Run core and plugin gap recovery, pagination after repair, prompt history, and stream isolation. +- Cover a dropped final change while idle, equal-revision checks, irrelevant agent streaming, + complete hydration beyond 100 turns, binding renewal/reconnect, and access revocation after readiness. +- Assert stable IDs, current turn durations, preserved loaded ranges, no deleted-row resurrection, + and no history rereads for equal revisions or fully covered irrelevant updates. +- Inspect existing unit tests first. Keep direct-SQL and timer edge cases at the deterministic + repository/client boundary when browser controls cannot express them; record this limitation. +- Add a focused mobile recovery scenario to an existing mobile spec if its current flow lacks it. + Mobile uses the Panels picker, full-height panel, touch controls, and one scroller. +- Use approved E2E helpers, no arbitrary sleeps or overlapping full suites. Read the E2E and + mobile-parity skills. Fresh worktrees need the frozen workspace install before pnpm commands. +- Record results at the final code commit and synchronize original Task 01/04/05 and companion + plan evidence without replacing historical counts. Do not mark any unrun matrix complete. + +## Out of scope + +UI redesign, plugin extraction, new auth contracts, and a generic full verification campaign. + +## Acceptance + +1. All named desktop and mobile suites pass after the final fixes, with actual counts and tested SHA. +2. The scenario matrix maps each required recovery condition to passing browser or explicitly named + lower-level evidence; mobile has real recovery evidence rather than only panel placement. +3. The original package's remaining gates are reconciled accurately; unresolved failures remain open. + +## Verification + +Run from the repository root. Run the browser commands sequentially with the guarded runner. + +```bash +(cd apps && pnpm install --frozen-lockfile) +(cd apps/web && pnpm e2e:run --project chromium tests/plugins/conversation-recovery.spec.ts tests/task/prompt-history-panel.spec.ts tests/session/session-stream-overload-isolation.spec.ts) +(cd apps/web && pnpm e2e:run --project mobile-chrome tests/plugins/mobile-prompt-history-plugin.spec.ts tests/task/mobile-prompt-history-panel.spec.ts tests/session/mobile-session-stream-overload-isolation.spec.ts) +(cd apps/web && pnpm exec vitest run lib/plugins/conversation-source-scope.test.tsx lib/plugins/conversation-reconciliation.test.ts lib/ws/client.test.ts) +(cd apps/web && pnpm run typecheck) +(cd apps/backend && go test -race ./internal/gateway/websocket -count=1) +python3 scripts/list-docs.py validate +python3 scripts/lint-spec-files.py --all +git diff --check +``` + +If fixes touch additional tests, add their exact commands to Results and run them too. +A later production-code change invalidates relevant earlier final-head evidence; rerun affected checks. + +## Files likely touched + +- `apps/web/e2e/tests/plugins/conversation-recovery.spec.ts` +- `apps/web/e2e/tests/plugins/mobile-prompt-history-plugin.spec.ts` +- `apps/web/e2e/tests/task/prompt-history-panel.spec.ts` +- `apps/web/e2e/tests/task/mobile-prompt-history-panel.spec.ts` +- `apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts` +- `apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts` +- `apps/web/e2e/helpers/ws-drop.ts` and existing fixture controls. +- `apps/web/lib/plugins/conversation-source-scope.ts`, `conversation-scope.tsx`, and associated tests. +- `apps/backend/internal/gateway/websocket/conversation_delivery.go` if a reproduced defect requires it. + +## Risks + +Long-lived authorization and binding tests can become slow or flaky. Use causal controls and bounded +fake-clock tests for timing mechanics, plus browser evidence for the resulting user-visible recovery. + +## Dependencies + +01-postgres-coverage, 02-office-migration, 03-task-service + +## Parallelism + +`sequential` + +## Inputs + +- [Host requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and [source reconciliation design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [Architecture decision](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Implementation commit `c0a048bc128f7ef9a1051caf95ed442627faf9df`; comparison base `88c6c0fe0a6ae5d25332070d603b99f7d9241645`. +- [Original package](../conversation-storage-replacement/plan.md) and its recorded evidence. + +## Results + +- The final guarded Docker desktop command passed 5/5 tests: core recovery (2), plugin recovery with pagination (1), session stream isolation (1), and Prompt History panel behavior (2). An earlier full-matrix run had one transient plugin-panel timeout; its isolated rerun and the final full matrix both passed. +- The final guarded Docker mobile command passed 5/5 tests: fixture-plugin Host navigation (1), native session-sheet stream isolation (1), Prompt History panel touch navigation (1), long-history touch loading (1), and cancellation after leaving Chat (1). +- The focused frontend Vitest command passed 41/41 tests. `pnpm run typecheck` passed. `go test -race ./internal/gateway/websocket -count=1` passed, including the synthetic-client authorization regression and task-event receipt projection regression. Specification catalog validation, full specification lint, and `git diff --check` passed. +- The browser helper now drops the v2 source-change notification for the targeted prompt, so recovery tests exercise the source reconciliation path while preserving compatibility frames. +- No layout, navigation, scrolling, or touch behavior changed. The mobile tests use the existing Panels picker, full-height panel, and single scroller. +- Tested code revision: `213492517315abb38697b765e91e6fe0ea5388c7`. diff --git a/docs/plans/conversation-storage-replacement/plan.md b/docs/plans/conversation-storage-replacement/plan.md new file mode 100644 index 0000000000..f4d6a2bbb7 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/plan.md @@ -0,0 +1,283 @@ +--- +created: 2026-09-16 +status: complete +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-005 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +system_design: + - ../../specs/plugins/system-design/conversation-source-reconciliation.md +legacy_specs: [] +--- + +# Implementation Plan: Conversation storage replacement + +## Overview + +Replace mandatory conversation copies with authorized reads of original messages and turns. +Keep the useful browser Host API and core Prompt History panel. +Deliver compatible readers first, migrate plugin and core consumers, then remove legacy writers and storage. +Ship the package as one release. Intermediate dual-path work orders are not release candidates. + +The user requested and authorized implementation of this design package on 2026-09-16. +The user later authorized committing the implementation and this follow-up plan. +Production database mutation remains outside the package. Branch delivery is +handled by the current implementation handoff. + +## Inputs and authority + +- [Requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md): new requirements 006 and 007 replace retired transport criteria 002.13, 002.15, and 002.16. +- [Design](../../specs/plugins/system-design/conversation-source-reconciliation.md) and [ADR](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- [Original package](../prompt-history-plugin-host/plan.md) and [recovery package](../pr-3588-conversation-recovery/plan.md) retain historical results. +- Source baseline: `88c6c0fe0a6ae5d25332070d603b99f7d9241645`. PR #3588 merged as `2b1d0cf7d1e118dbf7b57ce5243131bb49b9dae4`. +- Before implementation, resolve current main and recheck affected symbols and migrations. Do not treat this baseline as permanent. + +The plugin system owns the Host access contract. The task repository owns original records and atomic revision changes. +The selected design retains only current state, not durable delivery of intermediate versions. +The user selected incremental updates without hashes on 2026-09-16. +This revision replaces the earlier refresh-on-change approach. Cleanup requirements and Task 04 remain applicable. +The facade recovers deleted or inaccessible previously loaded scopes as terminal unavailable state without revealing which cause occurred. + +## Scope + +### In scope + +- Revision-only metadata, transaction-bound mutation receipts, source reads, and authorized incremental subscriptions. +- Plugin and core recovery, including direct SQL mutation and missed-event repair. +- Removal of copied payloads, obsolete triggers, event mirrors, ACK/poison persistence, and startup backfill. +- Safe SQLite/PostgreSQL upgrade, specific Host-file cleanup, storage evidence, and documentation. + +### Out of scope + +- External plugin implementation, publication, and a plugin-owned transcript archive. +- Core Prompt History removal, saved-layout migration, and UI redesign. +- Generic replay infrastructure, exactly-once delivery, historical snapshots, new feature flags, and automatic compaction. +- Message content hashes, hash backfill, and hashing on streaming writes. + +## Technical approach + +Task 01 adds source readers and revision metadata behind a private v2 route contract. +Task 02 moves plugin scopes to live changes by ID and discrepancy-driven recovery. +Task 03 migrates core delivery and repairs original message and turn caches. +Task 04 removes journal initialization before backfill can run, then removes legacy database objects and Host event files. +Task 05 proves parity and updates the API and operational reference. + +Exact components, cursor fields, transaction semantics, failure behavior, and migration order are defined in the system design. +A normal mutation updates only source records and one small revision per affected session for this feature. +The gateway checks only subscribed sessions. Complete change batches carry scoped current records or removals by ID. +Irrelevant changes carry coverage only and trigger no history reads. +Fully applied revisions remain separate from observed revisions. A counter check alone cannot advance applied state. +Normal updates use in-memory projection. Initial load, pagination, and discrepancy recovery use bounded source pages. + +## Tests + +New test names are implementation targets, not existing proof. +Each work order must record its actual RED failure and final GREEN commands. + +| Criteria | Target evidence | Owner | +| --------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----- | +| 002.1, 002.4, 002.7, 006.1-2, 006.5, 006.8 | `TestConversationSourcePages`, `TestConversationRevisionAtomicity`, and PostgreSQL variants in new `conversation_source*_test.go` | 01 | +| 002.2-3, 002.5-6, 002.9-10, 002.14, 002.17, 006.3-8 | Existing Host test files: revision race, loaded-range deletion, churn, expiry, access loss, independent panels, teardown | 02 | +| 006.3-4, 006.6, 006.8 | `TestConversationDeliveryRepairsMissedPublication`, `TestConversationDeliveryIdleHasNoReads`, authorization/slow-client variants | 02 | +| 002.5-6, 002.9, 005.3, 006.3-4, 006.6-7 | `use-session-recovery.test.tsx`, `client.test.ts`, gateway tests: rich state, optimistic messages, missed events, restart | 03 | +| 006.1-2, 006.8, 007.1-4 | `TestConversationJournalCleanupPreservesSources`, failure/replay/partial-schema/PostgreSQL variants and Host file cleanup tests | 04 | +| 006.1-2, 006.8 | `BenchmarkConversationSourceStorage`, 10k/100k rows, one session then multiple sessions | 04 | +| 006.9-12 | `TestConversationRevisionReceiptMatchesCommit`, complete multi-write receipt/reset, and concurrent-writer tests in source tests | 01 | +| 006.9-12 | Host/gateway tests: 100 agent updates with zero prompt rereads, matching ID updates, coverage-only batches, pagination after applied revisions | 02 | +| 006.10 | Drop 10-to-11, deliver 11-to-12, then report revision 12. Assert applied revision stays 10 until repair | 02 | +| 006.9-12 | Core tests: matching updates without hydration, turn completion, duplicate/out-of-order batches, and reset recovery | 03 | +| 005.1-3, 006.3-12, 007.4 | SDK tests, rendered fixtures, public-doc validators, spec validation | 05 | + +All abbreviated criteria use prefix `AC-PLUGINS-PROMPT-HISTORY-HOST-`. +PostgreSQL commands require a disposable `KANDEV_TEST_POSTGRES_DSN`. Skipped PostgreSQL tests do not satisfy the work order. +The migration matrix includes fresh, fully migrated, interrupted backfill, missing legacy objects, repeat boot, rollback, and source mutations after cleanup. + +## E2E tests + +All tests use managed fresh production builds and isolated fixtures. + +| File and project | Required outcome | Criteria | +| ------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | ----------------------------------- | +| `tests/plugins/conversation-recovery.spec.ts`, chromium | Plugin and core recover dropped events and current-state rebind without remounting | 002.5-6, 002.9, 006.3-6 | +| `tests/plugins/prompt-history-plugin.spec.ts`, chromium | Public SDK pagination, privacy, alias/favorite display, native navigation, and terminal state | 002.1, 002.4, 002.7, 005.2-3, 006.7 | +| `tests/task/prompt-history-panel.spec.ts`, chromium | Core ownership and current visible behavior remain valid | 005.3, 006.7 | +| `tests/plugins/mobile-prompt-history-plugin.spec.ts`, mobile-chrome | Touch panel entry, older-page loading, reconnect repair, alias display, and Chat navigation | 005.2-3, 006.3, 006.7 | +| `tests/session/session-stream-overload-isolation.spec.ts`, chromium, and its `mobile-` sibling, mobile-chrome | A slow stream cannot block another session and recovery stays bounded | 006.6, 006.8 | + +Inject changes between readiness/read, between pages, and before refresh commit through causal fixture controls. +Include equal timestamps, nullable task selection, two panels, deleted page boundaries, direct SQL edits, and no installed plugin. +After hydration, assert zero history rereads during 100 ordinary agent updates to a prompt-only panel. +Then drop one relevant change, deliver a later revision, and assert source repair before the view claims freshness. +Cover empty coverage batches, filter entry/exit, pagination after applied changes, concurrent commit/publication ordering, and bounded-buffer overflow. +These scenarios cover AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 through AC-PLUGINS-PROMPT-HISTORY-HOST-006.12. +Do not use elapsed sleeps as correctness evidence. Assert visible state after controlled mutation and recovery completion. + +## Mobile parity + +This package changes data synchronization only. It does not change rendered layout, controls, scrolling ownership, or navigation adapters. +The existing mobile panel is the exemplar: Panels entry, inset picker, full-height content, and one vertical scroller. +Existing touch expansion and Chat navigation remain covered in the mobile fixture. +No new ASCII UI preview is needed because no rendered UI change is proposed. + +## Work orders + +- [x] [Task 01: Add revision-backed source reads](task-01-source-reads.md) +- [x] [Task 02: Switch plugin scopes to source reconciliation](task-02-plugin-reconciliation.md) +- [x] [Task 03: Migrate core conversation delivery](task-03-core-reconciliation.md) +- [x] [Task 04: Remove legacy journal storage safely](task-04-retire-journal.md) +- [x] [Task 05: Prove parity and publish the contract](task-05-parity-and-docs.md) + +Execution order is 01 -> 02 -> 03 -> 04 -> 05. All work is sequential. +Shared schema and transport changes prevent independent release or parallel execution. + +## Verification results + +Initial design and incremental-update refinement validation passed on 2026-09-16: + +- `python3 scripts/list-docs.py validate`: 280 decisions and 960 specifications validated. +- `python3 scripts/lint-spec-files.test.py`: 36 tests passed. +- `python3 scripts/lint-spec-files.py --all`: passed. +- `git diff --check`: passed. +- All five work orders have valid requirement IDs, design links, and existing source paths. +- Catalog queries discover the replacement design and ADR. +- `git status --short -- docs/plans/conversation-storage-replacement` confirms the new package files are untracked and uncommitted. + Implementation, migration, performance, and E2E evidence completed on 2026-09-16: + +- Backend source, receipt, gateway, plugin, test-harness, and service checks passed, including targeted race runs. +- Frontend source reconciliation, core v2 delivery, Host facade, and isolation tests passed: 64 tests. +- Desktop recovery/parity coverage passed: 6 tests after the source-change harness correction. +- Mobile prompt-history and stream-isolation coverage passed: 2 tests. +- Plugin SDK tests and typecheck passed. +- Public docs, specification, i18n, SQL guard, formatting, and diff checks passed. +- The broad `GOCACHE=/tmp/kandev-go-cache make -C apps/backend test` target was attempted. It reported the pre-existing Office migration fixture failure `TestMigrate_PriorityIdempotent` (`description` is absent) and a separate failure in `internal/task/service`; focused conversation-related package checks passed. +- PostgreSQL variants were not run at this historical checkpoint because this workspace had no `KANDEV_TEST_POSTGRES_DSN`; the follow-up package now records their successful disposable-database execution. + Exact product commands are in each work order. Do not treat design lint as implementation verification. + +## Risks + +- Removing the journal before all callers migrate can break core Chat updates. +- Repeated delivery loss can still require expensive loaded-range recovery. + Use direct updates for normal delivery. Bound recovery and record read counts before release. +- One-time DROP and the existing startup backup can still take time on large databases. + This package removes payload backfill, not every source of startup I/O. +- Physical database shrink is separate from logical cleanup. +- Old browser wire state requires a full page reload. Mixed backend versions cannot write during cutover. +- File cleanup follows primary commit and needs independent idempotent retry. + +## Completion and documentation + +Keep all work orders pending until their tests pass. +At completion, record commands/results and mark the paired replacement design current. +Keep requirements active. Do not label requirement documents shipped. +Preserve historical companion results and link new evidence instead of overwriting old counts. +Update `PLUGIN-API.md`, public plugin authoring/manifest reference, and System database maintenance guidance during Task 05. +The implementation portion and the required follow-up gates are complete. The +public pages and active system design were updated during Task 05. Historical +companion plans retain their original context and now link to the replacement +package. The broad `make test` audit still reports unrelated process-probe, +home-config, and launcher failures in this workspace; those failures remain +explicitly recorded in the follow-up package rather than being attributed to +the conversation storage change. + + +## Review remediation + +The user authorized fixes for four review findings and requested no commit. + +- Recheck session access, active user status, plugin capability, and installation generation before delivery. Close revoked subscriptions with a terminal notification. +- Check subscribed source revisions every five seconds. Clients wait one second for pending updates before recovery. Equal revisions cause no history reads. Stop checks when subscriptions close. +- Renew short-lived plugin bindings before expiry, share concurrent renewal requests, and retry failed acquisition. +- Load every turn page at one revision before publishing hydrated history. Keep partial results private if a continuation fails. + +Focused remediation verification is recorded below. The existing PostgreSQL and broader verification gates remain unchanged. No commit was created. + +### Review remediation verification + +- Gateway: `go test -race ./internal/gateway/websocket` passed. `golangci-lint run ./internal/gateway/websocket/...` passed with zero issues after extracting subscription validation helpers. +- Frontend: the seven focused source-scope, reconciliation, Host facade, isolation, WebSocket, recovery, and message-handler files passed (91 tests). +- After the final test-only constant extraction, `pnpm exec vitest run lib/plugins/conversation-source-scope.test.tsx` passed (11 tests). +- `pnpm run typecheck` passed. Focused ESLint checks passed without errors; the reported duplicate-string warning was removed and the affected file passed again. +- `python3 scripts/list-docs.py validate`, `python3 scripts/lint-spec-files.py --all`, and `git diff --check` passed. +- These fixes change state and transport behavior only. No layout, navigation, scrolling, or touch behavior changed. Targeted tests cover both viewport-independent clients. Browser E2E and PostgreSQL checks were not rerun for this remediation. +- At this stage, all edits remained uncommitted, as requested. + + +## Large legacy SQLite verification + +The requested large-database verification passed on a 2.95 GB synthetic legacy +database. Backup, cleanup, task-repository initialization, and HTTP readiness +were measured separately. Both 60-second streaming runs completed without +write or readiness/probe failures. Active SQLite triggers, production workers, +and retired host files were audited. Manual compaction and subsequent readiness +passed; no automatic startup VACUUM was added. + +See [the verification report](verification/large-sqlite-upgrade.md) for workload, +commands, measurements, evidence, and limits. The public operations guide now +includes explicit post-upgrade compaction. PostgreSQL and other existing plan +gates remained open at this historical checkpoint. The changes were later +committed through the follow-up delivery work. + + +## Commit preparation + +The six reported SQLite-package lint findings were addressed with constant +reuse, moving unchanged trigger SQL to a package constant, and flattening one +error branch. `go test -race ./internal/task/repository/sqlite -run +'^TestConversation' -count=1` passed. The remaining PostgreSQL, backend failure, +and recovery E2E work will be tracked in a separate follow-up package. + + +## Committed implementation and remaining work + +Implementation is committed as `c0a048bc128f7ef9a1051caf95ed442627faf9df`. +Normal commit hooks passed, including Go lint and zero-warning web lint. +Earlier uncommitted and lint-failure notes describe their historical verification phase. + +The [remaining-gates package](../conversation-storage-follow-up/plan.md) owns +the completed PostgreSQL coverage, Office migration fixture correction, +task-service comparison, and desktop/mobile recovery checks after final +remediation. All package work orders now have recorded results. The broad +backend audit has unrelated failures in this workspace and is not presented as +conversation-storage evidence. +No automatic startup VACUUM was added. Physical compaction remains explicit. + +## Final review-fixup verification + +The delivery review follow-up closed the remaining implementation findings: + +- Source reads now report stale cursors as reconciliation conflicts, classify + session lookup failures as retryable upstream errors, and route turn reads + through the message repository source. +- Live source delivery rechecks the service under the hub lock, projects core + entities at the API boundary, and does not reset a conversation for a + receiptless session removal. +- Receipt publication projects transient entities, pending tool completion is + transaction-bound to turn completion, and PostgreSQL cleanup removes the + retired helper functions. Legacy cleanup runs before token-key loading. +- Plugin reconciliation preserves complete-session updates, retries transient + recovery failures, recognizes deleted pagination boundaries, and collapses + bounded pending changes into one recovery when the buffer limits are reached. +- The generated E2E fixture bundle and historical work-order references were + synchronized with their source files. + +The focused changed-package Go suite, Go lint, frontend reconciliation tests, +TypeScript, focused ESLint, Prettier, specification validators, and whitespace +checks passed after these changes. Browser E2E and PostgreSQL checks remain +CI-controlled gates for the delivery branch. + +## Live projection regression verification + +The delivery branch exposed two live core projection regressions during the +browser matrix: completed turns did not carry the runtime model snapshot, and +projected shell-output summaries were treated as empty output on a second +metadata projection. The fix keeps the core turn metadata allowlisted, carries +the completion output bit through the source receipt, and makes shell-output +projection idempotent. Plugin projections remain metadata-free. + +The focused backend models, task service, and gateway tests passed. The web +WebSocket client tests, TypeScript, focused ESLint, backend build, E2E fixture +build, and plugin package build passed. Desktop Chromium passed all 13 focused +chat cases, including model-selector recovery and shell-output disclosure. +Mobile Chromium passed the focused empty-turn case. Backend lint reported zero +issues and `git diff --check` passed. diff --git a/docs/plans/conversation-storage-replacement/task-01-source-reads.md b/docs/plans/conversation-storage-replacement/task-01-source-reads.md new file mode 100644 index 0000000000..addc68ef96 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/task-01-source-reads.md @@ -0,0 +1,118 @@ +--- +id: "01-source-reads" +title: "Add revision-backed source reads" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.8 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.11 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.12 +system_design: + - "../../specs/plugins/system-design/conversation-source-reconciliation.md" +--- + +# Task 01: Add revision-backed source reads + +## Summary + +Add revision-only schema and transactional source readers behind explicit Host-only v2 requests. +Keep the existing callers operational until their migration. This intermediate work order is not a release. + +## In scope + +- Own revision schema, both database dialects, message/turn page reads, and service interfaces. +- Capture transient mutation receipts with base/final revision and persisted results inside the source transaction. +- Cover normal creation, streaming append, tool, permission/clarification, and turn writers. Incomplete bulk coverage must return reset. +- Keep keyset cursor identity separate from expected applied revision. Add no content hashes or per-message version storage. +- Add v2 page/revision routes with existing authorization, sanitization, signed query binding, and private reconciliation errors. +- Prove updates, deletion cascades, session moves, rollback, concurrent writes, and absent revision row semantics. + +## Out of scope + +- Plugin/core caller migration and legacy cleanup. +- New public SDK signatures or a new external plugin. + +## Acceptance + +- Source-backed pages and revision checks pass privacy, ordering, filter, expiry, and concurrent-mutation tests. +- Schema setup does not read historical payloads. Revision rows remain bounded by session count. +- SQLite and real PostgreSQL tests prove atomic source reads, transaction-bound receipts, concurrent writers, and replayable setup. + +## Verification + +Run from the repository root. Obtain behavioral RED evidence before production edits. + +```bash +(cd apps/backend && go test -race ./internal/task/repository/sqlite ./internal/task/service ./internal/plugins -count=1) +: "${KANDEV_TEST_POSTGRES_DSN:?Set a disposable PostgreSQL test DSN}" +(cd apps/backend && go test -race ./internal/task/repository/sqlite ./internal/plugins -run 'ConversationSource|ConversationRevision' -count=1) +(cd apps/backend && go run ./cmd/sqlguard ./internal) +``` + +## Files likely touched + +- `apps/backend/internal/task/repository/sqlite/conversation_source.go (new)` +- `apps/backend/internal/task/repository/sqlite/conversation_source_test.go (new)` +- `apps/backend/internal/task/repository/sqlite/conversation_source_postgres_test.go (new)` +- `apps/backend/internal/task/repository/sqlite/base_schema.go` +- `apps/backend/internal/task/repository/sqlite/message.go` +- `apps/backend/internal/task/repository/sqlite/message_payload_replay.go` +- `apps/backend/internal/task/models/models.go` +- `apps/backend/internal/task/repository/interface.go` +- `apps/backend/internal/task/service/service_messages.go` +- `apps/backend/internal/task/service/service_turns.go` +- `apps/backend/internal/plugins/conversation_handlers.go` +- `apps/backend/internal/plugins/conversation_tokens.go` +- `apps/backend/internal/plugins/conversation_source_handlers_test.go (new)` + +## Dependencies + +None. + +## Risks + +- Do not use PostgreSQL READ COMMITTED across separate row and revision queries. +- Sampling the revision after commit can falsely certify an older payload as current. Capture it inside the mutation transaction. +- Avoid JavaScript precision loss and deadlocks when mutations affect several sessions. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and the frontmatter criteria. +- [System design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [ADR](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Existing conversation handler, journal, Host facade, and recovery tests provide the regression patterns. + +## Results + +- Added revision-only source metadata and SQLite/PostgreSQL trigger setup with bounded source page readers. +- Added transaction-bound receipts for message creation, updates, deletion, turn creation, and turn completion. +- Added v2 page and revision routes with signed cursors, authorization, sanitization, decimal revision strings, and private mismatch errors. +- Integrated message and turn service publication with receipt data while preserving compatibility callers. +- `GOCACHE=/tmp/kandev-go-cache go test -race ./internal/plugins ./internal/gateway/websocket ./internal/office/testharness -count=1`: passed. +- `GOCACHE=/tmp/kandev-go-cache go test -race ./internal/task/repository/sqlite -run 'Conversation|MessagePayload|PromptIndex' -count=1`: passed. +- `GOCACHE=/tmp/kandev-go-cache go test ./internal/task/service -run 'TestPublishMessageEvent|Conversation' -count=1`: passed. +- `GOCACHE=/tmp/kandev-go-cache go run ./cmd/sqlguard ./internal`: passed. +- Focused source, receipt, route, and gateway tests passed. +- The follow-up added the required PostgreSQL source, receipt, and cleanup variants. On disposable PostgreSQL 16.15, all three named PostgreSQL tests passed with the race detector; the SQLite/PostgreSQL upgrade conformance checks and SQL guard also passed. See the [follow-up Task 01 results](../conversation-storage-follow-up/task-01-postgres-coverage.md). + +## Remaining-gate handoff + +The follow-up package records the final PostgreSQL and related delivery evidence. +Existing results remain historical. diff --git a/docs/plans/conversation-storage-replacement/task-02-plugin-reconciliation.md b/docs/plans/conversation-storage-replacement/task-02-plugin-reconciliation.md new file mode 100644 index 0000000000..e4a9ef525b --- /dev/null +++ b/docs/plans/conversation-storage-replacement/task-02-plugin-reconciliation.md @@ -0,0 +1,133 @@ +--- +id: "02-plugin-reconciliation" +title: "Switch plugin scopes to source reconciliation" +status: done +wave: 2 +depends_on: + - "01-source-reads" +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.6 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.14 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.17 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.6 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.8 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.11 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.12 +system_design: + - "../../specs/plugins/system-design/conversation-source-reconciliation.md" +--- + +# Task 02: Switch plugin scopes to source reconciliation + +## Summary + +Implement the v2 incremental subscription and migrate the browser Host facade. +Preserve current public shapes and lifecycle outcomes. Core migration follows in Task 03. + +## In scope + +- Own gateway v2 registration, scoped change batches, coverage-only batches, revision checks, authorization, and slow-subscriber reset. +- Replace durable replay with ID upsert/removal and complete contiguous revision coverage. Use source repair only for discrepancies or lifecycle recovery. +- Preserve loaded range, join loadMore calls, apply message/turn changes atomically, and fence late responses. Pagination continues after fully applied revisions. +- Extend existing fixture recovery E2E with a plugin missed-notification and concurrent-pagination scenario. + +## Required regression cases + +- Deliver 100 ordinary agent updates to a prompt-only scope. Assert zero history rereads after initial hydration. +- Apply a matching update and turn completion by ID. Assert no bulk refresh and no duplicate records. +- Drop interval 10-to-11 and deliver 11-to-12. A revision check at 12 must not advance applied state past 10. +- Deliver irrelevant changes as empty coverage batches without exposing entity IDs or content. +- Cover duplicate, reordered, partially overlapping, malformed, and oversized batches with bounded recovery. +- Page after fully applied changes without restarting loaded history. Fence events arriving during the page read. +- Cover filter entry/exit and session moves, including removal from the previous selection. + +## Out of scope + +- Core consumer migration, legacy storage cleanup, plugin extraction, and rendered layout changes. + +## Acceptance + +- A plugin-only fixture applies ordinary changes by ID with zero history rereads, including unrelated agent streaming and turn completion. +- Authorization, task filters, teardown, stale generations, and terminal state pass existing and new regressions. +- Gap, malformed batch, direct SQL, reconnect, and expiry tests repair state without falsely advancing applied revision. Idle checks read no payloads. + +## Verification + +Run from the repository root. Obtain behavioral RED evidence before production edits. + +```bash +(cd apps/backend && go test -race ./internal/plugins ./internal/gateway/websocket -count=1) +(cd apps/web && pnpm exec vitest run lib/plugins/conversation-host.test.tsx lib/plugins/conversation-host-isolation.test.tsx) +(cd apps/web && pnpm run typecheck) +(cd apps && pnpm --filter @kandev/plugin-sdk typecheck) +(cd apps/web && pnpm e2e:run --project chromium tests/plugins/conversation-recovery.spec.ts) +``` + +Before the first pnpm command in a fresh worktree, run `(cd apps && pnpm install --frozen-lockfile)`. +Managed E2E rebuilds the application and fixture. Run desktop and mobile sequentially. + +## Files likely touched + +- `apps/backend/internal/gateway/websocket/client.go` +- `apps/backend/internal/gateway/websocket/hub.go` +- `apps/backend/internal/gateway/websocket/conversation_delivery.go (new)` +- `apps/backend/internal/gateway/websocket/conversation_delivery_test.go (new)` +- `apps/backend/internal/backendapp/helpers.go` +- `apps/web/lib/plugins/conversation-host.tsx` +- `apps/web/lib/plugins/conversation-scope.tsx` +- `apps/web/lib/plugins/conversation-event-projection.ts` +- `apps/web/lib/plugins/conversation-host.test.tsx` +- `apps/web/lib/plugins/conversation-host-isolation.test.tsx` +- `apps/web/e2e/tests/plugins/conversation-recovery.spec.ts` + +## Dependencies + +01-source-reads. + +## Risks + +- A latest-page refresh alone leaves older loaded deletions stale. +- Do not turn malformed or unauthorized tokens into infinite recovery loops. +- Keep the source revision as consistency authority. Timestamps alone do not order writes. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and the frontmatter criteria. +- [System design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [ADR](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Existing conversation handler, journal, Host facade, and recovery tests provide the regression patterns. + +## Results + +- Migrated plugin scopes to the source-backed v2 subscription with decimal revision reconciliation, bounded buffering, source-page recovery, and terminal removal handling. +- Preserved public Host hook shapes, task/filter semantics, loaded ranges, paging, independent panel scopes, and sanitized DTOs. +- Added source reconciliation tests for matching updates, revision gaps, recovery resubscription, coverage, malformed input, and terminal lifecycle. +- Added a managed desktop fixture that drops one plugin source change, verifies a fresh subscription, restores the missed message, and continues paging without remounting. +- `go test -race ./internal/plugins ./internal/gateway/websocket ./internal/office/testharness`: passed. +- Targeted frontend Vitest, TypeScript, ESLint, and managed desktop/plugin recovery checks: passed. + + +## Review remediation + +Binding renewal, failed-binding retry, complete turn pagination, terminal authorization removal, and revision-observation handling were fixed. The source-scope regressions pass (11 tests). + +See [the plan](plan.md#review-remediation-verification) for commands, results, and remaining package gates. No commit was created. diff --git a/docs/plans/conversation-storage-replacement/task-03-core-reconciliation.md b/docs/plans/conversation-storage-replacement/task-03-core-reconciliation.md new file mode 100644 index 0000000000..be9bb7c6fa --- /dev/null +++ b/docs/plans/conversation-storage-replacement/task-03-core-reconciliation.md @@ -0,0 +1,125 @@ +--- +id: "03-core-reconciliation" +title: "Migrate core conversation delivery" +status: done +wave: 3 +depends_on: + - "02-plugin-reconciliation" +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-005 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.6 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-005.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.6 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.11 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.12 +system_design: + - "../../specs/plugins/system-design/conversation-source-reconciliation.md" +--- + +# Task 03: Migrate core conversation delivery + +## Summary + +Remove the core dependency on journal replay while preserving ordinary live Chat updates. +Use contiguous change batches for normal delivery and connect discrepancy repair to the existing hydration coordinator and prove recovery in the rendered application. + +## In scope + +- Own core subscription, ID-based message/turn projection, applied-versus-observed revisions, and discrepancy-only recovery hydration. +- Migrate normal publication to transaction-bound receipts and project each change once. Do not run legacy and v2 payload delivery in parallel. +- Reject stale v1 subscriptions with an explicit error after cutover. Test that a full reload restores current transport. +- Preserve pending local messages, rich DTOs, cached-window invalidation, task/session selection, and terminal deletion. +- Replace journal-specific E2E assertions with visible recovery outcomes while preserving the old regression scenarios. + +## Out of scope + +- Legacy table/file cleanup and any UI layout, touch, or navigation redesign. + +## Acceptance + +- Mounted idle Chat and Prompt History recover current messages and turns after a missed event or backend restart. +- Contiguous streaming updates and turn completion apply by ID without broad hydration. Recovery preserves optimistic state and discards covered batches. +- Core and plugin views recover independently without cross-cache mutation or duplicate normal notifications. + +## Verification + +Run from the repository root. Obtain behavioral RED evidence before production edits. + +```bash +(cd apps/backend && go test -race ./internal/gateway/websocket ./internal/task/handlers ./internal/task/service ./internal/orchestrator -count=1) +(cd apps/web && pnpm exec vitest run lib/ws/client.test.ts hooks/domains/session/use-session-recovery.test.tsx lib/ws/handlers/messages.test.ts) +(cd apps/web && pnpm run typecheck) +(cd apps/web && pnpm e2e:run --project chromium tests/plugins/conversation-recovery.spec.ts tests/task/prompt-history-panel.spec.ts tests/session/session-stream-overload-isolation.spec.ts) +``` + +Before the first pnpm command in a fresh worktree, run `(cd apps && pnpm install --frozen-lockfile)`. +Managed E2E rebuilds the application and fixture. Run desktop and mobile sequentially. + +## Files likely touched + +- `apps/backend/internal/gateway/websocket/ordered_session_events.go` +- `apps/backend/internal/gateway/websocket/task_notifications.go` +- `apps/backend/internal/gateway/websocket/client.go` +- `apps/backend/internal/task/handlers/message_handlers.go` +- `apps/backend/internal/task/handlers/task_http_handlers.go` +- `apps/backend/internal/orchestrator/event_handlers_agent.go` +- `apps/backend/internal/task/service/service_messages.go` +- `apps/backend/internal/task/service/service_turns.go` +- `apps/web/lib/ws/client.ts` +- `apps/web/lib/ws/ordered-session-events.ts` +- `apps/web/hooks/domains/session/use-session-messages.ts` +- `apps/web/hooks/domains/session/use-session-turns.ts` +- `apps/web/hooks/domains/session/use-session-turns-hydration.ts` +- `apps/web/hooks/domains/session/use-session-message-fetch.ts` +- `apps/web/hooks/domains/session/use-session-recovery.test.tsx` +- `apps/web/e2e/tests/plugins/conversation-recovery.spec.ts` + +## Dependencies + +02-plugin-reconciliation. + +## Risks + +- Journal removal can otherwise suppress the only normal message publisher. +- A later revision check must not certify an unapplied earlier message. Test gaps independently of payload projection. +- Core hydration must not await the same readiness barrier that its own completion satisfies. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and the frontmatter criteria. +- [System design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [ADR](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Existing conversation handler, journal, Host facade, and recovery tests provide the regression patterns. + +## Results + +- Migrated core conversation delivery to v2 source subscriptions with ID-based rich message/turn projection and revision-gated hydration recovery. +- Retained optimistic messages, terminal removal, bounded pending changes, and existing core panel behavior while removing the old ordered journal runtime from production composition. +- Added client tests for matching updates, gaps, malformed/reset changes, recovery, and v2 transport selection. +- `go test -race ./internal/gateway/websocket`: passed. +- Targeted frontend Vitest and the managed core recovery, prompt-history panel, and stream-isolation desktop checks: passed. +- The follow-up reproduced the reported task-service package condition on the comparison base and current tree; both passed the full race package, with three current-tree repetitions. The gateway and frontend recovery gates also pass. See the [follow-up Task 03 results](../conversation-storage-follow-up/task-03-task-service.md). + + +## Review remediation + +Added the five-second subscribed-session revision worker and delivery authorization rechecks. Core revision observations wait one second for pending changes, and stop after unsubscribe. Gateway race tests and core client regression tests pass. + +See [the plan](plan.md#review-remediation-verification) for the historical +remediation commands and results. The follow-up package records final delivery +evidence. diff --git a/docs/plans/conversation-storage-replacement/task-04-retire-journal.md b/docs/plans/conversation-storage-replacement/task-04-retire-journal.md new file mode 100644 index 0000000000..f43b990c91 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/task-04-retire-journal.md @@ -0,0 +1,132 @@ +--- +id: "04-retire-journal" +title: "Remove legacy journal storage safely" +status: done +wave: 4 +depends_on: + - "03-core-reconciliation" +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.8 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.4 +system_design: + - "../../specs/plugins/system-design/conversation-source-reconciliation.md" +--- + +# Task 04: Remove legacy journal storage safely + +## Summary + +Remove the obsolete runtime and add the forward cleanup migration. +This work order establishes the final single-source storage contract for both fresh and upgraded installations. + +## In scope + +- Own atomic trigger/table cutover, PostgreSQL function removal, startup ordering, migration cancellation, and failure propagation. +- Remove journal readers, mirrors, durable ACK/poison APIs, maintenance workers, composition wiring, and retired wire types. +- Remove only the legacy Host event database and its SQLite sidecars after primary cleanup commits. +- Add pre-PR-schema fixtures, partial-backfill fixtures, failpoints, and source-preservation sentinels. +- Integrate new coverage with required-store conformance and the previous-Stable upgrade harness. Preserve tagged fixture provenance. +- Add a disposable benchmark for 10k/100k 1 KiB messages and record storage, write, and startup measurements. + +## Out of scope + +- Automatic VACUUM, deletion of backups or plugin storage, rolling mixed-version deployment, and unrelated database cleanup. + +## Acceptance + +- Fresh and legacy upgrades preserve source records byte-for-byte and leave no legacy writer or payload table. +- DDL failure rollback, repeated startup, direct writes, FK cascades, and real PostgreSQL pass targeted tests. +- Successful file cleanup removes only expected regular files. Failed cleanup is observable and retryable without reopening the old store. + +## Verification + +Run from the repository root. Obtain behavioral RED evidence before production edits. + +```bash +(cd apps/backend && go test -race ./internal/task/repository/sqlite ./internal/plugins ./internal/backendapp ./internal/persistence/storeconformance -count=1) +: "${KANDEV_TEST_POSTGRES_DSN:?Set a disposable PostgreSQL test DSN}" +(cd apps/backend && go test -race ./internal/task/repository/sqlite ./internal/plugins ./internal/persistence/storeconformance -run 'Conversation|PreviousStableUpgrade|UpgradeFixtureManifest' -count=1) +(cd apps/backend && go run ./cmd/sqlguard ./internal) +(cd apps/backend && go test ./internal/task/repository/sqlite -run '^$' -bench '^BenchmarkConversationSourceStorage$' -benchtime=1x -count=1) +rg -n 'conversation_message_versions|conversation_turn_versions|conversation_session_events|session-events.sqlite|SessionDeliveryDispatcher' apps/backend apps/web +``` + +## Files likely touched + +- `apps/backend/internal/task/repository/sqlite/conversation_journal.go` +- `apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup.go (new)` +- `apps/backend/internal/task/repository/sqlite/conversation_journal_cleanup_test.go (new)` +- `apps/backend/internal/task/repository/sqlite/conversation_source_bench_test.go (new)` +- `apps/backend/internal/task/repository/sqlite/base_schema.go` +- `apps/backend/internal/plugins/conversation_journal.go` +- `apps/backend/internal/plugins/conversation_stream.go` +- `apps/backend/internal/plugins/conversation_stream_service.go` +- `apps/backend/internal/plugins/service.go` +- `apps/backend/internal/backendapp/main.go` +- `apps/backend/internal/backendapp/services.go` +- `apps/backend/internal/backendapp/types.go` +- `apps/backend/internal/persistence/requiredstores/` +- `apps/backend/internal/persistence/storeconformance/upgrade_test.go` +- `apps/backend/internal/persistence/storeconformance/testdata/` +- `apps/backend/pkg/websocket/actions.go` +- `apps/web/lib/ws/ordered-session-events.ts` + +## Dependencies + +03-core-reconciliation. + +## Risks + +- A bare code revert leaves copy triggers installed. +- The legacy SQLite file is outside the primary database transaction. Retry cleanup without touching keys or plugin data. +- DROP frees logical storage but can cause one-time I/O and does not shrink the SQLite file. +- The rg audit permits only cleanup code and historical fixtures/tests. Inspect each match, not only its exit code. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and the frontmatter criteria. +- [System design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [ADR](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Existing conversation handler, journal, Host facade, and recovery tests provide the regression patterns. + +## Results + +- Removed legacy journal readers, writers, ordered delivery runtime, ACK/poison persistence, and startup backfill composition. +- Added idempotent SQLite/PostgreSQL cleanup for legacy conversation tables and triggers, with source-preservation and safe Host-file sidecar cleanup tests. +- Added the `BenchmarkConversationSourceStorage` benchmark and recorded 10k/100k-row runs in the implementation evidence. +- `go run ./cmd/sqlguard ./internal`: passed. +- Targeted cleanup, repository, plugin, gateway, and test-harness race checks: passed. +- The follow-up ran the PostgreSQL source and cleanup variants on disposable PostgreSQL 16.15. Source bytes and indexes survived injected cleanup rollback and retry; legacy tables, functions, and triggers were removed without removing unrelated objects. See the [follow-up Task 01 results](../conversation-storage-follow-up/task-01-postgres-coverage.md). + + +## Large legacy SQLite verification + +The requested large-database verification passed on a 2.95 GB synthetic legacy +database. Backup, cleanup, task-repository initialization, and HTTP readiness +were measured separately. Both 60-second streaming runs completed without +write or readiness/probe failures. Active SQLite triggers, production workers, +and retired host files were audited. Manual compaction and subsequent readiness +passed; no automatic startup VACUUM was added. + +See [the verification report](verification/large-sqlite-upgrade.md) for workload, +commands, measurements, evidence, and limits. The public operations guide now +includes explicit post-upgrade compaction. PostgreSQL and other existing plan +gates remain open. All changes remain uncommitted. + +## Remaining-gate handoff + +The follow-up package records the final PostgreSQL and recovery evidence. +Existing results remain historical. diff --git a/docs/plans/conversation-storage-replacement/task-05-parity-and-docs.md b/docs/plans/conversation-storage-replacement/task-05-parity-and-docs.md new file mode 100644 index 0000000000..9d739ca081 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/task-05-parity-and-docs.md @@ -0,0 +1,148 @@ +--- +id: "05-parity-and-docs" +title: "Prove parity and publish the contract" +status: done +wave: 5 +depends_on: + - "04-retire-journal" +plan: "plan.md" +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-005 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +acceptance_criteria: + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-005.1 + - AC-PLUGINS-PROMPT-HISTORY-HOST-005.2 + - AC-PLUGINS-PROMPT-HISTORY-HOST-005.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.3 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.5 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.6 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.7 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.8 + - AC-PLUGINS-PROMPT-HISTORY-HOST-007.4 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.9 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.10 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.11 + - AC-PLUGINS-PROMPT-HISTORY-HOST-006.12 +system_design: + - "../../specs/plugins/system-design/conversation-source-reconciliation.md" +--- + +# Task 05: Prove parity and publish the contract + +## Summary + +Complete the replacement scenario matrix on desktop and mobile, and synchronize supported API and upgrade documentation. +Record actual check results and promote the draft design only after all work orders pass. + +## In scope + +- Own final fixture scenarios for live ID updates, loaded-range deletion, reconnect, restart, malformed batches, access loss, and suspended tabs. +- Assert zero history rereads for normal streaming and irrelevant changes after hydration, on desktop and mobile. +- Prove missing revision coverage triggers repair even when a later event and revision check arrive. +- Document no content hashes, transient complete change batches, coverage-only messages, and recovery-only broader reads. +- Preserve alias, favorite, native navigation, and existing core panel tests. +- Update PLUGIN-API.md Host-only wire sections and public authoring/upgrade documentation at implementation cutover. +- Update scoped engineering guidance and all companion plan references/results that describe the retired transport. + +## Out of scope + +- External plugin creation/publication, built-in panel removal, saved-layout migration, and generic full-repository QA. + +## Acceptance + +- Desktop and mobile fixtures prove incremental updates without broad rereads, plus discrepancy recovery, through the unchanged public SDK. +- All work-order checks have recorded results. No stale replay or fixed-snapshot promise remains in the active reference. +- Documentation describes original-record reads, recovery semantics, logical cleanup, physical compaction, and backup-based downgrade. + +## Verification + +Run from the repository root. Obtain behavioral RED evidence before production edits. + +```bash +(cd apps/web && pnpm e2e:run --project chromium tests/plugins/conversation-recovery.spec.ts tests/plugins/prompt-history-plugin.spec.ts tests/task/prompt-history-panel.spec.ts tests/session/session-stream-overload-isolation.spec.ts) +(cd apps/web && pnpm e2e:run --project mobile-chrome tests/plugins/mobile-prompt-history-plugin.spec.ts tests/session/mobile-session-stream-overload-isolation.spec.ts) +(cd apps/web && pnpm run typecheck) +(cd apps && pnpm --filter @kandev/plugin-sdk test && pnpm --filter @kandev/plugin-sdk typecheck) +node --test scripts/validate-public-docs.test.mjs +node scripts/validate-public-docs.mjs +python3 scripts/list-docs.py validate +python3 scripts/lint-spec-files.test.py +python3 scripts/lint-spec-files.py --all +git diff --check +``` + +Before the first pnpm command in a fresh worktree, run `(cd apps && pnpm install --frozen-lockfile)`. +Managed E2E rebuilds the application and fixture. Run desktop and mobile sequentially. + +## Files likely touched + +- `apps/web/e2e/tests/plugins/conversation-recovery.spec.ts` +- `apps/web/e2e/tests/plugins/prompt-history-plugin.spec.ts` +- `apps/web/e2e/tests/plugins/mobile-prompt-history-plugin.spec.ts` +- `apps/web/e2e/fixtures/plugins/prompt-history-plugin/bundle.js` +- `apps/web/e2e/helpers/causal-waits.ts` +- `apps/backend/internal/office/testharness/routes.go` +- `docs/plans/plugins/PLUGIN-API.md` +- `docs/public/plugins-authoring.md` +- `docs/public/plugins-manifest.md` +- `docs/public/operations.md` +- `apps/backend/AGENTS.md` +- `apps/web/AGENTS.md` +- `docs/specs/plugins/system-design/conversation-source-reconciliation.md` +- `docs/plans/conversation-storage-replacement/` + +## Dependencies + +04-retire-journal. + +## Risks + +- A browser clock cannot simulate server expiry or restart. Use isolated harness controls and causal completion. +- An old frontend must not silently accept unsupported v1 ACK semantics. +- Do not replace historical validation counts with claims that the new implementation passed them. + +## Parallelism + +`sequential` + +## Inputs + +- [Requirements](../../specs/plugins/requirements/prompt-history-extraction-host.md) and the frontmatter criteria. +- [System design](../../specs/plugins/system-design/conversation-source-reconciliation.md). +- [ADR](../../decisions/2026-09-16-conversation-source-reconciliation.md). +- Existing conversation handler, journal, Host facade, and recovery tests provide the regression patterns. + +## Results + +- Updated the active Host API, public plugin authoring/manifest/operations references, source reconciliation design, ADR, and companion plans for the current-state v2 contract. +- Replaced the obsolete expiry mutation E2E seam with a source-change drop and recovery assertion. +- Desktop managed E2E passed 6/6 for core/plugin recovery, plugin parity, core panel behavior, and stream isolation. +- Mobile managed E2E passed 2/2 for prompt-history touch navigation and stream isolation. +- Frontend unit tests passed 64/64; TypeScript, ESLint, plugin SDK tests/typecheck, i18n checks, public-doc validation, specification validation, SQL guard, and `git diff --check` passed. +- The follow-up completed the dependent PostgreSQL, Office, task-service, and final recovery gates. The final guarded browser matrix passed desktop 5/5 and mobile 5/5, with 41 focused frontend tests and gateway race coverage passing on revision `213492517315abb38697b765e91e6fe0ea5388c7`. The historical 6/6 desktop and 2/2 mobile counts above remain unchanged. + + +## Large legacy SQLite verification + +The requested large-database verification passed on a 2.95 GB synthetic legacy +database. Backup, cleanup, task-repository initialization, and HTTP readiness +were measured separately. Both 60-second streaming runs completed without +write or readiness/probe failures. Active SQLite triggers, production workers, +and retired host files were audited. Manual compaction and subsequent readiness +passed; no automatic startup VACUUM was added. + +See [the verification report](verification/large-sqlite-upgrade.md) for workload, +commands, measurements, evidence, and limits. The public operations guide now +includes explicit post-upgrade compaction. PostgreSQL and other existing plan +gates remain open. All changes remain uncommitted. + +## Remaining-gate handoff + +The follow-up package records the final PostgreSQL, backend, and recovery +evidence. Existing results remain historical. diff --git a/docs/plans/conversation-storage-replacement/verification/corrected-benchmark.log b/docs/plans/conversation-storage-replacement/verification/corrected-benchmark.log new file mode 100644 index 0000000000..38ee69473c --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/corrected-benchmark.log @@ -0,0 +1,7 @@ +goos: linux +goarch: amd64 +pkg: github.com/kandev/kandev/internal/task/repository/sqlite +cpu: AMD Ryzen 5 7640HS w/ Radeon 760M Graphics +BenchmarkConversationSourceStorage/messages_10000-11 1 3540695383 ns/op 23363584 db-bytes 14845498 startup-ns +PASS +ok github.com/kandev/kandev/internal/task/repository/sqlite 3.637s diff --git a/docs/plans/conversation-storage-replacement/verification/large-sqlite-results.json b/docs/plans/conversation-storage-replacement/verification/large-sqlite-results.json new file mode 100644 index 0000000000..815e25286e --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/large-sqlite-results.json @@ -0,0 +1,125 @@ +{ + "date": "2026-09-16", + "scope": "Uncommitted replacement on 88c6c0fe0a6ae5d25332070d603b99f7d9241645", + "sqlite_version": "3.51.1", + "platform": "Linux-7.0.14-4-pve-x86_64-with-glibc2.41", + "fixture": { + "messages": 50000, + "message_content_bytes_before_final_suffix": 8190, + "versions": 100000, + "events": 100000, + "turn_versions": 0, + "legacy_bytes": 2953932800, + "backup_bytes": 2949378048 + }, + "phase_timings_ms": { + "backup": 10811.325203, + "open_and_backup": 10814.908, + "cleanup": 625.067, + "task_repository_reopen": 32.546, + "post_cleanup_process_to_ready": 2227.281, + "snapshot_inside_process_start": 1402.432516, + "remaining_process_start": 824.8484839999999 + }, + "repository_streaming": { + "duration_seconds": 60, + "workers": 8, + "updates": 44204, + "write_failures": 0, + "probes": 599, + "probe_failures": 0, + "worst_probe_ms": 115.891 + }, + "runtime_streaming": { + "process_to_http_ready_ms": 2227.281, + "ready": { + "service": "kandev", + "status": "ok", + "version": "large-upgrade-verification" + }, + "workers": 8, + "duration_seconds": 60, + "streams": [ + { + "writes": 5643, + "failures": 0, + "errors": [], + "max_write_ms": 82.34840724617243 + }, + { + "writes": 5636, + "failures": 0, + "errors": [], + "max_write_ms": 86.38314716517925 + }, + { + "writes": 5640, + "failures": 0, + "errors": [], + "max_write_ms": 96.79982252418995 + }, + { + "writes": 5634, + "failures": 0, + "errors": [], + "max_write_ms": 90.04011563956738 + }, + { + "writes": 5630, + "failures": 0, + "errors": [], + "max_write_ms": 92.23995544016361 + }, + { + "writes": 5642, + "failures": 0, + "errors": [], + "max_write_ms": 99.56210665404797 + }, + { + "writes": 5636, + "failures": 0, + "errors": [], + "max_write_ms": 86.35333180427551 + }, + { + "writes": 5654, + "failures": 0, + "errors": [], + "max_write_ms": 86.70796174556017 + } + ], + "remaining_legacy_host_files": [], + "ready_samples": 596, + "ready_failures": [], + "max_ready_ms": 14.306996949017048 + }, + "full_catalog_periodic_probes": { + "observed_after_readiness": 4, + "unhealthy": 0 + }, + "manual_compaction": { + "before_bytes": 2953932800, + "free_pages_before": 604101, + "manual_vacuum_ms": 2960.6228582561016, + "integrity_check": "ok", + "messages": 50000, + "free_pages_after": 0, + "after_bytes": 477069312 + }, + "post_compaction_ready_ms": 412.714, + "audit": { + "legacy_primary_tables_remaining": 0, + "legacy_copy_triggers_remaining": 0, + "legacy_host_files_remaining": 0, + "old_worker_frames": 0, + "new_revision_worker_frames": 1 + }, + "limits": [ + "Synthetic fixture, not a production database clone.", + "Legacy turn-version table is empty; all seven legacy trigger definitions are installed.", + "Two independent 60-second streaming runs; no long-duration soak or huge single-payload coverage.", + "PostgreSQL runtime migration not exercised.", + "Startup timings are staged measurements, not additive end-to-end upgrade latency." + ] +} diff --git a/docs/plans/conversation-storage-replacement/verification/large-sqlite-test.log b/docs/plans/conversation-storage-replacement/verification/large-sqlite-test.log new file mode 100644 index 0000000000..547e46f121 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/large-sqlite-test.log @@ -0,0 +1,15 @@ +=== RUN TestConversationLargeLegacyUpgrade + conversation_upgrade_large_test.go:47: seeded_messages=500 + conversation_upgrade_large_test.go:47: seeded_messages=10500 + conversation_upgrade_large_test.go:47: seeded_messages=20500 + conversation_upgrade_large_test.go:47: seeded_messages=30500 + conversation_upgrade_large_test.go:47: seeded_messages=40500 + conversation_upgrade_large_test.go:49: legacy_db_bytes=2953932800 + conversation_upgrade_large_test.go:65: open_and_backup_ms=10814.908 + conversation_upgrade_large_test.go:71: cleanup_ms=625.067 + conversation_upgrade_large_test.go:77: task_repository_startup_ms=32.546 + conversation_upgrade_large_test.go:94: post_cleanup_pages=721175 free_pages=605351 + conversation_upgrade_large_test.go:95: stream_seconds=60 workers=8 target_updates_per_second=800 updates=44204 write_failures=0 probes=599 probe_failures=0 worst_probe_ms=115.891 +--- PASS: TestConversationLargeLegacyUpgrade (163.02s) +PASS +ok github.com/kandev/kandev/internal/task/repository/sqlite 163.042s diff --git a/docs/plans/conversation-storage-replacement/verification/large-sqlite-upgrade.md b/docs/plans/conversation-storage-replacement/verification/large-sqlite-upgrade.md new file mode 100644 index 0000000000..324053d488 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/large-sqlite-upgrade.md @@ -0,0 +1,162 @@ +# Large legacy SQLite verification + +Date: 2026-09-16. Scope: the uncommitted replacement on +`88c6c0fe0a6ae5d25332070d603b99f7d9241645`. No commit was created. + +## Fixture and method + +The disposable fixture contained 50,000 agent messages with 8,190 bytes of +synthetic content each. One insert and one update per message produced 100,000 +message versions and 100,000 events through the actual historical copy triggers. +The five legacy tables and seven SQLite triggers came from the historical Go +SQL builders at the scope base. The turn-version table was empty; this run +measures message-heavy storage, not a large turn-history workload. + +The source database measured 2,953,932,800 bytes (2.75 GiB). The verified backup +measured 2,949,378,048 bytes. Kandev used SQLite 3.51.1 on Linux amd64 and Go 1.26. +These are single-run, warm-cache measurements on a shared development VM. +An initial fixture attempt with an invalid workspace was discarded and corrected. +No developer database or live instance was used. + +The opt-in test opens the real single-writer/read-pool pair through +`persistence.Provide`, takes its pre-migration snapshot, runs the production +cleanup transaction, and reopens the populated task repository. +It then calls the real required-store health checker while eight workers update +messages through `UpdateMessageWithConversationReceipt`. + +A separate isolated backend uses that upgraded database. After `/ready` returns +200, eight clients PATCH `/api/v1/_test/messages/large-<0..7>` at up to 100 Hz +per client for 60 seconds. That route uses the production receipt transaction +and publishes on the event bus. `/ready` is sampled every 100 ms. The normal +15-second, full-catalog persistence probe remains enabled. +This does not simulate agent adapters or browser subscribers. + +## Separate timings + +| Measurement | Seconds | Boundary | +| --- | ---: | --- | +| Pre-migration backup | 10.811 | Snapshot creation, validation, and publication, from the production log | +| Open plus backup | 10.815 | Entire `persistence.Provide` call | +| Journal cleanup | 0.625 | Production cleanup transaction only | +| Populated task repository reopen | 0.033 | Task schema initialization after cleanup | +| Backend process to HTTP readiness | 2.227 | Separate launch against the already-cleaned database | +| Snapshot inside that backend launch | 1.402 | Separate post-cleanup snapshot, included in 2.227 seconds | +| Remaining backend startup | 0.825 | Process-to-ready minus that launch's snapshot duration | + +These staged measurements must not be added and presented as a single cold +upgrade. Full backend startup includes owners beyond the task repository. +An initial backend launch lacked the agentctl executable; that environment was +corrected before the measured ready/load run. + +The former source benchmark incorrectly reopened an empty path for its startup +measurement. It now reopens the populated file and seeds a valid workspace. +The corrected 10,000-message benchmark passed: populated reopen took 14.8 ms. + +## Post-readiness writer results + +| Run | Updates | Write failures | Probe samples | Probe failures | Worst probe | +| --- | ---: | ---: | ---: | ---: | ---: | +| Repository, 8 workers, 60 seconds | 44,204 | 0 | 599 persistence checks | 0 | 115.891 ms | +| Full backend, 8 clients, 60 seconds | 45,115 | 0 | 596 HTTP readiness checks | 0 | 14.307 ms | + +The repository checks use the production two-second deadline and the shared +writer connection, with the task-store table set. The full backend logs show +four successful full-catalog periodic probes after readiness, at 15-second +intervals. There were no `writer ping failed`, `table probe failed`, or +`required persistence probe failed` messages. These results establish a bounded +load pass, not an unrestricted concurrency or long-duration guarantee. + +## Retirement audit + +The verified pre-upgrade backup contains all five legacy tables and all seven +copy triggers. After cleanup and reopening, the active conversation schema has +only `conversation_session_revisions` and these revision-only triggers: + +- `conversation_source_message_insert`, `conversation_source_message_update`, + `conversation_source_message_delete` +- `conversation_source_turn_insert`, `conversation_source_turn_update`, + `conversation_source_turn_delete` + +No active trigger inserts into the retired event or version tables. +Production source has no `SessionEventLog`, `SessionDeliveryDispatcher`, +`StartSessionEventMaintenanceWorker`, `maintainSessionEvents`, +`syncAllCommittedSessionEvents`, `SyncCommittedSessionEvents`, or ordered-journal +broadcast implementation. Remaining legacy names belong to removal code, +historical documentation, or explicit test fixtures. +A runtime goroutine capture contains one new `runConversationChecks` worker and +none of the retired worker frames. + +Seeded `.host/session-events.sqlite`, `-wal`, and `-shm` files were removed at +backend startup and did not reappear during load. PostgreSQL cleanup statements +cover its three historical copy-trigger names, but PostgreSQL execution remains +unverified. This audit does not remove unrelated workflow or logging journals. + +## Explicit compaction + +Cleanup left the live file at 2,953,932,800 bytes, with 605,351 free pages out of +721,175. `auto_vacuum` was zero. No startup compaction was added. + +After stopping the isolated backend, an explicit offline `VACUUM` took 2.961 +seconds and reduced the file to 477,069,312 bytes. Free pages fell to zero. +`PRAGMA integrity_check` returned `ok`; all 50,000 source messages remained. +The compacted database then reached HTTP readiness in 0.413 seconds. + +The [operations guide](../../../public/operations.md#compact-sqlite-after-the-conversation-upgrade) +now separates compaction from startup and from backup `VACUUM INTO`, explains +maintenance disk requirements, and gives explicit UI and offline procedures. +Keep rollback snapshots outside automatic retention: repeated pre-ready starts +can produce further snapshots and prune older ones. + +## Reproduction and retained evidence + +Run the opt-in test from `apps/backend`, with at least 10 GiB of free disk: + +```bash +KANDEV_LARGE_UPGRADE_DIR=/absolute/disposable/empty-directory \ + go test ./internal/task/repository/sqlite \ + -run '^TestConversationLargeLegacyUpgrade$' -count=1 -v -timeout=15m +``` + +It retains `/data/kandev.db`, backups, and `persistence.log`. +Only use a disposable fixture for the backend HTTP workload described above. +The measured backend was built from the current checkout and launched with +`__backend`, isolated home/database/ports, `KANDEV_E2E_MOCK=true`, mock providers, +and the agentctl binary on PATH. Temporary instances were stopped after each +run. Temporary databases and binaries are removed after evidence collection. + +For a complete disposable reproduction, run from `apps/backend`: + +```bash +scratch=$(mktemp -d) +KANDEV_LARGE_UPGRADE_DIR="$scratch/data" go test ./internal/task/repository/sqlite \ + -run '^TestConversationLargeLegacyUpgrade$' -count=1 -v -timeout=15m +go build -o "$scratch/kandev" -ldflags '-X main.Version=large-upgrade-verification' ./cmd/kandev +go build -o "$scratch/agentctl" ./cmd/agentctl +python3 ../../docs/plans/conversation-storage-replacement/verification/runtime_verify.py "$scratch" +``` + +Inspect `runtime-results.json` and the backend logs for failures. The diagnostic +script stops its backend on exit and retains its scratch directory for inspection. +Remove that invocation-owned directory when finished. The driver is for the +synthetic fixture only, not for a running instance or a production database. + +Retained evidence: [metrics](large-sqlite-results.json), +[test output](large-sqlite-test.log), [legacy inventory](legacy-backup-inventory.txt), +and [runtime persistence logs](runtime-health-evidence.log). +PostgreSQL, browser E2E, and the unrelated full-backend failures remain separate +plan gates; this verification does not mark them complete. + +## Validation status + +The large opt-in test, corrected benchmark, cleanup regression test, public-doc +validator, specification validators, and whitespace checks passed. The normal +opt-in test invocation skips the large workload unless its environment variable +is set. + +Package-wide `golangci-lint run ./internal/task/repository/sqlite/...` still +reports six issues in the pre-existing replacement implementation: function +length in `initSQLiteConversationSourceTriggers`, four constant-reuse findings +in `conversation_source.go` and `conversation_receipts.go`, and nested complexity +in `UpdateMessageWithConversationReceipt`. It reports none in the added large +fixture test or corrected benchmark. These findings remain an implementation +gate; this workload verification is not a full lint or merge-readiness pass. diff --git a/docs/plans/conversation-storage-replacement/verification/legacy-backup-inventory.txt b/docs/plans/conversation-storage-replacement/verification/legacy-backup-inventory.txt new file mode 100644 index 0000000000..2297101147 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/legacy-backup-inventory.txt @@ -0,0 +1,17 @@ +sqlite_version=3.51.1 +["table","conversation_journal_meta"] +["table","conversation_message_versions"] +["table","conversation_session_events"] +["table","conversation_session_streams"] +["table","conversation_turn_versions"] +["trigger","conversation_message_delete"] +["trigger","conversation_message_insert"] +["trigger","conversation_message_update"] +["trigger","conversation_session_delete"] +["trigger","conversation_turn_complete"] +["trigger","conversation_turn_delete"] +["trigger","conversation_turn_insert"] +task_session_messages=50000 +conversation_message_versions=100000 +conversation_turn_versions=0 +conversation_session_events=100000 diff --git a/docs/plans/conversation-storage-replacement/verification/runtime-health-evidence.log b/docs/plans/conversation-storage-replacement/verification/runtime-health-evidence.log new file mode 100644 index 0000000000..b87d59c8cd --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/runtime-health-evidence.log @@ -0,0 +1,10 @@ +2026-09-16T23:00:11.514+0100 INFO logger/logger.go:240 pre-migration backup taken {"from_version": "pre-meta", "to_version": "large-upgrade-verification", "path": "/tmp/kandev-conversation-upgrade/data/data/backups/kandev-pre-meta-20260916T220010Z.db", "size_bytes": 471957504, "duration": 1.345753579} +2026-09-16T23:01:07.725+0100 INFO logger/logger.go:240 pre-migration backup taken {"from_version": "pre-meta", "to_version": "large-upgrade-verification", "path": "/tmp/kandev-conversation-upgrade/data/data/backups/kandev-pre-meta-20260916T220106Z.db", "size_bytes": 476778496, "duration": 1.402432516} +2026-09-16T23:01:08.498+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:01:08.505+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:01:23.500+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:01:38.500+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:01:53.517+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:02:08.564+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:03:37.929+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} +2026-09-16T23:03:37.936+0100 INFO logger/logger.go:240 required persistence state updated {"state": "healthy", "store_ids": [], "error_class": "none"} diff --git a/docs/plans/conversation-storage-replacement/verification/runtime_verify.py b/docs/plans/conversation-storage-replacement/verification/runtime_verify.py new file mode 100644 index 0000000000..6b169ee864 --- /dev/null +++ b/docs/plans/conversation-storage-replacement/verification/runtime_verify.py @@ -0,0 +1,58 @@ +import os, subprocess, time, json, urllib.request, concurrent.futures, pathlib, socket, sqlite3, sys +root=pathlib.Path(sys.argv[1]).resolve() +if not (root/'data/data/kandev.db').is_file(): raise SystemExit('Expected disposable large-upgrade fixture at /data/data/kandev.db') +home=root/'runtime-home';home.mkdir(exist_ok=True) +legacy_host=home/'plugins/.host';legacy_host.mkdir(parents=True,exist_ok=True) +with sqlite3.connect(legacy_host/'session-events.sqlite') as connection: + connection.execute('CREATE TABLE IF NOT EXISTS legacy_fixture(payload TEXT)') + connection.execute("INSERT INTO legacy_fixture VALUES('synthetic legacy host journal')") +for suffix in ['-wal','-shm']:(legacy_host/('session-events.sqlite'+suffix)).touch() +# OS-selected loopback port; only this disposable backend owns the fixture. +sock=socket.socket();sock.bind(('127.0.0.1',0));port=sock.getsockname()[1];sock.close() +env={'PATH':str(root)+os.pathsep+os.environ['PATH'],'HOME':str(home),'KANDEV_HOME_DIR':str(home),'KANDEV_DATABASE_PATH':str(root/'data/data/kandev.db'),'KANDEV_SERVER_HOST':'127.0.0.1','KANDEV_SERVER_PORT':str(port),'KANDEV_AGENT_STANDALONE_PORT':str(port+1),'KANDEV_E2E_MOCK':'true','KANDEV_MOCK_AGENT':'true','KANDEV_MOCK_GITHUB':'true','KANDEV_MOCK_JIRA':'true','KANDEV_MOCK_LINEAR':'true','KANDEV_DEBUG_PPROF_ENABLED':'true','KANDEV_LOG_LEVEL':'info'} +log=open(root/'runtime.log','w');started=time.monotonic();p=subprocess.Popen([str(root/'kandev'),'__backend'],env=env,cwd=str(home),stdout=log,stderr=subprocess.STDOUT) +(root/'runtime.pid').write_text(str(p.pid)) +base=f'http://127.0.0.1:{port}' +def request(path,data=None): + req=urllib.request.Request(base+path,data=None if data is None else json.dumps(data).encode(),headers={'Content-Type':'application/json'},method='GET' if data is None else 'PATCH') + with urllib.request.urlopen(req,timeout=5) as r: return r.status,r.read() +try: + while time.monotonic()-started<180: + if p.poll() is not None: raise RuntimeError(f'backend exited {p.returncode}') + try: + status,body=request('/ready') + if status==200: break + except Exception: pass + time.sleep(.1) + else: raise RuntimeError('readiness deadline') + result={'process_to_http_ready_ms':round((time.monotonic()-started)*1000,3),'ready':json.loads(body),'workers':8,'duration_seconds':(0 if os.environ.get('KANDEV_VERIFY_READY_ONLY') else 60)} + print(json.dumps(result),flush=True) + deadline=time.monotonic()+(0 if os.environ.get('KANDEV_VERIFY_READY_ONLY') else 60) + def stream(worker): + success=0;errors=[];max_latency=0 + while time.monotonic(); - retry(): void; -} + interface PluginSessionMessagesQuery { + sessionId: string | null; + taskId?: string | null; + authorTypes?: readonly PluginConversationAuthor[]; + sort?: PluginConversationSort; + pageSize?: number; + } -interface PluginSessionTurnsState { - turns: readonly PluginConversationTurn[]; - loading: boolean; - hydrated: boolean; - error: PluginConversationError | null; - removed: boolean; - retry(): void; -} + interface PluginSessionMessagesState { + messages: readonly PluginConversationMessage[]; + loading: boolean; + hydrated: boolean; + loadingMore: boolean; + error: PluginConversationError | null; + hasMore: boolean; + removed: boolean; + loadMore(): Promise; + retry(): void; + } -interface PluginConversationApi { - useSessionMessages( - query: PluginSessionMessagesQuery, - ): PluginSessionMessagesState; - useSessionTurns( - sessionId: string | null, - taskId?: string | null, - ): PluginSessionTurnsState; - useMessageFavorite(sessionId: string | null, messageId: string): boolean; -} -``` + interface PluginSessionTurnsState { + turns: readonly PluginConversationTurn[]; + loading: boolean; + hydrated: boolean; + error: PluginConversationError | null; + removed: boolean; + retry(): void; + } -Within a task-panel render, `host.conversation` resolves the Host-injected -panel scope; `props.conversation.history` is the equivalent explicit handle. -Outside a panel scope, nullable session reads return empty state. `taskId` is tri-state: `undefined` (field omitted) inherits the active panel context task via Host-side resolution before sending, so the server never observes `undefined`; explicit `null` means session-scoped with no task filter and encodes as absent `task_id`; an explicit string must equal the panel context task or the query fails `400/invalid_query` without network activity. An explicit string sends as its value; empty `task_id=` is `400`. Event `task_id` and public DTO `taskId` are nullable (`string | null`), with `null` for session-scoped rows. Cache identity, cursor fingerprint, and live-event filtering all key on the tri-state tag, so `undefined`, `null`, and `"t1"` never collapse; wrong-session or wrong-task events are dropped before merge. -Live event matching is explicit: inherited `undefined` and an explicit task -string accept only events whose `session_id` and `task_id` equal the resolved -query context. Explicit `null` accepts every task ID in the selected session, -including event `task_id: null`. A `session.removed` barrier is accepted for -the selected session regardless of its event task ID and closes both hooks. + interface PluginConversationApi { + useSessionMessages(query: PluginSessionMessagesQuery): PluginSessionMessagesState; + useSessionTurns(sessionId: string | null, taskId?: string | null): PluginSessionTurnsState; + useMessageFavorite(sessionId: string | null, messageId: string): boolean; + } -`loadMore()` resolves to the number of newly projected messages. Joined calls -resolve to the same committed count. Exhausted, closed, and terminal handles -resolve `0` without network activity; retryable transport or renewal failures -reject with `PluginConversationError` while leaving committed state unchanged. -Non-retryable query errors also reject and remain in `error`; `retry()` clears -the error and starts a fresh snapshot only for a live, retryable handle. -Turn `retry()` follows the same live-handle rule: it clears a retryable error -and starts a fresh ready snapshot, while closed, removed, exhausted, and -non-retryable handles do not issue network requests. -Each mounted `PluginTaskPanel` is wrapped in a Host-owned scope provider with its own cache, consumer identity, and abort controller. `history` is the authoritative panel handle; global `host.conversation` resolves only the nearest scope. Unmount or any plugin/panel/task/session/generation change aborts in-flight work, releases that consumer, and closes the handle. Closed handles and calls outside a scope return the stable empty state without network activity; two panels never share scope state. Message pages are bounded, newest-first by default, deterministic keyset pages with opaque cursors. `loadMore` is idempotent and preserves cursor, sort, filters, and generation. The Host owns cutoff, sidecars, replay, gaps, deduplication, tombstones, and lifecycle abort; none is public DTO state. -Lifecycle invalidation also includes a presentation change. `loadMore` is -idempotent operationally: concurrent calls for the same committed continuation -join one in-flight request, and calls after `hasMore` is `false` resolve without -network activity. -On `session.removed`, the facade retains already projected rows, sets -`removed` to `true`, sets `hasMore` to `false` for messages, and makes -subsequent `loadMore` and retry operations no-ops without network activity. -The `removed` terminal state is observable and stable; subsequent retries and -pagination calls resolve without network activity. -Each panel scope owns a monotonic `continuation_revision` over `{cursor,snapshot_token,cutoff,fingerprint,generation}` starting at sentinel `0` on scope creation. Query identity change, explicit `retry`, and close/invalidate increment the revision (close moves to an invalid tombstone that rejects all in-flight completions). Renewal and `loadMore` serialize dispatch per scope and CAS-install only when the current revision still equals the captured one; every response is checked against both the revision and the full tuple. Renewal may change only expiry, token rotation, and cursor re-encoding while preserving cutoff and fingerprint. Failed renewal or failed page leaves the committed page state unchanged; close or retry invalidates pending completions. -The HTTP facade returns the public page shapes above. The Host transport carries the subscribe ACK's `snapshot_token` in a Host-only request header; it remains valid through the signed cursor/query expiry (maximum 10 minutes) and rotates only on rebind. The server validates its plugin/session/generation binding and committed cutoff, then returns internal row-sequence sidecars. The Host strips those sidecars before this API resolves. Cursors reject tampering or a different session, task, filter, sort, generation, or expiry; reads are not caller-selectable as-of queries. -The Host renews a continuation before 80% of expiry through its private `POST /api/plugins/{id}/conversation/continuation/renew` operation. It accepts the current cursor and snapshot token, returns replacements with the same cutoff/fingerprint and a new expiry, and never selects a new cutoff. If retention or generation cannot preserve the cutoff, `loadMore` leaves existing state unchanged and reports retryable `upstream_failure`; explicit `retry` starts a fresh snapshot. +Within a task-panel render, host.conversation resolves the Host-injected panel +scope; props.conversation.history is the equivalent explicit handle. Outside a +panel scope, nullable session reads return empty state. taskId is tri-state: +undefined inherits the active panel task, null selects every task in the session, +and an explicit string must equal the panel task. A mismatch fails before +network activity. Cache identity, cursor fingerprints, and live filtering retain +that tri-state distinction. -Endpoint error matrix: `GET /conversation/binding` `409/generation_superseded/true` is Host-loader-only (bounded retry, never import until success, `Cache-Control: no-store` on success and every error). `POST /conversation/continuation/renew` `409` maps to public retryable `upstream_failure` without changing the loaded cutoff, also `Cache-Control: no-store`. Plugin-visible errors never carry `generation_superseded`. -Binding tokens expire no later than 10 minutes after issuance; the loader -rebinds before expiry and never sends an expired token to plugin code. -The Host establishes ordered readiness before the initial snapshot. The Host-only -request uses the discriminated `SubscribeRequest` branches below: core requires -`wire_id`; plugin requires `plugin_id`, `generation`, `binding_token`, and -`consumer_id`; identities are mutually exclusive. Success returns the matching -identity, while failure has no success-only fields. Legacy payloads are -accepted only with all ordered fields absent. +Messages and turns are read from current source rows in bounded deterministic +keyset pages. The Host maps safe DTOs, strips system content and arbitrary +metadata, and keeps source cursors private. It owns page invalidation, retries, +deduplication, recovery, and lifecycle abort. The public state retains projected +rows when session.removed arrives, sets removed to true, and stops pagination and +retry without issuing more network requests. Each mounted panel has an +independent scope, cache, Host-minted identity, and abort controller. -#### Host-only conversation wire contract +The Host reconciles source notifications by epoch and decimal revision. A +matching complete receipt applies its operations by entity ID. A reset marker, +malformed payload, wrong scope, epoch change, revision gap, or failed operation +starts a fresh source read. Notifications received before their matching +snapshot commits remain buffered. A source read and its revision are checked +together, then the buffered changes are applied. There is no durable payload +journal, ACK protocol, poison queue, replay promise, content hash, or caller +selectable as-of read. -The Host-only snapshot header is `X-Kandev-Snapshot-Token`. HTTP page and -renewal shapes, plus ordered event payloads, are: +#### Host-only v2 conversation wire contract -```ts -type CoreSubscribeRequest = { - session_id: string; - consumer_kind: "core"; - last_seen_sequence?: number; - resume_token?: string; - wire_id: string; - plugin_id?: never; - generation?: never; - binding_token?: never; - consumer_id?: never; -}; -type PluginSubscribeRequest = { - session_id: string; - consumer_kind: "plugin"; - plugin_id: string; - generation: number; - binding_token: string; - last_seen_sequence?: number; - resume_token?: string; - consumer_id: string; - wire_id?: never; -}; -type SubscribeRequest = CoreSubscribeRequest | PluginSubscribeRequest; -type SessionWireFailureCode = - | "unauthorized" - | "invalid_binding" - | "generation_superseded" - | "invalid_request" - | "session_not_found" - | "cursor_expired" - | "forward_gap" - | "session_removed"; -type SessionWireError = { - code: SessionWireFailureCode; - message: string; - retryable: boolean; -}; -type SubscribeSuccessPayload = { - success: true; - session_id: string; - result: "fresh" | "replay" | "gap" | "invalid_resume"; - event_watermark: number; - snapshot_cutoff: number; - snapshot_token: string; - replay_from?: number; - replay_to?: number; - oldest_sequence?: number; - resume_token: string; -} & ( - | { wire_id: string; consumer_id?: never } - | { consumer_id: string; wire_id?: never } -); -type SubscribeFailurePayload = { - success: false; - session_id?: string; - error: SessionWireError; -}; -type SubscribeAck = { - id: string; - type: "response"; - action: "session.subscribe"; - payload: SubscribeSuccessPayload | SubscribeFailurePayload; -}; -type CoreSessionAckRequest = { - session_id: string; - consumer_kind: "core"; - sequence: number; - resume_token?: string; - wire_id: string; - plugin_id?: never; - generation?: never; - consumer_id?: never; -}; -type PluginSessionAckRequest = { - session_id: string; - consumer_kind: "plugin"; - plugin_id: string; - generation: number; - sequence: number; - resume_token?: string; - consumer_id: string; - wire_id?: never; -}; -type SessionAckRequest = CoreSessionAckRequest | PluginSessionAckRequest; -type SessionAckSuccessPayload = { - success: true; - session_id: string; - acknowledged_sequence: number; - resume_token: string; -} & ( - | { wire_id: string; consumer_id?: never } - | { consumer_id: string; wire_id?: never } -); -type SessionAckFailurePayload = { - success: false; - session_id?: string; - error: SessionWireError; -}; -type SessionAckResponse = { - id: string; - type: "response"; - action: "session.ack"; - payload: SessionAckSuccessPayload | SessionAckFailurePayload; -}; -type PluginConversationPage = { - messages: readonly PluginConversationMessage[]; - hasMore: boolean; - cursor: string | null; -}; -type PluginConversationTurnsPage = { - turns: readonly PluginConversationTurn[]; -}; -type ContinuationRenewRequest = { - cursor: string; - snapshot_token: string; -}; -type ContinuationRenewResponse = { - cursor: string; - snapshot_token: string; - cutoff: number; - fingerprint: string; - expires_at: string; -}; -type RawSessionEvent = { - type: "session.event"; - protocol_version: number; - event_type: string; - session_id: string; - task_id: string | null; - sequence: number; - event_id: string; - payload: unknown; -}; +The private subscribe actions are session.conversation.subscribe and +session.conversation.unsubscribe. A request uses this shape: -// Validation converts RawSessionEvent to this projected union only after -// protocol, identity, event-type, and payload checks succeed. -type SessionEventPayload = - | { event_type: "message.added"; message: PluginConversationMessage } - | { event_type: "message.updated"; message: PluginConversationMessage } - | { event_type: "message.deleted"; message_id: string } - | { event_type: "session.turn.started"; turn: PluginConversationTurn } - | { event_type: "session.turn.completed"; turn: PluginConversationTurn } - | { event_type: "session.removed" }; -type SessionEventBase = { - type: "session.event"; - protocol_version: 1; - session_id: string; - task_id: string | null; - sequence: number; - event_id: string; -}; -type SessionEvent = { - [T in SessionEventPayload["event_type"]]: SessionEventBase & { - event_type: T; - payload: Extract; - }; -}[SessionEventPayload["event_type"]]; -``` + type ConversationSubscribeRequest = { + protocol_version: 2; + scope_id: string; + session_id: string; + consumer_kind: "core" | "plugin"; + plugin_id?: string; + generation?: number; + binding_token?: string; + task_id?: string | null; + authors?: string[]; + sort?: "asc" | "desc"; + }; -The Host sends `X-Kandev-Snapshot-Token` on snapshot reads and validates the -renewal request before exposing only the public page state. `cursor` is -nullable only on a page with no next page; renewal receives and returns the -current non-null cursor. The Host maps event payloads to public DTOs and -strips protocol sidecars before plugin code sees them. Removal is terminal -whether it precedes or follows poison; the replacement cursor is closed. -Only `message.added`, `message.updated`, `message.deleted`, -`session.turn.started`, `session.turn.completed`, and `session.removed` use -this migrated contract. -The raw envelope is decoded with `protocol_version: number` and arbitrary -`event_type`/`payload`; `SessionEvent` is the validated projected union. -Validation requires `payload.type === event_type`; a mismatch is durable -poison, is not projected, and cannot advance ACK. Strict subscribe ACKs require -`session_id`, `result`, `event_watermark`, `snapshot_cutoff`, `snapshot_token`, -and `resume_token`. `fresh` forbids replay and oldest-sequence fields; -`replay` requires `replay_from` and `replay_to` with -`replay_from <= replay_to <= snapshot_cutoff`. -For `replay`, `replay_from` is the first delivered sequence (inclusive) and -equals `last_seen_sequence + 1`; `replay_to` is the final delivered sequence -(inclusive) and equals the committed `snapshot_cutoff`. -`replay` is returned only when `last_seen_sequence` was supplied; otherwise -the initial result is `fresh`. -`gap` requires `oldest_sequence` and forbids replay fields; `invalid_resume` -forbids replay fields and reports the replacement cursor state. The -`resume_token` is always the current token for the returned consumer state. +Plugin requests include plugin_id, generation, and binding_token. Core requests +use consumer_kind: "core". The server validates the session through the normal +user/workspace boundary and rejects stale legacy session.subscribe payloads with +ordered fields. Unsubscribe uses the same scope and binding identity. -Subscribe and ACK failures use the same outer response envelope with -`success:false`, the requested `session_id` when known, and -`error:{code,message,retryable}`; no success-only ordered fields are present. -Codes distinguish `unauthorized`, `invalid_binding`, `generation_superseded`, -`invalid_request`, `session_not_found`, `cursor_expired`, `forward_gap`, and -`session_removed`. Binding/generation failures are non-projection failures; -`generation_superseded` is retryable only for the Host loader, while expired -or retained-cursor failures produce `gap` reconciliation rather than a -plugin-visible subscribe exception. ACK rejects forward gaps and malformed -identity/range/token input without advancing the durable cursor. -Core wire identity: the Host mints one opaque `wireId` per core consumer mount and carries it in subscribe/ACK state as `wire_id`. Plugin panels receive a separate Host-minted opaque `consumer_id`, carried in subscribe/ACK state and never caller-supplied. Broker state is keyed by `(session,consumer_kind,plugin,generation,wire_id|consumer_id)`; each applicable identity survives reconnect through its persisted entry, while a new mount mints a new identity at its registration watermark. Identities transition `registered -> ready -> streaming -> suspended (reconnect window) -> released` and release on unsubscribe/unmount or after the reconnect plus token-retention window expires. Plugin panels from one plugin/generation never share a cursor. -Reconnect grace is 10 minutes after disconnect. Resume and snapshot tokens -expire no later than 10 minutes after issuance; delivery cursors and immutable -event rows remain for the maximum token expiry plus the reconnect grace. -Garbage collection runs only after those bounds and preserves terminal -tombstones until all resumable consumer identities are released. -If `session.removed` precedes poison, the rebind still produces terminal -removal and a closed replacement cursor; no later event is admitted. This is -the same terminal outcome as poison followed by removal. -If a previously authorized consumer reconnects to `session_removed` or a -terminal replacement cursor, the facade preserves committed rows, sets -`removed:true`, sets message `hasMore:false`, and makes `loadMore` and `retry` -no-ops. A fresh unauthorized or nonexistent lookup remains ordinary -`404/not_found` and never reveals deletion state. -Ordered event envelopes carry `protocol_version`; canonical policy registry is `{1:{ignorable:["session.workspace_sources.updated"]}}`. Only a type listed by its matching version may advance ACK without projection. Missing, malformed, or unsupported versions and other unknown types remain durable poison: the cursor stays before the event, diagnostics retain its ID/version, and advancement stops. Rebind atomically snapshots past poison at a new cutoff, commits that cutoff, returns a replacement cursor/token, and does not project the poison. When `session.removed` arrives after a poison cursor, rebind snapshots past both to a terminal cutoff: the outcome is terminal removal, the replacement cursor is terminally closed and admits no post-terminal events, and poison diagnostics remain durable. -Poison operations are owned by `SessionDeliveryDispatcher`: each record moves -`pending -> leased -> exhausted -> requeued`, with a 30-second lease, five -attempts, and exponential backoff capped at 30 seconds. Exhaustion records -event ID, protocol version, and last error, and emits structured -`session_event_poison_total` diagnostics. The Host-only -`session.event.poison.requeue` command requires an authenticated operator with -`session_events:requeue`; it accepts session, event ID, and expected owner epoch, -audits actor, prior state, attempt count, and timestamp, increments owner epoch, -resets attempts, and returns `pending` without changing sequence or payload. -Repeated requests after requeue return current state without mutation; stale -expected epochs are rejected without mutation. Startup reclaims expired leases -but never clears or skips poison. -Legacy projection coalescing: pending contiguous sequence ranges project at most once per `event_id`/sequence per wire, then ACK the highest contiguous sequence. Example: `seq 10 = message.updated id A` followed by `seq 11 = message.updated id A` coalesces to one projection of the latest image `A@11`, then a single `session.ack` for `11`; a `session.removed` barrier after `11` is a separate terminal projection. Same-`event_id` replays and same-sequence redeliveries are dropped; any event with `sequence <= acknowledged_sequence` is suppressed without a second projection. -After contiguous delivery the Host-only `session.ack` uses the discriminated -`SessionAckRequest` branches below: core requires `wire_id`; plugin requires -`plugin_id`, `generation`, and `consumer_id`; identities are mutually exclusive. -Success returns the matching identity and failure has no success-only fields. -The shared client retains the full response envelope, validates outer -IDs/types/actions before payload parsing, and retries by consumer sequence/token. -The Host validates binding/generation, strict integer/range rules, applicable -identity, current or prior token on retry, and contiguous advancement. Snapshot -tokens and sidecars never reach plugin code. Reload, disable, uninstall, panel -switch, or unmount revokes generation. + type ConversationSubscribeSuccess = { + success: true; + protocol_version: 2; + scope_id: string; + session_id: string; + epoch: string; + revision: string; + }; -Registry ownership matrix: every `register*` call and Host-owned resource is keyed by `(pluginId,generation,stageId,kind,localId)` and staged invisibly until commit. `kind` is the registration method (`route`, `navItem`, `settingsRoute`, `component`, `wsHandler`, `keybinding`, `integrationSettings`, `repositoryProvider`, `reviewProvider`, `taskAction`, `taskPanel`, `taskMenuAction`, `taskFilter`, `taskListFacet`, `translations`) or Host-owned resource (`styleLink`, `modal`, `conversationScope`, `bindingGrant`). A duplicate `(kind,localId)` within the same stage rejects the second call; a call from a foreign or retired `(pluginId,generation,stageId)` is rejected without side effects; stage abort discards all its staged entries without notifications. Commit atomically publishes the stage's entries and revokes the retired generation's entries in one linearization step, emitting notifications only for the committed delta. + type ConversationSubscribeFailure = { + success: false; + error: { + code: "invalid_request" | "invalid_binding" | "generation_superseded" + | "session_not_found" | "unauthorized" | "upstream_failure"; + message: string; + retryable: boolean; + }; + }; + type ConversationChangeOperation = { + kind: "upsert" | "remove"; + entity: "message" | "turn"; + id: string; + message?: object; + turn?: object; + }; + + type ConversationChangedPayload = { + protocol_version: 2; + scope_id: string; + session_id: string; + epoch: string; + base_revision: string; + revision: string; + reset?: boolean; + operations: ConversationChangeOperation[]; + }; + +The server publishes a changed payload only after the source transaction +commits. Complete mutation receipts carry the represented operations and the +base and committed revision. Incomplete receipts and uninstrumented writes +carry reset: true, so the client performs source reconciliation. An operation +may be filtered out for a task or author subscription while the revision still +advances; an empty operations array with matching revisions is a valid +coverage-only notification. + +Revision values are decimal strings because JavaScript numbers cannot safely +represent every database revision. The process epoch changes after restart or +restore. The Host accepts only a newer contiguous revision in the current epoch, +buffers changes until snapshots commit, and recovers on a gap or epoch change. +A terminal session.removed notification is delivered through the normal +notification channel and closes every matching source scope. + +HTTP pages are: +GET /api/plugins/{id}/conversation/v2/task-sessions/{sessionId}/messages +GET /api/plugins/{id}/conversation/v2/task-sessions/{sessionId}/turns +GET /api/plugins/{id}/conversation/v2/task-sessions/{sessionId}/revision +GET /api/plugins/{id}/conversation/binding + +Source pages return the public records plus private epoch, revision, cursor, and +hasMore fields. expected_revision is an optional decimal guard. Binding and all +error responses use Cache-Control: no-store. Stable public errors are +401/unauthenticated, 404/not_found, 400/invalid_query, and authorized +5xx/upstream_failure. Binding failures may use a Host-only retryable +generation_superseded response; it never reaches plugin code. + +The continuation-renew endpoint from the predecessor transport is not part of +the source contract. Load-more and retry use a current source read and preserve +the public state until that read succeeds. No browser code imports a persistence +store or writes conversation history. ## `registry: PluginRegistry` ```ts diff --git a/docs/plans/pr-3588-conversation-recovery/plan.md b/docs/plans/pr-3588-conversation-recovery/plan.md index 7b229b1113..8104675ef2 100644 --- a/docs/plans/pr-3588-conversation-recovery/plan.md +++ b/docs/plans/pr-3588-conversation-recovery/plan.md @@ -12,6 +12,14 @@ legacy_specs: [] # Implementation Plan: PR 3588 Conversation Recovery + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Overview Repair the three recovery defects found in PR #3588 at commit @@ -179,3 +187,9 @@ PR review. The broader durable transport scope decision remains separate. overwrite newer snapshot state, and deleted rows must not reappear. - Expiry, removal, access revocation, and panel replacement can race. - Real transport tests must use fresh binaries and bundles. + +## Replacement validation handoff + +The [remaining-gates package](../conversation-storage-follow-up/plan.md) owns the +replacement PostgreSQL matrix, backend failure remediation, and final recovery E2E evidence. +Historical counts in this package do not prove the replacement implementation. diff --git a/docs/plans/pr-3588-conversation-recovery/task-01-correct-replay-grants.md b/docs/plans/pr-3588-conversation-recovery/task-01-correct-replay-grants.md index 1ad73a5a02..aaa51bb7e9 100644 --- a/docs/plans/pr-3588-conversation-recovery/task-01-correct-replay-grants.md +++ b/docs/plans/pr-3588-conversation-recovery/task-01-correct-replay-grants.md @@ -1,7 +1,7 @@ --- id: "01-correct-replay-grants" title: "Correct replay grants" -status: complete +status: done wave: 1 depends_on: [] plan: "plan.md" @@ -11,8 +11,6 @@ requirements: acceptance_criteria: - AC-PLUGINS-PROMPT-HISTORY-HOST-002.5 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.13 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.15 - AC-PLUGINS-PROMPT-HISTORY-HOST-005.3 system_design: - "../../specs/plugins/system-design/conversation-recovery.md" @@ -20,6 +18,14 @@ system_design: # Task 01: Correct replay grants + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Summary Make every valid contiguous replay acknowledgment succeed. diff --git a/docs/plans/pr-3588-conversation-recovery/task-02-restore-core-snapshots.md b/docs/plans/pr-3588-conversation-recovery/task-02-restore-core-snapshots.md index 97972ae1e7..cdfdbe2129 100644 --- a/docs/plans/pr-3588-conversation-recovery/task-02-restore-core-snapshots.md +++ b/docs/plans/pr-3588-conversation-recovery/task-02-restore-core-snapshots.md @@ -1,7 +1,7 @@ --- id: "02-restore-core-snapshots" title: "Restore core snapshots during recovery" -status: complete +status: done wave: 2 depends_on: - "01-correct-replay-grants" @@ -13,7 +13,6 @@ acceptance_criteria: - AC-PLUGINS-PROMPT-HISTORY-HOST-002.5 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.6 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.9 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.15 - AC-PLUGINS-PROMPT-HISTORY-HOST-005.3 system_design: - "../../specs/plugins/system-design/conversation-recovery.md" @@ -21,6 +20,14 @@ system_design: # Task 02: Restore core snapshots during recovery + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Summary Repair core Chat after cursor replacement without a socket reconnect. diff --git a/docs/plans/pr-3588-conversation-recovery/task-03-recover-expired-continuations.md b/docs/plans/pr-3588-conversation-recovery/task-03-recover-expired-continuations.md index e7a1f3bea3..883f04650b 100644 --- a/docs/plans/pr-3588-conversation-recovery/task-03-recover-expired-continuations.md +++ b/docs/plans/pr-3588-conversation-recovery/task-03-recover-expired-continuations.md @@ -1,7 +1,7 @@ --- id: "03-recover-expired-continuations" title: "Recover expired plugin continuations" -status: complete +status: done wave: 3 depends_on: - "02-restore-core-snapshots" @@ -15,8 +15,6 @@ acceptance_criteria: - AC-PLUGINS-PROMPT-HISTORY-HOST-002.6 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.10 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.13 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.15 - AC-PLUGINS-PROMPT-HISTORY-HOST-005.3 system_design: - "../../specs/plugins/system-design/conversation-recovery.md" @@ -24,6 +22,14 @@ system_design: # Task 03: Recover expired plugin continuations + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Summary Let an idle plugin panel resume paging and live updates without remount. diff --git a/docs/plans/prompt-history-plugin-host/plan.md b/docs/plans/prompt-history-plugin-host/plan.md index 7e311c02f7..27e44239c3 100644 --- a/docs/plans/prompt-history-plugin-host/plan.md +++ b/docs/plans/prompt-history-plugin-host/plan.md @@ -14,6 +14,14 @@ legacy_specs: [] # Implementation Plan: Prompt History Plugin Host Prerequisites + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Objective Land the complete public browser Host boundary required by a future external prompt-history plugin while keeping the shipped core panel unchanged. The package adds capability-gated session reads, typed live reconciliation, scoped native navigation, host-owned prompt display dependencies, and the missing generic task-panel context. @@ -116,7 +124,7 @@ This package is complete when all six work orders are done and a fixture plugin The [PR #3588 recovery package](../pr-3588-conversation-recovery/plan.md) tracks three newly identified defects: replay grants, core snapshot repair, -and expired continuations. Its work orders remain pending. +and expired continuations. Its work orders are complete, with results recorded in that package. The completion statements and results below describe the original delivery; they do not prove these recovery cases. Tasks 02, 04, and 05 link to the new repair work and its exact verification matrix. The full PR review and @@ -133,3 +141,9 @@ ownership. Retention now runs from the live service maintenance lifecycle. All six work orders are complete. External plugin creation, publication, core removal, and saved-layout migration remain blocked on a later explicitly approved design and implementation package. + +## Replacement validation handoff + +The [remaining-gates package](../conversation-storage-follow-up/plan.md) owns the +replacement PostgreSQL matrix, backend failure remediation, and final recovery E2E evidence. +Historical counts in this package do not prove the replacement implementation. diff --git a/docs/plans/prompt-history-plugin-host/task-01-publish-browser-conversation-contract.md b/docs/plans/prompt-history-plugin-host/task-01-publish-browser-conversation-contract.md index 767c278566..56c9fe57a7 100644 --- a/docs/plans/prompt-history-plugin-host/task-01-publish-browser-conversation-contract.md +++ b/docs/plans/prompt-history-plugin-host/task-01-publish-browser-conversation-contract.md @@ -20,6 +20,14 @@ system_design: # Task 01: Publish Browser Conversation Contract + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Scope Publish the additive, runtime-free TypeScript contract before implementing either side. Define conversation DTOs, the `PluginConversationError` shape with stable codes and `retryable`, hook state types, task-panel context, visibility/title/navigation/favorite/mention/lifecycle contracts, and canonical documentation. This task publishes shapes only; runtime behavior belongs to later work orders. diff --git a/docs/plans/prompt-history-plugin-host/task-02-add-plugin-conversation-reads.md b/docs/plans/prompt-history-plugin-host/task-02-add-plugin-conversation-reads.md index 9b66ca2fae..a14b2c87b0 100644 --- a/docs/plans/prompt-history-plugin-host/task-02-add-plugin-conversation-reads.md +++ b/docs/plans/prompt-history-plugin-host/task-02-add-plugin-conversation-reads.md @@ -15,15 +15,20 @@ acceptance_criteria: - AC-PLUGINS-PROMPT-HISTORY-HOST-002.7 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.8 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.10 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.13 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.15 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.16 system_design: - "../../specs/plugins/system-design/prompt-history-extraction-host.md" --- # Task 02: Add Plugin Conversation Read Routes + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Scope Add the authenticated backend and Host-only transport prerequisites for browser diff --git a/docs/plans/prompt-history-plugin-host/task-03-extend-task-panel-capabilities.md b/docs/plans/prompt-history-plugin-host/task-03-extend-task-panel-capabilities.md index 4f94b2fec0..7ede1dffdc 100644 --- a/docs/plans/prompt-history-plugin-host/task-03-extend-task-panel-capabilities.md +++ b/docs/plans/prompt-history-plugin-host/task-03-extend-task-panel-capabilities.md @@ -27,6 +27,14 @@ system_design: # Task 03: Extend Generic Task Panel Capabilities + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Scope Extend the existing generic task-panel contribution rather than adding prompt-history-specific host branches. Add localized title resolution, typed visibility context, session kind, and a generation-bound native message-navigation capability on desktop and mobile. diff --git a/docs/plans/prompt-history-plugin-host/task-04-build-conversation-host-facade.md b/docs/plans/prompt-history-plugin-host/task-04-build-conversation-host-facade.md index 4aecd47fa6..fb22928f33 100644 --- a/docs/plans/prompt-history-plugin-host/task-04-build-conversation-host-facade.md +++ b/docs/plans/prompt-history-plugin-host/task-04-build-conversation-host-facade.md @@ -20,7 +20,6 @@ acceptance_criteria: - AC-PLUGINS-PROMPT-HISTORY-HOST-002.9 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.12 - AC-PLUGINS-PROMPT-HISTORY-HOST-002.14 - - AC-PLUGINS-PROMPT-HISTORY-HOST-002.15 - AC-PLUGINS-PROMPT-HISTORY-HOST-004.1 - AC-PLUGINS-PROMPT-HISTORY-HOST-004.2 - AC-PLUGINS-PROMPT-HISTORY-HOST-004.3 @@ -29,6 +28,14 @@ system_design: --- # Task 04: Build Browser Conversation Host Facade + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Scope Implement `host.conversation` and the curated prompt-mention renderer behind the published SDK types. Own request/subscription races, pagination, lifecycle cleanup, DTO mapping, favorite reactivity, and private custom-prompt access inside the host. diff --git a/docs/plans/prompt-history-plugin-host/task-05-prove-plugin-parity.md b/docs/plans/prompt-history-plugin-host/task-05-prove-plugin-parity.md index c98ba9baa6..57ff1bde62 100644 --- a/docs/plans/prompt-history-plugin-host/task-05-prove-plugin-parity.md +++ b/docs/plans/prompt-history-plugin-host/task-05-prove-plugin-parity.md @@ -19,6 +19,14 @@ system_design: # Task 05: Prove Prompt-History Plugin Parity + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Scope Add a test-only external-style fixture plugin that implements prompt history through `@kandev/plugin-sdk` and the published Host APIs only. Use it to prove the prerequisite boundary; do not ship it as the production plugin or delete core code. diff --git a/docs/plans/prompt-history-plugin-host/task-06-document-and-verify.md b/docs/plans/prompt-history-plugin-host/task-06-document-and-verify.md index 31029fe568..f3b3da40ef 100644 --- a/docs/plans/prompt-history-plugin-host/task-06-document-and-verify.md +++ b/docs/plans/prompt-history-plugin-host/task-06-document-and-verify.md @@ -19,6 +19,14 @@ system_design: # Task 06: Document and Verify Host Prerequisites + +## Replacement scope, 2026-09-16 + +The [conversation storage replacement](../conversation-storage-replacement/plan.md) owns the next implementation. +This file preserves historical scope and results. Do not execute its durable replay mechanics as new work. +The replacement work orders preserve public behavior and provide new source-reconciliation, upgrade, and E2E evidence. + + ## Scope Finish the public reference/how-to documentation and run the package-level validation matrix. Update artifact statuses/results only after the behavior and docs are proven. Keep external-plugin and core-removal work explicitly deferred. diff --git a/docs/public/operations.md b/docs/public/operations.md index 06690f158d..febd2fbbdf 100644 --- a/docs/public/operations.md +++ b/docs/public/operations.md @@ -460,6 +460,45 @@ Open **Settings > System > Data & Logs > Database** to see database size, WAL si These actions run as background system jobs. Closing the browser does not cancel a job. Check the page or logs for its terminal state before starting another maintenance operation. +Conversation history uses the task database as its source. A storage cutover +can remove the retired conversation journal tables and triggers in one startup +migration, after the normal pre-migration backup. The migration is transactional +and safe to retry after a failed attempt. Stop older Kandev backends before the +upgrade; a mixed old and new writer set is not supported. The cutover does not +run `VACUUM`, so use the existing **Vacuum** action later if you need to reduce +the SQLite file size. + +#### Compact SQLite after the conversation upgrade + +Compaction is an explicit maintenance operation, separate from backup, migration, +and readiness. Startup does not compact the live database. The pre-migration +snapshot uses `VACUUM INTO` to create a separate backup; it does not shrink the +live file. + +1. Wait for the upgraded backend to become ready. Check history before proceeding. +2. Copy the verified pre-upgrade backup outside the automatic retention directory + for rollback. Create and verify a current snapshot to preserve newer work. +3. Schedule a quiet maintenance period and pause agent runs. Ensure sufficient + free disk space for temporary files, the WAL, and retained backups. +4. Open **Settings > System > Data & Logs > Database** and select **Vacuum**. + Wait for the job to finish before starting other maintenance or resuming runs. +5. Check the reported reclaimed bytes, `/ready`, and conversation history. + Existing backup files are not compacted by this operation. + +For offline maintenance, stop every backend that uses the database, including +supervised services. Then run the following against the configured SQLite path: + +```bash +sqlite3 "/absolute/path/to/kandev.db" 'VACUUM;' +sqlite3 "/absolute/path/to/kandev.db" 'PRAGMA integrity_check;' +``` + +The integrity check must return `ok` before you restart Kandev. Do not delete +`-wal` or `-shm` files manually. `PRAGMA optimize` and a WAL checkpoint do not +replace compaction. SQLite documents a worst-case requirement of twice the +original database size in additional free space for +[VACUUM](https://www.sqlite.org/lang_vacuum.html). + ### PostgreSQL PostgreSQL is an external, operator-managed database. The Database page shows driver, database-level size, and schema version, but hides SQLite path, WAL, backup time, and maintenance controls. Kandev does not take a pre-migration PostgreSQL dump and the System Backups page is not a PostgreSQL backup/restore facility. Use your platform backup policy or `pg_dump`/`pg_restore`. diff --git a/docs/public/plugins-authoring.md b/docs/public/plugins-authoring.md index 664bc04df0..079b72e316 100644 --- a/docs/public/plugins-authoring.md +++ b/docs/public/plugins-authoring.md @@ -325,11 +325,10 @@ of truth and must be updated together when the contract changes: [explicit plugin utility selection](../decisions/2026-09-14-explicit-plugin-utility-selection.md), and [Browser conversation facade ADR](../decisions/2026-09-06-browser-plugin-conversation-facade.md). -The browser conversation facade ADR is still proposed while this prerequisite -package is under review. Until that ADR is accepted, normative authority is -split deliberately: requirements define observable behavior, the system design -defines Host architecture, and `PLUGIN-API.md` defines the Host-only wire -contract. +The [source reconciliation design](../specs/plugins/system-design/conversation-source-reconciliation.md) +defines the current storage and transport behavior. Requirements define +observable behavior, and `PLUGIN-API.md` defines the browser API and Host-only +v2 wire contract. ## Frontend contract @@ -401,7 +400,11 @@ Panel handles are independently scoped and become inert on unmount, identity change, disable, or reload. `host.conversation` is the equivalent nearest-scope accessor; outside a panel it returns stable empty state. Use the canonical [PLUGIN-API contract](../plans/plugins/PLUGIN-API.md) for DTOs, pagination, -ordered updates, lifecycle, and retryable errors. +revision-bound updates, lifecycle, recovery, and retryable errors. A complete +source mutation sends a transient revision receipt after commit. An incomplete +or uninstrumented mutation sends a reset marker, and the Host reads current +source rows to repair the panel. The transport does not retain a payload +journal or provide a replay guarantee. Browser route errors use `unauthenticated`/non-retryable for `401`, `not_found`/non-retryable for `404`, `invalid_query`/non-retryable for `400`, and `upstream_failure`/retryable for every authorized `5xx`. @@ -418,7 +421,7 @@ closing future reads. | --- | --- | --- | | Surface | `host.conversation` or `conversation.history` | `host.Messages().List` | | Runtime | Native UI bundle | Plugin server process | -| Data | Sanitized browser DTOs and ordered live updates | Typed paginated reader | +| Data | Sanitized browser DTOs and revision-bound updates | Typed paginated reader | | Forbidden shortcut | `host.store`, raw WS, `/api/v1` | Private application imports | ### Frontend hook/API matrix diff --git a/docs/public/plugins-manifest.md b/docs/public/plugins-manifest.md index 273ed45da5..575b62caf4 100644 --- a/docs/public/plugins-manifest.md +++ b/docs/public/plugins-manifest.md @@ -408,9 +408,10 @@ matched, so a plugin subscribed to `shell.output.*` reads the session id off `event_type` as its last segment. For an unsuffixed subject that string is just the subject itself (`task.created`). The table above documents backend `capabilities.events` bus vocabulary. The -browser conversation facade uses a separate Host-only ordered vocabulary for -live history, including `session.turn.started` and `session.turn.completed`; -those names do not change the backend subjects listed here. +browser conversation facade uses a separate Host-only revision-bound v2 +vocabulary for live history, including `session.turn.started` and +`session.turn.completed`; those names do not change the backend subjects listed +here. | Domain | Events | | -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/docs/specs/plugins/requirements/prompt-history-extraction-host.md b/docs/specs/plugins/requirements/prompt-history-extraction-host.md index df2edae2c1..24279cde55 100644 --- a/docs/specs/plugins/requirements/prompt-history-extraction-host.md +++ b/docs/specs/plugins/requirements/prompt-history-extraction-host.md @@ -34,7 +34,7 @@ does not extract or remove the shipped core panel. ### REQ-PLUGINS-PROMPT-HISTORY-HOST-002: Browser Conversation Host Boundary -**Intent:** Provide a sanitized, capability-gated, generation-safe browser conversation facade with deterministic pagination and ordered live reconciliation. +**Intent:** Provide a sanitized, capability-gated, generation-safe browser conversation facade with deterministic pagination and live state reconciliation. #### Acceptance criteria @@ -42,18 +42,15 @@ does not extract or remove the shipped core panel. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.2:** `host.conversation.useSessionMessages` exposes loading, hydrated, pagination, retryable error, and observable terminal `removed` state. `loadMore()` resolves the count of newly projected messages; concurrent calls for one continuation join; exhausted, closed, or terminal handles resolve `0` without network activity; failures reject with `PluginConversationError` and preserve committed state. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.3:** `host.conversation.useSessionTurns` returns typed turn IDs, start/completion timestamps, and freshness state. It applies the same task-selection rules and terminal lifecycle as messages. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.4:** Message pages preserve deterministic `(created_at,id)` ordering, use opaque keyset cursors, expose absolute prompt indexes that do not renumber after deletion, and can load the selected session without initializing or changing the chat transcript cache. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.5:** The Host establishes subscription readiness before resolving the initial snapshot, buffers and merges later adds and updates by stable ID, removes deleted messages, and reconciles after reconnect. No message created between subscription and snapshot is lost or duplicated. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.5:** The Host establishes readiness before the initial read and reconciles additions, updates, and deletions by stable ID. It repairs current state after reconnect. No message created between subscription and snapshot is lost or duplicated. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.6:** Requests, pages, and events are scoped to the requested session generation and every mounted panel owns an independent Host scope, cache, consumer identity, and abort controller. Session switch, unmount, plugin reload, disable, or uninstall aborts pending reads, releases the consumer, closes the handle, and prevents stale completion from mutating a replacement surface. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.7:** Authenticated browser read routes require an active plugin declaring `api_read:messages`, verify access to the task session, and reuse task services and repositories. Unknown or inactive plugins, missing capabilities, invalid cursors, and inaccessible sessions fail closed with typed HTTP errors. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.8:** External plugins do not need a Go backend to consume the browser facade. The existing capability-gated Go Host `Messages().List` API remains available for backend plugins but is not the browser contract and is not expanded solely for prompt history. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.9:** The turns facade consumes typed `session.turn.started` and `session.turn.completed` events after readiness, merges completion by turn ID and freshness, and updates derived prompt durations without retry or remount. Session removal is a terminal barrier for queued turn events. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.10:** Browser failures use `{"error":{"code":"...","message":"...","retryable":false|true}}`: 401 maps to `unauthenticated`, 404 to `not_found`, 400 to `invalid_query`, and authorized upstream 5xx to retryable `upstream_failure`. The first three are non-retryable. Binding-only `409/generation_superseded` is Host-loader-only; continuation-renewal 409 is retryable `upstream_failure`; binding and renewal responses use `Cache-Control: no-store`. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.9:** The turns facade observes turn starts and completions after readiness. It refreshes completion state and derived prompt durations without explicit retry or remount. Session removal is a terminal barrier for queued turn events. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.10:** Browser failures use `{"error":{"code":"...","message":"...","retryable":false|true}}`: 401 maps to `unauthenticated`, 404 to `not_found`, 400 to `invalid_query`, and authorized upstream 5xx to retryable `upstream_failure`. The first three are non-retryable. Binding-only `409/generation_superseded` stays internal to the Host loader. Private reconciliation conflicts recover internally or surface as retryable `upstream_failure`. Conversation responses use `Cache-Control: no-store`. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.11:** Test-only fixture controls can causally request message update/delete and turn-completion transitions. Each control response is tied to its emitted event, so parity tests do not use timing sleeps. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.12:** Every snapshot and public live message/turn DTO has a non-empty `updatedAt`. A timestamp-less unseen add may normalize to its valid `createdAt`; a timestamp-less update to an existing ID is rejected and repaired by the next snapshot. Fixture transitions provide explicit update/completion timestamps. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.13:** Subscription readiness yields a per-session monotonic watermark. Plugin subscribe requests carry an authenticated loader binding for plugin, consumer kind, generation, and a Host-minted per-panel `consumer_id`; core wires carry a distinct Host-minted `wire_id`. Each identity survives reconnect, is never shared, and is released on unsubscribe or after retention. Binding tokens expire no later than 10 minutes after issuance, and the loader rebinds before expiry. Strict results include event watermark, committed snapshot cutoff, and Host-only snapshot token. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.14:** Before buffering or merging, the Host applies the selected task/session matching rule. Inherited `undefined` and explicit task strings require exact session and task equality. Explicit `null` accepts every task ID in the selected session, including `event.task_id:null`. `session.removed` is accepted for the selected session regardless of event task ID and closes both hooks. Wrong-session and wrong-task events do not mutate selected state. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.15:** The facade consumes the production ordered WebSocket envelope and readiness handshake. Raw envelopes are validated against a versioned event registry before projection; outer and payload event types must match. Missing, malformed, unsupported-version, and unknown-type events are durable poison: advancement stops, diagnostics retain event ID and version, and atomic rebind returns a replacement cursor/token past the poison. `session.removed` is terminal whether it precedes or follows poison; its replacement cursor is closed. ACK accepts only the highest contiguous sequence and rejects forward gaps, malformed identity, and invalid tokens without advancing state. The protocol has explicit success and failure ACK envelopes, with no success-only fields on failure. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.16:** Event append, version rows, deletion tombstones, and sequence allocation share one transaction and lock. Delivery cursors persist ACK, owner epoch, lease, retry, and generation state. Poison dispatch is owned by `SessionDeliveryDispatcher`: records use `pending`, `leased`, `exhausted`, and `requeued` states, a 30-second lease, five attempts, exponential backoff capped at 30 seconds, immutable diagnostics, and audited idempotent operator requeue. Startup reclaims expired leases and never silently skips poison. Tokens and reconnect grace are each bounded at 10 minutes; retention preserves resumable cursors, event rows, and terminal tombstones through both bounds. - **AC-PLUGINS-PROMPT-HISTORY-HOST-002.17:** A previously authorized consumer that reconnects to a removed session enters the same terminal state as a live removal event: committed rows remain visible, `removed` is true, `hasMore` is false, and later reads and retries make no network request. A fresh unauthorized or nonexistent lookup remains an ordinary not-found response and does not reveal prior existence. ### REQ-PLUGINS-PROMPT-HISTORY-HOST-003: Task-Panel and Navigation Capabilities @@ -85,9 +82,67 @@ does not extract or remove the shipped core panel. - **AC-PLUGINS-PROMPT-HISTORY-HOST-005.1:** The SDK contract is runtime-free and assignable to the web internal aliases. Existing plugin registrations remain source-compatible; no public contract imports Kandev stores, React runtime, or backend types. - **AC-PLUGINS-PROMPT-HISTORY-HOST-005.2:** A fixture plugin consumes only the public SDK and proves message pagination, live add/update/delete, turn completion, removal, favorites, alias display, task navigation, and desktop/mobile panel placement. -- **AC-PLUGINS-PROMPT-HISTORY-HOST-005.3:** Parity verification compares the fixture plugin with the shipped core panel for ordering, filtering inputs, navigation, alias display, favorites, live updates, removal, and mobile behavior. It includes wrong-session, wrong-task, reconnect, gap, poison, and terminal-removal cases. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-005.3:** Parity verification compares the fixture plugin with the shipped core panel for ordering, filtering inputs, navigation, alias display, favorites, live updates, removal, and mobile behavior. It includes wrong-session, wrong-task, reconnect, missed-notification, malformed-notification, and terminal-removal cases. - **AC-PLUGINS-PROMPT-HISTORY-HOST-005.4:** The package documents the future extraction sequence. It leaves core registrations, layout IDs, stores, locale keys, and core tests untouched; external plugin creation, publication, saved-layout migration, and core removal require a later explicitly approved package. +## Replacement contract, 2026-09-16 + +The user requested a replacement for the storage architecture in PR #3588. +The plugin system owns this contract because it owns Host conversation access. +Task records remain the source of truth. +The new criteria define the replacement. The linked plans track implementation and remaining validation. +Existing panel, navigation, display, authorization, and SDK criteria still apply. + +### REQ-PLUGINS-PROMPT-HISTORY-HOST-006: Conversation access without transcript copies + +**Intent:** Preserve live conversation access without mandatory copies of conversation content. + +#### Acceptance criteria + +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.1:** Host conversation access shall not require persistent copies of message content, turn payloads, or historical versions. This applies with and without installed plugins. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.2:** An upgrade shall not copy or sanitize historical conversation content during startup. Subsequent writes shall not create transcript copies for plugin delivery. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.3:** Open conversation views shall converge to current authorized records after missed notifications, reconnect, server restart, and suspended-tab recovery. Recovery shall include additions, edits, deletions, and completed turns. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.4:** A settled conversation shall become current after the next successful revision check and refresh. Foreground checks shall occur at intervals no greater than five seconds. Recovery shall preserve committed rows on transient failure and expose the existing retryable error. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.5:** Concurrent changes during pagination shall trigger reconciliation without duplicate IDs, resurrected deletions, or silently missing records in the loaded range. Unchanged history shall retain deterministic keyset order. The facade shall not promise historical, fixed-time snapshots. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.6:** Malformed or lost notifications shall cause state repair without permanent replay blockage. Access loss, session removal, and lifecycle replacement shall prevent stale responses from restoring state. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.7:** Public hook signatures, safe DTO fields, native navigation, and core Prompt History ownership shall remain compatible. Desktop and mobile shall recover through the same data contract. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.8:** Idle installations shall perform no conversation payload scans for plugin synchronization. Synchronization work shall be limited to subscribed sessions and requested or already loaded history. + +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.9:** Normal live additions, updates, and deletions shall apply by stable message or turn ID. Unrelated agent output shall not trigger a prompt-history reread. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.10:** A view shall claim freshness only after a current source read or complete coverage of intervening changes. A later observed revision alone shall not hide a missed update. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.11:** Synchronization shall require no message content hashes, hash backfill, or hashing on message writes. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-006.12:** Pagination shall continue after fully applied live changes without rereading the loaded range solely because the revision advanced. + +### REQ-PLUGINS-PROMPT-HISTORY-HOST-007: Safe removal of legacy conversation storage + +**Intent:** Remove legacy duplication without loss of original conversations or unrelated plugin state. + +#### Acceptance criteria + +- **AC-PLUGINS-PROMPT-HISTORY-HOST-007.1:** Fresh installations and upgrades shall reach the same storage contract. Cleanup shall preserve original messages, turns, prompt indexes, timestamps, favorites, plugin packages, and unrelated plugin storage. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-007.2:** Repeated or interrupted upgrades shall recover safely. A failed database cutover shall prevent readiness and preserve a retryable database state. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-007.3:** Cleanup shall remove legacy conversation copies and stop their writers. Cleanup shall not require historical payload decoding or automatic database compaction. +- **AC-PLUGINS-PROMPT-HISTORY-HOST-007.4:** Upgrade documentation shall distinguish logical cleanup from disk-space reclamation. It shall describe backup restoration for downgrade and identify cleanup failures without logging conversation content. + +### Retired transport acceptance criteria + +Criteria 002.13, 002.15, and 002.16 retain their identifiers for historical traceability. +REQ-PLUGINS-PROMPT-HISTORY-HOST-006 replaces their durable replay requirements. +The following text records the prior contract and does not govern new implementation. + +- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.13:** Subscription readiness yields a per-session monotonic watermark. Plugin subscribe requests carry an authenticated loader binding for plugin, consumer kind, generation, and a Host-minted per-panel `consumer_id`; core wires carry a distinct Host-minted `wire_id`. Each identity survives reconnect, is never shared, and is released on unsubscribe or after retention. Binding tokens expire no later than 10 minutes after issuance, and the loader rebinds before expiry. Strict results include event watermark, committed snapshot cutoff, and Host-only snapshot token. + +- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.15:** The facade consumes the production ordered WebSocket envelope and readiness handshake. Raw envelopes are validated against a versioned event registry before projection; outer and payload event types must match. Missing, malformed, unsupported-version, and unknown-type events are durable poison: advancement stops, diagnostics retain event ID and version, and atomic rebind returns a replacement cursor/token past the poison. `session.removed` is terminal whether it precedes or follows poison; its replacement cursor is closed. ACK accepts only the highest contiguous sequence and rejects forward gaps, malformed identity, and invalid tokens without advancing state. The protocol has explicit success and failure ACK envelopes, with no success-only fields on failure. + +- **AC-PLUGINS-PROMPT-HISTORY-HOST-002.16:** Event append, version rows, deletion tombstones, and sequence allocation share one transaction and lock. Delivery cursors persist ACK, owner epoch, lease, retry, and generation state. Poison dispatch is owned by `SessionDeliveryDispatcher`: records use `pending`, `leased`, `exhausted`, and `requeued` states, a 30-second lease, five attempts, exponential backoff capped at 30 seconds, immutable diagnostics, and audited idempotent operator requeue. Startup reclaims expired leases and never silently skips poison. Tokens and reconnect grace are each bounded at 10 minutes; retention preserves resumable cursors, event rows, and terminal tombstones through both bounds. + +## Implementation plans + +- [Remaining validation and remediation](../../../plans/conversation-storage-follow-up/plan.md) owns the outstanding delivery gates. +- [Storage replacement](../../../plans/conversation-storage-replacement/plan.md) owns the new implementation and validation. +- The original Host and recovery packages retain their historical results. + Their durable replay work orders are not instructions to rebuild the retired transport. + ## Source inventory The implementation plan and system design record the current contact-point @@ -98,9 +153,9 @@ product behavior; this document is authoritative only for Host prerequisites. ## Decision and implementation boundary -This is a draft prerequisite package, not approval to implement or extract the -plugin. The plan assigns each criterion to one work order and stops after a -fixture parity proof. Any implementation must preserve the Host-owned security, +The original prerequisite package and storage replacement are implemented. +The linked plans track remaining validation and remediation. +Plugin extraction requires a separate approved package. Any implementation must preserve the Host-owned security, lifecycle, and transport invariants above. See [the system design](../system-design/prompt-history-extraction-host.md) and diff --git a/docs/specs/plugins/system-design/conversation-recovery.md b/docs/specs/plugins/system-design/conversation-recovery.md index 0bc3701ad6..b650434b00 100644 --- a/docs/specs/plugins/system-design/conversation-recovery.md +++ b/docs/specs/plugins/system-design/conversation-recovery.md @@ -1,5 +1,5 @@ --- -status: draft +status: superseded system: plugins requirements: - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 @@ -11,6 +11,16 @@ owners: # Conversation recovery + +## Storage replacement, 2026-09-16 + +The [source reconciliation design](conversation-source-reconciliation.md) now owns the intended conversation transport and storage replacement. +The [replacement plan](../../../plans/conversation-storage-replacement/plan.md) owns its implementation. +The durable journal, fixed-cutoff snapshots, ACKs, poison dispatch, and retention text here records the prior implementation. +It does not require the replacement to preserve those mechanisms. +Authorization, safe DTOs, lifecycle fencing, core compatibility, and visible recovery outcomes remain required. + + ## Boundary and references This document refines recovery in the diff --git a/docs/specs/plugins/system-design/conversation-source-reconciliation.md b/docs/specs/plugins/system-design/conversation-source-reconciliation.md new file mode 100644 index 0000000000..1873df7581 --- /dev/null +++ b/docs/specs/plugins/system-design/conversation-source-reconciliation.md @@ -0,0 +1,359 @@ +--- +status: current +system: plugins +created: 2026-09-16 +owners: + - kandev +requirements: + - REQ-PLUGINS-PROMPT-HISTORY-HOST-002 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-005 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-006 + - REQ-PLUGINS-PROMPT-HISTORY-HOST-007 +--- + +# Conversation source reconciliation + +## Purpose and authority + +This design replaces the durable conversation transport introduced by PR #3588. +The [Host requirement](../requirements/prompt-history-extraction-host.md) owns the public boundary. +The [Host prerequisite design](prompt-history-extraction-host.md) retains authority for panels, navigation, display adapters, safe DTOs, and plugin lifecycle. +This document replaces its journal, version, fixed-cutoff, ACK, poison, and delivery-retention sections. +It also replaces the transport mechanics in [Conversation recovery](conversation-recovery.md). +The public current-state outcomes of recovery remain required. + +The [decision](../../../decisions/2026-09-16-conversation-source-reconciliation.md) explains the storage tradeoff. +The implementation and its focused tests now match this design. + +## Requirement mapping + +| Requirement | Design sections | +| --- | --- | +| REQ-PLUGINS-PROMPT-HISTORY-HOST-002 | Source reads, Scope and authorization, Facade reconciliation | +| REQ-PLUGINS-PROMPT-HISTORY-HOST-005 | Compatibility, Core delivery | +| REQ-PLUGINS-PROMPT-HISTORY-HOST-006 | Revision model, Incremental delivery, Facade reconciliation | +| REQ-PLUGINS-PROMPT-HISTORY-HOST-007 | Upgrade and cleanup, Operational evidence | + +## Current boundaries + +- `internal/task/repository/sqlite/conversation_source.go` owns the revision table, revision-only triggers, and source-backed reads. +- `internal/task/repository/sqlite/conversation_receipts.go` returns transient mutation receipts from transaction-bound message and turn writers. +- `internal/plugins/conversation_handlers.go` owns source route authorization, safe DTO mapping, and revision guards. +- `internal/gateway/websocket/conversation_delivery.go` publishes post-commit v2 receipts to core and plugin scopes. +- `internal/gateway/websocket/client.go` rejects stale ordered session subscriptions after cutover. +- `apps/web/lib/plugins/conversation-source-scope.ts` and `conversation-host.tsx` own source reads, revision reconciliation, and terminal removal. +- `apps/web/lib/ws/client.ts` owns core source projection and recovery coordination. +- `apps/web/hooks/domains/session/use-session-messages.ts` and the existing message/turn hydration owners repair core state. + +## Revision model + +Use a `conversation_session_revisions` table in the task repository. +It contains `session_id` as primary key and a nonnegative `BIGINT revision`. +A foreign key removes the revision with its session. No payload, event ID, timestamp history, or per-message version is stored. +An existing session without a revision row reads as revision zero. +Initialization creates schema and triggers without enumerating messages or turns. + +Small database triggers advance the session revision on relevant message and turn inserts, updates, and deletes. +They replace payload-copy triggers at final cutover. +They cover direct SQL, tool-payload retention, bulk operations, and FK cascades without relying on event publication. +A move between sessions advances both surviving session revisions. +Trigger upserts select an existing session before insertion, so cascading session deletion cannot recreate an orphan revision. +Session absence itself is the terminal signal. No durable deletion tombstone is needed. + +Source mutation and revision increment share the transaction. +Concurrent PostgreSQL writers serialize through atomic row upserts and updates. +Callers that control multiple session mutations acquire revision locks in deterministic session order. +Arbitrary bulk SQL can still deadlock on PostgreSQL. A deadlock aborts the whole transaction and uses the existing caller retry policy. +SQLite uses its existing single-writer contract. +A failed mutation or trigger rolls back both source and revision changes. +Tests cover real multi-connection PostgreSQL writes and cascades. + +A process boot epoch supplements the database revision in Host-only responses. +A changed epoch forces reconciliation after restart or restore. +Revision tokens serialize as decimal strings to avoid JavaScript integer precision loss. +No client can select a past revision for an as-of read. +No message content hash, hash column, hash backfill, or per-write content hashing is added. +Cryptographic signatures on authorization and cursor tokens remain unrelated to content hashes. + +### Mutation receipts + +Normal message and turn writers return a transient mutation receipt from the source transaction. +It contains the session, base revision, committed revision, and every operation represented by that revision interval. +Operations carry stable IDs, previous selection/order fields, and the resulting persisted record or deletion identity. +The receipt is an in-memory result. No receipt, changed-ID journal, or old payload is persisted. + +The transaction creates the revision row lazily when absent, then locks it before reading its base value. +It captures the final revision and source result before commit, after all relevant triggers run. +Publication occurs only after commit. A later database read cannot assign a revision to an older event payload. +This prevents a concurrent writer from making incomplete data appear current. + +The normal single-record writer accounts for one revision increment. +A transaction with several increments must describe the complete interval, or publish a reset notification instead of a certified change batch. +Bulk writers and cascades without complete receipts use reset or periodic discrepancy detection. +Direct SQL still increments the revision even without any receipt or bus publication. +It therefore leaves a detectable gap. A process crash after commit has the same recovery behavior. +Cross-session moves produce a receipt per surviving session and preserve deterministic lock order. + +Migrate message creation, content append, tool updates, permission/clarification updates, and turn completion to this receipt boundary. +Do not duplicate original source writes to capture receipts. +Uninstrumented exceptional paths remain correct through recovery, but normal streaming cannot use reset as its default. + +## Source reads + +Extend `ConversationReader` through the task service and repository interfaces. +Remove plugin SQL access after caller migration. +The implemented methods read a message page, a turn page, and a revision/status tuple. +Private expansion routes are `GET /api/plugins/{pluginId}/conversation/v2/task-sessions/{sessionId}/{messages|turns|revision}`. +The existing binding route remains unchanged. Final source readers remain on these v2 paths. +Core readers expose revision metadata through their existing authorized first-party handlers. +Each page reads source rows, session existence, and revision in one short database transaction. +SQLite uses a consistent read transaction. PostgreSQL uses read-only repeatable-read isolation. +No transaction spans HTTP requests. + +Message order stays `(created_at,id)`. Preserve durable `prompt_seq`, tri-state task selection, author filters, and sort direction. +Message and turn queries use bounded pages with a maximum of 100 rows. +Turn keysets use `(started_at,id)` and the same task selection and revision binding. +The Host assembles turn pages internally while the public turns hook keeps its existing shape. +Safe DTO mapping occurs after authorized source reads. System blocks and arbitrary metadata remain excluded. + +The Host-only v2 page envelope adds `{epoch, revision, cursor, hasMore}` to the existing records. +Signed cursors bind the user, plugin generation, session, task selection, filter, sort, page size, keyset position, epoch, and expiry. +The current 10-minute cursor lifetime remains a bound, not a snapshot retention promise. +Cursors identify a query and keyset boundary, not a frozen revision. +A continuation separately sends the scope's fully applied revision as `expected_revision`. +The page transaction compares that revision with its source revision. +A mismatch returns private `409/reconciliation_required`, with no partial page commit. +The Host first drains already received contiguous batches and retries once with its newer applied revision. +If coverage is incomplete, it repairs the loaded range before continuation. +Ordinary applied updates therefore do not invalidate every pagination cursor or force a full refresh. +The Host handles that condition internally. Exhausted recovery maps to existing retryable `upstream_failure`. +Malformed or mismatched tokens remain `400/invalid_query`. Expired tokens trigger one fresh authorized read cycle in the Host. +No cutoff renewal route or stored token cursor is needed after migration. + +## Scope and authorization + +Preserve authenticated plugin binding, active-generation checks, and `api_read:messages`. +Every read, revision check, and subscription applies current session authorization. +No task or session selection can expand through a caller-provided token. +Bindings remain separate from pagination. The signing key remains because binding and cursor signatures still need it. + +The cache key includes panel, plugin generation, session, tri-state task selection, authors, sort, and page size. +Each panel owns its request generation and abort controller. +Unsubscribe, disable, uninstall, reload, and session switch cancel reads and invalidate pending commits. + +Fresh inaccessible or nonexistent lookups return the same `404/not_found` envelope. +For a handle with a previous successful authorized read, a later 404 closes the handle and stops reads. +It retains committed rows and exposes terminal `removed` state with the not-found error. +This state means that the scope is no longer available. It does not distinguish deletion from revoked access. +A 401 closes authorization with `unauthenticated`. No background retries continue after either terminal authorization outcome. +No durable receipt, retained deletion payload, or existence-disclosing lookup is added. + +## Incremental delivery + +Use the Host-only v2 subscription actions `session.conversation.subscribe` and +`session.conversation.unsubscribe` with `protocol_version: 2`. +Plugin requests carry plugin ID, generation, binding token, and Host-created scope identity. +Core requests carry their distinct Host-created scope identity. The server rejects mixed identity branches. +The server binds plugin scopes to their task selection and author filters before readiness. +Readiness identifies the subscription, process epoch, and current revision. It does not prove that a client loaded the source state. + +`session.conversation.changed` carries `{epoch, base_revision, revision, operations}` and the bound subscription identity. +Each operation is a message or turn upsert, or a removal by ID. +Upserts carry the current full safe DTO, not a text append delta. +Repeating an upsert replaces the same ID instead of appending text again. +Payloads are transient network data, not database copies or retained event history. + +The gateway projects receipts through the scope's authorization and selection rules. +An update that leaves a filter becomes removal. An update that enters a filter becomes an upsert. +An unrelated change sends an empty operations list for its revision interval. +That empty batch proves the interval has no effect on the selection without exposing another task's IDs or content. +Only a validated, complete receipt can produce such a coverage batch. +The plugin sees safe DTOs only. Core scopes use their existing richer authorized projections. + +The gateway can coalesce adjacent, fully accounted intervals for at most 100 ms. +It can retain only the last operation per entity when the batch preserves the final selected state and all covered revisions. +It must not coalesce across a missing interval or omit a relevant removal. +Uncertain coalescing produces reset, not a false claim of coverage. +An empty batch can be coalesced with adjacent covered batches without a payload read. + +### Applied revision and discrepancy checks + +Each scope tracks `appliedRevision` and `observedRevision` separately. +A successfully committed source snapshot establishes `appliedRevision = R`. +Only complete contiguous change batches or another source snapshot can advance it. +A revision check or readiness message advances only `observedRevision`. + +For a batch with base B and final revision R: + +- If B equals `appliedRevision`, apply all selected operations atomically, then set `appliedRevision = R`. +- If R is no greater than `appliedRevision`, discard it as already covered. +- If B exceeds `appliedRevision`, buffer briefly for an out-of-order predecessor, without claiming freshness. +- If the interval partially overlaps applied state, repair rather than infer which operations are safe to skip. +- If the epoch or scope differs, reject the batch and use the appropriate lifecycle recovery. + +An empty operations list follows the same rules. It advances coverage without changing records or triggering history reads. +After revision 10, receiving only a batch covering 11 to 12 cannot hide the missing 10 to 11 batch. +A check reporting revision 12 also cannot advance the applied revision from 10. + +Revision observations use `session.conversation.changed` with `check: true`, equal +`base_revision` and `revision`, and an empty operations list. These frames report +source state without proving delivery of any revision interval. +A `terminal: true` frame closes only the identified subscription when its session +or authorization is no longer available. It contains no message or turn payload. + +One gateway worker checks revision and existence for subscribed sessions every five seconds. +It batches bounded session-ID queries, rechecks authorization, and reads no payloads. +No subscribers means no checks. Checks equal to the applied revision cause no history reads. +A newer observed revision waits up to one second for pending delivery before requesting recovery. +An older revision or different epoch triggers a fresh authorized snapshot. +Reconnect and visibility return also reconcile before claiming freshness. + +Per-scope buffering is bounded by one second, 256 operations, and 1 MiB of encoded payload, whichever limit occurs first. +A larger batch, overflow, missing predecessor, malformed batch, or explicit reset schedules source recovery. +Large individual messages remain readable through source pages. The bounded live path can reset for them. +The existing socket backpressure path disconnects if it cannot deliver a reset safely. +No durable ACK, replay log, poison queue, or permanent per-consumer cursor is introduced. + +## Facade reconciliation + +### Initial load and repair + +Subscribe before reading source records. Buffer transient batches within the limits above during hydration. +Fetch the requested message range and turn pages at one common revision R. +If pages disagree, discard the staged cycle and start a fresh cycle. +Commit matching message pages, turns, cursor, and revision atomically for the scope. +Discard buffered batches already covered by R, then apply contiguous later batches. +Compare the resulting applied revision with the latest observed revision before claiming freshness. +A gap schedules repair. Polling detects loss after the final comparison. + +Repair replaces the loaded range by authoritative IDs, including deletions. +For a deleted boundary record, retain its key values as the range boundary. +Fetch from the current head through the retained boundary, with the equivalent upper boundary for ascending order. +A changed ordering key requires range repair unless the projection can prove complete range membership. +Do not initialize or mutate `messages.bySession` for a plugin read. + +### Normal live changes + +A matching message upsert replaces or adds that ID in deterministic order. +A removal deletes that ID. A turn upsert changes only that turn and the derived duration state. +Existing rows outside the loaded range remain unloaded. +New head records enter the loaded range without discarding older loaded records. +A move across selection boundaries uses the receipt's previous selection fields to remove obsolete entries. +Unchanged filtered projections receive coverage-only batches and perform no history read. + +Live content updates do not refresh messages or turns in bulk. +A prompt-only panel ignores agent output content, while its turn subscription still receives completion changes. +Unusual range-membership changes, missing coverage, and explicit reset use repair. + +### Pagination and failure + +`loadMore` extends the range by one requested page using the applied revision and signed keyset cursor. +Buffer batches during the request. Commit the page at its returned revision, then apply contiguous later batches. +Concurrent calls join. The result counts newly projected IDs against the original call's committed view. +A cursor does not require renewal merely because a fully applied message update advanced the revision. + +Only one repair runs per scope. Additional discrepancies set one dirty bit. +Automatic repair starts at most once per second per scope. +Each cycle attempts at most three consistent reads with existing bounded retry delays. +Persistent churn retains committed state and exposes retryable failure rather than false freshness. +The next scheduled check or explicit retry starts another bounded cycle. +Reconnect, expiry, authorization loss, removal, and generation changes retain the lifecycle rules in this design. + +## Core delivery + +Core consumers use the same source-backed v2 subscription as plugin consumers. +Preserve low-latency streaming through the same transaction-bound mutation receipts and applied-revision accounting. +Core projections retain rich DTOs and optimistic-message rules. Plugin projections remain sanitized and cache-isolated. +Use the existing core hydration coordinator only for initial load or repair. +A normal contiguous event updates its ID and does not invoke broad hydration. + +Remove the old journal adapter and avoid parallel legacy-plus-v2 projection of the same message. +Other non-conversation WebSocket traffic remains outside this revision protocol. +Transport deduplication is in-memory. The system does not promise durable exactly-once event delivery. + +Recovery refreshes messages and turns and removes stale cached server rows. +Preserve pending local messages and invalidate unloaded cache windows under existing pagination ownership. +Buffer only bounded transient batches during recovery, then discard covered revisions and apply contiguous successors. +Consumers without a mounted hydration owner remain dirty until an owner mounts. +Terminal session removal cancels hydration and prevents resurrection. + +Audit all message and turn publishers when removing journal fanout. +Normal source writers need complete receipts, not a second best-effort payload path with independently sampled revisions. +Include task deletion, workspace cascades, Quick Chat, Office, workflow cleanup, and direct repository deletion in revision/existence tests. +Exceptional deletions without receipts recover through the revision check or session absence. +No new universal deletion outbox is required for a current-state view. + +## Compatibility + +The public SDK hook and task-panel shapes do not change. +`PLUGIN-API.md` must describe current-state pagination and the replacement Host-only wire protocol at cutover. +Private wire v2 is the production conversation transport after this sequential +implementation package. +The final release contains one synchronization implementation and no fallback journal. +No intermediate work order is independently released. + +An old browser with v1 state receives an explicit unsupported-protocol outcome after cutover. +An existing browser tab requires a full reload to fetch the matching frontend. Document this upgrade step. +Compatibility-only legacy fixtures do not define the production conversation contract. +Reconnect and reload reauthorize plugins before data access. +Old plugin bundles still use the compatible Host facade supplied by the current application. +The existing minimum Host version is not changed merely because an internal transport changes. + +Desktop remains a dockview panel. Mobile retains Panels navigation, its inset picker, full-height panel, and one scroller. +No markup, controls, touch behavior, navigation layout, or copy change is proposed. +Existing mobile fixture tests prove the shared data path with touch navigation. +No new UI preview is necessary for this internal state replacement. + +## Upgrade and cleanup + +The package ships one forward migration, not a bare Git revert. +Stop old backend processes before migration. Existing startup backup and migration admission remain in force. +No mixed-version writers or rolling old/new backend deployment is supported during this cutover. + +Replace `initConversationJournalSchema` in schema initialization before it can run a historical backfill. +The cutover transaction removes the exact legacy SQLite triggers and PostgreSQL triggers/functions. +It installs revision-only triggers and removes the five legacy tables: +`conversation_session_streams`, `conversation_session_events`, `conversation_message_versions`, +`conversation_turn_versions`, `conversation_journal_meta`, plus no unrelated tables. +Use catalog-aware, explicit names. Do not use a wildcard or PostgreSQL `CASCADE` that can remove unrelated dependencies. +All source rows, indexes, and prompt sequences remain intact. +Use the repository migration context and propagate failures before readiness. + +Fresh, fully migrated, partially backfilled, and partially present legacy schemas take the same idempotent path. +Failure injection between DDL steps proves rollback. Reopen and repeat prove replay safety. +Large legacy table removal can still cause one-time I/O. No row decoding, copying, DELETE loop, or automatic VACUUM is permitted. + +Remove event worker composition, journal DB injection, and every runtime open of the Host event SQLite file. +After database cutover commits, remove only `.host/session-events.sqlite` and its `-wal` and `-shm` siblings. +Validate that each target is the expected regular file under the configured plugin root. Refuse symlinks and unexpected paths. +Do not remove `.host` itself, `conversation-token.key`, plugin storage, packages, approvals, or unrelated files. +File cleanup is idempotent and runs without opening or parsing the legacy database. +A file cleanup failure logs the exact non-content path and leaves an explicit retry on the next startup. +It does not restart the obsolete writer or roll back the successful primary cutover. + +Existing backup policy remains unchanged. PostgreSQL backups remain operator-managed. +SQLite DROP frees pages for reuse but does not guarantee a smaller database file. +Disk reclamation uses the existing operator compaction workflow after backup. +Never trigger startup VACUUM to improve a size screenshot. +Downgrade after cleanup requires a pre-upgrade backup and matching binary, not a journal recreation migration. + +## Operational evidence + +Log migration duration, cleanup completion/failure, and revision-refresh failures without payloads or credentials. +Use existing structured logging. No session IDs as metric labels and no new diagnostics dashboard. + +A disposable benchmark records 10k and 100k messages with fixed 1 KiB content before and after cutover. +Record database/WAL bytes, allocated/free pages, cold and repeated startup time, and source-write latency. +Also record history-read count and bytes for a large loaded prompt range during agent streaming. +After initial hydration, 100 contiguous agent updates must cause zero history rereads in a prompt-only scope. +Matching message updates and turn completion must update by ID without a bulk refresh. +Injected delivery loss must cause bounded repair and restore correct visible state. +Separate backup time, cleanup time, and ordinary boot time. +Structural gates require zero legacy payload tables/files after successful cleanup and no per-message plugin-history writes. +Revision-row count cannot exceed surviving session count and cannot grow with repeated writes to one session. +Timing results inform review. They do not invent a hardware-independent startup SLA. + +## Implementation plans + +- [Storage replacement](../../../plans/conversation-storage-replacement/plan.md) records the implementation and SQLite operational evidence. +- [Remaining gates](../../../plans/conversation-storage-follow-up/plan.md) owns PostgreSQL coverage, backend failures, and final recovery E2E checks. diff --git a/docs/specs/plugins/system-design/prompt-history-extraction-host.md b/docs/specs/plugins/system-design/prompt-history-extraction-host.md index c8bd242bfb..115a0691a3 100644 --- a/docs/specs/plugins/system-design/prompt-history-extraction-host.md +++ b/docs/specs/plugins/system-design/prompt-history-extraction-host.md @@ -14,6 +14,16 @@ owners: # Prompt History Plugin Host Prerequisites System Design + +## Storage replacement, 2026-09-16 + +The [source reconciliation design](conversation-source-reconciliation.md) now owns the intended conversation transport and storage replacement. +The [replacement plan](../../../plans/conversation-storage-replacement/plan.md) owns its implementation. +The durable journal, fixed-cutoff snapshots, ACKs, poison dispatch, and retention text here records the prior implementation. +It does not require the replacement to preserve those mechanisms. +Authorization, safe DTOs, lifecycle fencing, core compatibility, and visible recovery outcomes remain required. + + ## Purpose and boundaries This design adds the narrow public browser Host contracts required for a future external prompt-history plugin. The host continues to own authorization, transcript persistence, transport reconciliation, task-panel placement, native navigation, custom-prompt disclosure, and browser-local favorite state. The future plugin owns prompt-list derivation and presentation through those contracts. @@ -76,7 +86,7 @@ Classification meanings: |`internal/task/service.Service` and repository interfaces|Authorized task/session reads and message/turn access.|covered but contract change required|Add a narrow conversation read interface exposing actor/session authorization, plural author filters, deterministic keyset pages, turns, and minimal DTO inputs; plugin handlers call this interface, never SQLite directly.| |`internal/backendapp/helpers.go`, backend composition, and plugin route registration|Wires plugin handlers with their dependencies.|lacking API contract|Inject the narrow conversation read interface into `internal/plugins` and preserve normal user/workspace authorization; add composition-level tests proving inaccessible sessions do not leak existence.| |`internal/plugins/host_data.go`, mappers, `pkg/pluginsdk`, `plugin.proto`|Separate Go backend-plugin conversation contract.|unchanged|No browser prompt-history change; it is not used by the UI bundle.| -|Task message/turn event publication and WebSocket session subscription|Sanitized message/turn/removal events.|contract change|`SessionEventStream` is the sole durable per-session log, sequencer, and fanout. Mutation services append transactionally; migrate subject subscriptions and direct broadcasts. Client coalescing preserves sequence, holds gaps, and never publishes a second stream.| +|Task message/turn event publication and WebSocket session subscription|Sanitized message/turn/removal events.|contract change|Source transactions return transient receipts after commit. The v2 gateway publishes revision-bound changes to core and plugin scopes; source reads repair gaps and session absence is terminal.| ### Shared types, contracts, and documentation @@ -103,62 +113,119 @@ Classification meanings: ## Public browser API contract -[`PLUGIN-API.md`](../../../plans/plugins/PLUGIN-API.md#browser-conversation-facade) is the canonical declaration for the runtime-free TypeScript types, Host-only wire envelopes, retryability mapping, lifecycle, and compatibility policy. The SDK and web Host must remain structurally assignable to that declaration. +[PLUGIN-API.md](../../../plans/plugins/PLUGIN-API.md#hostconversation---live-paginated-session-history) +is the canonical declaration for the runtime-free TypeScript types, source +revision behavior, lifecycle, and private v2 wire envelopes. The SDK and web +Host remain structurally assignable to that declaration. Implementation invariants: -- Browser conversation state exposes sanitized message and turn DTOs, including non-empty `updatedAt`, nullable task selection, hydration, pagination, retry, and terminal removal state. It never exposes cursors, tokens, sidecars, event payloads, Kandev stores, or React runtime values. -- `taskId: undefined` inherits the active panel task, `null` selects all tasks in the session, and an explicit string must equal the panel task. Mismatches fail before network activity. -- `HostReact` includes structural `useLayoutEffect`; task-panel registration keeps literal `title` as the fallback and adds optional localized title, typed visibility, session kind, and scoped message navigation. -- The Host owns custom-prompt disclosure and read-only favorite observation. Plugins receive only `host.ui.PromptMentionText` and `host.conversation.useMessageFavorite`. -- Persisted update timestamps map to `updatedAt`; a missing row update timestamp falls back to `createdAt`. An unseen timestamp-less add may use `createdAt`, while a timestamp-less update is rejected for snapshot repair. Deletes remain terminal by ID. +- Browser conversation state exposes sanitized message and turn DTOs, including non-empty updatedAt, nullable task selection, hydration, pagination, retry, and terminal removal state. It never exposes cursors, revision tokens, source sidecars, event payloads, Kandev stores, or React runtime values. +- taskId: undefined inherits the active panel task, null selects all tasks in the session, and an explicit string must equal the panel task. Mismatches fail before network activity. +- The Host owns custom-prompt disclosure and read-only favorite observation. Plugins receive only host.ui.PromptMentionText and host.conversation.useMessageFavorite. +- Persisted update timestamps map to updatedAt; a missing row update timestamp falls back to createdAt. Deletes remain terminal by ID. +- Desktop and phone panels share the same scope, DTOs, filtering, pagination, and navigation capability. Phone presentation keeps the existing full-height Chat surface and local transcript scroll owner. ## Browser conversation facade -`buildHostApi` creates a plugin-scoped conversation implementation alongside `storage`. The task-panel wrapper injects `history` as an opaque, generation-bound handle; its hook IDs must match the wrapper's task/session context, while no panel context yields only the nullable empty state. Hook returns are immutable SDK DTOs; no SDK type imports `AppState`, Zustand, Kandev HTTP types, or session-event payloads. Before import, the loader calls authenticated `GET /api/plugins/{id}/conversation/binding`. Success is `{bindingToken,generation,expiresAt}` with RFC3339 UTC `expiresAt` <=10m; every response, including errors, sends `Cache-Control: no-store`. Plugin-visible errors use the canonical envelope with only `401/unauthenticated/false` or `404/not_found/false`; binding-only `409/generation_superseded/true` stays loader-internal with bounded retry and never reaches plugin code. On `401` or `404` the current load is skipped without disabling persisted state; the loader never imports until binding succeeds. revoked grants rebind or abort. `host.ts` keeps grants Host-only; reload stages grant/import/init, preserving old state until successor commit and fencing only pending state on failure. Disable, uninstall, or replacement revokes old grants. +buildHostApi creates a plugin-scoped conversation implementation alongside storage. +The task-panel wrapper injects history as an opaque, generation-bound handle. +No panel context yields the nullable empty state. Hook results are immutable SDK +DTOs and do not import AppState, Zustand, Kandev HTTP types, or WebSocket +payloads. -Messages are stored internally in ascending deterministic order and projected into requested order. Equal in-flight requests join per scope/query. Cache identity includes panel/plugin ID, session ID, tri-state task ID, author filter, sort, page size, and generation; cursor fingerprints use the same identity. Live matching requires exact session/task equality for inherited or explicit task strings, accepts every task ID for explicit null, and accepts `session.removed` for the selected session regardless of event task ID. Each panel has an independent cache, Host-minted consumer identity, and abort controller. Concurrent calls for one continuation join; exhausted and removed states perform no network activity. Reads never initialize `messages.bySession`. +Before plugin import, the loader calls authenticated +GET /api/plugins/{pluginId}/conversation/binding. Success contains +bindingToken, generation, and an RFC3339 UTC expiresAt no later than ten minutes +after issuance. Every response, including errors, sends Cache-Control: no-store. +Binding-only generation superseded responses stay inside the loader. A failed +binding skips the current load without disabling persisted state. Reload stages +grant, import, and initialization, preserving the old state until the successor +commits; disable, uninstall, replacement, and failed reload revoke the old grant. + +Messages and turns are read from current source pages in deterministic keyset +order. Equal in-flight requests join per scope and query. Cache identity includes +panel and plugin ID, session ID, tri-state task ID, author filter, sort, page +size, and generation. A source scope owns its cache, Host-minted scope ID, abort +controller, epoch, and decimal applied revision. Reads never initialize +messages.bySession. + +Live source changes require the matching scope ID and session ID. A complete +receipt applies operations by entity ID after its base revision matches the +scope's applied revision. A reset marker, malformed operation, wrong epoch, +revision gap, stale base, or failed operation starts a source read. Changes that +arrive before the relevant snapshot commits are buffered and drained after the +source page and revision are installed together. An empty operation list is a +valid coverage-only change when the revision interval is contiguous. + +The process epoch changes after restart or restore. Revisions are decimal +strings to avoid JavaScript precision loss. Source notifications are transient +and published after the source transaction commits. There is no durable payload +journal, ACK protocol, poison queue, replay promise, content hash, or +caller-selectable as-of read. Session removal arrives on the normal +session.removed channel, retains projected rows, sets removed, and closes +pagination and retry for every matching scope. ## Backend read routes -Add authenticated plugin-scoped routes before the wildcard webhook routes: +The browser Host uses these source-backed routes: -```text -GET /api/plugins/{pluginId}/conversation/task-sessions/{sessionId}/messages -GET /api/plugins/{pluginId}/conversation/task-sessions/{sessionId}/turns -GET /api/plugins/{pluginId}/conversation/binding -POST /api/plugins/{pluginId}/conversation/continuation/renew -``` +GET /api/plugins/{pluginId}/conversation/v2/task-sessions/{sessionId}/messages +GET /api/plugins/{pluginId}/conversation/v2/task-sessions/{sessionId}/turns +GET /api/plugins/{pluginId}/conversation/v2/task-sessions/{sessionId}/revision +GET /api/plugins/{pluginId}/conversation/binding -Queries bound `limit` (default 20, max 100), opaque keyset `cursor`, repeated supported `author_type`, and `sort`; invalid/empty filters return `400`. Public responses are `{ messages, hasMore, cursor }` with `cursor: string | null` and `{ turns }`; `cursor` is null when no next page exists. Exact Host-only wire request/response and event payload DTOs are canonical in the `PLUGIN-API.md` Host-only conversation wire contract. Internally responses carry `snapshotCutoff` and row-sequence sidecars. The Host sends the subscribe ACK's binding `snapshot_token` in the Host-only `X-Kandev-Snapshot-Token` header; server validation applies the committed cutoff and `rowSequence <= cutoff` to every page, seeking the cursor key directly. The cursor encodes `(created_at,id,direction,cutoff,fingerprint,expiry,generation)` and rejects tampering or mismatch with `400`; it is never an offset. Before 80% expiry Host-only `continuation/renew` accepts cursor/token and returns the same cutoff/fingerprint with new expiry; renewal never picks a new cutoff. Renewal success and error responses use `Cache-Control: no-store`. If retention/generation cannot preserve it, the Host-only renew returns `409`, surfaced to plugin code as retryable `upstream_failure`; `loadMore` leaves state/cursor unchanged and explicit retry starts fresh. Each panel scope has monotonic `continuation_revision` sentinel `0` over `{cursor,snapshot_token,cutoff,fingerprint,generation}`. Renewal and `loadMore` serialize per scope and CAS-install matching revisions; page, cursor, and `hasMore` commit atomically. Stale responses drop; failed renewal leaves the tuple unchanged; close/retry invalidates it. +The source message and turn readers require an authenticated identity, an active +plugin with api_read:messages, and normal task/session authorization. They call +task service and repository interfaces. Responses contain sanitized camel-case +DTOs and private epoch and revision metadata; arbitrary metadata, raw content, +and system blocks remain excluded. expected_revision is an optional decimal +guard. Bounded query filters preserve the existing author, task, sort, and +keyset semantics. First-party REST and Go RPC contracts remain unchanged. -Both handlers: +The private v2 WebSocket actions are +session.conversation.subscribe, +session.conversation.unsubscribe, and session.conversation.changed. +Subscribe binds the scope to the authenticated user, plugin capability, +generation, session, and optional task and author filters. Core uses the same +source reader with consumer_kind core. Plugin uses consumer_kind plugin with +plugin ID, generation, and binding token. Scope IDs are opaque and unique per +mounted consumer. -1. require an authenticated Kandev identity; -2. resolve an active installed plugin and require `api_read:messages`; -3. resolve the task session through the normal user/workspace authorization boundary; -4. call task services and repository interfaces, not SQL; -5. map to the narrow camel-case browser-plugin DTO; -6. strip Kandev system content and omit raw content and arbitrary metadata; -7. return stable error codes for authentication, capability, input, inaccessible session, and server failures. +A successful subscribe returns protocol_version 2, scope_id, session_id, epoch, +and decimal revision. A changed notification returns protocol_version 2, +scope_id, session_id, epoch, base_revision, revision, optional reset, and +operations. Message and turn operations carry only the fields needed for the +existing core projection and sanitized plugin DTO. The notification is +published only after commit. Filtering can produce a coverage-only empty +operation list while still advancing the revision. -Handlers use a conversation-specific resolver: absent, inactive, and capability-missing plugins collapse to sanitized `404` before session authorization, unlike `activeRecord`'s `503`. All responses use `{"error":{"code":"...","message":"...","retryable":false|true}}`; the stable errors are `401/unauthenticated/false`, `404/not_found/false`, `400/invalid_query/false`, and authorized `5xx/upstream_failure/true`. Binding and continuation-renewal success and error responses use `Cache-Control: no-store`. A narrow exact-path auth error-writer branch emits the same `401` envelope; no broad plugin-route exemption is allowed. +Core WebSocket hydration uses a per-session source subscription. Its reducer +validates scope, epoch, decimal revisions, contiguous intervals, and operation +shape. It applies message and turn operations by ID and triggers the existing +authorized snapshot recovery on a gap, reset, malformed value, or epoch change. +The core session state remains the product's projection owner. Normal +session.removed delivery remains terminal. -Message mapping reads only the allowlisted `metadata.sender_task_id` key; non-empty string values become `senderTaskId`, while missing, empty, or non-string values omit that field. Turns omit metadata/runtime; first-party REST and Go RPC contracts remain unchanged. - -`SessionEventStream` owns session sequencing and fanout for message, turn, and removal events. Subscribe is discriminated: a core request requires the Host-minted `wire_id`; a plugin request requires `plugin_id`, `generation`, `binding_token`, and its Host-minted `consumer_id`. Identity fields are mutually exclusive. Bindings check user, capability, and generation, and registration atomically captures the event watermark. - -The exact Host-only subscribe/ACK envelopes, legacy parsing rules, result states, and token field presence are canonical in the `PLUGIN-API.md` Host-only conversation wire contract. `SessionEventStream` validates outer IDs, session/binding identity, integer ranges, contiguous ACK advancement, and current/prior retry tokens. It binds resume and snapshot tokens to user, plugin, session, consumer, generation, and cutoff; snapshot commit precedes ACK and releases buffered post-cutoff events. `fresh`, `replay`, `gap`, and `invalid_resume` retain their documented reconciliation semantics. Events use `{type:"session.event",protocol_version,event_type,session_id,task_id, sequence,event_id,payload}`. Canonical policy is `{1:{ignorable:["session.workspace_sources.updated"]}}`; only a type listed by its matching version may advance ACK without projection. Missing, malformed, or unsupported versions and other unknown types remain durable poison: the cursor stays before the event, diagnostics retain its ID/version, and advancement stops. Rebind atomically snapshots past poison at a new cutoff, commits that cutoff, and returns a replacement token/cursor; the poison is not projected. `session.removed` is terminal regardless of whether it precedes or follows poison, and never becomes a legacy data update. - -Snapshot reads are current-state reads captured at committed cutoff, not an as-of query. `SessionMessageVersion` and `SessionTurnVersion` are immutable, keyed/indexed by `(entity_id,row_sequence)` plus session/task identity; each entity selects maximum `row_sequence <= cutoff`. A tombstone at or below cutoff excludes its entity, while a later tombstone remains post-cutoff; apply the terminal session barrier after selection. The adapter strips sidecars, seeds row sequences, buffers post-cutoff events, deduplicates reflected rows, then replays. Core delivery uses a per-wire `legacy-core/` cursor: reconnecting the same wire resumes its cursor, a new wire starts at its registration watermark, and each wire gets at most one projection per `event_id`/sequence. Projection precedes ACK; `message.added|updated|deleted` and `session.turn.started|session.turn.completed` map to existing core notifications, while removal is terminal. Merge uses ID with sequence as ordering/fencing authority; `updatedAt` is freshness only. - -The durable append API returns `appended`, `duplicate`, `late_terminal`, or `sequence_conflict`; identity/payload conflicts are poison and late-terminal events drop after fencing. Version, tombstone, and log rows allocate one locked per-session sequence in one transaction; immutable `SessionEventLog` rows and `SessionDeliveryCursor` rows hold ACK, identity, epoch, lease/retry/generation. `SessionDeliveryDispatcher` owns poison: 30s lease, five attempts, 30s-capped backoff. Host-only `session.event.poison.requeue` requires `session_events:requeue`, audits actor/epoch, is idempotent, and never auto-skips. Startup reclaims leases. Tokens and reconnect grace are 10m; GC and tombstones through both bounds. Bus publication is not replay truth. +Stale legacy session.subscribe or session.unsubscribe requests that carry +ordered stream fields are rejected after cutover. No plugin or core path +recreates a durable conversation stream. The source migration is one forward +startup cutover: revision-only triggers are installed, the five legacy +conversation tables are dropped transactionally, and the old +.host/session-events.sqlite file and sidecars are removed only after the +database cutover succeeds. The cleanup validates each target with lstat, +refuses symlinks and non-regular files, and can retry on the next startup. ## Test-only transition controls The fixture exposes response-bounded message patch/delete and turn-completion controls. Handlers persist before typed events; completion takes RFC3339. Fixture source is `apps/web/e2e/fixtures/plugins/prompt-history-plugin/`; `e2e-plugin-ui` cleans/builds before packaging, and archive checks source/output hashes, capability, min version, and panel key. -Deletes and session removal are terminal barriers. All production callers, including plugin `PluginOwnedTaskTree`, workspace resource/task FK cascades, task/session repositories, handoff/task-agent cascades, quick-chat expiry, profile/automation/office cleanup, workflow replacement/compensation, reference cleanup, and MCP handlers invoke one transaction-aware `SessionDeletionService`. It locks/captures sessions, reserves terminal sequences, and inserts unique outbox rows before deletion/commit; pre-listing, standalone deletes, and unobserved FK cascades are forbidden. Partial cascades, rollback, retries, and duplicates are idempotent; leased post-commit dispatch handles startup drain, recovery, backoff, requeue, duplicate ACK, and poison. +Deletes and session removal are terminal barriers. Source revision triggers cover +message and turn deletes, bulk operations, and foreign-key cascades. A missing +session is terminal for a current-state reader. A missed receipt, rollback, +retry, or process restart is repaired by a consistent source read; no durable +deletion payload, consumer cursor, delivery worker, or universal deletion +outbox is required for this view. ## Task-panel contract