From 75cb354fbe47d2a104ae6fdb3a4ffb6be757a6cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Flor=C3=AAncio?= Date: Mon, 3 Aug 2026 10:07:12 +0100 Subject: [PATCH] fix: isolate high-volume session stream traffic (#2175) --- .../backend/cmd/mock-agent/reasoning_burst.go | 51 +++ .../cmd/mock-agent/reasoning_burst_test.go | 60 +++ apps/backend/cmd/mock-agent/script.go | 5 + .../agent/runtime/lifecycle/manager_events.go | 3 + .../runtime/lifecycle/manager_events_test.go | 72 +++ .../runtime/lifecycle/manager_lifecycle.go | 1 + .../runtime/lifecycle/manager_streaming.go | 170 ++++---- .../runtime/lifecycle/stream_coalescer.go | 158 +++++++ .../lifecycle/stream_coalescer_test.go | 147 +++++++ .../internal/agent/runtime/lifecycle/types.go | 2 + .../server/adapter/transport/acp/adapter.go | 44 +- .../transport/acp/adapter_close_test.go | 36 ++ .../adapter/transport/acp/conversion_test.go | 54 +++ .../agentctl/server/process/manager.go | 4 +- .../internal/clarification/canceller.go | 1 + .../internal/clarification/canceller_test.go | 9 + .../internal/gateway/websocket/client.go | 211 +++++++-- .../websocket/client_send_queue_test.go | 349 +++++++++++++++ .../websocket/client_write_pump_test.go | 108 +++++ .../backend/internal/gateway/websocket/hub.go | 6 +- .../websocket/notification_scheduler.go | 410 ++++++++++++++++++ apps/web/app/office/tasks/[id]/page.tsx | 14 +- .../tasks/[id]/use-session-live-sync.test.tsx | 92 ++++ .../tasks/[id]/use-session-live-sync.ts | 66 +++ apps/web/e2e/helpers/api-client.ts | 1 + .../e2e/helpers/session-stream-overload.ts | 73 ++++ .../chat/agent-message-attribution.spec.ts | 33 +- .../e2e/tests/chat/message-pagination.spec.ts | 15 + ...-session-stream-overload-isolation.spec.ts | 101 +++++ .../session-stream-overload-isolation.spec.ts | 99 +++++ apps/web/lib/ws/handlers/messages.test.ts | 198 +++++++++ apps/web/lib/ws/handlers/messages.ts | 180 ++++++-- apps/web/lib/ws/handlers/turns.test.ts | 24 +- apps/web/lib/ws/handlers/turns.ts | 7 +- apps/web/lib/ws/router.ts | 10 +- apps/web/lib/ws/use-websocket.test.tsx | 4 +- apps/web/lib/ws/use-websocket.tsx | 7 +- ...late-replaceable-session-stream-traffic.md | 120 +++++ docs/decisions/INDEX.md | 1 + .../session-stream-overload-isolation/plan.md | 247 +++++++++++ .../task-01-coalesce-agent-stream-ingress.md | 108 +++++ ...02-isolate-replaceable-session-delivery.md | 104 +++++ .../task-03-bound-frontend-stream-work.md | 106 +++++ .../task-04-prove-overload-containment.md | 139 ++++++ .../platform/bounded-task-status-delivery.md | 82 +++- 45 files changed, 3518 insertions(+), 214 deletions(-) create mode 100644 apps/backend/cmd/mock-agent/reasoning_burst.go create mode 100644 apps/backend/cmd/mock-agent/reasoning_burst_test.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/stream_coalescer.go create mode 100644 apps/backend/internal/agent/runtime/lifecycle/stream_coalescer_test.go create mode 100644 apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_close_test.go create mode 100644 apps/backend/internal/gateway/websocket/client_write_pump_test.go create mode 100644 apps/backend/internal/gateway/websocket/notification_scheduler.go create mode 100644 apps/web/app/office/tasks/[id]/use-session-live-sync.test.tsx create mode 100644 apps/web/app/office/tasks/[id]/use-session-live-sync.ts create mode 100644 apps/web/e2e/helpers/session-stream-overload.ts create mode 100644 apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts create mode 100644 apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts create mode 100644 apps/web/lib/ws/handlers/messages.test.ts create mode 100644 docs/decisions/2026-08-02-isolate-replaceable-session-stream-traffic.md create mode 100644 docs/plans/session-stream-overload-isolation/plan.md create mode 100644 docs/plans/session-stream-overload-isolation/task-01-coalesce-agent-stream-ingress.md create mode 100644 docs/plans/session-stream-overload-isolation/task-02-isolate-replaceable-session-delivery.md create mode 100644 docs/plans/session-stream-overload-isolation/task-03-bound-frontend-stream-work.md create mode 100644 docs/plans/session-stream-overload-isolation/task-04-prove-overload-containment.md diff --git a/apps/backend/cmd/mock-agent/reasoning_burst.go b/apps/backend/cmd/mock-agent/reasoning_burst.go new file mode 100644 index 0000000000..43b2d0dec7 --- /dev/null +++ b/apps/backend/cmd/mock-agent/reasoning_burst.go @@ -0,0 +1,51 @@ +package main + +import ( + "fmt" + "strings" +) + +const ( + reasoningBurstDefaultCount = 1000 + reasoningBurstMaxCount = 100_000 +) + +// parseReasoningBurstCommand parses the deterministic script command used by +// overload tests. The command emits no delay and keeps the count visible in a +// final marker message so an E2E test can prove the producer actually ran. +func parseReasoningBurstCommand(line string) (count int, ok bool) { + for _, prefix := range []string{"e2e:reasoning_burst(", "e2e:reasoning-burst("} { + if !strings.HasPrefix(line, prefix) { + continue + } + count = extractIntArg(line, prefix) + if count <= 0 { + count = reasoningBurstDefaultCount + } + if count > reasoningBurstMaxCount { + count = reasoningBurstMaxCount + } + return count, true + } + return 0, false +} + +func reasoningBurstChunk(index int) string { + return fmt.Sprintf("reasoning-burst-%06d|", index) +} + +func reasoningBurstContent(count int) string { + var content strings.Builder + content.Grow(count * len("reasoning-burst-000000|")) + for index := 1; index <= count; index++ { + content.WriteString(reasoningBurstChunk(index)) + } + return content.String() +} + +func emitReasoningBurst(e *emitter, count int) { + for index := 1; index <= count; index++ { + e.thought(reasoningBurstChunk(index)) + } + e.text(fmt.Sprintf("reasoning-burst-produced:%d", count)) +} diff --git a/apps/backend/cmd/mock-agent/reasoning_burst_test.go b/apps/backend/cmd/mock-agent/reasoning_burst_test.go new file mode 100644 index 0000000000..45ce88d341 --- /dev/null +++ b/apps/backend/cmd/mock-agent/reasoning_burst_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "strings" + "testing" +) + +func TestParseReasoningBurstCommand(t *testing.T) { + for _, test := range []struct { + line string + count int + ok bool + }{ + {line: "e2e:reasoning_burst(12)", count: 12, ok: true}, + {line: "e2e:reasoning-burst(12)", count: 12, ok: true}, + {line: "e2e:reasoning_burst(0)", count: reasoningBurstDefaultCount, ok: true}, + {line: "e2e:message(12)", count: 0, ok: false}, + } { + count, ok := parseReasoningBurstCommand(test.line) + if count != test.count || ok != test.ok { + t.Errorf("parseReasoningBurstCommand(%q) = (%d, %v), want (%d, %v)", test.line, count, ok, test.count, test.ok) + } + } + if count, _ := parseReasoningBurstCommand("e2e:reasoning_burst(999999)"); count != reasoningBurstMaxCount { + t.Fatalf("burst count should be capped at %d, got %d", reasoningBurstMaxCount, count) + } +} + +func TestEmitReasoningBurstProducesExactContentAndMarker(t *testing.T) { + const count = 37 + e, mock := newTestEmitter() + emitReasoningBurst(e, count) + + updates := mock.getUpdates() + if len(updates) != count+1 { + t.Fatalf("expected %d updates, got %d", count+1, len(updates)) + } + var thought strings.Builder + for index := 0; index < count; index++ { + if !isThoughtUpdate(updates[index]) { + t.Fatalf("update %d should be a thought", index) + } + thought.WriteString(getThoughtContent(updates[index])) + } + if got, want := thought.String(), reasoningBurstContent(count); got != want { + t.Fatalf("reasoning content mismatch: got %q, want %q", got, want) + } + if got, want := getTextContent(updates[count]), "reasoning-burst-produced:37"; got != want { + t.Fatalf("produced marker = %q, want %q", got, want) + } +} + +func TestExecuteScriptReasoningBurst(t *testing.T) { + e, mock := newTestEmitter() + executeScript(e, "", "e2e:reasoning_burst(3)") + updates := mock.getUpdates() + if len(updates) != 4 { + t.Fatalf("expected three thought chunks plus marker, got %d", len(updates)) + } +} diff --git a/apps/backend/cmd/mock-agent/script.go b/apps/backend/cmd/mock-agent/script.go index d5c64b9e8b..dca44d6a93 100644 --- a/apps/backend/cmd/mock-agent/script.go +++ b/apps/backend/cmd/mock-agent/script.go @@ -40,6 +40,11 @@ func executeCommand(e *emitter, fullPrompt, line string) { text := extractStringArg(line, "e2e:thinking(") e.thought(text) + case strings.HasPrefix(line, "e2e:reasoning_burst(") || strings.HasPrefix(line, "e2e:reasoning-burst("): + if count, ok := parseReasoningBurstCommand(line); ok { + emitReasoningBurst(e, count) + } + case strings.HasPrefix(line, "e2e:delay("): ms := extractIntArg(line, "e2e:delay(") fixedDelay(ms) diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_events.go b/apps/backend/internal/agent/runtime/lifecycle/manager_events.go index d46d0c19cb..0e47cf937e 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_events.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_events.go @@ -391,6 +391,7 @@ func (m *Manager) handleToolCallEvent(execution *AgentExecution, event agentctl. // handleToolUpdateEvent stores completed tool results in session history. func (m *Manager) handleToolUpdateEvent(execution *AgentExecution, event agentctl.AgentEvent) { if event.ParentToolCallID == "" && isTerminalToolUpdate(event) { + m.flushStreamCoalescer(execution) execution.clearActiveTool(event.ToolCallID) } if m.historyManager != nil && execution.historyEnabled && execution.SessionID != "" && event.ToolStatus == toolStatusComplete { @@ -573,6 +574,7 @@ func (m *Manager) handleStreamDisconnect( return } + m.flushMessageBuffer(execution) m.flushAssistantHistory(execution) m.persistExecutorRunning(context.Background(), updated) m.publishStreamDisconnectError(execution, err) @@ -583,6 +585,7 @@ func (m *Manager) handleStreamDisconnect( // after promptDoneCh is signaled. Drain the partial assistant transcript // here as well as at prompt setup; the shared buffer lock makes either // path the single owner and prevents a later reset from dropping it. + m.flushMessageBuffer(execution) m.flushAssistantHistory(execution) if err := m.UpdateStatus(execution.ID, v1.AgentStatusFailed); err != nil { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go b/apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go index 94ae125a52..ee166e025b 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go @@ -176,6 +176,7 @@ func TestHandleAgentEvent_ProtocolMessageResumesAcrossToolCall(t *testing.T) { Text: "after tool", ProtocolMessageID: "acp-message-1", }) + mgr.flushStreamCoalescer(execution) messageEvents := streamEventsOfType(eventBus, "message_streaming") if len(messageEvents) != 2 { @@ -377,6 +378,76 @@ func TestHandleAgentEvent_LegacyThinkingFlushesBeforeProtocolThinking(t *testing } } +func TestHandleAgentEvent_ProtocolThinkingBurstIsCoalescedBeforeCompletion(t *testing.T) { + mgr, eventBus := createTestManagerWithTracking() + execution := createTestExecution("exec-1", "task-1", "session-1") + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatalf("add execution: %v", err) + } + + const chunkCount = 20 + for index := 0; index < chunkCount; index++ { + text := "b" + if index == 0 { + text = "a" + } + mgr.handleAgentEvent(execution, agentctl.AgentEvent{ + Type: "reasoning", + ReasoningText: text, + ProtocolMessageID: "burst-thinking", + }) + } + mgr.handleAgentEvent(execution, agentctl.AgentEvent{Type: "complete"}) + + var thinkingEvents []AgentStreamEventPayload + var streamed strings.Builder + for _, event := range eventBus.getStreamEvents() { + if event.Data == nil || event.Data.Type != "thinking_streaming" { + continue + } + thinkingEvents = append(thinkingEvents, event) + streamed.WriteString(event.Data.Text) + } + if got := len(thinkingEvents); got > 2 { + t.Fatalf("thinking publication count = %d, want at most 2 for %d chunks", got, chunkCount) + } + if got := streamed.String(); got != "abbbbbbbbbbbbbbbbbbb" { + t.Fatalf("coalesced thinking content = %q, want exact burst content", got) + } +} + +func TestHandleAgentEvent_LegacyNewlineBurstIsCoalescedBeforeCompletion(t *testing.T) { + mgr, eventBus := createTestManagerWithTracking() + execution := createTestExecution("exec-1", "task-1", "session-1") + if err := mgr.executionStore.Add(execution); err != nil { + t.Fatalf("add execution: %v", err) + } + + const chunkCount = 20 + var want strings.Builder + for index := 0; index < chunkCount; index++ { + text := "legacy chunk\n" + want.WriteString(text) + mgr.handleAgentEvent(execution, agentctl.AgentEvent{ + Type: "message_chunk", + Text: text, + }) + } + mgr.handleAgentEvent(execution, agentctl.AgentEvent{Type: "complete"}) + + messageEvents := streamEventsOfType(eventBus, "message_streaming") + if got := len(messageEvents); got > 2 { + t.Fatalf("message publication count = %d, want at most 2 for %d ID-less chunks", got, chunkCount) + } + var streamed strings.Builder + for _, event := range messageEvents { + streamed.WriteString(event.Data.Text) + } + if got := streamed.String(); got != want.String() { + t.Fatalf("coalesced legacy content = %q, want exact burst content", got) + } +} + func TestHandleAgentEvent_ProtocolAssistantHistoryPersistedOnceInWireOrder(t *testing.T) { mgr, _ := createTestManagerWithTracking() history, err := NewSessionHistoryManager(t.TempDir(), "", newTestLogger()) @@ -1029,6 +1100,7 @@ func TestHandleAgentEvent_SubagentToolCallDoesNotSplitStreaming(t *testing.T) { Type: "message_chunk", Text: "on the new PR.\n", }) + mgr.flushStreamCoalescer(execution) var streamingEvents []AgentStreamEventPayload for _, e := range eventBus.getStreamEvents() { diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go index 43d620f0dd..0f30ba726c 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go @@ -312,6 +312,7 @@ func (m *Manager) CleanupStaleExecutionBySessionID(ctx context.Context, sessionI func (m *Manager) RemoveExecution(executionID string) { m.releaseActivity(executionActivityKey(executionID)) if execution, ok := m.executionStore.Get(executionID); ok { + m.closeStreamCoalescer(execution) m.cleanupPassthroughMCPConfig(execution) m.setRuntimeInterest(execution.SessionID, false) } diff --git a/apps/backend/internal/agent/runtime/lifecycle/manager_streaming.go b/apps/backend/internal/agent/runtime/lifecycle/manager_streaming.go index b98e0fb64e..65e01b2988 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/manager_streaming.go +++ b/apps/backend/internal/agent/runtime/lifecycle/manager_streaming.go @@ -16,6 +16,69 @@ func (e *AgentExecution) clearProtocolMessageCorrelationLocked() { e.protocolThinkingIDs = nil } +func (m *Manager) streamCoalescer(execution *AgentExecution) *streamCoalescer { + execution.streamMu.Lock() + defer execution.streamMu.Unlock() + if execution.stream == nil { + execution.stream = newStreamCoalescer(defaultStreamCoalesceWindow, func(chunk coalescedStreamChunk) { + m.publishStreamingContentNow( + execution, + chunk.eventType, + chunk.messageID, + chunk.content, + chunk.isAppend, + ) + }) + } + return execution.stream +} + +func (m *Manager) flushStreamCoalescer(execution *AgentExecution) { + execution.streamMu.Lock() + stream := execution.stream + execution.streamMu.Unlock() + if stream != nil { + stream.flushBoundary() + } +} + +func (m *Manager) closeStreamCoalescer(execution *AgentExecution) { + execution.streamMu.Lock() + stream := execution.stream + execution.stream = nil + execution.streamMu.Unlock() + if stream != nil { + stream.close() + stats := stream.stats() + if stats.received > 0 { + m.logger.Info("agent stream coalescer closed", + zap.String("execution_id", execution.ID), + zap.String("session_id", execution.SessionID), + zap.Int("stream_chunks_received", stats.received), + zap.Int("stream_chunks_coalesced", stats.coalesced), + zap.Int("stream_segments_flushed", stats.flushed)) + } + } +} + +func (m *Manager) enqueueStreamingContent( + execution *AgentExecution, + eventType string, + messageID string, + content string, + isAppend bool, +) { + if content == "" { + return + } + m.streamCoalescer(execution).add(coalescedStreamChunk{ + eventType: eventType, + messageID: messageID, + content: content, + isAppend: isAppend, + }) +} + func (e *AgentExecution) resetStreamingStateLocked() { e.messageBuffer.Reset() e.thinkingBuffer.Reset() @@ -89,6 +152,12 @@ func resetStreamingStateWithHistory( historyManager *SessionHistoryManager, log *logger.Logger, ) { + execution.streamMu.Lock() + stream := execution.stream + execution.streamMu.Unlock() + if stream != nil { + stream.flushBoundary() + } execution.messageMu.Lock() content := execution.detachAssistantHistoryLocked() execution.resetStreamingStateLocked() @@ -133,6 +202,9 @@ func (m *Manager) flushPendingLegacyMessage(execution *AgentExecution) { return } if messageID != "" { + // The final direct append must follow any coalesced legacy chunk that + // was already emitted for this record. + m.flushStreamCoalescer(execution) m.publishStreamingMessageFinal(execution, messageID, trimmed) return } @@ -157,6 +229,9 @@ func (m *Manager) flushPendingLegacyThinking(execution *AgentExecution) { return } if messageID != "" { + // The final direct append must follow any coalesced legacy chunk that + // was already emitted for this record. + m.flushStreamCoalescer(execution) m.publishStreamingThinkingFinal(execution, messageID, trimmed) return } @@ -175,7 +250,7 @@ func (m *Manager) publishProtocolThinking( messageID, isAppend := protocolRecordID(&execution.protocolThinkingIDs, protocolMessageID) execution.messageMu.Unlock() - m.publishStreamingContent(execution, thinkingStreamingEventType, messageID, content, isAppend) + m.enqueueStreamingContent(execution, thinkingStreamingEventType, messageID, content, isAppend) } func (m *Manager) publishStreamingContent( @@ -184,6 +259,16 @@ func (m *Manager) publishStreamingContent( messageID string, content string, isAppend bool, +) { + m.enqueueStreamingContent(execution, eventType, messageID, content, isAppend) +} + +func (m *Manager) publishStreamingContentNow( + execution *AgentExecution, + eventType string, + messageID string, + content string, + isAppend bool, ) { event := AgentStreamEventData{ Type: eventType, @@ -222,32 +307,13 @@ func (m *Manager) publishStreamingMessage(execution *AgentExecution, content str } execution.messageMu.Unlock() - event := AgentStreamEventData{ - Type: "message_streaming", - Text: content, - MessageID: messageID, - IsAppend: isAppend, - } - - // Create payload manually to include streaming-specific fields - payload := &AgentStreamEventPayload{ - Type: "agent/event", - Timestamp: time.Now().UTC().Format(time.RFC3339Nano), - AgentID: execution.ID, - ExecutionID: execution.ID, - TaskID: execution.TaskID, - SessionID: execution.SessionID, - Data: &event, - } - m.logger.Debug("publishing streaming message", zap.String("execution_id", execution.ID), zap.String("message_id", messageID), zap.Bool("is_append", isAppend), zap.Int("content_length", len(content))) - // Publish the streaming event - orchestrator will handle create/append logic - m.eventPublisher.PublishAgentStreamEventPayload(payload) + m.enqueueStreamingContent(execution, "message_streaming", messageID, content, isAppend) } // flushMessageBuffer extracts any accumulated message from the buffer and returns it. @@ -255,6 +321,7 @@ func (m *Manager) publishStreamingMessage(execution *AgentExecution, content str // It also clears the currentMessageID to start fresh for the next message segment. // Additionally flushes any accumulated thinking content. func (m *Manager) flushMessageBuffer(execution *AgentExecution) string { + m.flushStreamCoalescer(execution) execution.messageMu.Lock() agentMessage := execution.messageBuffer.String() thinkingContent := execution.thinkingBuffer.String() @@ -311,29 +378,12 @@ func (m *Manager) flushMessageBuffer(execution *AgentExecution) string { // publishStreamingMessageFinal publishes the final chunk of a streaming message. // This is called during flush to append any remaining buffered content. func (m *Manager) publishStreamingMessageFinal(execution *AgentExecution, messageID, content string) { - event := AgentStreamEventData{ - Type: "message_streaming", - Text: content, - MessageID: messageID, - IsAppend: true, - } - - payload := &AgentStreamEventPayload{ - Type: "agent/event", - Timestamp: time.Now().UTC().Format(time.RFC3339Nano), - AgentID: execution.ID, - ExecutionID: execution.ID, - TaskID: execution.TaskID, - SessionID: execution.SessionID, - Data: &event, - } - m.logger.Debug("publishing final streaming message chunk", zap.String("execution_id", execution.ID), zap.String("message_id", messageID), zap.Int("content_length", len(content))) - m.eventPublisher.PublishAgentStreamEventPayload(payload) + m.publishStreamingContentNow(execution, "message_streaming", messageID, content, true) } // publishStreamingThinking publishes a streaming thinking event for real-time thinking updates. @@ -351,56 +401,18 @@ func (m *Manager) publishStreamingThinking(execution *AgentExecution, content st } execution.messageMu.Unlock() - event := AgentStreamEventData{ - Type: thinkingStreamingEventType, - Text: content, - MessageID: thinkingID, - IsAppend: isAppend, - MessageType: "thinking", - } - - // Create payload manually to include streaming-specific fields - payload := &AgentStreamEventPayload{ - Type: "agent/event", - Timestamp: time.Now().UTC().Format(time.RFC3339Nano), - AgentID: execution.ID, - ExecutionID: execution.ID, - TaskID: execution.TaskID, - SessionID: execution.SessionID, - Data: &event, - } - - // Publish the streaming event - orchestrator will handle create/append logic - m.eventPublisher.PublishAgentStreamEventPayload(payload) + m.enqueueStreamingContent(execution, thinkingStreamingEventType, thinkingID, content, isAppend) } // publishStreamingThinkingFinal publishes the final chunk of a streaming thinking message. // This is called during flush to append any remaining buffered thinking content. func (m *Manager) publishStreamingThinkingFinal(execution *AgentExecution, thinkingID, content string) { - event := AgentStreamEventData{ - Type: thinkingStreamingEventType, - Text: content, - MessageID: thinkingID, - IsAppend: true, - MessageType: "thinking", - } - - payload := &AgentStreamEventPayload{ - Type: "agent/event", - Timestamp: time.Now().UTC().Format(time.RFC3339Nano), - AgentID: execution.ID, - ExecutionID: execution.ID, - TaskID: execution.TaskID, - SessionID: execution.SessionID, - Data: &event, - } - m.logger.Debug("publishing final streaming thinking chunk", zap.String("execution_id", execution.ID), zap.String("thinking_id", thinkingID), zap.Int("content_length", len(content))) - m.eventPublisher.PublishAgentStreamEventPayload(payload) + m.publishStreamingContentNow(execution, thinkingStreamingEventType, thinkingID, content, true) } // updateExecutionError updates an execution with an error diff --git a/apps/backend/internal/agent/runtime/lifecycle/stream_coalescer.go b/apps/backend/internal/agent/runtime/lifecycle/stream_coalescer.go new file mode 100644 index 0000000000..aa5d0623fe --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/stream_coalescer.go @@ -0,0 +1,158 @@ +package lifecycle + +import ( + "sync" + "time" +) + +const defaultStreamCoalesceWindow = 100 * time.Millisecond + +type coalescedStreamChunk struct { + eventType string + messageID string + content string + isAppend bool +} + +// streamCoalescer combines adjacent append chunks for one execution. The +// first non-empty chunk of a record is emitted immediately; later chunks wait for the +// bounded window or an explicit lifecycle boundary. A single pending segment +// is intentional: combining across another message ID would change wire +// ordering. +type streamCoalescer struct { + emitMu sync.Mutex + mu sync.Mutex + window time.Duration + pending *coalescedStreamChunk + timer *time.Timer + closed bool + publish func(coalescedStreamChunk) + lastEventType string + lastMessageID string + forceImmediate bool + received int + coalesced int + flushed int +} + +type streamCoalescerStats struct { + received int + coalesced int + flushed int +} + +func newStreamCoalescer(window time.Duration, publish func(coalescedStreamChunk)) *streamCoalescer { + if window <= 0 { + window = defaultStreamCoalesceWindow + } + return &streamCoalescer{window: window, publish: publish} +} + +func (c *streamCoalescer) add(chunk coalescedStreamChunk) { + if chunk.content == "" { + return + } + + c.emitMu.Lock() + defer c.emitMu.Unlock() + + var ready []coalescedStreamChunk + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return + } + c.received++ + + sameAsLast := c.lastEventType == chunk.eventType && c.lastMessageID == chunk.messageID + immediate := !chunk.isAppend || c.forceImmediate || !sameAsLast + c.forceImmediate = false + switch { + case immediate: + ready = c.detachLocked(ready) + ready = append(ready, chunk) + case c.pending != nil && c.pending.eventType == chunk.eventType && c.pending.messageID == chunk.messageID: + c.pending.content += chunk.content + c.coalesced++ + default: + ready = c.detachLocked(ready) + pending := chunk + c.pending = &pending + c.coalesced++ + c.timer = time.AfterFunc(c.window, c.flush) + } + c.lastEventType = chunk.eventType + c.lastMessageID = chunk.messageID + c.mu.Unlock() + + c.publishReady(ready) +} + +func (c *streamCoalescer) flush() { + c.emitMu.Lock() + defer c.emitMu.Unlock() + c.mu.Lock() + ready := c.detachLocked(nil) + c.mu.Unlock() + c.publishReady(ready) +} + +func (c *streamCoalescer) flushBoundary() { + c.emitMu.Lock() + defer c.emitMu.Unlock() + c.mu.Lock() + c.forceImmediate = true + ready := c.detachLocked(nil) + c.mu.Unlock() + c.publishReady(ready) +} + +func (c *streamCoalescer) close() { + c.emitMu.Lock() + defer c.emitMu.Unlock() + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return + } + c.closed = true + ready := c.detachLocked(nil) + c.mu.Unlock() + c.publishReady(ready) +} + +func (c *streamCoalescer) detachLocked(ready []coalescedStreamChunk) []coalescedStreamChunk { + // A timer callback that already fired can be waiting on emitMu while add + // installs a later pending segment and timer. When that callback acquires + // emitMu it may detach the later segment immediately, collapsing its window; + // content and ordering remain correct because all detaches are serialized. + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + if c.pending != nil { + ready = append(ready, *c.pending) + c.pending = nil + c.flushed++ + } + return ready +} + +func (c *streamCoalescer) stats() streamCoalescerStats { + c.mu.Lock() + defer c.mu.Unlock() + return streamCoalescerStats{ + received: c.received, + coalesced: c.coalesced, + flushed: c.flushed, + } +} + +func (c *streamCoalescer) publishReady(chunks []coalescedStreamChunk) { + if c.publish == nil { + return + } + for _, chunk := range chunks { + c.publish(chunk) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/stream_coalescer_test.go b/apps/backend/internal/agent/runtime/lifecycle/stream_coalescer_test.go new file mode 100644 index 0000000000..f736e7d6fd --- /dev/null +++ b/apps/backend/internal/agent/runtime/lifecycle/stream_coalescer_test.go @@ -0,0 +1,147 @@ +package lifecycle + +import ( + "strings" + "sync" + "testing" + "testing/synctest" + "time" +) + +func TestStreamCoalescerFirstChunkIsImmediateAndAppendsAreWindowed(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + var got []coalescedStreamChunk + var gotMu sync.Mutex + coalescer := newStreamCoalescer(100*time.Millisecond, func(chunk coalescedStreamChunk) { + gotMu.Lock() + defer gotMu.Unlock() + got = append(got, chunk) + }) + + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "a"}) + gotMu.Lock() + if len(got) != 1 || got[0].content != "a" { + gotMu.Unlock() + t.Fatalf("first chunk = %#v, want immediate publication", got) + } + gotMu.Unlock() + + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "b", isAppend: true}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "c", isAppend: true}) + gotMu.Lock() + if len(got) != 1 { + gotMu.Unlock() + t.Fatalf("append chunks published before window: %d", len(got)) + } + gotMu.Unlock() + + time.Sleep(99 * time.Millisecond) + synctest.Wait() + gotMu.Lock() + if len(got) != 1 { + gotMu.Unlock() + t.Fatalf("append flushed before window: %d", len(got)) + } + gotMu.Unlock() + time.Sleep(1 * time.Millisecond) + synctest.Wait() + gotMu.Lock() + defer gotMu.Unlock() + if len(got) != 2 || got[1].content != "bc" || !got[1].isAppend { + t.Fatalf("windowed append = %#v, want one bc append", got) + } + }) +} + +func TestStreamCoalescerPreservesLargeBurstExactly(t *testing.T) { + var got []coalescedStreamChunk + coalescer := newStreamCoalescer(time.Hour, func(chunk coalescedStreamChunk) { + got = append(got, chunk) + }) + + const chunkCount = 30_000 + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "a"}) + for i := 1; i < chunkCount; i++ { + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "b", isAppend: true}) + } + coalescer.flush() + + if len(got) != 2 { + t.Fatalf("publication count = %d, want 2", len(got)) + } + var content strings.Builder + for _, chunk := range got { + content.WriteString(chunk.content) + } + if content.String() != "a"+strings.Repeat("b", chunkCount-1) { + t.Fatalf("burst content was not preserved exactly") + } +} + +func TestStreamCoalescerFlushesBeforeInterleavedRecord(t *testing.T) { + var got []coalescedStreamChunk + coalescer := newStreamCoalescer(time.Hour, func(chunk coalescedStreamChunk) { + got = append(got, chunk) + }) + + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "a", content: "a"}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "a", content: "1", isAppend: true}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "b", content: "b"}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "b", content: "2", isAppend: true}) + coalescer.flush() + + if len(got) != 4 { + t.Fatalf("publication count = %d, want 4 ordered segments", len(got)) + } + want := []string{"a", "1", "b", "2"} + for index, chunk := range got { + if chunk.content != want[index] { + t.Fatalf("chunk %d = %#v, want content %q", index, chunk, want[index]) + } + } +} + +func TestStreamCoalescerBoundaryMakesNextAppendImmediate(t *testing.T) { + var got []coalescedStreamChunk + coalescer := newStreamCoalescer(time.Hour, func(chunk coalescedStreamChunk) { + got = append(got, chunk) + }) + + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "a"}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "b", isAppend: true}) + coalescer.flushBoundary() + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "c", isAppend: true}) + + if len(got) != 3 || got[1].content != "b" || got[2].content != "c" { + t.Fatalf("boundary output = %#v, want separate b and immediate c segments", got) + } +} + +func TestStreamCoalescerCloseFlushesAndRejectsFutureChunks(t *testing.T) { + var got []coalescedStreamChunk + coalescer := newStreamCoalescer(time.Hour, func(chunk coalescedStreamChunk) { + got = append(got, chunk) + }) + + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "a"}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "b", isAppend: true}) + coalescer.close() + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "c", isAppend: true}) + coalescer.close() + + if len(got) != 2 || got[1].content != "b" { + t.Fatalf("close output = %#v, want first plus final b segment", got) + } +} + +func TestStreamCoalescerStatsCountReceivedMergedAndFlushedSegments(t *testing.T) { + coalescer := newStreamCoalescer(time.Hour, func(coalescedStreamChunk) {}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "a"}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "b", isAppend: true}) + coalescer.add(coalescedStreamChunk{eventType: "thinking_streaming", messageID: "m1", content: "c", isAppend: true}) + coalescer.flush() + + if got := coalescer.stats(); got.received != 3 || got.coalesced != 2 || got.flushed != 1 { + t.Fatalf("stats = %+v, want received=3 coalesced=2 flushed=1", got) + } +} diff --git a/apps/backend/internal/agent/runtime/lifecycle/types.go b/apps/backend/internal/agent/runtime/lifecycle/types.go index 9d115ec032..23a6f2dd49 100644 --- a/apps/backend/internal/agent/runtime/lifecycle/types.go +++ b/apps/backend/internal/agent/runtime/lifecycle/types.go @@ -117,6 +117,8 @@ type AgentExecution struct { messageBuffer strings.Builder thinkingBuffer strings.Builder messageMu sync.Mutex + streamMu sync.Mutex + stream *streamCoalescer // Legacy streaming message tracking for agents that omit protocol message IDs. // These are set when we create a streaming message and cleared on tool_call/complete. diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go index 3e52487808..f4f9d0832b 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go @@ -256,6 +256,11 @@ type Adapter struct { // Synchronization mu sync.RWMutex closed bool + // closedCh is closed as soon as shutdown begins so direct sendUpdate + // callers stop waiting before updatesCh is closed. updateSendWg lets Close + // wait for those callers before closing the channel. + closedCh chan struct{} + updateSendWg sync.WaitGroup // promptTurn tracks the in-flight session/prompt RPC so Cancel can interrupt it // and wait for acknowledgment before reporting success. @@ -337,6 +342,7 @@ func NewAdapter(cfg *shared.Config, log *logger.Logger) *Adapter { asyncTurnEpochs: make(map[string]uint64), lifetimeCtx: ctx, lifetimeCancel: cancel, + closedCh: make(chan struct{}), } a.wakeup = newWakeupScheduler(l, a.fireWakeup) // Start the update worker before returning so any caller that connects @@ -501,12 +507,38 @@ func (a *Adapter) SetPermissionHandler(handler PermissionHandler) { } // sendUpdate safely sends an event to the updates channel. -// It checks the closed flag under read-lock to prevent panics on closed channels. +// +// Stream events are lossless at this boundary: when the per-agent consumer is +// slower than the ACP producer, the ACP update worker applies backpressure to +// that agent instead of silently dropping transcript content. The lifetime +// context makes the wait cancelable during shutdown. Do not hold a.mu while +// waiting; Close needs the write lock to cancel the context and unblock us. func (a *Adapter) sendUpdate(event AgentEvent) { a.mu.RLock() - defer a.mu.RUnlock() - if !a.sendUpdateLocked(event) && !a.closed { - a.logger.Warn("updates channel full, dropping event", zap.String("type", event.Type)) + if a.closed { + a.mu.RUnlock() + return + } + updatesCh := a.updatesCh + lifetimeCtx := a.lifetimeCtx + if lifetimeCtx == nil { + lifetimeCtx = context.Background() + } + closedCh := a.closedCh + if closedCh != nil { + a.updateSendWg.Add(1) + } + a.mu.RUnlock() + if closedCh != nil { + defer a.updateSendWg.Done() + } + + select { + case updatesCh <- event: + case <-closedCh: + case <-lifetimeCtx.Done(): + // Shutdown cancels the wait. The event is intentionally discarded after + // the adapter has become terminal; no later transcript can be applied. } } @@ -532,6 +564,9 @@ func (a *Adapter) Close() error { return nil } a.closed = true + if a.closedCh != nil { + close(a.closedCh) + } a.mu.Unlock() a.logger.Info("closing ACP adapter") @@ -552,6 +587,7 @@ func (a *Adapter) Close() error { // handleACPUpdate may call sendUpdate, so updatesCh must remain open // until the worker is gone. a.workerWg.Wait() + a.updateSendWg.Wait() a.mu.Lock() a.clearCodexSubagentCorrelationsLocked("") a.clearPromptHandoffToolTrackingLocked() diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_close_test.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_close_test.go new file mode 100644 index 0000000000..5359d6f82d --- /dev/null +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/adapter_close_test.go @@ -0,0 +1,36 @@ +package acp + +import ( + "testing" + "time" + + "github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared" + "github.com/kandev/kandev/internal/agentctl/types/streams" + "github.com/kandev/kandev/internal/common/logger" +) + +func TestAdapterCloseUnblocksDirectUpdateSender(t *testing.T) { + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatalf("logger: %v", err) + } + a := NewAdapter(&shared.Config{AgentID: "close-test", WorkDir: t.TempDir()}, log) + for index := 0; index < cap(a.updatesCh); index++ { + a.updatesCh <- AgentEvent{Type: streams.EventTypeReasoning} + } + + senderDone := make(chan struct{}) + go func() { + a.sendUpdate(AgentEvent{Type: streams.EventTypeReasoning}) + close(senderDone) + }() + + if err := a.Close(); err != nil { + t.Fatalf("close adapter: %v", err) + } + select { + case <-senderDone: + case <-time.After(2 * time.Second): + t.Fatal("direct sendUpdate caller remained blocked during close") + } +} diff --git a/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go b/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go index 5ce3bbfb6d..ae0563e6fe 100644 --- a/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go +++ b/apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go @@ -3,6 +3,7 @@ package acp import ( "encoding/json" "testing" + "time" "github.com/coder/acp-go-sdk" "github.com/kandev/kandev/internal/agentctl/server/adapter/transport/shared" @@ -10,6 +11,59 @@ import ( "github.com/kandev/kandev/internal/common/logger" ) +func TestSendUpdateAppliesCancelableBackpressure(t *testing.T) { + a := newTestAdapter() + + for index := 0; index < cap(a.updatesCh); index++ { + a.sendUpdate(AgentEvent{Type: streams.EventTypeReasoning, ReasoningText: "chunk"}) + } + + done := make(chan struct{}) + go func() { + a.sendUpdate(AgentEvent{Type: streams.EventTypeReasoning, ReasoningText: "final"}) + close(done) + }() + + select { + case <-done: + t.Fatal("sendUpdate bypassed a full per-agent queue") + case <-time.After(20 * time.Millisecond): + } + + <-a.updatesCh + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("sendUpdate did not resume after queue space became available") + } + + if err := a.Close(); err != nil { + t.Fatalf("close adapter: %v", err) + } +} + +func TestSendUpdateBackpressureIsCanceledOnClose(t *testing.T) { + a := newTestAdapter() + for index := 0; index < cap(a.updatesCh); index++ { + a.sendUpdate(AgentEvent{Type: streams.EventTypeReasoning, ReasoningText: "chunk"}) + } + + done := make(chan struct{}) + go func() { + a.sendUpdate(AgentEvent{Type: streams.EventTypeReasoning, ReasoningText: "final"}) + close(done) + }() + + if err := a.Close(); err != nil { + t.Fatalf("close adapter: %v", err) + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("sendUpdate remained blocked after adapter close") + } +} + func TestConvertACPConfigOptions_PreservesDescriptions(t *testing.T) { description := "Controls how much reasoning the model performs." valueDescription := "Uses more reasoning for complex tasks." diff --git a/apps/backend/internal/agentctl/server/process/manager.go b/apps/backend/internal/agentctl/server/process/manager.go index a84a2ed1a3..1962219c8c 100644 --- a/apps/backend/internal/agentctl/server/process/manager.go +++ b/apps/backend/internal/agentctl/server/process/manager.go @@ -1489,8 +1489,8 @@ func (m *Manager) forwardUpdates(agentAdapter adapter.AgentAdapter, stopCh <-cha } select { case m.updatesCh <- update: - default: - m.logger.Warn("updates channel full, dropping notification") + case <-stopCh: + return } case <-stopCh: return diff --git a/apps/backend/internal/clarification/canceller.go b/apps/backend/internal/clarification/canceller.go index 1325e3f459..9dc07891f5 100644 --- a/apps/backend/internal/clarification/canceller.go +++ b/apps/backend/internal/clarification/canceller.go @@ -190,6 +190,7 @@ func (c *Canceller) publishMessageUpdated(ctx context.Context, msg *taskmodels.M "type": msgType, "requests_input": msg.RequestsInput, "created_at": msg.CreatedAt.Format(time.RFC3339), + "updated_at": msg.UpdatedAt.Format(time.RFC3339Nano), "metadata": msg.Metadata, } diff --git a/apps/backend/internal/clarification/canceller_test.go b/apps/backend/internal/clarification/canceller_test.go index 10cc7f3c0d..3112360074 100644 --- a/apps/backend/internal/clarification/canceller_test.go +++ b/apps/backend/internal/clarification/canceller_test.go @@ -151,12 +151,14 @@ func TestCanceller_NoMessagesToUpdate(t *testing.T) { func TestCanceller_PublishesMessageUpdatedEvent(t *testing.T) { msgs := map[string][]*taskmodels.Message{} c, _, eventBus := newTestCanceller(t, msgs) + updatedAt := time.Date(2026, time.August, 2, 20, 0, 0, 123456789, time.UTC) pendingID, _ := c.store.CreateRequest(&Request{SessionID: "s1"}) msgs[pendingID] = []*taskmodels.Message{{ ID: "m1", TaskSessionID: "s1", Metadata: map[string]any{"status": "pending"}, + UpdatedAt: updatedAt, }} c.DetachSessionAndNotify(context.Background(), "s1") @@ -164,6 +166,13 @@ func TestCanceller_PublishesMessageUpdatedEvent(t *testing.T) { if len(eventBus.events) != 1 { t.Fatalf("expected 1 event, got %d", len(eventBus.events)) } + data, ok := eventBus.events[0].Data.(map[string]any) + if !ok { + t.Fatalf("expected map event data, got %T", eventBus.events[0].Data) + } + if got := data["updated_at"]; got != updatedAt.Format(time.RFC3339Nano) { + t.Errorf("expected updated_at %q, got %#v", updatedAt.Format(time.RFC3339Nano), got) + } } // TestCanceller_MultiQuestion_MarksAllMessagesDetached confirms a multi-question diff --git a/apps/backend/internal/gateway/websocket/client.go b/apps/backend/internal/gateway/websocket/client.go index 44c5ac4953..c5005aef34 100644 --- a/apps/backend/internal/gateway/websocket/client.go +++ b/apps/backend/internal/gateway/websocket/client.go @@ -56,23 +56,47 @@ type Client struct { mu sync.RWMutex closed bool logger *logger.Logger + + // Replaceable session.message.updated traffic is scheduled separately from + // semantic notifications so one noisy session cannot fill the shared FIFO. + // Each session queue contains replaceable segments separated by semantic + // barriers. This prevents an update published after a barrier from overtaking + // that barrier while still allowing coalescing within the current segment. + replaceableByKey map[queuedReplaceableKey]outboundNotification + replaceableBySession map[string][]sessionNotificationQueueItem + replaceableCurrentByKey map[replaceableNotificationKey]queuedReplaceableKey + replaceableSessionOrder []string + replaceableRoundRobin int + nextReplaceableSequence uint64 + scheduledSemantic int + notificationWake chan struct{} + replaceableReplacements uint64 + replaceableEvictions uint64 + replaceableRejected uint64 + droppedSemantic uint64 + replaceablePerSessionLimit int + replaceableGlobalLimit int } // NewClient creates a new WebSocket client func NewClient(id string, identity authn.Identity, conn *websocket.Conn, hub *Hub, log *logger.Logger) *Client { return &Client{ - ID: id, - identity: identity, - conn: conn, - hub: hub, - send: make(chan []byte, 256), - controlSend: make(chan []byte, controlSendBufferSize), - subscriptions: make(map[string]bool), - sessionSubscriptions: make(map[string]bool), - sessionFocus: make(map[string]bool), - userSubscriptions: make(map[string]bool), - runSubscriptions: make(map[string]bool), - logger: log.WithFields(zap.String("client_id", id)), + ID: id, + identity: identity, + conn: conn, + hub: hub, + send: make(chan []byte, 256), + controlSend: make(chan []byte, controlSendBufferSize), + subscriptions: make(map[string]bool), + sessionSubscriptions: make(map[string]bool), + sessionFocus: make(map[string]bool), + userSubscriptions: make(map[string]bool), + runSubscriptions: make(map[string]bool), + replaceableByKey: make(map[queuedReplaceableKey]outboundNotification), + replaceableBySession: make(map[string][]sessionNotificationQueueItem), + replaceableCurrentByKey: make(map[replaceableNotificationKey]queuedReplaceableKey), + notificationWake: make(chan struct{}, 1), + logger: log.WithFields(zap.String("client_id", id)), } } @@ -654,19 +678,17 @@ func (c *Client) sendMessage(msg *ws.Message) { } func (c *Client) sendBytes(data []byte) bool { - c.mu.Lock() - defer c.mu.Unlock() - if c.closed { - return false - } + return c.enqueueNotification(newOutboundNotification(data, "")) +} - select { - case c.send <- data: - return true - default: - c.logger.Warn("Client send buffer full") - return false - } +// sendNotification queues a pre-marshalled notification with its typed action +// so the client scheduler can classify it once without reparsing the envelope. +func (c *Client) sendNotification(data []byte, action string) bool { + return c.sendNotificationFrame(newOutboundNotification(data, action)) +} + +func (c *Client) sendNotificationFrame(frame outboundNotification) bool { + return c.enqueueNotification(frame) } // sendControlBytes queues a response/error separately from notifications. @@ -732,6 +754,15 @@ func (c *Client) closeSendLocked() { if c.controlSend != nil { close(c.controlSend) } + if c.notificationWake != nil { + // The wake channel is deliberately left open: enqueueNotification checks + // closed while holding c.mu, so closing it here would create a + // send-on-closed race with a concurrent producer. + select { + case c.notificationWake <- struct{}{}: + default: + } + } } // WritePump pumps messages from the hub to the WebSocket connection @@ -744,45 +775,129 @@ func (c *Client) WritePump() { } }() + controlCh := c.controlSend + semanticCh := c.send + semanticSinceReplaceable := 0 + + writeMessage := func(message []byte) bool { + if err := c.conn.SetWriteDeadline(time.Now().Add(writeWait)); err != nil { + c.logger.Debug("failed to set write deadline", zap.Error(err)) + } + if err := c.conn.WriteMessage(websocket.TextMessage, message); err != nil { + c.logger.Debug("failed to write websocket message", zap.Error(err)) + return false + } + return true + } + writeClose := func() { + if err := c.conn.SetWriteDeadline(time.Now().Add(writeWait)); err != nil { + c.logger.Debug("failed to set write deadline", zap.Error(err)) + } + if err := c.conn.WriteMessage(websocket.CloseMessage, []byte{}); err != nil { + c.logger.Debug("failed to write close message", zap.Error(err)) + } + } + writePing := func() bool { + if err := c.conn.SetWriteDeadline(time.Now().Add(writeWait)); err != nil { + c.logger.Debug("failed to set write deadline", zap.Error(err)) + } + if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { + c.logger.Debug("failed to write websocket ping", zap.Error(err)) + return false + } + return true + } + for { - // Prefer control frames whenever one is already queued. The second - // select still lets stream traffic make progress when no response is - // waiting, while preventing notifications from starving ACKs. - var message []byte - var ok bool + // Correlated responses/errors always run first. A bounded semantic burst + // then gives each active session's replaceable queue a turn. select { - case message, ok = <-c.controlSend: + case <-ticker.C: + if !writePing() { + return + } default: + } + if controlCh != nil { select { - case message, ok = <-c.controlSend: - case message, ok = <-c.send: - case <-ticker.C: - if err := c.conn.SetWriteDeadline(time.Now().Add(writeWait)); err != nil { - c.logger.Debug("failed to set write deadline", zap.Error(err)) + case message, ok := <-controlCh: + if !ok { + controlCh = nil + continue } - if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { + if !writeMessage(message) { + return + } + continue + default: + } + } + + if semanticSinceReplaceable >= semanticPriorityBurst { + if frame, ok := c.popNextReplaceable(); ok { + semanticSinceReplaceable = 0 + if !writeMessage(frame.data) { return } continue } } - if err := c.conn.SetWriteDeadline(time.Now().Add(writeWait)); err != nil { - c.logger.Debug("failed to set write deadline", zap.Error(err)) - } - if !ok { - // Hub closed the channel. - if err := c.conn.WriteMessage(websocket.CloseMessage, []byte{}); err != nil { - c.logger.Debug("failed to write close message", zap.Error(err)) + if semanticCh != nil { + select { + case message, ok := <-semanticCh: + if !ok { + semanticCh = nil + continue + } + semanticSinceReplaceable++ + if !writeMessage(message) { + return + } + continue + default: } + } + + if frame, ok := c.popNextReplaceable(); ok { + semanticSinceReplaceable = 0 + if !writeMessage(frame.data) { + return + } + continue + } + + if controlCh == nil && semanticCh == nil && !c.hasReplaceable() { + writeClose() return } - // Send each message in its own WebSocket frame - // (previously batched with newlines, but clients expect one JSON object per frame) - if err := c.conn.WriteMessage(websocket.TextMessage, message); err != nil { - c.logger.Debug("failed to write websocket message", zap.Error(err)) - return + c.mu.RLock() + wakeCh := c.notificationWake + c.mu.RUnlock() + select { + case message, ok := <-controlCh: + if !ok { + controlCh = nil + continue + } + if !writeMessage(message) { + return + } + case message, ok := <-semanticCh: + if !ok { + semanticCh = nil + continue + } + semanticSinceReplaceable++ + if !writeMessage(message) { + return + } + case <-wakeCh: + case <-ticker.C: + if !writePing() { + return + } } } } diff --git a/apps/backend/internal/gateway/websocket/client_send_queue_test.go b/apps/backend/internal/gateway/websocket/client_send_queue_test.go index 04696fc902..93f5b2677a 100644 --- a/apps/backend/internal/gateway/websocket/client_send_queue_test.go +++ b/apps/backend/internal/gateway/websocket/client_send_queue_test.go @@ -2,6 +2,8 @@ package websocket import ( "encoding/json" + "reflect" + "slices" "testing" ws "github.com/kandev/kandev/pkg/websocket" @@ -66,3 +68,350 @@ func TestClient_ControlOverflowClosesConnection(t *testing.T) { } } } + +func TestClient_ReplaceableSessionMessageKeepsLatestFrame(t *testing.T) { + c := newTestClient("replaceable-session-message") + + first := []byte(`{"type":"notification","action":"session.message.updated","payload":{"session_id":"session-1","message_id":"message-1","content":"old"}}`) + latest := []byte(`{"type":"notification","action":"session.message.updated","payload":{"session_id":"session-1","message_id":"message-1","content":"latest"}}`) + + if !c.sendBytes(first) || !c.sendBytes(latest) { + t.Fatal("replaceable frames should be accepted") + } + if got := len(c.send); got != 0 { + t.Fatalf("replaceable frame leaked into semantic queue, got %d", got) + } + + frame, ok := c.popNextReplaceable() + if !ok { + t.Fatal("expected one replaceable frame") + } + var message ws.Message + if err := json.Unmarshal(frame.data, &message); err != nil { + t.Fatalf("decode queued frame: %v", err) + } + var payload struct { + Content string `json:"content"` + } + if err := json.Unmarshal(message.Payload, &payload); err != nil { + t.Fatalf("decode payload: %v", err) + } + if payload.Content != "latest" { + t.Fatalf("expected latest content, got %q", payload.Content) + } + if stats := c.notificationQueueStats(); stats.Depth != 0 || stats.Replacements != 1 { + t.Fatalf("unexpected replaceable stats: %+v", stats) + } +} + +func TestClient_ReplaceableQueuePreservesPositionAndSemanticBarrier(t *testing.T) { + c := newTestClient("replaceable-order") + + first, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": "session-1", "message_id": "message-1", "content": "first", + }) + if err != nil { + t.Fatalf("build first update: %v", err) + } + latest, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": "session-1", "message_id": "message-1", "content": "latest", + }) + if err != nil { + t.Fatalf("build latest update: %v", err) + } + deleted, err := ws.NewNotification(ws.ActionSessionMessageDeleted, map[string]any{ + "session_id": "session-1", "message_id": "message-1", + }) + if err != nil { + t.Fatalf("build delete: %v", err) + } + + firstBytes, _ := json.Marshal(first) + latestBytes, _ := json.Marshal(latest) + deletedBytes, _ := json.Marshal(deleted) + if !c.sendNotification(firstBytes, first.Action) || !c.sendNotification(latestBytes, latest.Action) { + t.Fatal("message updates should be accepted") + } + if !c.sendNotification(deletedBytes, deleted.Action) { + t.Fatal("semantic barrier should be accepted") + } + if got := len(c.send); got != 0 { + t.Fatalf("semantic barrier overtook replacement queue, got %d frames", got) + } + + frame, ok := c.popNextReplaceable() + if !ok { + t.Fatal("expected replacement frame") + } + var got ws.Message + if err := json.Unmarshal(frame.data, &got); err != nil { + t.Fatalf("decode replacement: %v", err) + } + if got.Action != ws.ActionSessionMessageUpdated { + t.Fatalf("expected message update first, got %q", got.Action) + } + var payload struct { + Content string `json:"content"` + } + if err := json.Unmarshal(got.Payload, &payload); err != nil { + t.Fatalf("decode replacement payload: %v", err) + } + if payload.Content != "latest" { + t.Fatalf("expected latest payload in original position, got %q", payload.Content) + } + + barrierFrame, ok := c.popNextReplaceable() + if !ok { + t.Fatal("expected semantic barrier after replacement") + } + var barrierMessage ws.Message + if err := json.Unmarshal(barrierFrame.data, &barrierMessage); err != nil { + t.Fatalf("decode semantic barrier: %v", err) + } + if barrierMessage.Action != ws.ActionSessionMessageDeleted { + t.Fatalf("expected delete barrier after replacement, got %q", barrierMessage.Action) + } +} + +func TestClient_ReplaceableQueueRoundRobinsSessions(t *testing.T) { + c := newTestClient("replaceable-round-robin") + for _, item := range []struct { + sessionID string + messageID string + content string + }{ + {sessionID: "session-a", messageID: "message-a1", content: "a1"}, + {sessionID: "session-a", messageID: "message-a2", content: "a2"}, + {sessionID: "session-b", messageID: "message-b1", content: "b1"}, + {sessionID: "session-b", messageID: "message-b2", content: "b2"}, + } { + message, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": item.sessionID, "message_id": item.messageID, "content": item.content, + }) + if err != nil { + t.Fatalf("build update: %v", err) + } + data, _ := json.Marshal(message) + if !c.sendNotification(data, message.Action) { + t.Fatalf("queue update %s/%s", item.sessionID, item.messageID) + } + } + + var got []string + for i := 0; i < 4; i++ { + frame, ok := c.popNextReplaceable() + if !ok { + t.Fatalf("missing frame %d", i) + } + var message ws.Message + if err := json.Unmarshal(frame.data, &message); err != nil { + t.Fatalf("decode frame %d: %v", i, err) + } + var payload struct { + MessageID string `json:"message_id"` + } + if err := json.Unmarshal(message.Payload, &payload); err != nil { + t.Fatalf("decode frame payload %d: %v", i, err) + } + got = append(got, payload.MessageID) + } + want := []string{"message-a1", "message-b1", "message-a2", "message-b2"} + if !reflect.DeepEqual(got, want) { + t.Fatalf("round-robin order = %v, want %v", got, want) + } +} + +func TestClient_ReplaceableQueueBoundsNoisySessionOnly(t *testing.T) { + c := newTestClient("replaceable-bounds") + c.replaceablePerSessionLimit = 2 + c.replaceableGlobalLimit = 4 + + queueUpdate := func(sessionID, messageID string) { + t.Helper() + message, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": sessionID, "message_id": messageID, + }) + if err != nil { + t.Fatalf("build update: %v", err) + } + data, _ := json.Marshal(message) + if !c.sendNotification(data, message.Action) { + t.Fatalf("queue update %s/%s", sessionID, messageID) + } + } + + queueUpdate("noisy", "n1") + queueUpdate("noisy", "n2") + queueUpdate("noisy", "n3") // evicts noisy/n1 + queueUpdate("quiet", "q1") + queueUpdate("quiet", "q2") + queueUpdate("quiet", "q3") // evicts quiet/q1, not noisy + if accepted := c.sendNotification([]byte(`{"type":"notification","action":"session.message.updated","payload":{"session_id":"other","message_id":"o1"}}`), ws.ActionSessionMessageUpdated); accepted { + t.Fatal("global overflow should reject a new session rather than evict another session") + } + + stats := c.notificationQueueStats() + if stats.Depth != 4 || stats.Evictions != 2 || stats.Rejected != 1 { + t.Fatalf("unexpected bounded queue stats: %+v", stats) + } + var got []string + for { + frame, ok := c.popNextReplaceable() + if !ok { + break + } + var message ws.Message + if err := json.Unmarshal(frame.data, &message); err != nil { + t.Fatalf("decode bounded frame: %v", err) + } + var payload struct { + MessageID string `json:"message_id"` + } + if err := json.Unmarshal(message.Payload, &payload); err != nil { + t.Fatalf("decode bounded payload: %v", err) + } + got = append(got, payload.MessageID) + } + for _, messageID := range []string{"n2", "n3", "q2", "q3"} { + if !slices.Contains(got, messageID) { + t.Fatalf("bounded queue lost %s: %v", messageID, got) + } + } + for _, messageID := range []string{"n1", "q1", "o1"} { + if slices.Contains(got, messageID) { + t.Fatalf("bounded queue retained evicted/rejected %s: %v", messageID, got) + } + } +} + +func TestClient_ReplaceableEvictionKeepsSessionOrderUnique(t *testing.T) { + c := newTestClient("replaceable-eviction-order") + c.replaceablePerSessionLimit = 2 + c.replaceableGlobalLimit = 2 + + queueUpdate := func(sessionID, messageID string) { + t.Helper() + message, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": sessionID, "message_id": messageID, + }) + if err != nil { + t.Fatalf("build update: %v", err) + } + data, _ := json.Marshal(message) + if !c.sendNotification(data, message.Action) { + t.Fatalf("queue update %s/%s", sessionID, messageID) + } + } + + queueUpdate("session-a", "message-a1") + queueUpdate("session-b", "message-b1") + queueUpdate("session-a", "message-a2") // evicts a1 while a remains registered + + c.mu.RLock() + order := append([]string(nil), c.replaceableSessionOrder...) + c.mu.RUnlock() + if !reflect.DeepEqual(order, []string{"session-a", "session-b"}) { + t.Fatalf("replaceable session order = %v, want one entry per session", order) + } +} + +func TestClient_SemanticBarrierPartitionsReplacementUpdates(t *testing.T) { + c := newTestClient("semantic-barrier-partition") + + queueUpdate := func(content string) { + t.Helper() + message, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": "session-1", "message_id": "message-1", "content": content, + }) + if err != nil { + t.Fatalf("build update: %v", err) + } + data, _ := json.Marshal(message) + if !c.sendNotification(data, message.Action) { + t.Fatalf("queue update %q", content) + } + } + queueUpdate("before") + + deleted, err := ws.NewNotification(ws.ActionSessionMessageDeleted, map[string]any{ + "session_id": "session-1", "message_id": "message-1", + }) + if err != nil { + t.Fatalf("build delete: %v", err) + } + deletedData, _ := json.Marshal(deleted) + if !c.sendNotification(deletedData, deleted.Action) { + t.Fatal("semantic barrier was not accepted") + } + queueUpdate("after") + + readAction := func() (string, string) { + t.Helper() + frame, ok := c.popNextReplaceable() + if !ok { + t.Fatal("expected scheduled frame") + } + var message ws.Message + if err := json.Unmarshal(frame.data, &message); err != nil { + t.Fatalf("decode scheduled frame: %v", err) + } + var payload struct { + Content string `json:"content"` + } + if len(message.Payload) > 0 { + if err := json.Unmarshal(message.Payload, &payload); err != nil { + t.Fatalf("decode scheduled payload: %v", err) + } + } + return message.Action, payload.Content + } + + if action, content := readAction(); action != ws.ActionSessionMessageUpdated || content != "before" { + t.Fatalf("first scheduled frame = (%q, %q), want before update", action, content) + } + if action, _ := readAction(); action != ws.ActionSessionMessageDeleted { + t.Fatalf("second scheduled frame = %q, want delete barrier", action) + } + if action, content := readAction(); action != ws.ActionSessionMessageUpdated || content != "after" { + t.Fatalf("third scheduled frame = (%q, %q), want after update", action, content) + } +} + +func TestClient_SemanticDropsAreCounted(t *testing.T) { + c := newTestClient("semantic-drop-count") + c.send = make(chan []byte, 1) + c.send <- []byte("occupied") + + if c.sendNotification([]byte(`{"type":"notification","action":"session.state_updated"}`), "session.state_updated") { + t.Fatal("semantic frame should be rejected when the queue is full") + } + if got := c.notificationQueueStats().DroppedSemantic; got != 1 { + t.Fatalf("dropped semantic count = %d, want 1", got) + } +} + +func TestClient_ReplaceableQueueCloseAndZeroValueAreSafe(t *testing.T) { + var zero Client + frame := []byte(`{"type":"notification","action":"session.message.updated","payload":{"session_id":"session-1","message_id":"message-1"}}`) + if !zero.sendBytes(frame) { + t.Fatal("zero-value client should accept replaceable traffic") + } + if _, ok := zero.popNextReplaceable(); !ok { + t.Fatal("zero-value replaceable traffic was not drainable") + } + if zero.sendBytes([]byte("semantic")) { + t.Fatal("zero-value client should reject semantic traffic without a channel") + } + + c := newTestClient("replaceable-close") + if !c.sendBytes(frame) { + t.Fatal("replaceable frame was not accepted before close") + } + c.closeSend() + if c.sendBytes(frame) { + t.Fatal("closed client accepted replaceable traffic") + } + if _, ok := c.popNextReplaceable(); !ok { + t.Fatal("queued replaceable frame was lost during close drain") + } +} diff --git a/apps/backend/internal/gateway/websocket/client_write_pump_test.go b/apps/backend/internal/gateway/websocket/client_write_pump_test.go new file mode 100644 index 0000000000..345b7480f7 --- /dev/null +++ b/apps/backend/internal/gateway/websocket/client_write_pump_test.go @@ -0,0 +1,108 @@ +package websocket + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + gorillaws "github.com/gorilla/websocket" + "github.com/kandev/kandev/internal/common/logger" + ws "github.com/kandev/kandev/pkg/websocket" +) + +func TestClientWritePumpPrioritizesControlAndFairReplaceable(t *testing.T) { + serverConnCh := make(chan *gorillaws.Conn, 1) + upgrader := gorillaws.Upgrader{CheckOrigin: func(*http.Request) bool { return true }} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + t.Errorf("upgrade websocket: %v", err) + return + } + serverConnCh <- conn + })) + defer server.Close() + + clientConn, _, err := gorillaws.DefaultDialer.Dial("ws"+server.URL[len("http"):], nil) + if err != nil { + t.Fatalf("dial websocket: %v", err) + } + defer func() { _ = clientConn.Close() }() + serverConn := <-serverConnCh + defer func() { _ = serverConn.Close() }() + + log, err := logger.NewLogger(logger.LoggingConfig{Level: "error", Format: "json"}) + if err != nil { + t.Fatalf("logger: %v", err) + } + c := &Client{ + ID: "write-pump-priority", + conn: serverConn, + send: make(chan []byte, 16), + controlSend: make(chan []byte, 16), + notificationWake: make(chan struct{}, 1), + logger: log, + } + c.controlSend <- []byte("control") + for i := 0; i < 9; i++ { + c.send <- []byte("semantic-" + string(rune('0'+i))) + } + update, err := ws.NewNotification(ws.ActionSessionMessageUpdated, map[string]any{ + "session_id": "noisy-session", + "message_id": "message-1", + "content": "replaceable", + }) + if err != nil { + t.Fatalf("build replaceable frame: %v", err) + } + updateBytes, err := json.Marshal(update) + if err != nil { + t.Fatalf("marshal replaceable frame: %v", err) + } + if !c.sendNotification(updateBytes, update.Action) { + t.Fatal("queue replaceable frame") + } + + done := make(chan struct{}) + go func() { + c.WritePump() + close(done) + }() + + var got []string + for len(got) < 11 { + messageType, data, readErr := clientConn.ReadMessage() + if readErr != nil { + t.Fatalf("read frame %d: %v", len(got), readErr) + } + if messageType != gorillaws.TextMessage { + t.Fatalf("frame %d type = %d, want text", len(got), messageType) + } + got = append(got, string(data)) + } + + if got[0] != "control" { + t.Fatalf("first frame = %q, want control", got[0]) + } + for index := 1; index <= 8; index++ { + want := "semantic-" + string(rune('0'+index-1)) + if got[index] != want { + t.Fatalf("frame %d = %q, want %q", index, got[index], want) + } + } + if got[9] != string(updateBytes) { + t.Fatalf("replaceable frame = %q, want %q", got[9], updateBytes) + } + if got[10] != "semantic-8" { + t.Fatalf("frame after replaceable = %q, want semantic-8", got[10]) + } + + c.closeSend() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("WritePump did not stop after queues closed") + } +} diff --git a/apps/backend/internal/gateway/websocket/hub.go b/apps/backend/internal/gateway/websocket/hub.go index 7078fef800..53d062d700 100644 --- a/apps/backend/internal/gateway/websocket/hub.go +++ b/apps/backend/internal/gateway/websocket/hub.go @@ -263,11 +263,12 @@ func (h *Hub) broadcastMessage(msg *ws.Message) { h.mu.RLock() defer h.mu.RUnlock() + frame := newOutboundNotification(data, msg.Action) // For now, broadcast to all clients // TODO: Add topic-based routing for task-specific notifications for client := range h.clients { - client.sendBytes(data) + client.sendNotificationFrame(frame) } } @@ -365,9 +366,10 @@ func (h *Hub) getSubscribersLocked(m map[string]map[*Client]bool, id string) []* // sendToClients delivers a pre-marshalled message to a list of clients. func (h *Hub) sendToClients(data []byte, clients []*Client, action string) int { + frame := newOutboundNotification(data, action) queued := 0 for _, client := range clients { - if client.sendBytes(data) { + if client.sendNotificationFrame(frame) { queued++ h.logger.Debug("Sent message to client", zap.String("client_id", client.ID), diff --git a/apps/backend/internal/gateway/websocket/notification_scheduler.go b/apps/backend/internal/gateway/websocket/notification_scheduler.go new file mode 100644 index 0000000000..85214c4b24 --- /dev/null +++ b/apps/backend/internal/gateway/websocket/notification_scheduler.go @@ -0,0 +1,410 @@ +package websocket + +import ( + "encoding/json" + "slices" + + ws "github.com/kandev/kandev/pkg/websocket" + "go.uber.org/zap" +) + +const ( + // replaceablePerSessionCapacity bounds the amount of distinct full-state + // message updates a single session can hold for one client. + replaceablePerSessionCapacity = 32 + // replaceableGlobalCapacity bounds all pending replaceable updates for one + // client. A noisy session can only evict its own oldest queued replacements. + replaceableGlobalCapacity = 256 + // semanticPriorityBurst prevents a busy semantic queue from starving the + // replaceable queues while still giving responses/control frames priority. + semanticPriorityBurst = 8 +) + +type outboundNotification struct { + data []byte + action string + sessionID string + messageID string +} + +type replaceableNotificationKey struct { + sessionID string + messageID string +} + +type queuedReplaceableKey struct { + replaceableNotificationKey + sequence uint64 +} + +type notificationQueueItemKind uint8 + +const ( + replaceableQueueItem notificationQueueItemKind = iota + semanticQueueItem +) + +type sessionNotificationQueueItem struct { + kind notificationQueueItemKind + key queuedReplaceableKey + frame outboundNotification +} + +func (n outboundNotification) replaceableKey() (replaceableNotificationKey, bool) { + if n.action != ws.ActionSessionMessageUpdated || n.sessionID == "" || n.messageID == "" { + return replaceableNotificationKey{}, false + } + return replaceableNotificationKey{sessionID: n.sessionID, messageID: n.messageID}, true +} + +// newOutboundNotification classifies one already-marshalled websocket frame. +// Hub broadcasts pass the action directly so the envelope is not reparsed for +// routing, while direct snapshots and tests can still use sendBytes safely. +func newOutboundNotification(data []byte, action string) outboundNotification { + frame := outboundNotification{data: data, action: action} + var envelope struct { + Action string `json:"action"` + Payload json.RawMessage `json:"payload"` + } + if err := json.Unmarshal(data, &envelope); err != nil { + return frame + } + if frame.action == "" { + frame.action = envelope.Action + } + if len(envelope.Payload) == 0 { + return frame + } + var identity struct { + SessionID string `json:"session_id"` + MessageID string `json:"message_id"` + } + if err := json.Unmarshal(envelope.Payload, &identity); err != nil { + return frame + } + frame.sessionID = identity.SessionID + frame.messageID = identity.MessageID + return frame +} + +func (c *Client) ensureNotificationSchedulerLocked() { + if c.replaceableByKey == nil { + c.replaceableByKey = make(map[queuedReplaceableKey]outboundNotification) + } + if c.replaceableBySession == nil { + c.replaceableBySession = make(map[string][]sessionNotificationQueueItem) + } + if c.replaceableCurrentByKey == nil { + c.replaceableCurrentByKey = make(map[replaceableNotificationKey]queuedReplaceableKey) + } + if c.notificationWake == nil { + c.notificationWake = make(chan struct{}, 1) + } +} + +func (c *Client) signalNotificationWake() { + c.mu.RLock() + wake := c.notificationWake + c.mu.RUnlock() + if wake == nil { + return + } + select { + case wake <- struct{}{}: + default: + } +} + +func (c *Client) enqueueNotification(frame outboundNotification) bool { + key, replaceable := frame.replaceableKey() + if replaceable { + return c.enqueueReplaceable(frame, key) + } + + c.mu.Lock() + if c.closed || c.send == nil { + c.mu.Unlock() + return false + } + c.ensureNotificationSchedulerLocked() + if frame.sessionID != "" && len(c.replaceableBySession[frame.sessionID]) > 0 { + accepted := c.enqueueSemanticBarrierLocked(frame) + c.mu.Unlock() + if accepted { + c.signalNotificationWake() + } + return accepted + } + accepted := c.enqueueSemanticLocked(frame.data) + c.mu.Unlock() + return accepted +} + +func (c *Client) enqueueSemanticLocked(data []byte) bool { + if c.send == nil { + c.droppedSemantic++ + return false + } + select { + case c.send <- data: + return true + default: + c.droppedSemantic++ + if c.logger != nil { + c.logger.Warn("Client send buffer full") + } + return false + } +} + +func (c *Client) enqueueSemanticBarrierLocked(frame outboundNotification) bool { + if c.scheduledSemantic >= cap(c.send) { + c.droppedSemantic++ + if c.logger != nil { + c.logger.Warn("Client semantic queue full; dropping notification", + queueActionField(frame.action), + queueSessionField(frame.sessionID)) + } + return false + } + c.replaceableBySession[frame.sessionID] = append( + c.replaceableBySession[frame.sessionID], + sessionNotificationQueueItem{kind: semanticQueueItem, frame: frame}, + ) + c.scheduledSemantic++ + c.clearCurrentReplaceableKeysLocked(frame.sessionID) + if !slices.Contains(c.replaceableSessionOrder, frame.sessionID) { + c.replaceableSessionOrder = append(c.replaceableSessionOrder, frame.sessionID) + } + return true +} + +func (c *Client) clearCurrentReplaceableKeysLocked(sessionID string) { + for key := range c.replaceableCurrentByKey { + if key.sessionID == sessionID { + delete(c.replaceableCurrentByKey, key) + } + } +} + +func (c *Client) enqueueReplaceable(frame outboundNotification, key replaceableNotificationKey) bool { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return false + } + c.ensureNotificationSchedulerLocked() + if queuedKey, exists := c.replaceableCurrentByKey[key]; exists { + // Keep the key in its current segment position: a full-state update is + // replaceable, but it must not cross a semantic barrier. + c.replaceableByKey[queuedKey] = frame + c.replaceableReplacements++ + depth := len(c.replaceableByKey) + c.mu.Unlock() + c.logReplaceablePressure("replacement", frame, depth) + c.signalNotificationWake() + return true + } + + perSessionLimit, globalLimit := c.replaceableCapacitiesLocked() + queue := c.replaceableBySession[frame.sessionID] + if c.replaceableSessionDepthLocked(queue) >= perSessionLimit || len(c.replaceableByKey) >= globalLimit { + // Only this session's oldest queued replacement may be evicted. If + // the global cap is occupied by other sessions, reject this frame. + if c.replaceableSessionDepthLocked(queue) == 0 { + c.replaceableRejected++ + depth := len(c.replaceableByKey) + c.mu.Unlock() + c.logReplaceablePressure("rejected", frame, depth) + return false + } + if c.removeOldestReplaceableLocked(frame.sessionID) { + c.replaceableEvictions++ + } + queue = c.replaceableBySession[frame.sessionID] + } + c.nextReplaceableSequence++ + queuedKey := queuedReplaceableKey{ + replaceableNotificationKey: key, + sequence: c.nextReplaceableSequence, + } + c.replaceableByKey[queuedKey] = frame + if !slices.Contains(c.replaceableSessionOrder, frame.sessionID) { + c.replaceableSessionOrder = append(c.replaceableSessionOrder, frame.sessionID) + } + c.replaceableBySession[frame.sessionID] = append(queue, sessionNotificationQueueItem{ + kind: replaceableQueueItem, + key: queuedKey, + }) + c.replaceableCurrentByKey[key] = queuedKey + depth := len(c.replaceableByKey) + c.mu.Unlock() + c.logReplaceablePressure("queued", frame, depth) + c.signalNotificationWake() + return true +} + +func (c *Client) replaceableCapacitiesLocked() (int, int) { + perSessionLimit := c.replaceablePerSessionLimit + if perSessionLimit <= 0 { + perSessionLimit = replaceablePerSessionCapacity + } + globalLimit := c.replaceableGlobalLimit + if globalLimit <= 0 { + globalLimit = replaceableGlobalCapacity + } + return perSessionLimit, globalLimit +} + +func (c *Client) replaceableSessionDepthLocked(queue []sessionNotificationQueueItem) int { + depth := 0 + for _, item := range queue { + if item.kind == replaceableQueueItem { + depth++ + } + } + return depth +} + +func (c *Client) removeOldestReplaceableLocked(sessionID string) bool { + queue := c.replaceableBySession[sessionID] + for index, item := range queue { + if item.kind != replaceableQueueItem { + continue + } + delete(c.replaceableByKey, item.key) + if current, ok := c.replaceableCurrentByKey[item.key.replaceableNotificationKey]; ok && current == item.key { + delete(c.replaceableCurrentByKey, item.key.replaceableNotificationKey) + } + queue = append(queue[:index], queue[index+1:]...) + // Keep the session registered in replaceableSessionOrder. The enqueue + // caller may immediately add another key for this session; popNextReplaceable + // owns removal from the round-robin order after the queue is truly drained. + c.replaceableBySession[sessionID] = queue + return true + } + return false +} + +func (c *Client) hasReplaceable() bool { + c.mu.RLock() + defer c.mu.RUnlock() + return len(c.replaceableByKey) > 0 || c.scheduledSemantic > 0 +} + +// popNextReplaceable returns one scheduled frame using round-robin session +// scheduling. Semantic barriers live in the same per-session sequence as +// replaceable updates, so a later update cannot overtake the barrier. +func (c *Client) popNextReplaceable() (outboundNotification, bool) { + c.mu.Lock() + defer c.mu.Unlock() + for len(c.replaceableSessionOrder) > 0 { + if c.replaceableRoundRobin >= len(c.replaceableSessionOrder) { + c.replaceableRoundRobin = 0 + } + index := c.replaceableRoundRobin + sessionID := c.replaceableSessionOrder[index] + queue := c.replaceableBySession[sessionID] + if len(queue) == 0 { + c.removeReplaceableSessionLocked(index, sessionID) + continue + } + + item := queue[0] + c.replaceableBySession[sessionID] = queue[1:] + if item.kind == semanticQueueItem { + c.scheduledSemantic-- + if len(queue) == 1 { + c.removeReplaceableSessionLocked(index, sessionID) + } else { + c.replaceableRoundRobin = (index + 1) % len(c.replaceableSessionOrder) + } + return item.frame, true + } + + frame, ok := c.replaceableByKey[item.key] + delete(c.replaceableByKey, item.key) + if current, currentOK := c.replaceableCurrentByKey[item.key.replaceableNotificationKey]; currentOK && current == item.key { + delete(c.replaceableCurrentByKey, item.key.replaceableNotificationKey) + } + if len(queue) == 1 { + c.removeReplaceableSessionLocked(index, sessionID) + } else { + c.replaceableRoundRobin = (index + 1) % len(c.replaceableSessionOrder) + } + if !ok { + continue + } + return frame, true + } + return outboundNotification{}, false +} + +func (c *Client) removeReplaceableSessionLocked(index int, sessionID string) { + delete(c.replaceableBySession, sessionID) + c.clearCurrentReplaceableKeysLocked(sessionID) + c.replaceableSessionOrder = append(c.replaceableSessionOrder[:index], c.replaceableSessionOrder[index+1:]...) + if len(c.replaceableSessionOrder) == 0 { + c.replaceableRoundRobin = 0 + return + } + if index >= len(c.replaceableSessionOrder) { + c.replaceableRoundRobin = 0 + } else { + c.replaceableRoundRobin = index + } +} + +func (c *Client) logReplaceablePressure(reason string, frame outboundNotification, depth int) { + if c.logger == nil { + return + } + c.logger.Debug("replaceable websocket queue update", + queueActionField(frame.action), + queueSessionField(frame.sessionID), + queueMessageField(frame.messageID), + queueReasonField(reason), + queueDepthField(depth)) +} + +func queueActionField(action string) zap.Field { + return zap.String("action", action) +} + +func queueSessionField(sessionID string) zap.Field { + return zap.String("session_id", sessionID) +} + +func queueMessageField(messageID string) zap.Field { + return zap.String("message_id", messageID) +} + +func queueReasonField(reason string) zap.Field { + return zap.String("reason", reason) +} + +func queueDepthField(depth int) zap.Field { + return zap.Int("replaceable_queue_depth", depth) +} + +// notificationQueueStats intentionally exposes only queue metadata. Message +// content never enters pressure logs or metrics. +type notificationQueueStats struct { + Depth int + Replacements uint64 + Evictions uint64 + Rejected uint64 + DroppedSemantic uint64 +} + +func (c *Client) notificationQueueStats() notificationQueueStats { + c.mu.RLock() + defer c.mu.RUnlock() + return notificationQueueStats{ + Depth: len(c.replaceableByKey), + Replacements: c.replaceableReplacements, + Evictions: c.replaceableEvictions, + Rejected: c.replaceableRejected, + DroppedSemantic: c.droppedSemantic, + } +} diff --git a/apps/web/app/office/tasks/[id]/page.tsx b/apps/web/app/office/tasks/[id]/page.tsx index d97cab7a83..7a91879c65 100644 --- a/apps/web/app/office/tasks/[id]/page.tsx +++ b/apps/web/app/office/tasks/[id]/page.tsx @@ -13,7 +13,6 @@ import { type TaskDecisionDTO, } from "@/lib/api/domains/office-api"; import { listTaskSessions } from "@/lib/api/domains/session-api"; -import { getWebSocketClient } from "@/lib/ws/connection"; import { OfficeSimplePane } from "@/components/task/simple/OfficeSimplePane"; import { TaskAdvancedMode } from "./task-advanced-mode"; import { IssueDetailSkeleton } from "./task-detail-skeleton"; @@ -28,6 +27,7 @@ import type { } from "./types"; import type { ActivityEntry, OfficeTask } from "@/lib/state/slices/office/types"; import type { TaskSession as ApiTaskSession } from "@/lib/types/http"; +import { useSessionLiveSyncSubscriptions } from "./use-session-live-sync"; type IssueDetailPageProps = { params: Promise<{ id: string }>; @@ -225,13 +225,11 @@ function useSessionLiveSync({ ); const connectionStatus = useAppStore((s) => s.connection.status); - useEffect(() => { - if (connectionStatus !== "connected" || baseSessions.length === 0 || !task) return; - const client = getWebSocketClient(); - if (!client) return; - const unsubs = baseSessions.map((s) => client.subscribeSession(s.id)); - return () => unsubs.forEach((u) => u()); - }, [connectionStatus, baseSessions, task]); + useSessionLiveSyncSubscriptions({ + connectionStatus, + taskId: task?.id ?? null, + sessionIds: baseSessions.map((session) => session.id), + }); // Refetch the task + comments whenever session state actually changes. // The dep is intentionally the joined session-state key (and the diff --git a/apps/web/app/office/tasks/[id]/use-session-live-sync.test.tsx b/apps/web/app/office/tasks/[id]/use-session-live-sync.test.tsx new file mode 100644 index 0000000000..cc580b059c --- /dev/null +++ b/apps/web/app/office/tasks/[id]/use-session-live-sync.test.tsx @@ -0,0 +1,92 @@ +import { renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { useSessionLiveSyncSubscriptions } from "./use-session-live-sync"; + +const mockWebSocketClient = vi.hoisted(() => ({ + subscribeSession: vi.fn(), +})); + +vi.mock("@/lib/ws/connection", () => ({ + getWebSocketClient: () => mockWebSocketClient, +})); + +describe("Office session live-sync membership", () => { + beforeEach(() => { + mockWebSocketClient.subscribeSession.mockReset(); + mockWebSocketClient.subscribeSession.mockImplementation(() => vi.fn()); + }); + + it("does not churn subscriptions when equivalent session objects are recreated", () => { + const first = ["session-b", "session-a"]; + const { rerender, unmount } = renderHook( + ({ sessionIds }) => + useSessionLiveSyncSubscriptions({ + connectionStatus: "connected", + taskId: "task-1", + sessionIds, + }), + { initialProps: { sessionIds: first } }, + ); + + expect(mockWebSocketClient.subscribeSession).toHaveBeenCalledTimes(2); + rerender({ sessionIds: ["session-a", "session-b"] }); + expect(mockWebSocketClient.subscribeSession).toHaveBeenCalledTimes(2); + + unmount(); + expect(mockWebSocketClient.subscribeSession.mock.results[0]?.value).toHaveBeenCalledTimes(1); + expect(mockWebSocketClient.subscribeSession.mock.results[1]?.value).toHaveBeenCalledTimes(1); + }); + + it("subscribes and unsubscribes only the membership delta", () => { + const unsubscriptions = new Map>(); + mockWebSocketClient.subscribeSession.mockImplementation((sessionId: string) => { + const unsubscribe = vi.fn(); + unsubscriptions.set(sessionId, unsubscribe); + return unsubscribe; + }); + + const { rerender, unmount } = renderHook( + ({ sessionIds }) => + useSessionLiveSyncSubscriptions({ + connectionStatus: "connected", + taskId: "task-1", + sessionIds, + }), + { initialProps: { sessionIds: ["session-a", "session-b"] } }, + ); + + rerender({ sessionIds: ["session-b", "session-c"] }); + expect(mockWebSocketClient.subscribeSession).toHaveBeenCalledTimes(3); + expect(unsubscriptions.get("session-a")).toHaveBeenCalledTimes(1); + expect(unsubscriptions.get("session-b")).toHaveBeenCalledTimes(0); + expect(unsubscriptions.get("session-c")).toBeDefined(); + + unmount(); + expect(unsubscriptions.get("session-b")).toHaveBeenCalledTimes(1); + expect(unsubscriptions.get("session-c")).toHaveBeenCalledTimes(1); + }); + + it("clears all subscriptions when the connection disconnects", () => { + const unsubscribe = vi.fn(); + mockWebSocketClient.subscribeSession.mockReturnValue(unsubscribe); + const initialProps: { connectionStatus: "connected" | "reconnecting" } = { + connectionStatus: "connected", + }; + + const { rerender, unmount } = renderHook( + ({ connectionStatus }: { connectionStatus: "connected" | "reconnecting" }) => + useSessionLiveSyncSubscriptions({ + connectionStatus, + taskId: "task-1", + sessionIds: ["session-a", "session-b"], + }), + { initialProps }, + ); + + expect(mockWebSocketClient.subscribeSession).toHaveBeenCalledTimes(2); + rerender({ connectionStatus: "reconnecting" }); + expect(unsubscribe).toHaveBeenCalledTimes(2); + unmount(); + expect(unsubscribe).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/web/app/office/tasks/[id]/use-session-live-sync.ts b/apps/web/app/office/tasks/[id]/use-session-live-sync.ts new file mode 100644 index 0000000000..d68f7b080f --- /dev/null +++ b/apps/web/app/office/tasks/[id]/use-session-live-sync.ts @@ -0,0 +1,66 @@ +import { useEffect, useMemo, useRef } from "react"; +import { getWebSocketClient } from "@/lib/ws/connection"; +import type { ConnectionStatus } from "@/lib/types/connection"; + +export type SessionLiveSyncParams = { + connectionStatus: ConnectionStatus; + taskId: string | null; + sessionIds: readonly string[]; +}; + +export function stableSessionIdsKey(sessionIds: readonly string[]): string { + return [...new Set(sessionIds.filter(Boolean))].sort().join(","); +} + +export function useSessionLiveSyncSubscriptions({ + connectionStatus, + taskId, + sessionIds, +}: SessionLiveSyncParams): string[] { + const membershipKey = useMemo(() => stableSessionIdsKey(sessionIds), [sessionIds]); + const stableSessionIds = useMemo( + () => (membershipKey ? membershipKey.split(",") : []), + [membershipKey], + ); + const subscriptionsRef = useRef(new Map void>()); + + useEffect(() => { + const subscriptions = subscriptionsRef.current; + const clearSubscriptions = () => { + for (const unsubscribe of subscriptions.values()) unsubscribe(); + subscriptions.clear(); + }; + + if (connectionStatus !== "connected" || !taskId) { + clearSubscriptions(); + return; + } + + const client = getWebSocketClient(); + if (!client) { + clearSubscriptions(); + return; + } + + const desired = new Set(stableSessionIds); + for (const [sessionId, unsubscribe] of subscriptions) { + if (desired.has(sessionId)) continue; + unsubscribe(); + subscriptions.delete(sessionId); + } + for (const sessionId of stableSessionIds) { + if (subscriptions.has(sessionId)) continue; + subscriptions.set(sessionId, client.subscribeSession(sessionId)); + } + }, [connectionStatus, membershipKey, stableSessionIds, taskId]); + + useEffect( + () => () => { + for (const unsubscribe of subscriptionsRef.current.values()) unsubscribe(); + subscriptionsRef.current.clear(); + }, + [], + ); + + return stableSessionIds; +} diff --git a/apps/web/e2e/helpers/api-client.ts b/apps/web/e2e/helpers/api-client.ts index becfcde42a..867f1ab830 100644 --- a/apps/web/e2e/helpers/api-client.ts +++ b/apps/web/e2e/helpers/api-client.ts @@ -1712,6 +1712,7 @@ export class ApiClient { id: string; content: string; author_type: string; + type?: string; raw_content?: string; metadata?: Record; }>; diff --git a/apps/web/e2e/helpers/session-stream-overload.ts b/apps/web/e2e/helpers/session-stream-overload.ts new file mode 100644 index 0000000000..5309491b5d --- /dev/null +++ b/apps/web/e2e/helpers/session-stream-overload.ts @@ -0,0 +1,73 @@ +import { expect, type Page } from "@playwright/test"; +import type { ApiClient } from "./api-client"; +import type { GatewayTrafficFrame } from "./ws-traffic"; + +export const REASONING_BURST_COUNT = 2_000; + +export function reasoningBurstPrompt(count = REASONING_BURST_COUNT): string { + return `e2e:reasoning_burst(${count})`; +} + +export function expectedReasoningContent(count = REASONING_BURST_COUNT): string { + let content = ""; + for (let index = 1; index <= count; index += 1) { + content += `reasoning-burst-${String(index).padStart(6, "0")}|`; + } + return content; +} + +export async function waitForExactReasoningBurst( + apiClient: ApiClient, + sessionId: string, + count = REASONING_BURST_COUNT, +): Promise<{ sourceChunks: number; reasoningBytes: number }> { + const expected = expectedReasoningContent(count); + const firstChunk = `reasoning-burst-${String(1).padStart(6, "0")}|`; + const findBurst = (messages: Awaited>["messages"]) => + messages.find((message) => String(message.metadata?.thinking ?? "").startsWith(firstChunk)); + let latestMessages: Awaited>["messages"] = []; + await expect + .poll( + async () => { + latestMessages = (await apiClient.listSessionMessages(sessionId)).messages; + const marker = latestMessages.some( + (message) => message.content === `reasoning-burst-produced:${count}`, + ); + const reasoning = findBurst(latestMessages); + return marker && reasoning?.metadata?.thinking === expected; + }, + { + timeout: 120_000, + message: `reasoning burst did not persist exact ${count}-chunk content`, + }, + ) + .toBe(true); + + const reasoning = findBurst(latestMessages); + return { + sourceChunks: count, + reasoningBytes: Buffer.byteLength(String(reasoning?.metadata?.thinking ?? ""), "utf8"), + }; +} + +export function noisyReceivedFrames( + frames: readonly GatewayTrafficFrame[], + sessionId: string, +): GatewayTrafficFrame[] { + return frames.filter( + (frame) => + frame.direction === "received" && + frame.sessionId === sessionId && + frame.action === "session.message.updated", + ); +} + +export async function assertNoHorizontalOverflow(page: Page, label: string): Promise { + const dimensions = await page.evaluate(() => ({ + scrollWidth: document.documentElement.scrollWidth, + clientWidth: document.documentElement.clientWidth, + })); + expect(dimensions.scrollWidth, `${label} scroll width`).toBeLessThanOrEqual( + dimensions.clientWidth, + ); +} diff --git a/apps/web/e2e/tests/chat/agent-message-attribution.spec.ts b/apps/web/e2e/tests/chat/agent-message-attribution.spec.ts index a053acb701..205c73657c 100644 --- a/apps/web/e2e/tests/chat/agent-message-attribution.spec.ts +++ b/apps/web/e2e/tests/chat/agent-message-attribution.spec.ts @@ -33,23 +33,25 @@ function senderBadge(session: SessionPage): Locator { return session.chat.locator("[data-testid='sender-task-badge']"); } -/** Poll the target's messages until the default `createIdleTarget` agent has - * emitted its "ready for instructions" reply — the cheapest signal that the - * session is idle and ready to receive a follow-up via the prompt path - * rather than the queue path. Avoids hard-coded sleeps. */ +/** Wait for the target session to enter the state where a cross-task prompt + * takes the synchronous path instead of being queued. The mock agent emits + * its text reply before this state transition, so message content alone is + * not a reliable readiness signal. */ async function waitForTargetIdle( apiClient: ApiClient, + taskId: string, sessionId: string, - marker = "ready for instructions", timeoutMs = 30_000, ): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const { messages } = await apiClient.listSessionMessages(sessionId); - if (messages.some((m) => m.content.includes(marker))) return; - await new Promise((r) => setTimeout(r, 250)); - } - throw new Error(`Target session ${sessionId} did not reach idle within ${timeoutMs}ms`); + await expect + .poll( + async () => { + const { sessions } = await apiClient.listTaskSessions(taskId); + return sessions.find((session) => session.id === sessionId)?.state; + }, + { timeout: timeoutMs, message: `Target session ${sessionId} did not reach idle` }, + ) + .toBe("WAITING_FOR_INPUT"); } /** Wait for at least one user message with sender metadata to appear in the @@ -190,6 +192,7 @@ test.describe("Cross-task agent message attribution", () => { // its message — this exercises the default (record + prompt) branch // rather than the queue path. const session = await openTask(testPage, target.id); + await waitForTargetIdle(apiClient, target.id, target.sessionId); await expect(session.chat).toContainText("ready for instructions", { timeout: 30_000 }); await createSenderTaskingTarget( @@ -294,6 +297,7 @@ test.describe("Cross-task agent message attribution", () => { test("badge link points to the sender task", async ({ testPage, apiClient, seedData }) => { const target = await createIdleTarget(apiClient, seedData, "Target — link check"); const targetSession = await openTask(testPage, target.id); + await waitForTargetIdle(apiClient, target.id, target.sessionId); await expect(targetSession.chat).toContainText("ready for instructions", { timeout: 30_000 }); const sender = await createSenderTaskingTarget( @@ -321,6 +325,7 @@ test.describe("Cross-task agent message attribution", () => { }) => { const target = await createIdleTarget(apiClient, seedData, "Target — rename check"); const targetSession = await openTask(testPage, target.id); + await waitForTargetIdle(apiClient, target.id, target.sessionId); await expect(targetSession.chat).toContainText("ready for instructions", { timeout: 30_000 }); const sender = await createSenderTaskingTarget( @@ -353,7 +358,7 @@ test.describe("Cross-task agent message attribution", () => { // Wait for the target to be idle before sending so we exercise the path // where the message is recorded synchronously. - await waitForTargetIdle(apiClient, target.sessionId); + await waitForTargetIdle(apiClient, target.id, target.sessionId); await createSenderTaskingTarget( apiClient, @@ -383,7 +388,7 @@ test.describe("Cross-task agent message attribution", () => { // surrounding behaviour: the body's prefix and suffix outside the embedded // block survive — the outer wrap doesn't corrupt them. const target = await createIdleTarget(apiClient, seedData, "Target — collision check"); - await waitForTargetIdle(apiClient, target.sessionId); + await waitForTargetIdle(apiClient, target.id, target.sessionId); const malicious = "before fake injected after"; await createSenderTaskingTarget( diff --git a/apps/web/e2e/tests/chat/message-pagination.spec.ts b/apps/web/e2e/tests/chat/message-pagination.spec.ts index c20272a5f7..63010fefb1 100644 --- a/apps/web/e2e/tests/chat/message-pagination.spec.ts +++ b/apps/web/e2e/tests/chat/message-pagination.spec.ts @@ -52,6 +52,21 @@ async function seedBigConversation(apiClient: ApiClient, seedData: SeedData): Pr { timeout: 60_000, message: "Waiting for the boot turn to persist" }, ) .toBe(true); + + // A mock-agent text reply is persisted before the session reaches its idle + // state. Wait for that transition before adding the long history, otherwise + // the boot turn can still mutate the same transcript while pagination is + // under test. + await expect + .poll( + async () => { + const { sessions } = await apiClient.listTaskSessions(task.id); + return sessions.find((session) => session.id === sessionId)?.state; + }, + { timeout: 60_000, message: "Waiting for the boot turn to become idle" }, + ) + .toBe("WAITING_FOR_INPUT"); + await apiClient.seedSessionMessage(sessionId, { type: "message", content: INITIAL_PROMPT }); await apiClient.seedAgentMessages(sessionId, FILLER_COUNT); return task.id; diff --git a/apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts b/apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts new file mode 100644 index 0000000000..3c860cee89 --- /dev/null +++ b/apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts @@ -0,0 +1,101 @@ +import { expect } from "@playwright/test"; +import { test } from "../../fixtures/test-base"; +import { SessionPage } from "../../pages/session-page"; +import { + assertNoHorizontalOverflow, + noisyReceivedFrames, + reasoningBurstPrompt, + REASONING_BURST_COUNT, + waitForExactReasoningBurst, +} from "../../helpers/session-stream-overload"; +import { attachGatewayTrafficCapture, summarizeGatewayTraffic } from "../../helpers/ws-traffic"; + +test.describe("mobile: session stream overload isolation", () => { + test("switches through the native session sheet while a sibling streams", async ({ + testPage, + apiClient, + seedData, + }) => { + test.setTimeout(180_000); + const capture = attachGatewayTrafficCapture(testPage); + const noisyTask = await apiClient.createTask( + seedData.workspaceId, + "Mobile noisy reasoning task", + { + agent_profile_id: seedData.agentProfileId, + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + const quietSession = await apiClient.seedTaskSession(noisyTask.id, { + state: "WAITING_FOR_INPUT", + agentProfileId: seedData.agentProfileId, + repositoryId: seedData.repositoryId, + sessionId: `mobile-quiet-${noisyTask.id}`, + }); + + await testPage.goto(`/t/${noisyTask.id}`); + const session = new SessionPage(testPage); + await testPage + .locator("[data-testid='mobile-task-layout']:visible") + .waitFor({ state: "visible", timeout: 30_000 }); + + const launched = await apiClient.launchSession({ + task_id: noisyTask.id, + agent_profile_id: seedData.agentProfileId, + executor_profile_id: seedData.worktreeExecutorProfileId, + workflow_step_id: seedData.startStepId, + prompt: reasoningBurstPrompt(), + }); + const noisySessionId = launched.session_id; + + await session.waitForLoad(); + const noisyPage = await testPage.context().newPage(); + const noisyCapture = attachGatewayTrafficCapture(noisyPage); + await noisyPage.goto(`/t/${noisyTask.id}`); + const noisySession = new SessionPage(noisyPage); + await noisySession.waitForLoad(); + + const pill = testPage.getByTestId("mobile-sessions-pill"); + await expect(pill).toBeVisible({ timeout: 30_000 }); + await pill.tap(); + const quietRow = testPage.getByTestId(`mobile-session-row-${quietSession.session_id}`); + await expect(quietRow).toBeVisible({ timeout: 30_000 }); + await quietRow.tap(); + await session.waitForLoad(); + await session.waitForChatIdle({ timeout: 60_000 }); + await session.sendMessageViaButton("mobile-quiet-followup"); + await session.expectChatResponseVisible("mobile-quiet-followup", 0, { timeout: 60_000 }); + + await waitForExactReasoningBurst(apiClient, noisySessionId); + const noisyFrames = noisyReceivedFrames(noisyCapture.frames, noisySessionId); + const quietSessionNoisyFrames = noisyReceivedFrames(capture.frames, noisySessionId); + expect(noisyFrames.length).toBeGreaterThan(0); + expect(noisyFrames.length).toBeLessThan(REASONING_BURST_COUNT); + expect( + quietSessionNoisyFrames, + "quiet mobile view must not receive noisy-session updates", + ).toHaveLength(0); + await assertNoHorizontalOverflow(testPage, "mobile session stream overload surface"); + + const evidence = { + sourceChunks: REASONING_BURST_COUNT, + gatewayReceivedUpdatedFrames: noisyFrames.length, + noisySessionId, + quietSessionId: quietSession.session_id, + quietResponse: "mobile-quiet-followup", + gateway: summarizeGatewayTraffic(capture.frames), + noisyGateway: summarizeGatewayTraffic(noisyCapture.frames), + }; + await test.info().attach("mobile-session-stream-overload-evidence.json", { + body: JSON.stringify(evidence, null, 2), + contentType: "application/json", + }); + test.info().annotations.push({ + type: "stream-overload-evidence", + description: JSON.stringify(evidence), + }); + await noisyPage.close(); + }); +}); diff --git a/apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts b/apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts new file mode 100644 index 0000000000..bd616c35c5 --- /dev/null +++ b/apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts @@ -0,0 +1,99 @@ +import { expect } from "@playwright/test"; +import { test } from "../../fixtures/test-base"; +import { SessionPage } from "../../pages/session-page"; +import { + assertNoHorizontalOverflow, + noisyReceivedFrames, + reasoningBurstPrompt, + REASONING_BURST_COUNT, + waitForExactReasoningBurst, +} from "../../helpers/session-stream-overload"; +import { attachGatewayTrafficCapture, summarizeGatewayTraffic } from "../../helpers/ws-traffic"; + +test.describe("Session stream overload isolation", () => { + test("keeps a quiet session usable while another session streams", async ({ + testPage, + apiClient, + seedData, + }) => { + test.setTimeout(180_000); + const quietCapture = attachGatewayTrafficCapture(testPage); + + const noisyTask = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Noisy reasoning isolation task", + seedData.agentProfileId, + { + description: reasoningBurstPrompt(), + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + if (!noisyTask.session_id) throw new Error("noisy task has no session"); + + // Keep a real subscribed browser open for the noisy session while the + // primary page navigates to the quiet task. This exercises the same + // per-client isolation boundary that failed during the incident. + const noisyPage = await testPage.context().newPage(); + const noisyCapture = attachGatewayTrafficCapture(noisyPage); + await noisyPage.goto(`/t/${noisyTask.id}`); + const noisySession = new SessionPage(noisyPage); + await noisySession.waitForLoad(); + + const quietTask = await apiClient.createTaskWithAgent( + seedData.workspaceId, + "Quiet session isolation task", + seedData.agentProfileId, + { + description: "/e2e:simple-message", + workflow_id: seedData.workflowId, + workflow_step_id: seedData.startStepId, + repository_ids: [seedData.repositoryId], + }, + ); + if (!quietTask.session_id) throw new Error("quiet task has no session"); + + await testPage.goto(`/t/${quietTask.id}`); + const quietSession = new SessionPage(testPage); + await quietSession.waitForLoad(); + await quietSession.waitForChatIdle({ timeout: 60_000 }); + await quietSession.sendMessageViaButton("quiet-session-followup"); + await quietSession.expectChatResponseVisible("quiet-session-followup", 0, { timeout: 60_000 }); + await assertNoHorizontalOverflow(testPage, "session stream overload desktop surface"); + + const exact = await waitForExactReasoningBurst(apiClient, noisyTask.session_id); + const noisyFrames = noisyReceivedFrames(noisyCapture.frames, noisyTask.session_id); + const quietSessionNoisyFrames = noisyReceivedFrames(quietCapture.frames, noisyTask.session_id); + expect( + noisyFrames.length, + "noisy session must produce observable update frames", + ).toBeGreaterThan(0); + expect(noisyFrames.length, "gateway delivery must be below source chunk count").toBeLessThan( + REASONING_BURST_COUNT, + ); + expect( + quietSessionNoisyFrames, + "quiet page must not receive noisy-session updates", + ).toHaveLength(0); + + const evidence = { + sourceChunks: exact.sourceChunks, + reasoningBytes: exact.reasoningBytes, + gatewayReceivedUpdatedFrames: noisyFrames.length, + quietSessionResponse: "quiet-session-followup", + quietGateway: summarizeGatewayTraffic(quietCapture.frames), + noisyGateway: summarizeGatewayTraffic(noisyCapture.frames), + }; + await test.info().attach("session-stream-overload-evidence.json", { + body: JSON.stringify(evidence, null, 2), + contentType: "application/json", + }); + test.info().annotations.push({ + type: "stream-overload-evidence", + description: JSON.stringify(evidence), + }); + + await noisyPage.close(); + }); +}); diff --git a/apps/web/lib/ws/handlers/messages.test.ts b/apps/web/lib/ws/handlers/messages.test.ts new file mode 100644 index 0000000000..1bd3a3f7b8 --- /dev/null +++ b/apps/web/lib/ws/handlers/messages.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it, vi } from "vitest"; +import type { StoreApi } from "zustand"; +import type { AppState } from "@/lib/state/store"; +import type { BackendMessageMap } from "@/lib/types/backend"; +import { createMessagesHandlerRegistration, createMessageUpdateScheduler } from "./messages"; + +type UpdatedMessage = BackendMessageMap["session.message.updated"]; +const TEST_TIMESTAMP = "2026-08-02T00:00:00.000Z"; + +function makePayload(sessionId: string, messageId: string, content: string) { + return { + task_id: "task-1", + session_id: sessionId, + message_id: messageId, + turn_id: "turn-1", + author_type: "agent" as const, + author_id: "agent-1", + content, + type: "message", + created_at: TEST_TIMESTAMP, + updated_at: "2026-08-02T00:00:01.000Z", + }; +} + +function makeUpdated(sessionId: string, messageId: string, content: string): UpdatedMessage { + return { + id: `${sessionId}-${messageId}`, + type: "notification", + action: "session.message.updated", + payload: makePayload(sessionId, messageId, content), + timestamp: TEST_TIMESTAMP, + }; +} + +function makeStore(currentMessages: Record = {}) { + const updateMessage = vi.fn(); + const addMessage = vi.fn(); + const removeMessage = vi.fn(); + const state = { + updateMessage, + addMessage, + removeMessage, + messages: { bySession: currentMessages }, + }; + return { + store: { getState: () => state } as unknown as StoreApi, + updateMessage, + addMessage, + removeMessage, + }; +} + +function makeFrameScheduler() { + let frame: (() => void) | null = null; + const schedule = vi.fn((callback: () => void) => { + frame = callback; + return 1; + }); + const cancel = vi.fn(() => { + frame = null; + }); + return { + schedule, + cancel, + runFrame: () => { + const callback = frame; + frame = null; + callback?.(); + }, + }; +} + +describe("session message frame scheduler", () => { + it("applies hundreds of same-key updates once with the newest full payload", () => { + const { store, updateMessage } = makeStore(); + const frame = makeFrameScheduler(); + const registration = createMessagesHandlerRegistration(store, frame); + const handler = registration.handlers["session.message.updated"]!; + + for (let index = 0; index < 300; index += 1) { + handler(makeUpdated("session-1", "message-1", `content-${index}`)); + } + + expect(updateMessage).not.toHaveBeenCalled(); + expect(frame.schedule).toHaveBeenCalledTimes(1); + frame.runFrame(); + + expect(updateMessage).toHaveBeenCalledTimes(1); + expect(updateMessage).toHaveBeenLastCalledWith( + expect.objectContaining({ + id: "message-1", + session_id: "session-1", + content: "content-299", + }), + ); + registration.dispose(); + }); + + it("updates different message keys once each in insertion order", () => { + const { store, updateMessage } = makeStore(); + const frame = makeFrameScheduler(); + const registration = createMessagesHandlerRegistration(store, frame); + const handler = registration.handlers["session.message.updated"]!; + + handler(makeUpdated("session-1", "message-a", "a1")); + handler(makeUpdated("session-2", "message-b", "b1")); + handler(makeUpdated("session-1", "message-a", "a2")); + frame.runFrame(); + + expect(updateMessage).toHaveBeenCalledTimes(2); + expect(updateMessage.mock.calls.map(([message]) => message.content)).toEqual(["a2", "b1"]); + registration.dispose(); + }); + + it("flushes pending updates before add and delete barriers", () => { + const { store, updateMessage, addMessage, removeMessage } = makeStore(); + const frame = makeFrameScheduler(); + const registration = createMessagesHandlerRegistration(store, frame); + const handlers = registration.handlers; + + handlers["session.message.updated"]!(makeUpdated("session-1", "message-1", "before-delete")); + handlers["session.message.deleted"]!({ + id: "delete-1", + type: "notification", + action: "session.message.deleted", + payload: makePayload("session-1", "message-1", "ignored"), + timestamp: TEST_TIMESTAMP, + }); + expect(updateMessage).toHaveBeenCalledTimes(1); + expect(removeMessage).toHaveBeenCalledWith("session-1", "message-1"); + frame.runFrame(); + expect(updateMessage).toHaveBeenCalledTimes(1); + + handlers["session.message.updated"]!(makeUpdated("session-1", "message-2", "before-add")); + handlers["session.message.added"]!({ + id: "add-1", + type: "notification", + action: "session.message.added", + payload: makePayload("session-1", "message-3", "added"), + timestamp: TEST_TIMESTAMP, + }); + expect(updateMessage).toHaveBeenCalledTimes(2); + expect(addMessage).toHaveBeenCalledTimes(1); + frame.runFrame(); + expect(updateMessage).toHaveBeenCalledTimes(2); + registration.dispose(); + }); + + it("clears scheduled work when the handler registration is disposed", () => { + const { store, updateMessage } = makeStore(); + const frame = makeFrameScheduler(); + const registration = createMessagesHandlerRegistration(store, frame); + registration.handlers["session.message.updated"]!( + makeUpdated("session-1", "message-1", "discarded"), + ); + + registration.dispose(); + expect(frame.cancel).toHaveBeenCalledTimes(1); + frame.runFrame(); + expect(updateMessage).not.toHaveBeenCalled(); + }); + + it("flushes a pending update as a turn-settle barrier", () => { + const { store, updateMessage } = makeStore(); + const frame = makeFrameScheduler(); + const scheduler = createMessageUpdateScheduler(store, frame); + scheduler.enqueue(makePayload("session-1", "message-1", "settled")); + + scheduler.flush(); + + expect(updateMessage).toHaveBeenCalledTimes(1); + expect(updateMessage).toHaveBeenLastCalledWith(expect.objectContaining({ content: "settled" })); + scheduler.dispose(); + }); +}); + +describe("session message snapshot ordering", () => { + it("does not apply a batched update older than a refetched snapshot", () => { + const { store, updateMessage } = makeStore({ + "session-1": [ + { + id: "message-1", + updated_at: "2026-08-02T00:00:02.000Z", + }, + ], + }); + const frame = makeFrameScheduler(); + const registration = createMessagesHandlerRegistration(store, frame); + + registration.handlers["session.message.updated"]!( + makeUpdated("session-1", "message-1", "stale"), + ); + frame.runFrame(); + + expect(updateMessage).not.toHaveBeenCalled(); + registration.dispose(); + }); +}); diff --git a/apps/web/lib/ws/handlers/messages.ts b/apps/web/lib/ws/handlers/messages.ts index b89740c488..d775a2bc39 100644 --- a/apps/web/lib/ws/handlers/messages.ts +++ b/apps/web/lib/ws/handlers/messages.ts @@ -1,58 +1,154 @@ import type { StoreApi } from "zustand"; import type { AppState } from "@/lib/state/store"; +import type { MessageAddedPayload } from "@/lib/types/backend"; import type { WsHandlers } from "@/lib/ws/handlers/types"; -import { sessionId, taskId, type MessageType } from "@/lib/types/http"; +import { sessionId, taskId, type Message, type MessageType } from "@/lib/types/http"; -export function registerMessagesHandlers(store: StoreApi): WsHandlers { +type MessagePayload = MessageAddedPayload; +type MessageUpdate = Omit & { + session_id: ReturnType; + task_id: ReturnType; +}; + +export type MessageFrameSchedulerOptions = { + schedule?: (callback: () => void) => number; + cancel?: (handle: number) => void; +}; + +export type MessageUpdateScheduler = { + enqueue: (payload: MessagePayload) => void; + flush: () => void; + dispose: () => void; +}; + +export type MessageHandlerRegistration = { + handlers: WsHandlers; + scheduler: MessageUpdateScheduler; + dispose: () => void; +}; + +function toMessage(payload: MessagePayload): MessageUpdate { return { + id: payload.message_id, + session_id: sessionId(payload.session_id), + task_id: taskId(payload.task_id), + turn_id: payload.turn_id, + author_type: payload.author_type, + author_id: payload.author_id, + content: payload.content, + raw_content: payload.raw_content, + type: (payload.type as MessageType) || "message", + metadata: payload.metadata, + requests_input: payload.requests_input, + created_at: payload.created_at, + updated_at: payload.updated_at, + }; +} + +function messageKey(payload: MessagePayload): string { + return `${payload.session_id}:${payload.message_id}`; +} + +function isOlderThanCurrentSnapshot(store: StoreApi, payload: MessagePayload): boolean { + const current = store + .getState() + .messages.bySession[ + sessionId(payload.session_id) + ]?.find((message) => message.id === payload.message_id); + if (!current?.updated_at) return false; + return !payload.updated_at || current.updated_at > payload.updated_at; +} + +function defaultSchedule(callback: () => void): number { + if (typeof requestAnimationFrame === "function") { + return requestAnimationFrame(() => callback()); + } + return globalThis.setTimeout(callback, 16) as unknown as number; +} + +function defaultCancel(handle: number): void { + if (typeof cancelAnimationFrame === "function") { + cancelAnimationFrame(handle); + return; + } + globalThis.clearTimeout(handle); +} + +export function createMessageUpdateScheduler( + store: StoreApi, + options: MessageFrameSchedulerOptions = {}, +): MessageUpdateScheduler { + const schedule = options.schedule ?? defaultSchedule; + const cancel = options.cancel ?? defaultCancel; + const pending = new Map(); + let scheduledHandle: number | null = null; + let disposed = false; + + const flush = () => { + if (scheduledHandle !== null) { + cancel(scheduledHandle); + scheduledHandle = null; + } + if (disposed || pending.size === 0) { + pending.clear(); + return; + } + const updates = [...pending.values()]; + pending.clear(); + for (const payload of updates) { + if (!payload.session_id || !payload.message_id) continue; + if (isOlderThanCurrentSnapshot(store, payload)) continue; + store.getState().updateMessage(toMessage(payload)); + } + }; + + const enqueue = (payload: MessagePayload) => { + if (disposed || !payload.session_id || !payload.message_id) return; + pending.set(messageKey(payload), payload); + if (scheduledHandle !== null) return; + scheduledHandle = schedule(() => { + scheduledHandle = null; + flush(); + }); + }; + + const dispose = () => { + if (disposed) return; + disposed = true; + if (scheduledHandle !== null) { + cancel(scheduledHandle); + scheduledHandle = null; + } + pending.clear(); + }; + + return { enqueue, flush, dispose }; +} + +export function createMessagesHandlerRegistration( + store: StoreApi, + options?: MessageFrameSchedulerOptions, +): MessageHandlerRegistration { + const scheduler = createMessageUpdateScheduler(store, options); + const handlers: WsHandlers = { "session.message.added": (message) => { const payload = message.payload; - if (!payload.session_id) { - return; - } - store.getState().addMessage({ - id: payload.message_id, - session_id: sessionId(payload.session_id), - task_id: taskId(payload.task_id), - turn_id: payload.turn_id, - author_type: payload.author_type, - author_id: payload.author_id, - content: payload.content, - raw_content: payload.raw_content, - type: (payload.type as MessageType) || "message", - metadata: payload.metadata, - requests_input: payload.requests_input, - created_at: payload.created_at, - updated_at: payload.updated_at, - }); + if (!payload.session_id) return; + scheduler.flush(); + store.getState().addMessage(toMessage(payload)); }, "session.message.updated": (message) => { - const payload = message.payload; - if (!payload.session_id) { - return; - } - store.getState().updateMessage({ - id: payload.message_id, - session_id: sessionId(payload.session_id), - task_id: taskId(payload.task_id), - turn_id: payload.turn_id, - author_type: payload.author_type, - author_id: payload.author_id, - content: payload.content, - raw_content: payload.raw_content, - type: (payload.type as MessageType) || "message", - metadata: payload.metadata, - requests_input: payload.requests_input, - created_at: payload.created_at, - updated_at: payload.updated_at, - }); + scheduler.enqueue(message.payload); }, "session.message.deleted": (message) => { const payload = message.payload; - if (!payload.session_id) { - return; - } + if (!payload.session_id) return; + // A delete is a semantic barrier. Applying pending updates before the + // removal prevents a stale animation-frame callback from resurrecting + // the deleted row. + scheduler.flush(); store.getState().removeMessage(sessionId(payload.session_id), payload.message_id); }, }; + return { handlers, scheduler, dispose: scheduler.dispose }; } diff --git a/apps/web/lib/ws/handlers/turns.test.ts b/apps/web/lib/ws/handlers/turns.test.ts index 4c0393cada..f735329fa1 100644 --- a/apps/web/lib/ws/handlers/turns.test.ts +++ b/apps/web/lib/ws/handlers/turns.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { create } from "zustand"; import { immer } from "zustand/middleware/immer"; import { createSessionSlice } from "@/lib/state/slices/session/session-slice"; @@ -12,6 +12,7 @@ const TURN_STARTED = "session.turn.started"; const TURN_COMPLETED = "session.turn.completed"; const NOTIFICATION = "notification"; const TURN_STARTED_AT = "2026-07-23T10:00:00.000Z"; +const TURN_COMPLETED_AT = "2026-07-23T10:01:00.000Z"; function makeStore() { return create()( @@ -48,7 +49,7 @@ function send( describe("session turn WebSocket handlers", () => { it("keeps a completed turn inactive when its delayed started event arrives", () => { const store = makeStore(); - const completed = "2026-07-23T10:01:00.000Z"; + const completed = TURN_COMPLETED_AT; send(store, TURN_COMPLETED, turn("turn-1", TURN_STARTED_AT, completed)); send(store, TURN_STARTED, turn("turn-1", TURN_STARTED_AT)); @@ -63,7 +64,7 @@ describe("session turn WebSocket handlers", () => { const store = makeStore(); send(store, TURN_STARTED, turn("turn-a", TURN_STARTED_AT)); - send(store, TURN_STARTED, turn("turn-b", "2026-07-23T10:01:00.000Z")); + send(store, TURN_STARTED, turn("turn-b", TURN_COMPLETED_AT)); send(store, TURN_COMPLETED, turn("turn-a", TURN_STARTED_AT, "2026-07-23T10:02:00.000Z")); expect(store.getState().turns.activeBySession[SESSION_ID]).toBe("turn-b"); @@ -71,7 +72,7 @@ describe("session turn WebSocket handlers", () => { it("tracks a normal turn from start through completion", () => { const store = makeStore(); - const completed = "2026-07-23T10:01:00.000Z"; + const completed = TURN_COMPLETED_AT; send(store, TURN_STARTED, turn("turn-1", TURN_STARTED_AT)); expect(store.getState().turns.activeBySession[SESSION_ID]).toBe("turn-1"); @@ -80,4 +81,19 @@ describe("session turn WebSocket handlers", () => { expect(store.getState().turns.activeBySession[SESSION_ID]).toBeNull(); expect(store.getState().turns.bySession[SESSION_ID][0].completed_at).toBe(completed); }); + + it("flushes batched message updates before completing a turn", () => { + const store = makeStore(); + const flush = vi.fn(); + const handler = registerTurnsHandlers(store as never, { flush })[TURN_COMPLETED]!; + + handler({ + id: "complete-1", + type: NOTIFICATION, + action: TURN_COMPLETED, + payload: turn("turn-1", TURN_STARTED_AT, TURN_COMPLETED_AT), + } as never); + + expect(flush).toHaveBeenCalledTimes(1); + }); }); diff --git a/apps/web/lib/ws/handlers/turns.ts b/apps/web/lib/ws/handlers/turns.ts index eb19be3a38..999d701707 100644 --- a/apps/web/lib/ws/handlers/turns.ts +++ b/apps/web/lib/ws/handlers/turns.ts @@ -4,6 +4,7 @@ import type { AppState } from "@/lib/state/store"; import type { WsHandlers } from "@/lib/ws/handlers/types"; import { sessionId, taskId } from "@/lib/types/http"; import { maybeEmitEmptyTurnNotice } from "@/lib/ws/handlers/empty-turn-notice"; +import type { MessageUpdateScheduler } from "@/lib/ws/handlers/messages"; const debug = createDebugLogger("session:turns"); @@ -23,7 +24,10 @@ function completePendingToolCalls(store: StoreApi, sessionId: string): } } -export function registerTurnsHandlers(store: StoreApi): WsHandlers { +export function registerTurnsHandlers( + store: StoreApi, + messageScheduler?: Pick, +): WsHandlers { return { "session.turn.started": (message) => { const payload = message.payload; @@ -53,6 +57,7 @@ export function registerTurnsHandlers(store: StoreApi): WsHandlers { if (!payload.session_id || !payload.id) { return; } + messageScheduler?.flush(); debug("turn.completed", { sessionId: payload.session_id, task_id: payload.task_id ?? "-", diff --git a/apps/web/lib/ws/router.ts b/apps/web/lib/ws/router.ts index 73d4d6c2a3..33f50fce27 100644 --- a/apps/web/lib/ws/router.ts +++ b/apps/web/lib/ws/router.ts @@ -13,7 +13,7 @@ import { registerSessionTodosHandlers } from "@/lib/ws/handlers/session-todos"; import { registerPromptUsageHandlers } from "@/lib/ws/handlers/prompt-usage"; import { registerWorkflowsHandlers } from "@/lib/ws/handlers/workflows"; -import { registerMessagesHandlers } from "@/lib/ws/handlers/messages"; +import { createMessagesHandlerRegistration } from "@/lib/ws/handlers/messages"; import { registerNotificationsHandlers } from "@/lib/ws/handlers/notifications"; import { registerDiffsHandlers } from "@/lib/ws/handlers/diffs"; import { registerExecutorsHandlers } from "@/lib/ws/handlers/executors"; @@ -37,7 +37,8 @@ import { registerOfficeHandlers } from "@/lib/ws/handlers/office"; import { registerRunHandlers } from "@/lib/ws/handlers/run"; export function registerWsHandlers(store: StoreApi) { - return { + const messages = createMessagesHandlerRegistration(store); + const handlers = { ...registerKanbanHandlers(store), ...registerTasksHandlers(store), ...registerTaskPlansHandlers(store), @@ -63,15 +64,16 @@ export function registerWsHandlers(store: StoreApi) { ...registerUsersHandlers(store), ...registerTerminalsHandlers(store), ...registerDiffsHandlers(store), - ...registerMessagesHandlers(store), + ...messages.handlers, ...registerNotificationsHandlers(store), ...registerSecretsHandlers(store), ...registerGitStatusHandlers(store), ...registerSystemEventsHandlers(store), - ...registerTurnsHandlers(store), + ...registerTurnsHandlers(store, messages.scheduler), ...registerGitHubHandlers(store), ...registerGitLabHandlers(store), ...registerOfficeHandlers(store), ...registerRunHandlers(), }; + return { handlers, dispose: messages.dispose }; } diff --git a/apps/web/lib/ws/use-websocket.test.tsx b/apps/web/lib/ws/use-websocket.test.tsx index 9d8bef902e..eb63cc3cd0 100644 --- a/apps/web/lib/ws/use-websocket.test.tsx +++ b/apps/web/lib/ws/use-websocket.test.tsx @@ -21,7 +21,9 @@ vi.mock("@/lib/ws/client", () => ({ }, })); -vi.mock("@/lib/ws/router", () => ({ registerWsHandlers: vi.fn(() => ({})) })); +vi.mock("@/lib/ws/router", () => ({ + registerWsHandlers: vi.fn(() => ({ handlers: {}, dispose: vi.fn() })), +})); vi.mock("@/lib/ws/connection", () => ({ setWebSocketClient: vi.fn() })); vi.mock("@/lib/debug/log", () => ({ createDebugLogger: () => vi.fn() })); diff --git a/apps/web/lib/ws/use-websocket.tsx b/apps/web/lib/ws/use-websocket.tsx index aca54ce79c..e358700420 100644 --- a/apps/web/lib/ws/use-websocket.tsx +++ b/apps/web/lib/ws/use-websocket.tsx @@ -45,13 +45,14 @@ export function useWebSocket(store: StoreApi, url: string) { client.connect(); setWebSocketClient(client); - const handlers = registerWsHandlers(store); - const unsubscribers = Object.entries(handlers).map(([type, handler]) => - client.on(type as keyof typeof handlers, handler as never), + const registration = registerWsHandlers(store); + const unsubscribers = Object.entries(registration.handlers).map(([type, handler]) => + client.on(type as keyof typeof registration.handlers, handler as never), ); return () => { unsubscribers.forEach((unsubscribe) => unsubscribe()); + registration.dispose(); connectionIssueMonitor.dispose(); client.disconnect(); setWebSocketClient(null); diff --git a/docs/decisions/2026-08-02-isolate-replaceable-session-stream-traffic.md b/docs/decisions/2026-08-02-isolate-replaceable-session-stream-traffic.md new file mode 100644 index 0000000000..776c97e367 --- /dev/null +++ b/docs/decisions/2026-08-02-isolate-replaceable-session-stream-traffic.md @@ -0,0 +1,120 @@ +# ADR-2026-08-02-isolate-replaceable-session-stream-traffic: Isolate Replaceable Session Stream Traffic + +**Status:** accepted +**Date:** 2026-08-02 +**Area:** backend, frontend, protocol + +## Context + +The bounded task-status design separated correlated WebSocket responses from +best-effort notifications and removed task-switcher subscriptions to inactive +session details. It did not bound the work produced by one actively subscribed +session. + +Incident task `69637afd-c193-4197-aa49-348ac2b0cfb3` exposed that remaining +failure mode. Its ACP execution emitted 28,967 unique `thinking_streaming` +events and 31,314 total stream events over about 41 minutes, peaking at 63 +reasoning chunks per second. Each tiny reasoning chunk caused an event-bus +publication, a read/concatenate/full-message database update, a full growing +`session.message.updated` payload, one shared WebSocket notification slot, and +one Zustand/React update. The notification queue overflowed briefly and +dropped five message updates. More importantly, continuous serialization and +browser state churn made the frontend unusable until the backend stopped. + +The same capture showed an independent amplifier in the Office task detail: +refetches replaced session objects, causing unchanged multi-session +subscriptions to be torn down and recreated. Some connections issued more than +100 subscribe/unsubscribe operations and snapshots in seconds. + +The backend remained running and correlated responses already had a separate +control queue. Therefore this was not a backend crash or response-priority +failure. It was unbounded replaceable work across the ingress, delivery, and +render boundaries. + +## Decision + +Kandev will treat agent text and reasoning chunks as a high-frequency ingress +format rather than a persistence or render contract. + +The lifecycle manager will coalesce adjacent chunks for the same execution, +stream kind, and Kandev message record. The first non-empty chunk creates the record +immediately; later chunks flush no more than once per 100 ms window during +continuous streaming. Semantic boundaries and teardown flush pending content +before they proceed. Coalescing is lossless: final persisted assistant and +reasoning content is byte-for-byte equivalent to the ordered input chunks. + +The per-agent ACP adapter/process handoff will apply cancellation-aware +backpressure when its bounded event channel fills. Normalized assistant and +reasoning chunks are not silently discarded before lifecycle coalescing; only a +terminal shutdown may cancel an outstanding send. Because each handoff belongs +to one agent instance, a noisy session slows its own producer rather than +blocking unrelated backend sessions. + +The WebSocket gateway will distinguish semantic notifications from +replaceable full-state stream notifications. `session.message.updated` is +replaceable by `(session_id, message_id)` because every payload contains the +complete current public message state. Each client gets bounded per-session +replaceable capacity. A newer update replaces the queued payload in place, and +session queues are drained fairly. Semantic notifications and correlated +control traffic have independent bounded capacity, so a noisy session cannot +occupy their slots. Overload may evict the oldest queued replaceable entry from +the offending session, including an entry that has not yet been superseded. +Because the database remains authoritative, reconnect, snapshot, and +turn-settle reconciliation repair any intermediate replacement that was not +delivered; authoritative persistence is never discarded. + +The frontend will coalesce replacement updates once more at the animation-frame +boundary and will key intentional multi-session subscriptions by stable session +identity. Add/delete/turn-settle handling remains ordered so frame batching +cannot resurrect a deleted message or hide a terminal state. + +Each layer will expose content-free counters and structured context for +received chunks, coalesced chunks, flushes, queued replacements, replacements, +and per-session evictions. Tests use small injected capacities and fake time; +production constants remain implementation details so they can be tuned +without changing the protocol. + +## Consequences + +A 63-chunk-per-second agent is reduced to at most ten append publications per +second for a continuously streaming message before gateway replacement and +browser-frame batching. Database writes, JSON size amplification, notification +slots, and React commits are bounded independently. Other sessions and +semantic state continue to move even if the noisy session remains active. + +The transcript remains authoritative and complete, but an observer may skip +intermediate visual states and jump directly to a newer full-message +replacement. This is acceptable because streaming updates are progressive +render hints, not individual durable occurrences. Reconnect and existing +session reconciliation repair missed intermediate delivery. + +The gateway becomes a small typed scheduler instead of a single notification +FIFO. Ordering tests must cover replacement-in-place, semantic barriers, +fairness, teardown, and zero-value clients. Lifecycle timers add teardown and +race obligations, including execution replacement and backend shutdown. + +No new mobile layout or user-facing copy is introduced. Desktop and mobile use +the same store and delivery behavior. + +## Alternatives Considered + +- **Increase the 256-frame notification queue.** Rejected because it converts + immediate drops into a larger stale backlog, consumes more memory, and leaves + database and browser work unbounded. +- **Rate-limit or terminate the ACP process.** Rejected as the primary repair + because agents may legitimately emit long reasoning streams, and killing the + producer loses useful work instead of removing redundant intermediate work. +- **Drop normalized events when the per-agent handoff queue is full.** Rejected + because it makes the supposedly lossless lifecycle coalescer observe an + incomplete stream. Cancellation-aware backpressure preserves transcript + content while keeping the pressure scoped to the producing agent instance. +- **Coalesce only in the browser.** Rejected because the event bus, database, + JSON encoding, and WebSocket queue would still process every chunk. +- **Coalesce only in the lifecycle manager.** Rejected because other producers, + older backends during rollout, or bursty full-state updates could still + pressure a client; each shared boundary needs its own containment. +- **Drop all reasoning updates.** Rejected because live reasoning is useful and + persisted transcripts must remain complete. +- **Give every session a separate WebSocket connection.** Rejected because it + multiplies connection lifecycle and authorization complexity while bounded + per-session scheduling provides isolation on the existing transport. diff --git a/docs/decisions/INDEX.md b/docs/decisions/INDEX.md index 49c96aabdf..428cd488d2 100644 --- a/docs/decisions/INDEX.md +++ b/docs/decisions/INDEX.md @@ -106,3 +106,4 @@ Read individual ADRs for full context. Create new ones via `/record decision` or | 2026-08-02-new-workspace-github-access-defaults | [Bootstrap New Workspaces From Host GitHub Access](2026-08-02-new-workspace-github-access-defaults.md) | accepted | backend, frontend, security | 2026-08-02 | | 2026-08-02-single-owner-agent-task-titles | [Assign Agent Task Titles to One Session](2026-08-02-single-owner-agent-task-titles.md) | accepted | backend, frontend, protocol, workflow | 2026-08-02 | | 2026-08-02-agent-terminal-diagnostics-over-stderr | [Capture Agent Terminal Diagnostics From Managed Stderr](2026-08-02-agent-terminal-diagnostics-over-stderr.md) | accepted | backend, frontend, protocol, security | 2026-08-02 | +| 2026-08-02-isolate-replaceable-session-stream-traffic | [Isolate Replaceable Session Stream Traffic](2026-08-02-isolate-replaceable-session-stream-traffic.md) | accepted | backend, frontend, protocol | 2026-08-02 | diff --git a/docs/plans/session-stream-overload-isolation/plan.md b/docs/plans/session-stream-overload-isolation/plan.md new file mode 100644 index 0000000000..5607ea3b23 --- /dev/null +++ b/docs/plans/session-stream-overload-isolation/plan.md @@ -0,0 +1,247 @@ +--- +spec: docs/specs/platform/bounded-task-status-delivery.md +decision: docs/decisions/2026-08-02-isolate-replaceable-session-stream-traffic.md +created: 2026-08-02 +status: complete +--- + +# Implementation Plan: Session Stream Overload Isolation + +## Overview + +Contain a single high-volume ACP session at three independent boundaries while +keeping the final transcript exact. Coalesce adjacent assistant/reasoning +chunks before the orchestrator performs read/append/write work, replace queued +full-message notifications instead of accumulating stale versions, and batch +browser store updates once per animation frame. Stabilize Office multi-session +membership so task refetches do not replay snapshots. Prove the contract with a +deterministic noisy mock-agent scenario on desktop and mobile. + +This extends the implemented bounded task-status delivery design. It does not +change task summaries, message persistence ownership, or the reserved +correlated-response queue. + +## Incident baseline + +- Task `69637afd-c193-4197-aa49-348ac2b0cfb3`, session + `89f7b247-cfb3-4a4b-8147-8826e13f1630`. +- The recorded launcher was `opencode-acp` using `opencode-go/kimi-k3`; the + logs do not identify this execution as Grok. +- 28,967 unique reasoning-stream events and 31,314 total stream events over + about 41 minutes, peaking at 63 reasoning events per second. +- Every chunk crossed lifecycle, orchestrator, message persistence, gateway, + JSON, Zustand, and React boundaries. +- Five `session.message.updated` frames were dropped when the 256-frame + notification queue filled. No control-queue overflow or backend crash was + recorded. +- Equivalent Office task refetches caused repeated unchanged session + unsubscribe/subscribe/snapshot cycles, further amplifying traffic. + +## Architecture + +### Lossless lifecycle coalescing + +Add an execution-owned coalescer in +`apps/backend/internal/agent/runtime/lifecycle`. It accepts already-normalized +assistant/reasoning chunks after legacy newline buffering and protocol ID to +Kandev ID mapping. The first non-empty chunk for a record publishes immediately with +`IsAppend=false`. Adjacent later chunks append to one pending segment and flush +at the 100 ms cadence with `IsAppend=true`. + +Keep ordering explicit. Before a tool call/result, permission request, +completion, error, stream disconnect, prompt-generation reset, execution +replacement, or manager shutdown, synchronously detach and publish pending +segments before publishing the boundary. Timer callbacks detach under the +execution mutex and publish outside it. Teardown stops timers and makes repeat +flushes idempotent. + +The coalescer operates before `handleThinkingStreamingEvent` / +`handleMessageStreamingEvent`, so the existing task service naturally performs +fewer `GetMessage`, concatenation, `UpdateMessage`, and full-message event +publications. It does not change stored schemas or final message shapes. + +### Session-isolated notification scheduling + +Replace the client's single best-effort notification channel with a scheduler +that retains three classes: + +1. the existing control queue for correlated responses/errors; +2. a bounded semantic FIFO for non-replaceable notifications; and +3. bounded per-session queues for replaceable `session.message.updated` + notifications. + +The hub passes action and routing identity to the client instead of only raw +bytes. For a replaceable message update, derive the key from `session_id` and +`message_id`. If the key is already pending, replace its bytes without moving +its position. A later semantic event for the same session therefore remains +behind the newest message state. Drain active session queues round-robin and +interleave them with the semantic FIFO after bounded control priority. + +Production capacities are named constants, but tests construct small queues. +On per-session pressure, evict or reject only queued replaceable entries for +that session. Never evict control or semantic notifications to admit a stream +replacement. Record content-free counters/log fields for action, client, +session, replacement, eviction, and queue depth. + +### Frontend batching and stable membership + +Extract a small replaceable-message scheduler beside +`lib/ws/handlers/messages.ts`. During one animation frame it retains only the +newest full payload per `(session_id, message_id)` and then invokes +`updateMessage` once per changed key. Added/deleted messages and turn-settle +events act as barriers: flush or cancel a pending replacement as appropriate so +wire order cannot revive deleted state. Clear scheduled work when the WS router +is disposed or replaced. + +In the Office task detail live-sync hook, derive a stable, sorted session-ID +membership key. Refetching equal session IDs or replacing the task object must +not rerun subscription cleanup/setup. A genuine membership change applies only +the added/removed IDs. The surface remains an intentional multi-session detail +view; this task removes churn rather than changing its product scope. + +### Observability and reconciliation + +Lifecycle and gateway logs/metrics must identify pressure without including +message content. Existing session snapshot/reconnect and turn-settle behavior +remain the reconciliation path; no new user-facing overload banner or protocol +action is required. + +## Tests + +### Backend + +- Use `testing/synctest` around the lifecycle coalescer. Assert immediate first + publication, one append per 100 ms window, exact 30,000-chunk concatenation, + independent assistant/reasoning IDs, boundary order, disconnect/shutdown + flush, and no timer/goroutine leak. +- Assert the orchestrator/task-service integration sees the reduced event count + and stores exact final assistant and thinking content. +- Construct gateway schedulers with tiny capacities. Assert replacement in + place, bounded per-session occupancy, round-robin progress, semantic barrier + order, control priority, noisy-session-only eviction, metrics, close safety, + and zero-value test-client compatibility. + +### Frontend + +- Feed hundreds of `session.message.updated` payloads for one message in a + single fake animation frame. Assert one store update with the newest complete + payload, while different message keys each update once. +- Cover add/delete/turn barriers, handler teardown, and separate sessions. +- Rerender Office live sync with new task/session object references but the + same IDs and assert no subscription churn; then change membership and assert + only the delta is subscribed/unsubscribed. + +## E2E tests + +Add a deterministic mock-agent command that emits a configurable reasoning +burst without sleeping or embedding thousands of commands in the prompt. A +new overload-isolation Playwright spec opens the noisy session and captures +gateway frames while interacting with a second session. + +- Desktop Chromium: emit a representative burst, navigate to the quiet task, + submit a message, and assert navigation/model controls/responding message + remain usable before the noisy run settles. +- Assert server-side burst count was actually produced, the final persisted + reasoning text is exact, received replacement frames are materially below + source chunks, no control response is lost, and the quiet session progresses. +- Mobile Chrome: repeat the navigation/quiet-session progress assertion through + the existing native task-switcher sheet and assert no horizontal overflow. +- Attach source-chunk, lifecycle-publication, gateway replacement/eviction, + received-frame, byte, and latency totals. Counts are diagnostics except for + the hard boundedness, exact-content, and cross-session-progress assertions. + +## Mobile design contract + +- **Desktop outcome / mobile entry:** both remain responsive while another + session streams. Desktop uses the existing task/sidebar navigation; phone + uses the existing task-switcher sheet. +- **Nearest exemplar:** keep + `apps/web/components/task/mobile/session-task-switcher-sheet.tsx` and shared + task/session stores unchanged in layout and navigation ownership. +- **Hierarchy / interaction:** no new control, gesture, dialog, or route. +- **Presentation / scroll:** the repair changes transport/state cadence only; + existing sheet scrolling, safe areas, and viewport bounds remain intact. +- **Shared state:** desktop and mobile consume the same batched message and + connection state. +- **Mobile proof:** run the overload scenario under `mobile-chrome`, switch to a + quiet task, verify progress, and assert `scrollWidth <= clientWidth`. + +## Implementation waves + +Implementation is sequential in the user-started session because Tasks 02 and +03 consume the delivery semantics established by the preceding task. These +task files do not authorize subagent work. + +- [x] [Task 01: Coalesce agent stream ingress](task-01-coalesce-agent-stream-ingress.md) +- [x] [Task 02: Isolate replaceable session delivery](task-02-isolate-replaceable-session-delivery.md) +- [x] [Task 03: Bound frontend stream work](task-03-bound-frontend-stream-work.md) +- [x] [Task 04: Prove overload containment](task-04-prove-overload-containment.md) + +## Verification + +All implementation waves are complete. The final verification evidence is: + +- `go test -race ./internal/agentctl/server/adapter/transport/acp ./internal/agentctl/server/process ./internal/agent/runtime/lifecycle ./internal/gateway/websocket ./cmd/mock-agent` — passed (2,633 tests). +- `make -C apps/backend test` — passed on the final run. +- `rtk proxy golangci-lint run ./...` — passed (0 issues). +- Focused frontend Vitest — passed (7 tests); web typecheck and lint — passed. +- Desktop Chromium overload E2E — passed (1 test, 13.4s). +- Mobile Chrome overload E2E — passed (1 test, 8.3s). + +The E2E evidence confirms exact persisted reasoning, bounded noisy-session +frames, quiet-session progress, and desktop/mobile viewport safety. The ACP +and process handoff now uses cancellation-aware per-agent backpressure so +normalized stream events are not silently discarded before lifecycle +coalescing; shutdown cancellation still releases a blocked noisy session. + +```bash +cd apps/backend && go test ./internal/agent/runtime/lifecycle/... \ + ./internal/orchestrator/... ./internal/task/service/... +cd apps/backend && go test ./internal/gateway/websocket/... +make -C apps/backend test +make -C apps/backend lint +cd apps/web && pnpm exec vitest run \ + lib/ws/handlers/messages.test.ts \ + 'app/office/tasks/[id]/use-session-live-sync.test.tsx' +cd apps/web && pnpm run typecheck +cd apps/web && pnpm run lint +cd apps/web && pnpm e2e:run tests/session/session-stream-overload-isolation.spec.ts \ + -- --project=chromium +cd apps/web && pnpm e2e:run tests/session/session-stream-overload-isolation.spec.ts \ + -- --project=mobile-chrome +``` + +## Risks + +- Timer callbacks racing completion could duplicate or reorder content. Detach + pending buffers atomically, publish outside locks, and make boundary flushes + idempotent. +- Protocol message IDs may interleave. Coalesce only adjacent chunks per typed + Kandev record and retain the first-seen queue position. +- A latest-wins gateway queue can reorder a message update past turn completion + if replacement positions are not stable. Preserve per-session enqueue order + and test semantic barriers. +- Strict control/semantic priority could starve replacement streams. Use + bounded priority bursts plus round-robin session scheduling. +- Animation-frame batching can be heavily throttled in background tabs. Keep + only complete latest state and rely on selected-session reconciliation when + the tab resumes. +- A quiet E2E producer would yield a false pass. Expose and assert the mock + burst's produced count before checking isolation. + +## Out of scope + +- Truncating stored assistant or reasoning content. +- Automatically stopping an agent because it streams heavily. +- Splitting sessions across multiple WebSocket connections. +- Changing task-summary fields, task-switcher layout, or message schema. +- Adding a user-facing overload state or runtime feature flag. + +## Documentation impact + +The behavioral contract is added to +`docs/specs/platform/bounded-task-status-delivery.md` and the shared boundary is +recorded in +`docs/decisions/2026-08-02-isolate-replaceable-session-stream-traffic.md`. +No public docs change is expected because commands, settings, public APIs, and +visible interaction patterns do not change. diff --git a/docs/plans/session-stream-overload-isolation/task-01-coalesce-agent-stream-ingress.md b/docs/plans/session-stream-overload-isolation/task-01-coalesce-agent-stream-ingress.md new file mode 100644 index 0000000000..b14a15dc72 --- /dev/null +++ b/docs/plans/session-stream-overload-isolation/task-01-coalesce-agent-stream-ingress.md @@ -0,0 +1,108 @@ +--- +id: "01-coalesce-agent-stream-ingress" +title: "Coalesce agent stream ingress" +status: done +wave: 1 +depends_on: [] +plan: "plan.md" +spec: "../../specs/platform/bounded-task-status-delivery.md" +--- + +# Task 01: Coalesce Agent Stream Ingress + +## Acceptance + +- The first non-empty assistant/reasoning chunk for a message publishes immediately; + adjacent append chunks publish no more than once per 100 ms window. +- A 30,000-chunk protocol reasoning stream persists byte-for-byte content with + a bounded publication/write count. +- Tool, permission, completion, error, disconnect, replacement, and shutdown + boundaries flush pending content before the boundary event. +- Interleaved message IDs/kinds preserve wire order, repeat flushes are + idempotent, and teardown leaks no timers or goroutines. +- Content-free counters/logs report received/coalesced/flushed chunk totals. + +## TDD sequence + +1. RED: add fake-time tests for immediate first publication, 100 ms append + cadence, exact burst concatenation, typed ID isolation, and semantic order. +2. RED: cover disconnect, prompt reset, execution replacement, shutdown, and a + timer callback racing a synchronous flush. +3. GREEN: add an execution-owned adjacent-segment coalescer, then route both + protocol and legacy streaming publications through it. +4. GREEN: flush at every lifecycle boundary and dispose the coalescer from all + execution teardown paths. +5. REFACTOR: isolate timer/segment mechanics, publish outside locks, and add + content-free observability without growing lifecycle event handlers. + +## Files likely touched + +- `apps/backend/internal/agent/runtime/lifecycle/types.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_events.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_streaming.go` +- lifecycle disconnect/reset/shutdown files that own execution teardown +- new `apps/backend/internal/agent/runtime/lifecycle/stream_coalescer.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go` +- new focused coalescer tests +- focused orchestrator/task-service streaming integration tests + +## Verification + +```bash +cd apps/backend && go test ./internal/agent/runtime/lifecycle/... \ + ./internal/orchestrator/... ./internal/task/service/... +``` + +## Dependencies + +None. This is the earliest amplification boundary and preserves the current +downstream message event contract. + +## Parallelism + +Sequential. Task 02 assumes the reduced but still defensive full-message update +stream produced here. + +## Inputs + +- Spec section **Session stream overload isolation**. +- Incident baseline: 28,967 reasoning events, peak 63/s. +- Existing protocol-ID and semantic-boundary tests in + `manager_events_test.go`. + +## Risks + +- Publishing while holding `messageMu` can deadlock callbacks; detach then + publish. +- A boundary missed during teardown can lose a final partial chunk. +- Coalescing across interleaved IDs or a tool boundary can corrupt transcript + order; only adjacent same-key chunks may combine. + +## Output contract + +Report exact input/published/write counts for the 30,000-chunk test, final +content equality, covered flush boundaries, metrics added, test commands, and +files changed. + +## Verification results + +- `go test -race ./internal/agent/runtime/lifecycle/...` — passed (1,059 tests + across lifecycle and lifecycle/skill packages). +- `go test ./internal/agent/runtime/lifecycle/... ./internal/orchestrator/... ./internal/task/service/...` — passed. +- The coalescer test feeds 30,000 chunks and emits two ordered segments while + preserving the exact concatenated content. +- Boundary coverage includes tool calls/results, completion, disconnect, + prompt handoff/reset, execution removal, timer flush, interleaved IDs, and + close-time stats logging. +- Added structured close-time counters for received, coalesced, and flushed + stream segments; no transcript content is logged. + +## Files changed + +- `apps/backend/internal/agent/runtime/lifecycle/stream_coalescer.go` +- `apps/backend/internal/agent/runtime/lifecycle/stream_coalescer_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/types.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_streaming.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_events.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_events_test.go` +- `apps/backend/internal/agent/runtime/lifecycle/manager_lifecycle.go` diff --git a/docs/plans/session-stream-overload-isolation/task-02-isolate-replaceable-session-delivery.md b/docs/plans/session-stream-overload-isolation/task-02-isolate-replaceable-session-delivery.md new file mode 100644 index 0000000000..094999327a --- /dev/null +++ b/docs/plans/session-stream-overload-isolation/task-02-isolate-replaceable-session-delivery.md @@ -0,0 +1,104 @@ +--- +id: "02-isolate-replaceable-session-delivery" +title: "Isolate replaceable session delivery" +status: done +wave: 2 +depends_on: ["01-coalesce-agent-stream-ingress"] +plan: "plan.md" +spec: "../../specs/platform/bounded-task-status-delivery.md" +--- + +# Task 02: Isolate Replaceable Session Delivery + +## Acceptance + +- `session.message.updated` uses one pending slot per client/session/message and + a newer full-state payload replaces the older payload in place. +- Replaceable capacity is bounded per session and globally; one session's + overflow cannot consume or evict control, semantic, or another session's + allowance. +- Active session queues make round-robin progress while control and semantic + traffic receive bounded priority. +- Message creation/deletion, turn completion, and state changes retain their + order relative to the newest queued replacement for that session. +- Replacement/eviction/depth telemetry carries IDs and action but no content. + +## TDD sequence + +1. RED: use tiny injected capacities to prove replacement-in-place, stable + ordering, per-session/global bounds, and noisy-session-only eviction. +2. RED: prove two quiet sessions, semantic notifications, and a correlated + response progress while one session continuously replaces updates. +3. RED: cover close/drain races, zero-value test clients, and existing control + overflow behavior. +4. GREEN: introduce a typed outbound notification scheduler and pass action, + session, and message routing identity from hub broadcasts. +5. GREEN: update `WritePump` to drain bounded priority bursts and round-robin + replaceable session queues. +6. REFACTOR: centralize classification/key extraction and add content-free + queue metrics/logs. + +## Files likely touched + +- `apps/backend/internal/gateway/websocket/client.go` +- `apps/backend/internal/gateway/websocket/hub.go` +- new gateway notification scheduler/classifier files +- `apps/backend/internal/gateway/websocket/client_send_queue_test.go` +- `apps/backend/internal/gateway/websocket/session_notifications_test.go` +- related hub broadcast and lifecycle/leak tests + +## Verification + +```bash +cd apps/backend && go test ./internal/gateway/websocket/... +cd apps/backend && go test -race ./internal/gateway/websocket/... +``` + +Result: `go test ./internal/gateway/websocket/... -count=1` passed (200 tests). +Race result: `go test -race ./internal/gateway/websocket/... -count=1` passed +(200 tests). + +Production capacities: 256 semantic FIFO frames, 32 distinct replaceable +message updates per session, and 256 replaceable updates per client. Control +responses/errors retain their independent 256-frame queue. Replacement keeps +its original per-session position; per-session pressure evicts only that +session's oldest replaceable entry, while a full global queue rejects an +incoming replacement whenever its session has no pending entry rather than +evicting another session. Queue logs include action, +client, session, message, reason, and depth only; content is never logged. + +Files changed: + +- `apps/backend/internal/gateway/websocket/client.go` +- `apps/backend/internal/gateway/websocket/hub.go` +- `apps/backend/internal/gateway/websocket/notification_scheduler.go` +- `apps/backend/internal/gateway/websocket/client_send_queue_test.go` + +## Dependencies + +Task 01 establishes lossless coalescing and boundary ordering before gateway +replacement is introduced. + +## Parallelism + +Sequential. Task 03 consumes the final replacement/barrier semantics. + +## Inputs + +- ADR **Isolate Replaceable Session Stream Traffic**. +- Existing reserved control queue and `client_send_queue_test.go`. +- Full-state `session.message.updated` payload from task service. + +## Risks + +- Moving a replacement to the back of a queue can overtake a terminal event; + update bytes in the existing slot. +- Absolute priority can starve stream rendering; use bounded priority bursts. +- Parsing identity from raw JSON in multiple places can drift; classify once + from the typed WebSocket message before enqueue. + +## Output contract + +Report production/test capacities, scheduling order, saturation measurements, +replacement and eviction behavior, metrics added, test results, and files +changed. diff --git a/docs/plans/session-stream-overload-isolation/task-03-bound-frontend-stream-work.md b/docs/plans/session-stream-overload-isolation/task-03-bound-frontend-stream-work.md new file mode 100644 index 0000000000..024b1652c9 --- /dev/null +++ b/docs/plans/session-stream-overload-isolation/task-03-bound-frontend-stream-work.md @@ -0,0 +1,106 @@ +--- +id: "03-bound-frontend-stream-work" +title: "Bound frontend stream work" +status: done +wave: 3 +depends_on: ["02-isolate-replaceable-session-delivery"] +plan: "plan.md" +spec: "../../specs/platform/bounded-task-status-delivery.md" +--- + +# Task 03: Bound Frontend Stream Work + +## Acceptance + +- During one animation frame, hundreds of updates for one message cause one + Zustand update with the newest complete payload. +- Different message keys each make progress once per frame; add/delete and + turn-settle barriers cannot resurrect or leave stale message state. +- Pending frame work is cleared when the router/client is disposed or replaced. +- Office task detail rerenders with equivalent task/session objects send no + subscription changes or snapshot replays; real membership changes apply only + the session-ID delta. +- Desktop and mobile share the same store behavior and no new copy/layout is + introduced. + +## TDD sequence + +1. RED: add message-handler tests with a fake animation-frame scheduler for + same-key replacement, multi-key fairness, barriers, and teardown. +2. RED: add an Office live-sync rerender test proving equivalent IDs do not + unsubscribe/resubscribe and changed membership applies only a delta. +3. GREEN: extract the frame-batched replacement helper and route + `session.message.updated` through it. +4. GREEN: extract/stabilize Office live sync around a sorted session-ID key and + incremental membership reconciliation. +5. REFACTOR: keep transport helpers independent of React components and verify + shared mobile/desktop state selectors do not need changes. + +## Files likely touched + +- `apps/web/lib/ws/handlers/messages.ts` +- new `apps/web/lib/ws/handlers/messages.test.ts` +- `apps/web/lib/ws/router.ts` or its disposal owner +- `apps/web/app/office/tasks/[id]/page.tsx` +- new extracted Office live-sync hook/helper and test +- focused session-slice tests if message merge semantics change + +## Verification + +```bash +cd apps/web && pnpm exec vitest run \ + lib/ws/handlers/messages.test.ts \ + 'app/office/tasks/[id]/use-session-live-sync.test.tsx' +cd apps/web && pnpm run typecheck +cd apps/web && pnpm run lint +``` + +## Dependencies + +Task 02 defines replacement and semantic-barrier ordering at the wire boundary. + +## Parallelism + +Sequential. Task 04 needs the complete backend/frontend containment stack. + +## Inputs + +- Existing `registerMessagesHandlers`, session slice `updateMessage`, and + Office `useSessionLiveSync` behavior. +- Mobile parity contract in the parent plan. + +## Risks + +- `requestAnimationFrame` is throttled in hidden tabs; retain only latest full + state and reconcile on resume. +- A queued update after delete can apply stale state unless deletion cancels + the key before store mutation. +- Sorting/memoizing IDs must not hide real membership changes or leak refcounts. + +## Output contract + +Report input versus store-update counts, barrier behavior, subscription call +counts across rerenders/membership changes, desktop/mobile shared-state proof, +test results, and files changed. + +## Verification results + +- `pnpm exec vitest run lib/ws/handlers/messages.test.ts 'app/office/tasks/[id]/use-session-live-sync.test.tsx'` — passed (7 tests). +- `pnpm run typecheck` — passed. +- `pnpm run lint` — passed for the web package. +- `session.message.updated` batching keeps one latest payload per key per + animation frame; add/delete and turn-complete events flush as barriers. +- Office live sync sorts and deduplicates IDs, reconciles only membership + deltas, and clears subscriptions on disconnect/unmount. Desktop and mobile + continue to consume the same WebSocket/store path; no layout or copy changed. + +## Files changed + +- `apps/web/lib/ws/handlers/messages.ts` +- `apps/web/lib/ws/handlers/messages.test.ts` +- `apps/web/lib/ws/handlers/turns.ts` +- `apps/web/lib/ws/router.ts` +- `apps/web/lib/ws/use-websocket.tsx` +- `apps/web/app/office/tasks/[id]/page.tsx` +- `apps/web/app/office/tasks/[id]/use-session-live-sync.ts` +- `apps/web/app/office/tasks/[id]/use-session-live-sync.test.tsx` diff --git a/docs/plans/session-stream-overload-isolation/task-04-prove-overload-containment.md b/docs/plans/session-stream-overload-isolation/task-04-prove-overload-containment.md new file mode 100644 index 0000000000..0c0fb28168 --- /dev/null +++ b/docs/plans/session-stream-overload-isolation/task-04-prove-overload-containment.md @@ -0,0 +1,139 @@ +--- +id: "04-prove-overload-containment" +title: "Prove overload containment" +status: done +wave: 4 +depends_on: + - "01-coalesce-agent-stream-ingress" + - "02-isolate-replaceable-session-delivery" + - "03-bound-frontend-stream-work" +plan: "plan.md" +spec: "../../specs/platform/bounded-task-status-delivery.md" +--- + +# Task 04: Prove Overload Containment + +## Acceptance + +- A deterministic mock ACP command emits a reported configurable reasoning + burst suitable for pressure tests without a huge prompt or real-time delay. +- While one session emits the burst, desktop and mobile can switch to a quiet + task/session and complete a correlated message action without waiting for the + noisy backlog. +- The noisy session's final persisted reasoning is byte-for-byte exact; + lifecycle publications, gateway pending work, and browser store work remain + bounded and materially below source chunk count. +- Playwright attaches source, publication, replacement/eviction, frame/byte, + latency, exact-content, and subscription-churn evidence. +- Broad backend/frontend checks pass and plan/task statuses record exact + command results. + +## TDD sequence + +1. RED: add mock-agent parser/emitter tests for a configurable reasoning burst + and an explicit produced-count marker. +2. RED: add the desktop pressure test and require exact transcript content plus + quiet-session navigation/action progress during the active burst. +3. RED: run the same interaction through the native mobile task-switcher sheet + and assert no horizontal overflow. +4. GREEN: fix only integration wiring or deterministic diagnostics; do not + weaken burst-produced, exact-content, boundedness, or cross-session-progress + assertions. +5. REFACTOR: reuse WebSocket traffic capture helpers, attach compact evidence, + run broad verification, and update plan/task statuses/results. + +## Files likely touched + +- `apps/backend/cmd/mock-agent/script.go` or a focused scenario file +- mock-agent parser/emitter tests +- `apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go` +- `apps/backend/internal/agentctl/server/process/manager.go` +- new `apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts` +- new `apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts` +- `apps/web/e2e/helpers/session-stream-overload.ts` +- `apps/web/e2e/helpers/ws-traffic.ts` +- focused API/session page helpers +- this plan and task files for completion evidence + +## Verification + +```bash +cd apps/backend && go test ./cmd/mock-agent/... +cd apps/web && pnpm e2e:run tests/session/session-stream-overload-isolation.spec.ts \ + -- --project=chromium +cd apps/web && pnpm e2e:run tests/session/session-stream-overload-isolation.spec.ts \ + -- --project=mobile-chrome +make -C apps/backend test +make -C apps/backend lint +cd apps/web && pnpm exec vitest run +cd apps/web && pnpm run typecheck +cd apps/web && pnpm run lint +``` + +## Dependencies + +Tasks 01–03 must be complete so E2E observes the final containment stack. + +## Parallelism + +Sequential final integration/QA task. + +## Inputs + +- Incident task/session and event-count baseline in the parent plan. +- Existing `session-stream-budget.spec.ts`, `ws-traffic.ts`, mock-agent script + commands, and mobile task-switcher E2E patterns. + +## Risks + +- A burst that finishes before interaction begins does not prove concurrency; + synchronize on a produced/start marker and hold completion until the quiet + action begins. +- Raw byte totals vary by fixture data and are diagnostic, not the correctness + oracle. +- E2E must verify server-side production count so coalescing cannot make a + quiet producer look successful. + +## Output contract + +Report the diagnostics exposed by the current proof harness: source chunk +count, exact persisted byte count, gateway frame/byte summaries, quiet-session +response, desktop/mobile results, verification commands, and files changed. +Lifecycle publication, DB/message update, replacement/eviction, maximum-depth, +browser-store update, latency, and content-hash counters are not exposed by +this E2E harness and are out of scope for this task. + +## Verification results + +- `go test ./cmd/mock-agent/...` — passed (184 tests); parser/default/cap, + exact content, and script execution are covered. +- `go test -race ./internal/agentctl/server/adapter/transport/acp ./internal/agentctl/server/process ./internal/agent/runtime/lifecycle ./internal/gateway/websocket ./cmd/mock-agent` — passed (2,633 tests). +- `make -C apps/backend test` — passed on the final run. +- `rtk proxy golangci-lint run ./...` — passed (0 issues). +- Desktop Chromium E2E — passed (1 test, 13.4s): exact 2,000-chunk persisted + reasoning, quiet follow-up response, no horizontal overflow, and fewer + received noisy replacements than source chunks. +- Mobile Chrome E2E — passed (1 test, 8.3s): native session-sheet switch, + quiet follow-up response, exact reasoning, bounded noisy updates, and no + horizontal overflow. +- A full web Vitest/typecheck/lint verification was completed for the changed + package; focused Vitest covered 7 new tests. + +The ACP/process handoff now applies cancellation-aware per-agent backpressure +instead of dropping normalized stream events when its bounded queue fills. The +E2E evidence attaches source chunk count, exact persisted byte count, gateway +frame/byte summaries, and quiet-session progress without transcript content. + +## Files changed + +- `apps/backend/cmd/mock-agent/reasoning_burst.go` +- `apps/backend/cmd/mock-agent/reasoning_burst_test.go` +- `apps/backend/cmd/mock-agent/script.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/adapter.go` +- `apps/backend/internal/agentctl/server/adapter/transport/acp/conversion_test.go` +- `apps/backend/internal/agentctl/server/process/manager.go` +- `apps/web/e2e/helpers/api-client.ts` +- `apps/web/e2e/helpers/session-stream-overload.ts` +- `apps/web/e2e/tests/session/session-stream-overload-isolation.spec.ts` +- `apps/web/e2e/tests/session/mobile-session-stream-overload-isolation.spec.ts` diff --git a/docs/specs/platform/bounded-task-status-delivery.md b/docs/specs/platform/bounded-task-status-delivery.md index 5689463198..da59ef2f39 100644 --- a/docs/specs/platform/bounded-task-status-delivery.md +++ b/docs/specs/platform/bounded-task-status-delivery.md @@ -1,6 +1,7 @@ --- -status: implemented +status: approved created: 2026-08-01 +updated: 2026-08-02 owner: kandev --- @@ -34,6 +35,9 @@ detail surface. refresh has a targeted action that does not alter focus. - Correlated WebSocket responses and errors are prioritized over unsolicited notifications and cannot be silently dropped by notification pressure. +- One session's streaming text or reasoning cannot monopolize persistence, + notification delivery, or browser state work. Intermediate replacement + updates are bounded while the final transcript remains lossless. - `message.add` uses a stable client-generated message ID so an uncertain send can be reconciled or retried without creating or dispatching a duplicate. - Existing desktop and mobile status/icon precedence remains unchanged. @@ -122,6 +126,57 @@ remain during migration, but switchers use the summary when present. notification hydration. Duplicate notifications and responses upsert the same frontend message. +## Session stream overload isolation + +Agent stream chunks are an ingress format, not a required persistence or +browser-render cadence. The lifecycle boundary coalesces adjacent assistant or +reasoning chunks for the same Kandev message record before publishing them to +the orchestrator: + +- the first non-empty chunk creates the streaming record without waiting for a + coalescing interval; +- subsequent adjacent chunks for that record publish at most once per 100 ms + window while streaming continues; +- concatenation preserves byte order and content exactly; coalescing may not + truncate, summarize, or rewrite agent output; and +- pending content flushes before a tool call/result, permission request, + completion, error, stream disconnect, execution replacement, or backend + shutdown so persisted transcript order remains authoritative. + +The per-agent ACP adapter and process handoff apply cancellation-aware +backpressure when their bounded handoff queues fill. They must not silently +drop normalized assistant/reasoning chunks before the lifecycle coalescer; a +noisy session may slow its own agent process, and shutdown cancellation may +discard only work after that session is terminal. This handoff is scoped to the +agent instance and must not block unrelated backend sessions. + +The WebSocket gateway treats `session.message.updated` as a replaceable full- +state notification keyed by client, session, and message. At most one unsent +replacement for a key occupies delivery capacity. A newer update replaces the +older payload in place. Replaceable stream notifications use bounded +per-session capacity and fair scheduling, separate from correlated control +traffic and semantic notifications such as message creation/deletion, task or +session state changes, pending-input occurrences, and turn completion. When a +session exceeds its replaceable allowance, that session's oldest queued +replaceable entry may be evicted; it cannot consume another session's +allowance or the semantic-notification lane. Reconnect, snapshot, and +turn-settle reconciliation repair any intermediate replacement that was not +delivered because the database remains authoritative. + +The frontend applies the same defense at the render boundary. It keeps only the +newest `session.message.updated` payload for each message during one animation +frame and performs one store update per changed message when the frame flushes. +Message add/delete and turn-settle events remain ordered barriers. Intentional +multi-session detail surfaces may subscribe to every session they display, but +rerendering or refetching equivalent session objects must not tear down and +recreate unchanged subscription membership. + +Overload handling is observable. Structured metrics/logs identify the client, +session, action, queue class, coalesced chunk count, replacement count, and +eviction count without logging transcript content. Reconnect, a selected- +session snapshot, or turn-settle reconciliation repairs any skipped +intermediate replacement. + ## Failure modes - If projection fails, the last valid summary remains visible, the failure is @@ -138,6 +193,10 @@ remain during migration, but switchers use the summary when present. reconciliation cannot determine acceptance. - Notification overload may coalesce or drop replaceable notifications, but it cannot silently discard a correlated response/error. +- If one session exceeds its replaceable stream allowance, other sessions and + semantic notifications continue to make progress. The noisy session retains + exact persisted content and converges through its newest replacement or + normal reconciliation. - If `status_summary` is temporarily absent during rollout, task rows use the existing coarse task fields and omit unavailable decorations; they do not subscribe to inactive sessions. @@ -166,6 +225,18 @@ remain during migration, but switchers use the summary when present. - **GIVEN** many background sessions are active, **WHEN** the selected agent publishes model configuration, **THEN** the selected chat retains its model selector and can submit a follow-up message. +- **GIVEN** one ACP session emits 30,000 tiny reasoning chunks for one message, + **WHEN** the stream crosses the lifecycle, persistence, gateway, and browser + boundaries, **THEN** the final reasoning content is byte-for-byte complete, + append publications occur no more than once per 100 ms window, and pending + replacement work remains bounded. +- **GIVEN** a noisy session is continuously streaming, **WHEN** the user opens + another task or submits a message in another selected session, **THEN** that + navigation and its correlated response complete without waiting for the + noisy session's backlog. +- **GIVEN** an Office task detail refetch returns new objects for the same set + of session IDs, **WHEN** React rerenders the page, **THEN** the client sends no + unsubscribe, subscribe, or initial-snapshot replay for unchanged membership. - **GIVEN** a phone viewport, **WHEN** task summaries change, **THEN** the existing task-switcher sheet shows the same badges and precedence as the desktop sidebar without new navigation, scroll, or touch behavior. @@ -182,7 +253,14 @@ remain during migration, but switchers use the summary when present. backend process crash; this contract prevents duplicate handler dispatch on client retry. - Treating a larger timeout or queue capacity as the fix. +- Limiting how much reasoning or assistant content an agent may ultimately + produce. This contract bounds intermediate work, not transcript length. +- Automatically stopping or penalizing an agent solely because it emits many + valid chunks. ## Implementation plan -See [`../../plans/bounded-task-status-delivery/plan.md`](../../plans/bounded-task-status-delivery/plan.md). +- Original delivery redesign: + [`../../plans/bounded-task-status-delivery/plan.md`](../../plans/bounded-task-status-delivery/plan.md) +- Stream overload repair: + [`../../plans/session-stream-overload-isolation/plan.md`](../../plans/session-stream-overload-isolation/plan.md)