From 0d9004f546d38e9fee68ec4a160c12ebcefdabac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Salavisa?= Date: Fri, 21 Aug 2026 22:36:22 +0100 Subject: [PATCH] refactor(office): dedupe truncateUTF8 into shared package (#2866) Co-authored-by: Kandev Agent Co-authored-by: Claude Opus 5 (1M context) --- .../sqlite/continuation_summaries.go | 21 +---- .../internal/office/summary/builder.go | 18 +---- .../internal/office/truncate/truncate.go | 23 ++++++ .../internal/office/truncate/truncate_test.go | 79 +++++++++++++++++++ 4 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 apps/backend/internal/office/truncate/truncate.go create mode 100644 apps/backend/internal/office/truncate/truncate_test.go diff --git a/apps/backend/internal/office/repository/sqlite/continuation_summaries.go b/apps/backend/internal/office/repository/sqlite/continuation_summaries.go index 5f1b152a1c..a149a31f1f 100644 --- a/apps/backend/internal/office/repository/sqlite/continuation_summaries.go +++ b/apps/backend/internal/office/repository/sqlite/continuation_summaries.go @@ -5,6 +5,8 @@ import ( "database/sql" "errors" "time" + + "github.com/kandev/kandev/internal/office/truncate" ) // continuationSummaryMaxBytes caps the markdown body stored in @@ -61,7 +63,7 @@ func (r *Repository) GetContinuationSummary( func (r *Repository) UpsertContinuationSummary( ctx context.Context, row AgentContinuationSummary, ) error { - row.Content = truncateUTF8(row.Content, continuationSummaryMaxBytes) + row.Content = truncate.UTF8(row.Content, continuationSummaryMaxBytes) if row.UpdatedAt.IsZero() { row.UpdatedAt = time.Now().UTC() } @@ -79,20 +81,3 @@ func (r *Repository) UpsertContinuationSummary( row.UpdatedAt, row.UpdatedByRunID) return err } - -// truncateUTF8 returns s truncated to at most maxBytes, cutting at a -// rune boundary. When the cut would split a multi-byte rune the -// straddling bytes are dropped. -func truncateUTF8(s string, maxBytes int) string { - if len(s) <= maxBytes { - return s - } - // Walk back from maxBytes to the start of the last fully-included - // rune. UTF-8 continuation bytes match 10xxxxxx (0x80..0xBF); the - // loop terminates as soon as we land on a non-continuation byte. - cut := maxBytes - for cut > 0 && (s[cut]&0xC0) == 0x80 { - cut-- - } - return s[:cut] -} diff --git a/apps/backend/internal/office/summary/builder.go b/apps/backend/internal/office/summary/builder.go index bf6e1a74c0..06d68536ee 100644 --- a/apps/backend/internal/office/summary/builder.go +++ b/apps/backend/internal/office/summary/builder.go @@ -19,6 +19,8 @@ import ( "sort" "strings" "time" + + "github.com/kandev/kandev/internal/office/truncate" ) // maxSummaryBytes mirrors the storage cap in the continuation @@ -100,7 +102,7 @@ func BuildSummary(in BuildInputs) string { out := strings.TrimRight(b.String(), "\n") + "\n" if len(out) > maxSummaryBytes { - return truncateUTF8(out, maxSummaryBytes) + return truncate.UTF8(out, maxSummaryBytes) } return out } @@ -238,17 +240,3 @@ func firstNonEmptyLine(s, skipPrefix string) string { } return "" } - -// truncateUTF8 cuts s to at most maxBytes at a UTF-8 rune boundary. -// Mirrors the helper in the repo so the summary package is self- -// contained and doesn't import sqlite. -func truncateUTF8(s string, maxBytes int) string { - if len(s) <= maxBytes { - return s - } - cut := maxBytes - for cut > 0 && (s[cut]&0xC0) == 0x80 { - cut-- - } - return s[:cut] -} diff --git a/apps/backend/internal/office/truncate/truncate.go b/apps/backend/internal/office/truncate/truncate.go new file mode 100644 index 0000000000..a2035ec154 --- /dev/null +++ b/apps/backend/internal/office/truncate/truncate.go @@ -0,0 +1,23 @@ +// Package truncate provides rune-safe byte truncation for office content. +// +// It exists so the cut lives in exactly one place. The implementations it +// replaces were byte-for-byte identical copies in separate packages, which +// is how a future correctness fix lands in one and silently misses another. +package truncate + +// UTF8 returns s truncated to at most maxBytes, cutting at a rune +// boundary. When the cut would split a multi-byte rune the straddling +// bytes are dropped, so the result is always valid UTF-8. +func UTF8(s string, maxBytes int) string { + if len(s) <= maxBytes { + return s + } + // Walk back from maxBytes to the start of the last fully-included + // rune. UTF-8 continuation bytes match 10xxxxxx (0x80..0xBF); the + // loop terminates as soon as we land on a non-continuation byte. + cut := maxBytes + for cut > 0 && (s[cut]&0xC0) == 0x80 { + cut-- + } + return s[:cut] +} diff --git a/apps/backend/internal/office/truncate/truncate_test.go b/apps/backend/internal/office/truncate/truncate_test.go new file mode 100644 index 0000000000..4650fd57e8 --- /dev/null +++ b/apps/backend/internal/office/truncate/truncate_test.go @@ -0,0 +1,79 @@ +package truncate_test + +import ( + "testing" + "unicode/utf8" + + "github.com/kandev/kandev/internal/office/truncate" +) + +func TestUTF8(t *testing.T) { + tests := []struct { + name string + in string + maxBytes int + want string + }{ + {name: "shorter than cap is unchanged", in: "hello", maxBytes: 8, want: "hello"}, + {name: "exactly at cap is unchanged", in: "hello", maxBytes: 5, want: "hello"}, + {name: "empty string", in: "", maxBytes: 4, want: ""}, + {name: "ascii cuts at the byte cap", in: "abcdef", maxBytes: 3, want: "abc"}, + {name: "zero cap yields empty", in: "abc", maxBytes: 0, want: ""}, + + // Multi-byte runes straddling the cut are dropped whole rather + // than sliced into invalid bytes. "é" is C3 A9 (2 bytes). + {name: "2-byte rune split on its lead byte", in: "aé", maxBytes: 1, want: "a"}, + {name: "2-byte rune split on its continuation byte", in: "aé", maxBytes: 2, want: "a"}, + {name: "2-byte rune fits exactly", in: "aé", maxBytes: 3, want: "aé"}, + + // "€" is E2 82 AC (3 bytes). + {name: "3-byte rune split on its lead byte", in: "a€", maxBytes: 1, want: "a"}, + {name: "3-byte rune split after one continuation byte", in: "a€", maxBytes: 2, want: "a"}, + {name: "3-byte rune split after two continuation bytes", in: "a€", maxBytes: 3, want: "a"}, + {name: "3-byte rune fits exactly", in: "a€", maxBytes: 4, want: "a€"}, + + // "😀" is F0 9F 98 80 (4 bytes). + {name: "4-byte rune split on its lead byte", in: "a😀", maxBytes: 1, want: "a"}, + {name: "4-byte rune split after one continuation byte", in: "a😀", maxBytes: 2, want: "a"}, + {name: "4-byte rune split after two continuation bytes", in: "a😀", maxBytes: 3, want: "a"}, + {name: "4-byte rune split after three continuation bytes", in: "a😀", maxBytes: 4, want: "a"}, + {name: "4-byte rune fits exactly", in: "a😀", maxBytes: 5, want: "a😀"}, + + // Nothing survives when the only rune straddles the cut. + {name: "sole multi-byte rune is dropped entirely", in: "😀", maxBytes: 3, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := truncate.UTF8(tt.in, tt.maxBytes) + if got != tt.want { + t.Errorf("UTF8(%q, %d) = %q, want %q", tt.in, tt.maxBytes, got, tt.want) + } + if len(got) > tt.maxBytes { + t.Errorf("UTF8(%q, %d) returned %d bytes, over the cap", tt.in, tt.maxBytes, len(got)) + } + if !utf8.ValidString(got) { + t.Errorf("UTF8(%q, %d) returned invalid UTF-8: %q", tt.in, tt.maxBytes, got) + } + }) + } +} + +// TestUTF8KeepsResultValidAcrossEveryCut walks every cut position of a +// mixed-width string: no cap may ever produce invalid UTF-8 or exceed +// its own budget. +func TestUTF8KeepsResultValidAcrossEveryCut(t *testing.T) { + const s = "aé€😀bñ漢字" + for maxBytes := 0; maxBytes <= len(s)+2; maxBytes++ { + got := truncate.UTF8(s, maxBytes) + if len(got) > maxBytes { + t.Fatalf("maxBytes=%d: got %d bytes", maxBytes, len(got)) + } + if !utf8.ValidString(got) { + t.Fatalf("maxBytes=%d: invalid UTF-8 %q", maxBytes, got) + } + if got != s[:len(got)] { + t.Fatalf("maxBytes=%d: %q is not a prefix of the input", maxBytes, got) + } + } +}