mentedb_core/
text.rs

1//! Small string helpers shared across the workspace.
2
3/// Truncate `s` to at most `max_bytes`, snapping the cut down to the nearest
4/// UTF-8 character boundary so a multi-byte character is never split.
5///
6/// Byte-index slicing like `&s[..s.len().min(n)]` panics when byte `n` lands in
7/// the middle of a multi-byte character (Cyrillic, CJK, emoji, accents). This
8/// returns a valid `&str` in every case, at the cost of up to three fewer bytes.
9pub fn truncate_on_char_boundary(s: &str, max_bytes: usize) -> &str {
10    if s.len() <= max_bytes {
11        return s;
12    }
13    let mut end = max_bytes;
14    while end > 0 && !s.is_char_boundary(end) {
15        end -= 1;
16    }
17    &s[..end]
18}
19
20#[cfg(test)]
21mod tests {
22    use super::*;
23
24    #[test]
25    fn ascii_truncates_exactly() {
26        assert_eq!(truncate_on_char_boundary("hello world", 5), "hello");
27    }
28
29    #[test]
30    fn short_strings_pass_through() {
31        assert_eq!(truncate_on_char_boundary("hi", 100), "hi");
32        assert_eq!(truncate_on_char_boundary("", 5), "");
33    }
34
35    #[test]
36    fn never_splits_a_multibyte_char() {
37        // Each Cyrillic char is 2 bytes; boundaries fall on even byte offsets.
38        // Naive `&s[..5]` would panic; we snap down to 4.
39        let s = "Привет";
40        let out = truncate_on_char_boundary(s, 5);
41        assert_eq!(out, "Пр");
42        assert!(s.starts_with(out));
43    }
44
45    #[test]
46    fn handles_emoji_boundaries() {
47        // 'a'(1) + emoji(4) + 'b'(1) = 6 bytes; boundaries at 0,1,5,6.
48        let s = "a😀b";
49        assert_eq!(truncate_on_char_boundary(s, 1), "a");
50        assert_eq!(truncate_on_char_boundary(s, 3), "a"); // mid-emoji -> snap to 1
51        assert_eq!(truncate_on_char_boundary(s, 5), "a😀");
52        assert_eq!(truncate_on_char_boundary(s, 6), "a😀b");
53    }
54
55    #[test]
56    fn cjk_at_a_splitting_offset_does_not_panic() {
57        // Each CJK char is 3 bytes; a 500-byte cut lands mid-char two times in
58        // three. This is the exact prod case behind the enrichment fix.
59        let s = "记忆".repeat(400); // 2400 bytes, boundaries every 3
60        let out = truncate_on_char_boundary(&s, 500);
61        assert!(out.len() <= 500);
62        assert!(s.starts_with(out));
63        assert!(s.is_char_boundary(out.len()));
64    }
65}
mentedb_core/text.rs

mentedb_core/
text.rs
