leviath_core/
text.rs

1//! Cutting a `&str` at a byte offset without splitting a character.
2//!
3//! Rust panics on `&s[..n]` when `n` lands inside a multi-byte character, and
4//! this workspace slices strings at fixed byte budgets in a lot of places:
5//! script-tool I/O caps, ACP frame chunking, region seed truncation, dashboard
6//! column fitting, log previews. Every one of those had grown its own
7//! `while !s.is_char_boundary(end) { end -= 1 }` loop with its own comment
8//! explaining why - and two sites (`lev test`'s response preview and `lev
9//! setup`'s key redactor) never grew one at all and panicked on any emoji.
10//!
11//! That failure has happened for real: a byte cut-off through a flag emoji
12//! inside a Rhai host function double-panicked and aborted the whole daemon.
13//! Keeping the walk-back in one tested place means a new
14//! truncation site cannot forget it. The workspace also denies
15//! `clippy::string_slice`, so reaching for a raw `&s[..n]` instead of these
16//! helpers is a compile error unless the boundary is proven at the call site.
17
18/// Largest byte index `<= max` that is a char boundary in `s`.
19///
20/// `max` past the end clamps to `s.len()`. The walk-back always terminates:
21/// byte 0 is a boundary in every string, including the empty one.
22pub fn floor_char_boundary(s: &str, max: usize) -> usize {
23    let mut end = max.min(s.len());
24    while !s.is_char_boundary(end) {
25        end -= 1;
26    }
27    end
28}
29
30/// `&s[..max]`, backed off to the nearest char boundary at or before `max`.
31///
32/// Returns all of `s` when `max` reaches the end. Callers append their own
33/// ellipsis or truncation marker - this only cuts.
34#[expect(
35    clippy::string_slice,
36    reason = "floor_char_boundary returns a char boundary by construction - this is the one raw \
37              slice the rest of the workspace defers to"
38)]
39pub fn truncate_at_boundary(s: &str, max: usize) -> &str {
40    &s[..floor_char_boundary(s, max)]
41}
42
43/// The workspace's one generic token estimate: bytes divided by four,
44/// rounded up.
45///
46/// Every context budget, eviction threshold, and truncation cap that has no
47/// exact tokenizer runs on this. It was open-coded across ~30 sites with
48/// three disagreeing formulas (`/4`, `/4 + 1`, `div_ceil(4)`), which meant
49/// the same text could count differently on the budgeting side and the
50/// truncation side of one decision. Rounding up (never 0 for non-empty text)
51/// is the safe direction for a budget: overestimating spends a token of
52/// headroom, underestimating overflows a window.
53///
54/// Provider-accuracy heuristics (e.g. the Anthropic-calibrated bytes/3.5 in
55/// `leviath-providers`) are deliberately separate: they estimate a specific
56/// tokenizer, this estimates "text-shaped budget units".
57pub fn estimate_tokens(s: &str) -> usize {
58    s.len().div_ceil(4)
59}
60
61#[cfg(test)]
62mod tests {
63    use super::*;
64
65    #[test]
66    fn estimate_tokens_rounds_up_and_never_zero_for_nonempty() {
67        assert_eq!(estimate_tokens(""), 0);
68        assert_eq!(estimate_tokens("abc"), 1);
69        assert_eq!(estimate_tokens("abcd"), 1);
70        assert_eq!(estimate_tokens("abcde"), 2);
71        // Bytes, not chars: one 3-byte character still costs one budget unit.
72        assert_eq!(estimate_tokens("\u{65e5}"), 1);
73    }
74
75    #[test]
76    fn floor_char_boundary_clamps_and_walks_back() {
77        // Past the end clamps to the length.
78        assert_eq!(floor_char_boundary("abc", 99), 3);
79        // Already a boundary - unchanged.
80        assert_eq!(floor_char_boundary("abc", 2), 2);
81        // Zero is always a boundary, so no walk-back happens.
82        assert_eq!(floor_char_boundary("日本語", 0), 0);
83        // Mid-character walks back to the start of that character. '🎉' is four
84        // bytes at 3..7, so 4, 5 and 6 all floor to 3.
85        let s = "abc🎉";
86        assert_eq!(floor_char_boundary(s, 4), 3);
87        assert_eq!(floor_char_boundary(s, 6), 3);
88        assert_eq!(floor_char_boundary(s, 7), 7);
89        // Walking back all the way to 0 when the first character straddles the cut.
90        assert_eq!(floor_char_boundary("🎉abc", 2), 0);
91        // Empty string: byte 0 is a boundary, so any max clamps to 0.
92        assert_eq!(floor_char_boundary("", 5), 0);
93    }
94
95    #[test]
96    fn truncate_at_boundary_never_splits_a_character() {
97        assert_eq!(truncate_at_boundary("abc", 99), "abc");
98        assert_eq!(truncate_at_boundary("abcdef", 3), "abc");
99        // The exact shape that panicked in issues #109/#115.
100        assert_eq!(truncate_at_boundary("abc🎉def", 5), "abc");
101        assert_eq!(truncate_at_boundary("🎉abc", 2), "");
102        assert_eq!(truncate_at_boundary("", 5), "");
103    }
104}
leviath_core/text.rs

leviath_core/
text.rs
