leviath_core/text.rs
1//! Cutting a `&str` at a byte offset without splitting a character.
2//!
3//! Rust panics on `&s[..n]` when `n` lands inside a multi-byte character, and
4//! this workspace slices strings at fixed byte budgets in a lot of places:
5//! script-tool I/O caps, ACP frame chunking, region seed truncation, dashboard
6//! column fitting, log previews. Every one of those had grown its own
7//! `while !s.is_char_boundary(end) { end -= 1 }` loop with its own comment
8//! explaining why - and two sites (`lev test`'s response preview and `lev
9//! setup`'s key redactor) never grew one at all and panicked on any emoji.
10//!
11//! That failure has happened for real: a byte cut-off through a flag emoji
12//! inside a Rhai host function double-panicked and aborted the whole daemon.
13//! Keeping the walk-back in one tested place means a new
14//! truncation site cannot forget it. The workspace also denies
15//! `clippy::string_slice`, so reaching for a raw `&s[..n]` instead of these
16//! helpers is a compile error unless the boundary is proven at the call site.
17
18/// Largest byte index `<= max` that is a char boundary in `s`.
19///
20/// `max` past the end clamps to `s.len()`. The walk-back always terminates:
21/// byte 0 is a boundary in every string, including the empty one.
22pub fn floor_char_boundary(s: &str, max: usize) -> usize {
23 let mut end = max.min(s.len());
24 while !s.is_char_boundary(end) {
25 end -= 1;
26 }
27 end
28}
29
30/// `&s[..max]`, backed off to the nearest char boundary at or before `max`.
31///
32/// Returns all of `s` when `max` reaches the end. Callers append their own
33/// ellipsis or truncation marker - this only cuts.
34#[expect(
35 clippy::string_slice,
36 reason = "floor_char_boundary returns a char boundary by construction - this is the one raw \
37 slice the rest of the workspace defers to"
38)]
39pub fn truncate_at_boundary(s: &str, max: usize) -> &str {
40 &s[..floor_char_boundary(s, max)]
41}
42
43/// The workspace's one generic token estimate: bytes divided by four,
44/// rounded up.
45///
46/// Every context budget, eviction threshold, and truncation cap that has no
47/// exact tokenizer runs on this. It was open-coded across ~30 sites with
48/// three disagreeing formulas (`/4`, `/4 + 1`, `div_ceil(4)`), which meant
49/// the same text could count differently on the budgeting side and the
50/// truncation side of one decision. Rounding up (never 0 for non-empty text)
51/// is the safe direction for a budget: overestimating spends a token of
52/// headroom, underestimating overflows a window.
53///
54/// Provider-accuracy heuristics (e.g. the Anthropic-calibrated bytes/3.5 in
55/// `leviath-providers`) are deliberately separate: they estimate a specific
56/// tokenizer, this estimates "text-shaped budget units".
57pub fn estimate_tokens(s: &str) -> usize {
58 s.len().div_ceil(4)
59}
60
61/// Substitute `{name}` placeholders in a template.
62///
63/// Plain sequential `str::replace`, the same scheme `CompactionConfig`'s
64/// `user_prompt` uses for `{content}` / `{region_name}` - not a template
65/// language. Placeholders absent from `vars` pass through untouched, so a
66/// nudge or required-region message can contain literal braces without
67/// escaping as long as they don't collide with a supported name.
68pub fn interpolate(template: &str, vars: &[(&str, &str)]) -> String {
69 let mut out = template.to_string();
70 for (name, value) in vars {
71 out = out.replace(&format!("{{{name}}}"), value);
72 }
73 out
74}
75
76#[cfg(test)]
77mod tests {
78 use super::*;
79
80 #[test]
81 fn estimate_tokens_rounds_up_and_never_zero_for_nonempty() {
82 assert_eq!(estimate_tokens(""), 0);
83 assert_eq!(estimate_tokens("abc"), 1);
84 assert_eq!(estimate_tokens("abcd"), 1);
85 assert_eq!(estimate_tokens("abcde"), 2);
86 // Bytes, not chars: one 3-byte character still costs one budget unit.
87 assert_eq!(estimate_tokens("\u{65e5}"), 1);
88 }
89
90 #[test]
91 fn interpolate_replaces_known_placeholders_and_keeps_the_rest() {
92 // Present, repeated, and absent placeholders in one template.
93 assert_eq!(
94 interpolate(
95 "populate {region} - yes, {region} - in stage {stage} {unknown}",
96 &[("region", "plan"), ("stage", "design")]
97 ),
98 "populate plan - yes, plan - in stage design {unknown}"
99 );
100 // No vars: the template passes through unchanged.
101 assert_eq!(interpolate("no placeholders", &[]), "no placeholders");
102 }
103
104 #[test]
105 fn floor_char_boundary_clamps_and_walks_back() {
106 // Past the end clamps to the length.
107 assert_eq!(floor_char_boundary("abc", 99), 3);
108 // Already a boundary - unchanged.
109 assert_eq!(floor_char_boundary("abc", 2), 2);
110 // Zero is always a boundary, so no walk-back happens.
111 assert_eq!(floor_char_boundary("日本語", 0), 0);
112 // Mid-character walks back to the start of that character. '🎉' is four
113 // bytes at 3..7, so 4, 5 and 6 all floor to 3.
114 let s = "abc🎉";
115 assert_eq!(floor_char_boundary(s, 4), 3);
116 assert_eq!(floor_char_boundary(s, 6), 3);
117 assert_eq!(floor_char_boundary(s, 7), 7);
118 // Walking back all the way to 0 when the first character straddles the cut.
119 assert_eq!(floor_char_boundary("🎉abc", 2), 0);
120 // Empty string: byte 0 is a boundary, so any max clamps to 0.
121 assert_eq!(floor_char_boundary("", 5), 0);
122 }
123
124 #[test]
125 fn truncate_at_boundary_never_splits_a_character() {
126 assert_eq!(truncate_at_boundary("abc", 99), "abc");
127 assert_eq!(truncate_at_boundary("abcdef", 3), "abc");
128 // The exact shape that panicked in issues #109/#115.
129 assert_eq!(truncate_at_boundary("abc🎉def", 5), "abc");
130 assert_eq!(truncate_at_boundary("🎉abc", 2), "");
131 assert_eq!(truncate_at_boundary("", 5), "");
132 }
133}