agent_core/
text.rs

1//! Shared UTF-8-safe, byte-budgeted text truncation (T2).
2//!
3//! [`BoundedText`] is the one workspace-wide utility for producing
4//! byte-bounded, valid-UTF-8 previews of arbitrary (user- or model-derived)
5//! strings. Production code must never byte-index-slice such strings
6//! directly; route through this type or [`crate::truncate_str`] (the
7//! borrowing primitive this type is built on).
8
9/// A byte-bounded, char-boundary-safe view of a string, with exact
10/// accounting of what was kept and what was cut.
11#[derive(Debug, Clone, PartialEq, Eq)]
12pub struct BoundedText {
13    /// The retained prefix — always valid UTF-8, at most the byte budget.
14    pub text: String,
15    /// Byte length of the original input.
16    pub original_bytes: usize,
17    /// Byte length of `text` (≤ budget, ≤ `original_bytes`).
18    pub retained_bytes: usize,
19    /// Whether anything was cut (`retained_bytes < original_bytes`).
20    pub truncated: bool,
21}
22
23impl BoundedText {
24    /// Truncate `s` to at most `max_bytes` bytes at a valid UTF-8 boundary.
25    /// Never panics; greedy (keeps the longest prefix that fits).
26    pub fn new(s: &str, max_bytes: usize) -> Self {
27        let text = crate::truncate_str(s, max_bytes);
28        Self {
29            original_bytes: s.len(),
30            retained_bytes: text.len(),
31            truncated: text.len() < s.len(),
32            text: text.to_string(),
33        }
34    }
35}
36
37#[cfg(test)]
38mod tests {
39    use super::*;
40
41    #[test]
42    fn bounded_text_reports_exact_accounting() {
43        let bt = BoundedText::new("hello", 10);
44        assert_eq!(bt.text, "hello");
45        assert_eq!(bt.original_bytes, 5);
46        assert_eq!(bt.retained_bytes, 5);
47        assert!(!bt.truncated);
48
49        let bt = BoundedText::new("hello", 3);
50        assert_eq!(bt.text, "hel");
51        assert_eq!(bt.original_bytes, 5);
52        assert_eq!(bt.retained_bytes, 3);
53        assert!(bt.truncated);
54    }
55
56    /// Property-style sweep: for a deterministic corpus of adversarial
57    /// Unicode inputs (multibyte Latin, emoji + ZWJ, CJK, combining marks,
58    /// LCG-generated mixes) and EVERY byte budget from 0 to len+2, the
59    /// constructor never panics, never exceeds the budget, always yields a
60    /// valid-UTF-8 prefix, accounts bytes exactly, and is greedy.
61    #[test]
62    fn bounded_text_property_sweep_never_exceeds_budget() {
63        let pool: Vec<char> = "aé汉字テ🌟👨\u{200D}👩\u{301}\u{300}あ€𝄞\u{FFFD}"
64            .chars()
65            .collect();
66        let mut corpus: Vec<String> = vec![
67            String::new(),
68            "plain ascii".into(),
69            "héllo wörld".into(),
70            "🌟👨\u{200D}👩\u{200D}👧\u{200D}👦🌟".into(),
71            "汉字漢字テスト한글".into(),
72            "e\u{301}\u{300}o\u{308}".into(),
73            "a🌟b汉c\u{301}d".into(),
74        ];
75        // Deterministic LCG generator — no new dependencies.
76        let mut state: u64 = 0x2545_F491_4F6C_DD1D;
77        for len in [1usize, 3, 7, 17, 33] {
78            let mut s = String::new();
79            for _ in 0..len {
80                state = state
81                    .wrapping_mul(6364136223846793005)
82                    .wrapping_add(1442695040888963407);
83                s.push(pool[(state >> 33) as usize % pool.len()]);
84            }
85            corpus.push(s);
86        }
87        for s in &corpus {
88            for budget in 0..=s.len() + 2 {
89                let bt = BoundedText::new(s, budget);
90                assert!(
91                    bt.retained_bytes <= budget,
92                    "budget exceeded: {s:?} @ {budget}"
93                );
94                assert_eq!(bt.retained_bytes, bt.text.len());
95                assert_eq!(bt.original_bytes, s.len());
96                assert!(s.starts_with(&bt.text), "not a prefix: {s:?} @ {budget}");
97                assert_eq!(bt.truncated, bt.retained_bytes < s.len());
98                // Greedy: the next char (if any) must not have fit.
99                if bt.truncated {
100                    let next_len = s[bt.retained_bytes..]
101                        .chars()
102                        .next()
103                        .map(char::len_utf8)
104                        .unwrap_or(0);
105                    assert!(
106                        bt.retained_bytes + next_len > budget,
107                        "not greedy: {s:?} @ {budget}"
108                    );
109                }
110            }
111        }
112    }
113}
agent_core/text.rs

agent_core/
text.rs
