microclaw_core/
text.rs

1pub fn floor_char_boundary(s: &str, mut index: usize) -> usize {
2    let len = s.len();
3    if index >= len {
4        return len;
5    }
6
7    while index > 0 && !s.is_char_boundary(index) {
8        index -= 1;
9    }
10
11    index
12}
13
14pub fn split_text(text: &str, max_len: usize) -> Vec<String> {
15    if text.len() <= max_len {
16        return vec![text.to_string()];
17    }
18
19    let mut chunks = Vec::new();
20    let mut remaining = text;
21    while !remaining.is_empty() {
22        let chunk_len = if remaining.len() <= max_len {
23            remaining.len()
24        } else {
25            let boundary = floor_char_boundary(remaining, max_len.min(remaining.len()));
26            remaining[..boundary]
27                .rfind('\n')
28                .map(|index| index + '\n'.len_utf8())
29                .unwrap_or(boundary)
30        };
31        chunks.push(remaining[..chunk_len].to_string());
32        remaining = &remaining[chunk_len..];
33    }
34    chunks
35}
36
37/// Remove private reasoning and textual protocol artifacts before content is
38/// handed to a user-facing channel. Real tool calls are structured blocks; a
39/// literal `[tool_use: ...]` line is therefore always an implementation leak.
40/// Cap on the excerpt embedded by [`quoted_context_prefix`].
41pub const QUOTED_CONTEXT_MAX_CHARS: usize = 400;
42
43/// Build the `[quoted from …]` prefix that channel adapters prepend when a
44/// user replies to an earlier message, so terse follow-ups ("why?", "fix
45/// it") keep their referent even after the original scrolled out of the
46/// session window. Returns `None` when the referenced content is empty.
47pub fn quoted_context_prefix(author: Option<&str>, excerpt: &str) -> Option<String> {
48    let trimmed = excerpt.trim();
49    if trimmed.is_empty() {
50        return None;
51    }
52    let mut capped: String = trimmed.chars().take(QUOTED_CONTEXT_MAX_CHARS).collect();
53    if capped.chars().count() < trimmed.chars().count() {
54        capped.push('…');
55    }
56    let author = author.map(str::trim).filter(|a| !a.is_empty());
57    Some(match author {
58        Some(a) => format!("[quoted from {a}: {capped}]\n"),
59        None => format!("[quoted: {capped}]\n"),
60    })
61}
62
63pub fn sanitize_user_visible_text(text: &str) -> String {
64    fn strip_tag_blocks(input: &str, open: &str, close: &str) -> String {
65        let mut result = String::with_capacity(input.len());
66        let mut rest = input;
67        while let Some(start) = rest.find(open) {
68            result.push_str(&rest[..start]);
69            if let Some(end) = rest[start..].find(close) {
70                rest = &rest[start + end + close.len()..];
71            } else {
72                rest = "";
73                break;
74            }
75        }
76        result.push_str(rest);
77        result
78    }
79
80    let mut visible = text.to_string();
81    for (open, close) in [
82        ("<think>", "</think>"),
83        ("<thought>", "</thought>"),
84        ("<thinking>", "</thinking>"),
85        ("<reasoning>", "</reasoning>"),
86    ] {
87        visible = strip_tag_blocks(&visible, open, close);
88    }
89
90    visible
91        .lines()
92        .filter(|line| {
93            let trimmed = line.trim();
94            !(trimmed.starts_with("[tool_use:") && trimmed.ends_with(']'))
95        })
96        .collect::<Vec<_>>()
97        .join("\n")
98        .trim()
99        .to_string()
100}
101
102#[cfg(test)]
103mod tests {
104    use super::{
105        quoted_context_prefix, sanitize_user_visible_text, split_text, QUOTED_CONTEXT_MAX_CHARS,
106    };
107
108    #[test]
109    fn quoted_context_includes_author_and_caps_length() {
110        let p = quoted_context_prefix(Some("alice"), "hello world").unwrap();
111        assert_eq!(p, "[quoted from alice: hello world]\n");
112        let p = quoted_context_prefix(None, "hi").unwrap();
113        assert_eq!(p, "[quoted: hi]\n");
114        assert!(quoted_context_prefix(Some("bob"), "   ").is_none());
115        let long = "x".repeat(QUOTED_CONTEXT_MAX_CHARS + 50);
116        let p = quoted_context_prefix(None, &long).unwrap();
117        assert!(p.contains('…'));
118        assert!(p.chars().count() < long.chars().count());
119    }
120
121    #[test]
122    fn sanitizes_private_reasoning_and_fake_tool_calls() {
123        let text = "<think>secret</think>\nVisible\n[tool_use: bash({\"command\":\"pwd\"})]";
124        assert_eq!(sanitize_user_visible_text(text), "Visible");
125    }
126
127    #[test]
128    fn split_text_respects_utf8_boundaries() {
129        let chunks = split_text("你好世界", 7);
130        assert_eq!(chunks, vec!["你好", "世界"]);
131    }
132
133    #[test]
134    fn split_text_preserves_every_byte_at_newline_boundaries() {
135        let text = "first paragraph\nsecond paragraph\nthird paragraph";
136        let chunks = split_text(text, 18);
137        assert!(chunks.iter().all(|chunk| chunk.len() <= 18));
138        assert_eq!(chunks.concat(), text);
139    }
140}
microclaw_core/text.rs

microclaw_core/
text.rs
