use once_cell::sync::Lazy;
use regex::Regex;
static WHITESPACE_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"\s+").unwrap());
static DECORATIVE_RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"[▶→←▼▲•·◆◇◊✓✗✔✘‣⁃◦]").unwrap());
pub fn compress_text(text: &str) -> String {
let clean = DECORATIVE_RE.replace_all(text, "");
let collapsed = WHITESPACE_RE.replace_all(&clean, " ");
collapsed.trim().to_string()
}
pub fn compress_block(text: &str) -> String {
let mut lines: Vec<String> = Vec::new();
let mut prev_blank = false;
for raw in text.lines() {
let line = compress_text(raw);
let blank = line.is_empty();
if blank && prev_blank {
continue;
}
lines.push(line);
prev_blank = blank;
}
lines.join("\n").trim().to_string()
}
fn is_url_punct(b: u8) -> bool {
matches!(
b,
b'/' | b':' | b'.' | b'?' | b'#' | b'&' | b'=' | b'%' | b'~'
)
}
fn cost_quarters(text: &str) -> usize {
text.len() + 2 * text.bytes().filter(|b| is_url_punct(*b)).count()
}
pub fn estimate_tokens(text: &str) -> usize {
cost_quarters(text) / 4
}
pub const TRUNCATION_MARKER: &str = "\n…[truncated]";
pub fn truncate_to_tokens(text: &str, max_tokens: usize) -> String {
truncate_to_tokens_at(text, max_tokens).0
}
pub fn truncate_to_tokens_at(text: &str, max_tokens: usize) -> (String, usize) {
if estimate_tokens(text) <= max_tokens {
return (text.to_string(), text.len());
}
let budget = (max_tokens * 4).saturating_sub(cost_quarters(TRUNCATION_MARKER));
let mut spent = 0usize;
let mut end = 0usize;
for (i, b) in text.bytes().enumerate() {
let next = spent + if is_url_punct(b) { 3 } else { 1 };
if next > budget {
break;
}
spent = next;
end = i + 1;
}
while end > 0 && !text.is_char_boundary(end) {
end -= 1;
}
end = snap_to_boundary(text, end);
if end == 0 {
end = text
.char_indices()
.nth(1)
.map(|(i, _)| i)
.unwrap_or(text.len());
}
(format!("{}{}", &text[..end], TRUNCATION_MARKER), end)
}
const SNAP_BACK_FRACTION: usize = 5;
fn snap_to_boundary(text: &str, end: usize) -> usize {
if end == 0 || end >= text.len() {
return end;
}
let floor = end - end / SNAP_BACK_FRACTION;
let head = &text[..end];
for pattern in ["\n\n", "\n", " "] {
if let Some(at) = head.rfind(pattern) {
let cut = at + pattern.len();
if cut >= floor {
return cut;
}
}
}
end
}
pub fn window_from(text: &str, offset: usize) -> &str {
let mut start = offset.min(text.len());
while start > 0 && !text.is_char_boundary(start) {
start -= 1;
}
&text[start..]
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn estimate_unchanged_for_plain_prose() {
assert_eq!(estimate_tokens(&"a".repeat(100)), 25);
}
#[test]
fn url_heavy_text_estimates_higher_than_prose_of_same_length() {
let urls = "[1] https://example.com/a/b?c=d#e\n\
[2] https://example.org/x/y/z?q=1\n\
[3] https://sub.example.net/path/to/thing\n\
[4] https://example.io/foo/bar/baz?k=v";
let prose = "x".repeat(urls.len());
assert_eq!(urls.len(), prose.len());
assert!(
estimate_tokens(urls) > estimate_tokens(&prose),
"urls={} prose={}",
estimate_tokens(urls),
estimate_tokens(&prose)
);
}
#[test]
fn truncate_respects_the_budget_it_reports() {
let text = "a".repeat(1000);
let out = truncate_to_tokens(&text, 20);
assert!(out.ends_with(TRUNCATION_MARKER));
assert!(
estimate_tokens(&out) <= 20,
"estimate {}",
estimate_tokens(&out)
);
}
#[test]
fn truncate_respects_budget_on_url_heavy_text() {
let urls = "[1] https://example.com/a/b?c=d#e\n".repeat(200);
let out = truncate_to_tokens(&urls, 50);
assert!(
estimate_tokens(&out) <= 50,
"estimate {}",
estimate_tokens(&out)
);
}
#[test]
fn truncate_is_a_noop_within_budget() {
let text = "short enough";
assert_eq!(truncate_to_tokens(text, 1000), text);
}
#[test]
fn windows_tile_the_document_exactly() {
let text = "Sentence number one is here. ".repeat(400);
let mut offset = 0usize;
let mut seen = String::new();
let mut windows = 0;
while offset < text.len() {
let remainder = window_from(&text, offset);
let (_, consumed) = truncate_to_tokens_at(remainder, 60);
assert!(consumed > 0, "a window must make progress");
seen.push_str(&remainder[..consumed]);
offset += consumed;
windows += 1;
assert!(windows < 1000, "paging failed to terminate");
}
assert!(windows > 5, "expected several windows, got {windows}");
assert_eq!(seen, text);
}
#[test]
fn a_pathologically_small_budget_still_makes_progress() {
let text = "ünicode text that will not fit";
let (out, consumed) = truncate_to_tokens_at(text, 1);
assert!(consumed > 0);
assert!(text.is_char_boundary(consumed));
assert!(out.starts_with('ü'));
}
#[test]
fn a_window_ends_on_a_word_boundary_when_one_is_near() {
let text = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu ".repeat(20);
let (_, consumed) = truncate_to_tokens_at(&text, 20);
assert!(
text[..consumed].ends_with(' '),
"cut mid-word: {:?}",
&text[consumed.saturating_sub(12)..consumed]
);
assert!(window_from(&text, consumed).starts_with(char::is_alphabetic));
}
#[test]
fn text_without_a_boundary_still_yields_a_window() {
let text = "x".repeat(500);
let (_, consumed) = truncate_to_tokens_at(&text, 20);
assert!(consumed > 0);
}
#[test]
fn consumption_covers_the_whole_text_when_nothing_is_cut() {
let text = "well within budget";
let (out, consumed) = truncate_to_tokens_at(text, 1000);
assert_eq!(out, text);
assert_eq!(consumed, text.len());
}
#[test]
fn window_clamps_past_the_end_and_snaps_to_a_char_boundary() {
let text = "héllo wörld";
assert_eq!(window_from(text, text.len() + 500), "");
assert_eq!(window_from(text, 2), &text[1..]);
}
#[test]
fn truncate_never_splits_a_utf8_char() {
let text = "é".repeat(500);
let out = truncate_to_tokens(&text, 10);
assert!(out.starts_with('é'));
assert!(estimate_tokens(&out) <= 10);
}
}