use std::collections::HashMap;
use std::collections::HashSet;
use crate::thrift::need::Need;
use crate::thrift::tokens::{max_tokens, Ceiling};
pub const DEFAULT_BOILERPLATE_SHARE: f32 = 0.6;
pub const MIN_PAGES_FOR_BOILERPLATE: usize = 3;
pub const DEFAULT_DATA_URI_LIMIT: usize = 256;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum TrimReason {
Boilerplate {
lines: usize,
bytes: usize,
},
DataUri {
count: usize,
bytes: usize,
},
LinkLines {
lines: usize,
bytes: usize,
},
Whitespace {
bytes: usize,
},
Truncated {
tokens: usize,
},
}
impl TrimReason {
pub fn bytes(&self) -> usize {
match self {
TrimReason::Boilerplate { bytes, .. }
| TrimReason::DataUri { bytes, .. }
| TrimReason::LinkLines { bytes, .. }
| TrimReason::Whitespace { bytes } => *bytes,
TrimReason::Truncated { .. } => 0,
}
}
}
#[derive(Debug, Clone, Default, PartialEq)]
pub struct Trimmed {
pub text: String,
pub reasons: Vec<TrimReason>,
}
impl Trimmed {
pub fn removed_bytes(&self) -> usize {
self.reasons.iter().map(TrimReason::bytes).sum()
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
#[non_exhaustive]
pub struct TrimSettings {
pub data_uri_limit: usize,
pub drop_link_lines: bool,
pub collapse_blanks: bool,
pub boilerplate_share: f32,
pub min_pages_for_boilerplate: usize,
}
impl Default for TrimSettings {
fn default() -> TrimSettings {
TrimSettings {
data_uri_limit: DEFAULT_DATA_URI_LIMIT,
drop_link_lines: true,
collapse_blanks: true,
boilerplate_share: DEFAULT_BOILERPLATE_SHARE,
min_pages_for_boilerplate: MIN_PAGES_FOR_BOILERPLATE,
}
}
}
impl TrimSettings {
pub fn for_need(need: &Need) -> TrimSettings {
TrimSettings {
drop_link_lines: !matches!(need, Need::Links),
..TrimSettings::default()
}
}
pub fn untouched() -> TrimSettings {
TrimSettings {
data_uri_limit: usize::MAX,
drop_link_lines: false,
collapse_blanks: false,
boilerplate_share: 1.1,
min_pages_for_boilerplate: usize::MAX,
}
}
}
#[derive(Debug, Clone, Default)]
pub struct Trimmer {
settings: TrimSettings,
furniture: HashSet<u64>,
repeated_lines: HashSet<u64>,
}
impl Trimmer {
pub fn new(settings: TrimSettings) -> Trimmer {
Trimmer {
settings,
furniture: HashSet::new(),
repeated_lines: HashSet::new(),
}
}
pub fn for_need(need: &Need) -> Trimmer {
match need {
Need::Raw => Trimmer::new(TrimSettings::untouched()),
other => Trimmer::new(TrimSettings::for_need(other)),
}
}
pub fn settings(&self) -> &TrimSettings {
&self.settings
}
pub fn learn<S: AsRef<str>>(&mut self, pages: &[S]) {
self.furniture.clear();
self.repeated_lines.clear();
if pages.len() < self.settings.min_pages_for_boilerplate {
return;
}
let mut line_pages: HashMap<u64, usize> = HashMap::new();
let mut pair_pages: HashMap<u64, usize> = HashMap::new();
for page in pages {
let lines = meaningful_lines(page.as_ref());
let mut seen_lines: HashSet<u64> = HashSet::new();
let mut seen_pairs: HashSet<u64> = HashSet::new();
for hash in &lines {
if seen_lines.insert(*hash) {
*line_pages.entry(*hash).or_insert(0) += 1;
}
}
for window in lines.windows(2) {
let [first, second] = window else { continue };
let pair = mix(*first, *second);
if seen_pairs.insert(pair) {
*pair_pages.entry(pair).or_insert(0) += 1;
}
}
}
let floor = (pages.len() as f32 * self.settings.boilerplate_share).floor() as usize;
let keep_above = |count: usize| count > floor;
self.repeated_lines = line_pages
.into_iter()
.filter(|(_, count)| keep_above(*count))
.map(|(hash, _)| hash)
.collect();
self.furniture = pair_pages
.into_iter()
.filter(|(_, count)| keep_above(*count))
.map(|(hash, _)| hash)
.collect();
}
pub fn knows_boilerplate(&self) -> bool {
!self.furniture.is_empty()
}
pub fn trim(&self, text: &str, max_tokens: Option<usize>) -> Trimmed {
let mut reasons = Vec::new();
let mut out = text.to_string();
if self.knows_boilerplate() {
let (next, lines, bytes) = self.strip_furniture(&out);
if lines > 0 {
reasons.push(TrimReason::Boilerplate { lines, bytes });
out = next;
}
}
if self.settings.data_uri_limit < usize::MAX {
let (next, count, bytes) = drop_data_uris(&out, self.settings.data_uri_limit);
if count > 0 {
reasons.push(TrimReason::DataUri { count, bytes });
out = next;
}
}
if self.settings.drop_link_lines {
let (next, lines, bytes) = drop_link_only_lines(&out);
if lines > 0 {
reasons.push(TrimReason::LinkLines { lines, bytes });
out = next;
}
}
if self.settings.collapse_blanks {
let before = out.len();
let collapsed = collapse_whitespace(&out);
if collapsed.len() < before {
reasons.push(TrimReason::Whitespace {
bytes: before - collapsed.len(),
});
}
out = collapsed;
}
if let Some(ceiling) = max_tokens {
let (next, dropped) = truncate_to_tokens(&out, ceiling);
if dropped > 0 {
reasons.push(TrimReason::Truncated { tokens: dropped });
out = next;
}
}
Trimmed { text: out, reasons }
}
fn strip_furniture(&self, text: &str) -> (String, usize, usize) {
let carried: Vec<(usize, u64)> = text
.lines()
.enumerate()
.filter_map(|(index, line)| {
let normal = normalize(line);
(!normal.is_empty()).then(|| (index, hash(&normal)))
})
.collect();
let mut furniture_lines: HashSet<usize> = HashSet::new();
for (position, (index, this)) in carried.iter().enumerate() {
if !self.repeated_lines.contains(this) {
continue;
}
let after_a_repeat = position
.checked_sub(1)
.and_then(|earlier| carried.get(earlier))
.is_some_and(|(_, before)| self.furniture.contains(&mix(*before, *this)));
let before_a_repeat = carried
.get(position + 1)
.is_some_and(|(_, after)| self.furniture.contains(&mix(*this, *after)));
if after_a_repeat || before_a_repeat {
furniture_lines.insert(*index);
}
}
let mut kept = String::with_capacity(text.len());
let mut dropped_lines = 0usize;
let mut dropped_bytes = 0usize;
for (index, line) in text.lines().enumerate() {
if furniture_lines.contains(&index) {
dropped_lines += 1;
dropped_bytes += line.len() + 1;
continue;
}
kept.push_str(line);
kept.push('\n');
}
(kept, dropped_lines, dropped_bytes)
}
}
fn meaningful_lines(text: &str) -> Vec<u64> {
text.lines()
.map(normalize)
.filter(|line| !line.is_empty())
.map(|line| hash(&line))
.collect()
}
fn normalize(line: &str) -> String {
line.split_whitespace().collect::<Vec<_>>().join(" ")
}
fn hash(text: &str) -> u64 {
let mut value: u64 = 0xcbf2_9ce4_8422_2325;
for byte in text.as_bytes() {
value ^= u64::from(*byte);
value = value.wrapping_mul(0x1000_0000_01b3);
}
value
}
fn mix(first: u64, second: u64) -> u64 {
let mut value = first ^ 0x9e37_79b9_7f4a_7c15;
value = value.wrapping_mul(0x1000_0000_01b3);
value ^= second;
value.wrapping_mul(0x1000_0000_01b3)
}
pub fn collapse_whitespace(text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut blank_run = 0usize;
let mut wrote_anything = false;
for line in text.lines() {
let squeezed = normalize(line);
if squeezed.is_empty() {
blank_run += 1;
continue;
}
if wrote_anything {
out.push('\n');
if blank_run > 0 {
out.push('\n');
}
}
out.push_str(&squeezed);
wrote_anything = true;
blank_run = 0;
}
out
}
pub fn drop_data_uris(text: &str, limit: usize) -> (String, usize, usize) {
let mut out = String::with_capacity(text.len());
let mut count = 0usize;
let mut bytes = 0usize;
let mut rest = text;
while let Some(start) = rest.find("data:") {
let (before, from_start) = rest.split_at(start);
let end = from_start
.find(|c: char| c.is_whitespace() || matches!(c, '"' | '\'' | ')' | '>' | '<'))
.unwrap_or(from_start.len());
let (uri, after) = from_start.split_at(end);
out.push_str(before);
if uri.len() > limit {
count += 1;
bytes += uri.len();
out.push_str(&format!("data:[dropped {} bytes]", uri.len()));
} else {
out.push_str(uri);
}
rest = after;
}
out.push_str(rest);
(out, count, bytes)
}
pub fn drop_link_only_lines(text: &str) -> (String, usize, usize) {
let mut out = String::with_capacity(text.len());
let mut lines = 0usize;
let mut bytes = 0usize;
for line in text.lines() {
if is_link_only(line) {
lines += 1;
bytes += line.len() + 1;
continue;
}
out.push_str(line);
out.push('\n');
}
(out, lines, bytes)
}
fn is_link_only(line: &str) -> bool {
let body = line
.trim()
.trim_start_matches(['-', '*', '+'])
.trim_start_matches(|c: char| c.is_ascii_digit())
.trim_start_matches(['.', ')'])
.trim();
if body.is_empty() {
return false;
}
if body.starts_with("http://") || body.starts_with("https://") {
return !body.contains(char::is_whitespace);
}
if let Some(rest) = body.strip_prefix('[') {
if let Some((label, target)) = rest.split_once("](") {
if let Some(target) = target.strip_suffix(')') {
return !label.contains(['[', ']']) && !target.contains(['(', ')']);
}
}
}
false
}
pub fn truncate_to_tokens(text: &str, ceiling: usize) -> (String, usize) {
let total = max_tokens(text);
if total <= ceiling {
return (text.to_string(), 0);
}
let marker_cost = approx_marker_tokens(total);
let room = ceiling.saturating_sub(marker_cost);
if room == 0 {
return (String::new(), total);
}
let cut = floor_to_character(text, last_boundary_within(text, room));
if cut == 0 {
return (String::new(), total);
}
let Some(kept) = text.get(..cut) else {
return (String::new(), total);
};
let kept = kept.trim_end();
let dropped = total.saturating_sub(max_tokens(kept));
let mut out = String::with_capacity(kept.len() + 32);
out.push_str(kept);
out.push_str(&format!("\n...[truncated {dropped} tokens]"));
(out, dropped)
}
fn approx_marker_tokens(total: usize) -> usize {
max_tokens(&format!("\n...[truncated {total} tokens]"))
}
fn floor_to_character(text: &str, at: usize) -> usize {
let mut at = at.min(text.len());
while at > 0 && !text.is_char_boundary(at) {
at -= 1;
}
at
}
fn ends_a_sentence(c: char) -> bool {
matches!(c, '.' | '!' | '?')
}
fn ends_a_wide_sentence(c: char) -> bool {
matches!(c, '。' | '!' | '?' | '…' | '.')
}
fn breaks_without_a_space(c: char) -> bool {
matches!(c,
'\u{3000}'..='\u{303f}'
| '\u{3040}'..='\u{30ff}'
| '\u{3400}'..='\u{4dbf}'
| '\u{4e00}'..='\u{9fff}'
| '\u{f900}'..='\u{faff}'
| '\u{ff00}'..='\u{ffef}'
)
}
fn hangs_on_the_character_before(c: char) -> bool {
matches!(c,
'\u{0300}'..='\u{036f}' | '\u{0483}'..='\u{0489}'
| '\u{0591}'..='\u{05bd}'
| '\u{0610}'..='\u{061a}'
| '\u{064b}'..='\u{065f}'
| '\u{0e31}' | '\u{0e34}'..='\u{0e3a}' | '\u{0e47}'..='\u{0e4e}'
| '\u{0900}'..='\u{0903}' | '\u{093a}'..='\u{094f}'
| '\u{1ab0}'..='\u{1aff}'
| '\u{1dc0}'..='\u{1dff}'
| '\u{200d}' | '\u{20d0}'..='\u{20f0}'
| '\u{3099}'..='\u{309a}' | '\u{fe00}'..='\u{fe0f}' | '\u{fe20}'..='\u{fe2f}'
| '\u{ff9e}'..='\u{ff9f}' | '\u{1f3fb}'..='\u{1f3ff}' | '\u{e0100}'..='\u{e01ef}'
)
}
fn hangs_on_the_character_after(c: char) -> bool {
c == '\u{200d}'
}
fn is_regional_indicator(c: char) -> bool {
matches!(c, '\u{1f1e6}'..='\u{1f1ff}')
}
fn last_boundary_within(text: &str, room: usize) -> usize {
let mut spent = Ceiling::new();
let mut sentence_end = 0usize;
let mut word_end = 0usize;
let mut character_end = 0usize;
let mut regional_run = 0usize;
let mut may_cut_before = true;
let mut after_a_space = false;
let mut chars = text.char_indices().peekable();
while let Some((index, c)) = chars.next() {
spent.push(c);
if spent.tokens() > room {
break;
}
let next_index = index + c.len_utf8();
let next = chars.peek().map(|(_, next)| *next);
let followed_by_space = next.map(char::is_whitespace).unwrap_or(true);
regional_run = if is_regional_indicator(c) {
regional_run + 1
} else {
0
};
let may_cut_after = !next.map(hangs_on_the_character_before).unwrap_or(false)
&& !hangs_on_the_character_after(c)
&& regional_run.is_multiple_of(2);
if may_cut_after {
if c.is_whitespace() {
if may_cut_before && !after_a_space {
word_end = index;
character_end = index;
}
} else {
character_end = next_index;
if ends_a_wide_sentence(c) || (ends_a_sentence(c) && followed_by_space) {
sentence_end = next_index;
}
if breaks_without_a_space(c) {
word_end = next_index;
}
}
}
may_cut_before = may_cut_after;
after_a_space = c.is_whitespace();
}
if sentence_end > 0 {
sentence_end
} else if word_end > 0 {
word_end
} else {
character_end
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::indexing_slicing,
clippy::string_slice
)]
use super::*;
use crate::thrift::tokens::max_tokens;
const AWKWARD: &[(&str, &str)] = &[
(
"accented",
"Café, naïve, Ærø. Prix réduit pour la rentrée scolaire.",
),
(
"currency",
"£19.99, €22.40, ¥3200, ₹1650. Livraison incluse.",
),
(
"han",
"钢制垫圈,四十毫米。冷压装配,不要加热外壳。同日发货。",
),
("emoji", "Ready 👍🏽 to ship 🧑🏾🔧 today. Packed 📦 and sealed."),
(
"rtl",
"الشحن في نفس اليوم. السعر يشمل الضريبة. الرجاء الاتصال بنا.",
),
(
"mixed",
"Café ☕ 钢制 £19.99 مرحبا 👍🏽. Second sentence, also mixed: naïve ¥3200.",
),
];
fn awkward_page() -> String {
let mut out = String::new();
for (name, line) in AWKWARD {
out.push_str(name);
out.push('\n');
out.push_str(line);
out.push_str("\n\n");
}
out
}
fn kept_part(out: &str) -> &str {
out.split("\n...[truncated").next().unwrap_or_default()
}
fn cut_between_characters(original: &str, kept: &str) -> bool {
original.starts_with(kept) && original.is_char_boundary(kept.len())
}
#[test]
fn truncation_lands_between_characters_whatever_the_script() {
for (name, line) in AWKWARD {
for ceiling in 0..=(max_tokens(line) + 4) {
let (out, _) = truncate_to_tokens(line, ceiling);
let kept = kept_part(&out);
assert!(
cut_between_characters(line, kept),
"{name} at {ceiling} cut at byte {} of {line:?}",
kept.len()
);
}
}
}
#[test]
fn a_ceiling_large_enough_to_keep_something_keeps_something() {
for (name, line) in AWKWARD {
let ceiling = max_tokens(line);
let (out, dropped) = truncate_to_tokens(line, ceiling);
assert_eq!(dropped, 0, "{name} was cut when it fitted");
assert_eq!(out, *line, "{name}");
let long = [*line; 6].join(" ");
let ceiling = max_tokens(&long) * 3 / 4;
let (out, dropped) = truncate_to_tokens(&long, ceiling);
let kept = kept_part(&out);
assert!(dropped > 0, "{name} was not cut at all");
assert!(!kept.is_empty(), "{name} kept nothing");
assert!(
kept.len() > long.len() / 2,
"{name} kept {} bytes",
kept.len()
);
assert!(cut_between_characters(&long, kept), "{name}");
}
}
#[test]
fn a_cut_never_separates_an_emoji_from_its_modifier() {
let line = "Ready 👍🏽 to ship 🧑🏾🔧 today. Packed 📦 and sealed, twice over.";
let follower = |c: char| matches!(c, '\u{1f3fb}'..='\u{1f3ff}' | '\u{200d}' | '\u{fe0f}');
for ceiling in 0..=(max_tokens(line) + 4) {
let (out, _) = truncate_to_tokens(line, ceiling);
let kept = kept_part(&out);
let Some(next) = line.get(kept.len()..).and_then(|rest| rest.chars().next()) else {
continue;
};
assert!(
!follower(next),
"ceiling {ceiling} cut before {next:?}, leaving {kept:?}"
);
}
}
#[test]
fn a_page_with_no_spaces_in_it_is_still_cut_somewhere_useful() {
let line = ["钢制垫圈,四十毫米。冷压装配,不要加热外壳。同日发货,价格含税。"; 8].concat();
let line = line.as_str();
let ceiling = max_tokens(line) * 3 / 4;
let (out, dropped) = truncate_to_tokens(line, ceiling);
let kept = kept_part(&out);
assert!(dropped > 0);
assert!(cut_between_characters(line, kept), "{kept:?}");
assert!(
kept.len() > line.len() / 2,
"kept {} bytes of {}",
kept.len(),
line.len()
);
assert!(kept.trim_end().ends_with('。'), "{kept:?}");
}
#[test]
fn a_price_in_euros_does_not_take_the_caller_down() {
let line = "Price: €22.40 including delivery. Next sentence here.";
for ceiling in 0..40 {
let (out, _) = truncate_to_tokens(line, ceiling);
assert!(
cut_between_characters(line, kept_part(&out)),
"ceiling {ceiling}: {out:?}"
);
}
let (out, _) = truncate_to_tokens(line, 40);
assert!(kept_part(&out).contains('€'), "{out:?}");
}
#[test]
fn every_cut_survives_awkward_text() {
let page = awkward_page();
let trimmer = Trimmer::new(TrimSettings::default());
for ceiling in [None, Some(0), Some(1), Some(8), Some(40), Some(4000)] {
let trimmed = trimmer.trim(&page, ceiling);
assert!(!trimmed.text.contains('\u{fffd}'), "ceiling {ceiling:?}");
for c in trimmed.text.chars().filter(|c| !c.is_whitespace()) {
assert!(
page.contains(c) || ".[]".contains(c) || c.is_ascii_alphanumeric(),
"ceiling {ceiling:?} produced {c:?}"
);
}
}
for out in [
collapse_whitespace(&page),
drop_data_uris(&page, 4).0,
drop_link_only_lines(&page).0,
] {
assert!(!out.contains('\u{fffd}'));
for line in out.lines().filter(|line| !line.is_empty()) {
assert!(page.contains(line), "{line:?} is not from the page");
}
}
}
#[test]
fn boilerplate_removal_survives_awkward_text() {
let bodies = ["钢制垫圈", "黄铜垫圈", "尼龙垫圈", "青铜垫圈"];
let pages: Vec<String> = bodies
.iter()
.map(|body| {
format!(
"مرحبا بكم\nCafé ☕\n£19.99\n\n{body},同日发货。\n\nالشحن اليوم\nNaïve 👍🏽\n"
)
})
.collect();
let mut trimmer = Trimmer::new(TrimSettings::default());
trimmer.learn(&pages);
for (index, page) in pages.iter().enumerate() {
let trimmed = trimmer.trim(page, Some(45));
assert!(!trimmed.text.contains('\u{fffd}'), "page {index}");
assert!(
!trimmed.text.is_empty(),
"page {index} came back with nothing"
);
}
let trimmed = trimmer.trim(&pages[0], None).text;
assert!(trimmed.contains("钢制垫圈"), "{trimmed}");
assert!(!trimmed.contains("Café"), "{trimmed}");
}
#[test]
fn a_data_uri_next_to_a_multibyte_character_is_cut_cleanly() {
let long = "z".repeat(400);
let text = format!("Prix: €22,40 <img src=\"data:image/png;base64,{long}\">。终わり");
let (out, count, _) = drop_data_uris(&text, 256);
assert_eq!(count, 1);
assert!(out.contains("€22,40"));
assert!(out.contains("。终わり"));
}
#[test]
fn a_link_line_in_another_script_is_still_a_link_line() {
let text = "[الشحن](https://example.com/shipping)\nCafé ☕ costs £3.\n";
let (out, lines, _) = drop_link_only_lines(text);
assert_eq!(lines, 1);
assert!(out.contains("Café ☕ costs £3."));
}
fn crawl(bodies: &[&str]) -> Vec<String> {
bodies
.iter()
.map(|body| {
format!(
"Example Shop\nHome\nProducts\nAbout\nContact\n\n{body}\n\n\
Terms of sale\nPrivacy notice\nCopyright Example Shop\n"
)
})
.collect()
}
#[test]
fn a_menu_and_a_footer_go_and_the_content_stays() {
let pages = crawl(&[
"The blue widget is made of steel and weighs two kilos.",
"The red widget is made of brass and weighs one kilo.",
"The green widget is made of wood and weighs nothing much.",
"The black widget is made of stone and weighs rather a lot.",
]);
let mut trimmer = Trimmer::new(TrimSettings::default());
trimmer.learn(&pages);
assert!(trimmer.knows_boilerplate());
let trimmed = trimmer.trim(&pages[0], None);
assert!(
trimmed.text.contains("blue widget"),
"content went: {:?}",
trimmed.text
);
for furniture in [
"Example Shop",
"Home",
"Products",
"About",
"Contact",
"Terms of sale",
"Privacy notice",
] {
assert!(
!trimmed.text.contains(furniture),
"{furniture} survived: {:?}",
trimmed.text
);
}
assert!(trimmed
.reasons
.iter()
.any(|r| matches!(r, TrimReason::Boilerplate { .. })));
}
#[test]
fn boilerplate_removal_is_most_of_the_reduction_on_a_crawl() {
let pages = crawl(&[
"The blue widget is made of steel.",
"The red widget is made of brass.",
"The green widget is made of wood.",
"The black widget is made of stone.",
"The white widget is made of bone.",
]);
let mut trimmer = Trimmer::new(TrimSettings::default());
trimmer.learn(&pages);
let before: usize = pages.iter().map(|p| p.len()).sum();
let after: usize = pages
.iter()
.map(|page| trimmer.trim(page, None).text.len())
.sum();
let ratio = after as f64 / before as f64;
assert!(ratio < 0.35, "kept {ratio} of the crawl");
}
#[test]
fn a_line_on_two_pages_of_five_is_not_furniture() {
let shared = "Free delivery on orders over fifty pounds.";
let mut pages = crawl(&[
"The blue widget is made of steel.",
"The red widget is made of brass.",
"The green widget is made of wood.",
"The black widget is made of stone.",
"The white widget is made of bone.",
]);
pages[0] = format!("{}\n{shared}\nMore about it.\n", pages[0]);
pages[1] = format!("{}\n{shared}\nMore about it.\n", pages[1]);
let mut trimmer = Trimmer::new(TrimSettings::default());
trimmer.learn(&pages);
let trimmed = trimmer.trim(&pages[0], None);
assert!(trimmed.text.contains(shared), "{:?}", trimmed.text);
}
#[test]
fn a_crawl_too_short_to_judge_loses_nothing() {
let pages = crawl(&["One widget.", "Another widget."]);
let mut trimmer = Trimmer::new(TrimSettings::default());
trimmer.learn(&pages);
assert!(!trimmer.knows_boilerplate());
assert!(trimmer.trim(&pages[0], None).text.contains("Home"));
}
#[test]
fn the_first_line_of_content_after_a_menu_survives() {
let pages = crawl(&[
"Widget: blue, steel, two kilos.",
"Widget: red, brass, one kilo.",
"Widget: green, wood, light.",
"Widget: black, stone, heavy.",
]);
let mut trimmer = Trimmer::new(TrimSettings::default());
trimmer.learn(&pages);
let trimmed = trimmer.trim(&pages[2], None);
assert!(trimmed.text.contains("green"), "{:?}", trimmed.text);
}
#[test]
fn blank_runs_and_repeated_spaces_collapse() {
let text = "One.\n\n\n\nTwo words here.\n \n\nThree. \n";
let out = collapse_whitespace(text);
assert_eq!(out, "One.\n\nTwo words here.\n\nThree.");
assert!(!out.contains("\n\n\n"));
}
#[test]
fn a_long_data_uri_goes_and_a_short_one_stays() {
let long = "x".repeat(400);
let text = format!(
"<img src=\"data:image/png;base64,{long}\"> and <img src=\"data:image/gif;base64,AAAA\">"
);
let (out, count, bytes) = drop_data_uris(&text, 256);
assert_eq!(count, 1);
assert!(bytes > 400);
assert!(out.contains("data:[dropped"));
assert!(out.contains("data:image/gif;base64,AAAA"));
assert!(out.len() < text.len() / 2);
}
#[test]
fn a_line_that_is_only_a_link_goes_and_a_sentence_with_one_stays() {
let text = "https://example.com/a\n\
- [Contact](https://example.com/contact)\n\
See https://example.com/a for the details.\n\
Ordinary prose.\n";
let (out, lines, bytes) = drop_link_only_lines(text);
assert_eq!(lines, 2);
assert!(bytes > 0);
assert!(out.contains("See https://example.com/a for the details."));
assert!(out.contains("Ordinary prose."));
assert!(!out.contains("- [Contact]"));
}
#[test]
fn link_lines_survive_when_the_links_are_what_was_asked_for() {
let trimmer = Trimmer::for_need(&Need::Links);
let trimmed = trimmer.trim("https://example.com/a\nhttps://example.com/b\n", None);
assert!(trimmed.text.contains("https://example.com/b"));
assert!(!trimmer.settings().drop_link_lines);
}
#[test]
fn truncation_lands_on_a_sentence_end() {
let text = "One sentence here. Two sentences here. Three sentences here. \
Four sentences here. Five sentences here. Six sentences here.";
let (out, dropped) = truncate_to_tokens(text, 60);
assert!(dropped > 0);
let body = out
.split("\n...[truncated")
.next()
.expect("the kept text")
.to_string();
assert!(body.ends_with('.'), "cut mid sentence: {body:?}");
assert!(text.starts_with(&body), "kept text is not a prefix");
assert!(out.contains(&format!("truncated {dropped} tokens")));
}
#[test]
fn what_comes_back_fits_the_ceiling_it_was_given() {
let text = "One sentence here. Two sentences here. Three sentences here. \
Four sentences here. Five sentences here. Six sentences here.";
for ceiling in [8usize, 12, 20, 30, 40] {
let (out, _) = truncate_to_tokens(text, ceiling);
let cost = max_tokens(&out);
assert!(cost <= ceiling, "{cost} tokens for a ceiling of {ceiling}");
}
}
#[test]
fn text_that_already_fits_is_returned_untouched() {
let text = "Short enough.";
let (out, dropped) = truncate_to_tokens(text, 100);
assert_eq!(out, text);
assert_eq!(dropped, 0);
}
#[test]
fn a_need_that_opted_out_loses_nothing() {
let trimmer = Trimmer::for_need(&Need::Raw);
let text = "One.\n\n\n\nTwo.\nhttps://example.com/a\n";
let trimmed = trimmer.trim(text, None);
assert_eq!(trimmed.text, text);
assert!(trimmed.reasons.is_empty());
}
#[test]
fn the_reasons_say_where_the_bytes_went() {
let long = "y".repeat(400);
let text = format!(
"One.\n\n\n\nhttps://example.com/a\n<img src=\"data:image/png;base64,{long}\">\n"
);
let trimmer = Trimmer::new(TrimSettings::default());
let trimmed = trimmer.trim(&text, None);
assert!(trimmed.removed_bytes() > 400);
assert!(trimmed
.reasons
.iter()
.any(|r| matches!(r, TrimReason::DataUri { .. })));
assert!(trimmed
.reasons
.iter()
.any(|r| matches!(r, TrimReason::LinkLines { .. })));
assert_eq!(TrimReason::Truncated { tokens: 4 }.bytes(), 0);
}
}