use std::io::Write;
use crate::domain::Document;
const BROTLI_LGWIN: u32 = 18;
const BROTLI_QUALITY: u32 = 6;
const MAX_PARAGRAPH_BYTES: usize = 256 * 1024;
pub fn compute(analysis: &mut Document) {
for para in analysis.paragraphs_mut() {
if para.word_count() > 50 && !para.in_blockquote && para.text.len() <= MAX_PARAGRAPH_BYTES {
para.compression_ratio = compression_ratio(¶.text);
}
}
}
fn compression_ratio(text: &str) -> Option<f64> {
let original = text.as_bytes();
if original.is_empty() {
return Some(1.0);
}
let mut compressed = Vec::new();
{
let mut writer =
brotli::CompressorWriter::new(&mut compressed, 4096, BROTLI_QUALITY, BROTLI_LGWIN);
writer.write_all(original).ok()?;
}
Some(compressed.len() as f64 / original.len() as f64)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{Paragraph, Section, Sentence, Token};
fn content_token(text: &str) -> Token {
Token {
id: 0,
text: text.to_string(),
lemma: text.to_lowercase(),
pos: "NOUN".to_string(),
xpos: String::new(),
feats: String::new(),
dep: "dep".to_string(),
head: 0,
deps: String::new(),
misc: String::new(),
is_punct: false,
}
}
#[test]
fn compression_ratio_bounds() {
let ratio = compression_ratio("Some text to compress for testing purposes.")
.expect("compression should succeed");
assert!(ratio > 0.0 && ratio <= 2.0);
}
#[test]
fn oversize_paragraph_is_skipped_quickly() {
let big_text = "a ".repeat(MAX_PARAGRAPH_BYTES); let big_text = big_text.repeat(2); assert!(big_text.len() > MAX_PARAGRAPH_BYTES);
let sections = vec![Section {
heading: None,
level: 0,
paragraphs: vec![Paragraph::new(big_text, false)],
}];
let mut analysis = Document::new(sections);
for para in analysis.paragraphs_mut() {
let tokens = (0..51).map(|j| content_token(&format!("w{j}"))).collect();
para.sentences.push(Sentence::new(String::new(), tokens));
}
let start = std::time::Instant::now();
compute(&mut analysis);
let elapsed = start.elapsed();
let paras: Vec<_> = analysis.paragraphs().collect();
assert!(
paras[0].compression_ratio.is_none(),
"oversize paragraph should be skipped"
);
assert!(
elapsed < std::time::Duration::from_millis(100),
"skipping oversize paragraph took {elapsed:?} (expected < 100ms)"
);
}
#[test]
fn compression_word_count_boundary() {
let text_50 = (0..50)
.map(|i| format!("word{i}"))
.collect::<Vec<_>>()
.join(" ");
let text_51 = (0..51)
.map(|i| format!("word{i}"))
.collect::<Vec<_>>()
.join(" ");
let sections = vec![Section {
heading: None,
level: 0,
paragraphs: vec![
Paragraph::new(text_50, false),
Paragraph::new(text_51, false),
],
}];
let mut analysis = Document::new(sections);
for (i, para) in analysis.paragraphs_mut().enumerate() {
let tokens = (0..(50 + i))
.map(|j| content_token(&format!("w{j}")))
.collect();
para.sentences.push(Sentence::new(String::new(), tokens));
}
compute(&mut analysis);
let paras: Vec<_> = analysis.paragraphs().collect();
assert!(
paras[0].compression_ratio.is_none(),
"50 words should not get compression"
);
assert!(
paras[1].compression_ratio.is_some(),
"51 words should get compression"
);
}
}