use std::io::Write;
use crate::domain::Document;
use crate::domain::limits::{MAX_COMPRESSION_PARAGRAPH_BYTES, MIN_COMPRESSION_WORDS};
const BROTLI_LGWIN: u32 = 18;
const BROTLI_QUALITY: u32 = 6;
pub(crate) fn compute(analysis: &mut Document) {
for para in analysis.paragraphs_mut() {
if para.word_count() >= MIN_COMPRESSION_WORDS
&& para.kind != crate::domain::ParagraphKind::Quote
&& para.text.len() <= MAX_COMPRESSION_PARAGRAPH_BYTES
{
para.metrics.compression_ratio = compression_ratio(¶.text);
}
}
}
fn compression_ratio(text: &str) -> Option<f64> {
let original = text.as_bytes();
if original.is_empty() {
return Some(1.0);
}
let mut compressed = Vec::new();
{
let mut writer =
brotli::CompressorWriter::new(&mut compressed, 4096, BROTLI_QUALITY, BROTLI_LGWIN);
writer.write_all(original).ok()?;
}
Some(compressed.len() as f64 / original.len() as f64)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::{Paragraph, ParagraphKind, Section, Sentence, Span, Token};
fn content_token(text: &str) -> Token {
Token {
id: 0,
span: Default::default(),
text: text.to_string(),
lemma: text.to_lowercase(),
pos: "NOUN".to_string(),
xpos: String::new(),
features: std::collections::BTreeMap::new(),
dep: "dep".to_string(),
head: 0,
deps: String::new(),
misc: String::new(),
is_punct: false,
}
}
#[test]
fn compression_ratio_bounds() {
let ratio = compression_ratio("Some text to compress for testing purposes.")
.expect("compression should succeed");
assert!(ratio > 0.0 && ratio <= 2.0);
}
#[test]
fn oversize_paragraph_is_skipped_quickly() {
let big_text = "a ".repeat(MAX_COMPRESSION_PARAGRAPH_BYTES); let big_text = big_text.repeat(2); assert!(big_text.len() > MAX_COMPRESSION_PARAGRAPH_BYTES);
let sections = vec![Section {
heading: None,
level: 0,
span: None,
paragraphs: vec![Paragraph::new(
big_text,
ParagraphKind::Paragraph,
Span::default(),
)],
}];
let mut analysis =
Document::new(crate::domain::Provenance::for_test(), sections, Vec::new());
for para in analysis.paragraphs_mut() {
let tokens = (0..51).map(|j| content_token(&format!("w{j}"))).collect();
para.sentences.push(Sentence::new(String::new(), tokens));
}
let start = std::time::Instant::now();
compute(&mut analysis);
let elapsed = start.elapsed();
let paras: Vec<_> = analysis.paragraphs().collect();
assert!(
paras[0].metrics.compression_ratio.is_none(),
"oversize paragraph should be skipped"
);
assert!(
elapsed < std::time::Duration::from_millis(100),
"skipping oversize paragraph took {elapsed:?} (expected < 100ms)"
);
}
#[test]
fn compression_word_count_boundary() {
let text_50 = (0..50)
.map(|i| format!("word{i}"))
.collect::<Vec<_>>()
.join(" ");
let text_51 = (0..51)
.map(|i| format!("word{i}"))
.collect::<Vec<_>>()
.join(" ");
let sections = vec![Section {
heading: None,
level: 0,
span: None,
paragraphs: vec![
Paragraph::new(text_50, ParagraphKind::Paragraph, Span::default()),
Paragraph::new(text_51, ParagraphKind::Paragraph, Span::default()),
],
}];
let mut analysis =
Document::new(crate::domain::Provenance::for_test(), sections, Vec::new());
for (i, para) in analysis.paragraphs_mut().enumerate() {
let tokens = (0..(50 + i))
.map(|j| content_token(&format!("w{j}")))
.collect();
para.sentences.push(Sentence::new(String::new(), tokens));
}
compute(&mut analysis);
let paras: Vec<_> = analysis.paragraphs().collect();
assert!(
paras[0].metrics.compression_ratio.is_none(),
"50 words should not get compression"
);
assert!(
paras[1].metrics.compression_ratio.is_some(),
"51 words should get compression"
);
}
}