matra 0.3.0

NLP library: UDPipe structured parse, base text metrics, summarization, keyphrase extraction. Rust core with Python bindings.
Documentation
//! Compression-ratio metric — brotli-compressed size / original size.
//!
//! Lower ratio = more compressible = more repetitive prose. A rough
//! proxy for surface redundancy in LLM-generated text.

use std::io::Write;

use crate::domain::Document;
use crate::domain::limits::{MAX_COMPRESSION_PARAGRAPH_BYTES, MIN_COMPRESSION_WORDS};

/// Brotli sliding-window size (log2 bytes). 18 = 256 KiB window.
///
/// Down from 22 (4 MiB) per Vector's HIGH finding: a 4 MiB window
/// per paragraph multiplies CPU pegging risk on adversarial input.
/// 18 is the safe-by-default ceiling for prose: long-range redundancy
/// across 256 KiB is more than enough for compression-as-redundancy-proxy
/// signal; beyond that the metric is measuring engine plumbing, not
/// linguistic structure.
const BROTLI_LGWIN: u32 = 18;

/// Brotli quality level (0–11). 6 is mid-range — balances time vs ratio
/// for our metric purpose. We are not optimizing storage; we are getting
/// a redundancy proxy. Higher quality only marginally tightens the ratio
/// on prose at significant CPU cost.
const BROTLI_QUALITY: u32 = 6;

/// Populate `ParagraphMetrics::compression_ratio` for every paragraph with
/// at least [`MIN_COMPRESSION_WORDS`] words that is not in a blockquote
/// and is at or under [`MAX_COMPRESSION_PARAGRAPH_BYTES`], whose 256 KiB
/// matches [`BROTLI_LGWIN`]'s window.
pub(crate) fn compute(analysis: &mut Document) {
    for para in analysis.paragraphs_mut() {
        if para.word_count() >= MIN_COMPRESSION_WORDS
            && para.kind != crate::domain::ParagraphKind::Quote
            && para.text.len() <= MAX_COMPRESSION_PARAGRAPH_BYTES
        {
            para.metrics.compression_ratio = compression_ratio(&para.text);
        }
    }
}

fn compression_ratio(text: &str) -> Option<f64> {
    let original = text.as_bytes();
    if original.is_empty() {
        return Some(1.0);
    }
    let mut compressed = Vec::new();
    {
        let mut writer =
            brotli::CompressorWriter::new(&mut compressed, 4096, BROTLI_QUALITY, BROTLI_LGWIN);
        writer.write_all(original).ok()?;
    }
    Some(compressed.len() as f64 / original.len() as f64)
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::domain::{Paragraph, ParagraphKind, Section, Sentence, Span, Token};

    fn content_token(text: &str) -> Token {
        Token {
            id: 0,
            span: Default::default(),
            text: text.to_string(),
            lemma: text.to_lowercase(),
            pos: "NOUN".to_string(),
            xpos: String::new(),
            features: std::collections::BTreeMap::new(),
            dep: "dep".to_string(),
            head: 0,
            deps: String::new(),
            misc: String::new(),
            is_punct: false,
        }
    }

    #[test]
    fn compression_ratio_bounds() {
        let ratio = compression_ratio("Some text to compress for testing purposes.")
            .expect("compression should succeed");
        assert!(ratio > 0.0 && ratio <= 2.0);
    }

    #[test]
    fn oversize_paragraph_is_skipped_quickly() {
        // 1 MiB single paragraph (4× the per-paragraph cap). Should be
        // skipped (compression_ratio = None) and the whole compute call
        // should return well under 100ms — we are measuring the gate,
        // not brotli.
        let big_text = "a ".repeat(MAX_COMPRESSION_PARAGRAPH_BYTES); // ~512 KiB after duplication, but we want > cap
        let big_text = big_text.repeat(2); // ~1 MiB, well over MAX_COMPRESSION_PARAGRAPH_BYTES
        assert!(big_text.len() > MAX_COMPRESSION_PARAGRAPH_BYTES);

        let sections = vec![Section {
            heading: None,
            level: 0,
            span: None,
            paragraphs: vec![Paragraph::new(
                big_text,
                ParagraphKind::Paragraph,
                Span::default(),
            )],
        }];
        let mut analysis =
            Document::new(crate::domain::Provenance::for_test(), sections, Vec::new());

        // Attach enough content tokens so word_count clears the >50 threshold.
        for para in analysis.paragraphs_mut() {
            let tokens = (0..51).map(|j| content_token(&format!("w{j}"))).collect();
            para.sentences.push(Sentence::new(String::new(), tokens));
        }
        let start = std::time::Instant::now();
        compute(&mut analysis);
        let elapsed = start.elapsed();

        let paras: Vec<_> = analysis.paragraphs().collect();
        assert!(
            paras[0].metrics.compression_ratio.is_none(),
            "oversize paragraph should be skipped"
        );
        assert!(
            elapsed < std::time::Duration::from_millis(100),
            "skipping oversize paragraph took {elapsed:?} (expected < 100ms)"
        );
    }

    #[test]
    fn compression_word_count_boundary() {
        let text_50 = (0..50)
            .map(|i| format!("word{i}"))
            .collect::<Vec<_>>()
            .join(" ");
        let text_51 = (0..51)
            .map(|i| format!("word{i}"))
            .collect::<Vec<_>>()
            .join(" ");

        let sections = vec![Section {
            heading: None,
            level: 0,
            span: None,
            paragraphs: vec![
                Paragraph::new(text_50, ParagraphKind::Paragraph, Span::default()),
                Paragraph::new(text_51, ParagraphKind::Paragraph, Span::default()),
            ],
        }];
        let mut analysis =
            Document::new(crate::domain::Provenance::for_test(), sections, Vec::new());

        // Attach enough content tokens so word_count clears the thresholds.
        for (i, para) in analysis.paragraphs_mut().enumerate() {
            let tokens = (0..(50 + i))
                .map(|j| content_token(&format!("w{j}")))
                .collect();
            para.sentences.push(Sentence::new(String::new(), tokens));
        }

        compute(&mut analysis);

        let paras: Vec<_> = analysis.paragraphs().collect();
        assert!(
            paras[0].metrics.compression_ratio.is_none(),
            "50 words should not get compression"
        );
        assert!(
            paras[1].metrics.compression_ratio.is_some(),
            "51 words should get compression"
        );
    }
}