text_mirror/convert/
differential.rs

1//! Differential comparison for legacy Office conversions.
2//!
3//! The primary converter's text is checked against a second,
4//! independent extraction. Agreement is silent. Divergence appends a
5//! structured warning. The one failing case is a primary that
6//! extracted under half of what the secondary found, because an
7//! artifact missing that much content must not present as converted.
8//! A secondary crash appends an unavailability warning and never
9//! sinks a sound primary conversion.
10
11use std::collections::HashMap;
12use std::io::Cursor;
13use std::panic::{AssertUnwindSafe, catch_unwind};
14
15use unicode_normalization::UnicodeNormalization;
16
17/// The secondary extractor identity carried in every differential
18/// warning and error string.
19pub const SECONDARY_ID: &str = "office_oxide/0.1.8";
20
21/// Length ratio at or above this bound counts toward agreement.
22pub const LEN_RATIO_BOUND: f64 = 0.90;
23
24/// Dice coefficient at or above this bound counts toward agreement.
25pub const DICE_BOUND: f64 = 0.95;
26
27/// Primary shorter than this fraction of the secondary is a failure.
28pub const FAILURE_FRACTION: f64 = 0.50;
29
30/// The two comparison metrics.
31///
32/// Length is the UTF-8 byte length of the normalized comparison text.
33/// That unit is the contract, because changing it would silently move
34/// verdicts near the thresholds.
35#[derive(Debug, Clone, Copy, PartialEq)]
36pub struct DiffMetrics {
37    /// Normalized length ratio, shorter over longer, 1.0 when equal.
38    pub len_ratio: f64,
39    /// Dice coefficient over token multisets.
40    pub dice: f64,
41}
42
43/// The comparison verdict.
44#[derive(Debug, Clone, PartialEq)]
45pub enum DiffVerdict {
46    /// Both metrics at or above their bounds. Nothing is recorded.
47    Agreement,
48    /// A metric fell below its bound but the primary holds enough
49    /// text. The conversion stands with a warning.
50    Divergence,
51    /// The primary extracted under half of the secondary's text. The
52    /// conversion fails.
53    Failure,
54}
55
56/// Normalizes text for comparison: NFC, every whitespace run
57/// collapsed to one space, trimmed. No case folding.
58pub fn normalize_for_diff(text: &str) -> String {
59    let composed: String = text.nfc().collect();
60    let mut out = String::with_capacity(composed.len());
61    let mut in_space = true;
62    for c in composed.chars() {
63        if c.is_whitespace() {
64            if !in_space {
65                out.push(' ');
66                in_space = true;
67            }
68        } else {
69            out.push(c);
70            in_space = false;
71        }
72    }
73    while out.ends_with(' ') {
74        out.pop();
75    }
76    out
77}
78
79fn dice_over_tokens(a: &str, b: &str) -> f64 {
80    let mut counts: HashMap<&str, (u64, u64)> = HashMap::new();
81    let mut total_a = 0u64;
82    let mut total_b = 0u64;
83    for token in a.split(' ').filter(|t| !t.is_empty()) {
84        counts.entry(token).or_default().0 += 1;
85        total_a += 1;
86    }
87    for token in b.split(' ').filter(|t| !t.is_empty()) {
88        counts.entry(token).or_default().1 += 1;
89        total_b += 1;
90    }
91    if total_a + total_b == 0 {
92        return 1.0;
93    }
94    let intersection: u64 = counts.values().map(|(x, y)| (*x).min(*y)).sum();
95    (2 * intersection) as f64 / (total_a + total_b) as f64
96}
97
98/// Compares two already-normalized texts and returns the metrics and
99/// the verdict per the thresholds.
100pub fn compare_texts(primary: &str, secondary: &str) -> (DiffMetrics, DiffVerdict) {
101    let primary_len = primary.len() as f64;
102    let secondary_len = secondary.len() as f64;
103    let len_ratio = if primary_len == 0.0 && secondary_len == 0.0 {
104        1.0
105    } else {
106        primary_len.min(secondary_len) / primary_len.max(secondary_len)
107    };
108    let dice = dice_over_tokens(primary, secondary);
109    let metrics = DiffMetrics { len_ratio, dice };
110    let verdict = if len_ratio >= LEN_RATIO_BOUND && dice >= DICE_BOUND {
111        DiffVerdict::Agreement
112    } else if primary_len < FAILURE_FRACTION * secondary_len {
113        DiffVerdict::Failure
114    } else {
115        DiffVerdict::Divergence
116    };
117    (metrics, verdict)
118}
119
120/// The structured divergence string used in warnings and error detail.
121/// Metrics print with four decimals so a reader can reproduce the
122/// threshold decision from the record alone.
123pub fn divergence_message(metrics: DiffMetrics) -> String {
124    format!(
125        "differential-divergence: secondary={SECONDARY_ID} len_ratio={:.4} dice={:.4}",
126        metrics.len_ratio, metrics.dice
127    )
128}
129
130/// The unavailability warning for a secondary that errored or panicked.
131pub fn unavailable_message(error_class: &str) -> String {
132    format!("differential-unavailable: {error_class}")
133}
134
135fn office_error_class(error: &office_oxide::OfficeError) -> &'static str {
136    use office_oxide::OfficeError;
137    match error {
138        OfficeError::Core(_) => "core",
139        OfficeError::Docx(_) => "docx",
140        OfficeError::Xlsx(_) => "xlsx",
141        OfficeError::Pptx(_) => "pptx",
142        OfficeError::Doc(_) => "doc",
143        OfficeError::Xls(_) => "xls",
144        OfficeError::Ppt(_) => "ppt",
145        OfficeError::UnsupportedFormat(_) => "unsupported_format",
146    }
147}
148
149/// Extracts the secondary's plain text for a legacy Office format.
150/// Errors and panics both come back as an error class string.
151pub fn secondary_text(bytes: &[u8], format_id: &str) -> std::result::Result<String, String> {
152    let format = match format_id {
153        "doc" => office_oxide::DocumentFormat::Doc,
154        "xls" => office_oxide::DocumentFormat::Xls,
155        "ppt" => office_oxide::DocumentFormat::Ppt,
156        other => return Err(format!("no secondary for format {other}")),
157    };
158    let owned = bytes.to_vec();
159    let extraction = catch_unwind(AssertUnwindSafe(move || {
160        office_oxide::Document::from_reader(Cursor::new(owned), format)
161            .map(|document| document.plain_text())
162    }));
163    match extraction {
164        Ok(Ok(text)) => Ok(text),
165        Ok(Err(error)) => Err(office_error_class(&error).to_string()),
166        Err(_) => Err("panic".to_string()),
167    }
168}
169
170/// What a full differential check concluded.
171#[derive(Debug, Clone, PartialEq)]
172pub enum DifferentialOutcome {
173    /// Both metrics inside their bounds. Nothing is recorded.
174    Agreement,
175    /// A warning string to append, for divergence or for a secondary
176    /// that could not run.
177    Warning(String),
178    /// The failure detail for a failed record with reason
179    /// `differential-divergence`.
180    Failure(String),
181}
182
183/// Runs the whole check: secondary extraction, normalization of both
184/// sides, metrics, and verdict.
185pub fn check(primary_text: &str, source: &[u8], format_id: &str) -> DifferentialOutcome {
186    match secondary_text(source, format_id) {
187        Err(class) => DifferentialOutcome::Warning(unavailable_message(&class)),
188        Ok(secondary) => {
189            let primary_norm = normalize_for_diff(primary_text);
190            let secondary_norm = normalize_for_diff(&secondary);
191            let (metrics, verdict) = compare_texts(&primary_norm, &secondary_norm);
192            match verdict {
193                DiffVerdict::Agreement => DifferentialOutcome::Agreement,
194                DiffVerdict::Divergence => {
195                    DifferentialOutcome::Warning(divergence_message(metrics))
196                }
197                DiffVerdict::Failure => DifferentialOutcome::Failure(format!(
198                    "secondary={SECONDARY_ID} len_ratio={:.4} dice={:.4}",
199                    metrics.len_ratio, metrics.dice
200                )),
201            }
202        }
203    }
204}
205
206#[cfg(test)]
207mod tests {
208    use super::*;
209
210    #[test]
211    fn normalize_collapses_whitespace_and_composes() {
212        assert_eq!(
213            normalize_for_diff("  a\t\tb\n\nc  cafe\u{301} "),
214            "a b c caf\u{e9}"
215        );
216    }
217
218    #[test]
219    fn identical_texts_agree() {
220        let (metrics, verdict) = compare_texts("alpha beta gamma", "alpha beta gamma");
221        assert_eq!(metrics.len_ratio, 1.0);
222        assert_eq!(metrics.dice, 1.0);
223        assert_eq!(verdict, DiffVerdict::Agreement);
224        let (_, verdict) = compare_texts("", "");
225        assert_eq!(verdict, DiffVerdict::Agreement);
226    }
227
228    #[test]
229    fn small_disagreement_diverges_with_a_warning_string() {
230        // Same length class, different tokens: dice falls, length holds.
231        let (metrics, verdict) = compare_texts(
232            "alpha beta gamma delta epsilon",
233            "alpha beta gamma delta zzzzzzz",
234        );
235        assert_eq!(verdict, DiffVerdict::Divergence);
236        let message = divergence_message(metrics);
237        assert!(message.starts_with("differential-divergence: secondary=office_oxide/0.1.8"));
238        assert!(message.contains("len_ratio=1.0000"));
239        assert!(message.contains("dice=0.8000"));
240    }
241
242    #[test]
243    fn four_decimals_separate_a_near_threshold_dice() {
244        // A dice of 0.949 fails the 0.95 bound and must not print as
245        // if it passed.
246        let metrics = DiffMetrics {
247            len_ratio: 1.0,
248            dice: 0.949,
249        };
250        let message = divergence_message(metrics);
251        assert!(message.contains("dice=0.9490"));
252        assert!(!message.contains("dice=0.95"));
253    }
254
255    #[test]
256    fn short_primary_fails_and_short_secondary_only_warns() {
257        let long = "one two three four five six seven eight nine ten";
258        let short = "one two three";
259        let (_, verdict) = compare_texts(short, long);
260        assert_eq!(verdict, DiffVerdict::Failure);
261        // The same gap the other way is the secondary's weakness.
262        let (_, verdict) = compare_texts(long, short);
263        assert_eq!(verdict, DiffVerdict::Divergence);
264    }
265
266    #[test]
267    fn secondary_garbage_reports_an_error_class() {
268        let class = secondary_text(b"not an office file", "doc").unwrap_err();
269        assert!(!class.is_empty());
270    }
271
272    #[test]
273    fn unavailable_message_names_the_class() {
274        assert_eq!(
275            unavailable_message("panic"),
276            "differential-unavailable: panic"
277        );
278    }
279}
text_mirror/convert/differential.rs

text_mirror/convert/
differential.rs
