setzer_import/scrub/
check.rs

1//! Does the copy show the problem? Import the original and the copy and
2//! compare what the author saw go wrong: the error, the warnings, and the
3//! shape of the book (its language, and the sequence of section and block
4//! kinds).
5//!
6//! Warning messages quote the manuscript ("the words after the first
7//! break"), so they differ in a copy by design; a warning is compared by
8//! its kind, its location, its section and what it names (a picture, a
9//! footnote label, a front-matter key, a fence).
10
11use setzer_model::{Block, Book};
12
13use super::{Kind, Scrubbed};
14use crate::{ImportError, Imported, WarningDetail, WarningKind};
15
16/// The result of [`check`].
17#[derive(Debug, Clone, PartialEq, Eq)]
18pub struct Verdict {
19    /// Whether the copy imports like the original.
20    pub shows_problem: bool,
21    /// What differs, in words for the person who reads the report. Empty
22    /// when `shows_problem`.
23    pub differences: Vec<String>,
24}
25
26type Key = (WarningKind, usize, Option<usize>, Option<WarningDetail>);
27
28/// What an import came to, minus the text.
29struct Summary {
30    error: Option<String>,
31    warnings: Vec<Key>,
32    shape: Vec<String>,
33}
34
35fn summarize(result: Result<Imported, ImportError>) -> Summary {
36    match result {
37        Err(e) => Summary {
38            error: Some(e.to_string()),
39            warnings: Vec::new(),
40            shape: Vec::new(),
41        },
42        Ok(im) => Summary {
43            error: None,
44            warnings: im
45                .warnings
46                .into_iter()
47                .map(|w| (w.kind, w.location, w.section, w.detail))
48                .collect(),
49            shape: shape(&im.book),
50        },
51    }
52}
53
54/// One line per section and per block: the kinds, nothing of the text.
55fn shape(book: &Book) -> Vec<String> {
56    // The language chooses quotes and hyphenation: a copy that reads as
57    // another language does not show a quote bug.
58    let mut out = vec![format!("language {}", book.metadata.lang.0)];
59    for (i, s) in book.sections.iter().enumerate() {
60        let epigraph = if s.epigraph.is_some() {
61            " +epigraph"
62        } else {
63            ""
64        };
65        out.push(format!(
66            "section {i}: {:?} {:?}{epigraph}",
67            s.kind, s.numbering
68        ));
69        for (j, b) in s.blocks.iter().enumerate() {
70            out.push(format!("section {i}, block {j}: {}", block_kind(b)));
71        }
72    }
73    out
74}
75
76fn block_kind(b: &Block) -> String {
77    match b {
78        Block::Paragraph { .. } => "paragraph".into(),
79        Block::Heading { level, .. } => format!("heading {level}"),
80        Block::SceneBreak { .. } => "scene break".into(),
81        Block::BlockQuote { blocks, .. } => {
82            let kinds: Vec<String> = blocks.iter().map(block_kind).collect();
83            format!("quote [{}]", kinds.join(", "))
84        }
85        Block::Verse { lines, .. } => format!("verse of {} lines", lines.len()),
86        // A picture's place, side and width are what the importer reads from
87        // a .docx's `wp:anchor` (the copy must not change them).
88        Block::Image {
89            placement, side, ..
90        } => {
91            let width = b
92                .image_width()
93                .map_or("auto".to_string(), |w| format!("{w}%"));
94            format!(
95                "image {} side {} width {width}",
96                format!("{placement:?}").to_lowercase(),
97                format!("{side:?}").to_lowercase()
98            )
99        }
100        Block::List { ordered, items, .. } => {
101            let kind = if *ordered { "numbered" } else { "bulleted" };
102            format!("{kind} list of {} items", items.len())
103        }
104        Block::Table(table) => {
105            let l = table.layout();
106            format!(
107                "table of {} rows, {} columns, {} header rows",
108                l.rows, l.columns, l.header_rows
109            )
110        }
111        Block::PageBreak { .. } => "page break".into(),
112    }
113}
114
115/// Import the original and the copy and compare. `original_images` gives
116/// the original Markdown manuscript's pictures, as for `scrub`; the copy's
117/// are its own.
118pub fn check(
119    kind: Kind,
120    original: &[u8],
121    original_images: impl FnMut(&str) -> Option<Vec<u8>>,
122    copy: &Scrubbed,
123) -> Verdict {
124    let (a, b) = match kind {
125        Kind::Docx => (
126            summarize(crate::docx::import(original)),
127            summarize(crate::docx::import(&copy.bytes)),
128        ),
129        Kind::Markdown => {
130            let strip = |b: &[u8]| {
131                std::str::from_utf8(b).map(|s| s.strip_prefix('\u{feff}').unwrap_or(s).to_string())
132            };
133            let (Ok(orig), Ok(copy_text)) = (strip(original), strip(&copy.bytes)) else {
134                return Verdict {
135                    shows_problem: false,
136                    differences: vec!["the original or the copy is not UTF-8 text".into()],
137                };
138            };
139            let copy_images = |p: &str| {
140                copy.images
141                    .iter()
142                    .find(|(path, _)| path == p)
143                    .map(|(_, bytes)| bytes.clone())
144            };
145            (
146                summarize(crate::markdown::import(&orig, original_images)),
147                summarize(crate::markdown::import(&copy_text, copy_images)),
148            )
149        }
150    };
151    compare(&a, &b)
152}
153
154fn compare(a: &Summary, b: &Summary) -> Verdict {
155    let mut differences = Vec::new();
156    if a.error != b.error {
157        differences.push(format!(
158            "the import's error differs: the original gives {}, the copy {}",
159            describe(a.error.as_deref()),
160            describe(b.error.as_deref())
161        ));
162    }
163    if a.warnings != b.warnings {
164        let only = |x: &[Key], y: &[Key]| -> Vec<String> {
165            let mut rest: Vec<&Key> = y.iter().collect();
166            x.iter()
167                .filter(|k| match rest.iter().position(|r| r == k) {
168                    Some(at) => {
169                        rest.remove(at);
170                        false
171                    }
172                    None => true,
173                })
174                .map(|(kind, loc, _, detail)| match detail {
175                    Some(d) => format!("{kind:?} at {loc} ({d:?})"),
176                    None => format!("{kind:?} at {loc}"),
177                })
178                .collect()
179        };
180        let (lost, gained) = (
181            only(&a.warnings, &b.warnings),
182            only(&b.warnings, &a.warnings),
183        );
184        if lost.is_empty() && gained.is_empty() {
185            differences.push("the warnings come in a different order".into());
186        }
187        if !lost.is_empty() {
188            differences.push(format!("the copy lacks the warnings: {}", lost.join("; ")));
189        }
190        if !gained.is_empty() {
191            differences.push(format!("the copy adds the warnings: {}", gained.join("; ")));
192        }
193    }
194    if a.shape != b.shape {
195        let at = a
196            .shape
197            .iter()
198            .zip(&b.shape)
199            .position(|(x, y)| x != y)
200            .unwrap_or_else(|| a.shape.len().min(b.shape.len()));
201        let line = |s: &Summary| s.shape.get(at).cloned().unwrap_or_else(|| "nothing".into());
202        differences.push(format!(
203            "the book's shape differs from line {} on: the original has \"{}\", the copy \"{}\" \
204             ({} and {} lines)",
205            at + 1,
206            line(a),
207            line(b),
208            a.shape.len(),
209            b.shape.len()
210        ));
211    }
212    Verdict {
213        shows_problem: differences.is_empty(),
214        differences,
215    }
216}
217
218fn describe(error: Option<&str>) -> String {
219    error.map_or_else(|| "none".into(), |e| format!("\"{e}\""))
220}
221
222#[cfg(test)]
223mod tests {
224    use super::*;
225
226    fn markdown(src: &str) -> Summary {
227        summarize(crate::markdown::import(src, |_| None))
228    }
229
230    #[test]
231    fn equal_shapes_and_warnings_show_the_problem() {
232        // links.md / footnotes.md lines: a missing note warns.
233        let a =
234            markdown("Title: T\n\n# One\n\nA mark without a definition[^nowhere] stays as text.\n");
235        let b =
236            markdown("Title: T\n\n# One\n\nA mark without a definition[^nowhere] stays as text.\n");
237        let v = compare(&a, &b);
238        assert!(v.shows_problem && v.differences.is_empty());
239    }
240
241    #[test]
242    fn a_lost_warning_a_new_block_and_another_error_are_reported() {
243        let a =
244            markdown("Title: T\n\n# One\n\nA mark without a definition[^nowhere] stays as text.\n");
245        let b = markdown(
246            "Title: T\n\n# One\n\nA mark without a definition stays as text.\n\n* * *\n\nMore.\n",
247        );
248        let v = compare(&a, &b);
249        assert!(!v.shows_problem);
250        let all = v.differences.join("\n");
251        assert!(
252            all.contains("lacks the warnings") && all.contains("FootnoteMissing"),
253            "{all}"
254        );
255        assert!(all.contains("shape differs"), "{all}");
256        let empty = summarize(crate::markdown::import("\n", |_| None));
257        let v = compare(&a, &empty);
258        assert!(v.differences.iter().any(|d| d.contains("error differs")));
259    }
260}
setzer_import/scrub/check.rs

setzer_import/scrub/
check.rs
