Skip to main content

common/parser_tools/
content_parser.rs

1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3use crate::parser_tools::list_grouper::default_list_suffix;
4
5/// An inline image recovered by a parser, before it becomes an `ImageAnchor`.
6///
7/// `src` is whatever the source document pointed at — a relative path, a bare
8/// name, a URL. Parsers do not resolve it; that is the embedding application's
9/// job, and this crate never touches the filesystem.
10#[derive(Debug, Clone, Default, PartialEq, Eq)]
11pub struct ParsedImage {
12    pub src: String,
13    pub alt: String,
14    /// Display size in pixels, `0` when the source did not state one.
15    ///
16    /// Djot and HTML both carry these as attributes (`{width=800}`,
17    /// `<img width=…>`); Markdown has no syntax for them, so a Markdown import
18    /// leaves both zero and the caller supplies intrinsic dimensions.
19    pub width: i64,
20    pub height: i64,
21}
22
23/// A parsed inline span with formatting info.
24///
25/// A span carries *either* text or an image, never both: an image span's `text`
26/// is empty and its description lives in [`ParsedImage::alt`], so alt text
27/// cannot leak into the block's prose (and therefore cannot be counted as
28/// manuscript words or matched by a search).
29#[derive(Debug, Clone, Default)]
30pub struct ParsedSpan {
31    pub text: String,
32    pub bold: bool,
33    pub italic: bool,
34    pub underline: bool,
35    pub strikeout: bool,
36    pub code: bool,
37    /// Superscript (djot `^x^`). Maps to `CharVerticalAlignment::SuperScript`.
38    pub superscript: bool,
39    /// Subscript (djot `~x~`). Maps to `CharVerticalAlignment::SubScript`.
40    pub subscript: bool,
41    pub link_href: Option<String>,
42    /// Set when this span is an inline image rather than text.
43    pub image: Option<ParsedImage>,
44    /// Set when this span is a footnote *reference* rather than text — the
45    /// label naming the note, never the number a reader sees.
46    pub footnote_ref: Option<String>,
47    /// The text keeps its whitespace as it is: it came from an element styled
48    /// `white-space: pre`, `pre-wrap` or `break-spaces`, where a tab or two spaces in a
49    /// row are the text's own. An HTML reader collapses none of it.
50    pub preserve_whitespace: bool,
51}
52
53/// A parsed table cell containing inline spans.
54#[derive(Debug, Clone)]
55pub struct ParsedTableCell {
56    pub spans: Vec<ParsedSpan>,
57}
58
59/// A parsed table extracted from markdown or HTML.
60#[derive(Debug, Clone)]
61pub struct ParsedTable {
62    /// Number of header rows (typically 1 for markdown tables).
63    pub header_rows: usize,
64    /// All rows (header + body), each containing cells with their inline spans.
65    pub rows: Vec<Vec<ParsedTableCell>>,
66    /// Blockquote nesting depth at the point the table appeared
67    /// (0 = not inside a blockquote), mirroring `ParsedBlock::blockquote_depth`.
68    pub blockquote_depth: u32,
69    /// How many of those quotations open at the table, mirroring
70    /// `ParsedBlock::blockquote_opens`.
71    pub blockquote_opens: u32,
72}
73
74impl ParsedTable {
75    /// How many columns the table needs to hold every cell it has: as many as its widest
76    /// row has cells. A row shorter than that is completed with empty cells by whoever
77    /// builds the table.
78    ///
79    /// The first row does not say it. The rows of a Djot table differ in length when their
80    /// source does, and an HTML table's first row is often one heading cell spanning every
81    /// column: a table sized from its first row kept the cells of a longer row in the model,
82    /// where they showed, while every export writes a row up to the table's width only, so
83    /// the first save dropped them.
84    pub fn column_count(&self) -> usize {
85        self.rows.iter().map(Vec::len).max().unwrap_or(0)
86    }
87}
88
89/// How many cells a table may reach once its short rows are completed (see
90/// [`ParsedTable::column_count`]), however few it has of its own: a table this size loads
91/// in a moment.
92const SQUARED_CELLS_ALWAYS: usize = 4096;
93
94/// How many times the cells it has of its own a larger table may reach once its short
95/// rows are completed.
96///
97/// A table whose rows differ in length because its source wrote them so, a wide heading
98/// row over rows of one cell or a few, grows to about as many times its own cells as it has
99/// columns: a dozen or so for a table a writer makes. The tables that made hundreds of
100/// thousands of cells out of a few kilobytes grow hundreds of times. At four times, a table
101/// of ten columns heading five hundred rows of one cell was read as paragraphs, its grid
102/// lost for a table that loads in a few milliseconds.
103const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
104
105/// Push `table`, which has `own_cells` cells of its own in its source, onto `elements`: as
106/// a table, or as the paragraphs of its cells when completing its short rows would make it
107/// many times that.
108///
109/// Every row of a loaded table is completed to the widest one. A few kilobytes of a table
110/// whose first row is wide and whose other rows hold one cell each, or of HTML cells each
111/// spanning a thousand columns, made hundreds of thousands of empty cells, and held a load
112/// or a paste for seconds. Such a table is read as the paragraphs of its cells that hold
113/// anything, in reading order, at the table's quotation depth: every word stays, the grid
114/// goes.
115fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
116    let squared = table.rows.len().saturating_mul(table.column_count());
117    let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
118    if squared <= allowed {
119        elements.push(ParsedElement::Table(table));
120        return;
121    }
122    elements.extend(
123        ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
124            .into_iter()
125            .filter(|block| spans_carry_content(&block.spans))
126            .map(ParsedElement::Block),
127    );
128}
129
130/// Record that a quotation opened at the first block or table of `elements` from `first`
131/// on: the quotation that began where `elements` stood at `first` and closes now. Nothing
132/// when the quotation holds none.
133fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
134    let Some(rest) = elements.get_mut(first..) else {
135        return;
136    };
137    for element in rest {
138        match element {
139            ParsedElement::Block(block) => {
140                block.blockquote_opens += 1;
141                return;
142            }
143            ParsedElement::Table(table) => {
144                table.blockquote_opens += 1;
145                return;
146            }
147            ParsedElement::FootnoteDefinition { .. } => {}
148        }
149    }
150}
151
152/// The cells a table's rows hold, for [`push_table`].
153fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
154    rows.iter().map(Vec::len).sum()
155}
156
157/// A parsed element: either a block or a table.
158#[derive(Debug, Clone)]
159pub enum ParsedElement {
160    Block(ParsedBlock),
161    Table(ParsedTable),
162    /// A footnote definition: the label its references name, and the blocks
163    /// making up its body.
164    ///
165    /// Separate from `Block` because a definition is not part of the flow it was
166    /// written in — it belongs wherever the output format puts notes, which the
167    /// importer expresses by giving it a detached frame of its own.
168    FootnoteDefinition {
169        label: String,
170        blocks: Vec<ParsedBlock>,
171    },
172}
173
174impl ParsedElement {
175    /// Extract blocks, flattening tables into one block per cell.
176    /// Use when table structure is not needed.
177    pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
178        let mut blocks = Vec::new();
179        for elem in elements {
180            match elem {
181                ParsedElement::Block(b) => blocks.push(b),
182                // A definition is not part of the flow it was written in — it
183                // belongs wherever the format puts notes. Flattening it in
184                // would splice a note's body into the prose at the point the
185                // definition happened to be typed.
186                ParsedElement::FootnoteDefinition { .. } => {}
187                ParsedElement::Table(t) => {
188                    for row in t.rows {
189                        for cell in row {
190                            blocks.push(ParsedBlock {
191                                spans: cell.spans,
192                                heading_level: None,
193                                list_style: None,
194                                list_indent: 0,
195                                list_prefix: String::new(),
196                                list_suffix: String::new(),
197                                list_start: None,
198                                marker: None,
199                                is_code_block: false,
200                                code_language: None,
201                                blockquote_depth: t.blockquote_depth,
202                                blockquote_opens: 0,
203                                line_height: None,
204                                non_breakable_lines: None,
205                                page_break_before: None,
206                                front_matter: None,
207                                direction: None,
208                                background_color: None,
209                                alignment: None,
210                                top_margin: None,
211                                text_indent: None,
212                                semantic_role: None,
213                            });
214                        }
215                    }
216                }
217            }
218        }
219        if blocks.is_empty() {
220            blocks.push(ParsedBlock {
221                spans: vec![ParsedSpan {
222                    text: String::new(),
223                    ..Default::default()
224                }],
225                heading_level: None,
226                list_style: None,
227                list_indent: 0,
228                list_prefix: String::new(),
229                list_suffix: String::new(),
230                list_start: None,
231                marker: None,
232                is_code_block: false,
233                code_language: None,
234                blockquote_depth: 0,
235                blockquote_opens: 0,
236                line_height: None,
237                non_breakable_lines: None,
238                page_break_before: None,
239                front_matter: None,
240                direction: None,
241                background_color: None,
242                alignment: None,
243                top_margin: None,
244                text_indent: None,
245                semantic_role: None,
246            });
247        }
248        blocks
249    }
250}
251
252/// A parsed block (paragraph, heading, list item, code block)
253///
254/// `Default` is an unformatted, empty paragraph — the shape a caller wants when
255/// it has prose and no structure to give it. Deriving it also means a field
256/// added later reaches those callers as "absent" rather than as a compile error
257/// they would fix by guessing a value.
258#[derive(Debug, Clone, Default)]
259pub struct ParsedBlock {
260    pub spans: Vec<ParsedSpan>,
261    pub heading_level: Option<i64>,
262    pub list_style: Option<ListStyle>,
263    pub list_indent: u32,
264    /// Ordered-list delimiter prefix (e.g. `"("` for djot `(1)` lists; empty
265    /// otherwise). Stored on the `List` entity for round-trip fidelity.
266    pub list_prefix: String,
267    /// Ordered-list delimiter suffix (`"."` for `1.`, `")"` for `1)`/`(1)`;
268    /// empty for unordered lists). Markdown reads it from the source, Djot from
269    /// its list style, and HTML gives an ordered list
270    /// [`default_list_suffix`].
271    pub list_suffix: String,
272    /// The number the first item of the list this item belongs to wears, when it is not 1
273    /// (Djot and Markdown `3.`, HTML `<ol start="3">`). Every item of a list carries it.
274    /// Stored on the `List` entity.
275    pub list_start: Option<i64>,
276    /// Task-list checkbox marker (djot `- [ ]` / `- [x]`). Maps to
277    /// `Block.fmt_marker`. `None` for non-task blocks.
278    pub marker: Option<MarkerType>,
279    pub is_code_block: bool,
280    pub code_language: Option<String>,
281    pub blockquote_depth: u32,
282    /// How many of the quotations the block stands in open at it, the innermost ones:
283    /// quotations as deep as the element's before it but not the same ones. `0` when the
284    /// block goes on in the quotations the element before it stands in, as deep as it
285    /// stands. Two quotations one after the other (Djot and Markdown `> a` and `> b` with a
286    /// blank line between, HTML `<blockquote>` twice) were read as one, since the depth
287    /// alone does not tell them apart.
288    pub blockquote_opens: u32,
289    pub line_height: Option<i64>,
290    pub non_breakable_lines: Option<bool>,
291    /// Start this block on a new page (djot `{page_break_before=true}`). Maps to
292    /// `Block.fmt_page_break_before`. `None` when absent.
293    pub page_break_before: Option<bool>,
294    /// This block opens the document as front matter (djot `{front_matter=true}`). Maps to
295    /// `Block.fmt_front_matter`. `None` when absent.
296    pub front_matter: Option<bool>,
297    pub direction: Option<TextDirection>,
298    pub background_color: Option<String>,
299    /// Paragraph alignment (djot `{alignment=left|right|center|justify}`). Maps
300    /// to `Block.fmt_alignment`. `None` when no alignment attribute is present.
301    pub alignment: Option<Alignment>,
302    /// This block's own space-above (djot `{top_margin=<int>}`). Maps to
303    /// `Block.fmt_top_margin` and overrides the document-wide paragraph
304    /// spacing for this block alone. `None` when absent.
305    pub top_margin: Option<i64>,
306    /// This block's own first-line indent (djot `{text_indent=<int>}`). Maps to
307    /// `Block.fmt_text_indent` and overrides the document-wide first-line
308    /// indent for this block alone. `None` when absent.
309    pub text_indent: Option<i64>,
310    /// The enclosing blockquote's semantic role (djot `{semantic_role=epigraph}`),
311    /// written on the quote's first block because block attributes are the only channel
312    /// djot offers. The importer lifts it onto the `Frame`, where it belongs.
313    pub semantic_role: Option<SemanticRole>,
314}
315
316impl ParsedBlock {
317    /// Returns `true` when this block carries no block-level formatting,
318    /// meaning its content is purely inline.
319    pub fn is_inline_only(&self) -> bool {
320        self.heading_level.is_none()
321            && self.list_style.is_none()
322            && !self.is_code_block
323            && self.blockquote_depth == 0
324            && self.line_height.is_none()
325            && self.non_breakable_lines.is_none()
326            && self.page_break_before.is_none()
327            && self.front_matter.is_none()
328            && self.direction.is_none()
329            && self.background_color.is_none()
330            && self.alignment.is_none()
331            && self.top_margin.is_none()
332            && self.text_indent.is_none()
333    }
334}
335
336// ─── Markdown parsing ────────────────────────────────────────────────
337
338/// Labels referenced as `[^label]` in `markdown` with no `[^label]: ...`
339/// definition anywhere in the document.
340///
341/// pulldown-cmark's default footnote mode (`Options::ENABLE_FOOTNOTES`, i.e.
342/// GitHub's syntax) only emits `Event::FootnoteReference` for a label some
343/// `Tag::FootnoteDefinition` actually defines (verified against 0.13) — an
344/// undefined reference silently decomposes into three ordinary `Text` events
345/// (`"["`, `"^label"`, `"]"`) instead, indistinguishable from a reader having
346/// typed literal brackets. `Options::ENABLE_OLD_FOOTNOTES` recognises it, but
347/// as `parse_markdown`'s doc comment explains, trades away multi-paragraph
348/// definition bodies to get there. Neither flag alone gives both, so
349/// `parse_markdown` hands pulldown-cmark a synthetic empty definition for
350/// every such label instead — real enough for it to recognise the reference,
351/// thrown away again before `elements` is returned.
352///
353/// Finding "every such label" is two parts: which labels pulldown-cmark's own
354/// block scanner (a first pass, at the caller's `options`) already recognises
355/// as truly defined, and which `[^…]`-shaped spans exist in the raw text at
356/// all. The second part is a plain scan, not a parser — it can over-match
357/// (inside a code span, inside a fenced block, a coincidental `[^x]:` in the
358/// middle of a sentence that real block parsing would never treat as a
359/// definition), but over-matching only ever produces an unused synthetic
360/// definition: pulldown-cmark's real, correct parse of the augmented text is
361/// what decides whether any span actually becomes a reference, and
362/// `parse_markdown` drops every synthesized definition regardless of whether
363/// that happened.
364fn dangling_footnote_labels(
365    markdown: &str,
366    options: pulldown_cmark::Options,
367) -> std::collections::BTreeSet<String> {
368    use pulldown_cmark::{Event, Parser, Tag};
369
370    let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
371    for event in Parser::new_ext(markdown, options) {
372        if let Event::Start(Tag::FootnoteDefinition(label)) = event {
373            defined.insert(label.to_string());
374        }
375    }
376
377    let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
378    let bytes = markdown.as_bytes();
379    let mut search_from = 0usize;
380    while let Some(rel) = markdown[search_from..].find("[^") {
381        let open = search_from + rel;
382        let label_start = open + 2;
383        let Some(close_rel) = markdown[label_start..].find(']') else {
384            break;
385        };
386        let close = label_start + close_rel;
387        let label = &markdown[label_start..close];
388        // `]:` immediately after is definition syntax, not a reference —
389        // pulldown-cmark's own scan above already supplied the ground truth
390        // for which labels are truly defined; this only needs to avoid
391        // counting a definition's own label as a "reference" candidate.
392        let looks_like_definition = bytes.get(close + 1) == Some(&b':');
393        if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
394            referenced.insert(label.to_string());
395        }
396        search_from = close + 1;
397    }
398
399    referenced.difference(&defined).cloned().collect()
400}
401
402/// The delimiter the ordered list that opens at byte `start` of `source` wears: `")"` for
403/// `1)`, `"."` for `1.`.
404///
405/// pulldown-cmark reports a list as `Tag::List(Some(first))` and says nothing of the
406/// delimiter, but the range of that event starts at the first item's marker (or at the
407/// indentation before it, or after the `> ` of a quotation it stands in), never inside the
408/// item's content. So the line is read from `start`: blanks and `>` are skipped, then the
409/// digits, and the character after them is the delimiter. Anything else reads as `"."`,
410/// the delimiter CommonMark gives an ordered list by default.
411fn markdown_ordered_suffix(source: &str, start: usize) -> &'static str {
412    let Some(rest) = source.get(start..) else {
413        return ".";
414    };
415    let line = rest.split('\n').next().unwrap_or("");
416    let after_blanks = line.trim_start_matches([' ', '\t', '>']);
417    let after_digits = after_blanks.trim_start_matches(|c: char| c.is_ascii_digit());
418    if after_digits.starts_with(')') {
419        ")"
420    } else {
421        "."
422    }
423}
424
425/// Read `markdown` into blocks and tables.
426///
427/// The parser reads nested containers without recursing, whatever their depth. The
428/// quotations the text holds are kept within
429/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
430/// `quote_depth`): the writers walk one call deeper for each, and a text nested 500
431/// quotations deep loaded, then aborted the process at its first save.
432pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
433    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
434
435    // ENABLE_FOOTNOTES is required for pulldown-cmark to parse `[^label]`
436    // references and `[^label]: body` definitions at all — without it both
437    // fall through as plain link-reference-shaped text. Deliberately NOT
438    // `ENABLE_OLD_FOOTNOTES` too: that variant makes a reference with no
439    // definition survive (see `dangling_footnote_labels`), but in exchange
440    // breaks multi-paragraph definition bodies — a blank line before an
441    // indented continuation escapes the definition and becomes a sibling
442    // code block, which is exactly the shape this crate's own Markdown
443    // exporter writes for a note with more than one paragraph. Getting both
444    // is `dangling_footnote_labels`'s job, not an `Options` flag's.
445    let options = Options::ENABLE_STRIKETHROUGH
446        | Options::ENABLE_TABLES
447        | Options::ENABLE_TASKLISTS
448        | Options::ENABLE_FOOTNOTES;
449
450    // See `dangling_footnote_labels`: pulldown-cmark's GFM-style footnotes
451    // only recognise `[^label]` as a reference when some definition exists
452    // for it anywhere in the document, so an undefined one — the normal
453    // state for a host that owns note bodies itself — has to be given a
454    // throwaway definition before parsing, or it silently becomes the
455    // literal text "[^label]" instead of surviving as a reference.
456    let dangling = dangling_footnote_labels(markdown, options);
457    let augmented_owner;
458    let source: &str = if dangling.is_empty() {
459        markdown
460    } else {
461        augmented_owner = dangling
462            .iter()
463            .fold(markdown.to_string(), |mut acc, label| {
464                acc.push_str("\n\n[^");
465                acc.push_str(label);
466                acc.push_str("]:\n");
467                acc
468            });
469        &augmented_owner
470    };
471    let parser = Parser::new_ext(source, options).into_offset_iter();
472
473    let mut elements: Vec<ParsedElement> = Vec::new();
474    let mut current_spans: Vec<ParsedSpan> = Vec::new();
475    let mut current_heading: Option<i64> = None;
476    let mut current_list_style: Option<ListStyle> = None;
477    let mut current_list_start: Option<i64> = None;
478    let mut current_list_suffix = String::new();
479    // The task-list checkbox of the item being read, which only its first block wears.
480    let mut current_marker: Option<MarkerType> = None;
481    let mut is_code_block = false;
482    let mut code_language: Option<String> = None;
483    let mut blockquote_depth: u32 = 0;
484    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
485    let mut quotations_open: Vec<usize> = Vec::new();
486    let mut in_block = false;
487
488    // Formatting state stack
489    let mut bold = false;
490    let mut italic = false;
491    let mut strikeout = false;
492    let mut link_href: Option<String> = None;
493    // Set between an image's Start and End; its alt text arrives as Text events.
494    let mut pending_image: Option<ParsedImage> = None;
495
496    // The label and element index of the footnote definition currently open,
497    // if any. Mirrors `parse_djot`'s `footnote_open`: a definition's body is
498    // ordinary block content, parsed by the same machinery as everything
499    // else, then lifted back out at `End` into its own top-level element so
500    // it never becomes part of the flow it was written in. CommonMark
501    // footnote definitions cannot nest, so one slot suffices.
502    let mut footnote_open: Option<(String, usize)> = None;
503
504    // List style stack for nested lists (also tracks nesting depth), and the start of each
505    // of those lists beside it.
506    let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
507    let mut list_start_stack: Vec<Option<i64>> = Vec::new();
508    // The delimiter each of those lists wears (`"."` or `")"`; empty for a bullet list).
509    let mut list_suffix_stack: Vec<String> = Vec::new();
510    let mut current_list_indent: u32 = 0;
511
512    // Table tracking state
513    let mut in_table = false;
514    let mut in_table_head = false;
515    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
516    let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
517    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
518    let mut table_header_rows: usize = 0;
519
520    for (event, range) in parser {
521        match event {
522            Event::Start(Tag::Paragraph) => {
523                in_block = true;
524                current_heading = None;
525                is_code_block = false;
526            }
527            Event::End(TagEnd::Paragraph) => {
528                if !current_spans.is_empty() || in_block {
529                    elements.push(ParsedElement::Block(ParsedBlock {
530                        spans: std::mem::take(&mut current_spans),
531                        heading_level: current_heading.take(),
532                        list_style: current_list_style.clone(),
533                        list_indent: current_list_indent,
534                        list_prefix: String::new(),
535                        list_suffix: current_list_suffix.clone(),
536                        list_start: current_list_start,
537                        marker: current_marker.clone(),
538                        is_code_block: false,
539                        code_language: None,
540                        blockquote_depth,
541                        blockquote_opens: 0,
542                        line_height: None,
543                        non_breakable_lines: None,
544                        page_break_before: None,
545                        front_matter: None,
546                        direction: None,
547                        background_color: None,
548                        alignment: None,
549                        top_margin: None,
550                        text_indent: None,
551                        semantic_role: None,
552                    }));
553                }
554                in_block = false;
555                current_list_style = None;
556                current_list_start = None;
557                current_list_suffix.clear();
558                current_marker = None;
559            }
560            Event::Start(Tag::Heading { level, .. }) => {
561                in_block = true;
562                current_heading = Some(heading_level_to_i64(level));
563                is_code_block = false;
564            }
565            Event::End(TagEnd::Heading(_)) => {
566                elements.push(ParsedElement::Block(ParsedBlock {
567                    spans: std::mem::take(&mut current_spans),
568                    heading_level: current_heading.take(),
569                    list_style: None,
570                    list_indent: 0,
571                    list_prefix: String::new(),
572                    list_suffix: String::new(),
573                    list_start: None,
574                    marker: None,
575                    is_code_block: false,
576                    code_language: None,
577                    blockquote_depth,
578                    blockquote_opens: 0,
579                    line_height: None,
580                    non_breakable_lines: None,
581                    page_break_before: None,
582                    front_matter: None,
583                    direction: None,
584                    background_color: None,
585                    alignment: None,
586                    top_margin: None,
587                    text_indent: None,
588                    semantic_role: None,
589                }));
590                in_block = false;
591            }
592            Event::Start(Tag::List(ordered)) => {
593                let style = if ordered.is_some() {
594                    Some(ListStyle::Decimal)
595                } else {
596                    Some(ListStyle::Disc)
597                };
598                list_stack.push(style);
599                list_start_stack.push(ordered.and_then(list_start_of));
600                list_suffix_stack.push(if ordered.is_some() {
601                    markdown_ordered_suffix(source, range.start).to_string()
602                } else {
603                    String::new()
604                });
605            }
606            Event::TaskListMarker(checked) => {
607                current_marker = Some(if checked {
608                    MarkerType::Checked
609                } else {
610                    MarkerType::Unchecked
611                });
612            }
613            Event::End(TagEnd::List(_)) => {
614                list_stack.pop();
615                list_start_stack.pop();
616                list_suffix_stack.pop();
617            }
618            Event::Start(Tag::Item) => {
619                // Flush any accumulated spans from the parent item before
620                // starting a child item in a tight list
621                if !current_spans.is_empty() {
622                    elements.push(ParsedElement::Block(ParsedBlock {
623                        spans: std::mem::take(&mut current_spans),
624                        heading_level: None,
625                        list_style: current_list_style.clone(),
626                        list_indent: current_list_indent,
627                        list_prefix: String::new(),
628                        list_suffix: current_list_suffix.clone(),
629                        list_start: current_list_start,
630                        marker: current_marker.clone(),
631                        is_code_block: false,
632                        code_language: None,
633                        blockquote_depth,
634                        blockquote_opens: 0,
635                        line_height: None,
636                        non_breakable_lines: None,
637                        page_break_before: None,
638                        front_matter: None,
639                        direction: None,
640                        background_color: None,
641                        alignment: None,
642                        top_margin: None,
643                        text_indent: None,
644                        semantic_role: None,
645                    }));
646                }
647                in_block = true;
648                current_list_style = list_stack.last().cloned().flatten();
649                current_list_start = list_start_stack.last().copied().flatten();
650                current_list_suffix = list_suffix_stack.last().cloned().unwrap_or_default();
651                // Set after the flush above, which wrote the parent item with its own box.
652                current_marker = None;
653                current_list_indent = if list_stack.is_empty() {
654                    0
655                } else {
656                    (list_stack.len() - 1) as u32
657                };
658            }
659            Event::End(TagEnd::Item) => {
660                // The paragraph inside the item will have already been flushed,
661                // but if there was no inner paragraph (tight list), flush now.
662                if !current_spans.is_empty() {
663                    elements.push(ParsedElement::Block(ParsedBlock {
664                        spans: std::mem::take(&mut current_spans),
665                        heading_level: None,
666                        list_style: current_list_style.clone(),
667                        list_indent: current_list_indent,
668                        list_prefix: String::new(),
669                        list_suffix: current_list_suffix.clone(),
670                        list_start: current_list_start,
671                        marker: current_marker.clone(),
672                        is_code_block: false,
673                        code_language: None,
674                        blockquote_depth,
675                        blockquote_opens: 0,
676                        line_height: None,
677                        non_breakable_lines: None,
678                        page_break_before: None,
679                        front_matter: None,
680                        direction: None,
681                        background_color: None,
682                        alignment: None,
683                        top_margin: None,
684                        text_indent: None,
685                        semantic_role: None,
686                    }));
687                }
688                in_block = false;
689                current_list_style = None;
690                current_list_start = None;
691                current_list_suffix.clear();
692                current_marker = None;
693            }
694            Event::Start(Tag::CodeBlock(kind)) => {
695                in_block = true;
696                is_code_block = true;
697                code_language = match &kind {
698                    pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
699                        Some(lang.to_string())
700                    }
701                    _ => None,
702                };
703            }
704            Event::End(TagEnd::CodeBlock) => {
705                // pulldown-cmark appends a trailing '\n' to code block text — strip it
706                if let Some(last) = current_spans.last_mut()
707                    && last.text.ends_with('\n')
708                {
709                    last.text.truncate(last.text.len() - 1);
710                }
711                elements.push(ParsedElement::Block(ParsedBlock {
712                    spans: std::mem::take(&mut current_spans),
713                    heading_level: None,
714                    list_style: None,
715                    list_indent: 0,
716                    list_prefix: String::new(),
717                    list_suffix: String::new(),
718                    list_start: None,
719                    marker: None,
720                    is_code_block: true,
721                    code_language: code_language.take(),
722                    blockquote_depth,
723                    blockquote_opens: 0,
724                    line_height: None,
725                    non_breakable_lines: None,
726                    page_break_before: None,
727                    front_matter: None,
728                    direction: None,
729                    background_color: None,
730                    alignment: None,
731                    top_margin: None,
732                    text_indent: None,
733                    semantic_role: None,
734                }));
735                in_block = false;
736                is_code_block = false;
737            }
738            // ─── Table events ───────────────────────────────────────
739            Event::Start(Tag::Table(_)) => {
740                in_table = true;
741                in_table_head = false;
742                table_rows.clear();
743                current_row_cells.clear();
744                current_cell_spans.clear();
745                table_header_rows = 0;
746            }
747            Event::End(TagEnd::Table) => {
748                let own_cells = cells_in_rows(&table_rows);
749                push_table(
750                    &mut elements,
751                    ParsedTable {
752                        header_rows: table_header_rows,
753                        rows: std::mem::take(&mut table_rows),
754                        blockquote_depth,
755                        blockquote_opens: 0,
756                    },
757                    own_cells,
758                );
759                in_table = false;
760            }
761            Event::Start(Tag::TableHead) => {
762                in_table_head = true;
763                current_row_cells.clear();
764            }
765            Event::End(TagEnd::TableHead) => {
766                // Flush the header row
767                table_rows.push(std::mem::take(&mut current_row_cells));
768                table_header_rows += 1;
769                in_table_head = false;
770            }
771            Event::Start(Tag::TableRow) => {
772                current_row_cells.clear();
773            }
774            Event::End(TagEnd::TableRow) if !in_table_head => {
775                // Body rows only — header row is flushed in End(TableHead)
776                table_rows.push(std::mem::take(&mut current_row_cells));
777            }
778            Event::Start(Tag::TableCell) => {
779                current_cell_spans.clear();
780            }
781            Event::End(TagEnd::TableCell) => {
782                current_row_cells.push(ParsedTableCell {
783                    spans: std::mem::take(&mut current_cell_spans),
784                });
785            }
786            // ─── Inline formatting ──────────────────────────────────
787            Event::Start(Tag::Emphasis) => {
788                italic = true;
789            }
790            Event::End(TagEnd::Emphasis) => {
791                italic = false;
792            }
793            Event::Start(Tag::Strong) => {
794                bold = true;
795            }
796            Event::End(TagEnd::Strong) => {
797                bold = false;
798            }
799            Event::Start(Tag::Strikethrough) => {
800                strikeout = true;
801            }
802            Event::End(TagEnd::Strikethrough) => {
803                strikeout = false;
804            }
805            Event::Start(Tag::Link { dest_url, .. }) => {
806                link_href = Some(dest_url.to_string());
807            }
808            Event::End(TagEnd::Link) => {
809                link_href = None;
810            }
811            // Markdown has no syntax for display size, so width/height stay 0
812            // and the caller supplies the image's intrinsic dimensions.
813            Event::Start(Tag::Image { dest_url, .. }) => {
814                pending_image = Some(ParsedImage {
815                    src: dest_url.to_string(),
816                    alt: String::new(),
817                    width: 0,
818                    height: 0,
819                });
820            }
821            Event::End(TagEnd::Image) => {
822                if let Some(image) = pending_image.take() {
823                    let span = ParsedSpan {
824                        text: String::new(),
825                        bold,
826                        italic,
827                        underline: false,
828                        strikeout,
829                        code: false,
830                        superscript: false,
831                        subscript: false,
832                        link_href: link_href.clone(),
833                        image: Some(image),
834                        footnote_ref: None,
835                        preserve_whitespace: false,
836                    };
837                    if in_table {
838                        current_cell_spans.push(span);
839                    } else {
840                        if !in_block {
841                            in_block = true;
842                        }
843                        current_spans.push(span);
844                    }
845                }
846            }
847            Event::Text(text) => {
848                // Inside an image this is its alt text, which pulldown-cmark
849                // emits as an ordinary Text event. Without this guard it fell
850                // through and landed in the paragraph as prose — the image was
851                // dropped and its description silently became manuscript text.
852                if let Some(img) = pending_image.as_mut() {
853                    img.alt.push_str(&text);
854                    continue;
855                }
856                let span = ParsedSpan {
857                    text: text.to_string(),
858                    bold,
859                    italic,
860                    underline: false,
861                    strikeout,
862                    code: is_code_block,
863                    superscript: false,
864                    subscript: false,
865                    link_href: link_href.clone(),
866                    image: None,
867                    footnote_ref: None,
868                    preserve_whitespace: false,
869                };
870                if in_table {
871                    current_cell_spans.push(span);
872                } else {
873                    if !in_block {
874                        in_block = true;
875                    }
876                    current_spans.push(span);
877                }
878            }
879            Event::Code(text) => {
880                let span = ParsedSpan {
881                    text: text.to_string(),
882                    bold,
883                    italic,
884                    underline: false,
885                    strikeout,
886                    code: true,
887                    superscript: false,
888                    subscript: false,
889                    link_href: link_href.clone(),
890                    image: None,
891                    footnote_ref: None,
892                    preserve_whitespace: false,
893                };
894                if in_table {
895                    current_cell_spans.push(span);
896                } else {
897                    if !in_block {
898                        in_block = true;
899                    }
900                    current_spans.push(span);
901                }
902            }
903            Event::SoftBreak => {
904                let span = ParsedSpan {
905                    text: " ".to_string(),
906                    bold,
907                    italic,
908                    underline: false,
909                    strikeout,
910                    code: false,
911                    superscript: false,
912                    subscript: false,
913                    link_href: link_href.clone(),
914                    image: None,
915                    footnote_ref: None,
916                    preserve_whitespace: false,
917                };
918                if in_table {
919                    current_cell_spans.push(span);
920                } else {
921                    current_spans.push(span);
922                }
923            }
924            // A hard break (two spaces or a backslash at the end of the line) is a line
925            // of the same block, as in the Djot reader. A table cell is one line, and so
926            // is an image's description.
927            Event::HardBreak if pending_image.is_some() => {
928                if let Some(img) = pending_image.as_mut() {
929                    img.alt.push(' ');
930                }
931            }
932            Event::HardBreak => {
933                let span = ParsedSpan {
934                    text: if in_table { " " } else { LINE_SEPARATOR_STR }.to_string(),
935                    bold,
936                    italic,
937                    underline: false,
938                    strikeout,
939                    code: false,
940                    superscript: false,
941                    subscript: false,
942                    link_href: link_href.clone(),
943                    image: None,
944                    footnote_ref: None,
945                    preserve_whitespace: false,
946                };
947                if in_table {
948                    current_cell_spans.push(span);
949                } else {
950                    current_spans.push(span);
951                }
952            }
953            Event::Start(Tag::BlockQuote(_)) => {
954                blockquote_depth += 1;
955                quotations_open.push(elements.len());
956            }
957            Event::End(TagEnd::BlockQuote(_)) => {
958                blockquote_depth = blockquote_depth.saturating_sub(1);
959                if let Some(first) = quotations_open.pop() {
960                    mark_quotation_opened(&mut elements, first);
961                }
962            }
963            // ── Footnote definitions ──
964            //
965            // `[^label]: body` opens a block-level container; its body is
966            // ordinary paragraphs/lists/etc., pushed onto `elements` by the
967            // ordinary machinery above. `End` drains everything pushed since
968            // `Start` back out into one `FootnoteDefinition`, exactly as
969            // `parse_djot` does for the same `[^label]: body` syntax.
970            // Flushing any spans left open by an interrupted block first
971            // matches every other container boundary in this parser
972            // (List/Item/BlockQuote).
973            Event::Start(Tag::FootnoteDefinition(label)) => {
974                if !current_spans.is_empty() {
975                    elements.push(ParsedElement::Block(ParsedBlock {
976                        spans: std::mem::take(&mut current_spans),
977                        heading_level: current_heading.take(),
978                        list_style: current_list_style.clone(),
979                        list_indent: current_list_indent,
980                        list_prefix: String::new(),
981                        list_suffix: current_list_suffix.clone(),
982                        list_start: current_list_start,
983                        marker: current_marker.clone(),
984                        is_code_block: false,
985                        code_language: None,
986                        blockquote_depth,
987                        blockquote_opens: 0,
988                        line_height: None,
989                        non_breakable_lines: None,
990                        page_break_before: None,
991                        front_matter: None,
992                        direction: None,
993                        background_color: None,
994                        alignment: None,
995                        top_margin: None,
996                        text_indent: None,
997                        semantic_role: None,
998                    }));
999                }
1000                footnote_open = Some((label.to_string(), elements.len()));
1001            }
1002            Event::End(TagEnd::FootnoteDefinition) => {
1003                if !current_spans.is_empty() {
1004                    elements.push(ParsedElement::Block(ParsedBlock {
1005                        spans: std::mem::take(&mut current_spans),
1006                        heading_level: current_heading.take(),
1007                        list_style: current_list_style.clone(),
1008                        list_indent: current_list_indent,
1009                        list_prefix: String::new(),
1010                        list_suffix: current_list_suffix.clone(),
1011                        list_start: current_list_start,
1012                        marker: current_marker.clone(),
1013                        is_code_block: false,
1014                        code_language: None,
1015                        blockquote_depth,
1016                        blockquote_opens: 0,
1017                        line_height: None,
1018                        non_breakable_lines: None,
1019                        page_break_before: None,
1020                        front_matter: None,
1021                        direction: None,
1022                        background_color: None,
1023                        alignment: None,
1024                        top_margin: None,
1025                        text_indent: None,
1026                        semantic_role: None,
1027                    }));
1028                }
1029                if let Some((label, start)) = footnote_open.take() {
1030                    let blocks: Vec<ParsedBlock> = elements
1031                        .drain(start..)
1032                        .filter_map(|e| match e {
1033                            ParsedElement::Block(b) => Some(b),
1034                            // A table inside a footnote is not representable
1035                            // as note content, same limitation as djot.
1036                            _ => None,
1037                        })
1038                        .collect();
1039                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
1040                }
1041            }
1042            // A footnote reference. pulldown-cmark emits this whether or not
1043            // a matching `[^label]:` definition exists anywhere in the
1044            // document — a dangling reference (the normal state for a host
1045            // that owns note bodies itself) must survive just the same,
1046            // mirroring `parse_djot`'s `E::FootnoteReference` handling.
1047            Event::FootnoteReference(label) => {
1048                let span = ParsedSpan {
1049                    text: String::new(),
1050                    bold,
1051                    italic,
1052                    underline: false,
1053                    strikeout,
1054                    code: false,
1055                    superscript: false,
1056                    subscript: false,
1057                    link_href: link_href.clone(),
1058                    image: None,
1059                    footnote_ref: Some(label.to_string()),
1060                    preserve_whitespace: false,
1061                };
1062                if in_table {
1063                    current_cell_spans.push(span);
1064                } else {
1065                    if !in_block {
1066                        in_block = true;
1067                    }
1068                    current_spans.push(span);
1069                }
1070            }
1071            _ => {}
1072        }
1073    }
1074
1075    // Flush any remaining content
1076    if !current_spans.is_empty() {
1077        elements.push(ParsedElement::Block(ParsedBlock {
1078            spans: std::mem::take(&mut current_spans),
1079            heading_level: current_heading,
1080            list_style: current_list_style,
1081            list_indent: current_list_indent,
1082            list_prefix: String::new(),
1083            list_suffix: current_list_suffix.clone(),
1084            list_start: current_list_start,
1085            marker: current_marker.clone(),
1086            is_code_block,
1087            code_language: code_language.take(),
1088            blockquote_depth,
1089            blockquote_opens: 0,
1090            line_height: None,
1091            non_breakable_lines: None,
1092            page_break_before: None,
1093            front_matter: None,
1094            direction: None,
1095            background_color: None,
1096            alignment: None,
1097            top_margin: None,
1098            text_indent: None,
1099            semantic_role: None,
1100        }));
1101    }
1102
1103    // Drop the throwaway definitions `dangling_footnote_labels` asked for —
1104    // they exist only to make pulldown-cmark recognise the reference as
1105    // real, never to become a note the document owns. `elements` cannot
1106    // become empty from this: a label reached `dangling` only via a
1107    // reference actually present in `markdown`, so its surrounding block
1108    // survives even with the synthetic definition gone.
1109    if !dangling.is_empty() {
1110        elements.retain(
1111            |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1112        );
1113    }
1114
1115    // If no elements were parsed, create a single empty paragraph
1116    if elements.is_empty() {
1117        elements.push(ParsedElement::Block(ParsedBlock {
1118            spans: vec![ParsedSpan {
1119                text: String::new(),
1120                ..Default::default()
1121            }],
1122            heading_level: None,
1123            list_style: None,
1124            list_indent: 0,
1125            list_prefix: String::new(),
1126            list_suffix: String::new(),
1127            list_start: None,
1128            marker: None,
1129            is_code_block: false,
1130            code_language: None,
1131            blockquote_depth: 0,
1132            blockquote_opens: 0,
1133            line_height: None,
1134            non_breakable_lines: None,
1135            page_break_before: None,
1136            front_matter: None,
1137            direction: None,
1138            background_color: None,
1139            alignment: None,
1140            top_margin: None,
1141            text_indent: None,
1142            semantic_role: None,
1143        }));
1144    }
1145
1146    // Quotations past the ceiling a document holds are flattened into the deepest kept
1147    // (see `quote_depth`): the writers walk one call deeper for each.
1148    crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1149    elements
1150}
1151
1152fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1153    use pulldown_cmark::HeadingLevel;
1154    match level {
1155        HeadingLevel::H1 => 1,
1156        HeadingLevel::H2 => 2,
1157        HeadingLevel::H3 => 3,
1158        HeadingLevel::H4 => 4,
1159        HeadingLevel::H5 => 5,
1160        HeadingLevel::H6 => 6,
1161    }
1162}
1163
1164// ─── HTML parsing ────────────────────────────────────────────────────
1165
1166use scraper::Node;
1167
1168/// Parsed CSS block-level styles from an inline `style` attribute.
1169#[derive(Debug, Clone, Default)]
1170struct BlockStyles {
1171    line_height: Option<i64>,
1172    non_breakable_lines: Option<bool>,
1173    page_break_before: Option<bool>,
1174    direction: Option<TextDirection>,
1175    background_color: Option<String>,
1176    /// `white-space` asks for the source's own spacing to be kept verbatim.
1177    /// Separate from `non_breakable_lines` because the two do not coincide:
1178    /// `nowrap` forbids wrapping but still collapses runs of spaces, so a
1179    /// block carrying it must be normalised like any other.
1180    preserve_whitespace: Option<bool>,
1181}
1182
1183/// Parse relevant CSS properties from an inline style string.
1184/// Handles: line-height, white-space, break-before/page-break-before, direction,
1185/// background-color.
1186fn parse_block_styles(style: &str) -> BlockStyles {
1187    let mut result = BlockStyles::default();
1188    for part in style.split(';') {
1189        let part = part.trim();
1190        if let Some((prop, val)) = part.split_once(':') {
1191            let prop = prop.trim().to_ascii_lowercase();
1192            let val = val.trim();
1193            match prop.as_str() {
1194                "line-height" => {
1195                    // Try parsing as a plain number (multiplier)
1196                    if let Ok(v) = val.parse::<f64>() {
1197                        result.line_height = Some((v * 1000.0) as i64);
1198                    }
1199                }
1200                "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1201                    result.non_breakable_lines = Some(true);
1202                    if val != "nowrap" {
1203                        result.preserve_whitespace = Some(true);
1204                    }
1205                }
1206                // CSS3 `break-before` and its CSS2 predecessor `page-break-before` mean
1207                // the same thing; both are read because both are written (browsers still
1208                // want the legacy spelling, and so do most EPUB engines).
1209                "break-before" | "page-break-before" => {
1210                    result.page_break_before = match val.to_ascii_lowercase().as_str() {
1211                        "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1212                        "avoid" | "auto" => Some(false),
1213                        _ => None,
1214                    };
1215                }
1216                "direction" => {
1217                    if val.eq_ignore_ascii_case("rtl") {
1218                        result.direction = Some(TextDirection::RightToLeft);
1219                    } else if val.eq_ignore_ascii_case("ltr") {
1220                        result.direction = Some(TextDirection::LeftToRight);
1221                    }
1222                }
1223                "background-color" | "background" => {
1224                    result.background_color = Some(val.to_string());
1225                }
1226                _ => {}
1227            }
1228        }
1229    }
1230    result
1231}
1232
1233/// What an inline `style` says of the whitespace in an element's text: `Some(true)` when
1234/// `white-space` keeps it as it is (`pre`, `pre-wrap`, `break-spaces`), `Some(false)` when
1235/// it collapses it (`normal`, `nowrap`, `pre-line`), `None` when the style does not say.
1236/// The last declaration wins, as in CSS, unless an earlier one is `!important` and it is
1237/// not.
1238fn style_keeps_whitespace(style: &str) -> Option<bool> {
1239    let mut kept = None;
1240    let mut important_kept = false;
1241    for part in style.split(';') {
1242        let Some((property, value)) = part.split_once(':') else {
1243            continue;
1244        };
1245        if !property.trim().eq_ignore_ascii_case("white-space") {
1246            continue;
1247        }
1248        let value = value.trim().to_ascii_lowercase();
1249        let (value, important) = value
1250            .strip_suffix("important")
1251            .and_then(|rest| rest.trim_end().strip_suffix('!'))
1252            .map_or((value.as_str(), false), |rest| (rest.trim_end(), true));
1253        let says = match value {
1254            "pre" | "pre-wrap" | "break-spaces" => true,
1255            "normal" | "nowrap" | "pre-line" => false,
1256            _ => continue,
1257        };
1258        if important || !important_kept {
1259            kept = Some(says);
1260            important_kept |= important;
1261        }
1262    }
1263    kept
1264}
1265
1266/// The `white-space` declarations of a document's `<style>` blocks that this reader honours:
1267/// the rules whose every selector is `*` or a bare tag name (`p, li { white-space: pre-wrap; }`
1268/// is what `QTextDocument::toHtml` writes). Not a CSS engine: a class, an id, an attribute, a
1269/// pseudo-class, a combinator, and everything inside an at-rule, is skipped.
1270///
1271/// Resolved once, when the sheet is read, into the one declaration that wins for each tag
1272/// the sheet names and the one that wins for `*`: a lookup for an element then builds nothing.
1273/// The declaration is put in front of the element's inline `style` attribute, so the one
1274/// precedence function, [`style_keeps_whitespace`], decides between the sheet and an element's
1275/// own `style` as it does between two declarations of one attribute.
1276#[derive(Debug, Default)]
1277struct SheetWhitespace {
1278    /// What every element gets (`*` rules): the winning keyword and whether it is `!important`.
1279    universal: Option<(&'static str, bool)>,
1280    /// What a tag the sheet names gets: the winner among the `*` rules and then the rules
1281    /// naming it, each in source order, so that a rule naming the tag beats `*` as its
1282    /// specificity says.
1283    per_tag: std::collections::HashMap<String, (&'static str, bool)>,
1284}
1285
1286/// One rule of a sheet: the tags it names (`*` for every element) and its `white-space`
1287/// declarations as `(keyword, important)`.
1288struct WhitespaceRule {
1289    selectors: Vec<String>,
1290    declarations: Vec<(&'static str, bool)>,
1291}
1292
1293/// The declaration that wins when `next` follows `current` in the cascade: the later one,
1294/// unless `current` is `!important` and `next` is not.
1295fn cascade(
1296    current: Option<(&'static str, bool)>,
1297    next: (&'static str, bool),
1298) -> Option<(&'static str, bool)> {
1299    match current {
1300        Some((_, true)) if !next.1 => current,
1301        _ => Some(next),
1302    }
1303}
1304
1305impl SheetWhitespace {
1306    /// The sheet of `css`, in one pass over its rules: time and memory in proportion to the
1307    /// rules and their selectors, whatever the sheet holds.
1308    fn from_css(css: &str) -> Self {
1309        let rules = parse_whitespace_rules(css);
1310        let mut sheet = SheetWhitespace::default();
1311        for rule in rules
1312            .iter()
1313            .filter(|r| r.selectors.iter().any(|s| s == "*"))
1314        {
1315            for declaration in &rule.declarations {
1316                sheet.universal = cascade(sheet.universal, *declaration);
1317            }
1318        }
1319        for rule in &rules {
1320            for tag in rule.selectors.iter().filter(|s| *s != "*") {
1321                let winner = sheet
1322                    .per_tag
1323                    .entry(tag.clone())
1324                    .or_insert_with(|| sheet.universal.unwrap_or(("", false)));
1325                let mut current = (!winner.0.is_empty()).then_some(*winner);
1326                for declaration in &rule.declarations {
1327                    current = cascade(current, *declaration);
1328                }
1329                if let Some(resolved) = current {
1330                    *winner = resolved;
1331                }
1332            }
1333        }
1334        sheet.per_tag.retain(|_, (keyword, _)| !keyword.is_empty());
1335        sheet
1336    }
1337
1338    /// The declaration that applies to an element named `tag`, `None` when none does.
1339    fn declaration_for(&self, tag: &str) -> Option<(&'static str, bool)> {
1340        self.per_tag.get(tag).copied().or(self.universal)
1341    }
1342
1343    fn is_empty(&self) -> bool {
1344        self.universal.is_none() && self.per_tag.is_empty()
1345    }
1346
1347    /// Whether the sheet keeps the whitespace of the document's body, which every element
1348    /// without a rule of its own inherits. A sheet cannot style the `<html>` and `<body>`
1349    /// elements of a fragment (the HTML parser drops them), so what they say is the state
1350    /// the walk starts from; `body` beats `html`, as the inheritance goes.
1351    fn initial_keeps_whitespace(&self) -> Option<bool> {
1352        let of = |tag: &str| {
1353            self.declaration_for(tag).and_then(|(keyword, important)| {
1354                style_keeps_whitespace(&format!(
1355                    "white-space: {keyword}{};",
1356                    if important { " !important" } else { "" }
1357                ))
1358            })
1359        };
1360        of("body").or_else(|| of("html"))
1361    }
1362}
1363
1364/// The `white-space` rules of `css` that [`SheetWhitespace`] honours.
1365///
1366/// One pass that reads the text as bytes and cuts it only at ASCII delimiters, so that
1367/// nothing in it, an unterminated comment, string or block, a stray `}` or text that is not
1368/// ASCII, can panic or loop.
1369fn parse_whitespace_rules(css: &str) -> Vec<WhitespaceRule> {
1370    // Comments, and the `<!--` and `-->` a stylesheet inside HTML is wrapped in (Word and
1371    // Outlook do), which browsers ignore as tokens.
1372    let mut text = String::with_capacity(css.len());
1373    let mut rest = css;
1374    while let Some(start) = rest.find("/*") {
1375        text.push_str(&rest[..start]);
1376        text.push(' ');
1377        match rest[start + 2..].find("*/") {
1378            Some(end) => rest = &rest[start + 2 + end + 2..],
1379            None => {
1380                rest = "";
1381                break;
1382            }
1383        }
1384    }
1385    text.push_str(rest);
1386    let text = text.replace("<!--", " ").replace("-->", " ");
1387    let bytes = text.as_bytes();
1388
1389    // The first of `stops` at or after `from` outside a string, or the end.
1390    let scan = |from: usize, stops: &[u8]| -> usize {
1391        let mut i = from;
1392        while i < bytes.len() {
1393            match bytes[i] {
1394                b'\\' => i += 1,
1395                quote @ (b'"' | b'\'') => {
1396                    i += 1;
1397                    while i < bytes.len() && bytes[i] != quote {
1398                        if bytes[i] == b'\\' {
1399                            i += 1;
1400                        }
1401                        i += 1;
1402                    }
1403                }
1404                b if stops.contains(&b) => return i,
1405                _ => {}
1406            }
1407            i += 1;
1408        }
1409        bytes.len()
1410    };
1411    // The `}` closing the block whose `{` is at `open`, counting nested braces, or the end.
1412    let block_end = |open: usize| -> usize {
1413        let mut depth = 0usize;
1414        let mut i = open;
1415        while i < bytes.len() {
1416            i = scan(i, b"{}");
1417            match bytes.get(i) {
1418                Some(b'{') => depth += 1,
1419                Some(b'}') => {
1420                    depth = depth.saturating_sub(1);
1421                    if depth == 0 {
1422                        return i;
1423                    }
1424                }
1425                _ => return bytes.len(),
1426            }
1427            i += 1;
1428        }
1429        bytes.len()
1430    };
1431
1432    let mut rules = Vec::new();
1433    let mut pos = 0usize;
1434    while pos < bytes.len() {
1435        let prelude_end = scan(pos, b"{;}");
1436        let prelude = text[pos..prelude_end].trim();
1437        match bytes.get(prelude_end) {
1438            // A statement (`@import url(x);`) or a stray `}`: nothing to keep.
1439            Some(b';') | Some(b'}') => pos = prelude_end + 1,
1440            Some(b'{') => {
1441                let close = block_end(prelude_end);
1442                if !prelude.starts_with('@') {
1443                    let body = &text[prelude_end + 1..close];
1444                    if let Some(rule) = whitespace_rule(prelude, body) {
1445                        rules.push(rule);
1446                    }
1447                }
1448                pos = close + 1;
1449            }
1450            _ => break,
1451        }
1452    }
1453    rules
1454}
1455
1456/// The rule `selectors { body }` makes, when every selector is `*` or a bare tag name and
1457/// the body holds a valid `white-space` declaration.
1458fn whitespace_rule(selectors: &str, body: &str) -> Option<WhitespaceRule> {
1459    let selectors: Vec<String> = selectors
1460        .split(',')
1461        .map(|s| s.trim().to_ascii_lowercase())
1462        .collect();
1463    let bare = |s: &String| {
1464        s == "*" || (!s.is_empty() && s.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
1465    };
1466    if !selectors.iter().all(bare) {
1467        return None;
1468    }
1469    let mut declarations = Vec::new();
1470    // The declarations of the rule itself: a nested rule (`p { span { white-space: pre } }`)
1471    // is another rule's, and a block in the body is taken out before it is split.
1472    let own;
1473    let body = if body.contains('{') {
1474        let mut text = String::with_capacity(body.len());
1475        let mut depth = 0usize;
1476        for ch in body.chars() {
1477            match ch {
1478                '{' => depth += 1,
1479                '}' if depth > 0 => {
1480                    depth -= 1;
1481                    if depth == 0 {
1482                        // What ended the nested rule ends a declaration too.
1483                        text.push(';');
1484                    }
1485                }
1486                _ if depth == 0 => text.push(ch),
1487                _ => {}
1488            }
1489        }
1490        own = text;
1491        own.as_str()
1492    } else {
1493        body
1494    };
1495    for part in body.split(';') {
1496        let Some((property, value)) = part.split_once(':') else {
1497            continue;
1498        };
1499        if !property.trim().eq_ignore_ascii_case("white-space") {
1500            continue;
1501        }
1502        let value = value.trim().to_ascii_lowercase();
1503        let (value, important) = value
1504            .strip_suffix("important")
1505            .and_then(|rest| rest.trim_end().strip_suffix('!'))
1506            .map_or((value.as_str(), false), |rest| (rest.trim_end(), true));
1507        // Only the keywords CSS defines; any other value is dropped, as a browser does.
1508        let keyword = match value {
1509            "normal" => "normal",
1510            "nowrap" => "nowrap",
1511            "pre" => "pre",
1512            "pre-wrap" => "pre-wrap",
1513            "pre-line" => "pre-line",
1514            "break-spaces" => "break-spaces",
1515            _ => continue,
1516        };
1517        declarations.push((keyword, important));
1518    }
1519    (!declarations.is_empty()).then_some(WhitespaceRule {
1520        selectors,
1521        declarations,
1522    })
1523}
1524
1525/// What the whitespace of an element's text is, from the sheet and the element's own
1526/// `style`: the sheet's declarations first, so that the inline ones win unless the sheet's
1527/// is `!important`. `None` when neither says.
1528fn element_keeps_whitespace(
1529    el: &scraper::node::Element,
1530    sheet: Option<&SheetWhitespace>,
1531) -> Option<bool> {
1532    let inline = el.attr("style");
1533    match sheet.and_then(|s| s.declaration_for(el.name())) {
1534        None => inline.and_then(style_keeps_whitespace),
1535        Some((keyword, important)) => style_keeps_whitespace(&format!(
1536            "white-space: {keyword}{};{}",
1537            if important { " !important" } else { "" },
1538            inline.unwrap_or("")
1539        )),
1540    }
1541}
1542
1543/// The CSS of the `<style>` blocks of `fragment` that apply to a screen: no `type` or
1544/// `text/css`, no `media` or `all` or `screen`.
1545fn html_sheet(fragment: &scraper::Html) -> Option<SheetWhitespace> {
1546    let mut css = String::new();
1547    for node in fragment.tree.nodes() {
1548        let Node::Element(el) = node.value() else {
1549            continue;
1550        };
1551        if el.name() != "style" {
1552            continue;
1553        }
1554        let type_ok = el
1555            .attr("type")
1556            .is_none_or(|t| t.trim().is_empty() || t.trim().eq_ignore_ascii_case("text/css"));
1557        let media_ok = el.attr("media").is_none_or(|m| {
1558            let m = m.trim();
1559            m.is_empty() || m.eq_ignore_ascii_case("all") || m.eq_ignore_ascii_case("screen")
1560        });
1561        if !(type_ok && media_ok) {
1562            continue;
1563        }
1564        for child in node.children() {
1565            if let Node::Text(text) = child.value() {
1566                css.push_str(text);
1567                css.push('\n');
1568            }
1569        }
1570    }
1571    if css.is_empty() {
1572        return None;
1573    }
1574    let sheet = SheetWhitespace::from_css(&css);
1575    (!sheet.is_empty()).then_some(sheet)
1576}
1577
1578/// Whether `name` is an element that sets its content apart as a block, in the sense of
1579/// "blanks next to it are layout".
1580fn is_block_level_name(name: &str) -> bool {
1581    matches!(
1582        name,
1583        "address"
1584            | "article"
1585            | "aside"
1586            | "blockquote"
1587            | "caption"
1588            | "center"
1589            | "dd"
1590            | "details"
1591            | "dialog"
1592            | "dir"
1593            | "div"
1594            | "dl"
1595            | "dt"
1596            | "fieldset"
1597            | "figcaption"
1598            | "figure"
1599            | "footer"
1600            | "form"
1601            | "h1"
1602            | "h2"
1603            | "h3"
1604            | "h4"
1605            | "h5"
1606            | "h6"
1607            | "header"
1608            | "hgroup"
1609            | "hr"
1610            | "li"
1611            | "main"
1612            | "menu"
1613            | "nav"
1614            | "ol"
1615            | "p"
1616            | "pre"
1617            | "search"
1618            | "section"
1619            | "summary"
1620            | "table"
1621            | "tbody"
1622            | "td"
1623            | "tfoot"
1624            | "th"
1625            | "thead"
1626            | "tr"
1627            | "ul"
1628    )
1629}
1630
1631/// Whether `name` is an element the reader turns into a block of its own: its content is
1632/// flushed apart from the text around it. The block-level names beyond these (a `<section>`,
1633/// a `<dt>`) are recursed into the run they stand in, so their texts join it.
1634fn is_block_in_the_reader(name: &str) -> bool {
1635    matches!(
1636        name,
1637        "p" | "div"
1638            | "h1"
1639            | "h2"
1640            | "h3"
1641            | "h4"
1642            | "h5"
1643            | "h6"
1644            | "li"
1645            | "pre"
1646            | "blockquote"
1647            | "ul"
1648            | "ol"
1649            | "table"
1650    )
1651}
1652
1653/// What a blank text node is, where whitespace is kept.
1654#[derive(Debug, PartialEq, Eq)]
1655enum BlankText {
1656    /// Content: kept as it is.
1657    Content,
1658    /// The layout between blocks, or at the edge of the element holding them: nothing.
1659    Layout,
1660    /// The layout between two block-level elements the reader does not split a run at
1661    /// (`<dt>a</dt>\n<dd>b</dd>`, two sections): the boundary a browser shows between two
1662    /// block boxes, which is one line break where whitespace is kept (and one space in a
1663    /// table cell, which is a single paragraph). The texts of the two stay apart, one line
1664    /// each, and not joined into a word.
1665    Boundary,
1666}
1667
1668/// Classify a text node of a parent whose whitespace is kept: only blanks, with a block-level
1669/// element on one side and a block-level element or the edge of the parent on the other, is
1670/// layout. Blanks between two inline siblings (`<b>a</b> <i>b</i>`) are the space between two
1671/// words, and blanks alone in an element (`<p> </p>`) are the content a writer put there;
1672/// both stay. Decided from the two neighbours, so the cost does not depend on how many
1673/// siblings there are.
1674fn blank_text_kind(text: &str, node: ego_tree::NodeRef<Node>) -> BlankText {
1675    if !text.chars().all(is_html_space) {
1676        return BlankText::Content;
1677    }
1678    // The neighbour on a side, skipping comments: `None` at the edge of the parent.
1679    let neighbour = |next: bool| {
1680        let mut at = node;
1681        loop {
1682            let sibling = if next {
1683                at.next_sibling()
1684            } else {
1685                at.prev_sibling()
1686            };
1687            match sibling {
1688                Some(s) if matches!(s.value(), Node::Comment(_)) => at = s,
1689                other => return other,
1690            }
1691        }
1692    };
1693    // How a neighbour sets blanks apart: `None` for text, and for an element or the edge of
1694    // the parent: whether it is block-level, whether the reader splits a run at it, and
1695    // whether it is the edge.
1696    let side = |sibling: Option<ego_tree::NodeRef<Node>>| match sibling {
1697        None => Some((false, false, true)),
1698        Some(s) => match s.value() {
1699            Node::Element(el) => Some((
1700                is_block_level_name(el.name()),
1701                is_block_in_the_reader(el.name()),
1702                false,
1703            )),
1704            _ => None,
1705        },
1706    };
1707    match (side(neighbour(false)), side(neighbour(true))) {
1708        (Some(before), Some(after)) if before.0 || after.0 => {
1709            if before.1 || after.1 || before.2 || after.2 {
1710                BlankText::Layout
1711            } else {
1712                BlankText::Boundary
1713            }
1714        }
1715        _ => BlankText::Content,
1716    }
1717}
1718
1719/// Write `text`, a run of a paragraph's text, into `out` as HTML: each of its lines (it
1720/// separates them with [`LINE_SEPARATOR`]) escaped by `escape`, the lines joined by
1721/// `line_break`, and a line whose whitespace an HTML reader would collapse put in a
1722/// `<span style="white-space: pre-wrap">`, where browsers, Qt and this crate's own HTML
1723/// reader keep it as it is.
1724///
1725/// A line is wrapped when it holds a tab or another HTML space but the plain space, two
1726/// spaces in a row, a space at its start when `after_space` (the text before it on its
1727/// line ends in whitespace, or the line starts with it), or a space at its end when
1728/// `ends_line` (its line ends with it). Written bare, a tab came back as a space and a
1729/// writer's double space as one, so the HTML of a text read back was not the HTML it had
1730/// been read from. Text an HTML reader keeps as it is goes out bare, as before.
1731///
1732/// `after_space` for the next run is [`html_text_ends_open`].
1733pub fn push_html_text(
1734    out: &mut String,
1735    text: &str,
1736    after_space: bool,
1737    ends_line: bool,
1738    escape: impl Fn(&str) -> String,
1739    line_break: &str,
1740) {
1741    let mut lines = text.split(LINE_SEPARATOR).peekable();
1742    let mut first = true;
1743    while let Some(line) = lines.next() {
1744        if !first {
1745            out.push_str(line_break);
1746        }
1747        let opens_on_space = if first { after_space } else { true };
1748        let ends = ends_line || lines.peek().is_some();
1749        first = false;
1750        let collapses = line.chars().any(|ch| ch != ' ' && is_html_space(ch))
1751            || line.contains("  ")
1752            || (opens_on_space && line.starts_with(' '))
1753            || (ends && line.ends_with(' '));
1754        if collapses {
1755            out.push_str("<span style=\"white-space: pre-wrap\">");
1756            out.push_str(&escape(line));
1757            out.push_str("</span>");
1758        } else {
1759            out.push_str(&escape(line));
1760        }
1761    }
1762}
1763
1764/// Whether a paragraph's line ends before `rest`, the inline content after a run of its
1765/// text: the `ends_line` of that run (see [`push_html_text`]). It does when nothing but
1766/// empty content follows, or when the next text starts a line.
1767pub fn html_line_ends_before<'a>(
1768    rest: impl IntoIterator<Item = &'a crate::format_runs::InlineContent>,
1769) -> bool {
1770    use crate::format_runs::InlineContent;
1771    for content in rest {
1772        match content {
1773            InlineContent::Empty => {}
1774            InlineContent::Text(text) if text.is_empty() => {}
1775            InlineContent::Text(text) => return text.starts_with(LINE_SEPARATOR),
1776            _ => return false,
1777        }
1778    }
1779    true
1780}
1781
1782/// Whether the text of a paragraph's line, `text` added to what came before it on its
1783/// line, ends in whitespace an HTML reader would collapse a space after it into: the
1784/// `after_space` of the run after it (see [`push_html_text`]). `after_space` says the same
1785/// of the text before `text`, and is the answer when `text` is empty.
1786pub fn html_text_ends_open(text: &str, after_space: bool) -> bool {
1787    match text.chars().next_back() {
1788        None => after_space,
1789        Some(ch) => ch == LINE_SEPARATOR || is_html_space(ch),
1790    }
1791}
1792
1793/// The five characters HTML calls whitespace.
1794///
1795/// Deliberately not `char::is_whitespace`, which is `true` for U+00A0: a
1796/// no-break space is a character the writer typed — the one French typography
1797/// puts before a colon, holding "Attention&nbsp;:" together — and collapsing it
1798/// away is a silent edit of the text, not of its layout.
1799fn is_html_space(ch: char) -> bool {
1800    matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1801}
1802
1803/// The box a Markdown task item writes after its list marker: `[x] ` or `[ ] `, nothing for
1804/// an item that is no task. Shared by the two Markdown writers.
1805pub fn task_box_markdown(marker: Option<&MarkerType>) -> &'static str {
1806    match marker {
1807        Some(MarkerType::Checked) => "[x] ",
1808        Some(MarkerType::Unchecked) => "[ ] ",
1809        _ => "",
1810    }
1811}
1812
1813/// The checkbox an HTML list item opens with when it is a task, nothing otherwise, inside
1814/// the `<label>` that names it: the item's text, which [`task_checkbox_end_html`] closes the
1815/// label after, is the checkbox's accessible name (a bare checkbox reads as "checkbox,
1816/// checked" and the text as another item). Written as well-formed XML (`disabled="disabled"`,
1817/// a closed tag) because the same markup is packaged into EPUB XHTML chapters, which a
1818/// minimised attribute and an unclosed input would break in a strict reader. Shared by the
1819/// two HTML writers; `html_item_checkbox` reads it back, through the label.
1820pub fn task_checkbox_html(marker: Option<&MarkerType>) -> &'static str {
1821    match marker {
1822        Some(MarkerType::Checked) => {
1823            "<label><input type=\"checkbox\" disabled=\"disabled\" checked=\"checked\"/> "
1824        }
1825        Some(MarkerType::Unchecked) => "<label><input type=\"checkbox\" disabled=\"disabled\"/> ",
1826        _ => "",
1827    }
1828}
1829
1830/// What closes [`task_checkbox_html`]'s label, after the item's inline text.
1831pub fn task_checkbox_end_html(marker: Option<&MarkerType>) -> &'static str {
1832    match marker {
1833        Some(MarkerType::Checked | MarkerType::Unchecked) => "</label>",
1834        _ => "",
1835    }
1836}
1837
1838/// The task-list checkbox an HTML list item opens with: the first thing it holds, apart
1839/// from blanks, is an `<input type="checkbox">`, bare or inside inline wrappers such as
1840/// `<label>` and `<span>`. `checked` makes it [`MarkerType::Checked`]; `disabled` and
1841/// every other attribute change nothing. A checkbox further on is a form control in the
1842/// text, not the item's box. A box inside a `<p>` of the item, which is how a loose GitHub task
1843/// list is rendered (`<li><p><input type="checkbox"> text</p></li>`), is not read.
1844fn html_item_checkbox(item: ego_tree::NodeRef<Node>) -> Option<MarkerType> {
1845    /// `Some(answer)` once something decided it, `None` while only blanks were seen.
1846    fn scan(node: ego_tree::NodeRef<Node>, depth: usize) -> Option<Option<MarkerType>> {
1847        for child in node.children() {
1848            match child.value() {
1849                Node::Text(text) => {
1850                    if !text.chars().all(is_html_space) {
1851                        return Some(None);
1852                    }
1853                }
1854                Node::Element(el) => {
1855                    let name = el.name();
1856                    if name == "input" {
1857                        let is_checkbox = el
1858                            .attr("type")
1859                            .is_some_and(|t| t.trim().eq_ignore_ascii_case("checkbox"));
1860                        return Some(is_checkbox.then(|| {
1861                            if el.attr("checked").is_some() {
1862                                MarkerType::Checked
1863                            } else {
1864                                MarkerType::Unchecked
1865                            }
1866                        }));
1867                    }
1868                    if matches!(name, "label" | "span") && depth < 8 {
1869                        if let Some(decided) = scan(child, depth + 1) {
1870                            return Some(decided);
1871                        }
1872                    } else {
1873                        return Some(None);
1874                    }
1875                }
1876                _ => {}
1877            }
1878        }
1879        None
1880    }
1881    scan(item, 0).flatten()
1882}
1883
1884/// Whether a span run holds anything a reader would see.
1885fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1886    spans
1887        .iter()
1888        .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1889}
1890
1891/// Apply the CSS `white-space: normal` rules to one block's inline spans: every
1892/// run of HTML whitespace collapses to a single space, and the runs at the
1893/// block's two ends disappear entirely.
1894///
1895/// Per block rather than per text node, because a run crosses span boundaries —
1896/// `<b>word</b>\n<i>next</i>` is one collapsible run split over three nodes —
1897/// and because only the block knows where its own ends are.
1898///
1899/// Without this, an exporter's own source formatting reaches the document as
1900/// text. LibreOffice writes a newline after every `<p …>` and hard-wraps the
1901/// prose inside it at about seventy columns, so pasting a whole book put a
1902/// blank line before every paragraph and a break through the middle of every
1903/// sentence.
1904fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1905    fn is_replaced(span: &ParsedSpan) -> bool {
1906        span.image.is_some() || span.footnote_ref.is_some()
1907    }
1908
1909    // Collapse within each span first. Nothing non-empty is emptied here, which
1910    // is what lets the second pass read `is_empty()` as "this span held no text
1911    // to begin with" — the marker a `<br>` leaves behind.
1912    for span in spans.iter_mut() {
1913        if is_replaced(span) || span.preserve_whitespace {
1914            continue;
1915        }
1916        let mut out = String::with_capacity(span.text.len());
1917        let mut in_run = false;
1918        for ch in span.text.chars() {
1919            if is_html_space(ch) {
1920                if !in_run {
1921                    out.push(' ');
1922                    in_run = true;
1923                }
1924            } else {
1925                out.push(ch);
1926                in_run = false;
1927            }
1928        }
1929        span.text = out;
1930    }
1931
1932    // The block's start counts as "already ended on a space", so a run opening
1933    // the block is dropped outright. A line break (the span a `<br>` leaves) ends one
1934    // line and starts the next, so the runs either side of it go as the block's own
1935    // ends do: `x <br> y` is two lines, "x" and "y".
1936    let mut prev_ends_with_space = true;
1937    // The last span holding text since the line began, whose trailing space a line
1938    // break takes off.
1939    let mut line_end: Option<usize> = None;
1940    for i in 0..spans.len() {
1941        let span = &mut spans[i];
1942        if is_replaced(span) {
1943            prev_ends_with_space = false;
1944            line_end = None;
1945            continue;
1946        }
1947        if span.text.is_empty() {
1948            // A span that held no text to begin with. It contributes none, but it
1949            // still ends the collapsible run either side of it.
1950            prev_ends_with_space = false;
1951            continue;
1952        }
1953        if span.text == LINE_SEPARATOR_STR {
1954            if let Some(end) = line_end.take()
1955                && spans[end].text.ends_with(' ')
1956            {
1957                spans[end].text.pop();
1958            }
1959            prev_ends_with_space = true;
1960            continue;
1961        }
1962        // Kept whitespace is the text's own, as a replaced element is content: a space
1963        // after it is not collapsed into it, and a line break takes none of it off.
1964        if span.preserve_whitespace {
1965            prev_ends_with_space = false;
1966            line_end = None;
1967            continue;
1968        }
1969        if prev_ends_with_space && span.text.starts_with(' ') {
1970            span.text.remove(0);
1971        }
1972        // A span emptied by that drop leaves the run open rather than closing
1973        // it: the space it carried belongs to the run its neighbours share.
1974        if !span.text.is_empty() {
1975            prev_ends_with_space = span.text.ends_with(' ');
1976            line_end = Some(i);
1977        }
1978    }
1979
1980    // The block's trailing run, in whichever span still holds it. Replaced
1981    // content stops the walk: a space before a trailing image is between two
1982    // pieces of content, not after the last one. A line break at the very end
1983    // starts no line of its own, as a browser shows it (a second one does: `a<br><br>`
1984    // is "a" and an empty line), so the last one goes, with the run before it.
1985    let mut trailing_break_dropped = false;
1986    for span in spans.iter_mut().rev() {
1987        if is_replaced(span) {
1988            break;
1989        }
1990        if span.text.is_empty() {
1991            continue;
1992        }
1993        if span.text == LINE_SEPARATOR_STR {
1994            if trailing_break_dropped {
1995                break;
1996            }
1997            span.text.clear();
1998            trailing_break_dropped = true;
1999            continue;
2000        }
2001        if span.preserve_whitespace {
2002            break;
2003        }
2004        if span.text.ends_with(' ') {
2005            span.text.pop();
2006        }
2007        break;
2008    }
2009}
2010
2011/// `block` as one block per line of its text, a line ending at each `\n`.
2012///
2013/// A `\n` is not a line break inside a block, which the model holds as
2014/// [`LINE_SEPARATOR`]: in the rope it separates two blocks, so a new line of this kind
2015/// is a new block, which is what a plain-text paste makes of one. Text that kept its
2016/// source's line breaks (`<pre>`, `white-space: pre`) held them inside one block, where
2017/// the editor showed them as lines and the Djot writer had nothing to write them as.
2018/// Written raw, a blank line ended the paragraph and the lines after it were read back
2019/// as the lists, headings and quotations their markers spelled, however deep.
2020///
2021/// Each line keeps the block's own formatting and its spans' formatting, but for the
2022/// page break, which stays with the first line: it is where the block starts, and on
2023/// every line it put each on a page of its own. An empty line is an empty block. The
2024/// line break that ends the last line starts no line of its own, as a browser shows it.
2025/// A block without a line break is returned as it is.
2026///
2027/// Every line is built from copies of the block and of the span it comes from that
2028/// hold no text, so the split costs in proportion to the text: built from whole copies,
2029/// each line copied the whole text again, and a paste of 64,000 preformatted lines
2030/// took seconds.
2031pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
2032    let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
2033    if !block.spans.iter().any(breaks) {
2034        return vec![block];
2035    }
2036    let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
2037    for mut span in std::mem::take(&mut block.spans) {
2038        if !breaks(&span) {
2039            push_to_last_line(&mut lines, span);
2040            continue;
2041        }
2042        let text = std::mem::take(&mut span.text);
2043        for (i, piece) in text.split('\n').enumerate() {
2044            if i > 0 {
2045                lines.push(Vec::new());
2046            }
2047            if !piece.is_empty() {
2048                push_to_last_line(
2049                    &mut lines,
2050                    ParsedSpan {
2051                        text: piece.to_string(),
2052                        ..span.clone()
2053                    },
2054                );
2055            }
2056        }
2057    }
2058    if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
2059        lines.pop();
2060    }
2061    let later_line = ParsedBlock {
2062        page_break_before: None,
2063        blockquote_opens: 0,
2064        // The box is the item's, which its first line holds.
2065        marker: None,
2066        ..block.clone()
2067    };
2068    let mut first_line = Some(block);
2069    lines
2070        .into_iter()
2071        .map(|spans| {
2072            let spans = if spans.is_empty() {
2073                vec![ParsedSpan::default()]
2074            } else {
2075                spans
2076            };
2077            match first_line.take() {
2078                Some(first) => ParsedBlock { spans, ..first },
2079                None => ParsedBlock {
2080                    spans,
2081                    ..later_line.clone()
2082                },
2083            }
2084        })
2085        .collect()
2086}
2087
2088/// Append `span` to the last of `lines`, which [`split_block_at_line_breaks`] keeps
2089/// non-empty.
2090fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
2091    match lines.last_mut() {
2092        Some(line) => line.push(span),
2093        None => lines.push(vec![span]),
2094    }
2095}
2096
2097pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
2098    ParsedElement::flatten_to_blocks(parse_html_elements(html))
2099}
2100
2101/// The attribute an HTML producer uses to say "this element *is* a footnote
2102/// reference", and the label it names.
2103///
2104/// A data attribute on an otherwise-ordinary element, rather than a literal
2105/// `[^label]` in the text, because the text a producer emits is *escaped* on the
2106/// way to Djot — [`escape_djot_inline`](crate::parser_tools::djot_escape::escape_djot_inline)
2107/// neutralises `[`, `]` and `^`, so a reference smuggled through as characters
2108/// arrives as `\[\^1\]` and is prose rather than a reference. It has to be a
2109/// node before it is serialised, which is what this makes it.
2110///
2111/// [`ParsedSpan::footnote_ref`] is the same field the Markdown/Djot reader
2112/// already fills, so everything downstream — the document model, the Djot
2113/// writer, numbering — needs no change at all.
2114pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
2115
2116/// Build a footnote-reference span from any element carrying
2117/// [`HTML_FOOTNOTE_ATTR`].
2118fn html_footnote_span(
2119    el: &scraper::node::Element,
2120    link_href: Option<String>,
2121) -> Option<ParsedSpan> {
2122    let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
2123    if label.is_empty() {
2124        return None;
2125    }
2126    Some(ParsedSpan {
2127        text: String::new(),
2128        link_href,
2129        footnote_ref: Some(label.to_string()),
2130        ..Default::default()
2131    })
2132}
2133
2134/// Build an inline-image span from an `<img>` element, if it has a usable
2135/// source.
2136///
2137/// `<img>` was matched by none of the HTML walker's three tag dispatches, so it
2138/// fell into their wildcard arms and was dropped whole — silently, and unlike
2139/// Markdown not even leaving its alt text behind. That is the path a browser or
2140/// Word paste travels.
2141fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
2142    let src = el.attr("src")?;
2143    if src.is_empty() {
2144        return None;
2145    }
2146    let dim = |name: &str| -> i64 {
2147        el.attr(name)
2148            .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
2149            .filter(|n| *n > 0)
2150            .unwrap_or(0)
2151    };
2152    Some(ParsedSpan {
2153        text: String::new(),
2154        link_href,
2155        image: Some(ParsedImage {
2156            src: src.to_string(),
2157            alt: el.attr("alt").unwrap_or_default().to_string(),
2158            width: dim("width"),
2159            height: dim("height"),
2160        }),
2161        ..Default::default()
2162    })
2163}
2164
2165/// Whether `node` is a `<ul>` or an `<ol>`.
2166fn is_html_list(node: &Node) -> bool {
2167    node.as_element()
2168        .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
2169}
2170
2171/// Move every list item nested deeper than
2172/// [`MAX_LIST_LEVELS`](crate::parser_tools::list_depth::MAX_LIST_LEVELS)
2173/// lists into the deepest list kept, right after the item holding it, in
2174/// document order: a list nested past that depth comes out flattened to it,
2175/// every item kept, in its order.
2176///
2177/// The reader recurses a few elements deep per list level and stops at
2178/// `MAX_RECURSION_DEPTH`, so a list nested past some eighty levels lost every
2179/// item below that, pasted or loaded. The editing gestures nest no deeper than
2180/// this depth, and an insertion brings any list back to it (see `list_depth`),
2181/// so HTML holding more is read at that depth to begin with. The tree is
2182/// changed in place, without recursion, before it is read.
2183fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
2184    use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
2185    use ego_tree::iter::Edge;
2186
2187    // The lists one level past the deepest kept, in document order.
2188    let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
2189    let mut depth = 0usize;
2190    for edge in tree.root().traverse() {
2191        match edge {
2192            Edge::Open(node) if is_html_list(node.value()) => {
2193                if depth == MAX_LIST_LEVELS {
2194                    too_deep.push(node.id());
2195                }
2196                depth += 1;
2197            }
2198            Edge::Close(node) if is_html_list(node.value()) => {
2199                depth = depth.saturating_sub(1);
2200            }
2201            _ => {}
2202        }
2203    }
2204
2205    for list in too_deep {
2206        // The child of the deepest kept list that holds `list` (its `<li>`, or
2207        // `list` itself when a list sits right inside another): the items go
2208        // after it.
2209        let Some(list_ref) = tree.get(list) else {
2210            continue;
2211        };
2212        let mut holder = None;
2213        let mut current = list_ref;
2214        while let Some(parent) = current.parent() {
2215            if is_html_list(parent.value()) {
2216                holder = Some(current.id());
2217                break;
2218            }
2219            current = parent;
2220        }
2221        let Some(holder) = holder else {
2222            continue;
2223        };
2224        let mut after = holder;
2225        let items: Vec<ego_tree::NodeId> = list_ref
2226            .descendants()
2227            .filter(|node| {
2228                node.value()
2229                    .as_element()
2230                    .is_some_and(|el| el.name() == "li")
2231            })
2232            .map(|node| node.id())
2233            .collect();
2234        for item in items {
2235            // Each node placed here sits in the deepest kept list, so it has a
2236            // parent to be placed beside.
2237            if tree.get(after).and_then(|node| node.parent()).is_none() {
2238                break;
2239            }
2240            if let Some(mut node) = tree.get_mut(after) {
2241                node.insert_id_after(item);
2242                after = item;
2243            }
2244        }
2245        // What is left of the list is its items' wrappers, or text standing in
2246        // it outside any item. It goes after the items when it holds any text
2247        // (it is already beside them when it sits right in the deepest kept
2248        // list), and away when it holds none.
2249        let holds_text = tree.get(list).is_some_and(|node| {
2250            node.descendants()
2251                .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
2252        });
2253        if !holds_text {
2254            if let Some(mut node) = tree.get_mut(list) {
2255                node.detach();
2256            }
2257        } else if holder != list
2258            && tree.get(after).and_then(|node| node.parent()).is_some()
2259            && let Some(mut node) = tree.get_mut(after)
2260        {
2261            node.insert_id_after(list);
2262        }
2263    }
2264}
2265
2266/// How deep a walk of an HTML tree may go on the caller's own stack: a debug build spends
2267/// between 5 and 8 KiB of stack on each element level it follows, so this is at most some
2268/// 256 KiB, what any thread a host reads on has to spare.
2269const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
2270
2271/// The deepest an HTML reader follows elements: a subtree nested deeper is read as its
2272/// text, without recursing (see `read_html_tree`).
2273const MAX_HTML_READ_DEPTH: usize = 256;
2274
2275/// The stack a reader of a deeper tree gets on a thread of its own, before what each level
2276/// adds: a spawned thread's.
2277const HTML_READER_STACK: usize = 2 << 20;
2278
2279/// The stack each element level of a deeper tree adds for its reader: two to three times
2280/// what a debug build spends on one.
2281const HTML_STACK_PER_LEVEL: usize = 16 << 10;
2282
2283/// `value` read as HTML reads an integer attribute such as `<ol start>`: after any leading
2284/// white space, an optional sign and the digits up to the first character that is not one.
2285/// `None` when no digit follows. A number past what 64 bits hold reads as the largest they
2286/// hold on its side of 0. A browser numbers `<ol start="5x">` from 5, where a strict parse
2287/// of the whole value numbered it from 1.
2288fn html_integer(value: &str) -> Option<i64> {
2289    let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
2290    let (negative, digits) = match value.as_bytes().first() {
2291        Some(b'-') => (true, &value[1..]),
2292        Some(b'+') => (false, &value[1..]),
2293        _ => (false, value),
2294    };
2295    let digits: &str = digits
2296        .split(|c: char| !c.is_ascii_digit())
2297        .next()
2298        .unwrap_or("");
2299    if digits.is_empty() {
2300        return None;
2301    }
2302    let magnitude = digits.bytes().fold(0i64, |number, digit| {
2303        number
2304            .saturating_mul(10)
2305            .saturating_add(i64::from(digit - b'0'))
2306    });
2307    Some(if negative { -magnitude } else { magnitude })
2308}
2309
2310/// How many elements deep the deepest node of `tree` is. A loop over the tree's edges,
2311/// never a recursion.
2312fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
2313    use ego_tree::iter::Edge;
2314    let mut depth = 0usize;
2315    let mut deepest = 0usize;
2316    for edge in tree.root().traverse() {
2317        match edge {
2318            Edge::Open(node) if node.value().is_element() => {
2319                depth += 1;
2320                deepest = deepest.max(depth);
2321            }
2322            Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
2323            _ => {}
2324        }
2325    }
2326    deepest
2327}
2328
2329/// Read `html` into blocks and tables.
2330///
2331/// The reader recurses once per element level it follows, up to `MAX_HTML_READ_DEPTH`,
2332/// and a stack overflow aborts the process: a debug build ran out of a spawned thread's
2333/// stack on a paste nested some 200 elements deep, and a host reading on a smaller thread
2334/// sooner. A tree nested deeper than a caller's stack is sure to hold is read on a thread
2335/// of its own, with a stack sized for its depth; when no thread can be started, it is read
2336/// on the caller's stack to `HTML_DEPTH_ON_THE_CALLERS_STACK` levels, each subtree
2337/// deeper than that read as its text. The quotations it holds are kept within
2338/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
2339/// `quote_depth`).
2340///
2341/// Whitespace is kept where an element's `white-space` says so: in its inline `style`, and
2342/// in a `<style>` block whose rule names only `*` or bare tags, as
2343/// `p, li { white-space: pre-wrap; }` does (what Qt writes). A rule with a class, id,
2344/// attribute, pseudo-class or combinator in its selector, a rule inside an at-rule such as
2345/// `@media`, a `<style>` of another `type` or for `print`, and every other property, is
2346/// ignored. Blanks alone between two blocks stay layout; between two inline siblings they are
2347/// the space between two words. Known limits: a rule with a class or id is not applied, and
2348/// neither is what it overrides (`p.plain { white-space: normal }` over `p { white-space: pre }`
2349/// leaves the paragraphs of the class kept); `pre-line` and `nowrap` collapse all whitespace,
2350/// as they do in an inline `style`; and a `pre` from a sheet, unlike one in an inline `style`,
2351/// does not mark the block non-breakable.
2352pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
2353    use scraper::Html;
2354
2355    let read = |depth_limit: usize, html: &str| {
2356        let mut fragment = Html::parse_fragment(html);
2357        flatten_deep_html_lists(&mut fragment.tree);
2358        let mut elements = read_html_tree(&fragment, depth_limit);
2359        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
2360        elements
2361    };
2362    let mut fragment = Html::parse_fragment(html);
2363    flatten_deep_html_lists(&mut fragment.tree);
2364    let depth = html_tree_depth(&fragment.tree);
2365    if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
2366        let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
2367        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
2368        return elements;
2369    }
2370    drop(fragment);
2371    let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
2372    let owned = html.to_owned();
2373    let on_its_own_thread = std::thread::Builder::new()
2374        .stack_size(stack)
2375        .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
2376        .ok()
2377        .and_then(|reader| reader.join().ok());
2378    on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
2379}
2380
2381/// Read `fragment`, following its elements `depth_limit` deep at most: a subtree nested
2382/// deeper is read as its text (see `subtree_text`). The walk recurses once per element
2383/// level it follows.
2384fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
2385    let mut elements: Vec<ParsedElement> = Vec::new();
2386
2387    // Walk the DOM tree starting from the root
2388    let root = fragment.root_element();
2389
2390    #[derive(Clone, Default)]
2391    struct FmtState {
2392        bold: bool,
2393        italic: bool,
2394        underline: bool,
2395        strikeout: bool,
2396        code: bool,
2397        superscript: bool,
2398        subscript: bool,
2399        link_href: Option<String>,
2400        /// Under an element whose `white-space` keeps whitespace as it is.
2401        preserve_whitespace: bool,
2402        /// The `white-space` rules of the document's `<style>` blocks, shared by every state.
2403        sheet: Option<std::rc::Rc<SheetWhitespace>>,
2404    }
2405
2406    /// The list an element stands in: its style, and the number its first item wears when
2407    /// it is not 1 (`<ol start="3">`).
2408    type HtmlList = (Option<ListStyle>, Option<i64>);
2409
2410    /// Elements whose text content is machinery, not prose.
2411    ///
2412    /// `<style>` and `<script>` hold raw text the HTML spec never renders, and
2413    /// `<head>` and its metadata children carry none a reader is meant to see.
2414    /// Every walker below recurses into unknown tags on the assumption that
2415    /// they are inline wrappers, which turns a stylesheet into paragraphs —
2416    /// and Word, Google Docs and Chrome all put a `<style>` block in the
2417    /// `text/html` flavour they publish to the clipboard, so this is the
2418    /// ordinary paste, not an exotic one.
2419    fn is_metadata_tag(tag: &str) -> bool {
2420        matches!(
2421            tag,
2422            "head"
2423                | "style"
2424                | "script"
2425                | "title"
2426                | "meta"
2427                | "link"
2428                | "base"
2429                | "noscript"
2430                | "template"
2431        )
2432    }
2433
2434    /// The text of everything under `node`, for a subtree nested deeper than
2435    /// the reader follows (see [`MAX_HTML_READ_DEPTH`]): read without recursing,
2436    /// with a space where a block ends so that words do not run together, and
2437    /// without what [`is_metadata_tag`] names.
2438    ///
2439    /// Past that depth the reader used to stop and keep nothing: text nested
2440    /// deeply enough in lists, quotations or plain `<div>`s was dropped from a
2441    /// paste without a trace.
2442    fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
2443        use ego_tree::iter::Edge;
2444        let mut text = String::new();
2445        // How many elements deep the walk is inside a metadata element.
2446        let mut hidden = 0usize;
2447        for edge in node.traverse() {
2448            match edge {
2449                Edge::Open(open) => match open.value() {
2450                    Node::Element(el) => {
2451                        if hidden > 0 || is_metadata_tag(el.name()) {
2452                            hidden += 1;
2453                        } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
2454                            text.push(' ');
2455                        }
2456                    }
2457                    Node::Text(t) if hidden == 0 => text.push_str(t),
2458                    _ => {}
2459                },
2460                Edge::Close(close) => {
2461                    if hidden > 0 && close.value().is_element() {
2462                        hidden -= 1;
2463                    }
2464                }
2465            }
2466        }
2467        text
2468    }
2469
2470    /// A span of `text` in `state`'s formatting.
2471    fn text_span(text: String, state: &FmtState) -> ParsedSpan {
2472        ParsedSpan {
2473            text,
2474            bold: state.bold,
2475            italic: state.italic,
2476            underline: state.underline,
2477            strikeout: state.strikeout,
2478            code: state.code,
2479            superscript: state.superscript,
2480            subscript: state.subscript,
2481            link_href: state.link_href.clone(),
2482            image: None,
2483            footnote_ref: None,
2484            preserve_whitespace: state.preserve_whitespace,
2485        }
2486    }
2487
2488    /// The space a cell puts between two blocks of its own: layout, collapsed whatever the
2489    /// state's whitespace, where a kept one would stay a space at the cell's edge.
2490    fn separator_span(state: &FmtState) -> ParsedSpan {
2491        let mut span = text_span(" ".to_string(), state);
2492        span.preserve_whitespace = false;
2493        span
2494    }
2495
2496    /// Collect inline spans from a `<td>` or `<th>` cell element.
2497    fn collect_cell_spans(
2498        node: ego_tree::NodeRef<Node>,
2499        state: &FmtState,
2500        spans: &mut Vec<ParsedSpan>,
2501        depth: usize,
2502        limit: usize,
2503    ) {
2504        if depth > limit {
2505            let text = subtree_text(node);
2506            if !text.is_empty() {
2507                spans.push(text_span(text, state));
2508            }
2509            return;
2510        }
2511        for child in node.children() {
2512            match child.value() {
2513                Node::Text(text) => {
2514                    let t = text.text.to_string();
2515                    // The blanks between two blocks are layout, even where whitespace is kept.
2516                    let blank = if state.preserve_whitespace {
2517                        blank_text_kind(&t, child)
2518                    } else {
2519                        BlankText::Content
2520                    };
2521                    if blank == BlankText::Boundary {
2522                        // One space between two texts, none where the cell has no text yet
2523                        // or the text before ends in a blank already.
2524                        if spans
2525                            .last()
2526                            .is_some_and(|last| !last.text.ends_with(is_html_space))
2527                        {
2528                            spans.push(separator_span(state));
2529                        }
2530                    } else if !t.is_empty() && blank == BlankText::Content {
2531                        spans.push(ParsedSpan {
2532                            text: t,
2533                            bold: state.bold,
2534                            italic: state.italic,
2535                            underline: state.underline,
2536                            strikeout: state.strikeout,
2537                            code: state.code,
2538                            superscript: state.superscript,
2539                            subscript: state.subscript,
2540                            link_href: state.link_href.clone(),
2541                            image: None,
2542                            footnote_ref: None,
2543                            preserve_whitespace: state.preserve_whitespace,
2544                        });
2545                    }
2546                }
2547                Node::Element(el) => {
2548                    let tag = el.name();
2549                    if is_metadata_tag(tag) {
2550                        continue;
2551                    }
2552                    let mut new_state = state.clone();
2553                    if let Some(keep) = element_keeps_whitespace(el, state.sheet.as_deref()) {
2554                        new_state.preserve_whitespace = keep;
2555                    }
2556                    match tag {
2557                        // Checked by attribute rather than by tag name: the
2558                        // producer picks the element (a `<sup>` renders sensibly
2559                        // in a browser), and only the attribute is a contract.
2560                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2561                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2562                            {
2563                                spans.push(span);
2564                            }
2565                            continue;
2566                        }
2567                        "b" | "strong" => new_state.bold = true,
2568                        "i" | "em" => new_state.italic = true,
2569                        "u" | "ins" => new_state.underline = true,
2570                        "s" | "del" | "strike" => new_state.strikeout = true,
2571                        "code" => new_state.code = true,
2572                        "sup" => new_state.superscript = true,
2573                        "sub" => new_state.subscript = true,
2574                        "a" => {
2575                            if let Some(href) = el.attr("href") {
2576                                new_state.link_href = Some(href.to_string());
2577                            }
2578                        }
2579                        "img" => {
2580                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2581                                spans.push(span);
2582                            }
2583                            continue;
2584                        }
2585                        // A cell is read as one paragraph, so a line break, or a
2586                        // paragraph or list item of its own, is a space between
2587                        // words. Read as nothing, a cell of two paragraphs, which
2588                        // the HTML writer puts out as `One<br/>Two`, came back as
2589                        // the one word `OneTwo`.
2590                        "br" => {
2591                            spans.push(separator_span(state));
2592                            continue;
2593                        }
2594                        "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
2595                        | "h3" | "h4" | "h5" | "h6" => {
2596                            spans.push(separator_span(state));
2597                        }
2598                        _ => {}
2599                    }
2600                    collect_cell_spans(child, &new_state, spans, depth + 1, limit);
2601                }
2602                _ => {}
2603            }
2604        }
2605    }
2606
2607    /// The widest a `colspan` makes a row: HTML's own limit on the attribute. A row of
2608    /// ordinary cells may be wider; only the empty cells a span adds stop here.
2609    const MAX_HTML_COLSPAN: usize = 1000;
2610
2611    /// Parse a `<table>` element into a ParsedTable, with the number of `<td>` and `<th>`
2612    /// cells it has (see `push_table`). `depth` is how deep the walk stands at the table,
2613    /// and `limit` how deep it goes.
2614    fn parse_table_element(
2615        table_node: ego_tree::NodeRef<Node>,
2616        state: &FmtState,
2617        depth: usize,
2618        limit: usize,
2619    ) -> (ParsedTable, usize) {
2620        let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
2621        let mut header_rows: usize = 0;
2622        let mut own_cells: usize = 0;
2623
2624        /// `walk` is how deep the walk stands at `node` and how deep it goes.
2625        fn collect_rows(
2626            node: ego_tree::NodeRef<Node>,
2627            state: &FmtState,
2628            rows: &mut Vec<Vec<ParsedTableCell>>,
2629            header_rows: &mut usize,
2630            own_cells: &mut usize,
2631            in_thead: bool,
2632            walk: (usize, usize),
2633        ) {
2634            let (depth, limit) = walk;
2635            for child in node.children() {
2636                if let Node::Element(el) = child.value() {
2637                    match el.name() {
2638                        "thead" => collect_rows(
2639                            child,
2640                            state,
2641                            rows,
2642                            header_rows,
2643                            own_cells,
2644                            true,
2645                            (depth + 1, limit),
2646                        ),
2647                        "tbody" | "tfoot" => collect_rows(
2648                            child,
2649                            state,
2650                            rows,
2651                            header_rows,
2652                            own_cells,
2653                            false,
2654                            (depth + 1, limit),
2655                        ),
2656                        "tr" => {
2657                            let mut cells: Vec<ParsedTableCell> = Vec::new();
2658                            // The cells of the row up to its last `<td>` or `<th>`: the
2659                            // empty cells a span adds after it hold no cell's place.
2660                            let mut own_end = 0;
2661                            for td in child.children() {
2662                                if let Node::Element(td_el) = td.value()
2663                                    && matches!(td_el.name(), "td" | "th")
2664                                {
2665                                    let mut spans = Vec::new();
2666                                    // A cell starts from the whitespace the table stands in
2667                                    // (the sheet's, with the table's own), then its own. It
2668                                    // takes none of the formatting around the table: a table
2669                                    // in a `<b>` or an `<a>` is not made of bold cells or links.
2670                                    let mut cell_state = FmtState {
2671                                        preserve_whitespace: state.preserve_whitespace,
2672                                        sheet: state.sheet.clone(),
2673                                        ..FmtState::default()
2674                                    };
2675                                    if let Some(keep) =
2676                                        element_keeps_whitespace(td_el, state.sheet.as_deref())
2677                                    {
2678                                        cell_state.preserve_whitespace = keep;
2679                                    }
2680                                    collect_cell_spans(
2681                                        td,
2682                                        &cell_state,
2683                                        &mut spans,
2684                                        depth + 2,
2685                                        limit,
2686                                    );
2687                                    collapse_inline_whitespace(&mut spans);
2688                                    if spans.is_empty() {
2689                                        spans.push(ParsedSpan::default());
2690                                    }
2691                                    cells.push(ParsedTableCell { spans });
2692                                    *own_cells += 1;
2693                                    own_end = cells.len();
2694                                    // A cell spanning columns covers the cells after it
2695                                    // in its row: empty ones hold its place, so the cells
2696                                    // that follow stay in their own columns and the
2697                                    // table is as wide as the columns it spans.
2698                                    let span = td_el
2699                                        .attr("colspan")
2700                                        .and_then(|value| value.trim().parse::<usize>().ok())
2701                                        .unwrap_or(1)
2702                                        .clamp(1, MAX_HTML_COLSPAN);
2703                                    for _ in 1..span {
2704                                        if cells.len() >= MAX_HTML_COLSPAN {
2705                                            break;
2706                                        }
2707                                        cells.push(ParsedTableCell {
2708                                            spans: vec![ParsedSpan::default()],
2709                                        });
2710                                    }
2711                                }
2712                            }
2713                            // A span closing a row widens the table only as far as another
2714                            // row's cells reach: `<th colspan="1000">` over rows of one
2715                            // cell is a table of one column, not of a thousand.
2716                            cells.truncate(own_end);
2717                            if !cells.is_empty() {
2718                                rows.push(cells);
2719                                if in_thead {
2720                                    *header_rows += 1;
2721                                }
2722                            }
2723                        }
2724                        _ => {}
2725                    }
2726                }
2727            }
2728        }
2729
2730        collect_rows(
2731            table_node,
2732            state,
2733            &mut rows,
2734            &mut header_rows,
2735            &mut own_cells,
2736            false,
2737            (depth, limit),
2738        );
2739
2740        // Tables without explicit <thead> but with <th> cells: treat first row as header
2741        if header_rows == 0 && !rows.is_empty() {
2742            header_rows = 1;
2743        }
2744
2745        (
2746            ParsedTable {
2747                header_rows,
2748                rows,
2749                // The caller (`walk_node`) sets the real depth: this helper has
2750                // no visibility into the surrounding blockquote nesting.
2751                blockquote_depth: 0,
2752                blockquote_opens: 0,
2753            },
2754            own_cells,
2755        )
2756    }
2757
2758    #[allow(clippy::too_many_arguments)]
2759    fn walk_node(
2760        node: ego_tree::NodeRef<Node>,
2761        state: &FmtState,
2762        elements: &mut Vec<ParsedElement>,
2763        current_list: &HtmlList,
2764        blockquote_depth: u32,
2765        list_depth: u32,
2766        depth: usize,
2767        limit: usize,
2768    ) {
2769        if depth > limit {
2770            let text = subtree_text(node);
2771            if !text.trim().is_empty() {
2772                let mut spans = vec![text_span(text, state)];
2773                collapse_inline_whitespace(&mut spans);
2774                elements.push(ParsedElement::Block(ParsedBlock {
2775                    spans,
2776                    blockquote_depth,
2777                    blockquote_opens: 0,
2778                    ..ParsedBlock::default()
2779                }));
2780            }
2781            return;
2782        }
2783        match node.value() {
2784            Node::Element(el) => {
2785                let tag = el.name();
2786                if is_metadata_tag(tag) {
2787                    return;
2788                }
2789                let mut new_state = state.clone();
2790                if let Some(keep) = element_keeps_whitespace(el, state.sheet.as_deref()) {
2791                    new_state.preserve_whitespace = keep;
2792                }
2793                let mut new_list = current_list.clone();
2794                let mut bq_depth = blockquote_depth;
2795                let mut new_list_depth = list_depth;
2796
2797                // Determine if this is a block-level element
2798                let is_block_tag = matches!(
2799                    tag,
2800                    "p" | "div"
2801                        | "h1"
2802                        | "h2"
2803                        | "h3"
2804                        | "h4"
2805                        | "h5"
2806                        | "h6"
2807                        | "li"
2808                        | "pre"
2809                        | "br"
2810                        | "blockquote"
2811                        | "body"
2812                        | "html"
2813                );
2814
2815                // Update formatting state
2816                match tag {
2817                    "b" | "strong" => new_state.bold = true,
2818                    "i" | "em" => new_state.italic = true,
2819                    "u" | "ins" => new_state.underline = true,
2820                    "s" | "del" | "strike" => new_state.strikeout = true,
2821                    "code" => new_state.code = true,
2822                    "sup" => new_state.superscript = true,
2823                    "sub" => new_state.subscript = true,
2824                    "a" => {
2825                        if let Some(href) = el.attr("href") {
2826                            new_state.link_href = Some(href.to_string());
2827                        }
2828                    }
2829                    "ul" => {
2830                        new_list = (Some(ListStyle::Disc), None);
2831                        new_list_depth = list_depth + 1;
2832                    }
2833                    "ol" => {
2834                        // A list numbered from below 0 is kept from 1: no saved syntax
2835                        // but HTML writes one. A start past what every format holds is
2836                        // read as the largest it holds (see `list_start`).
2837                        let start = el
2838                            .attr("start")
2839                            .and_then(html_integer)
2840                            .and_then(crate::parser_tools::list_start::list_start);
2841                        new_list = (Some(ListStyle::Decimal), start);
2842                        new_list_depth = list_depth + 1;
2843                    }
2844                    "blockquote" => {
2845                        bq_depth += 1;
2846                    }
2847                    _ => {}
2848                }
2849
2850                // Determine heading level
2851                let heading_level = match tag {
2852                    "h1" => Some(1),
2853                    "h2" => Some(2),
2854                    "h3" => Some(3),
2855                    "h4" => Some(4),
2856                    "h5" => Some(5),
2857                    "h6" => Some(6),
2858                    _ => None,
2859                };
2860
2861                let is_code_block = tag == "pre";
2862
2863                // Extract code language from <pre><code class="language-xxx">
2864                let code_language = if is_code_block {
2865                    node.children().find_map(|child| {
2866                        if let Node::Element(cel) = child.value()
2867                            && cel.name() == "code"
2868                            && let Some(cls) = cel.attr("class")
2869                        {
2870                            return cls
2871                                .split_whitespace()
2872                                .find_map(|c| c.strip_prefix("language-"))
2873                                .map(|l| l.to_string());
2874                        }
2875                        None
2876                    })
2877                } else {
2878                    None
2879                };
2880
2881                // Extract CSS styles from block-level elements
2882                let css = if is_block_tag {
2883                    el.attr("style").map(parse_block_styles).unwrap_or_default()
2884                } else {
2885                    BlockStyles::default()
2886                };
2887
2888                if tag == "table" {
2889                    // Parse table structure into a ParsedTable
2890                    let (mut parsed_table, own_cells) =
2891                        parse_table_element(node, &new_state, depth, limit);
2892                    if !parsed_table.rows.is_empty() {
2893                        parsed_table.blockquote_depth = bq_depth;
2894                        push_table(elements, parsed_table, own_cells);
2895                    }
2896                    return;
2897                }
2898
2899                if tag == "br" {
2900                    // <br> creates a new block
2901                    elements.push(ParsedElement::Block(ParsedBlock {
2902                        spans: vec![ParsedSpan {
2903                            text: String::new(),
2904                            ..Default::default()
2905                        }],
2906                        heading_level: None,
2907                        list_style: None,
2908                        list_indent: 0,
2909                        list_prefix: String::new(),
2910                        list_suffix: String::new(),
2911                        list_start: None,
2912                        marker: None,
2913                        is_code_block: false,
2914                        code_language: None,
2915                        blockquote_depth: bq_depth,
2916                        blockquote_opens: 0,
2917                        line_height: None,
2918                        non_breakable_lines: None,
2919                        page_break_before: None,
2920                        front_matter: None,
2921                        direction: None,
2922                        background_color: None,
2923                        alignment: None,
2924                        top_margin: None,
2925                        text_indent: None,
2926                        semantic_role: None,
2927                    }));
2928                    return;
2929                }
2930
2931                if tag == "blockquote" {
2932                    let first = elements.len();
2933                    // A quotation holding inline content alone is one paragraph of it, as
2934                    // a `<p>` is, its `<br>`s line breaks: verse is pasted from the web as
2935                    // `<blockquote>a line<br>the next</blockquote>`. Walked child by child,
2936                    // each line was a paragraph of its own and each `<br>` an empty one.
2937                    let inline_only = node.children().all(|child| match child.value() {
2938                        Node::Element(el) => !matches!(
2939                            el.name(),
2940                            "p" | "div"
2941                                | "h1"
2942                                | "h2"
2943                                | "h3"
2944                                | "h4"
2945                                | "h5"
2946                                | "h6"
2947                                | "li"
2948                                | "pre"
2949                                | "blockquote"
2950                                | "ul"
2951                                | "ol"
2952                                | "table"
2953                        ),
2954                        _ => true,
2955                    });
2956                    if inline_only {
2957                        let mut spans: Vec<ParsedSpan> = Vec::new();
2958                        let mut nested_elements: Vec<ParsedElement> = Vec::new();
2959                        collect_inline_spans(
2960                            node,
2961                            &new_state,
2962                            &mut spans,
2963                            &new_list,
2964                            &mut nested_elements,
2965                            bq_depth,
2966                            new_list_depth,
2967                            depth + 1,
2968                            limit,
2969                        );
2970                        collapse_inline_whitespace(&mut spans);
2971                        if spans_carry_content(&spans) {
2972                            elements.push(ParsedElement::Block(ParsedBlock {
2973                                spans,
2974                                blockquote_depth: bq_depth,
2975                                blockquote_opens: 0,
2976                                ..ParsedBlock::default()
2977                            }));
2978                        }
2979                        elements.append(&mut nested_elements);
2980                    } else {
2981                        // A container: recurse into children with increased depth.
2982                        for child in node.children() {
2983                            walk_node(
2984                                child,
2985                                &new_state,
2986                                elements,
2987                                &new_list,
2988                                bq_depth,
2989                                new_list_depth,
2990                                depth + 1,
2991                                limit,
2992                            );
2993                        }
2994                    }
2995                    mark_quotation_opened(elements, first);
2996                } else if is_block_tag && tag != "br" {
2997                    // Start collecting spans for a new block.
2998                    // Use a temporary buffer so that nested block-level
2999                    // elements (e.g. sub-lists inside <li>) are collected
3000                    // separately and appended *after* the parent block.
3001                    let mut spans: Vec<ParsedSpan> = Vec::new();
3002                    let mut nested_elements: Vec<ParsedElement> = Vec::new();
3003                    collect_inline_spans(
3004                        node,
3005                        &new_state,
3006                        &mut spans,
3007                        &new_list,
3008                        &mut nested_elements,
3009                        bq_depth,
3010                        new_list_depth,
3011                        depth + 1,
3012                        limit,
3013                    );
3014
3015                    let (list_style_for_block, list_start_for_block) = if tag == "li" {
3016                        new_list.clone()
3017                    } else {
3018                        (None, None)
3019                    };
3020
3021                    let list_indent_for_block = if tag == "li" {
3022                        new_list_depth.saturating_sub(1)
3023                    } else {
3024                        0
3025                    };
3026
3027                    if !(is_code_block || css.preserve_whitespace == Some(true)) {
3028                        collapse_inline_whitespace(&mut spans);
3029                    }
3030
3031                    // Whitespace between block-level children is layout, not
3032                    // content. `<body>` holds a newline after every `</p>` an
3033                    // exporter writes, and keeping those as the container's own
3034                    // inline run is what closed a pasted book with a paragraph
3035                    // of four thousand blank lines. A block that produced no
3036                    // children keeps its empty run instead: `<p> </p>` is an
3037                    // empty paragraph the writer meant to be there.
3038                    let own_run_is_content =
3039                        spans_carry_content(&spans) || nested_elements.is_empty();
3040
3041                    if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
3042                        // A code block is verbatim text, which no save can write an image or
3043                        // a note's reference into: a `<pre>` holding one is read as prose,
3044                        // which keeps it, one paragraph for each of its lines (see below).
3045                        let is_code_block = is_code_block
3046                            && !spans
3047                                .iter()
3048                                .any(|span| span.image.is_some() || span.footnote_ref.is_some());
3049                        let list_style_for_block_is_some = list_style_for_block.is_some();
3050                        let block = ParsedBlock {
3051                            spans,
3052                            heading_level,
3053                            list_suffix: list_style_for_block
3054                                .as_ref()
3055                                .map(|style| default_list_suffix(style).to_string())
3056                                .unwrap_or_default(),
3057                            list_style: list_style_for_block,
3058                            list_indent: list_indent_for_block,
3059                            list_prefix: String::new(),
3060                            list_start: list_start_for_block,
3061                            // Only a list item has a box: an `<li>` outside any list is read
3062                            // as a paragraph (see `is_inline_only`) and has none to keep.
3063                            marker: if list_style_for_block_is_some {
3064                                html_item_checkbox(node)
3065                            } else {
3066                                None
3067                            },
3068                            is_code_block,
3069                            code_language: code_language.filter(|_| is_code_block),
3070                            blockquote_depth: bq_depth,
3071                            blockquote_opens: 0,
3072                            line_height: css.line_height,
3073                            non_breakable_lines: css.non_breakable_lines,
3074                            page_break_before: css.page_break_before,
3075                            front_matter: None,
3076                            direction: css.direction,
3077                            background_color: css.background_color,
3078                            alignment: None,
3079                            top_margin: None,
3080                            text_indent: None,
3081                            semantic_role: None,
3082                        };
3083                        // Kept whitespace keeps its line breaks, and a line break in
3084                        // prose is a new block to the model. Only a code block holds
3085                        // its lines in one block.
3086                        if is_code_block {
3087                            elements.push(ParsedElement::Block(block));
3088                        } else {
3089                            elements.extend(
3090                                split_block_at_line_breaks(block)
3091                                    .into_iter()
3092                                    .map(ParsedElement::Block),
3093                            );
3094                        }
3095                    }
3096                    // Append nested block elements after the parent block
3097                    elements.append(&mut nested_elements);
3098                } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
3099                    // Container elements: recurse into children
3100                    for child in node.children() {
3101                        walk_node(
3102                            child,
3103                            &new_state,
3104                            elements,
3105                            &new_list,
3106                            bq_depth,
3107                            new_list_depth,
3108                            depth + 1,
3109                            limit,
3110                        );
3111                    }
3112                } else {
3113                    // Inline element or unknown: recurse
3114                    for child in node.children() {
3115                        walk_node(
3116                            child,
3117                            &new_state,
3118                            elements,
3119                            current_list,
3120                            bq_depth,
3121                            list_depth,
3122                            depth + 1,
3123                            limit,
3124                        );
3125                    }
3126                }
3127            }
3128            Node::Text(text) => {
3129                // Bare text not in a block (a list's text outside any item, `<ol>a</ol>`)
3130                // is a paragraph, its whitespace collapsed as a paragraph's is. Only
3131                // trimmed, a tab in it reached the model, and a newline, which separates
3132                // two blocks there, and the HTML written from it read back otherwise.
3133                //
3134                // Blanks alone are the layout between blocks, whitespace kept or not.
3135                let mut spans = vec![text_span(text.text.to_string(), state)];
3136                collapse_inline_whitespace(&mut spans);
3137                if spans_carry_content(&spans) && !text.chars().all(is_html_space) {
3138                    let block = ParsedBlock {
3139                        spans,
3140                        blockquote_depth,
3141                        blockquote_opens: 0,
3142                        ..ParsedBlock::default()
3143                    };
3144                    elements.extend(
3145                        split_block_at_line_breaks(block)
3146                            .into_iter()
3147                            .map(ParsedElement::Block),
3148                    );
3149                }
3150            }
3151            _ => {
3152                // Document, Comment, etc. — recurse children
3153                for child in node.children() {
3154                    walk_node(
3155                        child,
3156                        state,
3157                        elements,
3158                        current_list,
3159                        blockquote_depth,
3160                        list_depth,
3161                        depth + 1,
3162                        limit,
3163                    );
3164                }
3165            }
3166        }
3167    }
3168
3169    /// Collect inline spans from a block-level element's children.
3170    /// If a nested block-level element is encountered, it is flushed as a
3171    /// separate block.
3172    #[allow(clippy::too_many_arguments)]
3173    fn collect_inline_spans(
3174        node: ego_tree::NodeRef<Node>,
3175        state: &FmtState,
3176        spans: &mut Vec<ParsedSpan>,
3177        current_list: &HtmlList,
3178        elements: &mut Vec<ParsedElement>,
3179        blockquote_depth: u32,
3180        list_depth: u32,
3181        depth: usize,
3182        limit: usize,
3183    ) {
3184        if depth > limit {
3185            let text = subtree_text(node);
3186            if !text.is_empty() {
3187                spans.push(text_span(text, state));
3188            }
3189            return;
3190        }
3191        for child in node.children() {
3192            match child.value() {
3193                Node::Text(text) => {
3194                    let t = text.text.to_string();
3195                    // The blanks between two blocks are layout, even where whitespace is kept.
3196                    let blank = if state.preserve_whitespace {
3197                        blank_text_kind(&t, child)
3198                    } else {
3199                        BlankText::Content
3200                    };
3201                    if blank == BlankText::Boundary {
3202                        // One line break where whitespace is kept: the block is split there,
3203                        // as it is at any other newline. None when nothing comes before it
3204                        // in the run (the elements before it hold blocks of their own), or
3205                        // when the line has ended already; one that ends the run is no line
3206                        // of its own, as a browser shows it.
3207                        if spans.last().is_some_and(|last| !last.text.ends_with('\n')) {
3208                            spans.push(text_span("\n".to_string(), state));
3209                        }
3210                    } else if !t.is_empty() && blank == BlankText::Content {
3211                        spans.push(ParsedSpan {
3212                            text: t,
3213                            bold: state.bold,
3214                            italic: state.italic,
3215                            underline: state.underline,
3216                            strikeout: state.strikeout,
3217                            code: state.code,
3218                            superscript: state.superscript,
3219                            subscript: state.subscript,
3220                            link_href: state.link_href.clone(),
3221                            image: None,
3222                            footnote_ref: None,
3223                            preserve_whitespace: state.preserve_whitespace,
3224                        });
3225                    }
3226                }
3227                Node::Element(el) => {
3228                    let tag = el.name();
3229                    if is_metadata_tag(tag) {
3230                        continue;
3231                    }
3232                    let mut new_state = state.clone();
3233                    if let Some(keep) = element_keeps_whitespace(el, state.sheet.as_deref()) {
3234                        new_state.preserve_whitespace = keep;
3235                    }
3236
3237                    match tag {
3238                        // Checked by attribute rather than by tag name: the
3239                        // producer picks the element (a `<sup>` renders sensibly
3240                        // in a browser), and only the attribute is a contract.
3241                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
3242                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
3243                            {
3244                                spans.push(span);
3245                            }
3246                            continue;
3247                        }
3248                        "b" | "strong" => new_state.bold = true,
3249                        "i" | "em" => new_state.italic = true,
3250                        "u" | "ins" => new_state.underline = true,
3251                        "s" | "del" | "strike" => new_state.strikeout = true,
3252                        "code" => new_state.code = true,
3253                        "sup" => new_state.superscript = true,
3254                        "sub" => new_state.subscript = true,
3255                        "a" => {
3256                            if let Some(href) = el.attr("href") {
3257                                new_state.link_href = Some(href.to_string());
3258                            }
3259                        }
3260                        "img" => {
3261                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
3262                                spans.push(span);
3263                            }
3264                            continue;
3265                        }
3266                        _ => {}
3267                    }
3268
3269                    // Check for nested block elements
3270                    let nested_block = matches!(
3271                        tag,
3272                        "p" | "div"
3273                            | "h1"
3274                            | "h2"
3275                            | "h3"
3276                            | "h4"
3277                            | "h5"
3278                            | "h6"
3279                            | "li"
3280                            | "pre"
3281                            | "blockquote"
3282                            | "ul"
3283                            | "ol"
3284                    );
3285
3286                    if tag == "br" {
3287                        // A line break inside the block, in the formatting around it.
3288                        // `collapse_inline_whitespace` then drops the spaces either side,
3289                        // as a browser lays them out.
3290                        spans.push(text_span(LINE_SEPARATOR_STR.to_string(), state));
3291                    } else if nested_block || tag == "table" {
3292                        // Flush as separate element
3293                        walk_node(
3294                            child,
3295                            &new_state,
3296                            elements,
3297                            current_list,
3298                            blockquote_depth,
3299                            list_depth,
3300                            depth + 1,
3301                            limit,
3302                        );
3303                    } else {
3304                        // Inline element: recurse
3305                        collect_inline_spans(
3306                            child,
3307                            &new_state,
3308                            spans,
3309                            current_list,
3310                            elements,
3311                            blockquote_depth,
3312                            list_depth,
3313                            depth + 1,
3314                            limit,
3315                        );
3316                    }
3317                }
3318                _ => {}
3319            }
3320        }
3321    }
3322
3323    // What the sheet says of `<body>` and `<html>` is where the walk starts: the HTML parser
3324    // drops both elements from a fragment, so no element is left to carry it. Bare text
3325    // outside any element inherits only that.
3326    let sheet = html_sheet(fragment).map(std::rc::Rc::new);
3327    let initial_state = FmtState {
3328        preserve_whitespace: sheet
3329            .as_deref()
3330            .and_then(SheetWhitespace::initial_keeps_whitespace)
3331            .unwrap_or(false),
3332        sheet,
3333        ..FmtState::default()
3334    };
3335    // Treat the root element as a block-level container so that
3336    // top-level inline elements (e.g. `<b>Bold</b> <em>Italic</em>`)
3337    // are grouped into a single block instead of becoming separate blocks.
3338    let mut root_spans: Vec<ParsedSpan> = Vec::new();
3339    collect_inline_spans(
3340        *root,
3341        &initial_state,
3342        &mut root_spans,
3343        &(None, None),
3344        &mut elements,
3345        0,
3346        0,
3347        0,
3348        depth_limit,
3349    );
3350    collapse_inline_whitespace(&mut root_spans);
3351    // Only when something survived: the root's direct text children are the
3352    // newlines between top-level blocks in every document an exporter writes.
3353    if spans_carry_content(&root_spans) {
3354        // Text that kept its line breaks is one block for each line, as for any other block.
3355        let root_block = ParsedBlock {
3356            spans: root_spans,
3357            heading_level: None,
3358            list_style: None,
3359            list_indent: 0,
3360            list_prefix: String::new(),
3361            list_suffix: String::new(),
3362            list_start: None,
3363            marker: None,
3364            is_code_block: false,
3365            code_language: None,
3366            blockquote_depth: 0,
3367            blockquote_opens: 0,
3368            line_height: None,
3369            non_breakable_lines: None,
3370            page_break_before: None,
3371            front_matter: None,
3372            direction: None,
3373            background_color: None,
3374            alignment: None,
3375            top_margin: None,
3376            text_indent: None,
3377            semantic_role: None,
3378        };
3379        elements.extend(
3380            split_block_at_line_breaks(root_block)
3381                .into_iter()
3382                .map(ParsedElement::Block),
3383        );
3384    }
3385
3386    // If no elements were parsed, create a single empty paragraph
3387    if elements.is_empty() {
3388        elements.push(ParsedElement::Block(ParsedBlock {
3389            spans: vec![ParsedSpan {
3390                text: String::new(),
3391                ..Default::default()
3392            }],
3393            heading_level: None,
3394            list_style: None,
3395            list_indent: 0,
3396            list_prefix: String::new(),
3397            list_suffix: String::new(),
3398            list_start: None,
3399            marker: None,
3400            is_code_block: false,
3401            code_language: None,
3402            blockquote_depth: 0,
3403            blockquote_opens: 0,
3404            line_height: None,
3405            non_breakable_lines: None,
3406            page_break_before: None,
3407            front_matter: None,
3408            direction: None,
3409            background_color: None,
3410            alignment: None,
3411            top_margin: None,
3412            text_indent: None,
3413            semantic_role: None,
3414        }));
3415    }
3416
3417    elements
3418}
3419
3420/// Convert a `ParsedSpan` (parser output) into the `CharacterFormat` used by
3421/// `FormatRun`. `is_code_block` forces `monospace` as the font family for
3422/// every span inside a code block.
3423pub fn character_format_from_span(
3424    span: &ParsedSpan,
3425    is_code_block: bool,
3426) -> crate::format_runs::CharacterFormat {
3427    use crate::entities::CharVerticalAlignment;
3428    crate::format_runs::CharacterFormat {
3429        font_bold: if span.bold { Some(true) } else { None },
3430        font_italic: if span.italic { Some(true) } else { None },
3431        font_underline: if span.underline { Some(true) } else { None },
3432        font_strikeout: if span.strikeout { Some(true) } else { None },
3433        font_family: if span.code || is_code_block {
3434            Some("monospace".to_string())
3435        } else {
3436            None
3437        },
3438        anchor_href: span.link_href.clone(),
3439        is_anchor: if span.link_href.is_some() {
3440            Some(true)
3441        } else {
3442            None
3443        },
3444        vertical_alignment: if span.superscript {
3445            Some(CharVerticalAlignment::SuperScript)
3446        } else if span.subscript {
3447            Some(CharVerticalAlignment::SubScript)
3448        } else {
3449            None
3450        },
3451        ..Default::default()
3452    }
3453}
3454
3455/// Translate a slice of parsed spans into `(plain_text, format_runs)`.
3456///
3457/// One non-default span yields one `FormatRun`; spans with empty
3458/// `CharacterFormat` (no decoration, no link, no code) emit no run, since an
3459/// absent run means "inherit default formatting" in the new model. Adjacent
3460/// runs with identical formats are coalesced via `coalesce_in_place` so the
3461/// resulting vector satisfies `debug_assert_well_formed`.
3462///
3463/// Returns the concatenated `plain_text` of all spans and a sorted,
3464/// non-overlapping, coalesced `Vec<FormatRun>`. Both safe to feed straight
3465/// into the store under the dual-write bridge.
3466pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
3467    use crate::format_runs::{
3468        CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
3469    };
3470
3471    let mut plain_text = String::new();
3472    let mut runs: Vec<FormatRun> = Vec::new();
3473    let mut images: Vec<ImageAnchor> = Vec::new();
3474    let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
3475    let default = CharacterFormat::default();
3476
3477    for span in spans {
3478        let byte_start = plain_text.len() as u32;
3479
3480        if let Some(label) = &span.footnote_ref {
3481            // A reference occupies one U+FFFC, exactly as an image does, so
3482            // every downstream offset treats the two alike.
3483            plain_text.push('\u{FFFC}');
3484            // Raised, always — whatever the surrounding run is doing.
3485            //
3486            // A footnote marker is superscript by definition, in every
3487            // typographic tradition and in every reader that renders djot. The
3488            // ambient `superscript` flag the span carries is the *prose's*, and
3489            // prose is not superscript, so taking it verbatim sets a note's
3490            // number on the baseline in the middle of a sentence — which reads
3491            // as a stray digit the writer typed rather than as a reference.
3492            //
3493            // It is set here, on the anchor, rather than at render time so that
3494            // every consumer agrees: the editor raises it, the exporters that
3495            // carry character formatting carry it, and the djot writer knows to
3496            // emit `[^label]` *without* wrapping it in `^…^` (see
3497            // `a_reference_is_not_wrapped_in_superscript_markers`).
3498            let mut format = character_format_from_span(span, is_code_block);
3499            format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
3500            footnote_refs.push(FootnoteRefAnchor {
3501                byte_offset: byte_start,
3502                label: label.clone(),
3503                format,
3504            });
3505            continue;
3506        }
3507
3508        if let Some(image) = &span.image {
3509            // An image occupies one U+FFFC in the text, exactly as
3510            // `insert_image` mirrors into the rope, so every downstream offset
3511            // calculation treats a parsed image and an inserted one alike.
3512            plain_text.push('\u{FFFC}');
3513            images.push(ImageAnchor {
3514                byte_offset: byte_start,
3515                name: image.src.clone(),
3516                alt: image.alt.clone(),
3517                width: image.width,
3518                height: image.height,
3519                quality: 100,
3520                format: character_format_from_span(span, is_code_block),
3521            });
3522            continue;
3523        }
3524
3525        plain_text.push_str(&span.text);
3526        let byte_end = plain_text.len() as u32;
3527        if byte_start == byte_end {
3528            continue;
3529        }
3530        let format = character_format_from_span(span, is_code_block);
3531        if format == default {
3532            continue;
3533        }
3534        runs.push(FormatRun {
3535            byte_start,
3536            byte_end,
3537            format,
3538        });
3539    }
3540    coalesce_in_place(&mut runs);
3541    ParsedInline {
3542        plain_text,
3543        runs,
3544        images,
3545        footnote_refs,
3546    }
3547}
3548
3549/// The three parallel things a block stores, as recovered from parsed spans.
3550///
3551/// Returned as a struct rather than a tuple because it grew a third member
3552/// (images) after nine call sites already destructured a pair — and every one
3553/// of those sites has to decide what to do with images, so a silent
3554/// tuple-arity change would have been the wrong kind of easy.
3555#[derive(Debug, Clone, Default)]
3556pub struct ParsedInline {
3557    pub plain_text: String,
3558    pub runs: Vec<crate::format_runs::FormatRun>,
3559    pub images: Vec<crate::format_runs::ImageAnchor>,
3560    pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
3561}
3562
3563// ─── Djot parsing ────────────────────────────────────────────────────
3564
3565/// Map a jotdown unordered/task bullet marker to a model `ListStyle`.
3566///
3567/// The mapping is a stable bijection (`-`↔Disc, `*`↔Circle, `+`↔Square) so the
3568/// djot exporter can recover the exact bullet character for a lossless
3569/// round-trip.
3570fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
3571    use jotdown::ListBulletType as B;
3572    match b {
3573        B::Dash => ListStyle::Disc,
3574        B::Star => ListStyle::Circle,
3575        B::Plus => ListStyle::Square,
3576    }
3577}
3578
3579/// Map a jotdown ordered-list numbering scheme to a model `ListStyle`.
3580fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
3581    use jotdown::OrderedListNumbering as N;
3582    match n {
3583        N::Decimal => ListStyle::Decimal,
3584        N::AlphaLower => ListStyle::LowerAlpha,
3585        N::AlphaUpper => ListStyle::UpperAlpha,
3586        N::RomanLower => ListStyle::LowerRoman,
3587        N::RomanUpper => ListStyle::UpperRoman,
3588    }
3589}
3590
3591/// Map a jotdown ordered-list delimiter to the `(prefix, suffix)` affixes
3592/// stored on the `List` entity (`1.` → `("", ".")`, `1)` → `("", ")")`,
3593/// `(1)` → `("(", ")")`).
3594fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
3595    use jotdown::OrderedListStyle as S;
3596    match style {
3597        S::Period => (String::new(), ".".to_string()),
3598        S::Paren => (String::new(), ")".to_string()),
3599        S::ParenParen => ("(".to_string(), ")".to_string()),
3600    }
3601}
3602
3603/// Optional block-level style attributes carried on a djot block through its
3604/// `{key=value}` block attributes. All `None` when the block has no such
3605/// attributes (or they were filtered out by [`DjotImportOptions`]).
3606#[derive(Debug, Clone, Default)]
3607struct DjotBlockStyle {
3608    alignment: Option<Alignment>,
3609    line_height: Option<i64>,
3610    non_breakable_lines: Option<bool>,
3611    page_break_before: Option<bool>,
3612    front_matter: Option<bool>,
3613    direction: Option<TextDirection>,
3614    background_color: Option<String>,
3615    top_margin: Option<i64>,
3616    text_indent: Option<i64>,
3617    semantic_role: Option<SemanticRole>,
3618}
3619
3620impl DjotBlockStyle {
3621    /// Overlay the `Some` fields of `other` onto `self`, leaving `self`'s
3622    /// existing values for any field `other` does not set. Used to combine a
3623    /// heading's enclosing-`Section` attributes with any on the heading itself.
3624    fn merge_from(&mut self, other: DjotBlockStyle) {
3625        if other.alignment.is_some() {
3626            self.alignment = other.alignment;
3627        }
3628        if other.line_height.is_some() {
3629            self.line_height = other.line_height;
3630        }
3631        if other.non_breakable_lines.is_some() {
3632            self.non_breakable_lines = other.non_breakable_lines;
3633        }
3634        if other.page_break_before.is_some() {
3635            self.page_break_before = other.page_break_before;
3636        }
3637        if other.front_matter.is_some() {
3638            self.front_matter = other.front_matter;
3639        }
3640        if other.direction.is_some() {
3641            self.direction = other.direction;
3642        }
3643        if other.background_color.is_some() {
3644            self.background_color = other.background_color;
3645        }
3646        if other.top_margin.is_some() {
3647            self.top_margin = other.top_margin;
3648        }
3649        if other.text_indent.is_some() {
3650            self.text_indent = other.text_indent;
3651        }
3652        if other.semantic_role.is_some() {
3653            self.semantic_role = other.semantic_role.clone();
3654        }
3655    }
3656}
3657
3658/// Read the round-tripped block-style attributes off a djot block's
3659/// [`jotdown::Attributes`], honouring the import [`DjotImportOptions`]. Keys are
3660/// the model field names (`alignment`, `line_height`, `direction`,
3661/// `non_breakable_lines`, `page_break_before`, `front_matter`, `background_color`,
3662/// `top_margin`, `text_indent`, `semantic_role`); unrecognised values are ignored.
3663fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
3664    let mut style = DjotBlockStyle::default();
3665
3666    if opts.alignment
3667        && let Some(v) = attrs.get_value("alignment")
3668    {
3669        style.alignment = match v.to_string().as_str() {
3670            "left" => Some(Alignment::Left),
3671            "right" => Some(Alignment::Right),
3672            "center" => Some(Alignment::Center),
3673            "justify" => Some(Alignment::Justify),
3674            _ => None,
3675        };
3676    }
3677    if opts.line_height
3678        && let Some(v) = attrs.get_value("line_height")
3679    {
3680        style.line_height = v.to_string().parse::<i64>().ok();
3681    }
3682    if opts.direction
3683        && let Some(v) = attrs.get_value("direction")
3684    {
3685        style.direction = match v.to_string().as_str() {
3686            "ltr" => Some(TextDirection::LeftToRight),
3687            "rtl" => Some(TextDirection::RightToLeft),
3688            _ => None,
3689        };
3690    }
3691    if opts.non_breakable_lines
3692        && let Some(v) = attrs.get_value("non_breakable_lines")
3693    {
3694        style.non_breakable_lines = match v.to_string().as_str() {
3695            "true" => Some(true),
3696            "false" => Some(false),
3697            _ => None,
3698        };
3699    }
3700    if opts.page_break_before
3701        && let Some(v) = attrs.get_value("page_break_before")
3702    {
3703        style.page_break_before = match v.to_string().as_str() {
3704            "true" => Some(true),
3705            "false" => Some(false),
3706            _ => None,
3707        };
3708    }
3709    if opts.front_matter
3710        && let Some(v) = attrs.get_value("front_matter")
3711    {
3712        style.front_matter = match v.to_string().as_str() {
3713            "true" => Some(true),
3714            "false" => Some(false),
3715            _ => None,
3716        };
3717    }
3718    if opts.background_color
3719        && let Some(v) = attrs.get_value("background_color")
3720    {
3721        style.background_color = Some(v.to_string());
3722    }
3723    if opts.top_margin
3724        && let Some(v) = attrs.get_value("top_margin")
3725    {
3726        style.top_margin = v.to_string().parse::<i64>().ok();
3727    }
3728    if opts.text_indent
3729        && let Some(v) = attrs.get_value("text_indent")
3730    {
3731        style.text_indent = v.to_string().parse::<i64>().ok();
3732    }
3733    if opts.semantic_role
3734        && let Some(v) = attrs.get_value("semantic_role")
3735    {
3736        style.semantic_role = match v.to_string().as_str() {
3737            "epigraph" => Some(SemanticRole::Epigraph),
3738            // An unknown role is dropped, not guessed at — the same way an unknown
3739            // alignment value above is. A future role read by an older build then
3740            // degrades to a plain blockquote, which is what it looks like anyway.
3741            _ => None,
3742        };
3743    }
3744
3745    style
3746}
3747
3748/// The start a list keeps on its entity, from the number its first item is written with:
3749/// `None` for 1, which every list starts at unless it says otherwise, and a number past what
3750/// every format holds read as the largest it holds (see `list_start`).
3751fn list_start_of(start: u64) -> Option<i64> {
3752    crate::parser_tools::list_start::list_start_from_unsigned(start)
3753}
3754
3755/// Push a finished block into `elements`, applying the djot block-level fields
3756/// plus any round-tripped block-style attributes carried in `style`.
3757#[allow(clippy::too_many_arguments)]
3758fn djot_push_block(
3759    elements: &mut Vec<ParsedElement>,
3760    spans: Vec<ParsedSpan>,
3761    heading_level: Option<i64>,
3762    list_style: Option<ListStyle>,
3763    list_indent: u32,
3764    list_prefix: String,
3765    list_suffix: String,
3766    list_start: Option<i64>,
3767    marker: Option<MarkerType>,
3768    is_code_block: bool,
3769    code_language: Option<String>,
3770    blockquote_depth: u32,
3771    style: DjotBlockStyle,
3772) {
3773    elements.push(ParsedElement::Block(ParsedBlock {
3774        spans,
3775        heading_level,
3776        list_style,
3777        list_indent,
3778        list_prefix,
3779        list_suffix,
3780        list_start,
3781        marker,
3782        is_code_block,
3783        code_language,
3784        blockquote_depth,
3785        blockquote_opens: 0,
3786        line_height: style.line_height,
3787        non_breakable_lines: style.non_breakable_lines,
3788        page_break_before: style.page_break_before,
3789        front_matter: style.front_matter,
3790        direction: style.direction,
3791        background_color: style.background_color,
3792        alignment: style.alignment,
3793        top_margin: style.top_margin,
3794        text_indent: style.text_indent,
3795        semantic_role: style.semantic_role.clone(),
3796    }));
3797}
3798
3799/// A Djot document whose structure cannot be computed, as its source: one plain
3800/// paragraph for each line that holds text, with that line's text as it is written, its
3801/// leading and trailing whitespace aside.
3802///
3803/// A paragraph a line is what keeps the text the same through a save. The Djot writer
3804/// escapes whatever a paragraph opens with and whatever in it would read as markup, so
3805/// the saved text reads back as these same paragraphs. Shown as one paragraph, the
3806/// source kept its line breaks through a save, read back as the same document the parser
3807/// could not take, and every save escaped its backslashes once more. Blank lines are left
3808/// out because the reader keeps no block for an empty paragraph, and a line's outer
3809/// whitespace because it is the source's indentation, which the parser does not read as
3810/// text in a line it can take either.
3811fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
3812    djot.lines()
3813        .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
3814        .filter(|line| !line.is_empty())
3815        .map(|line| {
3816            ParsedElement::Block(ParsedBlock {
3817                spans: vec![ParsedSpan {
3818                    text: line.to_string(),
3819                    ..Default::default()
3820                }],
3821                ..Default::default()
3822            })
3823        })
3824        .collect()
3825}
3826
3827/// Parse djot source into the shared [`ParsedElement`] intermediate, mirroring
3828/// [`parse_markdown`]. Uses the [`jotdown`] pull parser.
3829///
3830/// Constructs the document model cannot represent are dropped, and their text
3831/// content is discarded so it never leaks into the document: footnotes, math,
3832/// fenced divs, raw blocks/inline, thematic breaks, description lists,
3833/// captions, symbols, link-reference definitions, and highlight/`mark`. Inline
3834/// images keep their alt text as plain text (the image itself is not modelled),
3835/// matching the Markdown importer. Smart-punctuation events are normalised to
3836/// their canonical Unicode characters so the model→djot→model round-trip is a
3837/// fixpoint.
3838///
3839/// Standalone paragraphs and headings additionally carry the optional
3840/// block-style attributes selected by `options` — paragraph alignment, line
3841/// height, text direction, non-breakable lines and background color — read from
3842/// djot `{key=value}` block attributes (see [`DjotImportOptions`]). List items,
3843/// code blocks and table cells normalise their block styling away.
3844///
3845/// Known model limitations (normalised, not preserved on round-trip):
3846/// table column alignment, and list tight/loose.
3847///
3848/// A CRLF line ending is read as the LF it stands for, so a source with Windows line endings
3849/// reads as the same text with Unix ones. A lone carriage return is kept as text: it is how
3850/// [`plain_text_to_djot`](crate::parser_tools::plain_text_to_djot) writes one, and what it
3851/// guarantees to give back.
3852pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
3853    use jotdown::{Container as C, Event as E, ListKind};
3854
3855    let djot = crate::parser_tools::djot_depth::crlf_to_lf(djot);
3856    let djot: &str = &djot;
3857
3858    // `jotdown` recurses with no limit of its own, once per nested container and
3859    // once per line of a paragraph that leaves an inline opener waiting, and a
3860    // stack overflow **aborts the process**: it cannot be caught by
3861    // `catch_unwind`, so there is no recovering from it after the fact and the
3862    // only safe move is not to start. It also panics on a heading deeper than it
3863    // can number, which a caller on an interface thread does not survive either.
3864    // Input is not always the author's own: a project bundle is mailed and
3865    // shared, and an imported document comes from whoever sent it.
3866    //
3867    // `djot_depth::parsable` is the door that keeps the parser within its limits,
3868    // and it degrades rather than refuses, so the signature stays the same and no
3869    // word is lost. Such a heading is escaped into text, as it was written. A
3870    // document nested too deeply by its indentation, which is how a deeply nested
3871    // list is written, is read with the indentation cut: the list comes out
3872    // flattened below a fixed depth, every item and word kept. This used to hand
3873    // back a pasted list nested two hundred levels as one paragraph of markup. A
3874    // long paragraph is parsed on a stack sized for it, and the block holding one
3875    // too long for any stack is shown as its lines, the rest of the document as it
3876    // was.
3877    //
3878    // A document still nested too deeply once flattened, or one the parser panicked
3879    // on, comes back as its lines, one plain paragraph each (see
3880    // `djot_source_lines`). Its structure is not computed, which is the honest
3881    // answer for a document whose structure cannot be computed without ending the
3882    // process, and its text is still there for the writer to see and repair.
3883    let parsable = crate::parser_tools::djot_depth::parsable(djot);
3884    let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
3885        return djot_source_lines(djot);
3886    };
3887
3888    let mut elements: Vec<ParsedElement> = Vec::new();
3889    let mut current_spans: Vec<ParsedSpan> = Vec::new();
3890    let mut current_heading: Option<i64> = None;
3891    let mut is_code_block = false;
3892    let mut code_language: Option<String> = None;
3893    let mut blockquote_depth: u32 = 0;
3894    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
3895    let mut quotations_open: Vec<usize> = Vec::new();
3896    // Block-style attributes captured from a standalone paragraph/heading's djot
3897    // `{…}` block attributes, consumed when that block is flushed.
3898    let mut pending_style = DjotBlockStyle::default();
3899
3900    // Inline formatting state.
3901    let mut bold = false;
3902    let mut italic = false;
3903    let mut underline = false;
3904    let mut strikeout = false;
3905    let mut code = false;
3906    let mut superscript = false;
3907    let mut subscript = false;
3908    let mut link_href: Option<String> = None;
3909    // Set between an image's Start and End. Its alt text arrives as ordinary
3910    // `Str` events in between, so it has to be captured rather than emitted.
3911    let mut pending_image: Option<ParsedImage> = None;
3912
3913    // List nesting: each entry is (style, prefix, suffix, start); depth = indent + 1.
3914    let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3915    // Context applied to the next flushed block while inside a list item.
3916    let mut cur_list_style: Option<ListStyle> = None;
3917    let mut cur_list_prefix = String::new();
3918    let mut cur_list_suffix = String::new();
3919    let mut cur_list_start: Option<i64> = None;
3920    let mut cur_list_indent: u32 = 0;
3921    let mut cur_marker: Option<MarkerType> = None;
3922
3923    // Table accumulation.
3924    let mut in_table_cell = false;
3925    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3926    let mut current_row: Vec<ParsedTableCell> = Vec::new();
3927    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3928    let mut table_header_rows: usize = 0;
3929    let mut row_is_head = false;
3930
3931    // Subtree-skip depth for unrepresentable containers (their entire content
3932    // is dropped). Incremented on the dropped container's `Start` and on every
3933    // nested `Start`; decremented on every `End`.
3934    let mut skip_depth: u32 = 0;
3935
3936    // The label and element index of the footnote definition currently open,
3937    // if any. Djot has no footnote inside a footnote, so one slot is enough
3938    // where the dropped containers need a depth counter.
3939    let mut footnote_open: Option<(String, usize)> = None;
3940
3941    // Push one inline span carrying the current formatting state into the
3942    // active sink (table cell or block). A macro (not a closure) to avoid
3943    // borrowing `current_spans`/`current_cell_spans` across the formatting
3944    // state reads.
3945    macro_rules! push_text {
3946        ($t:expr) => {{
3947            // Alt text belongs to the image, not to the paragraph. While an
3948            // image is open every text event is diverted into its description,
3949            // which is what keeps a photo's caption out of the manuscript's
3950            // word count and out of the search corpus.
3951            if let Some(img) = pending_image.as_mut() {
3952                img.alt.push_str(($t).as_ref());
3953            } else {
3954                let sp = ParsedSpan {
3955                    text: ($t).to_string(),
3956                    bold,
3957                    italic,
3958                    underline,
3959                    strikeout,
3960                    code,
3961                    superscript,
3962                    subscript,
3963                    link_href: link_href.clone(),
3964                    image: None,
3965                    footnote_ref: None,
3966                    preserve_whitespace: false,
3967                };
3968                if in_table_cell {
3969                    current_cell_spans.push(sp);
3970                } else {
3971                    current_spans.push(sp);
3972                }
3973            }
3974        }};
3975    }
3976
3977    // Push a completed inline image span into the active sink.
3978    macro_rules! push_image {
3979        ($img:expr) => {{
3980            let sp = ParsedSpan {
3981                text: String::new(),
3982                bold,
3983                italic,
3984                underline,
3985                strikeout,
3986                code,
3987                superscript,
3988                subscript,
3989                link_href: link_href.clone(),
3990                image: Some($img),
3991                footnote_ref: None,
3992                preserve_whitespace: false,
3993            };
3994            if in_table_cell {
3995                current_cell_spans.push(sp);
3996            } else {
3997                current_spans.push(sp);
3998            }
3999        }};
4000    }
4001
4002    // Enter a list item, flushing any unterminated inline content first and
4003    // capturing the list context + task marker for the item's block.
4004    macro_rules! enter_item {
4005        ($marker:expr) => {{
4006            if !current_spans.is_empty() {
4007                djot_push_block(
4008                    &mut elements,
4009                    std::mem::take(&mut current_spans),
4010                    None,
4011                    cur_list_style.clone(),
4012                    cur_list_indent,
4013                    cur_list_prefix.clone(),
4014                    cur_list_suffix.clone(),
4015                    cur_list_start,
4016                    cur_marker.clone(),
4017                    false,
4018                    None,
4019                    blockquote_depth,
4020                    DjotBlockStyle::default(),
4021                );
4022            }
4023            let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
4024                ListStyle::Disc,
4025                String::new(),
4026                String::new(),
4027                None,
4028            ));
4029            cur_list_style = Some(style);
4030            cur_list_prefix = prefix;
4031            cur_list_suffix = suffix;
4032            cur_list_start = start;
4033            cur_list_indent = list_stack.len().saturating_sub(1) as u32;
4034            cur_marker = $marker;
4035        }};
4036    }
4037
4038    // A hard break: a line break inside the block, [`LINE_SEPARATOR`] in its text, so the
4039    // next line is more of the same paragraph, list item, quotation or heading. A table
4040    // cell is one line, and an image's description is one too, so there it is a space.
4041    macro_rules! line_break {
4042        () => {{
4043            if in_table_cell || pending_image.is_some() {
4044                push_text!(" ");
4045            } else {
4046                push_text!(LINE_SEPARATOR_STR);
4047            }
4048        }};
4049    }
4050
4051    // A line ending inside a code span (see `E::Str` below): a new block, with the list,
4052    // quotation, heading and block style the line was in. A table cell is one line, so
4053    // there it is a space.
4054    //
4055    // The page break is the one style that stays with the first line: it is where the
4056    // block starts, and copied to every line it put each on a page of its own.
4057    macro_rules! split_line {
4058        () => {{
4059            if in_table_cell {
4060                push_text!(" ");
4061            } else if !current_spans.is_empty() && current_heading.is_some() {
4062                // The line before a hard break in a heading is a line of that
4063                // heading, as the line after it is when the heading ends. Read as
4064                // a paragraph, a heading written over two lines came back as a
4065                // paragraph and a heading.
4066                djot_push_block(
4067                    &mut elements,
4068                    std::mem::take(&mut current_spans),
4069                    current_heading,
4070                    None,
4071                    0,
4072                    String::new(),
4073                    String::new(),
4074                    None,
4075                    None,
4076                    false,
4077                    None,
4078                    blockquote_depth,
4079                    pending_style.clone(),
4080                );
4081                pending_style.page_break_before = None;
4082            } else if !current_spans.is_empty() {
4083                djot_push_block(
4084                    &mut elements,
4085                    std::mem::take(&mut current_spans),
4086                    None,
4087                    cur_list_style.clone(),
4088                    cur_list_indent,
4089                    cur_list_prefix.clone(),
4090                    cur_list_suffix.clone(),
4091                    cur_list_start,
4092                    cur_marker.clone(),
4093                    is_code_block,
4094                    code_language.clone(),
4095                    blockquote_depth,
4096                    pending_style.clone(),
4097                );
4098                pending_style.page_break_before = None;
4099            }
4100        }};
4101    }
4102
4103    for event in events {
4104        if skip_depth > 0 {
4105            match event {
4106                E::Start(..) => skip_depth += 1,
4107                E::End(_) => skip_depth -= 1,
4108                _ => {}
4109            }
4110            continue;
4111        }
4112
4113        match event {
4114            // ── Transparent wrappers (unwrap, keep content) ──
4115            E::Start(C::Document, _) | E::End(C::Document) => {}
4116            E::Start(C::Section { .. }, attrs) => {
4117                // A heading's block attributes attach to its enclosing Section,
4118                // not the heading itself; capture them for the heading's flush.
4119                if list_stack.is_empty() {
4120                    pending_style.merge_from(block_attrs_to_style(&attrs, options));
4121                }
4122            }
4123            E::End(C::Section { .. }) => {}
4124            E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
4125
4126            // ── Blockquote ──
4127            E::Start(C::Blockquote, _) => {
4128                blockquote_depth += 1;
4129                quotations_open.push(elements.len());
4130            }
4131            E::End(C::Blockquote) => {
4132                blockquote_depth = blockquote_depth.saturating_sub(1);
4133                if let Some(first) = quotations_open.pop() {
4134                    mark_quotation_opened(&mut elements, first);
4135                }
4136            }
4137
4138            // ── Lists ──
4139            E::Start(C::List { kind, .. }, _) => {
4140                let (style, prefix, suffix, start) = match kind {
4141                    ListKind::Unordered(b) | ListKind::Task(b) => {
4142                        (djot_bullet_style(b), String::new(), String::new(), None)
4143                    }
4144                    ListKind::Ordered {
4145                        numbering,
4146                        style,
4147                        start,
4148                    } => {
4149                        let (p, s) = djot_ordered_affixes(style);
4150                        (djot_ordered_style(numbering), p, s, list_start_of(start))
4151                    }
4152                };
4153                list_stack.push((style, prefix, suffix, start));
4154            }
4155            E::End(C::List { .. }) => {
4156                list_stack.pop();
4157                cur_list_style = None;
4158                cur_list_start = None;
4159                cur_marker = None;
4160            }
4161            E::Start(C::ListItem, _) => enter_item!(None),
4162            E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
4163                MarkerType::Checked
4164            } else {
4165                MarkerType::Unchecked
4166            })),
4167            E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
4168                // Tight item without a wrapping paragraph (defensive flush).
4169                if !current_spans.is_empty() {
4170                    djot_push_block(
4171                        &mut elements,
4172                        std::mem::take(&mut current_spans),
4173                        None,
4174                        cur_list_style.clone(),
4175                        cur_list_indent,
4176                        cur_list_prefix.clone(),
4177                        cur_list_suffix.clone(),
4178                        cur_list_start,
4179                        cur_marker.clone(),
4180                        false,
4181                        None,
4182                        blockquote_depth,
4183                        DjotBlockStyle::default(),
4184                    );
4185                }
4186                cur_list_style = None;
4187                cur_list_start = None;
4188                cur_marker = None;
4189            }
4190
4191            // ── Headings, paragraphs, code blocks ──
4192            E::Start(C::Heading { level, .. }, attrs) => {
4193                current_heading = Some(level as i64);
4194                // The block-style attributes live on the enclosing Section;
4195                // merge any placed directly on the heading without clearing them.
4196                pending_style.merge_from(block_attrs_to_style(&attrs, options));
4197            }
4198            E::End(C::Heading { .. }) => {
4199                djot_push_block(
4200                    &mut elements,
4201                    std::mem::take(&mut current_spans),
4202                    current_heading.take(),
4203                    None,
4204                    0,
4205                    String::new(),
4206                    String::new(),
4207                    None,
4208                    None,
4209                    false,
4210                    None,
4211                    blockquote_depth,
4212                    std::mem::take(&mut pending_style),
4213                );
4214            }
4215            E::Start(C::Paragraph, attrs) => {
4216                current_heading = None;
4217                // Block attributes only apply to standalone paragraphs;
4218                // list-item paragraphs normalise their styling away (matching
4219                // the exporter).
4220                pending_style = if list_stack.is_empty() {
4221                    block_attrs_to_style(&attrs, options)
4222                } else {
4223                    DjotBlockStyle::default()
4224                };
4225            }
4226            E::End(C::Paragraph) => {
4227                if !current_spans.is_empty() {
4228                    djot_push_block(
4229                        &mut elements,
4230                        std::mem::take(&mut current_spans),
4231                        None,
4232                        cur_list_style.clone(),
4233                        cur_list_indent,
4234                        cur_list_prefix.clone(),
4235                        cur_list_suffix.clone(),
4236                        cur_list_start,
4237                        cur_marker.clone(),
4238                        false,
4239                        None,
4240                        blockquote_depth,
4241                        std::mem::take(&mut pending_style),
4242                    );
4243                }
4244                cur_list_style = None;
4245                cur_list_start = None;
4246                cur_marker = None;
4247            }
4248            E::Start(C::CodeBlock { language }, _) => {
4249                is_code_block = true;
4250                code_language = if language.is_empty() {
4251                    None
4252                } else {
4253                    Some(language.to_string())
4254                };
4255            }
4256            E::End(C::CodeBlock { .. }) => {
4257                // Strip the single trailing newline jotdown appends.
4258                if let Some(last) = current_spans.last_mut()
4259                    && last.text.ends_with('\n')
4260                {
4261                    last.text.pop();
4262                }
4263                djot_push_block(
4264                    &mut elements,
4265                    std::mem::take(&mut current_spans),
4266                    None,
4267                    None,
4268                    0,
4269                    String::new(),
4270                    String::new(),
4271                    None,
4272                    None,
4273                    true,
4274                    code_language.take(),
4275                    blockquote_depth,
4276                    DjotBlockStyle::default(),
4277                );
4278                is_code_block = false;
4279            }
4280
4281            // ── Tables ──
4282            E::Start(C::Table, _) => {
4283                table_rows.clear();
4284                current_row.clear();
4285                current_cell_spans.clear();
4286                table_header_rows = 0;
4287            }
4288            E::End(C::Table) => {
4289                let own_cells = cells_in_rows(&table_rows);
4290                push_table(
4291                    &mut elements,
4292                    ParsedTable {
4293                        header_rows: table_header_rows,
4294                        rows: std::mem::take(&mut table_rows),
4295                        blockquote_depth,
4296                        blockquote_opens: 0,
4297                    },
4298                    own_cells,
4299                );
4300            }
4301            E::Start(C::TableRow { head }, _) => {
4302                row_is_head = head;
4303                current_row.clear();
4304            }
4305            E::End(C::TableRow { .. }) => {
4306                if row_is_head {
4307                    table_header_rows += 1;
4308                }
4309                table_rows.push(std::mem::take(&mut current_row));
4310            }
4311            E::Start(C::TableCell { .. }, _) => {
4312                in_table_cell = true;
4313                current_cell_spans.clear();
4314            }
4315            E::End(C::TableCell { .. }) => {
4316                in_table_cell = false;
4317                current_row.push(ParsedTableCell {
4318                    spans: std::mem::take(&mut current_cell_spans),
4319                });
4320            }
4321
4322            // ── Inline formatting ──
4323            E::Start(C::Strong, _) => bold = true,
4324            E::End(C::Strong) => bold = false,
4325            E::Start(C::Emphasis, _) => italic = true,
4326            E::End(C::Emphasis) => italic = false,
4327            E::Start(C::Verbatim, _) => code = true,
4328            E::End(C::Verbatim) => code = false,
4329            E::Start(C::Superscript, _) => superscript = true,
4330            E::End(C::Superscript) => superscript = false,
4331            E::Start(C::Subscript, _) => subscript = true,
4332            E::End(C::Subscript) => subscript = false,
4333            E::Start(C::Insert, _) => underline = true,
4334            E::End(C::Insert) => underline = false,
4335            E::Start(C::Delete, _) => strikeout = true,
4336            E::End(C::Delete) => strikeout = false,
4337            // Highlight/mark and bare spans have no model field — keep the text.
4338            E::Start(C::Mark, _) | E::End(C::Mark) => {}
4339            E::Start(C::Span, _) | E::End(C::Span) => {}
4340            E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
4341            E::End(C::Link(..)) => link_href = None,
4342            // Inline images. Djot writes display size as inline attributes
4343            // (`![alt](src){width=800 height=600}`), which jotdown hands over
4344            // on the `Start` event — verified against jotdown 0.10, including
4345            // quoted values and images mid-sentence.
4346            E::Start(C::Image(src, _), attrs) => {
4347                let attr_num = |key: &str| -> i64 {
4348                    attrs
4349                        .get_value(key)
4350                        .map(|v| v.to_string())
4351                        .and_then(|v| v.trim().parse::<i64>().ok())
4352                        .filter(|n| *n > 0)
4353                        .unwrap_or(0)
4354                };
4355                pending_image = Some(ParsedImage {
4356                    // The writer percent-encodes what the lexer would act on in a
4357                    // source; the source is a name, so it is decoded back.
4358                    src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
4359                    alt: String::new(),
4360                    width: attr_num("width"),
4361                    height: attr_num("height"),
4362                });
4363            }
4364            E::End(C::Image(..)) => {
4365                if let Some(img) = pending_image.take() {
4366                    push_image!(img);
4367                }
4368            }
4369
4370            // ── Footnote definitions ──
4371            //
4372            // The body is ordinary block content, so it is parsed by the
4373            // ordinary machinery: flush whatever inline run is open, note where
4374            // this definition's blocks start, and let them accumulate. `End`
4375            // lifts them back out. Nesting cannot occur — djot has no footnote
4376            // inside a footnote — so one mark suffices where the dropped
4377            // containers below need a depth counter.
4378            E::Start(C::Footnote { label }, _) => {
4379                if !current_spans.is_empty() {
4380                    djot_push_block(
4381                        &mut elements,
4382                        std::mem::take(&mut current_spans),
4383                        None,
4384                        cur_list_style.clone(),
4385                        cur_list_indent,
4386                        cur_list_prefix.clone(),
4387                        cur_list_suffix.clone(),
4388                        cur_list_start,
4389                        cur_marker.clone(),
4390                        false,
4391                        None,
4392                        blockquote_depth,
4393                        DjotBlockStyle::default(),
4394                    );
4395                }
4396                footnote_open = Some((label.to_string(), elements.len()));
4397            }
4398            E::End(C::Footnote { .. }) => {
4399                if !current_spans.is_empty() {
4400                    djot_push_block(
4401                        &mut elements,
4402                        std::mem::take(&mut current_spans),
4403                        None,
4404                        cur_list_style.clone(),
4405                        cur_list_indent,
4406                        cur_list_prefix.clone(),
4407                        cur_list_suffix.clone(),
4408                        cur_list_start,
4409                        cur_marker.clone(),
4410                        false,
4411                        None,
4412                        blockquote_depth,
4413                        DjotBlockStyle::default(),
4414                    );
4415                }
4416                if let Some((label, start)) = footnote_open.take() {
4417                    let blocks: Vec<ParsedBlock> = elements
4418                        .drain(start..)
4419                        .filter_map(|e| match e {
4420                            ParsedElement::Block(b) => Some(b),
4421                            // A table inside a footnote is not representable as
4422                            // note content; its cells would have to become
4423                            // blocks and lose their structure either way.
4424                            _ => None,
4425                        })
4426                        .collect();
4427                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
4428                }
4429            }
4430
4431            // ── Unrepresentable containers: drop the entire subtree ──
4432            E::Start(
4433                C::Math { .. }
4434                | C::RawBlock { .. }
4435                | C::RawInline { .. }
4436                | C::DescriptionList
4437                | C::DescriptionDetails
4438                | C::DescriptionTerm
4439                | C::Caption
4440                | C::LinkDefinition { .. },
4441                _,
4442            ) => skip_depth = 1,
4443
4444            // ── Text + atoms ──
4445            // A line ending inside a code span is kept by the parser as a `\n` in the
4446            // span's text. It is read as a new block with the list, quotation or heading
4447            // the line was in, as a `\n` pasted into a block is (see
4448            // `split_block_at_line_breaks`). Kept inside the span, the Djot writer saved
4449            // the code run's lines as blocks of their own, which reloaded as two blocks: a
4450            // document that did not settle into the Djot it saves as. Such a span is what
4451            // an older version wrote for a code run holding a line break, a pasted
4452            // `<pre><code>` passage above all, and reading its lines as blocks is how that
4453            // passage reads in the editor. It is not a hard break: nothing in it asked for
4454            // a line of the same paragraph.
4455            E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
4456                for (i, piece) in s.split('\n').enumerate() {
4457                    if i > 0 {
4458                        split_line!();
4459                    }
4460                    if !piece.is_empty() {
4461                        push_text!(piece);
4462                    }
4463                }
4464            }
4465            E::Str(s) => push_text!(s.as_ref()),
4466            E::Softbreak => push_text!(" "),
4467            E::LeftSingleQuote => push_text!("\u{2018}"),
4468            E::RightSingleQuote => push_text!("\u{2019}"),
4469            E::LeftDoubleQuote => push_text!("\u{201C}"),
4470            E::RightDoubleQuote => push_text!("\u{201D}"),
4471            E::Ellipsis => push_text!("\u{2026}"),
4472            E::EnDash => push_text!("\u{2013}"),
4473            E::EmDash => push_text!("\u{2014}"),
4474            E::NonBreakingSpace => push_text!("\u{00A0}"),
4475            E::Hardbreak => line_break!(),
4476            // A footnote reference. jotdown emits this purely syntactically —
4477            // it never checks that a matching `[^label]:` exists anywhere — so a
4478            // reference whose definition lives outside this document (the normal
4479            // state for a host that owns note bodies itself) arrives here just
4480            // the same, and must survive.
4481            E::FootnoteReference(label) => {
4482                let sp = ParsedSpan {
4483                    text: String::new(),
4484                    bold,
4485                    italic,
4486                    underline,
4487                    strikeout,
4488                    code,
4489                    superscript,
4490                    subscript,
4491                    link_href: link_href.clone(),
4492                    image: None,
4493                    footnote_ref: Some(label.to_string()),
4494                    preserve_whitespace: false,
4495                };
4496                if in_table_cell {
4497                    current_cell_spans.push(sp);
4498                } else {
4499                    current_spans.push(sp);
4500                }
4501            }
4502            // Symbols, escapes, blanklines, thematic breaks and dangling block
4503            // attributes carry no representable content.
4504            E::Symbol(_) => {}
4505            E::Escape | E::Blankline => {}
4506            E::ThematicBreak(_) | E::Attributes(_) => {}
4507
4508            // Ends of dropped containers (never reached at skip_depth 0) and any
4509            // future variants.
4510            _ => {}
4511        }
4512    }
4513
4514    // Flush any trailing inline content (defensive — Document End closes blocks).
4515    if !current_spans.is_empty() {
4516        djot_push_block(
4517            &mut elements,
4518            std::mem::take(&mut current_spans),
4519            current_heading.take(),
4520            cur_list_style.clone(),
4521            cur_list_indent,
4522            cur_list_prefix.clone(),
4523            cur_list_suffix.clone(),
4524            cur_list_start,
4525            cur_marker.clone(),
4526            is_code_block,
4527            code_language.take(),
4528            blockquote_depth,
4529            std::mem::take(&mut pending_style),
4530        );
4531    }
4532
4533    // An empty document still yields a single empty paragraph (matches
4534    // `parse_markdown`).
4535    if elements.is_empty() {
4536        djot_push_block(
4537            &mut elements,
4538            vec![ParsedSpan {
4539                text: String::new(),
4540                ..Default::default()
4541            }],
4542            None,
4543            None,
4544            0,
4545            String::new(),
4546            String::new(),
4547            None,
4548            None,
4549            false,
4550            None,
4551            0,
4552            DjotBlockStyle::default(),
4553        );
4554    }
4555
4556    elements
4557}
4558
4559#[cfg(test)]
4560mod tests {
4561    use super::*;
4562
4563    #[test]
4564    fn test_style_keeps_whitespace_reads_important() {
4565        assert_eq!(style_keeps_whitespace("white-space: pre-wrap"), Some(true));
4566        assert_eq!(
4567            style_keeps_whitespace("white-space: pre-wrap !important"),
4568            Some(true)
4569        );
4570        assert_eq!(
4571            style_keeps_whitespace("white-space:PRE!IMPORTANT; color: red"),
4572            Some(true)
4573        );
4574        assert_eq!(
4575            style_keeps_whitespace("white-space: pre ! important; white-space: normal"),
4576            Some(true),
4577            "a later declaration that is not important does not win"
4578        );
4579        assert_eq!(
4580            style_keeps_whitespace("white-space: pre; white-space: normal !important"),
4581            Some(false)
4582        );
4583        assert_eq!(
4584            style_keeps_whitespace("white-space: pre; white-space: normal"),
4585            Some(false)
4586        );
4587        assert_eq!(style_keeps_whitespace("white-space: inherit"), None);
4588    }
4589
4590    /// Helper: flatten parse_markdown output to blocks for tests that don't care about tables.
4591    fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
4592        ParsedElement::flatten_to_blocks(parse_markdown(md))
4593    }
4594
4595    #[test]
4596    fn test_parse_markdown_simple_paragraph() {
4597        let blocks = parse_markdown_blocks("Hello **world**");
4598        assert_eq!(blocks.len(), 1);
4599        assert!(blocks[0].spans.len() >= 2);
4600        // "Hello " is plain, "world" is bold
4601        let plain_span = blocks[0]
4602            .spans
4603            .iter()
4604            .find(|s| s.text.contains("Hello"))
4605            .unwrap();
4606        assert!(!plain_span.bold);
4607        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
4608        assert!(bold_span.bold);
4609    }
4610
4611    #[test]
4612    fn test_parse_markdown_heading() {
4613        let blocks = parse_markdown_blocks("# Title");
4614        assert_eq!(blocks.len(), 1);
4615        assert_eq!(blocks[0].heading_level, Some(1));
4616        assert_eq!(blocks[0].spans[0].text, "Title");
4617    }
4618
4619    #[test]
4620    fn test_parse_markdown_list() {
4621        let blocks = parse_markdown_blocks("- item1\n- item2");
4622        assert!(blocks.len() >= 2);
4623        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4624        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4625    }
4626
4627    /// Helper: extract (is_table, blockquote_depth) per element for nesting assertions.
4628    fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
4629        elements
4630            .iter()
4631            .map(|e| match e {
4632                ParsedElement::Block(b) => (false, b.blockquote_depth),
4633                ParsedElement::Table(t) => (true, t.blockquote_depth),
4634                // Definitions carry no blockquote nesting of their own; this
4635                // helper exists for the nesting assertions and never sees one.
4636                ParsedElement::FootnoteDefinition { .. } => (false, 0),
4637            })
4638            .collect()
4639    }
4640
4641    #[test]
4642    fn test_parse_markdown_table_in_blockquote_records_depth() {
4643        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
4644        assert_eq!(element_depths(&elements), vec![(true, 1)]);
4645    }
4646
4647    #[test]
4648    fn test_parse_markdown_text_then_table_in_blockquote() {
4649        let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
4650        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
4651    }
4652
4653    #[test]
4654    fn test_parse_markdown_table_after_blockquote_closes() {
4655        let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
4656        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
4657    }
4658
4659    #[test]
4660    fn test_parse_markdown_table_in_nested_blockquote() {
4661        let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
4662        assert_eq!(element_depths(&elements), vec![(true, 2)]);
4663    }
4664
4665    #[test]
4666    fn test_parse_markdown_list_in_blockquote_records_depth() {
4667        let elements = parse_markdown("> - item1\n> - item2");
4668        let depths = element_depths(&elements);
4669        assert_eq!(depths, vec![(false, 1), (false, 1)]);
4670        for e in &elements {
4671            if let ParsedElement::Block(b) = e {
4672                assert_eq!(b.list_style, Some(ListStyle::Disc));
4673            }
4674        }
4675    }
4676
4677    #[test]
4678    fn test_parse_html_table_in_blockquote_records_depth() {
4679        let elements = parse_html_elements(
4680            "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
4681        );
4682        assert_eq!(element_depths(&elements), vec![(true, 1)]);
4683    }
4684
4685    #[test]
4686    fn test_parse_html_table_after_blockquote() {
4687        let elements = parse_html_elements(
4688            "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
4689        );
4690        let depths = element_depths(&elements);
4691        // The blockquote paragraph carries depth 1; the table is outside (depth 0).
4692        assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
4693        assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
4694    }
4695
4696    #[test]
4697    fn test_flatten_to_blocks_propagates_blockquote_depth() {
4698        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
4699        let blocks = ParsedElement::flatten_to_blocks(elements);
4700        assert!(!blocks.is_empty());
4701        for b in &blocks {
4702            assert_eq!(b.blockquote_depth, 1);
4703        }
4704    }
4705
4706    #[test]
4707    fn test_parse_html_simple() {
4708        let blocks = parse_html("<p>Hello <b>world</b></p>");
4709        assert_eq!(blocks.len(), 1);
4710        assert!(blocks[0].spans.len() >= 2);
4711        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
4712        assert!(bold_span.bold);
4713    }
4714
4715    #[test]
4716    fn test_parse_html_multiple_paragraphs() {
4717        let blocks = parse_html("<p>A</p><p>B</p>");
4718        assert_eq!(blocks.len(), 2);
4719    }
4720
4721    #[test]
4722    fn test_parse_html_heading() {
4723        let blocks = parse_html("<h2>Subtitle</h2>");
4724        assert_eq!(blocks.len(), 1);
4725        assert_eq!(blocks[0].heading_level, Some(2));
4726    }
4727
4728    #[test]
4729    fn test_parse_html_list() {
4730        let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
4731        assert!(blocks.len() >= 2);
4732        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4733    }
4734
4735    #[test]
4736    fn test_parse_markdown_code_block() {
4737        let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
4738        assert_eq!(blocks.len(), 1);
4739        assert!(blocks[0].is_code_block);
4740        assert!(blocks[0].spans[0].code);
4741        // pulldown-cmark appends a trailing \n to code block text — verify it's stripped
4742        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4743        assert_eq!(
4744            text, "fn main() {}",
4745            "code block text should not have trailing newline"
4746        );
4747    }
4748
4749    #[test]
4750    fn test_parse_markdown_nested_formatting() {
4751        let blocks = parse_markdown_blocks("***bold italic***");
4752        assert_eq!(blocks.len(), 1);
4753        let span = &blocks[0].spans[0];
4754        assert!(span.bold);
4755        assert!(span.italic);
4756    }
4757
4758    #[test]
4759    fn test_parse_markdown_link() {
4760        let blocks = parse_markdown_blocks("[click](http://example.com)");
4761        assert_eq!(blocks.len(), 1);
4762        let span = &blocks[0].spans[0];
4763        assert_eq!(span.text, "click");
4764        assert_eq!(span.link_href, Some("http://example.com".to_string()));
4765    }
4766
4767    #[test]
4768    fn test_parse_markdown_empty() {
4769        let blocks = parse_markdown_blocks("");
4770        assert_eq!(blocks.len(), 1);
4771        assert!(blocks[0].spans[0].text.is_empty());
4772    }
4773
4774    #[test]
4775    fn test_parse_html_empty() {
4776        let blocks = parse_html("");
4777        assert_eq!(blocks.len(), 1);
4778        assert!(blocks[0].spans[0].text.is_empty());
4779    }
4780
4781    #[test]
4782    fn test_parse_html_nested_formatting() {
4783        let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
4784        assert_eq!(blocks.len(), 1);
4785        let span = &blocks[0].spans[0];
4786        assert!(span.bold);
4787        assert!(span.italic);
4788    }
4789
4790    #[test]
4791    fn test_parse_html_link() {
4792        let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
4793        assert_eq!(blocks.len(), 1);
4794        let span = &blocks[0].spans[0];
4795        assert_eq!(span.text, "click");
4796        assert_eq!(span.link_href, Some("http://example.com".to_string()));
4797    }
4798
4799    #[test]
4800    fn test_parse_html_ordered_list() {
4801        let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
4802        assert!(blocks.len() >= 2);
4803        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4804    }
4805
4806    #[test]
4807    fn test_parse_markdown_ordered_list() {
4808        let blocks = parse_markdown_blocks("1. first\n2. second");
4809        assert!(blocks.len() >= 2);
4810        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4811    }
4812
4813    #[test]
4814    fn test_parse_html_blockquote_nested() {
4815        let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
4816        assert!(blocks.len() >= 3);
4817    }
4818
4819    #[test]
4820    fn test_parse_block_styles_line_height() {
4821        let styles = parse_block_styles("line-height: 1.5");
4822        assert_eq!(styles.line_height, Some(1500));
4823    }
4824
4825    #[test]
4826    fn test_parse_block_styles_direction_rtl() {
4827        let styles = parse_block_styles("direction: rtl");
4828        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4829    }
4830
4831    #[test]
4832    fn test_parse_block_styles_background_color() {
4833        let styles = parse_block_styles("background-color: #ff0000");
4834        assert_eq!(styles.background_color, Some("#ff0000".to_string()));
4835    }
4836
4837    #[test]
4838    fn test_parse_block_styles_white_space_pre() {
4839        let styles = parse_block_styles("white-space: pre");
4840        assert_eq!(styles.non_breakable_lines, Some(true));
4841    }
4842
4843    #[test]
4844    fn test_parse_block_styles_multiple() {
4845        let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
4846        assert_eq!(styles.line_height, Some(2000));
4847        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4848        assert_eq!(styles.background_color, Some("blue".to_string()));
4849    }
4850
4851    #[test]
4852    fn test_parse_html_block_styles_extracted() {
4853        let blocks = parse_html(
4854            r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
4855        );
4856        assert_eq!(blocks.len(), 1);
4857        assert_eq!(blocks[0].line_height, Some(1500));
4858        assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
4859        assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
4860    }
4861
4862    /// The shape every word processor publishes on the clipboard: a newline
4863    /// after the opening tag, the prose hard-wrapped inside it, and a newline
4864    /// between every pair of blocks. None of it is text.
4865    #[test]
4866    fn test_parse_html_collapses_exporter_line_wrapping() {
4867        let blocks = parse_html(
4868            "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
4869             <p>\nsecond para</p>\n</body>",
4870        );
4871        let texts: Vec<String> = blocks
4872            .iter()
4873            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4874            .collect();
4875        assert_eq!(texts, vec!["first line wrapped here", "second para"]);
4876    }
4877
4878    /// The reported bug: pasting a whole LibreOffice book ended in one block
4879    /// holding every newline the exporter had written between its paragraphs —
4880    /// four thousand blank lines under the last sentence.
4881    #[test]
4882    fn test_parse_html_drops_whitespace_between_blocks() {
4883        let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
4884        assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
4885        assert!(
4886            blocks
4887                .iter()
4888                .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
4889            "no block keeps a literal newline"
4890        );
4891    }
4892
4893    /// A run split across inline elements is still one run, and it collapses to
4894    /// one space rather than disappearing between them.
4895    #[test]
4896    fn test_parse_html_collapses_across_span_boundaries() {
4897        let blocks = parse_html("<p>  <b>bold</b>\n  <i>italic</i>  </p>");
4898        assert_eq!(blocks.len(), 1);
4899        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4900        assert_eq!(text, "bold italic");
4901    }
4902
4903    /// U+00A0 is content, not layout — the space holding "Attention&nbsp;:"
4904    /// together in French must survive a paste.
4905    #[test]
4906    fn test_parse_html_keeps_no_break_space() {
4907        let blocks = parse_html("<p>\nAttention&nbsp;: ici</p>");
4908        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4909        assert_eq!(text, "Attention\u{a0}: ici");
4910    }
4911
4912    /// An empty paragraph is a paragraph. Only the whitespace *between* blocks
4913    /// goes; a block the writer opened stays, however little is in it.
4914    #[test]
4915    fn test_parse_html_keeps_empty_paragraph() {
4916        let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
4917        assert_eq!(blocks.len(), 3);
4918        let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
4919        assert_eq!(text, "");
4920    }
4921
4922    /// `<pre>` and `white-space: pre` mean the source's spacing *is* the
4923    /// content. `nowrap` does not: it forbids wrapping and collapses runs like
4924    /// any other block, which is why it is not in the same test.
4925    #[test]
4926    fn test_parse_html_preserves_whitespace_in_pre() {
4927        // The newline straight after `<pre>` is the one the HTML parser itself
4928        // drops, per spec; every newline after that is content.
4929        let blocks = parse_html("<pre>\nfn main() {\n    let x = 1;\n}</pre>");
4930        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4931        assert_eq!(text, "fn main() {\n    let x = 1;\n}");
4932
4933        // Outside a code block a line break is a new block, and each line keeps its
4934        // spaces.
4935        let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n  b</p>");
4936        let texts: Vec<String> = styled
4937            .iter()
4938            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4939            .collect();
4940        assert_eq!(texts, ["a", "  b"]);
4941
4942        let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n  b</p>");
4943        let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4944        assert_eq!(text, "a b");
4945    }
4946
4947    /// Table cells are blocks too, and an exporter indents them just as
4948    /// generously as it indents paragraphs.
4949    #[test]
4950    fn test_parse_html_collapses_table_cell_whitespace() {
4951        let elements = parse_html_elements(
4952            "<table>\n<tr>\n<td>\n  one\n</td>\n<td>\n  two\n</td>\n</tr>\n</table>",
4953        );
4954        let table = elements
4955            .iter()
4956            .find_map(|e| match e {
4957                ParsedElement::Table(t) => Some(t),
4958                _ => None,
4959            })
4960            .expect("a table");
4961        let cells: Vec<String> = table.rows[0]
4962            .iter()
4963            .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4964            .collect();
4965        assert_eq!(cells, vec!["one", "two"]);
4966    }
4967
4968    #[test]
4969    fn test_parse_html_white_space_pre() {
4970        let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4971        assert_eq!(blocks.len(), 1);
4972        assert_eq!(blocks[0].non_breakable_lines, Some(true));
4973    }
4974
4975    #[test]
4976    fn test_parse_html_no_styles_returns_none() {
4977        let blocks = parse_html("<p>plain</p>");
4978        assert_eq!(blocks.len(), 1);
4979        assert_eq!(blocks[0].line_height, None);
4980        assert_eq!(blocks[0].direction, None);
4981        assert_eq!(blocks[0].background_color, None);
4982        assert_eq!(blocks[0].non_breakable_lines, None);
4983    }
4984
4985    #[test]
4986    fn test_parse_markdown_nested_list_indent() {
4987        let md = "- top\n  - nested\n    - deep";
4988        let blocks = parse_markdown_blocks(md);
4989        assert_eq!(blocks.len(), 3);
4990        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4991        assert_eq!(blocks[0].list_indent, 0);
4992        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4993        assert_eq!(blocks[1].list_indent, 1);
4994        assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4995        assert_eq!(blocks[2].list_indent, 2);
4996    }
4997
4998    #[test]
4999    fn test_parse_markdown_nested_ordered_list_indent() {
5000        let md = "1. first\n   1. nested\n   2. nested2";
5001        let blocks = parse_markdown_blocks(md);
5002        assert_eq!(blocks.len(), 3);
5003        assert_eq!(blocks[0].list_indent, 0);
5004        assert_eq!(blocks[1].list_indent, 1);
5005        assert_eq!(blocks[2].list_indent, 1);
5006    }
5007
5008    #[test]
5009    fn test_parse_html_nested_list_indent() {
5010        let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
5011        let blocks = parse_html(html);
5012        assert!(blocks.len() >= 2);
5013        assert_eq!(blocks[0].list_indent, 0);
5014        assert_eq!(blocks[1].list_indent, 1);
5015    }
5016
5017    #[test]
5018    fn test_parse_markdown_table() {
5019        let md = "| A | B |\n|---|---|\n| 1 | 2 |";
5020        let elements = parse_markdown(md);
5021        assert_eq!(elements.len(), 1);
5022        match &elements[0] {
5023            ParsedElement::Table(table) => {
5024                assert_eq!(table.header_rows, 1);
5025                assert_eq!(table.rows.len(), 2); // 1 header + 1 body
5026                // Header row
5027                assert_eq!(table.rows[0].len(), 2);
5028                assert_eq!(table.rows[0][0].spans[0].text, "A");
5029                assert_eq!(table.rows[0][1].spans[0].text, "B");
5030                // Body row
5031                assert_eq!(table.rows[1].len(), 2);
5032                assert_eq!(table.rows[1][0].spans[0].text, "1");
5033                assert_eq!(table.rows[1][1].spans[0].text, "2");
5034            }
5035            _ => panic!("Expected ParsedElement::Table"),
5036        }
5037    }
5038
5039    #[test]
5040    fn test_parse_markdown_table_with_formatting() {
5041        let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
5042        let elements = parse_markdown(md);
5043        assert_eq!(elements.len(), 1);
5044        match &elements[0] {
5045            ParsedElement::Table(table) => {
5046                assert_eq!(table.rows.len(), 2);
5047                // Header: bold cell
5048                assert!(table.rows[0][0].spans[0].bold);
5049                // Header: code cell
5050                assert!(table.rows[0][1].spans[0].code);
5051                // Header: italic cell
5052                assert!(table.rows[0][2].spans[0].italic);
5053                // Body: strikeout cell
5054                assert!(table.rows[1][0].spans[0].strikeout);
5055                // Body: link cell
5056                assert_eq!(
5057                    table.rows[1][2].spans[0].link_href,
5058                    Some("http://x.com".to_string())
5059                );
5060            }
5061            _ => panic!("Expected ParsedElement::Table"),
5062        }
5063    }
5064
5065    #[test]
5066    fn test_parse_markdown_mixed_content_with_table() {
5067        let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
5068        let elements = parse_markdown(md);
5069        assert_eq!(elements.len(), 3);
5070        assert!(matches!(&elements[0], ParsedElement::Block(_)));
5071        assert!(matches!(&elements[1], ParsedElement::Table(_)));
5072        assert!(matches!(&elements[2], ParsedElement::Block(_)));
5073    }
5074}
5075
5076#[cfg(test)]
5077mod djot_tests {
5078    use super::*;
5079    use crate::entities::MarkerType;
5080
5081    fn blocks(d: &str) -> Vec<ParsedBlock> {
5082        ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
5083    }
5084
5085    fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
5086        b.spans.iter().find(|s| pred(s)).expect("span not found")
5087    }
5088
5089    #[test]
5090    fn paragraph_bold_italic() {
5091        let b = blocks("normal *bold* _italic_");
5092        assert_eq!(b.len(), 1);
5093        assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
5094        assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
5095    }
5096
5097    #[test]
5098    fn heading_levels() {
5099        assert_eq!(blocks("# H1")[0].heading_level, Some(1));
5100        assert_eq!(blocks("### H3")[0].heading_level, Some(3));
5101        assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
5102    }
5103
5104    #[test]
5105    fn unordered_bullet_styles_are_distinct() {
5106        assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
5107        assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
5108        assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
5109    }
5110
5111    #[test]
5112    fn ordered_delimiters() {
5113        let period = blocks("1. a");
5114        assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
5115        assert_eq!(period[0].list_prefix, "");
5116        assert_eq!(period[0].list_suffix, ".");
5117
5118        let paren = blocks("1) a");
5119        assert_eq!(paren[0].list_suffix, ")");
5120        assert_eq!(paren[0].list_prefix, "");
5121
5122        let paren_paren = blocks("(1) a");
5123        assert_eq!(paren_paren[0].list_prefix, "(");
5124        assert_eq!(paren_paren[0].list_suffix, ")");
5125    }
5126
5127    #[test]
5128    fn task_list_markers() {
5129        let b = blocks("- [ ] a\n- [x] b");
5130        assert_eq!(b.len(), 2);
5131        assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
5132        assert_eq!(b[1].marker, Some(MarkerType::Checked));
5133    }
5134
5135    #[test]
5136    fn code_block_with_language() {
5137        let b = blocks("```rust\nfn main() {}\n```");
5138        assert_eq!(b.len(), 1);
5139        assert!(b[0].is_code_block);
5140        assert_eq!(b[0].code_language.as_deref(), Some("rust"));
5141        let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
5142        assert_eq!(text, "fn main() {}");
5143    }
5144
5145    #[test]
5146    fn link_href() {
5147        let b = blocks("[text](http://example.com)");
5148        let s = first_span_with(&b[0], |s| s.text == "text");
5149        assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
5150    }
5151
5152    #[test]
5153    fn superscript_subscript() {
5154        assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
5155        assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
5156    }
5157
5158    #[test]
5159    fn delete_insert_verbatim() {
5160        assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
5161        assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
5162        assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
5163    }
5164
5165    #[test]
5166    fn blockquote_depth() {
5167        let els = parse_djot("> quoted", &DjotImportOptions::default());
5168        match &els[0] {
5169            ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
5170            _ => panic!("expected block"),
5171        }
5172    }
5173
5174    #[test]
5175    fn nested_list_indent() {
5176        // Djot nests a sub-list only when a blank line separates it from the
5177        // parent item and it is indented to the parent's content column
5178        // (2 spaces per level). Without the blank line the markers fold into
5179        // the paragraph as lazy continuation.
5180        let b = blocks("- a\n\n  - b\n\n    - c");
5181        assert_eq!(b.len(), 3);
5182        assert_eq!(b[0].list_indent, 0);
5183        assert_eq!(b[1].list_indent, 1);
5184        assert_eq!(b[2].list_indent, 2);
5185    }
5186
5187    #[test]
5188    fn table_parsed_as_table() {
5189        let els = parse_djot(
5190            "| a | b |\n|---|---|\n| c | d |",
5191            &DjotImportOptions::default(),
5192        );
5193        assert_eq!(els.len(), 1);
5194        match &els[0] {
5195            ParsedElement::Table(t) => {
5196                assert_eq!(t.header_rows, 1);
5197                assert_eq!(t.rows.len(), 2);
5198                assert_eq!(t.rows[0][0].spans[0].text, "a");
5199                assert_eq!(t.rows[1][1].spans[0].text, "d");
5200            }
5201            _ => panic!("expected table"),
5202        }
5203    }
5204
5205    #[test]
5206    fn smart_punctuation_normalised_to_unicode() {
5207        let text: String = blocks("a... b---c")[0]
5208            .spans
5209            .iter()
5210            .map(|s| s.text.as_str())
5211            .collect();
5212        assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
5213        assert!(text.contains('\u{2014}'), "em dash: {text:?}");
5214    }
5215
5216    #[test]
5217    fn unrepresentable_constructs_dropped_without_leaking_text() {
5218        // Thematic break between two paragraphs: no extra block, no stray text.
5219        let b = blocks("para1\n\n---\n\npara2");
5220        assert_eq!(b.len(), 2);
5221        assert_eq!(
5222            b[0].spans
5223                .iter()
5224                .map(|s| s.text.as_str())
5225                .collect::<String>(),
5226            "para1"
5227        );
5228        assert_eq!(
5229            b[1].spans
5230                .iter()
5231                .map(|s| s.text.as_str())
5232                .collect::<String>(),
5233            "para2"
5234        );
5235
5236        // Fenced div is unwrapped: its content survives, the fence does not.
5237        let d = blocks(":::\ninside\n:::");
5238        let joined: String = d
5239            .iter()
5240            .flat_map(|b| b.spans.iter())
5241            .map(|s| s.text.as_str())
5242            .collect();
5243        assert_eq!(joined, "inside");
5244
5245        // Inline math content is dropped, surrounding text kept.
5246        let m = blocks("before $`E=mc^2` after");
5247        let joined: String = m
5248            .iter()
5249            .flat_map(|b| b.spans.iter())
5250            .map(|s| s.text.as_str())
5251            .collect();
5252        assert!(joined.contains("before"), "{joined:?}");
5253        assert!(joined.contains("after"), "{joined:?}");
5254        assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
5255    }
5256
5257    #[test]
5258    fn empty_document_yields_one_empty_block() {
5259        let b = blocks("");
5260        assert_eq!(b.len(), 1);
5261        assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
5262    }
5263
5264    #[test]
5265    fn block_attributes_parse_into_block() {
5266        let b = blocks(
5267            "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
5268        );
5269        assert_eq!(b.len(), 1);
5270        assert_eq!(b[0].alignment, Some(Alignment::Center));
5271        assert_eq!(b[0].line_height, Some(1500));
5272        assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
5273        assert_eq!(b[0].non_breakable_lines, Some(true));
5274        assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
5275    }
5276
5277    #[test]
5278    fn spacing_block_attributes_parse_into_block() {
5279        // `top_margin` / `text_indent` let one block override the document-wide
5280        // paragraph spacing and first-line indent — what a scene break needs for
5281        // the paragraph that follows it.
5282        let b = blocks("{top_margin=24 text_indent=0}\nhello");
5283        assert_eq!(b.len(), 1);
5284        assert_eq!(b[0].top_margin, Some(24));
5285        assert_eq!(b[0].text_indent, Some(0));
5286    }
5287
5288    #[test]
5289    fn a_zero_text_indent_is_distinct_from_an_absent_one() {
5290        // The whole point of the attribute: `Some(0)` means "explicitly no
5291        // indent", which must not collapse to `None` ("use the document
5292        // default") — otherwise a scene break could not suppress the indent.
5293        let explicit = blocks("{text_indent=0}\nhello");
5294        let absent = blocks("hello");
5295        assert_eq!(explicit[0].text_indent, Some(0));
5296        assert_eq!(absent[0].text_indent, None);
5297        assert!(!explicit[0].is_inline_only());
5298        assert!(absent[0].is_inline_only());
5299    }
5300
5301    #[test]
5302    fn spacing_block_attributes_respect_import_options() {
5303        let src = "{top_margin=24 text_indent=0}\nhello";
5304        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
5305        assert_eq!(b[0].top_margin, None);
5306        assert_eq!(b[0].text_indent, None);
5307    }
5308
5309    #[test]
5310    fn block_attributes_on_heading() {
5311        let b = blocks("{alignment=right}\n# Title");
5312        assert_eq!(b[0].heading_level, Some(1));
5313        assert_eq!(b[0].alignment, Some(Alignment::Right));
5314    }
5315
5316    #[test]
5317    fn block_attributes_respect_import_options() {
5318        // With every optional attribute disabled, the `{…}` block attributes are
5319        // parsed and discarded — only the core paragraph survives.
5320        let src = "{alignment=center line_height=1500}\nhello";
5321        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
5322        assert_eq!(b[0].alignment, None);
5323        assert_eq!(b[0].line_height, None);
5324        assert_eq!(
5325            b[0].spans
5326                .iter()
5327                .map(|s| s.text.as_str())
5328                .collect::<String>(),
5329            "hello"
5330        );
5331    }
5332
5333    #[test]
5334    fn list_item_block_attributes_are_dropped() {
5335        // Block attributes only bind to standalone paragraphs/headings; a list
5336        // item normalises them away (symmetric with the exporter).
5337        let b = blocks("{alignment=center}\n- item");
5338        assert!(b.iter().all(|blk| blk.alignment.is_none()));
5339    }
5340
5341    #[test]
5342    fn unknown_alignment_value_is_ignored() {
5343        let b = blocks("{alignment=sideways}\nhello");
5344        assert_eq!(b[0].alignment, None);
5345    }
5346}
5347
5348// ─── Cheap plain-text extraction ─────────────────────────────────────
5349
5350/// The `U+FFFC OBJECT REPLACEMENT CHARACTER` that stands for a table in the document's
5351/// text.
5352///
5353/// A table is not prose, but it *occupies a position* in the flow: the import mirrors this
5354/// single sentinel into the rope where the table sits
5355/// (`rope_helpers::rope_append_table_anchor`), then the cells as ordinary blocks. Anything
5356/// reconstructing the text a search runs against has to reproduce it, or every offset after
5357/// the first table is short by the two characters (the sentinel and its separator) that the
5358/// document really holds there.
5359pub const TABLE_ANCHOR: &str = "\u{FFFC}";
5360
5361/// `U+2028 LINE SEPARATOR`: a line break inside a block, the one Shift+Enter makes in a
5362/// word processor.
5363///
5364/// A block's text holds it as one character, where it starts a new line of the same
5365/// paragraph, list item, quotation or heading. It is the character Qt's `QTextDocument`
5366/// uses for the same thing. It cannot be `\n`: the rope already uses `\n` between two
5367/// blocks, so a search, an offset or a plain-text export could not tell a line from a
5368/// paragraph.
5369///
5370/// The readers make one of every hard break they meet (Djot's `\` at the end of a line,
5371/// Markdown's, an HTML `<br>` inside a paragraph), and each writer puts it back as its
5372/// format's own line break. A table cell is one line in Djot and Markdown, so there a
5373/// line break is read as a space.
5374pub const LINE_SEPARATOR: char = '\u{2028}';
5375
5376/// [`LINE_SEPARATOR`] as text, for the readers' span builders.
5377const LINE_SEPARATOR_STR: &str = "\u{2028}";
5378
5379/// One span's contribution to the addressable text.
5380///
5381/// An inline object carries no prose but **does** occupy one `U+FFFC` in the
5382/// document (`format_runs_from_spans` mirrors it there), so it has to occupy one
5383/// here too. Leaving it out makes this string shorter than the document it
5384/// claims to be byte-identical to, and every offset past the object — every
5385/// search hit, every comment anchor — lands a character early.
5386fn span_prose(span: &ParsedSpan, out: &mut String) {
5387    if span.image.is_some() || span.footnote_ref.is_some() {
5388        out.push('\u{FFFC}');
5389        return;
5390    }
5391    out.push_str(&span.text);
5392}
5393
5394fn block_prose(block: &ParsedBlock) -> String {
5395    let mut prose = String::new();
5396    for span in &block.spans {
5397        span_prose(span, &mut prose);
5398    }
5399    prose
5400}
5401
5402fn cell_prose(cell: &ParsedTableCell) -> String {
5403    let mut prose = String::new();
5404    for span in &cell.spans {
5405        span_prose(span, &mut prose);
5406    }
5407    prose
5408}
5409
5410/// The prose of a Djot document, with no entities, no store, and no threads.
5411///
5412/// [`parse_djot`] and [`ParsedElement::flatten_to_blocks`] were both already `pub`;
5413/// nothing chained them. This does, and that is the whole trick: it stops at the
5414/// *parse*, where a full import goes on to create a `Block` entity per paragraph, list
5415/// item and table cell, mirror each into the rope, and write its format runs.
5416///
5417/// # Why a project-wide search needs this
5418///
5419/// A host app searching a manuscript must ask "does this scene contain that word" of
5420/// **thousands** of Djot rows, on every keystroke. Doing that by importing each one into
5421/// a document is not a slow feature, it is a frozen app.
5422///
5423/// And searching the Djot *source* instead — the tempting shortcut — is simply wrong:
5424/// the source is markup. `http` matches inside a link's URL, `*` matches an emphasis
5425/// marker, and an occurrence count taken from the source does not agree with what a
5426/// replace re-derives inside the parsed document. Where a replace guards itself with
5427/// "the text moved under me, skip this field", a count taken from markup makes that
5428/// guard fire on perfectly good rows.
5429///
5430/// # The contract
5431///
5432/// The result is **byte-identical to the text the document searches** for the same Djot:
5433/// each block's spans concatenated, blocks joined by a single `\n`, and a table announced
5434/// by its [`TABLE_ANCHOR`] sentinel — exactly the string the import mirrors into the rope
5435/// and that `build_full_text_via_store` recomposes. So an offset found here is an offset
5436/// the document agrees with. On a live document the same string is served by
5437/// `TextDocument::to_addressable_text()`, which reads it off the search plumbing itself.
5438///
5439/// A property in `djot_roundtrip_tests` pins that across the whole generated feature set;
5440/// without it this would be a second, silently-diverging definition of "the text".
5441///
5442/// **Known exception: a footnote *definition*'s body.** This function treats it as out of
5443/// flow and omits it — matching `character_count()`, which does not count it — but the
5444/// live document mirrors its blocks into the rope, so today an in-document search DOES
5445/// run over the note's body and `to_addressable_text()` includes it. The live document
5446/// holds every body after the main text, so this string is the start of that one, and
5447/// every offset of the main text agrees between the two; they differ by the bodies at
5448/// the end. Which side is wrong is an open product question (should search see notes?),
5449/// pinned as current behaviour in
5450/// `addressable_text_tests::footnote_bodies_are_searched_in_the_live_document`.
5451///
5452/// ⚠ It is **not** the same as `TextDocument::to_plain_text()`. That is the human-readable
5453/// *export* — same prose, same order, but with every object anchor omitted, so its offsets
5454/// drift by two characters per preceding table. The authority is what a search sees,
5455/// because that is what a replace edits.
5456/// (`to_plain_text` once also *ordered* a blockquote's prose differently; that was a bug,
5457/// fixed and pinned by `plain_text_order_tests` — see
5458/// `claude_reviews/text-document-plain-text-ordering.md`. The anchors are now the two
5459/// views' only difference.)
5460pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
5461    // Deliberately NOT `ParsedElement::flatten_to_blocks`: that helper drops a table's
5462    // anchor and yields only its cells, which would silently shift every offset in a
5463    // document containing a table by the two characters the document actually holds there.
5464    let elements = parse_djot(djot, options);
5465
5466    // Sized from the source: the prose is always shorter than the markup that carries it,
5467    // so this allocates once and never grows.
5468    let mut out = String::with_capacity(djot.len());
5469
5470    // The separator is decided by "is this the first block", NOT by "is the output still
5471    // empty". An EMPTY block (an empty code fence, say) is still a block: the document
5472    // holds an empty line for it, and an emptiness test would swallow both the block and
5473    // its separator, shifting every offset after it by one.
5474    let mut first = true;
5475    let push = |text: &str, out: &mut String, first: &mut bool| {
5476        if *first {
5477            *first = false;
5478        } else {
5479            out.push('\n');
5480        }
5481        out.push_str(text);
5482    };
5483
5484    for element in &elements {
5485        match element {
5486            ParsedElement::Block(block) => {
5487                push(&block_prose(block), &mut out, &mut first);
5488            }
5489            // A note's body is **out of flow**: it is not laid out where its
5490            // definition was written, it is not part of a copied fragment, and
5491            // the document does not count its characters. So it is not part of
5492            // the addressable text either — which is the one thing that has to
5493            // stay true here, since `character_count()` and this string are
5494            // compared directly.
5495            //
5496            // Its blocks do live in the rope, because they have to live
5497            // somewhere; they are simply not addressable prose.
5498            ParsedElement::FootnoteDefinition { .. } => {}
5499            ParsedElement::Table(table) => {
5500                // The import mirrors a table into the rope as a lone `U+FFFC` sentinel
5501                // followed by its cells, one per block (`rope_append_table_anchor`). The
5502                // sentinel occupies a real position in the text the document searches, so
5503                // it has to occupy one here too — otherwise every offset after a table is
5504                // short by two characters, and a snippet taken from this string would be
5505                // sliced in the wrong place.
5506                push(TABLE_ANCHOR, &mut out, &mut first);
5507                // A row shorter than the widest holds empty cells in the document, one
5508                // position each: the import completes it (see `ParsedTable::column_count`).
5509                let columns = table.column_count();
5510                for row in &table.rows {
5511                    for column in 0..columns {
5512                        let prose = row.get(column).map(cell_prose).unwrap_or_default();
5513                        push(&prose, &mut out, &mut first);
5514                    }
5515                }
5516            }
5517        }
5518    }
5519    out
5520}
5521
5522#[cfg(test)]
5523mod html_footnote_tests {
5524    use super::*;
5525
5526    /// The reason this exists at all: a producer cannot smuggle a reference
5527    /// through as text, because the Djot escaper neutralises `[`, `^` and `]`.
5528    /// Pinned here so the escaping rule and this workaround cannot drift apart.
5529    #[test]
5530    fn a_literal_reference_in_text_would_be_escaped_into_prose() {
5531        let blocks = parse_html("<p>The ferry.[^1]</p>");
5532        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
5533        assert!(
5534            text.contains("[^1]"),
5535            "the parser keeps it as characters: {text:?}"
5536        );
5537        assert!(
5538            blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
5539            "text is text — it must not become a reference by accident"
5540        );
5541    }
5542
5543    #[test]
5544    fn an_attributed_element_becomes_a_real_footnote_reference() {
5545        let blocks = parse_html(
5546            r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
5547        );
5548        let refs: Vec<&str> = blocks
5549            .iter()
5550            .flat_map(|b| b.spans.iter())
5551            .filter_map(|s| s.footnote_ref.as_deref())
5552            .collect();
5553        assert_eq!(refs, vec!["3"]);
5554
5555        // And it carries no text of its own, so it cannot be counted as words or
5556        // matched by a search.
5557        let marker = blocks[0]
5558            .spans
5559            .iter()
5560            .find(|s| s.footnote_ref.is_some())
5561            .expect("the reference span");
5562        assert!(marker.text.is_empty(), "{marker:?}");
5563    }
5564
5565    /// The attribute is the contract, not the tag: a producer may use whatever
5566    /// element renders sensibly in a browser.
5567    #[test]
5568    fn any_element_carrying_the_attribute_works() {
5569        for html in [
5570            r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
5571            r#"<p>a<span data-footnote-ref="1"></span></p>"#,
5572            r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
5573        ] {
5574            let blocks = parse_html(html);
5575            assert_eq!(
5576                blocks
5577                    .iter()
5578                    .flat_map(|b| b.spans.iter())
5579                    .filter_map(|s| s.footnote_ref.as_deref())
5580                    .collect::<Vec<_>>(),
5581                vec!["1"],
5582                "failed for {html}"
5583            );
5584        }
5585    }
5586
5587    #[test]
5588    fn an_empty_or_absent_label_is_not_a_reference() {
5589        for html in [
5590            r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
5591            r#"<p>a<sup></sup></p>"#,
5592        ] {
5593            let blocks = parse_html(html);
5594            assert!(
5595                blocks
5596                    .iter()
5597                    .flat_map(|b| b.spans.iter())
5598                    .all(|s| s.footnote_ref.is_none()),
5599                "failed for {html}"
5600            );
5601        }
5602    }
5603}
5604
5605#[cfg(test)]
5606mod html_depth_tests {
5607    //! HTML nested more deeply than the reader follows, and HTML table cells.
5608
5609    use super::*;
5610
5611    /// The text of each block, in order.
5612    fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
5613        blocks
5614            .iter()
5615            .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
5616            .collect()
5617    }
5618
5619    /// An HTML list nested past the deepest level an insertion keeps comes in with every
5620    /// item, in its order, the deeper ones side by side at that level. The reader stopped
5621    /// descending at its recursion limit and dropped every item below it (the eighty-sixth
5622    /// level on).
5623    #[test]
5624    fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
5625        use crate::parser_tools::list_depth::MAX_LIST_INDENT;
5626        let depth = 300;
5627        let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
5628        html.push_str(&"</li></ol>".repeat(depth));
5629        let blocks = parse_html(&html);
5630        let items: Vec<(String, u32)> = blocks
5631            .iter()
5632            .filter(|block| block.list_style.is_some())
5633            .map(|block| {
5634                let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
5635                (text, block.list_indent)
5636            })
5637            .collect();
5638        let expected: Vec<(String, u32)> = (0..depth)
5639            .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
5640            .collect();
5641        assert_eq!(items, expected);
5642    }
5643
5644    /// Text outside any item of a list nested too deeply stays, after the items.
5645    #[test]
5646    fn text_in_a_deep_html_list_outside_its_items_is_kept() {
5647        let depth = 40;
5648        let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
5649        html.push_str("<ul>loose words<li>last</li></ul>");
5650        html.push_str(&"</li></ul>".repeat(depth));
5651        let texts = block_texts(&parse_html(&html)).join(" ");
5652        assert!(texts.contains("loose words"), "{texts:?}");
5653        assert!(texts.contains("last"), "{texts:?}");
5654        assert!(texts.contains("item 39"), "{texts:?}");
5655    }
5656
5657    /// Past the depth the reader follows, it used to keep nothing: the words at the bottom
5658    /// of three hundred nested `<div>`s, or quotations, were dropped from the paste.
5659    #[test]
5660    fn html_nested_past_the_reader_depth_keeps_its_words() {
5661        for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
5662            let html = format!(
5663                "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
5664                open.repeat(300),
5665                close.repeat(300)
5666            );
5667            let texts = block_texts(&parse_html(&html)).join(" ");
5668            let words: Vec<&str> = texts.split_whitespace().collect();
5669            assert_eq!(
5670                words,
5671                ["Top.", "Deep", "words", "here.", "End."],
5672                "for {open}"
5673            );
5674        }
5675    }
5676
5677    /// The same inside a table cell, read as one paragraph.
5678    #[test]
5679    fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
5680        let html = format!(
5681            "<table><tr><td>{}deep words{}</td></tr></table>",
5682            "<span>".repeat(300),
5683            "</span>".repeat(300)
5684        );
5685        let elements = parse_html_elements(&html);
5686        let Some(ParsedElement::Table(table)) = elements.first() else {
5687            panic!("a table: {elements:?}");
5688        };
5689        let text: String = table.rows[0][0]
5690            .spans
5691            .iter()
5692            .map(|span| span.text.as_str())
5693            .collect();
5694        assert_eq!(text, "deep words");
5695    }
5696
5697    /// A cell is one paragraph, so its line breaks and paragraphs are spaces between
5698    /// words. The HTML writer puts a cell's two paragraphs out as `One<br/>Two`, and the
5699    /// reader read them back as the one word `OneTwo`.
5700    #[test]
5701    fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
5702        for cell in [
5703            "One<br/>Two",
5704            "<p>One</p><p>Two</p>",
5705            "<ul><li>One</li><li>Two</li></ul>",
5706        ] {
5707            let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
5708            let Some(ParsedElement::Table(table)) = elements.first() else {
5709                panic!("a table: {elements:?}");
5710            };
5711            let text: String = table.rows[0][0]
5712                .spans
5713                .iter()
5714                .map(|span| span.text.as_str())
5715                .collect();
5716            assert_eq!(text, "One Two", "for {cell}");
5717        }
5718    }
5719}
5720
5721#[cfg(test)]
5722mod list_delimiter_tests {
5723    use super::*;
5724
5725    fn md(source: &str) -> Vec<ParsedBlock> {
5726        ParsedElement::flatten_to_blocks(parse_markdown(source))
5727    }
5728
5729    fn html(source: &str) -> Vec<ParsedBlock> {
5730        ParsedElement::flatten_to_blocks(parse_html_elements(source))
5731    }
5732
5733    /// The suffix of every list item of `source`, in order.
5734    fn suffixes(source: &str) -> Vec<String> {
5735        md(source)
5736            .into_iter()
5737            .filter(|b| b.list_style.is_some())
5738            .map(|b| b.list_suffix)
5739            .collect()
5740    }
5741
5742    #[test]
5743    fn a_period_and_a_paren_are_told_apart() {
5744        assert_eq!(suffixes("1. a\n2. b"), [".", "."]);
5745        assert_eq!(suffixes("1) a\n2) b"), [")", ")"]);
5746        let numbered = md("3) a");
5747        assert_eq!(numbered[0].list_suffix, ")");
5748        assert_eq!(numbered[0].list_start, Some(3));
5749        let numbered = md("3. a");
5750        assert_eq!(numbered[0].list_suffix, ".");
5751        assert_eq!(numbered[0].list_start, Some(3));
5752    }
5753
5754    #[test]
5755    fn a_bullet_list_has_no_suffix() {
5756        assert_eq!(suffixes("- a\n- b"), ["", ""]);
5757        assert_eq!(suffixes("- 1) x"), [")"]);
5758        let nested = md("- x\n  1) y");
5759        assert_eq!(nested[0].list_suffix, "");
5760        assert_eq!(nested[1].list_suffix, ")");
5761    }
5762
5763    /// The suffix is read from the range pulldown-cmark reports for the list. Each case
5764    /// writes `)` so that a range starting anywhere else than at the marker reads `.` and
5765    /// fails here, and the cases cover every place a list can open.
5766    #[test]
5767    fn the_range_of_a_list_starts_at_its_marker_wherever_it_stands() {
5768        for source in [
5769            "1) a",
5770            "  1) a\n  2) b",
5771            "> 1) a",
5772            "> > 1) a",
5773            "> - 1) y\n>   2) z",
5774            "text\n1) a",
5775            "1) a\n\n2) b",
5776            "1. a\n   1) b",
5777            "- x\n  1) y",
5778            "- a\n\n    1) b",
5779            "- 1) y",
5780            "1. 2) y",
5781            "x[^n]\n\n[^n]: 1) a",
5782            "[^x] text\n\n1) a",
5783            "1) a\n2) b\n   - c\n3) d",
5784        ] {
5785            // The blocks of the flow and those of the notes.
5786            let mut every_block = Vec::new();
5787            for element in parse_markdown(source) {
5788                match element {
5789                    ParsedElement::Block(b) => every_block.push(b),
5790                    ParsedElement::FootnoteDefinition { blocks, .. } => every_block.extend(blocks),
5791                    ParsedElement::Table(_) => {}
5792                }
5793            }
5794            let found = every_block
5795                .into_iter()
5796                .filter(|b| b.list_style == Some(ListStyle::Decimal))
5797                .map(|b| b.list_suffix)
5798                .collect::<Vec<_>>();
5799            assert!(!found.is_empty(), "no numbered item in {source:?}");
5800            assert!(
5801                found.iter().any(|s| s == ")"),
5802                "{source:?} read {found:?}, the innermost ordered list wears ')'"
5803            );
5804        }
5805    }
5806
5807    #[test]
5808    fn nested_lists_each_wear_their_own_delimiter() {
5809        let blocks = md("1. a\n   1) b\n2. c");
5810        let suffixes: Vec<_> = blocks.iter().map(|b| b.list_suffix.as_str()).collect();
5811        assert_eq!(suffixes, [".", ")", "."]);
5812    }
5813
5814    #[test]
5815    fn a_change_of_delimiter_starts_a_new_list() {
5816        // `1. a` then `1) b`: both start at 1, so only the suffix tells them apart.
5817        let blocks = md("1. a\n1) b");
5818        let suffixes: Vec<_> = blocks.iter().map(|b| b.list_suffix.as_str()).collect();
5819        assert_eq!(suffixes, [".", ")"]);
5820    }
5821
5822    #[test]
5823    fn the_suffix_does_not_leak_into_a_following_block() {
5824        let blocks = md("1) a\n\nplain");
5825        assert_eq!(blocks[0].list_suffix, ")");
5826        assert_eq!(blocks[1].list_style, None);
5827        assert_eq!(blocks[1].list_suffix, "");
5828    }
5829
5830    #[test]
5831    fn a_loose_list_keeps_the_suffix_on_every_item() {
5832        assert_eq!(suffixes("1) a\n\n2) b\n\n3) c"), [")", ")", ")"]);
5833    }
5834
5835    #[test]
5836    fn the_html_reader_gives_an_ordered_list_a_period() {
5837        let blocks = html("<ol><li>a</li></ol>");
5838        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
5839        assert_eq!(blocks[0].list_suffix, ".");
5840        let blocks = html("<ol start=\"3\"><li>a</li></ol>");
5841        assert_eq!(blocks[0].list_suffix, ".");
5842        assert_eq!(blocks[0].list_start, Some(3));
5843        let blocks = html("<ul><li>a</li></ul>");
5844        assert_eq!(blocks[0].list_suffix, "");
5845    }
5846
5847    #[test]
5848    fn the_default_suffix_follows_the_style() {
5849        for style in [
5850            ListStyle::Decimal,
5851            ListStyle::LowerAlpha,
5852            ListStyle::UpperAlpha,
5853            ListStyle::LowerRoman,
5854            ListStyle::UpperRoman,
5855        ] {
5856            assert_eq!(default_list_suffix(&style), ".");
5857        }
5858        for style in [ListStyle::Disc, ListStyle::Circle, ListStyle::Square] {
5859            assert_eq!(default_list_suffix(&style), "");
5860        }
5861    }
5862}
5863
5864#[cfg(test)]
5865mod task_list_tests {
5866    use super::*;
5867
5868    fn md(source: &str) -> Vec<ParsedBlock> {
5869        ParsedElement::flatten_to_blocks(parse_markdown(source))
5870    }
5871
5872    fn html(source: &str) -> Vec<ParsedBlock> {
5873        ParsedElement::flatten_to_blocks(parse_html_elements(source))
5874    }
5875
5876    fn text(block: &ParsedBlock) -> String {
5877        block.spans.iter().map(|s| s.text.as_str()).collect()
5878    }
5879
5880    fn markers(blocks: &[ParsedBlock]) -> Vec<Option<MarkerType>> {
5881        blocks.iter().map(|b| b.marker.clone()).collect()
5882    }
5883
5884    #[test]
5885    fn markdown_task_items_keep_their_state_and_lose_the_box_text() {
5886        let blocks = md("- [x] done\n- [ ] todo\n- plain");
5887        assert_eq!(
5888            markers(&blocks),
5889            [Some(MarkerType::Checked), Some(MarkerType::Unchecked), None]
5890        );
5891        assert_eq!(text(&blocks[0]), "done");
5892        assert_eq!(text(&blocks[1]), "todo");
5893        assert_eq!(text(&blocks[2]), "plain");
5894    }
5895
5896    #[test]
5897    fn ordered_and_starred_items_and_a_capital_x() {
5898        assert_eq!(markers(&md("1. [x] a")), [Some(MarkerType::Checked)]);
5899        assert_eq!(markers(&md("* [X] cap")), [Some(MarkerType::Checked)]);
5900        assert_eq!(markers(&md("3) [ ] a")), [Some(MarkerType::Unchecked)]);
5901    }
5902
5903    #[test]
5904    fn a_nested_tight_item_has_its_own_box_and_the_parent_keeps_its_own() {
5905        let blocks = md("- [x] a\n  - [ ] b\n- c");
5906        assert_eq!(
5907            markers(&blocks),
5908            [Some(MarkerType::Checked), Some(MarkerType::Unchecked), None]
5909        );
5910        assert_eq!(text(&blocks[0]), "a");
5911        assert_eq!(text(&blocks[1]), "b");
5912    }
5913
5914    #[test]
5915    fn only_the_first_block_of_an_item_wears_the_box() {
5916        let blocks = md("- [ ] a\n\n  second para\n- [x] b");
5917        assert_eq!(text(&blocks[0]), "a");
5918        assert_eq!(blocks[0].marker, Some(MarkerType::Unchecked));
5919        assert_eq!(text(&blocks[1]), "second para");
5920        assert_eq!(blocks[1].marker, None);
5921        assert_eq!(text(&blocks[2]), "b");
5922        assert_eq!(blocks[2].marker, Some(MarkerType::Checked));
5923    }
5924
5925    #[test]
5926    fn a_task_item_in_a_quotation_is_read() {
5927        let blocks = md("> - [x] a");
5928        assert_eq!(blocks[0].marker, Some(MarkerType::Checked));
5929        assert_eq!(blocks[0].blockquote_depth, 1);
5930    }
5931
5932    #[test]
5933    fn an_empty_task_item_does_not_leak_into_the_next_one() {
5934        // `- [x]` alone is text; `- [x] ` is a box with nothing after it, which makes no
5935        // block (an empty item never did) and must not hand its box to the next item.
5936        let blocks = md("- [x]");
5937        assert_eq!(markers(&blocks), [None]);
5938        assert_eq!(text(&blocks[0]), "[x]");
5939        let blocks = md("- [x] \n- next");
5940        assert_eq!(text(blocks.last().unwrap()), "next");
5941        assert_eq!(blocks.last().unwrap().marker, None);
5942    }
5943
5944    #[test]
5945    fn a_footnote_with_a_task_list_reads_the_boxes() {
5946        let elements = parse_markdown("x[^n]\n\n[^n]: - [x] a\n");
5947        let note = elements
5948            .into_iter()
5949            .find_map(|e| match e {
5950                ParsedElement::FootnoteDefinition { blocks, .. } => Some(blocks),
5951                _ => None,
5952            })
5953            .expect("a note");
5954        assert!(note.iter().any(|b| b.marker == Some(MarkerType::Checked)));
5955    }
5956
5957    #[test]
5958    fn a_line_break_split_keeps_the_box_on_the_first_line_only() {
5959        let block = ParsedBlock {
5960            spans: vec![ParsedSpan {
5961                text: "one\ntwo".into(),
5962                preserve_whitespace: true,
5963                ..Default::default()
5964            }],
5965            list_style: Some(ListStyle::Disc),
5966            marker: Some(MarkerType::Checked),
5967            ..Default::default()
5968        };
5969        let lines = split_block_at_line_breaks(block);
5970        assert_eq!(
5971            markers(&lines),
5972            [Some(MarkerType::Checked), None],
5973            "the item's box is its first line's"
5974        );
5975    }
5976
5977    #[test]
5978    fn html_checkbox_items_are_read() {
5979        let blocks = html(
5980            r#"<ul><li><input type="checkbox" checked> a</li><li><input type="checkbox" disabled> b</li><li>c</li></ul>"#,
5981        );
5982        assert_eq!(
5983            markers(&blocks),
5984            [Some(MarkerType::Checked), Some(MarkerType::Unchecked), None]
5985        );
5986        assert_eq!(text(&blocks[0]), "a");
5987        assert_eq!(text(&blocks[1]), "b");
5988    }
5989
5990    #[test]
5991    fn html_checkbox_through_a_label_and_in_any_case() {
5992        let blocks = html(r#"<ul><li><label><input type="checkbox"> a</label></li></ul>"#);
5993        assert_eq!(blocks[0].marker, Some(MarkerType::Unchecked));
5994        assert_eq!(text(&blocks[0]), "a");
5995        let blocks = html(r#"<ul><li><input type="CHECKBOX" checked="checked"/> a</li></ul>"#);
5996        assert_eq!(blocks[0].marker, Some(MarkerType::Checked));
5997    }
5998
5999    #[test]
6000    fn html_checkbox_that_is_not_first_or_not_a_checkbox_sets_nothing() {
6001        let blocks = html(r#"<ul><li>a <input type="checkbox" checked></li></ul>"#);
6002        assert_eq!(blocks[0].marker, None);
6003        assert_eq!(text(&blocks[0]), "a");
6004        let blocks = html(r#"<ul><li><input type="text"> a</li></ul>"#);
6005        assert_eq!(blocks[0].marker, None);
6006    }
6007
6008    #[test]
6009    fn html_checkbox_outside_a_list_sets_nothing() {
6010        let blocks = html(r#"<li><input type="checkbox" checked> a</li>"#);
6011        assert!(blocks.iter().all(|b| b.marker.is_none()));
6012    }
6013
6014    #[test]
6015    fn the_checkbox_writers_are_read_back() {
6016        for marker in [MarkerType::Checked, MarkerType::Unchecked] {
6017            let written = task_checkbox_html(Some(&marker));
6018            let blocks = html(&format!("<ul><li>{written}a</li></ul>"));
6019            assert_eq!(blocks[0].marker, Some(marker));
6020            assert_eq!(text(&blocks[0]), "a");
6021        }
6022        assert_eq!(task_checkbox_html(None), "");
6023        assert_eq!(task_box_markdown(Some(&MarkerType::Checked)), "[x] ");
6024        assert_eq!(task_box_markdown(Some(&MarkerType::Unchecked)), "[ ] ");
6025        assert_eq!(task_box_markdown(None), "");
6026    }
6027}
6028
6029#[cfg(test)]
6030mod html_sheet_whitespace_tests {
6031    use super::*;
6032    use proptest::prelude::*;
6033
6034    fn blocks(source: &str) -> Vec<ParsedBlock> {
6035        ParsedElement::flatten_to_blocks(parse_html_elements(source))
6036    }
6037
6038    fn texts(source: &str) -> Vec<String> {
6039        blocks(source)
6040            .iter()
6041            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
6042            .collect()
6043    }
6044
6045    /// `QTextDocument::toHtml` as Qt 6 writes it, its sheet included.
6046    const QT_HEAD: &str = r#"<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd"><html><head><meta name="qrichtext" content="1" /><meta charset="utf-8" /><style type="text/css">
6047p, li { white-space: pre-wrap; }
6048hr { height: 1px; border-width: 0; }
6049li.unchecked::marker { content: "\2610"; }
6050li.checked::marker { content: "\2612"; }
6051</style></head>"#;
6052
6053    fn with_sheet(css: &str, body: &str) -> String {
6054        format!("<html><head><style>{css}</style></head><body>{body}</body></html>")
6055    }
6056
6057    #[test]
6058    fn the_qt_sheet_keeps_tabs_and_runs_of_spaces() {
6059        let html = format!(
6060            "{QT_HEAD}<body style=\" font-family:'Sans'; font-size:9pt;\">\n<p style=\" margin-top:0px;\">a\tb   c</p></body></html>"
6061        );
6062        assert_eq!(texts(&html), ["a\tb   c"]);
6063        let html = format!("{QT_HEAD}<body>\n<ul>\n<li>a\tb   c</li></ul></body></html>");
6064        assert_eq!(texts(&html), ["a\tb   c"]);
6065        let html = format!(
6066            "{QT_HEAD}<body><table><tr><td><p>a\tb   c</p></td></tr></table></body></html>"
6067        );
6068        let elements = parse_html_elements(&html);
6069        let cell = elements
6070            .iter()
6071            .find_map(|e| match e {
6072                ParsedElement::Table(t) => Some(
6073                    t.rows[0][0]
6074                        .spans
6075                        .iter()
6076                        .map(|s| s.text.as_str())
6077                        .collect::<String>(),
6078                ),
6079                _ => None,
6080            })
6081            .expect("a table");
6082        assert!(cell.contains("a\tb   c"), "{cell:?}");
6083    }
6084
6085    #[test]
6086    fn without_a_sheet_the_whitespace_collapses_as_before() {
6087        assert_eq!(texts("<p>a\tb   c</p>"), ["a b c"]);
6088    }
6089
6090    /// The sheet must not change the block: a `white-space` in a block's own `style` makes
6091    /// it non-breakable, and a sheet's must not.
6092    #[test]
6093    fn a_sheet_rule_does_not_make_blocks_non_breakable() {
6094        let html = format!("{QT_HEAD}<body><p>a\tb</p></body></html>");
6095        let parsed = blocks(&html);
6096        assert_eq!(parsed.len(), 1);
6097        assert_eq!(parsed[0].non_breakable_lines, None);
6098        assert!(parsed[0].is_inline_only());
6099    }
6100
6101    #[test]
6102    fn only_simple_selectors_count() {
6103        for css in [
6104            "p.x { white-space: pre }",
6105            "#a p { white-space: pre }",
6106            "p:first-child { white-space: pre }",
6107            "p > b { white-space: pre }",
6108            "p[class] { white-space: pre }",
6109            "p, p.x { white-space: pre }",
6110            "@media print { p { white-space: pre } }",
6111            "@font-face { font-family: x; white-space: pre }",
6112            "/* p { white-space: pre } */",
6113            "/* } */ p { color: red }",
6114            "p { color: red }",
6115            "p { white-space: inherit }",
6116        ] {
6117            assert_eq!(texts(&with_sheet(css, "<p>a\tb</p>")), ["a b"], "{css}");
6118        }
6119    }
6120
6121    #[test]
6122    fn a_style_of_another_kind_is_not_read() {
6123        for head in [
6124            r#"<style type="text/less">p { white-space: pre }</style>"#,
6125            r#"<style media="print">p { white-space: pre }</style>"#,
6126        ] {
6127            assert_eq!(
6128                texts(&format!(
6129                    "<html><head>{head}</head><body><p>a\tb</p></body></html>"
6130                )),
6131                ["a b"],
6132                "{head}"
6133            );
6134        }
6135        for head in [
6136            r#"<style type="TEXT/CSS" media="screen">p { white-space: pre }</style>"#,
6137            r#"<style media="all">p { white-space: pre }</style>"#,
6138        ] {
6139            assert_eq!(
6140                texts(&format!(
6141                    "<html><head>{head}</head><body><p>a\tb</p></body></html>"
6142                )),
6143                ["a\tb"],
6144                "{head}"
6145            );
6146        }
6147    }
6148
6149    #[test]
6150    fn the_later_rule_and_the_more_specific_one_win() {
6151        let case = |css: &str| texts(&with_sheet(css, "<p>a\tb</p>"));
6152        assert_eq!(
6153            case("p { white-space: pre } p { white-space: normal }"),
6154            ["a b"]
6155        );
6156        assert_eq!(
6157            case("p { white-space: normal } p { white-space: pre }"),
6158            ["a\tb"]
6159        );
6160        // A rule naming the tag beats `*`, whichever comes first.
6161        assert_eq!(
6162            case("* { white-space: pre-wrap } p { white-space: normal }"),
6163            ["a b"]
6164        );
6165        assert_eq!(
6166            case("p { white-space: normal } * { white-space: pre-wrap }"),
6167            ["a b"]
6168        );
6169        assert_eq!(case("p { white-space: pre; white-space: normal }"), ["a b"]);
6170        assert_eq!(case("p { white-space: pre; white-space: bogus }"), ["a\tb"]);
6171        assert_eq!(case("P { white-space: pre }"), ["a\tb"]);
6172        assert_eq!(case("p, b { white-space: pre }"), ["a\tb"]);
6173        // One member too complex for us drops the whole rule, as an unknown one would.
6174        assert_eq!(case("p, p.x { white-space: pre }"), ["a b"]);
6175    }
6176
6177    #[test]
6178    fn inline_style_and_importance_decide_between_sheet_and_element() {
6179        let case = |css: &str, style: &str| {
6180            texts(&with_sheet(css, &format!("<p style=\"{style}\">a\tb</p>")))
6181        };
6182        assert_eq!(
6183            case("p { white-space: pre-wrap }", "white-space: normal"),
6184            ["a b"]
6185        );
6186        assert_eq!(
6187            case(
6188                "p { white-space: pre-wrap !important }",
6189                "white-space: normal"
6190            ),
6191            ["a\tb"]
6192        );
6193        assert_eq!(
6194            case(
6195                "p { white-space: pre-wrap !important }",
6196                "white-space: normal !important"
6197            ),
6198            ["a b"]
6199        );
6200    }
6201
6202    #[test]
6203    fn body_html_and_star_start_the_walk_without_making_blocks_of_the_layout() {
6204        for selector in ["body", "html", "*", "body, html"] {
6205            let html = with_sheet(
6206                &format!("{selector} {{ white-space: pre-wrap }}"),
6207                "<p>a\tb</p>\n<p>c</p>\n<ul>\n<li>d</li>\n<li>e<ul>\n<li>f</li>\n</ul></li>\n</ul>\n<table>\n<tr>\n<td>g\th</td>\n</tr>\n</table>\n",
6208            );
6209            let parsed = ParsedElement::flatten_to_blocks(parse_html_elements(&html));
6210            let found: Vec<String> = parsed
6211                .iter()
6212                .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
6213                .collect();
6214            assert_eq!(
6215                found,
6216                ["a\tb", "c", "d", "e", "f", "g\th"],
6217                "{selector}: {found:?}"
6218            );
6219        }
6220    }
6221
6222    /// Text outside any element inherits what the sheet says of `<body>`: the rule naming
6223    /// it beats `*`, and `*` reaches the body before `html` does.
6224    #[test]
6225    fn bare_text_starts_from_the_body_as_the_cascade_gives_it() {
6226        let case = |css: &str| texts(&with_sheet(css, "a\tb"));
6227        assert_eq!(
6228            case("body { white-space: normal } * { white-space: pre }"),
6229            ["a b"]
6230        );
6231        assert_eq!(
6232            case("* { white-space: pre } body { white-space: normal }"),
6233            ["a b"]
6234        );
6235        assert_eq!(
6236            case("html { white-space: normal } * { white-space: pre }"),
6237            ["a\tb"]
6238        );
6239        assert_eq!(case("html { white-space: pre }"), ["a\tb"]);
6240        assert_eq!(
6241            case("body { white-space: pre } html { white-space: normal }"),
6242            ["a\tb"]
6243        );
6244    }
6245
6246    #[test]
6247    fn the_style_may_come_after_the_content_or_be_split() {
6248        let html = "<p>a\tb</p><style>p { white-space: pre-wrap }</style>";
6249        assert_eq!(texts(html), ["a\tb"]);
6250        let html =
6251            "<style>p { color: red }</style><style>p { white-space: pre-wrap }</style><p>a\tb</p>";
6252        assert_eq!(texts(html), ["a\tb"]);
6253    }
6254
6255    #[test]
6256    fn a_deep_document_with_a_sheet_is_read_on_its_own_thread() {
6257        let depth = HTML_DEPTH_ON_THE_CALLERS_STACK + 20;
6258        let html = format!(
6259            "<style>p {{ white-space: pre-wrap }}</style>{}<p>a\tb</p>{}",
6260            "<div>".repeat(depth),
6261            "</div>".repeat(depth)
6262        );
6263        assert_eq!(texts(&html), ["a\tb"]);
6264    }
6265
6266    /// The sheets Word and Qt put on the clipboard, wrapped as they are.
6267    #[test]
6268    fn a_word_sheet_in_comment_markers_is_skipped_without_losing_the_next_rule() {
6269        let css = "\n<!--\n /* Font Definitions */\n @font-face\n\t{font-family:\"Cambria Math\";\n\tpanose-1:2 4 5 3 5 4 6 3 2 4;}\n /* Style Definitions */\n p.MsoNormal, li.MsoNormal, div.MsoNormal\n\t{margin:0cm; font-size:11.0pt;}\n@page WordSection1\n\t{size:612.0pt 792.0pt; margin:72.0pt 72.0pt 72.0pt 72.0pt;}\ndiv.WordSection1\n\t{page:WordSection1;}\n /* List Definitions */\n @list l0:level1\n\t{mso-level-number-format:bullet;}\np { white-space: pre-wrap }\n-->\n";
6270        assert_eq!(texts(&with_sheet(css, "<p>a\tb</p>")), ["a\tb"]);
6271        // The first rule right after the opening marker is not lost either.
6272        assert_eq!(
6273            texts(&with_sheet(
6274                "<!-- p { white-space: pre-wrap } -->",
6275                "<p>a\tb</p>"
6276            )),
6277            ["a\tb"]
6278        );
6279    }
6280
6281    #[test]
6282    fn a_blank_paragraph_stays_content_and_blanks_between_blocks_do_not() {
6283        let html = with_sheet(
6284            "p { white-space: pre-wrap }",
6285            "<p>a</p>\n  <p>   </p>\n<p>b</p>",
6286        );
6287        assert_eq!(texts(&html), ["a", "   ", "b"]);
6288    }
6289
6290    #[test]
6291    fn a_hostile_sheet_neither_panics_nor_loops() {
6292        for css in [
6293            "/* unterminated",
6294            "p { white-space: pre",
6295            "p { \"unterminated",
6296            "} } }",
6297            "{ { {",
6298            "p {",
6299            "@media {",
6300            "\u{e9}\u{4e2d}\u{1f600} p { white-space: pre }",
6301            "p\\",
6302            "\\",
6303            "'",
6304            "<!-- -->",
6305            "p { white-space: \u{e9}pre }",
6306        ] {
6307            let _ = texts(&with_sheet(css, "<p>a</p>"));
6308        }
6309    }
6310
6311    /// A document with no <style> block and no inline `white-space` reads as it did before
6312    /// sheets were read. What an inline `white-space` changes is that blanks next to a block
6313    /// are layout, but the blank between two inline siblings is still the space between two
6314    /// words, whatever the element's own `white-space` says.
6315    #[test]
6316    fn blanks_between_inline_siblings_are_content() {
6317        for html in [
6318            r#"<div style="white-space: pre-wrap"><b>a</b> <i>b</i><p>c</p></div>"#,
6319            r#"<ul style="white-space: pre-wrap"><li><b>a</b> <i>b</i><ul><li>c</li></ul></li></ul>"#,
6320            "<style>li { white-space: pre-wrap }</style><ul><li><b>a</b> <i>b</i><ul><li>c</li></ul></li></ul>",
6321            "<style>* { white-space: pre-wrap }</style><div><b>a</b> <i>b</i><p>c</p></div>",
6322        ] {
6323            let read = texts(html);
6324            assert!(read.iter().any(|t| t == "a b"), "{html}: {read:?}");
6325            assert!(!read.iter().any(|t| t == "ab"), "{html}: {read:?}");
6326        }
6327        // And inside a cell.
6328        let cell = ParsedElement::flatten_to_blocks(parse_html_elements(
6329            r#"<table><tr><td style="white-space: pre-wrap"><b>a</b> <b>b</b><p>c</p></td></tr></table>"#,
6330        ));
6331        let cell: String = cell
6332            .iter()
6333            .flat_map(|b| b.spans.iter().map(|s| s.text.as_str()))
6334            .collect();
6335        assert!(cell.contains("a b"), "{cell:?}");
6336    }
6337
6338    /// Blanks between blocks are layout whatever the block elements are called, HTML5
6339    /// sectioning and grouping included.
6340    #[test]
6341    fn blanks_between_html5_containers_are_layout() {
6342        let sections = "<section>\n<p>a</p>\n</section>\n<section>\n<p>b</p>\n</section>\n";
6343        for selector in ["body", "html", "*"] {
6344            let html = with_sheet(&format!("{selector} {{ white-space: pre-wrap }}"), sections);
6345            assert_eq!(texts(&html), ["a", "b"], "{selector}");
6346        }
6347        for tag in [
6348            "article", "main", "header", "footer", "nav", "aside", "figure", "dl",
6349        ] {
6350            let html = with_sheet(
6351                "* { white-space: pre-wrap }",
6352                &format!("<{tag}>\n<p>a</p>\n</{tag}>\n<{tag}>\n<p>b</p>\n</{tag}>\n"),
6353            );
6354            assert_eq!(texts(&html), ["a", "b"], "{tag}");
6355        }
6356    }
6357
6358    /// The decision is made from the two neighbours of a blank, so a parent with a hundred
6359    /// thousand children is read in proportion to them.
6360    #[test]
6361    #[ignore = "a timing test: its bound is wide, but wall-clock time depends on the machine; run with --ignored"]
6362    fn a_parent_of_many_children_is_read_in_linear_time() {
6363        let children = "<b>x</b> ".repeat(100_000);
6364        let html =
6365            format!("<style>* {{ white-space: pre-wrap }}</style><div>{children}<p>end</p></div>");
6366        let started = std::time::Instant::now();
6367        let read = texts(&html);
6368        assert!(read.len() >= 2);
6369        assert!(
6370            started.elapsed() < std::time::Duration::from_secs(60),
6371            "100,000 children took {:?}",
6372            started.elapsed()
6373        );
6374    }
6375
6376    /// A table in a bold or a link makes no bold or linked cells: a cell starts from the
6377    /// whitespace around the table and nothing else.
6378    #[test]
6379    fn a_cell_takes_the_whitespace_around_the_table_and_no_formatting() {
6380        let parsed = parse_html_elements(
6381            r#"<b><a href="http://x"><table><tr><td>x</td></tr></table></a></b>"#,
6382        );
6383        let cell = parsed
6384            .iter()
6385            .find_map(|e| match e {
6386                ParsedElement::Table(t) => Some(t.rows[0][0].spans.clone()),
6387                _ => None,
6388            })
6389            .expect("a table");
6390        assert!(
6391            cell.iter().all(|s| !s.bold && s.link_href.is_none()),
6392            "{cell:?}"
6393        );
6394    }
6395
6396    /// A kept whitespace does not keep the space a cell puts between its blocks.
6397    #[test]
6398    fn a_cell_of_blocks_under_kept_whitespace_has_no_leading_space() {
6399        let parsed = parse_html_elements(&with_sheet(
6400            "* { white-space: pre-wrap }",
6401            "<table><tr><td><p>x</p></td></tr></table>",
6402        ));
6403        let cell: String = parsed
6404            .iter()
6405            .find_map(|e| match e {
6406                ParsedElement::Table(t) => {
6407                    Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6408                }
6409                _ => None,
6410            })
6411            .expect("a table");
6412        assert_eq!(cell, "x");
6413        let parsed = parse_html_elements(&with_sheet(
6414            "td { white-space: pre-wrap }",
6415            "<table><tr><td>a\tb   c</td></tr></table>",
6416        ));
6417        let cell: String = parsed
6418            .iter()
6419            .find_map(|e| match e {
6420                ParsedElement::Table(t) => {
6421                    Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6422                }
6423                _ => None,
6424            })
6425            .expect("a table");
6426        assert_eq!(cell, "a\tb   c");
6427    }
6428
6429    /// Bare text that kept its line breaks is one block for each line, as it is anywhere else.
6430    #[test]
6431    fn bare_text_with_a_line_break_under_a_kept_whitespace_is_two_blocks() {
6432        let html = "<style>body { white-space: pre-wrap }</style>a\nb";
6433        assert_eq!(texts(html), ["a", "b"]);
6434    }
6435
6436    /// The declarations of a nested rule are that rule's, not the rule that holds it.
6437    #[test]
6438    fn a_nested_rule_does_not_leak_into_the_rule_around_it() {
6439        let case = |css: &str| texts(&with_sheet(css, "<p>a\tb</p>"));
6440        assert_eq!(
6441            case("p { span { white-space: normal; white-space: pre; } }"),
6442            ["a b"]
6443        );
6444        assert_eq!(
6445            case("p { span { white-space: pre } white-space: pre-wrap }"),
6446            ["a\tb"]
6447        );
6448        assert_eq!(
6449            case("p { white-space: pre-wrap; & span { white-space: normal } }"),
6450            ["a\tb"]
6451        );
6452    }
6453
6454    /// `pre-line` and `nowrap` from a sheet collapse the whitespace, as they do inline, and a
6455    /// sheet's `pre` marks no block non-breakable, as an inline one does.
6456    #[test]
6457    fn pre_line_nowrap_and_pre_from_a_sheet() {
6458        for keyword in ["pre-line", "nowrap"] {
6459            // `*` keeps the whitespace and the tag's rule, which comes after it, collapses it:
6460            // the text collapses only if the sheet's `pre-line` or `nowrap` was read.
6461            let html = with_sheet(
6462                &format!("* {{ white-space: pre }} p {{ white-space: {keyword} }}"),
6463                "<p>a\tb   c</p>",
6464            );
6465            assert_eq!(texts(&html), ["a b c"], "{keyword}");
6466        }
6467        let html = with_sheet("p { white-space: pre }", "<p>a\tb</p>");
6468        let parsed = blocks(&html);
6469        assert_eq!(parsed[0].non_breakable_lines, None);
6470        assert_eq!(texts(&html), ["a\tb"]);
6471    }
6472
6473    /// Block-level elements the reader does not split a run at (a definition list, the HTML5
6474    /// sections) join their texts into one run, and a browser shows each on a line of its
6475    /// own: where whitespace is kept, the blank between two of them is one line break, so
6476    /// each is a line, and none is merged into a word with the next or given a stray space.
6477    #[test]
6478    fn block_level_elements_the_reader_does_not_split_are_a_line_each() {
6479        let dl = "<dl>\n<dt>a</dt>\n<dd>b</dd>\n</dl>";
6480        assert_eq!(
6481            texts(&format!("<div style=\"white-space: pre-wrap\">{dl}</div>")),
6482            ["a", "b"]
6483        );
6484        assert_eq!(
6485            texts(&with_sheet(
6486                "* { white-space: pre-wrap }",
6487                &format!("<div>{dl}</div>")
6488            )),
6489            ["a", "b"]
6490        );
6491        assert_eq!(
6492            texts(&with_sheet(
6493                "* { white-space: pre-wrap }",
6494                "<section>A</section> <section>B</section>"
6495            )),
6496            ["A", "B"]
6497        );
6498        assert_eq!(
6499            texts(&with_sheet(
6500                "body { white-space: pre-wrap }",
6501                "<header>Title</header>\n<main>Body</main>"
6502            )),
6503            ["Title", "Body"]
6504        );
6505        assert_eq!(
6506            texts(&with_sheet(
6507                "* { white-space: pre-wrap }",
6508                "<div><article>A</article>\n<aside>B</aside></div>"
6509            )),
6510            ["A", "B"]
6511        );
6512        // A block-level element beside an inline one.
6513        assert_eq!(
6514            texts(&with_sheet(
6515                "* { white-space: pre-wrap }",
6516                "<div><section>A</section> <b>x</b></div>"
6517            )),
6518            ["A", "x"]
6519        );
6520    }
6521
6522    /// Where whitespace collapses the same blank is one space, and there is never two.
6523    #[test]
6524    fn block_level_elements_the_reader_does_not_split_are_a_word_gap_where_whitespace_collapses() {
6525        let dl = "<dl>\n<dt>a</dt>\n<dd>b</dd>\n</dl>";
6526        assert_eq!(
6527            texts(&with_sheet(
6528                "body { white-space: pre-wrap }",
6529                &format!("<div style=\"white-space: normal\">{dl}</div>")
6530            )),
6531            ["a b"]
6532        );
6533        assert_eq!(texts(&format!("<div>{dl}</div>")), ["a b"]);
6534        // Text that ends in a blank before the blank does not double it.
6535        assert_eq!(
6536            texts("<div><dl><dt>a </dt>\n<dd> b</dd></dl></div>"),
6537            ["a b"]
6538        );
6539        assert_eq!(
6540            texts(&with_sheet(
6541                "body { white-space: pre-wrap }",
6542                "<div style=\"white-space: normal\"><dl><dt>a </dt>\n<dd> b</dd></dl></div>"
6543            )),
6544            ["a b"]
6545        );
6546        // And in a cell, the one space.
6547        let parsed = parse_html_elements(&with_sheet(
6548            "* { white-space: pre-wrap }",
6549            "<table><tr><td><section>A</section> <section>B</section></td></tr></table>",
6550        ));
6551        let cell: String = parsed
6552            .iter()
6553            .find_map(|e| match e {
6554                ParsedElement::Table(t) => {
6555                    Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6556                }
6557                _ => None,
6558            })
6559            .expect("a table");
6560        assert_eq!(cell, "A B");
6561        // The text before the blank ends in a blank already: no second space.
6562        let parsed = parse_html_elements(&with_sheet(
6563            "* { white-space: pre-wrap }",
6564            "<table><tr><td><section>A </section> <section>B</section></td></tr></table>",
6565        ));
6566        let cell: String = parsed
6567            .iter()
6568            .find_map(|e| match e {
6569                ParsedElement::Table(t) => {
6570                    Some(t.rows[0][0].spans.iter().map(|s| s.text.as_str()).collect())
6571                }
6572                _ => None,
6573            })
6574            .expect("a table");
6575        assert_eq!(cell, "A B");
6576    }
6577
6578    /// A boundary adds a line only where there is a line to end: not at the start of a run,
6579    /// not twice in a row, and not for elements that hold blocks of their own.
6580    #[test]
6581    fn boundaries_add_no_empty_lines() {
6582        let html = with_sheet(
6583            "* { white-space: pre-wrap }",
6584            "<section>x</section>\n<section><p>p</p></section>\n<section>y</section>",
6585        );
6586        let mut read = texts(&html);
6587        read.sort();
6588        assert_eq!(read, ["p", "x", "y"], "{html}");
6589        let html = with_sheet(
6590            "* { white-space: pre-wrap }",
6591            "<section><p>p</p></section>\n<section><p>q</p></section>",
6592        );
6593        let mut read = texts(&html);
6594        read.sort();
6595        assert_eq!(read, ["p", "q"], "{html}");
6596    }
6597
6598    /// Where whitespace is kept, a line keeps the blank that ends its own text: one
6599    /// line break is added between two elements and no space either side of it.
6600    #[test]
6601    fn a_kept_line_keeps_its_own_blanks_and_gains_none() {
6602        assert_eq!(
6603            texts(&with_sheet(
6604                "* { white-space: pre-wrap }",
6605                "<dl><dt>a </dt>\n<dd> b</dd></dl>"
6606            )),
6607            ["a ", " b"]
6608        );
6609    }
6610
6611    /// A sheet is resolved once into one winning declaration per tag, a small value and not
6612    /// the text of a style. (The cost in bytes allocated is pinned in
6613    /// `tests/html_sheet_cost_tests.rs`, where it is counted and not timed.)
6614    #[test]
6615    fn a_large_sheet_resolves_into_one_declaration_per_tag() {
6616        let mut css = String::new();
6617        for i in 0..4000 {
6618            css.push_str("* { white-space: pre }\n");
6619            css.push_str(&format!("t{i} {{ white-space: pre }}\n"));
6620        }
6621        let sheet = SheetWhitespace::from_css(&css);
6622        assert_eq!(sheet.per_tag.len(), 4000);
6623        assert_eq!(sheet.declaration_for("t17"), Some(("pre", false)));
6624        assert_eq!(sheet.declaration_for("p"), Some(("pre", false)));
6625
6626        let selectors: Vec<String> = (0..20_000).map(|i| format!("t{i}")).collect();
6627        let css = format!("{} {{ white-space: pre }}", selectors.join(","));
6628        let sheet = SheetWhitespace::from_css(&css);
6629        assert_eq!(sheet.per_tag.len(), 20_000);
6630        assert_eq!(sheet.declaration_for("p"), None);
6631    }
6632
6633    proptest! {
6634        #[test]
6635        fn the_rule_scanner_survives_any_text(css in "\\PC{0,200}") {
6636            let _ = parse_whitespace_rules(&css);
6637        }
6638
6639        #[test]
6640        fn the_rule_scanner_survives_css_shaped_noise(
6641            css in "[a-z*.#:>,@{}();!/\"' \\\\\n\u{e9}-]{0,120}"
6642        ) {
6643            let _ = parse_whitespace_rules(&css);
6644        }
6645    }
6646}
6647
6648#[cfg(test)]
6649mod djot_crlf_tests {
6650    use super::*;
6651
6652    fn parse(djot: &str) -> Vec<ParsedElement> {
6653        parse_djot(djot, &DjotImportOptions::default())
6654    }
6655
6656    /// The text of every span of every block, table cell and note, in order.
6657    fn all_text(elements: &[ParsedElement]) -> Vec<String> {
6658        let mut out = Vec::new();
6659        for element in elements {
6660            match element {
6661                ParsedElement::Block(b) => out.extend(b.spans.iter().map(|s| s.text.clone())),
6662                ParsedElement::Table(t) => {
6663                    for row in &t.rows {
6664                        for cell in row {
6665                            out.extend(cell.spans.iter().map(|s| s.text.clone()));
6666                        }
6667                    }
6668                }
6669                ParsedElement::FootnoteDefinition { blocks, .. } => {
6670                    for b in blocks {
6671                        out.extend(b.spans.iter().map(|s| s.text.clone()));
6672                    }
6673                }
6674            }
6675        }
6676        out
6677    }
6678
6679    /// A CRLF source reads as its LF twin, for every construct that holds a line ending.
6680    #[test]
6681    fn a_crlf_document_reads_as_its_lf_twin() {
6682        for lf in [
6683            "a\nb\n\nc",
6684            "```rust\nlet x = 1;\nlet y = 2;\n```\n",
6685            "`a\nb` and *c\nd*",
6686            "a\\\nb",
6687            "- one\n  - nested\n- two\n",
6688            "1. one\n2. two\n",
6689            "| a | b |\n|---|---|\n| c | d |\n",
6690            "> quoted\n> lines\n",
6691            "# Heading\n\nbody\n",
6692            "text[^n]\n\n[^n]: the note\n",
6693        ] {
6694            let crlf = lf.replace('\n', "\r\n");
6695            let from_crlf = parse(&crlf);
6696            let from_lf = parse(lf);
6697            assert_eq!(format!("{from_crlf:?}"), format!("{from_lf:?}"), "{lf:?}");
6698            assert!(
6699                all_text(&from_crlf).iter().all(|t| !t.contains('\r')),
6700                "a carriage return in {lf:?}: {:?}",
6701                all_text(&from_crlf)
6702            );
6703        }
6704    }
6705
6706    /// The first paragraph of "a\r\nb\r\n\r\nc" is "a", a soft break, "b", not two lines.
6707    #[test]
6708    fn a_soft_break_stays_a_soft_break() {
6709        let blocks = ParsedElement::flatten_to_blocks(parse("a\r\nb\r\n\r\nc"));
6710        assert_eq!(blocks.len(), 2);
6711        let first: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
6712        assert_eq!(first, "a b");
6713    }
6714
6715    /// Pin, not mechanism: a document too deep to parse is shown as its lines, and
6716    /// `str::lines` takes a CRLF off already.
6717    #[test]
6718    fn a_too_deep_crlf_document_shows_lines_without_carriage_returns() {
6719        let deep = format!("{}deep\r\nsecond line\r\n", "- ".repeat(700));
6720        assert!(
6721            all_text(&parse(&deep)).iter().all(|t| !t.contains('\r')),
6722            "{:?}",
6723            all_text(&parse(&deep))
6724        );
6725    }
6726
6727    /// The guarantee the carriage-return rule is cut around: any string comes back as it
6728    /// was through `plain_text_to_djot` and `djot_to_plain_text`, a lone `\r` and a `\r\n`
6729    /// included. Goes red if a lone `\r` is read as a line ending.
6730    #[test]
6731    fn the_plain_text_round_trip_keeps_every_carriage_return() {
6732        use crate::parser_tools::plain_text_to_djot;
6733        for text in [
6734            "a\rb",
6735            "\rcarriage return\r",
6736            "a\r\nb",
6737            "a\r\n\r\nb",
6738            "x\r",
6739            "a\r\rb",
6740        ] {
6741            let djot = plain_text_to_djot(text);
6742            let back = djot_to_plain_text(&djot, &DjotImportOptions::default());
6743            assert_eq!(back, text, "{text:?} written as {djot:?}");
6744        }
6745    }
6746
6747    /// A lone carriage return in Djot source stays text.
6748    #[test]
6749    fn a_lone_carriage_return_is_text() {
6750        let text = all_text(&parse("a\rb")).concat();
6751        assert_eq!(text, "a\rb");
6752    }
6753
6754    /// Pins of the other readers, which normalise a CRLF themselves: not mechanism tests of
6755    /// this change.
6756    #[test]
6757    fn markdown_and_html_already_read_a_crlf_as_a_line_ending() {
6758        let md = ParsedElement::flatten_to_blocks(parse_markdown("a\r\nb\r\n\r\nc"));
6759        let md_text: Vec<String> = md
6760            .iter()
6761            .flat_map(|b| b.spans.iter().map(|s| s.text.clone()))
6762            .collect();
6763        assert!(md_text.iter().all(|t| !t.contains('\r')), "{md_text:?}");
6764        let md = ParsedElement::flatten_to_blocks(parse_markdown("a\rb\n\nc"));
6765        assert!(
6766            md.iter()
6767                .all(|b| b.spans.iter().all(|s| !s.text.contains('\r'))),
6768            "a lone CR in Markdown"
6769        );
6770        let html = ParsedElement::flatten_to_blocks(parse_html_elements("<pre>a\r\nb</pre>"));
6771        let html_text: Vec<String> = html
6772            .iter()
6773            .flat_map(|b| b.spans.iter().map(|s| s.text.clone()))
6774            .collect();
6775        assert!(html_text.iter().all(|t| !t.contains('\r')), "{html_text:?}");
6776    }
6777
6778    #[test]
6779    fn crlf_to_lf_borrows_when_there_is_nothing_to_change() {
6780        use crate::parser_tools::djot_depth::crlf_to_lf;
6781        assert!(matches!(
6782            crlf_to_lf("a\nb\rc"),
6783            std::borrow::Cow::Borrowed(_)
6784        ));
6785        assert_eq!(crlf_to_lf("a\r\nb\r\n\r\n"), "a\nb\n\n");
6786        assert_eq!(crlf_to_lf("\r\r\n\r"), "\r\n\r");
6787    }
6788}