Skip to main content

common/parser_tools/
content_parser.rs

1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4/// An inline image recovered by a parser, before it becomes an `ImageAnchor`.
5///
6/// `src` is whatever the source document pointed at — a relative path, a bare
7/// name, a URL. Parsers do not resolve it; that is the embedding application's
8/// job, and this crate never touches the filesystem.
9#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11    pub src: String,
12    pub alt: String,
13    /// Display size in pixels, `0` when the source did not state one.
14    ///
15    /// Djot and HTML both carry these as attributes (`{width=800}`,
16    /// `<img width=…>`); Markdown has no syntax for them, so a Markdown import
17    /// leaves both zero and the caller supplies intrinsic dimensions.
18    pub width: i64,
19    pub height: i64,
20}
21
22/// A parsed inline span with formatting info.
23///
24/// A span carries *either* text or an image, never both: an image span's `text`
25/// is empty and its description lives in [`ParsedImage::alt`], so alt text
26/// cannot leak into the block's prose (and therefore cannot be counted as
27/// manuscript words or matched by a search).
28#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30    pub text: String,
31    pub bold: bool,
32    pub italic: bool,
33    pub underline: bool,
34    pub strikeout: bool,
35    pub code: bool,
36    /// Superscript (djot `^x^`). Maps to `CharVerticalAlignment::SuperScript`.
37    pub superscript: bool,
38    /// Subscript (djot `~x~`). Maps to `CharVerticalAlignment::SubScript`.
39    pub subscript: bool,
40    pub link_href: Option<String>,
41    /// Set when this span is an inline image rather than text.
42    pub image: Option<ParsedImage>,
43    /// Set when this span is a footnote *reference* rather than text — the
44    /// label naming the note, never the number a reader sees.
45    pub footnote_ref: Option<String>,
46    /// The text keeps its whitespace as it is: it came from an element styled
47    /// `white-space: pre`, `pre-wrap` or `break-spaces`, where a tab or two spaces in a
48    /// row are the text's own. An HTML reader collapses none of it.
49    pub preserve_whitespace: bool,
50}
51
52/// A parsed table cell containing inline spans.
53#[derive(Debug, Clone)]
54pub struct ParsedTableCell {
55    pub spans: Vec<ParsedSpan>,
56}
57
58/// A parsed table extracted from markdown or HTML.
59#[derive(Debug, Clone)]
60pub struct ParsedTable {
61    /// Number of header rows (typically 1 for markdown tables).
62    pub header_rows: usize,
63    /// All rows (header + body), each containing cells with their inline spans.
64    pub rows: Vec<Vec<ParsedTableCell>>,
65    /// Blockquote nesting depth at the point the table appeared
66    /// (0 = not inside a blockquote), mirroring `ParsedBlock::blockquote_depth`.
67    pub blockquote_depth: u32,
68    /// How many of those quotations open at the table, mirroring
69    /// `ParsedBlock::blockquote_opens`.
70    pub blockquote_opens: u32,
71}
72
73impl ParsedTable {
74    /// How many columns the table needs to hold every cell it has: as many as its widest
75    /// row has cells. A row shorter than that is completed with empty cells by whoever
76    /// builds the table.
77    ///
78    /// The first row does not say it. The rows of a Djot table differ in length when their
79    /// source does, and an HTML table's first row is often one heading cell spanning every
80    /// column: a table sized from its first row kept the cells of a longer row in the model,
81    /// where they showed, while every export writes a row up to the table's width only, so
82    /// the first save dropped them.
83    pub fn column_count(&self) -> usize {
84        self.rows.iter().map(Vec::len).max().unwrap_or(0)
85    }
86}
87
88/// How many cells a table may reach once its short rows are completed (see
89/// [`ParsedTable::column_count`]), however few it has of its own: a table this size loads
90/// in a moment.
91const SQUARED_CELLS_ALWAYS: usize = 4096;
92
93/// How many times the cells it has of its own a larger table may reach once its short
94/// rows are completed.
95///
96/// A table whose rows differ in length because its source wrote them so, a wide heading
97/// row over rows of one cell or a few, grows to about as many times its own cells as it has
98/// columns: a dozen or so for a table a writer makes. The tables that made hundreds of
99/// thousands of cells out of a few kilobytes grow hundreds of times. At four times, a table
100/// of ten columns heading five hundred rows of one cell was read as paragraphs, its grid
101/// lost for a table that loads in a few milliseconds.
102const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
103
104/// Push `table`, which has `own_cells` cells of its own in its source, onto `elements`: as
105/// a table, or as the paragraphs of its cells when completing its short rows would make it
106/// many times that.
107///
108/// Every row of a loaded table is completed to the widest one. A few kilobytes of a table
109/// whose first row is wide and whose other rows hold one cell each, or of HTML cells each
110/// spanning a thousand columns, made hundreds of thousands of empty cells, and held a load
111/// or a paste for seconds. Such a table is read as the paragraphs of its cells that hold
112/// anything, in reading order, at the table's quotation depth: every word stays, the grid
113/// goes.
114fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
115    let squared = table.rows.len().saturating_mul(table.column_count());
116    let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
117    if squared <= allowed {
118        elements.push(ParsedElement::Table(table));
119        return;
120    }
121    elements.extend(
122        ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
123            .into_iter()
124            .filter(|block| spans_carry_content(&block.spans))
125            .map(ParsedElement::Block),
126    );
127}
128
129/// Record that a quotation opened at the first block or table of `elements` from `first`
130/// on: the quotation that began where `elements` stood at `first` and closes now. Nothing
131/// when the quotation holds none.
132fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
133    let Some(rest) = elements.get_mut(first..) else {
134        return;
135    };
136    for element in rest {
137        match element {
138            ParsedElement::Block(block) => {
139                block.blockquote_opens += 1;
140                return;
141            }
142            ParsedElement::Table(table) => {
143                table.blockquote_opens += 1;
144                return;
145            }
146            ParsedElement::FootnoteDefinition { .. } => {}
147        }
148    }
149}
150
151/// The cells a table's rows hold, for [`push_table`].
152fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
153    rows.iter().map(Vec::len).sum()
154}
155
156/// A parsed element: either a block or a table.
157#[derive(Debug, Clone)]
158pub enum ParsedElement {
159    Block(ParsedBlock),
160    Table(ParsedTable),
161    /// A footnote definition: the label its references name, and the blocks
162    /// making up its body.
163    ///
164    /// Separate from `Block` because a definition is not part of the flow it was
165    /// written in — it belongs wherever the output format puts notes, which the
166    /// importer expresses by giving it a detached frame of its own.
167    FootnoteDefinition {
168        label: String,
169        blocks: Vec<ParsedBlock>,
170    },
171}
172
173impl ParsedElement {
174    /// Extract blocks, flattening tables into one block per cell.
175    /// Use when table structure is not needed.
176    pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
177        let mut blocks = Vec::new();
178        for elem in elements {
179            match elem {
180                ParsedElement::Block(b) => blocks.push(b),
181                // A definition is not part of the flow it was written in — it
182                // belongs wherever the format puts notes. Flattening it in
183                // would splice a note's body into the prose at the point the
184                // definition happened to be typed.
185                ParsedElement::FootnoteDefinition { .. } => {}
186                ParsedElement::Table(t) => {
187                    for row in t.rows {
188                        for cell in row {
189                            blocks.push(ParsedBlock {
190                                spans: cell.spans,
191                                heading_level: None,
192                                list_style: None,
193                                list_indent: 0,
194                                list_prefix: String::new(),
195                                list_suffix: String::new(),
196                                list_start: None,
197                                marker: None,
198                                is_code_block: false,
199                                code_language: None,
200                                blockquote_depth: t.blockquote_depth,
201                                blockquote_opens: 0,
202                                line_height: None,
203                                non_breakable_lines: None,
204                                page_break_before: None,
205                                front_matter: None,
206                                direction: None,
207                                background_color: None,
208                                alignment: None,
209                                top_margin: None,
210                                text_indent: None,
211                                semantic_role: None,
212                            });
213                        }
214                    }
215                }
216            }
217        }
218        if blocks.is_empty() {
219            blocks.push(ParsedBlock {
220                spans: vec![ParsedSpan {
221                    text: String::new(),
222                    ..Default::default()
223                }],
224                heading_level: None,
225                list_style: None,
226                list_indent: 0,
227                list_prefix: String::new(),
228                list_suffix: String::new(),
229                list_start: None,
230                marker: None,
231                is_code_block: false,
232                code_language: None,
233                blockquote_depth: 0,
234                blockquote_opens: 0,
235                line_height: None,
236                non_breakable_lines: None,
237                page_break_before: None,
238                front_matter: None,
239                direction: None,
240                background_color: None,
241                alignment: None,
242                top_margin: None,
243                text_indent: None,
244                semantic_role: None,
245            });
246        }
247        blocks
248    }
249}
250
251/// A parsed block (paragraph, heading, list item, code block)
252///
253/// `Default` is an unformatted, empty paragraph — the shape a caller wants when
254/// it has prose and no structure to give it. Deriving it also means a field
255/// added later reaches those callers as "absent" rather than as a compile error
256/// they would fix by guessing a value.
257#[derive(Debug, Clone, Default)]
258pub struct ParsedBlock {
259    pub spans: Vec<ParsedSpan>,
260    pub heading_level: Option<i64>,
261    pub list_style: Option<ListStyle>,
262    pub list_indent: u32,
263    /// Ordered-list delimiter prefix (e.g. `"("` for djot `(1)` lists; empty
264    /// otherwise). Stored on the `List` entity for round-trip fidelity.
265    pub list_prefix: String,
266    /// Ordered-list delimiter suffix (`"."` for `1.`, `")"` for `1)`/`(1)`;
267    /// empty for unordered lists).
268    pub list_suffix: String,
269    /// The number the first item of the list this item belongs to wears, when it is not 1
270    /// (Djot and Markdown `3.`, HTML `<ol start="3">`). Every item of a list carries it.
271    /// Stored on the `List` entity.
272    pub list_start: Option<i64>,
273    /// Task-list checkbox marker (djot `- [ ]` / `- [x]`). Maps to
274    /// `Block.fmt_marker`. `None` for non-task blocks.
275    pub marker: Option<MarkerType>,
276    pub is_code_block: bool,
277    pub code_language: Option<String>,
278    pub blockquote_depth: u32,
279    /// How many of the quotations the block stands in open at it, the innermost ones:
280    /// quotations as deep as the element's before it but not the same ones. `0` when the
281    /// block goes on in the quotations the element before it stands in, as deep as it
282    /// stands. Two quotations one after the other (Djot and Markdown `> a` and `> b` with a
283    /// blank line between, HTML `<blockquote>` twice) were read as one, since the depth
284    /// alone does not tell them apart.
285    pub blockquote_opens: u32,
286    pub line_height: Option<i64>,
287    pub non_breakable_lines: Option<bool>,
288    /// Start this block on a new page (djot `{page_break_before=true}`). Maps to
289    /// `Block.fmt_page_break_before`. `None` when absent.
290    pub page_break_before: Option<bool>,
291    /// This block opens the document as front matter (djot `{front_matter=true}`). Maps to
292    /// `Block.fmt_front_matter`. `None` when absent.
293    pub front_matter: Option<bool>,
294    pub direction: Option<TextDirection>,
295    pub background_color: Option<String>,
296    /// Paragraph alignment (djot `{alignment=left|right|center|justify}`). Maps
297    /// to `Block.fmt_alignment`. `None` when no alignment attribute is present.
298    pub alignment: Option<Alignment>,
299    /// This block's own space-above (djot `{top_margin=<int>}`). Maps to
300    /// `Block.fmt_top_margin` and overrides the document-wide paragraph
301    /// spacing for this block alone. `None` when absent.
302    pub top_margin: Option<i64>,
303    /// This block's own first-line indent (djot `{text_indent=<int>}`). Maps to
304    /// `Block.fmt_text_indent` and overrides the document-wide first-line
305    /// indent for this block alone. `None` when absent.
306    pub text_indent: Option<i64>,
307    /// The enclosing blockquote's semantic role (djot `{semantic_role=epigraph}`),
308    /// written on the quote's first block because block attributes are the only channel
309    /// djot offers. The importer lifts it onto the `Frame`, where it belongs.
310    pub semantic_role: Option<SemanticRole>,
311}
312
313impl ParsedBlock {
314    /// Returns `true` when this block carries no block-level formatting,
315    /// meaning its content is purely inline.
316    pub fn is_inline_only(&self) -> bool {
317        self.heading_level.is_none()
318            && self.list_style.is_none()
319            && !self.is_code_block
320            && self.blockquote_depth == 0
321            && self.line_height.is_none()
322            && self.non_breakable_lines.is_none()
323            && self.page_break_before.is_none()
324            && self.front_matter.is_none()
325            && self.direction.is_none()
326            && self.background_color.is_none()
327            && self.alignment.is_none()
328            && self.top_margin.is_none()
329            && self.text_indent.is_none()
330    }
331}
332
333// ─── Markdown parsing ────────────────────────────────────────────────
334
335/// Labels referenced as `[^label]` in `markdown` with no `[^label]: ...`
336/// definition anywhere in the document.
337///
338/// pulldown-cmark's default footnote mode (`Options::ENABLE_FOOTNOTES`, i.e.
339/// GitHub's syntax) only emits `Event::FootnoteReference` for a label some
340/// `Tag::FootnoteDefinition` actually defines (verified against 0.13) — an
341/// undefined reference silently decomposes into three ordinary `Text` events
342/// (`"["`, `"^label"`, `"]"`) instead, indistinguishable from a reader having
343/// typed literal brackets. `Options::ENABLE_OLD_FOOTNOTES` recognises it, but
344/// as `parse_markdown`'s doc comment explains, trades away multi-paragraph
345/// definition bodies to get there. Neither flag alone gives both, so
346/// `parse_markdown` hands pulldown-cmark a synthetic empty definition for
347/// every such label instead — real enough for it to recognise the reference,
348/// thrown away again before `elements` is returned.
349///
350/// Finding "every such label" is two parts: which labels pulldown-cmark's own
351/// block scanner (a first pass, at the caller's `options`) already recognises
352/// as truly defined, and which `[^…]`-shaped spans exist in the raw text at
353/// all. The second part is a plain scan, not a parser — it can over-match
354/// (inside a code span, inside a fenced block, a coincidental `[^x]:` in the
355/// middle of a sentence that real block parsing would never treat as a
356/// definition), but over-matching only ever produces an unused synthetic
357/// definition: pulldown-cmark's real, correct parse of the augmented text is
358/// what decides whether any span actually becomes a reference, and
359/// `parse_markdown` drops every synthesized definition regardless of whether
360/// that happened.
361fn dangling_footnote_labels(
362    markdown: &str,
363    options: pulldown_cmark::Options,
364) -> std::collections::BTreeSet<String> {
365    use pulldown_cmark::{Event, Parser, Tag};
366
367    let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
368    for event in Parser::new_ext(markdown, options) {
369        if let Event::Start(Tag::FootnoteDefinition(label)) = event {
370            defined.insert(label.to_string());
371        }
372    }
373
374    let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
375    let bytes = markdown.as_bytes();
376    let mut search_from = 0usize;
377    while let Some(rel) = markdown[search_from..].find("[^") {
378        let open = search_from + rel;
379        let label_start = open + 2;
380        let Some(close_rel) = markdown[label_start..].find(']') else {
381            break;
382        };
383        let close = label_start + close_rel;
384        let label = &markdown[label_start..close];
385        // `]:` immediately after is definition syntax, not a reference —
386        // pulldown-cmark's own scan above already supplied the ground truth
387        // for which labels are truly defined; this only needs to avoid
388        // counting a definition's own label as a "reference" candidate.
389        let looks_like_definition = bytes.get(close + 1) == Some(&b':');
390        if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
391            referenced.insert(label.to_string());
392        }
393        search_from = close + 1;
394    }
395
396    referenced.difference(&defined).cloned().collect()
397}
398
399/// Read `markdown` into blocks and tables.
400///
401/// The parser reads nested containers without recursing, whatever their depth. The
402/// quotations the text holds are kept within
403/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
404/// `quote_depth`): the writers walk one call deeper for each, and a text nested 500
405/// quotations deep loaded, then aborted the process at its first save.
406pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
407    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
408
409    // ENABLE_FOOTNOTES is required for pulldown-cmark to parse `[^label]`
410    // references and `[^label]: body` definitions at all — without it both
411    // fall through as plain link-reference-shaped text. Deliberately NOT
412    // `ENABLE_OLD_FOOTNOTES` too: that variant makes a reference with no
413    // definition survive (see `dangling_footnote_labels`), but in exchange
414    // breaks multi-paragraph definition bodies — a blank line before an
415    // indented continuation escapes the definition and becomes a sibling
416    // code block, which is exactly the shape this crate's own Markdown
417    // exporter writes for a note with more than one paragraph. Getting both
418    // is `dangling_footnote_labels`'s job, not an `Options` flag's.
419    let options = Options::ENABLE_STRIKETHROUGH
420        | Options::ENABLE_TABLES
421        | Options::ENABLE_TASKLISTS
422        | Options::ENABLE_FOOTNOTES;
423
424    // See `dangling_footnote_labels`: pulldown-cmark's GFM-style footnotes
425    // only recognise `[^label]` as a reference when some definition exists
426    // for it anywhere in the document, so an undefined one — the normal
427    // state for a host that owns note bodies itself — has to be given a
428    // throwaway definition before parsing, or it silently becomes the
429    // literal text "[^label]" instead of surviving as a reference.
430    let dangling = dangling_footnote_labels(markdown, options);
431    let augmented_owner;
432    let source: &str = if dangling.is_empty() {
433        markdown
434    } else {
435        augmented_owner = dangling
436            .iter()
437            .fold(markdown.to_string(), |mut acc, label| {
438                acc.push_str("\n\n[^");
439                acc.push_str(label);
440                acc.push_str("]:\n");
441                acc
442            });
443        &augmented_owner
444    };
445    let parser = Parser::new_ext(source, options);
446
447    let mut elements: Vec<ParsedElement> = Vec::new();
448    let mut current_spans: Vec<ParsedSpan> = Vec::new();
449    let mut current_heading: Option<i64> = None;
450    let mut current_list_style: Option<ListStyle> = None;
451    let mut current_list_start: Option<i64> = None;
452    let mut is_code_block = false;
453    let mut code_language: Option<String> = None;
454    let mut blockquote_depth: u32 = 0;
455    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
456    let mut quotations_open: Vec<usize> = Vec::new();
457    let mut in_block = false;
458
459    // Formatting state stack
460    let mut bold = false;
461    let mut italic = false;
462    let mut strikeout = false;
463    let mut link_href: Option<String> = None;
464    // Set between an image's Start and End; its alt text arrives as Text events.
465    let mut pending_image: Option<ParsedImage> = None;
466
467    // The label and element index of the footnote definition currently open,
468    // if any. Mirrors `parse_djot`'s `footnote_open`: a definition's body is
469    // ordinary block content, parsed by the same machinery as everything
470    // else, then lifted back out at `End` into its own top-level element so
471    // it never becomes part of the flow it was written in. CommonMark
472    // footnote definitions cannot nest, so one slot suffices.
473    let mut footnote_open: Option<(String, usize)> = None;
474
475    // List style stack for nested lists (also tracks nesting depth), and the start of each
476    // of those lists beside it.
477    let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
478    let mut list_start_stack: Vec<Option<i64>> = Vec::new();
479    let mut current_list_indent: u32 = 0;
480
481    // Table tracking state
482    let mut in_table = false;
483    let mut in_table_head = false;
484    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
485    let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
486    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
487    let mut table_header_rows: usize = 0;
488
489    for event in parser {
490        match event {
491            Event::Start(Tag::Paragraph) => {
492                in_block = true;
493                current_heading = None;
494                is_code_block = false;
495            }
496            Event::End(TagEnd::Paragraph) => {
497                if !current_spans.is_empty() || in_block {
498                    elements.push(ParsedElement::Block(ParsedBlock {
499                        spans: std::mem::take(&mut current_spans),
500                        heading_level: current_heading.take(),
501                        list_style: current_list_style.clone(),
502                        list_indent: current_list_indent,
503                        list_prefix: String::new(),
504                        list_suffix: String::new(),
505                        list_start: current_list_start,
506                        marker: None,
507                        is_code_block: false,
508                        code_language: None,
509                        blockquote_depth,
510                        blockquote_opens: 0,
511                        line_height: None,
512                        non_breakable_lines: None,
513                        page_break_before: None,
514                        front_matter: None,
515                        direction: None,
516                        background_color: None,
517                        alignment: None,
518                        top_margin: None,
519                        text_indent: None,
520                        semantic_role: None,
521                    }));
522                }
523                in_block = false;
524                current_list_style = None;
525                current_list_start = None;
526            }
527            Event::Start(Tag::Heading { level, .. }) => {
528                in_block = true;
529                current_heading = Some(heading_level_to_i64(level));
530                is_code_block = false;
531            }
532            Event::End(TagEnd::Heading(_)) => {
533                elements.push(ParsedElement::Block(ParsedBlock {
534                    spans: std::mem::take(&mut current_spans),
535                    heading_level: current_heading.take(),
536                    list_style: None,
537                    list_indent: 0,
538                    list_prefix: String::new(),
539                    list_suffix: String::new(),
540                    list_start: None,
541                    marker: None,
542                    is_code_block: false,
543                    code_language: None,
544                    blockquote_depth,
545                    blockquote_opens: 0,
546                    line_height: None,
547                    non_breakable_lines: None,
548                    page_break_before: None,
549                    front_matter: None,
550                    direction: None,
551                    background_color: None,
552                    alignment: None,
553                    top_margin: None,
554                    text_indent: None,
555                    semantic_role: None,
556                }));
557                in_block = false;
558            }
559            Event::Start(Tag::List(ordered)) => {
560                let style = if ordered.is_some() {
561                    Some(ListStyle::Decimal)
562                } else {
563                    Some(ListStyle::Disc)
564                };
565                list_stack.push(style);
566                list_start_stack.push(ordered.and_then(list_start_of));
567            }
568            Event::End(TagEnd::List(_)) => {
569                list_stack.pop();
570                list_start_stack.pop();
571            }
572            Event::Start(Tag::Item) => {
573                // Flush any accumulated spans from the parent item before
574                // starting a child item in a tight list
575                if !current_spans.is_empty() {
576                    elements.push(ParsedElement::Block(ParsedBlock {
577                        spans: std::mem::take(&mut current_spans),
578                        heading_level: None,
579                        list_style: current_list_style.clone(),
580                        list_indent: current_list_indent,
581                        list_prefix: String::new(),
582                        list_suffix: String::new(),
583                        list_start: current_list_start,
584                        marker: None,
585                        is_code_block: false,
586                        code_language: None,
587                        blockquote_depth,
588                        blockquote_opens: 0,
589                        line_height: None,
590                        non_breakable_lines: None,
591                        page_break_before: None,
592                        front_matter: None,
593                        direction: None,
594                        background_color: None,
595                        alignment: None,
596                        top_margin: None,
597                        text_indent: None,
598                        semantic_role: None,
599                    }));
600                }
601                in_block = true;
602                current_list_style = list_stack.last().cloned().flatten();
603                current_list_start = list_start_stack.last().copied().flatten();
604                current_list_indent = if list_stack.is_empty() {
605                    0
606                } else {
607                    (list_stack.len() - 1) as u32
608                };
609            }
610            Event::End(TagEnd::Item) => {
611                // The paragraph inside the item will have already been flushed,
612                // but if there was no inner paragraph (tight list), flush now.
613                if !current_spans.is_empty() {
614                    elements.push(ParsedElement::Block(ParsedBlock {
615                        spans: std::mem::take(&mut current_spans),
616                        heading_level: None,
617                        list_style: current_list_style.clone(),
618                        list_indent: current_list_indent,
619                        list_prefix: String::new(),
620                        list_suffix: String::new(),
621                        list_start: current_list_start,
622                        marker: None,
623                        is_code_block: false,
624                        code_language: None,
625                        blockquote_depth,
626                        blockquote_opens: 0,
627                        line_height: None,
628                        non_breakable_lines: None,
629                        page_break_before: None,
630                        front_matter: None,
631                        direction: None,
632                        background_color: None,
633                        alignment: None,
634                        top_margin: None,
635                        text_indent: None,
636                        semantic_role: None,
637                    }));
638                }
639                in_block = false;
640                current_list_style = None;
641                current_list_start = None;
642            }
643            Event::Start(Tag::CodeBlock(kind)) => {
644                in_block = true;
645                is_code_block = true;
646                code_language = match &kind {
647                    pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
648                        Some(lang.to_string())
649                    }
650                    _ => None,
651                };
652            }
653            Event::End(TagEnd::CodeBlock) => {
654                // pulldown-cmark appends a trailing '\n' to code block text — strip it
655                if let Some(last) = current_spans.last_mut()
656                    && last.text.ends_with('\n')
657                {
658                    last.text.truncate(last.text.len() - 1);
659                }
660                elements.push(ParsedElement::Block(ParsedBlock {
661                    spans: std::mem::take(&mut current_spans),
662                    heading_level: None,
663                    list_style: None,
664                    list_indent: 0,
665                    list_prefix: String::new(),
666                    list_suffix: String::new(),
667                    list_start: None,
668                    marker: None,
669                    is_code_block: true,
670                    code_language: code_language.take(),
671                    blockquote_depth,
672                    blockquote_opens: 0,
673                    line_height: None,
674                    non_breakable_lines: None,
675                    page_break_before: None,
676                    front_matter: None,
677                    direction: None,
678                    background_color: None,
679                    alignment: None,
680                    top_margin: None,
681                    text_indent: None,
682                    semantic_role: None,
683                }));
684                in_block = false;
685                is_code_block = false;
686            }
687            // ─── Table events ───────────────────────────────────────
688            Event::Start(Tag::Table(_)) => {
689                in_table = true;
690                in_table_head = false;
691                table_rows.clear();
692                current_row_cells.clear();
693                current_cell_spans.clear();
694                table_header_rows = 0;
695            }
696            Event::End(TagEnd::Table) => {
697                let own_cells = cells_in_rows(&table_rows);
698                push_table(
699                    &mut elements,
700                    ParsedTable {
701                        header_rows: table_header_rows,
702                        rows: std::mem::take(&mut table_rows),
703                        blockquote_depth,
704                        blockquote_opens: 0,
705                    },
706                    own_cells,
707                );
708                in_table = false;
709            }
710            Event::Start(Tag::TableHead) => {
711                in_table_head = true;
712                current_row_cells.clear();
713            }
714            Event::End(TagEnd::TableHead) => {
715                // Flush the header row
716                table_rows.push(std::mem::take(&mut current_row_cells));
717                table_header_rows += 1;
718                in_table_head = false;
719            }
720            Event::Start(Tag::TableRow) => {
721                current_row_cells.clear();
722            }
723            Event::End(TagEnd::TableRow) if !in_table_head => {
724                // Body rows only — header row is flushed in End(TableHead)
725                table_rows.push(std::mem::take(&mut current_row_cells));
726            }
727            Event::Start(Tag::TableCell) => {
728                current_cell_spans.clear();
729            }
730            Event::End(TagEnd::TableCell) => {
731                current_row_cells.push(ParsedTableCell {
732                    spans: std::mem::take(&mut current_cell_spans),
733                });
734            }
735            // ─── Inline formatting ──────────────────────────────────
736            Event::Start(Tag::Emphasis) => {
737                italic = true;
738            }
739            Event::End(TagEnd::Emphasis) => {
740                italic = false;
741            }
742            Event::Start(Tag::Strong) => {
743                bold = true;
744            }
745            Event::End(TagEnd::Strong) => {
746                bold = false;
747            }
748            Event::Start(Tag::Strikethrough) => {
749                strikeout = true;
750            }
751            Event::End(TagEnd::Strikethrough) => {
752                strikeout = false;
753            }
754            Event::Start(Tag::Link { dest_url, .. }) => {
755                link_href = Some(dest_url.to_string());
756            }
757            Event::End(TagEnd::Link) => {
758                link_href = None;
759            }
760            // Markdown has no syntax for display size, so width/height stay 0
761            // and the caller supplies the image's intrinsic dimensions.
762            Event::Start(Tag::Image { dest_url, .. }) => {
763                pending_image = Some(ParsedImage {
764                    src: dest_url.to_string(),
765                    alt: String::new(),
766                    width: 0,
767                    height: 0,
768                });
769            }
770            Event::End(TagEnd::Image) => {
771                if let Some(image) = pending_image.take() {
772                    let span = ParsedSpan {
773                        text: String::new(),
774                        bold,
775                        italic,
776                        underline: false,
777                        strikeout,
778                        code: false,
779                        superscript: false,
780                        subscript: false,
781                        link_href: link_href.clone(),
782                        image: Some(image),
783                        footnote_ref: None,
784                        preserve_whitespace: false,
785                    };
786                    if in_table {
787                        current_cell_spans.push(span);
788                    } else {
789                        if !in_block {
790                            in_block = true;
791                        }
792                        current_spans.push(span);
793                    }
794                }
795            }
796            Event::Text(text) => {
797                // Inside an image this is its alt text, which pulldown-cmark
798                // emits as an ordinary Text event. Without this guard it fell
799                // through and landed in the paragraph as prose — the image was
800                // dropped and its description silently became manuscript text.
801                if let Some(img) = pending_image.as_mut() {
802                    img.alt.push_str(&text);
803                    continue;
804                }
805                let span = ParsedSpan {
806                    text: text.to_string(),
807                    bold,
808                    italic,
809                    underline: false,
810                    strikeout,
811                    code: is_code_block,
812                    superscript: false,
813                    subscript: false,
814                    link_href: link_href.clone(),
815                    image: None,
816                    footnote_ref: None,
817                    preserve_whitespace: false,
818                };
819                if in_table {
820                    current_cell_spans.push(span);
821                } else {
822                    if !in_block {
823                        in_block = true;
824                    }
825                    current_spans.push(span);
826                }
827            }
828            Event::Code(text) => {
829                let span = ParsedSpan {
830                    text: text.to_string(),
831                    bold,
832                    italic,
833                    underline: false,
834                    strikeout,
835                    code: true,
836                    superscript: false,
837                    subscript: false,
838                    link_href: link_href.clone(),
839                    image: None,
840                    footnote_ref: None,
841                    preserve_whitespace: false,
842                };
843                if in_table {
844                    current_cell_spans.push(span);
845                } else {
846                    if !in_block {
847                        in_block = true;
848                    }
849                    current_spans.push(span);
850                }
851            }
852            Event::SoftBreak => {
853                let span = ParsedSpan {
854                    text: " ".to_string(),
855                    bold,
856                    italic,
857                    underline: false,
858                    strikeout,
859                    code: false,
860                    superscript: false,
861                    subscript: false,
862                    link_href: link_href.clone(),
863                    image: None,
864                    footnote_ref: None,
865                    preserve_whitespace: false,
866                };
867                if in_table {
868                    current_cell_spans.push(span);
869                } else {
870                    current_spans.push(span);
871                }
872            }
873            // A hard break (two spaces or a backslash at the end of the line) is a line
874            // of the same block, as in the Djot reader. A table cell is one line, and so
875            // is an image's description.
876            Event::HardBreak if pending_image.is_some() => {
877                if let Some(img) = pending_image.as_mut() {
878                    img.alt.push(' ');
879                }
880            }
881            Event::HardBreak => {
882                let span = ParsedSpan {
883                    text: if in_table { " " } else { LINE_SEPARATOR_STR }.to_string(),
884                    bold,
885                    italic,
886                    underline: false,
887                    strikeout,
888                    code: false,
889                    superscript: false,
890                    subscript: false,
891                    link_href: link_href.clone(),
892                    image: None,
893                    footnote_ref: None,
894                    preserve_whitespace: false,
895                };
896                if in_table {
897                    current_cell_spans.push(span);
898                } else {
899                    current_spans.push(span);
900                }
901            }
902            Event::Start(Tag::BlockQuote(_)) => {
903                blockquote_depth += 1;
904                quotations_open.push(elements.len());
905            }
906            Event::End(TagEnd::BlockQuote(_)) => {
907                blockquote_depth = blockquote_depth.saturating_sub(1);
908                if let Some(first) = quotations_open.pop() {
909                    mark_quotation_opened(&mut elements, first);
910                }
911            }
912            // ── Footnote definitions ──
913            //
914            // `[^label]: body` opens a block-level container; its body is
915            // ordinary paragraphs/lists/etc., pushed onto `elements` by the
916            // ordinary machinery above. `End` drains everything pushed since
917            // `Start` back out into one `FootnoteDefinition`, exactly as
918            // `parse_djot` does for the same `[^label]: body` syntax.
919            // Flushing any spans left open by an interrupted block first
920            // matches every other container boundary in this parser
921            // (List/Item/BlockQuote).
922            Event::Start(Tag::FootnoteDefinition(label)) => {
923                if !current_spans.is_empty() {
924                    elements.push(ParsedElement::Block(ParsedBlock {
925                        spans: std::mem::take(&mut current_spans),
926                        heading_level: current_heading.take(),
927                        list_style: current_list_style.clone(),
928                        list_indent: current_list_indent,
929                        list_prefix: String::new(),
930                        list_suffix: String::new(),
931                        list_start: current_list_start,
932                        marker: None,
933                        is_code_block: false,
934                        code_language: None,
935                        blockquote_depth,
936                        blockquote_opens: 0,
937                        line_height: None,
938                        non_breakable_lines: None,
939                        page_break_before: None,
940                        front_matter: None,
941                        direction: None,
942                        background_color: None,
943                        alignment: None,
944                        top_margin: None,
945                        text_indent: None,
946                        semantic_role: None,
947                    }));
948                }
949                footnote_open = Some((label.to_string(), elements.len()));
950            }
951            Event::End(TagEnd::FootnoteDefinition) => {
952                if !current_spans.is_empty() {
953                    elements.push(ParsedElement::Block(ParsedBlock {
954                        spans: std::mem::take(&mut current_spans),
955                        heading_level: current_heading.take(),
956                        list_style: current_list_style.clone(),
957                        list_indent: current_list_indent,
958                        list_prefix: String::new(),
959                        list_suffix: String::new(),
960                        list_start: current_list_start,
961                        marker: None,
962                        is_code_block: false,
963                        code_language: None,
964                        blockquote_depth,
965                        blockquote_opens: 0,
966                        line_height: None,
967                        non_breakable_lines: None,
968                        page_break_before: None,
969                        front_matter: None,
970                        direction: None,
971                        background_color: None,
972                        alignment: None,
973                        top_margin: None,
974                        text_indent: None,
975                        semantic_role: None,
976                    }));
977                }
978                if let Some((label, start)) = footnote_open.take() {
979                    let blocks: Vec<ParsedBlock> = elements
980                        .drain(start..)
981                        .filter_map(|e| match e {
982                            ParsedElement::Block(b) => Some(b),
983                            // A table inside a footnote is not representable
984                            // as note content, same limitation as djot.
985                            _ => None,
986                        })
987                        .collect();
988                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
989                }
990            }
991            // A footnote reference. pulldown-cmark emits this whether or not
992            // a matching `[^label]:` definition exists anywhere in the
993            // document — a dangling reference (the normal state for a host
994            // that owns note bodies itself) must survive just the same,
995            // mirroring `parse_djot`'s `E::FootnoteReference` handling.
996            Event::FootnoteReference(label) => {
997                let span = ParsedSpan {
998                    text: String::new(),
999                    bold,
1000                    italic,
1001                    underline: false,
1002                    strikeout,
1003                    code: false,
1004                    superscript: false,
1005                    subscript: false,
1006                    link_href: link_href.clone(),
1007                    image: None,
1008                    footnote_ref: Some(label.to_string()),
1009                    preserve_whitespace: false,
1010                };
1011                if in_table {
1012                    current_cell_spans.push(span);
1013                } else {
1014                    if !in_block {
1015                        in_block = true;
1016                    }
1017                    current_spans.push(span);
1018                }
1019            }
1020            _ => {}
1021        }
1022    }
1023
1024    // Flush any remaining content
1025    if !current_spans.is_empty() {
1026        elements.push(ParsedElement::Block(ParsedBlock {
1027            spans: std::mem::take(&mut current_spans),
1028            heading_level: current_heading,
1029            list_style: current_list_style,
1030            list_indent: current_list_indent,
1031            list_prefix: String::new(),
1032            list_suffix: String::new(),
1033            list_start: current_list_start,
1034            marker: None,
1035            is_code_block,
1036            code_language: code_language.take(),
1037            blockquote_depth,
1038            blockquote_opens: 0,
1039            line_height: None,
1040            non_breakable_lines: None,
1041            page_break_before: None,
1042            front_matter: None,
1043            direction: None,
1044            background_color: None,
1045            alignment: None,
1046            top_margin: None,
1047            text_indent: None,
1048            semantic_role: None,
1049        }));
1050    }
1051
1052    // Drop the throwaway definitions `dangling_footnote_labels` asked for —
1053    // they exist only to make pulldown-cmark recognise the reference as
1054    // real, never to become a note the document owns. `elements` cannot
1055    // become empty from this: a label reached `dangling` only via a
1056    // reference actually present in `markdown`, so its surrounding block
1057    // survives even with the synthetic definition gone.
1058    if !dangling.is_empty() {
1059        elements.retain(
1060            |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1061        );
1062    }
1063
1064    // If no elements were parsed, create a single empty paragraph
1065    if elements.is_empty() {
1066        elements.push(ParsedElement::Block(ParsedBlock {
1067            spans: vec![ParsedSpan {
1068                text: String::new(),
1069                ..Default::default()
1070            }],
1071            heading_level: None,
1072            list_style: None,
1073            list_indent: 0,
1074            list_prefix: String::new(),
1075            list_suffix: String::new(),
1076            list_start: None,
1077            marker: None,
1078            is_code_block: false,
1079            code_language: None,
1080            blockquote_depth: 0,
1081            blockquote_opens: 0,
1082            line_height: None,
1083            non_breakable_lines: None,
1084            page_break_before: None,
1085            front_matter: None,
1086            direction: None,
1087            background_color: None,
1088            alignment: None,
1089            top_margin: None,
1090            text_indent: None,
1091            semantic_role: None,
1092        }));
1093    }
1094
1095    // Quotations past the ceiling a document holds are flattened into the deepest kept
1096    // (see `quote_depth`): the writers walk one call deeper for each.
1097    crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1098    elements
1099}
1100
1101fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1102    use pulldown_cmark::HeadingLevel;
1103    match level {
1104        HeadingLevel::H1 => 1,
1105        HeadingLevel::H2 => 2,
1106        HeadingLevel::H3 => 3,
1107        HeadingLevel::H4 => 4,
1108        HeadingLevel::H5 => 5,
1109        HeadingLevel::H6 => 6,
1110    }
1111}
1112
1113// ─── HTML parsing ────────────────────────────────────────────────────
1114
1115use scraper::Node;
1116
1117/// Parsed CSS block-level styles from an inline `style` attribute.
1118#[derive(Debug, Clone, Default)]
1119struct BlockStyles {
1120    line_height: Option<i64>,
1121    non_breakable_lines: Option<bool>,
1122    page_break_before: Option<bool>,
1123    direction: Option<TextDirection>,
1124    background_color: Option<String>,
1125    /// `white-space` asks for the source's own spacing to be kept verbatim.
1126    /// Separate from `non_breakable_lines` because the two do not coincide:
1127    /// `nowrap` forbids wrapping but still collapses runs of spaces, so a
1128    /// block carrying it must be normalised like any other.
1129    preserve_whitespace: Option<bool>,
1130}
1131
1132/// Parse relevant CSS properties from an inline style string.
1133/// Handles: line-height, white-space, break-before/page-break-before, direction,
1134/// background-color.
1135fn parse_block_styles(style: &str) -> BlockStyles {
1136    let mut result = BlockStyles::default();
1137    for part in style.split(';') {
1138        let part = part.trim();
1139        if let Some((prop, val)) = part.split_once(':') {
1140            let prop = prop.trim().to_ascii_lowercase();
1141            let val = val.trim();
1142            match prop.as_str() {
1143                "line-height" => {
1144                    // Try parsing as a plain number (multiplier)
1145                    if let Ok(v) = val.parse::<f64>() {
1146                        result.line_height = Some((v * 1000.0) as i64);
1147                    }
1148                }
1149                "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1150                    result.non_breakable_lines = Some(true);
1151                    if val != "nowrap" {
1152                        result.preserve_whitespace = Some(true);
1153                    }
1154                }
1155                // CSS3 `break-before` and its CSS2 predecessor `page-break-before` mean
1156                // the same thing; both are read because both are written (browsers still
1157                // want the legacy spelling, and so do most EPUB engines).
1158                "break-before" | "page-break-before" => {
1159                    result.page_break_before = match val.to_ascii_lowercase().as_str() {
1160                        "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1161                        "avoid" | "auto" => Some(false),
1162                        _ => None,
1163                    };
1164                }
1165                "direction" => {
1166                    if val.eq_ignore_ascii_case("rtl") {
1167                        result.direction = Some(TextDirection::RightToLeft);
1168                    } else if val.eq_ignore_ascii_case("ltr") {
1169                        result.direction = Some(TextDirection::LeftToRight);
1170                    }
1171                }
1172                "background-color" | "background" => {
1173                    result.background_color = Some(val.to_string());
1174                }
1175                _ => {}
1176            }
1177        }
1178    }
1179    result
1180}
1181
1182/// What an inline `style` says of the whitespace in an element's text: `Some(true)` when
1183/// `white-space` keeps it as it is (`pre`, `pre-wrap`, `break-spaces`), `Some(false)` when
1184/// it collapses it (`normal`, `nowrap`, `pre-line`), `None` when the style does not say.
1185/// The last declaration wins, as in CSS, unless an earlier one is `!important` and it is
1186/// not.
1187fn style_keeps_whitespace(style: &str) -> Option<bool> {
1188    let mut kept = None;
1189    let mut important_kept = false;
1190    for part in style.split(';') {
1191        let Some((property, value)) = part.split_once(':') else {
1192            continue;
1193        };
1194        if !property.trim().eq_ignore_ascii_case("white-space") {
1195            continue;
1196        }
1197        let value = value.trim().to_ascii_lowercase();
1198        let (value, important) = value
1199            .strip_suffix("important")
1200            .and_then(|rest| rest.trim_end().strip_suffix('!'))
1201            .map_or((value.as_str(), false), |rest| (rest.trim_end(), true));
1202        let says = match value {
1203            "pre" | "pre-wrap" | "break-spaces" => true,
1204            "normal" | "nowrap" | "pre-line" => false,
1205            _ => continue,
1206        };
1207        if important || !important_kept {
1208            kept = Some(says);
1209            important_kept |= important;
1210        }
1211    }
1212    kept
1213}
1214
1215/// Write `text`, a run of a paragraph's text, into `out` as HTML: each of its lines (it
1216/// separates them with [`LINE_SEPARATOR`]) escaped by `escape`, the lines joined by
1217/// `line_break`, and a line whose whitespace an HTML reader would collapse put in a
1218/// `<span style="white-space: pre-wrap">`, where browsers, Qt and this crate's own HTML
1219/// reader keep it as it is.
1220///
1221/// A line is wrapped when it holds a tab or another HTML space but the plain space, two
1222/// spaces in a row, a space at its start when `after_space` (the text before it on its
1223/// line ends in whitespace, or the line starts with it), or a space at its end when
1224/// `ends_line` (its line ends with it). Written bare, a tab came back as a space and a
1225/// writer's double space as one, so the HTML of a text read back was not the HTML it had
1226/// been read from. Text an HTML reader keeps as it is goes out bare, as before.
1227///
1228/// `after_space` for the next run is [`html_text_ends_open`].
1229pub fn push_html_text(
1230    out: &mut String,
1231    text: &str,
1232    after_space: bool,
1233    ends_line: bool,
1234    escape: impl Fn(&str) -> String,
1235    line_break: &str,
1236) {
1237    let mut lines = text.split(LINE_SEPARATOR).peekable();
1238    let mut first = true;
1239    while let Some(line) = lines.next() {
1240        if !first {
1241            out.push_str(line_break);
1242        }
1243        let opens_on_space = if first { after_space } else { true };
1244        let ends = ends_line || lines.peek().is_some();
1245        first = false;
1246        let collapses = line.chars().any(|ch| ch != ' ' && is_html_space(ch))
1247            || line.contains("  ")
1248            || (opens_on_space && line.starts_with(' '))
1249            || (ends && line.ends_with(' '));
1250        if collapses {
1251            out.push_str("<span style=\"white-space: pre-wrap\">");
1252            out.push_str(&escape(line));
1253            out.push_str("</span>");
1254        } else {
1255            out.push_str(&escape(line));
1256        }
1257    }
1258}
1259
1260/// Whether a paragraph's line ends before `rest`, the inline content after a run of its
1261/// text: the `ends_line` of that run (see [`push_html_text`]). It does when nothing but
1262/// empty content follows, or when the next text starts a line.
1263pub fn html_line_ends_before<'a>(
1264    rest: impl IntoIterator<Item = &'a crate::format_runs::InlineContent>,
1265) -> bool {
1266    use crate::format_runs::InlineContent;
1267    for content in rest {
1268        match content {
1269            InlineContent::Empty => {}
1270            InlineContent::Text(text) if text.is_empty() => {}
1271            InlineContent::Text(text) => return text.starts_with(LINE_SEPARATOR),
1272            _ => return false,
1273        }
1274    }
1275    true
1276}
1277
1278/// Whether the text of a paragraph's line, `text` added to what came before it on its
1279/// line, ends in whitespace an HTML reader would collapse a space after it into: the
1280/// `after_space` of the run after it (see [`push_html_text`]). `after_space` says the same
1281/// of the text before `text`, and is the answer when `text` is empty.
1282pub fn html_text_ends_open(text: &str, after_space: bool) -> bool {
1283    match text.chars().next_back() {
1284        None => after_space,
1285        Some(ch) => ch == LINE_SEPARATOR || is_html_space(ch),
1286    }
1287}
1288
1289/// The five characters HTML calls whitespace.
1290///
1291/// Deliberately not `char::is_whitespace`, which is `true` for U+00A0: a
1292/// no-break space is a character the writer typed — the one French typography
1293/// puts before a colon, holding "Attention&nbsp;:" together — and collapsing it
1294/// away is a silent edit of the text, not of its layout.
1295fn is_html_space(ch: char) -> bool {
1296    matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1297}
1298
1299/// Whether a span run holds anything a reader would see.
1300fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1301    spans
1302        .iter()
1303        .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1304}
1305
1306/// Apply the CSS `white-space: normal` rules to one block's inline spans: every
1307/// run of HTML whitespace collapses to a single space, and the runs at the
1308/// block's two ends disappear entirely.
1309///
1310/// Per block rather than per text node, because a run crosses span boundaries —
1311/// `<b>word</b>\n<i>next</i>` is one collapsible run split over three nodes —
1312/// and because only the block knows where its own ends are.
1313///
1314/// Without this, an exporter's own source formatting reaches the document as
1315/// text. LibreOffice writes a newline after every `<p …>` and hard-wraps the
1316/// prose inside it at about seventy columns, so pasting a whole book put a
1317/// blank line before every paragraph and a break through the middle of every
1318/// sentence.
1319fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1320    fn is_replaced(span: &ParsedSpan) -> bool {
1321        span.image.is_some() || span.footnote_ref.is_some()
1322    }
1323
1324    // Collapse within each span first. Nothing non-empty is emptied here, which
1325    // is what lets the second pass read `is_empty()` as "this span held no text
1326    // to begin with" — the marker a `<br>` leaves behind.
1327    for span in spans.iter_mut() {
1328        if is_replaced(span) || span.preserve_whitespace {
1329            continue;
1330        }
1331        let mut out = String::with_capacity(span.text.len());
1332        let mut in_run = false;
1333        for ch in span.text.chars() {
1334            if is_html_space(ch) {
1335                if !in_run {
1336                    out.push(' ');
1337                    in_run = true;
1338                }
1339            } else {
1340                out.push(ch);
1341                in_run = false;
1342            }
1343        }
1344        span.text = out;
1345    }
1346
1347    // The block's start counts as "already ended on a space", so a run opening
1348    // the block is dropped outright. A line break (the span a `<br>` leaves) ends one
1349    // line and starts the next, so the runs either side of it go as the block's own
1350    // ends do: `x <br> y` is two lines, "x" and "y".
1351    let mut prev_ends_with_space = true;
1352    // The last span holding text since the line began, whose trailing space a line
1353    // break takes off.
1354    let mut line_end: Option<usize> = None;
1355    for i in 0..spans.len() {
1356        let span = &mut spans[i];
1357        if is_replaced(span) {
1358            prev_ends_with_space = false;
1359            line_end = None;
1360            continue;
1361        }
1362        if span.text.is_empty() {
1363            // A span that held no text to begin with. It contributes none, but it
1364            // still ends the collapsible run either side of it.
1365            prev_ends_with_space = false;
1366            continue;
1367        }
1368        if span.text == LINE_SEPARATOR_STR {
1369            if let Some(end) = line_end.take()
1370                && spans[end].text.ends_with(' ')
1371            {
1372                spans[end].text.pop();
1373            }
1374            prev_ends_with_space = true;
1375            continue;
1376        }
1377        // Kept whitespace is the text's own, as a replaced element is content: a space
1378        // after it is not collapsed into it, and a line break takes none of it off.
1379        if span.preserve_whitespace {
1380            prev_ends_with_space = false;
1381            line_end = None;
1382            continue;
1383        }
1384        if prev_ends_with_space && span.text.starts_with(' ') {
1385            span.text.remove(0);
1386        }
1387        // A span emptied by that drop leaves the run open rather than closing
1388        // it: the space it carried belongs to the run its neighbours share.
1389        if !span.text.is_empty() {
1390            prev_ends_with_space = span.text.ends_with(' ');
1391            line_end = Some(i);
1392        }
1393    }
1394
1395    // The block's trailing run, in whichever span still holds it. Replaced
1396    // content stops the walk: a space before a trailing image is between two
1397    // pieces of content, not after the last one. A line break at the very end
1398    // starts no line of its own, as a browser shows it (a second one does: `a<br><br>`
1399    // is "a" and an empty line), so the last one goes, with the run before it.
1400    let mut trailing_break_dropped = false;
1401    for span in spans.iter_mut().rev() {
1402        if is_replaced(span) {
1403            break;
1404        }
1405        if span.text.is_empty() {
1406            continue;
1407        }
1408        if span.text == LINE_SEPARATOR_STR {
1409            if trailing_break_dropped {
1410                break;
1411            }
1412            span.text.clear();
1413            trailing_break_dropped = true;
1414            continue;
1415        }
1416        if span.preserve_whitespace {
1417            break;
1418        }
1419        if span.text.ends_with(' ') {
1420            span.text.pop();
1421        }
1422        break;
1423    }
1424}
1425
1426/// `block` as one block per line of its text, a line ending at each `\n`.
1427///
1428/// A `\n` is not a line break inside a block, which the model holds as
1429/// [`LINE_SEPARATOR`]: in the rope it separates two blocks, so a new line of this kind
1430/// is a new block, which is what a plain-text paste makes of one. Text that kept its
1431/// source's line breaks (`<pre>`, `white-space: pre`) held them inside one block, where
1432/// the editor showed them as lines and the Djot writer had nothing to write them as.
1433/// Written raw, a blank line ended the paragraph and the lines after it were read back
1434/// as the lists, headings and quotations their markers spelled, however deep.
1435///
1436/// Each line keeps the block's own formatting and its spans' formatting, but for the
1437/// page break, which stays with the first line: it is where the block starts, and on
1438/// every line it put each on a page of its own. An empty line is an empty block. The
1439/// line break that ends the last line starts no line of its own, as a browser shows it.
1440/// A block without a line break is returned as it is.
1441///
1442/// Every line is built from copies of the block and of the span it comes from that
1443/// hold no text, so the split costs in proportion to the text: built from whole copies,
1444/// each line copied the whole text again, and a paste of 64,000 preformatted lines
1445/// took seconds.
1446pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1447    let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1448    if !block.spans.iter().any(breaks) {
1449        return vec![block];
1450    }
1451    let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1452    for mut span in std::mem::take(&mut block.spans) {
1453        if !breaks(&span) {
1454            push_to_last_line(&mut lines, span);
1455            continue;
1456        }
1457        let text = std::mem::take(&mut span.text);
1458        for (i, piece) in text.split('\n').enumerate() {
1459            if i > 0 {
1460                lines.push(Vec::new());
1461            }
1462            if !piece.is_empty() {
1463                push_to_last_line(
1464                    &mut lines,
1465                    ParsedSpan {
1466                        text: piece.to_string(),
1467                        ..span.clone()
1468                    },
1469                );
1470            }
1471        }
1472    }
1473    if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1474        lines.pop();
1475    }
1476    let later_line = ParsedBlock {
1477        page_break_before: None,
1478        blockquote_opens: 0,
1479        ..block.clone()
1480    };
1481    let mut first_line = Some(block);
1482    lines
1483        .into_iter()
1484        .map(|spans| {
1485            let spans = if spans.is_empty() {
1486                vec![ParsedSpan::default()]
1487            } else {
1488                spans
1489            };
1490            match first_line.take() {
1491                Some(first) => ParsedBlock { spans, ..first },
1492                None => ParsedBlock {
1493                    spans,
1494                    ..later_line.clone()
1495                },
1496            }
1497        })
1498        .collect()
1499}
1500
1501/// Append `span` to the last of `lines`, which [`split_block_at_line_breaks`] keeps
1502/// non-empty.
1503fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1504    match lines.last_mut() {
1505        Some(line) => line.push(span),
1506        None => lines.push(vec![span]),
1507    }
1508}
1509
1510pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1511    ParsedElement::flatten_to_blocks(parse_html_elements(html))
1512}
1513
1514/// The attribute an HTML producer uses to say "this element *is* a footnote
1515/// reference", and the label it names.
1516///
1517/// A data attribute on an otherwise-ordinary element, rather than a literal
1518/// `[^label]` in the text, because the text a producer emits is *escaped* on the
1519/// way to Djot — [`escape_djot_inline`](crate::parser_tools::djot_escape::escape_djot_inline)
1520/// neutralises `[`, `]` and `^`, so a reference smuggled through as characters
1521/// arrives as `\[\^1\]` and is prose rather than a reference. It has to be a
1522/// node before it is serialised, which is what this makes it.
1523///
1524/// [`ParsedSpan::footnote_ref`] is the same field the Markdown/Djot reader
1525/// already fills, so everything downstream — the document model, the Djot
1526/// writer, numbering — needs no change at all.
1527pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1528
1529/// Build a footnote-reference span from any element carrying
1530/// [`HTML_FOOTNOTE_ATTR`].
1531fn html_footnote_span(
1532    el: &scraper::node::Element,
1533    link_href: Option<String>,
1534) -> Option<ParsedSpan> {
1535    let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1536    if label.is_empty() {
1537        return None;
1538    }
1539    Some(ParsedSpan {
1540        text: String::new(),
1541        link_href,
1542        footnote_ref: Some(label.to_string()),
1543        ..Default::default()
1544    })
1545}
1546
1547/// Build an inline-image span from an `<img>` element, if it has a usable
1548/// source.
1549///
1550/// `<img>` was matched by none of the HTML walker's three tag dispatches, so it
1551/// fell into their wildcard arms and was dropped whole — silently, and unlike
1552/// Markdown not even leaving its alt text behind. That is the path a browser or
1553/// Word paste travels.
1554fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1555    let src = el.attr("src")?;
1556    if src.is_empty() {
1557        return None;
1558    }
1559    let dim = |name: &str| -> i64 {
1560        el.attr(name)
1561            .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1562            .filter(|n| *n > 0)
1563            .unwrap_or(0)
1564    };
1565    Some(ParsedSpan {
1566        text: String::new(),
1567        link_href,
1568        image: Some(ParsedImage {
1569            src: src.to_string(),
1570            alt: el.attr("alt").unwrap_or_default().to_string(),
1571            width: dim("width"),
1572            height: dim("height"),
1573        }),
1574        ..Default::default()
1575    })
1576}
1577
1578/// Whether `node` is a `<ul>` or an `<ol>`.
1579fn is_html_list(node: &Node) -> bool {
1580    node.as_element()
1581        .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1582}
1583
1584/// Move every list item nested deeper than
1585/// [`MAX_LIST_LEVELS`](crate::parser_tools::list_depth::MAX_LIST_LEVELS)
1586/// lists into the deepest list kept, right after the item holding it, in
1587/// document order: a list nested past that depth comes out flattened to it,
1588/// every item kept, in its order.
1589///
1590/// The reader recurses a few elements deep per list level and stops at
1591/// `MAX_RECURSION_DEPTH`, so a list nested past some eighty levels lost every
1592/// item below that, pasted or loaded. The editing gestures nest no deeper than
1593/// this depth, and an insertion brings any list back to it (see `list_depth`),
1594/// so HTML holding more is read at that depth to begin with. The tree is
1595/// changed in place, without recursion, before it is read.
1596fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1597    use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1598    use ego_tree::iter::Edge;
1599
1600    // The lists one level past the deepest kept, in document order.
1601    let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1602    let mut depth = 0usize;
1603    for edge in tree.root().traverse() {
1604        match edge {
1605            Edge::Open(node) if is_html_list(node.value()) => {
1606                if depth == MAX_LIST_LEVELS {
1607                    too_deep.push(node.id());
1608                }
1609                depth += 1;
1610            }
1611            Edge::Close(node) if is_html_list(node.value()) => {
1612                depth = depth.saturating_sub(1);
1613            }
1614            _ => {}
1615        }
1616    }
1617
1618    for list in too_deep {
1619        // The child of the deepest kept list that holds `list` (its `<li>`, or
1620        // `list` itself when a list sits right inside another): the items go
1621        // after it.
1622        let Some(list_ref) = tree.get(list) else {
1623            continue;
1624        };
1625        let mut holder = None;
1626        let mut current = list_ref;
1627        while let Some(parent) = current.parent() {
1628            if is_html_list(parent.value()) {
1629                holder = Some(current.id());
1630                break;
1631            }
1632            current = parent;
1633        }
1634        let Some(holder) = holder else {
1635            continue;
1636        };
1637        let mut after = holder;
1638        let items: Vec<ego_tree::NodeId> = list_ref
1639            .descendants()
1640            .filter(|node| {
1641                node.value()
1642                    .as_element()
1643                    .is_some_and(|el| el.name() == "li")
1644            })
1645            .map(|node| node.id())
1646            .collect();
1647        for item in items {
1648            // Each node placed here sits in the deepest kept list, so it has a
1649            // parent to be placed beside.
1650            if tree.get(after).and_then(|node| node.parent()).is_none() {
1651                break;
1652            }
1653            if let Some(mut node) = tree.get_mut(after) {
1654                node.insert_id_after(item);
1655                after = item;
1656            }
1657        }
1658        // What is left of the list is its items' wrappers, or text standing in
1659        // it outside any item. It goes after the items when it holds any text
1660        // (it is already beside them when it sits right in the deepest kept
1661        // list), and away when it holds none.
1662        let holds_text = tree.get(list).is_some_and(|node| {
1663            node.descendants()
1664                .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1665        });
1666        if !holds_text {
1667            if let Some(mut node) = tree.get_mut(list) {
1668                node.detach();
1669            }
1670        } else if holder != list
1671            && tree.get(after).and_then(|node| node.parent()).is_some()
1672            && let Some(mut node) = tree.get_mut(after)
1673        {
1674            node.insert_id_after(list);
1675        }
1676    }
1677}
1678
1679/// How deep a walk of an HTML tree may go on the caller's own stack: a debug build spends
1680/// between 5 and 8 KiB of stack on each element level it follows, so this is at most some
1681/// 256 KiB, what any thread a host reads on has to spare.
1682const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1683
1684/// The deepest an HTML reader follows elements: a subtree nested deeper is read as its
1685/// text, without recursing (see `read_html_tree`).
1686const MAX_HTML_READ_DEPTH: usize = 256;
1687
1688/// The stack a reader of a deeper tree gets on a thread of its own, before what each level
1689/// adds: a spawned thread's.
1690const HTML_READER_STACK: usize = 2 << 20;
1691
1692/// The stack each element level of a deeper tree adds for its reader: two to three times
1693/// what a debug build spends on one.
1694const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1695
1696/// `value` read as HTML reads an integer attribute such as `<ol start>`: after any leading
1697/// white space, an optional sign and the digits up to the first character that is not one.
1698/// `None` when no digit follows. A number past what 64 bits hold reads as the largest they
1699/// hold on its side of 0. A browser numbers `<ol start="5x">` from 5, where a strict parse
1700/// of the whole value numbered it from 1.
1701fn html_integer(value: &str) -> Option<i64> {
1702    let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1703    let (negative, digits) = match value.as_bytes().first() {
1704        Some(b'-') => (true, &value[1..]),
1705        Some(b'+') => (false, &value[1..]),
1706        _ => (false, value),
1707    };
1708    let digits: &str = digits
1709        .split(|c: char| !c.is_ascii_digit())
1710        .next()
1711        .unwrap_or("");
1712    if digits.is_empty() {
1713        return None;
1714    }
1715    let magnitude = digits.bytes().fold(0i64, |number, digit| {
1716        number
1717            .saturating_mul(10)
1718            .saturating_add(i64::from(digit - b'0'))
1719    });
1720    Some(if negative { -magnitude } else { magnitude })
1721}
1722
1723/// How many elements deep the deepest node of `tree` is. A loop over the tree's edges,
1724/// never a recursion.
1725fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1726    use ego_tree::iter::Edge;
1727    let mut depth = 0usize;
1728    let mut deepest = 0usize;
1729    for edge in tree.root().traverse() {
1730        match edge {
1731            Edge::Open(node) if node.value().is_element() => {
1732                depth += 1;
1733                deepest = deepest.max(depth);
1734            }
1735            Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1736            _ => {}
1737        }
1738    }
1739    deepest
1740}
1741
1742/// Read `html` into blocks and tables.
1743///
1744/// The reader recurses once per element level it follows, up to `MAX_HTML_READ_DEPTH`,
1745/// and a stack overflow aborts the process: a debug build ran out of a spawned thread's
1746/// stack on a paste nested some 200 elements deep, and a host reading on a smaller thread
1747/// sooner. A tree nested deeper than a caller's stack is sure to hold is read on a thread
1748/// of its own, with a stack sized for its depth; when no thread can be started, it is read
1749/// on the caller's stack to `HTML_DEPTH_ON_THE_CALLERS_STACK` levels, each subtree
1750/// deeper than that read as its text. The quotations it holds are kept within
1751/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
1752/// `quote_depth`).
1753pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1754    use scraper::Html;
1755
1756    let read = |depth_limit: usize, html: &str| {
1757        let mut fragment = Html::parse_fragment(html);
1758        flatten_deep_html_lists(&mut fragment.tree);
1759        let mut elements = read_html_tree(&fragment, depth_limit);
1760        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1761        elements
1762    };
1763    let mut fragment = Html::parse_fragment(html);
1764    flatten_deep_html_lists(&mut fragment.tree);
1765    let depth = html_tree_depth(&fragment.tree);
1766    if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1767        let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1768        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1769        return elements;
1770    }
1771    drop(fragment);
1772    let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1773    let owned = html.to_owned();
1774    let on_its_own_thread = std::thread::Builder::new()
1775        .stack_size(stack)
1776        .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1777        .ok()
1778        .and_then(|reader| reader.join().ok());
1779    on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1780}
1781
1782/// Read `fragment`, following its elements `depth_limit` deep at most: a subtree nested
1783/// deeper is read as its text (see `subtree_text`). The walk recurses once per element
1784/// level it follows.
1785fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1786    let mut elements: Vec<ParsedElement> = Vec::new();
1787
1788    // Walk the DOM tree starting from the root
1789    let root = fragment.root_element();
1790
1791    #[derive(Clone, Default)]
1792    struct FmtState {
1793        bold: bool,
1794        italic: bool,
1795        underline: bool,
1796        strikeout: bool,
1797        code: bool,
1798        superscript: bool,
1799        subscript: bool,
1800        link_href: Option<String>,
1801        /// Under an element whose `white-space` keeps whitespace as it is.
1802        preserve_whitespace: bool,
1803    }
1804
1805    /// The list an element stands in: its style, and the number its first item wears when
1806    /// it is not 1 (`<ol start="3">`).
1807    type HtmlList = (Option<ListStyle>, Option<i64>);
1808
1809    /// Elements whose text content is machinery, not prose.
1810    ///
1811    /// `<style>` and `<script>` hold raw text the HTML spec never renders, and
1812    /// `<head>` and its metadata children carry none a reader is meant to see.
1813    /// Every walker below recurses into unknown tags on the assumption that
1814    /// they are inline wrappers, which turns a stylesheet into paragraphs —
1815    /// and Word, Google Docs and Chrome all put a `<style>` block in the
1816    /// `text/html` flavour they publish to the clipboard, so this is the
1817    /// ordinary paste, not an exotic one.
1818    fn is_metadata_tag(tag: &str) -> bool {
1819        matches!(
1820            tag,
1821            "head"
1822                | "style"
1823                | "script"
1824                | "title"
1825                | "meta"
1826                | "link"
1827                | "base"
1828                | "noscript"
1829                | "template"
1830        )
1831    }
1832
1833    /// The text of everything under `node`, for a subtree nested deeper than
1834    /// the reader follows (see [`MAX_HTML_READ_DEPTH`]): read without recursing,
1835    /// with a space where a block ends so that words do not run together, and
1836    /// without what [`is_metadata_tag`] names.
1837    ///
1838    /// Past that depth the reader used to stop and keep nothing: text nested
1839    /// deeply enough in lists, quotations or plain `<div>`s was dropped from a
1840    /// paste without a trace.
1841    fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1842        use ego_tree::iter::Edge;
1843        let mut text = String::new();
1844        // How many elements deep the walk is inside a metadata element.
1845        let mut hidden = 0usize;
1846        for edge in node.traverse() {
1847            match edge {
1848                Edge::Open(open) => match open.value() {
1849                    Node::Element(el) => {
1850                        if hidden > 0 || is_metadata_tag(el.name()) {
1851                            hidden += 1;
1852                        } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1853                            text.push(' ');
1854                        }
1855                    }
1856                    Node::Text(t) if hidden == 0 => text.push_str(t),
1857                    _ => {}
1858                },
1859                Edge::Close(close) => {
1860                    if hidden > 0 && close.value().is_element() {
1861                        hidden -= 1;
1862                    }
1863                }
1864            }
1865        }
1866        text
1867    }
1868
1869    /// A span of `text` in `state`'s formatting.
1870    fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1871        ParsedSpan {
1872            text,
1873            bold: state.bold,
1874            italic: state.italic,
1875            underline: state.underline,
1876            strikeout: state.strikeout,
1877            code: state.code,
1878            superscript: state.superscript,
1879            subscript: state.subscript,
1880            link_href: state.link_href.clone(),
1881            image: None,
1882            footnote_ref: None,
1883            preserve_whitespace: state.preserve_whitespace,
1884        }
1885    }
1886
1887    /// Collect inline spans from a `<td>` or `<th>` cell element.
1888    fn collect_cell_spans(
1889        node: ego_tree::NodeRef<Node>,
1890        state: &FmtState,
1891        spans: &mut Vec<ParsedSpan>,
1892        depth: usize,
1893        limit: usize,
1894    ) {
1895        if depth > limit {
1896            let text = subtree_text(node);
1897            if !text.is_empty() {
1898                spans.push(text_span(text, state));
1899            }
1900            return;
1901        }
1902        for child in node.children() {
1903            match child.value() {
1904                Node::Text(text) => {
1905                    let t = text.text.to_string();
1906                    if !t.is_empty() {
1907                        spans.push(ParsedSpan {
1908                            text: t,
1909                            bold: state.bold,
1910                            italic: state.italic,
1911                            underline: state.underline,
1912                            strikeout: state.strikeout,
1913                            code: state.code,
1914                            superscript: state.superscript,
1915                            subscript: state.subscript,
1916                            link_href: state.link_href.clone(),
1917                            image: None,
1918                            footnote_ref: None,
1919                            preserve_whitespace: state.preserve_whitespace,
1920                        });
1921                    }
1922                }
1923                Node::Element(el) => {
1924                    let tag = el.name();
1925                    if is_metadata_tag(tag) {
1926                        continue;
1927                    }
1928                    let mut new_state = state.clone();
1929                    if let Some(keep) = el.attr("style").and_then(style_keeps_whitespace) {
1930                        new_state.preserve_whitespace = keep;
1931                    }
1932                    match tag {
1933                        // Checked by attribute rather than by tag name: the
1934                        // producer picks the element (a `<sup>` renders sensibly
1935                        // in a browser), and only the attribute is a contract.
1936                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1937                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1938                            {
1939                                spans.push(span);
1940                            }
1941                            continue;
1942                        }
1943                        "b" | "strong" => new_state.bold = true,
1944                        "i" | "em" => new_state.italic = true,
1945                        "u" | "ins" => new_state.underline = true,
1946                        "s" | "del" | "strike" => new_state.strikeout = true,
1947                        "code" => new_state.code = true,
1948                        "sup" => new_state.superscript = true,
1949                        "sub" => new_state.subscript = true,
1950                        "a" => {
1951                            if let Some(href) = el.attr("href") {
1952                                new_state.link_href = Some(href.to_string());
1953                            }
1954                        }
1955                        "img" => {
1956                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1957                                spans.push(span);
1958                            }
1959                            continue;
1960                        }
1961                        // A cell is read as one paragraph, so a line break, or a
1962                        // paragraph or list item of its own, is a space between
1963                        // words. Read as nothing, a cell of two paragraphs, which
1964                        // the HTML writer puts out as `One<br/>Two`, came back as
1965                        // the one word `OneTwo`.
1966                        "br" => {
1967                            spans.push(text_span(" ".to_string(), state));
1968                            continue;
1969                        }
1970                        "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1971                        | "h3" | "h4" | "h5" | "h6" => {
1972                            spans.push(text_span(" ".to_string(), state));
1973                        }
1974                        _ => {}
1975                    }
1976                    collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1977                }
1978                _ => {}
1979            }
1980        }
1981    }
1982
1983    /// The widest a `colspan` makes a row: HTML's own limit on the attribute. A row of
1984    /// ordinary cells may be wider; only the empty cells a span adds stop here.
1985    const MAX_HTML_COLSPAN: usize = 1000;
1986
1987    /// Parse a `<table>` element into a ParsedTable, with the number of `<td>` and `<th>`
1988    /// cells it has (see `push_table`). `depth` is how deep the walk stands at the table,
1989    /// and `limit` how deep it goes.
1990    fn parse_table_element(
1991        table_node: ego_tree::NodeRef<Node>,
1992        depth: usize,
1993        limit: usize,
1994    ) -> (ParsedTable, usize) {
1995        let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1996        let mut header_rows: usize = 0;
1997        let mut own_cells: usize = 0;
1998
1999        /// `walk` is how deep the walk stands at `node` and how deep it goes.
2000        fn collect_rows(
2001            node: ego_tree::NodeRef<Node>,
2002            rows: &mut Vec<Vec<ParsedTableCell>>,
2003            header_rows: &mut usize,
2004            own_cells: &mut usize,
2005            in_thead: bool,
2006            walk: (usize, usize),
2007        ) {
2008            let (depth, limit) = walk;
2009            for child in node.children() {
2010                if let Node::Element(el) = child.value() {
2011                    match el.name() {
2012                        "thead" => collect_rows(
2013                            child,
2014                            rows,
2015                            header_rows,
2016                            own_cells,
2017                            true,
2018                            (depth + 1, limit),
2019                        ),
2020                        "tbody" | "tfoot" => collect_rows(
2021                            child,
2022                            rows,
2023                            header_rows,
2024                            own_cells,
2025                            false,
2026                            (depth + 1, limit),
2027                        ),
2028                        "tr" => {
2029                            let mut cells: Vec<ParsedTableCell> = Vec::new();
2030                            // The cells of the row up to its last `<td>` or `<th>`: the
2031                            // empty cells a span adds after it hold no cell's place.
2032                            let mut own_end = 0;
2033                            for td in child.children() {
2034                                if let Node::Element(td_el) = td.value()
2035                                    && matches!(td_el.name(), "td" | "th")
2036                                {
2037                                    let mut spans = Vec::new();
2038                                    let state = FmtState::default();
2039                                    collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
2040                                    collapse_inline_whitespace(&mut spans);
2041                                    if spans.is_empty() {
2042                                        spans.push(ParsedSpan::default());
2043                                    }
2044                                    cells.push(ParsedTableCell { spans });
2045                                    *own_cells += 1;
2046                                    own_end = cells.len();
2047                                    // A cell spanning columns covers the cells after it
2048                                    // in its row: empty ones hold its place, so the cells
2049                                    // that follow stay in their own columns and the
2050                                    // table is as wide as the columns it spans.
2051                                    let span = td_el
2052                                        .attr("colspan")
2053                                        .and_then(|value| value.trim().parse::<usize>().ok())
2054                                        .unwrap_or(1)
2055                                        .clamp(1, MAX_HTML_COLSPAN);
2056                                    for _ in 1..span {
2057                                        if cells.len() >= MAX_HTML_COLSPAN {
2058                                            break;
2059                                        }
2060                                        cells.push(ParsedTableCell {
2061                                            spans: vec![ParsedSpan::default()],
2062                                        });
2063                                    }
2064                                }
2065                            }
2066                            // A span closing a row widens the table only as far as another
2067                            // row's cells reach: `<th colspan="1000">` over rows of one
2068                            // cell is a table of one column, not of a thousand.
2069                            cells.truncate(own_end);
2070                            if !cells.is_empty() {
2071                                rows.push(cells);
2072                                if in_thead {
2073                                    *header_rows += 1;
2074                                }
2075                            }
2076                        }
2077                        _ => {}
2078                    }
2079                }
2080            }
2081        }
2082
2083        collect_rows(
2084            table_node,
2085            &mut rows,
2086            &mut header_rows,
2087            &mut own_cells,
2088            false,
2089            (depth, limit),
2090        );
2091
2092        // Tables without explicit <thead> but with <th> cells: treat first row as header
2093        if header_rows == 0 && !rows.is_empty() {
2094            header_rows = 1;
2095        }
2096
2097        (
2098            ParsedTable {
2099                header_rows,
2100                rows,
2101                // The caller (`walk_node`) sets the real depth: this helper has
2102                // no visibility into the surrounding blockquote nesting.
2103                blockquote_depth: 0,
2104                blockquote_opens: 0,
2105            },
2106            own_cells,
2107        )
2108    }
2109
2110    #[allow(clippy::too_many_arguments)]
2111    fn walk_node(
2112        node: ego_tree::NodeRef<Node>,
2113        state: &FmtState,
2114        elements: &mut Vec<ParsedElement>,
2115        current_list: &HtmlList,
2116        blockquote_depth: u32,
2117        list_depth: u32,
2118        depth: usize,
2119        limit: usize,
2120    ) {
2121        if depth > limit {
2122            let text = subtree_text(node);
2123            if !text.trim().is_empty() {
2124                let mut spans = vec![text_span(text, state)];
2125                collapse_inline_whitespace(&mut spans);
2126                elements.push(ParsedElement::Block(ParsedBlock {
2127                    spans,
2128                    blockquote_depth,
2129                    blockquote_opens: 0,
2130                    ..ParsedBlock::default()
2131                }));
2132            }
2133            return;
2134        }
2135        match node.value() {
2136            Node::Element(el) => {
2137                let tag = el.name();
2138                if is_metadata_tag(tag) {
2139                    return;
2140                }
2141                let mut new_state = state.clone();
2142                if let Some(keep) = el.attr("style").and_then(style_keeps_whitespace) {
2143                    new_state.preserve_whitespace = keep;
2144                }
2145                let mut new_list = current_list.clone();
2146                let mut bq_depth = blockquote_depth;
2147                let mut new_list_depth = list_depth;
2148
2149                // Determine if this is a block-level element
2150                let is_block_tag = matches!(
2151                    tag,
2152                    "p" | "div"
2153                        | "h1"
2154                        | "h2"
2155                        | "h3"
2156                        | "h4"
2157                        | "h5"
2158                        | "h6"
2159                        | "li"
2160                        | "pre"
2161                        | "br"
2162                        | "blockquote"
2163                        | "body"
2164                        | "html"
2165                );
2166
2167                // Update formatting state
2168                match tag {
2169                    "b" | "strong" => new_state.bold = true,
2170                    "i" | "em" => new_state.italic = true,
2171                    "u" | "ins" => new_state.underline = true,
2172                    "s" | "del" | "strike" => new_state.strikeout = true,
2173                    "code" => new_state.code = true,
2174                    "sup" => new_state.superscript = true,
2175                    "sub" => new_state.subscript = true,
2176                    "a" => {
2177                        if let Some(href) = el.attr("href") {
2178                            new_state.link_href = Some(href.to_string());
2179                        }
2180                    }
2181                    "ul" => {
2182                        new_list = (Some(ListStyle::Disc), None);
2183                        new_list_depth = list_depth + 1;
2184                    }
2185                    "ol" => {
2186                        // A list numbered from below 0 is kept from 1: no saved syntax
2187                        // but HTML writes one. A start past what every format holds is
2188                        // read as the largest it holds (see `list_start`).
2189                        let start = el
2190                            .attr("start")
2191                            .and_then(html_integer)
2192                            .and_then(crate::parser_tools::list_start::list_start);
2193                        new_list = (Some(ListStyle::Decimal), start);
2194                        new_list_depth = list_depth + 1;
2195                    }
2196                    "blockquote" => {
2197                        bq_depth += 1;
2198                    }
2199                    _ => {}
2200                }
2201
2202                // Determine heading level
2203                let heading_level = match tag {
2204                    "h1" => Some(1),
2205                    "h2" => Some(2),
2206                    "h3" => Some(3),
2207                    "h4" => Some(4),
2208                    "h5" => Some(5),
2209                    "h6" => Some(6),
2210                    _ => None,
2211                };
2212
2213                let is_code_block = tag == "pre";
2214
2215                // Extract code language from <pre><code class="language-xxx">
2216                let code_language = if is_code_block {
2217                    node.children().find_map(|child| {
2218                        if let Node::Element(cel) = child.value()
2219                            && cel.name() == "code"
2220                            && let Some(cls) = cel.attr("class")
2221                        {
2222                            return cls
2223                                .split_whitespace()
2224                                .find_map(|c| c.strip_prefix("language-"))
2225                                .map(|l| l.to_string());
2226                        }
2227                        None
2228                    })
2229                } else {
2230                    None
2231                };
2232
2233                // Extract CSS styles from block-level elements
2234                let css = if is_block_tag {
2235                    el.attr("style").map(parse_block_styles).unwrap_or_default()
2236                } else {
2237                    BlockStyles::default()
2238                };
2239
2240                if tag == "table" {
2241                    // Parse table structure into a ParsedTable
2242                    let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2243                    if !parsed_table.rows.is_empty() {
2244                        parsed_table.blockquote_depth = bq_depth;
2245                        push_table(elements, parsed_table, own_cells);
2246                    }
2247                    return;
2248                }
2249
2250                if tag == "br" {
2251                    // <br> creates a new block
2252                    elements.push(ParsedElement::Block(ParsedBlock {
2253                        spans: vec![ParsedSpan {
2254                            text: String::new(),
2255                            ..Default::default()
2256                        }],
2257                        heading_level: None,
2258                        list_style: None,
2259                        list_indent: 0,
2260                        list_prefix: String::new(),
2261                        list_suffix: String::new(),
2262                        list_start: None,
2263                        marker: None,
2264                        is_code_block: false,
2265                        code_language: None,
2266                        blockquote_depth: bq_depth,
2267                        blockquote_opens: 0,
2268                        line_height: None,
2269                        non_breakable_lines: None,
2270                        page_break_before: None,
2271                        front_matter: None,
2272                        direction: None,
2273                        background_color: None,
2274                        alignment: None,
2275                        top_margin: None,
2276                        text_indent: None,
2277                        semantic_role: None,
2278                    }));
2279                    return;
2280                }
2281
2282                if tag == "blockquote" {
2283                    let first = elements.len();
2284                    // A quotation holding inline content alone is one paragraph of it, as
2285                    // a `<p>` is, its `<br>`s line breaks: verse is pasted from the web as
2286                    // `<blockquote>a line<br>the next</blockquote>`. Walked child by child,
2287                    // each line was a paragraph of its own and each `<br>` an empty one.
2288                    let inline_only = node.children().all(|child| match child.value() {
2289                        Node::Element(el) => !matches!(
2290                            el.name(),
2291                            "p" | "div"
2292                                | "h1"
2293                                | "h2"
2294                                | "h3"
2295                                | "h4"
2296                                | "h5"
2297                                | "h6"
2298                                | "li"
2299                                | "pre"
2300                                | "blockquote"
2301                                | "ul"
2302                                | "ol"
2303                                | "table"
2304                        ),
2305                        _ => true,
2306                    });
2307                    if inline_only {
2308                        let mut spans: Vec<ParsedSpan> = Vec::new();
2309                        let mut nested_elements: Vec<ParsedElement> = Vec::new();
2310                        collect_inline_spans(
2311                            node,
2312                            &new_state,
2313                            &mut spans,
2314                            &new_list,
2315                            &mut nested_elements,
2316                            bq_depth,
2317                            new_list_depth,
2318                            depth + 1,
2319                            limit,
2320                        );
2321                        collapse_inline_whitespace(&mut spans);
2322                        if spans_carry_content(&spans) {
2323                            elements.push(ParsedElement::Block(ParsedBlock {
2324                                spans,
2325                                blockquote_depth: bq_depth,
2326                                blockquote_opens: 0,
2327                                ..ParsedBlock::default()
2328                            }));
2329                        }
2330                        elements.append(&mut nested_elements);
2331                    } else {
2332                        // A container: recurse into children with increased depth.
2333                        for child in node.children() {
2334                            walk_node(
2335                                child,
2336                                &new_state,
2337                                elements,
2338                                &new_list,
2339                                bq_depth,
2340                                new_list_depth,
2341                                depth + 1,
2342                                limit,
2343                            );
2344                        }
2345                    }
2346                    mark_quotation_opened(elements, first);
2347                } else if is_block_tag && tag != "br" {
2348                    // Start collecting spans for a new block.
2349                    // Use a temporary buffer so that nested block-level
2350                    // elements (e.g. sub-lists inside <li>) are collected
2351                    // separately and appended *after* the parent block.
2352                    let mut spans: Vec<ParsedSpan> = Vec::new();
2353                    let mut nested_elements: Vec<ParsedElement> = Vec::new();
2354                    collect_inline_spans(
2355                        node,
2356                        &new_state,
2357                        &mut spans,
2358                        &new_list,
2359                        &mut nested_elements,
2360                        bq_depth,
2361                        new_list_depth,
2362                        depth + 1,
2363                        limit,
2364                    );
2365
2366                    let (list_style_for_block, list_start_for_block) = if tag == "li" {
2367                        new_list.clone()
2368                    } else {
2369                        (None, None)
2370                    };
2371
2372                    let list_indent_for_block = if tag == "li" {
2373                        new_list_depth.saturating_sub(1)
2374                    } else {
2375                        0
2376                    };
2377
2378                    if !(is_code_block || css.preserve_whitespace == Some(true)) {
2379                        collapse_inline_whitespace(&mut spans);
2380                    }
2381
2382                    // Whitespace between block-level children is layout, not
2383                    // content. `<body>` holds a newline after every `</p>` an
2384                    // exporter writes, and keeping those as the container's own
2385                    // inline run is what closed a pasted book with a paragraph
2386                    // of four thousand blank lines. A block that produced no
2387                    // children keeps its empty run instead: `<p> </p>` is an
2388                    // empty paragraph the writer meant to be there.
2389                    let own_run_is_content =
2390                        spans_carry_content(&spans) || nested_elements.is_empty();
2391
2392                    if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2393                        // A code block is verbatim text, which no save can write an image or
2394                        // a note's reference into: a `<pre>` holding one is read as prose,
2395                        // which keeps it, one paragraph for each of its lines (see below).
2396                        let is_code_block = is_code_block
2397                            && !spans
2398                                .iter()
2399                                .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2400                        let block = ParsedBlock {
2401                            spans,
2402                            heading_level,
2403                            list_style: list_style_for_block,
2404                            list_indent: list_indent_for_block,
2405                            list_prefix: String::new(),
2406                            list_suffix: String::new(),
2407                            list_start: list_start_for_block,
2408                            marker: None,
2409                            is_code_block,
2410                            code_language: code_language.filter(|_| is_code_block),
2411                            blockquote_depth: bq_depth,
2412                            blockquote_opens: 0,
2413                            line_height: css.line_height,
2414                            non_breakable_lines: css.non_breakable_lines,
2415                            page_break_before: css.page_break_before,
2416                            front_matter: None,
2417                            direction: css.direction,
2418                            background_color: css.background_color,
2419                            alignment: None,
2420                            top_margin: None,
2421                            text_indent: None,
2422                            semantic_role: None,
2423                        };
2424                        // Kept whitespace keeps its line breaks, and a line break in
2425                        // prose is a new block to the model. Only a code block holds
2426                        // its lines in one block.
2427                        if is_code_block {
2428                            elements.push(ParsedElement::Block(block));
2429                        } else {
2430                            elements.extend(
2431                                split_block_at_line_breaks(block)
2432                                    .into_iter()
2433                                    .map(ParsedElement::Block),
2434                            );
2435                        }
2436                    }
2437                    // Append nested block elements after the parent block
2438                    elements.append(&mut nested_elements);
2439                } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2440                    // Container elements: recurse into children
2441                    for child in node.children() {
2442                        walk_node(
2443                            child,
2444                            &new_state,
2445                            elements,
2446                            &new_list,
2447                            bq_depth,
2448                            new_list_depth,
2449                            depth + 1,
2450                            limit,
2451                        );
2452                    }
2453                } else {
2454                    // Inline element or unknown: recurse
2455                    for child in node.children() {
2456                        walk_node(
2457                            child,
2458                            &new_state,
2459                            elements,
2460                            current_list,
2461                            bq_depth,
2462                            list_depth,
2463                            depth + 1,
2464                            limit,
2465                        );
2466                    }
2467                }
2468            }
2469            Node::Text(text) => {
2470                // Bare text not in a block (a list's text outside any item, `<ol>a</ol>`)
2471                // is a paragraph, its whitespace collapsed as a paragraph's is. Only
2472                // trimmed, a tab in it reached the model, and a newline, which separates
2473                // two blocks there, and the HTML written from it read back otherwise.
2474                let mut spans = vec![text_span(text.text.to_string(), state)];
2475                collapse_inline_whitespace(&mut spans);
2476                if spans_carry_content(&spans) {
2477                    let block = ParsedBlock {
2478                        spans,
2479                        blockquote_depth,
2480                        blockquote_opens: 0,
2481                        ..ParsedBlock::default()
2482                    };
2483                    elements.extend(
2484                        split_block_at_line_breaks(block)
2485                            .into_iter()
2486                            .map(ParsedElement::Block),
2487                    );
2488                }
2489            }
2490            _ => {
2491                // Document, Comment, etc. — recurse children
2492                for child in node.children() {
2493                    walk_node(
2494                        child,
2495                        state,
2496                        elements,
2497                        current_list,
2498                        blockquote_depth,
2499                        list_depth,
2500                        depth + 1,
2501                        limit,
2502                    );
2503                }
2504            }
2505        }
2506    }
2507
2508    /// Collect inline spans from a block-level element's children.
2509    /// If a nested block-level element is encountered, it is flushed as a
2510    /// separate block.
2511    #[allow(clippy::too_many_arguments)]
2512    fn collect_inline_spans(
2513        node: ego_tree::NodeRef<Node>,
2514        state: &FmtState,
2515        spans: &mut Vec<ParsedSpan>,
2516        current_list: &HtmlList,
2517        elements: &mut Vec<ParsedElement>,
2518        blockquote_depth: u32,
2519        list_depth: u32,
2520        depth: usize,
2521        limit: usize,
2522    ) {
2523        if depth > limit {
2524            let text = subtree_text(node);
2525            if !text.is_empty() {
2526                spans.push(text_span(text, state));
2527            }
2528            return;
2529        }
2530        for child in node.children() {
2531            match child.value() {
2532                Node::Text(text) => {
2533                    let t = text.text.to_string();
2534                    if !t.is_empty() {
2535                        spans.push(ParsedSpan {
2536                            text: t,
2537                            bold: state.bold,
2538                            italic: state.italic,
2539                            underline: state.underline,
2540                            strikeout: state.strikeout,
2541                            code: state.code,
2542                            superscript: state.superscript,
2543                            subscript: state.subscript,
2544                            link_href: state.link_href.clone(),
2545                            image: None,
2546                            footnote_ref: None,
2547                            preserve_whitespace: state.preserve_whitespace,
2548                        });
2549                    }
2550                }
2551                Node::Element(el) => {
2552                    let tag = el.name();
2553                    if is_metadata_tag(tag) {
2554                        continue;
2555                    }
2556                    let mut new_state = state.clone();
2557                    if let Some(keep) = el.attr("style").and_then(style_keeps_whitespace) {
2558                        new_state.preserve_whitespace = keep;
2559                    }
2560
2561                    match tag {
2562                        // Checked by attribute rather than by tag name: the
2563                        // producer picks the element (a `<sup>` renders sensibly
2564                        // in a browser), and only the attribute is a contract.
2565                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2566                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2567                            {
2568                                spans.push(span);
2569                            }
2570                            continue;
2571                        }
2572                        "b" | "strong" => new_state.bold = true,
2573                        "i" | "em" => new_state.italic = true,
2574                        "u" | "ins" => new_state.underline = true,
2575                        "s" | "del" | "strike" => new_state.strikeout = true,
2576                        "code" => new_state.code = true,
2577                        "sup" => new_state.superscript = true,
2578                        "sub" => new_state.subscript = true,
2579                        "a" => {
2580                            if let Some(href) = el.attr("href") {
2581                                new_state.link_href = Some(href.to_string());
2582                            }
2583                        }
2584                        "img" => {
2585                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2586                                spans.push(span);
2587                            }
2588                            continue;
2589                        }
2590                        _ => {}
2591                    }
2592
2593                    // Check for nested block elements
2594                    let nested_block = matches!(
2595                        tag,
2596                        "p" | "div"
2597                            | "h1"
2598                            | "h2"
2599                            | "h3"
2600                            | "h4"
2601                            | "h5"
2602                            | "h6"
2603                            | "li"
2604                            | "pre"
2605                            | "blockquote"
2606                            | "ul"
2607                            | "ol"
2608                    );
2609
2610                    if tag == "br" {
2611                        // A line break inside the block, in the formatting around it.
2612                        // `collapse_inline_whitespace` then drops the spaces either side,
2613                        // as a browser lays them out.
2614                        spans.push(text_span(LINE_SEPARATOR_STR.to_string(), state));
2615                    } else if nested_block || tag == "table" {
2616                        // Flush as separate element
2617                        walk_node(
2618                            child,
2619                            &new_state,
2620                            elements,
2621                            current_list,
2622                            blockquote_depth,
2623                            list_depth,
2624                            depth + 1,
2625                            limit,
2626                        );
2627                    } else {
2628                        // Inline element: recurse
2629                        collect_inline_spans(
2630                            child,
2631                            &new_state,
2632                            spans,
2633                            current_list,
2634                            elements,
2635                            blockquote_depth,
2636                            list_depth,
2637                            depth + 1,
2638                            limit,
2639                        );
2640                    }
2641                }
2642                _ => {}
2643            }
2644        }
2645    }
2646
2647    let initial_state = FmtState::default();
2648    // Treat the root element as a block-level container so that
2649    // top-level inline elements (e.g. `<b>Bold</b> <em>Italic</em>`)
2650    // are grouped into a single block instead of becoming separate blocks.
2651    let mut root_spans: Vec<ParsedSpan> = Vec::new();
2652    collect_inline_spans(
2653        *root,
2654        &initial_state,
2655        &mut root_spans,
2656        &(None, None),
2657        &mut elements,
2658        0,
2659        0,
2660        0,
2661        depth_limit,
2662    );
2663    collapse_inline_whitespace(&mut root_spans);
2664    // Only when something survived: the root's direct text children are the
2665    // newlines between top-level blocks in every document an exporter writes.
2666    if spans_carry_content(&root_spans) {
2667        elements.push(ParsedElement::Block(ParsedBlock {
2668            spans: root_spans,
2669            heading_level: None,
2670            list_style: None,
2671            list_indent: 0,
2672            list_prefix: String::new(),
2673            list_suffix: String::new(),
2674            list_start: None,
2675            marker: None,
2676            is_code_block: false,
2677            code_language: None,
2678            blockquote_depth: 0,
2679            blockquote_opens: 0,
2680            line_height: None,
2681            non_breakable_lines: None,
2682            page_break_before: None,
2683            front_matter: None,
2684            direction: None,
2685            background_color: None,
2686            alignment: None,
2687            top_margin: None,
2688            text_indent: None,
2689            semantic_role: None,
2690        }));
2691    }
2692
2693    // If no elements were parsed, create a single empty paragraph
2694    if elements.is_empty() {
2695        elements.push(ParsedElement::Block(ParsedBlock {
2696            spans: vec![ParsedSpan {
2697                text: String::new(),
2698                ..Default::default()
2699            }],
2700            heading_level: None,
2701            list_style: None,
2702            list_indent: 0,
2703            list_prefix: String::new(),
2704            list_suffix: String::new(),
2705            list_start: None,
2706            marker: None,
2707            is_code_block: false,
2708            code_language: None,
2709            blockquote_depth: 0,
2710            blockquote_opens: 0,
2711            line_height: None,
2712            non_breakable_lines: None,
2713            page_break_before: None,
2714            front_matter: None,
2715            direction: None,
2716            background_color: None,
2717            alignment: None,
2718            top_margin: None,
2719            text_indent: None,
2720            semantic_role: None,
2721        }));
2722    }
2723
2724    elements
2725}
2726
2727/// Convert a `ParsedSpan` (parser output) into the `CharacterFormat` used by
2728/// `FormatRun`. `is_code_block` forces `monospace` as the font family for
2729/// every span inside a code block.
2730pub fn character_format_from_span(
2731    span: &ParsedSpan,
2732    is_code_block: bool,
2733) -> crate::format_runs::CharacterFormat {
2734    use crate::entities::CharVerticalAlignment;
2735    crate::format_runs::CharacterFormat {
2736        font_bold: if span.bold { Some(true) } else { None },
2737        font_italic: if span.italic { Some(true) } else { None },
2738        font_underline: if span.underline { Some(true) } else { None },
2739        font_strikeout: if span.strikeout { Some(true) } else { None },
2740        font_family: if span.code || is_code_block {
2741            Some("monospace".to_string())
2742        } else {
2743            None
2744        },
2745        anchor_href: span.link_href.clone(),
2746        is_anchor: if span.link_href.is_some() {
2747            Some(true)
2748        } else {
2749            None
2750        },
2751        vertical_alignment: if span.superscript {
2752            Some(CharVerticalAlignment::SuperScript)
2753        } else if span.subscript {
2754            Some(CharVerticalAlignment::SubScript)
2755        } else {
2756            None
2757        },
2758        ..Default::default()
2759    }
2760}
2761
2762/// Translate a slice of parsed spans into `(plain_text, format_runs)`.
2763///
2764/// One non-default span yields one `FormatRun`; spans with empty
2765/// `CharacterFormat` (no decoration, no link, no code) emit no run, since an
2766/// absent run means "inherit default formatting" in the new model. Adjacent
2767/// runs with identical formats are coalesced via `coalesce_in_place` so the
2768/// resulting vector satisfies `debug_assert_well_formed`.
2769///
2770/// Returns the concatenated `plain_text` of all spans and a sorted,
2771/// non-overlapping, coalesced `Vec<FormatRun>`. Both safe to feed straight
2772/// into the store under the dual-write bridge.
2773pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2774    use crate::format_runs::{
2775        CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2776    };
2777
2778    let mut plain_text = String::new();
2779    let mut runs: Vec<FormatRun> = Vec::new();
2780    let mut images: Vec<ImageAnchor> = Vec::new();
2781    let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2782    let default = CharacterFormat::default();
2783
2784    for span in spans {
2785        let byte_start = plain_text.len() as u32;
2786
2787        if let Some(label) = &span.footnote_ref {
2788            // A reference occupies one U+FFFC, exactly as an image does, so
2789            // every downstream offset treats the two alike.
2790            plain_text.push('\u{FFFC}');
2791            // Raised, always — whatever the surrounding run is doing.
2792            //
2793            // A footnote marker is superscript by definition, in every
2794            // typographic tradition and in every reader that renders djot. The
2795            // ambient `superscript` flag the span carries is the *prose's*, and
2796            // prose is not superscript, so taking it verbatim sets a note's
2797            // number on the baseline in the middle of a sentence — which reads
2798            // as a stray digit the writer typed rather than as a reference.
2799            //
2800            // It is set here, on the anchor, rather than at render time so that
2801            // every consumer agrees: the editor raises it, the exporters that
2802            // carry character formatting carry it, and the djot writer knows to
2803            // emit `[^label]` *without* wrapping it in `^…^` (see
2804            // `a_reference_is_not_wrapped_in_superscript_markers`).
2805            let mut format = character_format_from_span(span, is_code_block);
2806            format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2807            footnote_refs.push(FootnoteRefAnchor {
2808                byte_offset: byte_start,
2809                label: label.clone(),
2810                format,
2811            });
2812            continue;
2813        }
2814
2815        if let Some(image) = &span.image {
2816            // An image occupies one U+FFFC in the text, exactly as
2817            // `insert_image` mirrors into the rope, so every downstream offset
2818            // calculation treats a parsed image and an inserted one alike.
2819            plain_text.push('\u{FFFC}');
2820            images.push(ImageAnchor {
2821                byte_offset: byte_start,
2822                name: image.src.clone(),
2823                alt: image.alt.clone(),
2824                width: image.width,
2825                height: image.height,
2826                quality: 100,
2827                format: character_format_from_span(span, is_code_block),
2828            });
2829            continue;
2830        }
2831
2832        plain_text.push_str(&span.text);
2833        let byte_end = plain_text.len() as u32;
2834        if byte_start == byte_end {
2835            continue;
2836        }
2837        let format = character_format_from_span(span, is_code_block);
2838        if format == default {
2839            continue;
2840        }
2841        runs.push(FormatRun {
2842            byte_start,
2843            byte_end,
2844            format,
2845        });
2846    }
2847    coalesce_in_place(&mut runs);
2848    ParsedInline {
2849        plain_text,
2850        runs,
2851        images,
2852        footnote_refs,
2853    }
2854}
2855
2856/// The three parallel things a block stores, as recovered from parsed spans.
2857///
2858/// Returned as a struct rather than a tuple because it grew a third member
2859/// (images) after nine call sites already destructured a pair — and every one
2860/// of those sites has to decide what to do with images, so a silent
2861/// tuple-arity change would have been the wrong kind of easy.
2862#[derive(Debug, Clone, Default)]
2863pub struct ParsedInline {
2864    pub plain_text: String,
2865    pub runs: Vec<crate::format_runs::FormatRun>,
2866    pub images: Vec<crate::format_runs::ImageAnchor>,
2867    pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2868}
2869
2870// ─── Djot parsing ────────────────────────────────────────────────────
2871
2872/// Map a jotdown unordered/task bullet marker to a model `ListStyle`.
2873///
2874/// The mapping is a stable bijection (`-`↔Disc, `*`↔Circle, `+`↔Square) so the
2875/// djot exporter can recover the exact bullet character for a lossless
2876/// round-trip.
2877fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2878    use jotdown::ListBulletType as B;
2879    match b {
2880        B::Dash => ListStyle::Disc,
2881        B::Star => ListStyle::Circle,
2882        B::Plus => ListStyle::Square,
2883    }
2884}
2885
2886/// Map a jotdown ordered-list numbering scheme to a model `ListStyle`.
2887fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2888    use jotdown::OrderedListNumbering as N;
2889    match n {
2890        N::Decimal => ListStyle::Decimal,
2891        N::AlphaLower => ListStyle::LowerAlpha,
2892        N::AlphaUpper => ListStyle::UpperAlpha,
2893        N::RomanLower => ListStyle::LowerRoman,
2894        N::RomanUpper => ListStyle::UpperRoman,
2895    }
2896}
2897
2898/// Map a jotdown ordered-list delimiter to the `(prefix, suffix)` affixes
2899/// stored on the `List` entity (`1.` → `("", ".")`, `1)` → `("", ")")`,
2900/// `(1)` → `("(", ")")`).
2901fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2902    use jotdown::OrderedListStyle as S;
2903    match style {
2904        S::Period => (String::new(), ".".to_string()),
2905        S::Paren => (String::new(), ")".to_string()),
2906        S::ParenParen => ("(".to_string(), ")".to_string()),
2907    }
2908}
2909
2910/// Optional block-level style attributes carried on a djot block through its
2911/// `{key=value}` block attributes. All `None` when the block has no such
2912/// attributes (or they were filtered out by [`DjotImportOptions`]).
2913#[derive(Debug, Clone, Default)]
2914struct DjotBlockStyle {
2915    alignment: Option<Alignment>,
2916    line_height: Option<i64>,
2917    non_breakable_lines: Option<bool>,
2918    page_break_before: Option<bool>,
2919    front_matter: Option<bool>,
2920    direction: Option<TextDirection>,
2921    background_color: Option<String>,
2922    top_margin: Option<i64>,
2923    text_indent: Option<i64>,
2924    semantic_role: Option<SemanticRole>,
2925}
2926
2927impl DjotBlockStyle {
2928    /// Overlay the `Some` fields of `other` onto `self`, leaving `self`'s
2929    /// existing values for any field `other` does not set. Used to combine a
2930    /// heading's enclosing-`Section` attributes with any on the heading itself.
2931    fn merge_from(&mut self, other: DjotBlockStyle) {
2932        if other.alignment.is_some() {
2933            self.alignment = other.alignment;
2934        }
2935        if other.line_height.is_some() {
2936            self.line_height = other.line_height;
2937        }
2938        if other.non_breakable_lines.is_some() {
2939            self.non_breakable_lines = other.non_breakable_lines;
2940        }
2941        if other.page_break_before.is_some() {
2942            self.page_break_before = other.page_break_before;
2943        }
2944        if other.front_matter.is_some() {
2945            self.front_matter = other.front_matter;
2946        }
2947        if other.direction.is_some() {
2948            self.direction = other.direction;
2949        }
2950        if other.background_color.is_some() {
2951            self.background_color = other.background_color;
2952        }
2953        if other.top_margin.is_some() {
2954            self.top_margin = other.top_margin;
2955        }
2956        if other.text_indent.is_some() {
2957            self.text_indent = other.text_indent;
2958        }
2959        if other.semantic_role.is_some() {
2960            self.semantic_role = other.semantic_role.clone();
2961        }
2962    }
2963}
2964
2965/// Read the round-tripped block-style attributes off a djot block's
2966/// [`jotdown::Attributes`], honouring the import [`DjotImportOptions`]. Keys are
2967/// the model field names (`alignment`, `line_height`, `direction`,
2968/// `non_breakable_lines`, `page_break_before`, `front_matter`, `background_color`,
2969/// `top_margin`, `text_indent`, `semantic_role`); unrecognised values are ignored.
2970fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2971    let mut style = DjotBlockStyle::default();
2972
2973    if opts.alignment
2974        && let Some(v) = attrs.get_value("alignment")
2975    {
2976        style.alignment = match v.to_string().as_str() {
2977            "left" => Some(Alignment::Left),
2978            "right" => Some(Alignment::Right),
2979            "center" => Some(Alignment::Center),
2980            "justify" => Some(Alignment::Justify),
2981            _ => None,
2982        };
2983    }
2984    if opts.line_height
2985        && let Some(v) = attrs.get_value("line_height")
2986    {
2987        style.line_height = v.to_string().parse::<i64>().ok();
2988    }
2989    if opts.direction
2990        && let Some(v) = attrs.get_value("direction")
2991    {
2992        style.direction = match v.to_string().as_str() {
2993            "ltr" => Some(TextDirection::LeftToRight),
2994            "rtl" => Some(TextDirection::RightToLeft),
2995            _ => None,
2996        };
2997    }
2998    if opts.non_breakable_lines
2999        && let Some(v) = attrs.get_value("non_breakable_lines")
3000    {
3001        style.non_breakable_lines = match v.to_string().as_str() {
3002            "true" => Some(true),
3003            "false" => Some(false),
3004            _ => None,
3005        };
3006    }
3007    if opts.page_break_before
3008        && let Some(v) = attrs.get_value("page_break_before")
3009    {
3010        style.page_break_before = match v.to_string().as_str() {
3011            "true" => Some(true),
3012            "false" => Some(false),
3013            _ => None,
3014        };
3015    }
3016    if opts.front_matter
3017        && let Some(v) = attrs.get_value("front_matter")
3018    {
3019        style.front_matter = match v.to_string().as_str() {
3020            "true" => Some(true),
3021            "false" => Some(false),
3022            _ => None,
3023        };
3024    }
3025    if opts.background_color
3026        && let Some(v) = attrs.get_value("background_color")
3027    {
3028        style.background_color = Some(v.to_string());
3029    }
3030    if opts.top_margin
3031        && let Some(v) = attrs.get_value("top_margin")
3032    {
3033        style.top_margin = v.to_string().parse::<i64>().ok();
3034    }
3035    if opts.text_indent
3036        && let Some(v) = attrs.get_value("text_indent")
3037    {
3038        style.text_indent = v.to_string().parse::<i64>().ok();
3039    }
3040    if opts.semantic_role
3041        && let Some(v) = attrs.get_value("semantic_role")
3042    {
3043        style.semantic_role = match v.to_string().as_str() {
3044            "epigraph" => Some(SemanticRole::Epigraph),
3045            // An unknown role is dropped, not guessed at — the same way an unknown
3046            // alignment value above is. A future role read by an older build then
3047            // degrades to a plain blockquote, which is what it looks like anyway.
3048            _ => None,
3049        };
3050    }
3051
3052    style
3053}
3054
3055/// The start a list keeps on its entity, from the number its first item is written with:
3056/// `None` for 1, which every list starts at unless it says otherwise, and a number past what
3057/// every format holds read as the largest it holds (see `list_start`).
3058fn list_start_of(start: u64) -> Option<i64> {
3059    crate::parser_tools::list_start::list_start_from_unsigned(start)
3060}
3061
3062/// Push a finished block into `elements`, applying the djot block-level fields
3063/// plus any round-tripped block-style attributes carried in `style`.
3064#[allow(clippy::too_many_arguments)]
3065fn djot_push_block(
3066    elements: &mut Vec<ParsedElement>,
3067    spans: Vec<ParsedSpan>,
3068    heading_level: Option<i64>,
3069    list_style: Option<ListStyle>,
3070    list_indent: u32,
3071    list_prefix: String,
3072    list_suffix: String,
3073    list_start: Option<i64>,
3074    marker: Option<MarkerType>,
3075    is_code_block: bool,
3076    code_language: Option<String>,
3077    blockquote_depth: u32,
3078    style: DjotBlockStyle,
3079) {
3080    elements.push(ParsedElement::Block(ParsedBlock {
3081        spans,
3082        heading_level,
3083        list_style,
3084        list_indent,
3085        list_prefix,
3086        list_suffix,
3087        list_start,
3088        marker,
3089        is_code_block,
3090        code_language,
3091        blockquote_depth,
3092        blockquote_opens: 0,
3093        line_height: style.line_height,
3094        non_breakable_lines: style.non_breakable_lines,
3095        page_break_before: style.page_break_before,
3096        front_matter: style.front_matter,
3097        direction: style.direction,
3098        background_color: style.background_color,
3099        alignment: style.alignment,
3100        top_margin: style.top_margin,
3101        text_indent: style.text_indent,
3102        semantic_role: style.semantic_role.clone(),
3103    }));
3104}
3105
3106/// A Djot document whose structure cannot be computed, as its source: one plain
3107/// paragraph for each line that holds text, with that line's text as it is written, its
3108/// leading and trailing whitespace aside.
3109///
3110/// A paragraph a line is what keeps the text the same through a save. The Djot writer
3111/// escapes whatever a paragraph opens with and whatever in it would read as markup, so
3112/// the saved text reads back as these same paragraphs. Shown as one paragraph, the
3113/// source kept its line breaks through a save, read back as the same document the parser
3114/// could not take, and every save escaped its backslashes once more. Blank lines are left
3115/// out because the reader keeps no block for an empty paragraph, and a line's outer
3116/// whitespace because it is the source's indentation, which the parser does not read as
3117/// text in a line it can take either.
3118fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
3119    djot.lines()
3120        .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
3121        .filter(|line| !line.is_empty())
3122        .map(|line| {
3123            ParsedElement::Block(ParsedBlock {
3124                spans: vec![ParsedSpan {
3125                    text: line.to_string(),
3126                    ..Default::default()
3127                }],
3128                ..Default::default()
3129            })
3130        })
3131        .collect()
3132}
3133
3134/// Parse djot source into the shared [`ParsedElement`] intermediate, mirroring
3135/// [`parse_markdown`]. Uses the [`jotdown`] pull parser.
3136///
3137/// Constructs the document model cannot represent are dropped, and their text
3138/// content is discarded so it never leaks into the document: footnotes, math,
3139/// fenced divs, raw blocks/inline, thematic breaks, description lists,
3140/// captions, symbols, link-reference definitions, and highlight/`mark`. Inline
3141/// images keep their alt text as plain text (the image itself is not modelled),
3142/// matching the Markdown importer. Smart-punctuation events are normalised to
3143/// their canonical Unicode characters so the model→djot→model round-trip is a
3144/// fixpoint.
3145///
3146/// Standalone paragraphs and headings additionally carry the optional
3147/// block-style attributes selected by `options` — paragraph alignment, line
3148/// height, text direction, non-breakable lines and background color — read from
3149/// djot `{key=value}` block attributes (see [`DjotImportOptions`]). List items,
3150/// code blocks and table cells normalise their block styling away.
3151///
3152/// Known model limitations (normalised, not preserved on round-trip):
3153/// table column alignment, and list tight/loose.
3154pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
3155    use jotdown::{Container as C, Event as E, ListKind};
3156
3157    // `jotdown` recurses with no limit of its own, once per nested container and
3158    // once per line of a paragraph that leaves an inline opener waiting, and a
3159    // stack overflow **aborts the process**: it cannot be caught by
3160    // `catch_unwind`, so there is no recovering from it after the fact and the
3161    // only safe move is not to start. It also panics on a heading deeper than it
3162    // can number, which a caller on an interface thread does not survive either.
3163    // Input is not always the author's own: a project bundle is mailed and
3164    // shared, and an imported document comes from whoever sent it.
3165    //
3166    // `djot_depth::parsable` is the door that keeps the parser within its limits,
3167    // and it degrades rather than refuses, so the signature stays the same and no
3168    // word is lost. Such a heading is escaped into text, as it was written. A
3169    // document nested too deeply by its indentation, which is how a deeply nested
3170    // list is written, is read with the indentation cut: the list comes out
3171    // flattened below a fixed depth, every item and word kept. This used to hand
3172    // back a pasted list nested two hundred levels as one paragraph of markup. A
3173    // long paragraph is parsed on a stack sized for it, and the block holding one
3174    // too long for any stack is shown as its lines, the rest of the document as it
3175    // was.
3176    //
3177    // A document still nested too deeply once flattened, or one the parser panicked
3178    // on, comes back as its lines, one plain paragraph each (see
3179    // `djot_source_lines`). Its structure is not computed, which is the honest
3180    // answer for a document whose structure cannot be computed without ending the
3181    // process, and its text is still there for the writer to see and repair.
3182    let parsable = crate::parser_tools::djot_depth::parsable(djot);
3183    let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
3184        return djot_source_lines(djot);
3185    };
3186
3187    let mut elements: Vec<ParsedElement> = Vec::new();
3188    let mut current_spans: Vec<ParsedSpan> = Vec::new();
3189    let mut current_heading: Option<i64> = None;
3190    let mut is_code_block = false;
3191    let mut code_language: Option<String> = None;
3192    let mut blockquote_depth: u32 = 0;
3193    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
3194    let mut quotations_open: Vec<usize> = Vec::new();
3195    // Block-style attributes captured from a standalone paragraph/heading's djot
3196    // `{…}` block attributes, consumed when that block is flushed.
3197    let mut pending_style = DjotBlockStyle::default();
3198
3199    // Inline formatting state.
3200    let mut bold = false;
3201    let mut italic = false;
3202    let mut underline = false;
3203    let mut strikeout = false;
3204    let mut code = false;
3205    let mut superscript = false;
3206    let mut subscript = false;
3207    let mut link_href: Option<String> = None;
3208    // Set between an image's Start and End. Its alt text arrives as ordinary
3209    // `Str` events in between, so it has to be captured rather than emitted.
3210    let mut pending_image: Option<ParsedImage> = None;
3211
3212    // List nesting: each entry is (style, prefix, suffix, start); depth = indent + 1.
3213    let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
3214    // Context applied to the next flushed block while inside a list item.
3215    let mut cur_list_style: Option<ListStyle> = None;
3216    let mut cur_list_prefix = String::new();
3217    let mut cur_list_suffix = String::new();
3218    let mut cur_list_start: Option<i64> = None;
3219    let mut cur_list_indent: u32 = 0;
3220    let mut cur_marker: Option<MarkerType> = None;
3221
3222    // Table accumulation.
3223    let mut in_table_cell = false;
3224    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
3225    let mut current_row: Vec<ParsedTableCell> = Vec::new();
3226    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
3227    let mut table_header_rows: usize = 0;
3228    let mut row_is_head = false;
3229
3230    // Subtree-skip depth for unrepresentable containers (their entire content
3231    // is dropped). Incremented on the dropped container's `Start` and on every
3232    // nested `Start`; decremented on every `End`.
3233    let mut skip_depth: u32 = 0;
3234
3235    // The label and element index of the footnote definition currently open,
3236    // if any. Djot has no footnote inside a footnote, so one slot is enough
3237    // where the dropped containers need a depth counter.
3238    let mut footnote_open: Option<(String, usize)> = None;
3239
3240    // Push one inline span carrying the current formatting state into the
3241    // active sink (table cell or block). A macro (not a closure) to avoid
3242    // borrowing `current_spans`/`current_cell_spans` across the formatting
3243    // state reads.
3244    macro_rules! push_text {
3245        ($t:expr) => {{
3246            // Alt text belongs to the image, not to the paragraph. While an
3247            // image is open every text event is diverted into its description,
3248            // which is what keeps a photo's caption out of the manuscript's
3249            // word count and out of the search corpus.
3250            if let Some(img) = pending_image.as_mut() {
3251                img.alt.push_str(($t).as_ref());
3252            } else {
3253                let sp = ParsedSpan {
3254                    text: ($t).to_string(),
3255                    bold,
3256                    italic,
3257                    underline,
3258                    strikeout,
3259                    code,
3260                    superscript,
3261                    subscript,
3262                    link_href: link_href.clone(),
3263                    image: None,
3264                    footnote_ref: None,
3265                    preserve_whitespace: false,
3266                };
3267                if in_table_cell {
3268                    current_cell_spans.push(sp);
3269                } else {
3270                    current_spans.push(sp);
3271                }
3272            }
3273        }};
3274    }
3275
3276    // Push a completed inline image span into the active sink.
3277    macro_rules! push_image {
3278        ($img:expr) => {{
3279            let sp = ParsedSpan {
3280                text: String::new(),
3281                bold,
3282                italic,
3283                underline,
3284                strikeout,
3285                code,
3286                superscript,
3287                subscript,
3288                link_href: link_href.clone(),
3289                image: Some($img),
3290                footnote_ref: None,
3291                preserve_whitespace: false,
3292            };
3293            if in_table_cell {
3294                current_cell_spans.push(sp);
3295            } else {
3296                current_spans.push(sp);
3297            }
3298        }};
3299    }
3300
3301    // Enter a list item, flushing any unterminated inline content first and
3302    // capturing the list context + task marker for the item's block.
3303    macro_rules! enter_item {
3304        ($marker:expr) => {{
3305            if !current_spans.is_empty() {
3306                djot_push_block(
3307                    &mut elements,
3308                    std::mem::take(&mut current_spans),
3309                    None,
3310                    cur_list_style.clone(),
3311                    cur_list_indent,
3312                    cur_list_prefix.clone(),
3313                    cur_list_suffix.clone(),
3314                    cur_list_start,
3315                    cur_marker.clone(),
3316                    false,
3317                    None,
3318                    blockquote_depth,
3319                    DjotBlockStyle::default(),
3320                );
3321            }
3322            let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3323                ListStyle::Disc,
3324                String::new(),
3325                String::new(),
3326                None,
3327            ));
3328            cur_list_style = Some(style);
3329            cur_list_prefix = prefix;
3330            cur_list_suffix = suffix;
3331            cur_list_start = start;
3332            cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3333            cur_marker = $marker;
3334        }};
3335    }
3336
3337    // A hard break: a line break inside the block, [`LINE_SEPARATOR`] in its text, so the
3338    // next line is more of the same paragraph, list item, quotation or heading. A table
3339    // cell is one line, and an image's description is one too, so there it is a space.
3340    macro_rules! line_break {
3341        () => {{
3342            if in_table_cell || pending_image.is_some() {
3343                push_text!(" ");
3344            } else {
3345                push_text!(LINE_SEPARATOR_STR);
3346            }
3347        }};
3348    }
3349
3350    // A line ending inside a code span (see `E::Str` below): a new block, with the list,
3351    // quotation, heading and block style the line was in. A table cell is one line, so
3352    // there it is a space.
3353    //
3354    // The page break is the one style that stays with the first line: it is where the
3355    // block starts, and copied to every line it put each on a page of its own.
3356    macro_rules! split_line {
3357        () => {{
3358            if in_table_cell {
3359                push_text!(" ");
3360            } else if !current_spans.is_empty() && current_heading.is_some() {
3361                // The line before a hard break in a heading is a line of that
3362                // heading, as the line after it is when the heading ends. Read as
3363                // a paragraph, a heading written over two lines came back as a
3364                // paragraph and a heading.
3365                djot_push_block(
3366                    &mut elements,
3367                    std::mem::take(&mut current_spans),
3368                    current_heading,
3369                    None,
3370                    0,
3371                    String::new(),
3372                    String::new(),
3373                    None,
3374                    None,
3375                    false,
3376                    None,
3377                    blockquote_depth,
3378                    pending_style.clone(),
3379                );
3380                pending_style.page_break_before = None;
3381            } else if !current_spans.is_empty() {
3382                djot_push_block(
3383                    &mut elements,
3384                    std::mem::take(&mut current_spans),
3385                    None,
3386                    cur_list_style.clone(),
3387                    cur_list_indent,
3388                    cur_list_prefix.clone(),
3389                    cur_list_suffix.clone(),
3390                    cur_list_start,
3391                    cur_marker.clone(),
3392                    is_code_block,
3393                    code_language.clone(),
3394                    blockquote_depth,
3395                    pending_style.clone(),
3396                );
3397                pending_style.page_break_before = None;
3398            }
3399        }};
3400    }
3401
3402    for event in events {
3403        if skip_depth > 0 {
3404            match event {
3405                E::Start(..) => skip_depth += 1,
3406                E::End(_) => skip_depth -= 1,
3407                _ => {}
3408            }
3409            continue;
3410        }
3411
3412        match event {
3413            // ── Transparent wrappers (unwrap, keep content) ──
3414            E::Start(C::Document, _) | E::End(C::Document) => {}
3415            E::Start(C::Section { .. }, attrs) => {
3416                // A heading's block attributes attach to its enclosing Section,
3417                // not the heading itself; capture them for the heading's flush.
3418                if list_stack.is_empty() {
3419                    pending_style.merge_from(block_attrs_to_style(&attrs, options));
3420                }
3421            }
3422            E::End(C::Section { .. }) => {}
3423            E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3424
3425            // ── Blockquote ──
3426            E::Start(C::Blockquote, _) => {
3427                blockquote_depth += 1;
3428                quotations_open.push(elements.len());
3429            }
3430            E::End(C::Blockquote) => {
3431                blockquote_depth = blockquote_depth.saturating_sub(1);
3432                if let Some(first) = quotations_open.pop() {
3433                    mark_quotation_opened(&mut elements, first);
3434                }
3435            }
3436
3437            // ── Lists ──
3438            E::Start(C::List { kind, .. }, _) => {
3439                let (style, prefix, suffix, start) = match kind {
3440                    ListKind::Unordered(b) | ListKind::Task(b) => {
3441                        (djot_bullet_style(b), String::new(), String::new(), None)
3442                    }
3443                    ListKind::Ordered {
3444                        numbering,
3445                        style,
3446                        start,
3447                    } => {
3448                        let (p, s) = djot_ordered_affixes(style);
3449                        (djot_ordered_style(numbering), p, s, list_start_of(start))
3450                    }
3451                };
3452                list_stack.push((style, prefix, suffix, start));
3453            }
3454            E::End(C::List { .. }) => {
3455                list_stack.pop();
3456                cur_list_style = None;
3457                cur_list_start = None;
3458                cur_marker = None;
3459            }
3460            E::Start(C::ListItem, _) => enter_item!(None),
3461            E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3462                MarkerType::Checked
3463            } else {
3464                MarkerType::Unchecked
3465            })),
3466            E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3467                // Tight item without a wrapping paragraph (defensive flush).
3468                if !current_spans.is_empty() {
3469                    djot_push_block(
3470                        &mut elements,
3471                        std::mem::take(&mut current_spans),
3472                        None,
3473                        cur_list_style.clone(),
3474                        cur_list_indent,
3475                        cur_list_prefix.clone(),
3476                        cur_list_suffix.clone(),
3477                        cur_list_start,
3478                        cur_marker.clone(),
3479                        false,
3480                        None,
3481                        blockquote_depth,
3482                        DjotBlockStyle::default(),
3483                    );
3484                }
3485                cur_list_style = None;
3486                cur_list_start = None;
3487                cur_marker = None;
3488            }
3489
3490            // ── Headings, paragraphs, code blocks ──
3491            E::Start(C::Heading { level, .. }, attrs) => {
3492                current_heading = Some(level as i64);
3493                // The block-style attributes live on the enclosing Section;
3494                // merge any placed directly on the heading without clearing them.
3495                pending_style.merge_from(block_attrs_to_style(&attrs, options));
3496            }
3497            E::End(C::Heading { .. }) => {
3498                djot_push_block(
3499                    &mut elements,
3500                    std::mem::take(&mut current_spans),
3501                    current_heading.take(),
3502                    None,
3503                    0,
3504                    String::new(),
3505                    String::new(),
3506                    None,
3507                    None,
3508                    false,
3509                    None,
3510                    blockquote_depth,
3511                    std::mem::take(&mut pending_style),
3512                );
3513            }
3514            E::Start(C::Paragraph, attrs) => {
3515                current_heading = None;
3516                // Block attributes only apply to standalone paragraphs;
3517                // list-item paragraphs normalise their styling away (matching
3518                // the exporter).
3519                pending_style = if list_stack.is_empty() {
3520                    block_attrs_to_style(&attrs, options)
3521                } else {
3522                    DjotBlockStyle::default()
3523                };
3524            }
3525            E::End(C::Paragraph) => {
3526                if !current_spans.is_empty() {
3527                    djot_push_block(
3528                        &mut elements,
3529                        std::mem::take(&mut current_spans),
3530                        None,
3531                        cur_list_style.clone(),
3532                        cur_list_indent,
3533                        cur_list_prefix.clone(),
3534                        cur_list_suffix.clone(),
3535                        cur_list_start,
3536                        cur_marker.clone(),
3537                        false,
3538                        None,
3539                        blockquote_depth,
3540                        std::mem::take(&mut pending_style),
3541                    );
3542                }
3543                cur_list_style = None;
3544                cur_list_start = None;
3545                cur_marker = None;
3546            }
3547            E::Start(C::CodeBlock { language }, _) => {
3548                is_code_block = true;
3549                code_language = if language.is_empty() {
3550                    None
3551                } else {
3552                    Some(language.to_string())
3553                };
3554            }
3555            E::End(C::CodeBlock { .. }) => {
3556                // Strip the single trailing newline jotdown appends.
3557                if let Some(last) = current_spans.last_mut()
3558                    && last.text.ends_with('\n')
3559                {
3560                    last.text.pop();
3561                }
3562                djot_push_block(
3563                    &mut elements,
3564                    std::mem::take(&mut current_spans),
3565                    None,
3566                    None,
3567                    0,
3568                    String::new(),
3569                    String::new(),
3570                    None,
3571                    None,
3572                    true,
3573                    code_language.take(),
3574                    blockquote_depth,
3575                    DjotBlockStyle::default(),
3576                );
3577                is_code_block = false;
3578            }
3579
3580            // ── Tables ──
3581            E::Start(C::Table, _) => {
3582                table_rows.clear();
3583                current_row.clear();
3584                current_cell_spans.clear();
3585                table_header_rows = 0;
3586            }
3587            E::End(C::Table) => {
3588                let own_cells = cells_in_rows(&table_rows);
3589                push_table(
3590                    &mut elements,
3591                    ParsedTable {
3592                        header_rows: table_header_rows,
3593                        rows: std::mem::take(&mut table_rows),
3594                        blockquote_depth,
3595                        blockquote_opens: 0,
3596                    },
3597                    own_cells,
3598                );
3599            }
3600            E::Start(C::TableRow { head }, _) => {
3601                row_is_head = head;
3602                current_row.clear();
3603            }
3604            E::End(C::TableRow { .. }) => {
3605                if row_is_head {
3606                    table_header_rows += 1;
3607                }
3608                table_rows.push(std::mem::take(&mut current_row));
3609            }
3610            E::Start(C::TableCell { .. }, _) => {
3611                in_table_cell = true;
3612                current_cell_spans.clear();
3613            }
3614            E::End(C::TableCell { .. }) => {
3615                in_table_cell = false;
3616                current_row.push(ParsedTableCell {
3617                    spans: std::mem::take(&mut current_cell_spans),
3618                });
3619            }
3620
3621            // ── Inline formatting ──
3622            E::Start(C::Strong, _) => bold = true,
3623            E::End(C::Strong) => bold = false,
3624            E::Start(C::Emphasis, _) => italic = true,
3625            E::End(C::Emphasis) => italic = false,
3626            E::Start(C::Verbatim, _) => code = true,
3627            E::End(C::Verbatim) => code = false,
3628            E::Start(C::Superscript, _) => superscript = true,
3629            E::End(C::Superscript) => superscript = false,
3630            E::Start(C::Subscript, _) => subscript = true,
3631            E::End(C::Subscript) => subscript = false,
3632            E::Start(C::Insert, _) => underline = true,
3633            E::End(C::Insert) => underline = false,
3634            E::Start(C::Delete, _) => strikeout = true,
3635            E::End(C::Delete) => strikeout = false,
3636            // Highlight/mark and bare spans have no model field — keep the text.
3637            E::Start(C::Mark, _) | E::End(C::Mark) => {}
3638            E::Start(C::Span, _) | E::End(C::Span) => {}
3639            E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3640            E::End(C::Link(..)) => link_href = None,
3641            // Inline images. Djot writes display size as inline attributes
3642            // (`![alt](src){width=800 height=600}`), which jotdown hands over
3643            // on the `Start` event — verified against jotdown 0.10, including
3644            // quoted values and images mid-sentence.
3645            E::Start(C::Image(src, _), attrs) => {
3646                let attr_num = |key: &str| -> i64 {
3647                    attrs
3648                        .get_value(key)
3649                        .map(|v| v.to_string())
3650                        .and_then(|v| v.trim().parse::<i64>().ok())
3651                        .filter(|n| *n > 0)
3652                        .unwrap_or(0)
3653                };
3654                pending_image = Some(ParsedImage {
3655                    // The writer percent-encodes what the lexer would act on in a
3656                    // source; the source is a name, so it is decoded back.
3657                    src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3658                    alt: String::new(),
3659                    width: attr_num("width"),
3660                    height: attr_num("height"),
3661                });
3662            }
3663            E::End(C::Image(..)) => {
3664                if let Some(img) = pending_image.take() {
3665                    push_image!(img);
3666                }
3667            }
3668
3669            // ── Footnote definitions ──
3670            //
3671            // The body is ordinary block content, so it is parsed by the
3672            // ordinary machinery: flush whatever inline run is open, note where
3673            // this definition's blocks start, and let them accumulate. `End`
3674            // lifts them back out. Nesting cannot occur — djot has no footnote
3675            // inside a footnote — so one mark suffices where the dropped
3676            // containers below need a depth counter.
3677            E::Start(C::Footnote { label }, _) => {
3678                if !current_spans.is_empty() {
3679                    djot_push_block(
3680                        &mut elements,
3681                        std::mem::take(&mut current_spans),
3682                        None,
3683                        cur_list_style.clone(),
3684                        cur_list_indent,
3685                        cur_list_prefix.clone(),
3686                        cur_list_suffix.clone(),
3687                        cur_list_start,
3688                        cur_marker.clone(),
3689                        false,
3690                        None,
3691                        blockquote_depth,
3692                        DjotBlockStyle::default(),
3693                    );
3694                }
3695                footnote_open = Some((label.to_string(), elements.len()));
3696            }
3697            E::End(C::Footnote { .. }) => {
3698                if !current_spans.is_empty() {
3699                    djot_push_block(
3700                        &mut elements,
3701                        std::mem::take(&mut current_spans),
3702                        None,
3703                        cur_list_style.clone(),
3704                        cur_list_indent,
3705                        cur_list_prefix.clone(),
3706                        cur_list_suffix.clone(),
3707                        cur_list_start,
3708                        cur_marker.clone(),
3709                        false,
3710                        None,
3711                        blockquote_depth,
3712                        DjotBlockStyle::default(),
3713                    );
3714                }
3715                if let Some((label, start)) = footnote_open.take() {
3716                    let blocks: Vec<ParsedBlock> = elements
3717                        .drain(start..)
3718                        .filter_map(|e| match e {
3719                            ParsedElement::Block(b) => Some(b),
3720                            // A table inside a footnote is not representable as
3721                            // note content; its cells would have to become
3722                            // blocks and lose their structure either way.
3723                            _ => None,
3724                        })
3725                        .collect();
3726                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3727                }
3728            }
3729
3730            // ── Unrepresentable containers: drop the entire subtree ──
3731            E::Start(
3732                C::Math { .. }
3733                | C::RawBlock { .. }
3734                | C::RawInline { .. }
3735                | C::DescriptionList
3736                | C::DescriptionDetails
3737                | C::DescriptionTerm
3738                | C::Caption
3739                | C::LinkDefinition { .. },
3740                _,
3741            ) => skip_depth = 1,
3742
3743            // ── Text + atoms ──
3744            // A line ending inside a code span is kept by the parser as a `\n` in the
3745            // span's text. It is read as a new block with the list, quotation or heading
3746            // the line was in, as a `\n` pasted into a block is (see
3747            // `split_block_at_line_breaks`). Kept inside the span, the Djot writer saved
3748            // the code run's lines as blocks of their own, which reloaded as two blocks: a
3749            // document that did not settle into the Djot it saves as. Such a span is what
3750            // an older version wrote for a code run holding a line break, a pasted
3751            // `<pre><code>` passage above all, and reading its lines as blocks is how that
3752            // passage reads in the editor. It is not a hard break: nothing in it asked for
3753            // a line of the same paragraph.
3754            E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3755                for (i, piece) in s.split('\n').enumerate() {
3756                    if i > 0 {
3757                        split_line!();
3758                    }
3759                    if !piece.is_empty() {
3760                        push_text!(piece);
3761                    }
3762                }
3763            }
3764            E::Str(s) => push_text!(s.as_ref()),
3765            E::Softbreak => push_text!(" "),
3766            E::LeftSingleQuote => push_text!("\u{2018}"),
3767            E::RightSingleQuote => push_text!("\u{2019}"),
3768            E::LeftDoubleQuote => push_text!("\u{201C}"),
3769            E::RightDoubleQuote => push_text!("\u{201D}"),
3770            E::Ellipsis => push_text!("\u{2026}"),
3771            E::EnDash => push_text!("\u{2013}"),
3772            E::EmDash => push_text!("\u{2014}"),
3773            E::NonBreakingSpace => push_text!("\u{00A0}"),
3774            E::Hardbreak => line_break!(),
3775            // A footnote reference. jotdown emits this purely syntactically —
3776            // it never checks that a matching `[^label]:` exists anywhere — so a
3777            // reference whose definition lives outside this document (the normal
3778            // state for a host that owns note bodies itself) arrives here just
3779            // the same, and must survive.
3780            E::FootnoteReference(label) => {
3781                let sp = ParsedSpan {
3782                    text: String::new(),
3783                    bold,
3784                    italic,
3785                    underline,
3786                    strikeout,
3787                    code,
3788                    superscript,
3789                    subscript,
3790                    link_href: link_href.clone(),
3791                    image: None,
3792                    footnote_ref: Some(label.to_string()),
3793                    preserve_whitespace: false,
3794                };
3795                if in_table_cell {
3796                    current_cell_spans.push(sp);
3797                } else {
3798                    current_spans.push(sp);
3799                }
3800            }
3801            // Symbols, escapes, blanklines, thematic breaks and dangling block
3802            // attributes carry no representable content.
3803            E::Symbol(_) => {}
3804            E::Escape | E::Blankline => {}
3805            E::ThematicBreak(_) | E::Attributes(_) => {}
3806
3807            // Ends of dropped containers (never reached at skip_depth 0) and any
3808            // future variants.
3809            _ => {}
3810        }
3811    }
3812
3813    // Flush any trailing inline content (defensive — Document End closes blocks).
3814    if !current_spans.is_empty() {
3815        djot_push_block(
3816            &mut elements,
3817            std::mem::take(&mut current_spans),
3818            current_heading.take(),
3819            cur_list_style.clone(),
3820            cur_list_indent,
3821            cur_list_prefix.clone(),
3822            cur_list_suffix.clone(),
3823            cur_list_start,
3824            cur_marker.clone(),
3825            is_code_block,
3826            code_language.take(),
3827            blockquote_depth,
3828            std::mem::take(&mut pending_style),
3829        );
3830    }
3831
3832    // An empty document still yields a single empty paragraph (matches
3833    // `parse_markdown`).
3834    if elements.is_empty() {
3835        djot_push_block(
3836            &mut elements,
3837            vec![ParsedSpan {
3838                text: String::new(),
3839                ..Default::default()
3840            }],
3841            None,
3842            None,
3843            0,
3844            String::new(),
3845            String::new(),
3846            None,
3847            None,
3848            false,
3849            None,
3850            0,
3851            DjotBlockStyle::default(),
3852        );
3853    }
3854
3855    elements
3856}
3857
3858#[cfg(test)]
3859mod tests {
3860    use super::*;
3861
3862    #[test]
3863    fn test_style_keeps_whitespace_reads_important() {
3864        assert_eq!(style_keeps_whitespace("white-space: pre-wrap"), Some(true));
3865        assert_eq!(
3866            style_keeps_whitespace("white-space: pre-wrap !important"),
3867            Some(true)
3868        );
3869        assert_eq!(
3870            style_keeps_whitespace("white-space:PRE!IMPORTANT; color: red"),
3871            Some(true)
3872        );
3873        assert_eq!(
3874            style_keeps_whitespace("white-space: pre ! important; white-space: normal"),
3875            Some(true),
3876            "a later declaration that is not important does not win"
3877        );
3878        assert_eq!(
3879            style_keeps_whitespace("white-space: pre; white-space: normal !important"),
3880            Some(false)
3881        );
3882        assert_eq!(
3883            style_keeps_whitespace("white-space: pre; white-space: normal"),
3884            Some(false)
3885        );
3886        assert_eq!(style_keeps_whitespace("white-space: inherit"), None);
3887    }
3888
3889    /// Helper: flatten parse_markdown output to blocks for tests that don't care about tables.
3890    fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3891        ParsedElement::flatten_to_blocks(parse_markdown(md))
3892    }
3893
3894    #[test]
3895    fn test_parse_markdown_simple_paragraph() {
3896        let blocks = parse_markdown_blocks("Hello **world**");
3897        assert_eq!(blocks.len(), 1);
3898        assert!(blocks[0].spans.len() >= 2);
3899        // "Hello " is plain, "world" is bold
3900        let plain_span = blocks[0]
3901            .spans
3902            .iter()
3903            .find(|s| s.text.contains("Hello"))
3904            .unwrap();
3905        assert!(!plain_span.bold);
3906        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3907        assert!(bold_span.bold);
3908    }
3909
3910    #[test]
3911    fn test_parse_markdown_heading() {
3912        let blocks = parse_markdown_blocks("# Title");
3913        assert_eq!(blocks.len(), 1);
3914        assert_eq!(blocks[0].heading_level, Some(1));
3915        assert_eq!(blocks[0].spans[0].text, "Title");
3916    }
3917
3918    #[test]
3919    fn test_parse_markdown_list() {
3920        let blocks = parse_markdown_blocks("- item1\n- item2");
3921        assert!(blocks.len() >= 2);
3922        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3923        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3924    }
3925
3926    /// Helper: extract (is_table, blockquote_depth) per element for nesting assertions.
3927    fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3928        elements
3929            .iter()
3930            .map(|e| match e {
3931                ParsedElement::Block(b) => (false, b.blockquote_depth),
3932                ParsedElement::Table(t) => (true, t.blockquote_depth),
3933                // Definitions carry no blockquote nesting of their own; this
3934                // helper exists for the nesting assertions and never sees one.
3935                ParsedElement::FootnoteDefinition { .. } => (false, 0),
3936            })
3937            .collect()
3938    }
3939
3940    #[test]
3941    fn test_parse_markdown_table_in_blockquote_records_depth() {
3942        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3943        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3944    }
3945
3946    #[test]
3947    fn test_parse_markdown_text_then_table_in_blockquote() {
3948        let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3949        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3950    }
3951
3952    #[test]
3953    fn test_parse_markdown_table_after_blockquote_closes() {
3954        let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3955        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3956    }
3957
3958    #[test]
3959    fn test_parse_markdown_table_in_nested_blockquote() {
3960        let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3961        assert_eq!(element_depths(&elements), vec![(true, 2)]);
3962    }
3963
3964    #[test]
3965    fn test_parse_markdown_list_in_blockquote_records_depth() {
3966        let elements = parse_markdown("> - item1\n> - item2");
3967        let depths = element_depths(&elements);
3968        assert_eq!(depths, vec![(false, 1), (false, 1)]);
3969        for e in &elements {
3970            if let ParsedElement::Block(b) = e {
3971                assert_eq!(b.list_style, Some(ListStyle::Disc));
3972            }
3973        }
3974    }
3975
3976    #[test]
3977    fn test_parse_html_table_in_blockquote_records_depth() {
3978        let elements = parse_html_elements(
3979            "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3980        );
3981        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3982    }
3983
3984    #[test]
3985    fn test_parse_html_table_after_blockquote() {
3986        let elements = parse_html_elements(
3987            "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3988        );
3989        let depths = element_depths(&elements);
3990        // The blockquote paragraph carries depth 1; the table is outside (depth 0).
3991        assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3992        assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3993    }
3994
3995    #[test]
3996    fn test_flatten_to_blocks_propagates_blockquote_depth() {
3997        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3998        let blocks = ParsedElement::flatten_to_blocks(elements);
3999        assert!(!blocks.is_empty());
4000        for b in &blocks {
4001            assert_eq!(b.blockquote_depth, 1);
4002        }
4003    }
4004
4005    #[test]
4006    fn test_parse_html_simple() {
4007        let blocks = parse_html("<p>Hello <b>world</b></p>");
4008        assert_eq!(blocks.len(), 1);
4009        assert!(blocks[0].spans.len() >= 2);
4010        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
4011        assert!(bold_span.bold);
4012    }
4013
4014    #[test]
4015    fn test_parse_html_multiple_paragraphs() {
4016        let blocks = parse_html("<p>A</p><p>B</p>");
4017        assert_eq!(blocks.len(), 2);
4018    }
4019
4020    #[test]
4021    fn test_parse_html_heading() {
4022        let blocks = parse_html("<h2>Subtitle</h2>");
4023        assert_eq!(blocks.len(), 1);
4024        assert_eq!(blocks[0].heading_level, Some(2));
4025    }
4026
4027    #[test]
4028    fn test_parse_html_list() {
4029        let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
4030        assert!(blocks.len() >= 2);
4031        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4032    }
4033
4034    #[test]
4035    fn test_parse_markdown_code_block() {
4036        let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
4037        assert_eq!(blocks.len(), 1);
4038        assert!(blocks[0].is_code_block);
4039        assert!(blocks[0].spans[0].code);
4040        // pulldown-cmark appends a trailing \n to code block text — verify it's stripped
4041        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4042        assert_eq!(
4043            text, "fn main() {}",
4044            "code block text should not have trailing newline"
4045        );
4046    }
4047
4048    #[test]
4049    fn test_parse_markdown_nested_formatting() {
4050        let blocks = parse_markdown_blocks("***bold italic***");
4051        assert_eq!(blocks.len(), 1);
4052        let span = &blocks[0].spans[0];
4053        assert!(span.bold);
4054        assert!(span.italic);
4055    }
4056
4057    #[test]
4058    fn test_parse_markdown_link() {
4059        let blocks = parse_markdown_blocks("[click](http://example.com)");
4060        assert_eq!(blocks.len(), 1);
4061        let span = &blocks[0].spans[0];
4062        assert_eq!(span.text, "click");
4063        assert_eq!(span.link_href, Some("http://example.com".to_string()));
4064    }
4065
4066    #[test]
4067    fn test_parse_markdown_empty() {
4068        let blocks = parse_markdown_blocks("");
4069        assert_eq!(blocks.len(), 1);
4070        assert!(blocks[0].spans[0].text.is_empty());
4071    }
4072
4073    #[test]
4074    fn test_parse_html_empty() {
4075        let blocks = parse_html("");
4076        assert_eq!(blocks.len(), 1);
4077        assert!(blocks[0].spans[0].text.is_empty());
4078    }
4079
4080    #[test]
4081    fn test_parse_html_nested_formatting() {
4082        let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
4083        assert_eq!(blocks.len(), 1);
4084        let span = &blocks[0].spans[0];
4085        assert!(span.bold);
4086        assert!(span.italic);
4087    }
4088
4089    #[test]
4090    fn test_parse_html_link() {
4091        let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
4092        assert_eq!(blocks.len(), 1);
4093        let span = &blocks[0].spans[0];
4094        assert_eq!(span.text, "click");
4095        assert_eq!(span.link_href, Some("http://example.com".to_string()));
4096    }
4097
4098    #[test]
4099    fn test_parse_html_ordered_list() {
4100        let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
4101        assert!(blocks.len() >= 2);
4102        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4103    }
4104
4105    #[test]
4106    fn test_parse_markdown_ordered_list() {
4107        let blocks = parse_markdown_blocks("1. first\n2. second");
4108        assert!(blocks.len() >= 2);
4109        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
4110    }
4111
4112    #[test]
4113    fn test_parse_html_blockquote_nested() {
4114        let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
4115        assert!(blocks.len() >= 3);
4116    }
4117
4118    #[test]
4119    fn test_parse_block_styles_line_height() {
4120        let styles = parse_block_styles("line-height: 1.5");
4121        assert_eq!(styles.line_height, Some(1500));
4122    }
4123
4124    #[test]
4125    fn test_parse_block_styles_direction_rtl() {
4126        let styles = parse_block_styles("direction: rtl");
4127        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4128    }
4129
4130    #[test]
4131    fn test_parse_block_styles_background_color() {
4132        let styles = parse_block_styles("background-color: #ff0000");
4133        assert_eq!(styles.background_color, Some("#ff0000".to_string()));
4134    }
4135
4136    #[test]
4137    fn test_parse_block_styles_white_space_pre() {
4138        let styles = parse_block_styles("white-space: pre");
4139        assert_eq!(styles.non_breakable_lines, Some(true));
4140    }
4141
4142    #[test]
4143    fn test_parse_block_styles_multiple() {
4144        let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
4145        assert_eq!(styles.line_height, Some(2000));
4146        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
4147        assert_eq!(styles.background_color, Some("blue".to_string()));
4148    }
4149
4150    #[test]
4151    fn test_parse_html_block_styles_extracted() {
4152        let blocks = parse_html(
4153            r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
4154        );
4155        assert_eq!(blocks.len(), 1);
4156        assert_eq!(blocks[0].line_height, Some(1500));
4157        assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
4158        assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
4159    }
4160
4161    /// The shape every word processor publishes on the clipboard: a newline
4162    /// after the opening tag, the prose hard-wrapped inside it, and a newline
4163    /// between every pair of blocks. None of it is text.
4164    #[test]
4165    fn test_parse_html_collapses_exporter_line_wrapping() {
4166        let blocks = parse_html(
4167            "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
4168             <p>\nsecond para</p>\n</body>",
4169        );
4170        let texts: Vec<String> = blocks
4171            .iter()
4172            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4173            .collect();
4174        assert_eq!(texts, vec!["first line wrapped here", "second para"]);
4175    }
4176
4177    /// The reported bug: pasting a whole LibreOffice book ended in one block
4178    /// holding every newline the exporter had written between its paragraphs —
4179    /// four thousand blank lines under the last sentence.
4180    #[test]
4181    fn test_parse_html_drops_whitespace_between_blocks() {
4182        let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
4183        assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
4184        assert!(
4185            blocks
4186                .iter()
4187                .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
4188            "no block keeps a literal newline"
4189        );
4190    }
4191
4192    /// A run split across inline elements is still one run, and it collapses to
4193    /// one space rather than disappearing between them.
4194    #[test]
4195    fn test_parse_html_collapses_across_span_boundaries() {
4196        let blocks = parse_html("<p>  <b>bold</b>\n  <i>italic</i>  </p>");
4197        assert_eq!(blocks.len(), 1);
4198        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4199        assert_eq!(text, "bold italic");
4200    }
4201
4202    /// U+00A0 is content, not layout — the space holding "Attention&nbsp;:"
4203    /// together in French must survive a paste.
4204    #[test]
4205    fn test_parse_html_keeps_no_break_space() {
4206        let blocks = parse_html("<p>\nAttention&nbsp;: ici</p>");
4207        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4208        assert_eq!(text, "Attention\u{a0}: ici");
4209    }
4210
4211    /// An empty paragraph is a paragraph. Only the whitespace *between* blocks
4212    /// goes; a block the writer opened stays, however little is in it.
4213    #[test]
4214    fn test_parse_html_keeps_empty_paragraph() {
4215        let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
4216        assert_eq!(blocks.len(), 3);
4217        let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
4218        assert_eq!(text, "");
4219    }
4220
4221    /// `<pre>` and `white-space: pre` mean the source's spacing *is* the
4222    /// content. `nowrap` does not: it forbids wrapping and collapses runs like
4223    /// any other block, which is why it is not in the same test.
4224    #[test]
4225    fn test_parse_html_preserves_whitespace_in_pre() {
4226        // The newline straight after `<pre>` is the one the HTML parser itself
4227        // drops, per spec; every newline after that is content.
4228        let blocks = parse_html("<pre>\nfn main() {\n    let x = 1;\n}</pre>");
4229        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4230        assert_eq!(text, "fn main() {\n    let x = 1;\n}");
4231
4232        // Outside a code block a line break is a new block, and each line keeps its
4233        // spaces.
4234        let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n  b</p>");
4235        let texts: Vec<String> = styled
4236            .iter()
4237            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
4238            .collect();
4239        assert_eq!(texts, ["a", "  b"]);
4240
4241        let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n  b</p>");
4242        let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
4243        assert_eq!(text, "a b");
4244    }
4245
4246    /// Table cells are blocks too, and an exporter indents them just as
4247    /// generously as it indents paragraphs.
4248    #[test]
4249    fn test_parse_html_collapses_table_cell_whitespace() {
4250        let elements = parse_html_elements(
4251            "<table>\n<tr>\n<td>\n  one\n</td>\n<td>\n  two\n</td>\n</tr>\n</table>",
4252        );
4253        let table = elements
4254            .iter()
4255            .find_map(|e| match e {
4256                ParsedElement::Table(t) => Some(t),
4257                _ => None,
4258            })
4259            .expect("a table");
4260        let cells: Vec<String> = table.rows[0]
4261            .iter()
4262            .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
4263            .collect();
4264        assert_eq!(cells, vec!["one", "two"]);
4265    }
4266
4267    #[test]
4268    fn test_parse_html_white_space_pre() {
4269        let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4270        assert_eq!(blocks.len(), 1);
4271        assert_eq!(blocks[0].non_breakable_lines, Some(true));
4272    }
4273
4274    #[test]
4275    fn test_parse_html_no_styles_returns_none() {
4276        let blocks = parse_html("<p>plain</p>");
4277        assert_eq!(blocks.len(), 1);
4278        assert_eq!(blocks[0].line_height, None);
4279        assert_eq!(blocks[0].direction, None);
4280        assert_eq!(blocks[0].background_color, None);
4281        assert_eq!(blocks[0].non_breakable_lines, None);
4282    }
4283
4284    #[test]
4285    fn test_parse_markdown_nested_list_indent() {
4286        let md = "- top\n  - nested\n    - deep";
4287        let blocks = parse_markdown_blocks(md);
4288        assert_eq!(blocks.len(), 3);
4289        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4290        assert_eq!(blocks[0].list_indent, 0);
4291        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4292        assert_eq!(blocks[1].list_indent, 1);
4293        assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4294        assert_eq!(blocks[2].list_indent, 2);
4295    }
4296
4297    #[test]
4298    fn test_parse_markdown_nested_ordered_list_indent() {
4299        let md = "1. first\n   1. nested\n   2. nested2";
4300        let blocks = parse_markdown_blocks(md);
4301        assert_eq!(blocks.len(), 3);
4302        assert_eq!(blocks[0].list_indent, 0);
4303        assert_eq!(blocks[1].list_indent, 1);
4304        assert_eq!(blocks[2].list_indent, 1);
4305    }
4306
4307    #[test]
4308    fn test_parse_html_nested_list_indent() {
4309        let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4310        let blocks = parse_html(html);
4311        assert!(blocks.len() >= 2);
4312        assert_eq!(blocks[0].list_indent, 0);
4313        assert_eq!(blocks[1].list_indent, 1);
4314    }
4315
4316    #[test]
4317    fn test_parse_markdown_table() {
4318        let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4319        let elements = parse_markdown(md);
4320        assert_eq!(elements.len(), 1);
4321        match &elements[0] {
4322            ParsedElement::Table(table) => {
4323                assert_eq!(table.header_rows, 1);
4324                assert_eq!(table.rows.len(), 2); // 1 header + 1 body
4325                // Header row
4326                assert_eq!(table.rows[0].len(), 2);
4327                assert_eq!(table.rows[0][0].spans[0].text, "A");
4328                assert_eq!(table.rows[0][1].spans[0].text, "B");
4329                // Body row
4330                assert_eq!(table.rows[1].len(), 2);
4331                assert_eq!(table.rows[1][0].spans[0].text, "1");
4332                assert_eq!(table.rows[1][1].spans[0].text, "2");
4333            }
4334            _ => panic!("Expected ParsedElement::Table"),
4335        }
4336    }
4337
4338    #[test]
4339    fn test_parse_markdown_table_with_formatting() {
4340        let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4341        let elements = parse_markdown(md);
4342        assert_eq!(elements.len(), 1);
4343        match &elements[0] {
4344            ParsedElement::Table(table) => {
4345                assert_eq!(table.rows.len(), 2);
4346                // Header: bold cell
4347                assert!(table.rows[0][0].spans[0].bold);
4348                // Header: code cell
4349                assert!(table.rows[0][1].spans[0].code);
4350                // Header: italic cell
4351                assert!(table.rows[0][2].spans[0].italic);
4352                // Body: strikeout cell
4353                assert!(table.rows[1][0].spans[0].strikeout);
4354                // Body: link cell
4355                assert_eq!(
4356                    table.rows[1][2].spans[0].link_href,
4357                    Some("http://x.com".to_string())
4358                );
4359            }
4360            _ => panic!("Expected ParsedElement::Table"),
4361        }
4362    }
4363
4364    #[test]
4365    fn test_parse_markdown_mixed_content_with_table() {
4366        let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4367        let elements = parse_markdown(md);
4368        assert_eq!(elements.len(), 3);
4369        assert!(matches!(&elements[0], ParsedElement::Block(_)));
4370        assert!(matches!(&elements[1], ParsedElement::Table(_)));
4371        assert!(matches!(&elements[2], ParsedElement::Block(_)));
4372    }
4373}
4374
4375#[cfg(test)]
4376mod djot_tests {
4377    use super::*;
4378    use crate::entities::MarkerType;
4379
4380    fn blocks(d: &str) -> Vec<ParsedBlock> {
4381        ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4382    }
4383
4384    fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4385        b.spans.iter().find(|s| pred(s)).expect("span not found")
4386    }
4387
4388    #[test]
4389    fn paragraph_bold_italic() {
4390        let b = blocks("normal *bold* _italic_");
4391        assert_eq!(b.len(), 1);
4392        assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4393        assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4394    }
4395
4396    #[test]
4397    fn heading_levels() {
4398        assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4399        assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4400        assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4401    }
4402
4403    #[test]
4404    fn unordered_bullet_styles_are_distinct() {
4405        assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4406        assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4407        assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4408    }
4409
4410    #[test]
4411    fn ordered_delimiters() {
4412        let period = blocks("1. a");
4413        assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4414        assert_eq!(period[0].list_prefix, "");
4415        assert_eq!(period[0].list_suffix, ".");
4416
4417        let paren = blocks("1) a");
4418        assert_eq!(paren[0].list_suffix, ")");
4419        assert_eq!(paren[0].list_prefix, "");
4420
4421        let paren_paren = blocks("(1) a");
4422        assert_eq!(paren_paren[0].list_prefix, "(");
4423        assert_eq!(paren_paren[0].list_suffix, ")");
4424    }
4425
4426    #[test]
4427    fn task_list_markers() {
4428        let b = blocks("- [ ] a\n- [x] b");
4429        assert_eq!(b.len(), 2);
4430        assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4431        assert_eq!(b[1].marker, Some(MarkerType::Checked));
4432    }
4433
4434    #[test]
4435    fn code_block_with_language() {
4436        let b = blocks("```rust\nfn main() {}\n```");
4437        assert_eq!(b.len(), 1);
4438        assert!(b[0].is_code_block);
4439        assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4440        let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4441        assert_eq!(text, "fn main() {}");
4442    }
4443
4444    #[test]
4445    fn link_href() {
4446        let b = blocks("[text](http://example.com)");
4447        let s = first_span_with(&b[0], |s| s.text == "text");
4448        assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4449    }
4450
4451    #[test]
4452    fn superscript_subscript() {
4453        assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4454        assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4455    }
4456
4457    #[test]
4458    fn delete_insert_verbatim() {
4459        assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4460        assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4461        assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4462    }
4463
4464    #[test]
4465    fn blockquote_depth() {
4466        let els = parse_djot("> quoted", &DjotImportOptions::default());
4467        match &els[0] {
4468            ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4469            _ => panic!("expected block"),
4470        }
4471    }
4472
4473    #[test]
4474    fn nested_list_indent() {
4475        // Djot nests a sub-list only when a blank line separates it from the
4476        // parent item and it is indented to the parent's content column
4477        // (2 spaces per level). Without the blank line the markers fold into
4478        // the paragraph as lazy continuation.
4479        let b = blocks("- a\n\n  - b\n\n    - c");
4480        assert_eq!(b.len(), 3);
4481        assert_eq!(b[0].list_indent, 0);
4482        assert_eq!(b[1].list_indent, 1);
4483        assert_eq!(b[2].list_indent, 2);
4484    }
4485
4486    #[test]
4487    fn table_parsed_as_table() {
4488        let els = parse_djot(
4489            "| a | b |\n|---|---|\n| c | d |",
4490            &DjotImportOptions::default(),
4491        );
4492        assert_eq!(els.len(), 1);
4493        match &els[0] {
4494            ParsedElement::Table(t) => {
4495                assert_eq!(t.header_rows, 1);
4496                assert_eq!(t.rows.len(), 2);
4497                assert_eq!(t.rows[0][0].spans[0].text, "a");
4498                assert_eq!(t.rows[1][1].spans[0].text, "d");
4499            }
4500            _ => panic!("expected table"),
4501        }
4502    }
4503
4504    #[test]
4505    fn smart_punctuation_normalised_to_unicode() {
4506        let text: String = blocks("a... b---c")[0]
4507            .spans
4508            .iter()
4509            .map(|s| s.text.as_str())
4510            .collect();
4511        assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4512        assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4513    }
4514
4515    #[test]
4516    fn unrepresentable_constructs_dropped_without_leaking_text() {
4517        // Thematic break between two paragraphs: no extra block, no stray text.
4518        let b = blocks("para1\n\n---\n\npara2");
4519        assert_eq!(b.len(), 2);
4520        assert_eq!(
4521            b[0].spans
4522                .iter()
4523                .map(|s| s.text.as_str())
4524                .collect::<String>(),
4525            "para1"
4526        );
4527        assert_eq!(
4528            b[1].spans
4529                .iter()
4530                .map(|s| s.text.as_str())
4531                .collect::<String>(),
4532            "para2"
4533        );
4534
4535        // Fenced div is unwrapped: its content survives, the fence does not.
4536        let d = blocks(":::\ninside\n:::");
4537        let joined: String = d
4538            .iter()
4539            .flat_map(|b| b.spans.iter())
4540            .map(|s| s.text.as_str())
4541            .collect();
4542        assert_eq!(joined, "inside");
4543
4544        // Inline math content is dropped, surrounding text kept.
4545        let m = blocks("before $`E=mc^2` after");
4546        let joined: String = m
4547            .iter()
4548            .flat_map(|b| b.spans.iter())
4549            .map(|s| s.text.as_str())
4550            .collect();
4551        assert!(joined.contains("before"), "{joined:?}");
4552        assert!(joined.contains("after"), "{joined:?}");
4553        assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4554    }
4555
4556    #[test]
4557    fn empty_document_yields_one_empty_block() {
4558        let b = blocks("");
4559        assert_eq!(b.len(), 1);
4560        assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4561    }
4562
4563    #[test]
4564    fn block_attributes_parse_into_block() {
4565        let b = blocks(
4566            "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4567        );
4568        assert_eq!(b.len(), 1);
4569        assert_eq!(b[0].alignment, Some(Alignment::Center));
4570        assert_eq!(b[0].line_height, Some(1500));
4571        assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4572        assert_eq!(b[0].non_breakable_lines, Some(true));
4573        assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4574    }
4575
4576    #[test]
4577    fn spacing_block_attributes_parse_into_block() {
4578        // `top_margin` / `text_indent` let one block override the document-wide
4579        // paragraph spacing and first-line indent — what a scene break needs for
4580        // the paragraph that follows it.
4581        let b = blocks("{top_margin=24 text_indent=0}\nhello");
4582        assert_eq!(b.len(), 1);
4583        assert_eq!(b[0].top_margin, Some(24));
4584        assert_eq!(b[0].text_indent, Some(0));
4585    }
4586
4587    #[test]
4588    fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4589        // The whole point of the attribute: `Some(0)` means "explicitly no
4590        // indent", which must not collapse to `None` ("use the document
4591        // default") — otherwise a scene break could not suppress the indent.
4592        let explicit = blocks("{text_indent=0}\nhello");
4593        let absent = blocks("hello");
4594        assert_eq!(explicit[0].text_indent, Some(0));
4595        assert_eq!(absent[0].text_indent, None);
4596        assert!(!explicit[0].is_inline_only());
4597        assert!(absent[0].is_inline_only());
4598    }
4599
4600    #[test]
4601    fn spacing_block_attributes_respect_import_options() {
4602        let src = "{top_margin=24 text_indent=0}\nhello";
4603        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4604        assert_eq!(b[0].top_margin, None);
4605        assert_eq!(b[0].text_indent, None);
4606    }
4607
4608    #[test]
4609    fn block_attributes_on_heading() {
4610        let b = blocks("{alignment=right}\n# Title");
4611        assert_eq!(b[0].heading_level, Some(1));
4612        assert_eq!(b[0].alignment, Some(Alignment::Right));
4613    }
4614
4615    #[test]
4616    fn block_attributes_respect_import_options() {
4617        // With every optional attribute disabled, the `{…}` block attributes are
4618        // parsed and discarded — only the core paragraph survives.
4619        let src = "{alignment=center line_height=1500}\nhello";
4620        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4621        assert_eq!(b[0].alignment, None);
4622        assert_eq!(b[0].line_height, None);
4623        assert_eq!(
4624            b[0].spans
4625                .iter()
4626                .map(|s| s.text.as_str())
4627                .collect::<String>(),
4628            "hello"
4629        );
4630    }
4631
4632    #[test]
4633    fn list_item_block_attributes_are_dropped() {
4634        // Block attributes only bind to standalone paragraphs/headings; a list
4635        // item normalises them away (symmetric with the exporter).
4636        let b = blocks("{alignment=center}\n- item");
4637        assert!(b.iter().all(|blk| blk.alignment.is_none()));
4638    }
4639
4640    #[test]
4641    fn unknown_alignment_value_is_ignored() {
4642        let b = blocks("{alignment=sideways}\nhello");
4643        assert_eq!(b[0].alignment, None);
4644    }
4645}
4646
4647// ─── Cheap plain-text extraction ─────────────────────────────────────
4648
4649/// The `U+FFFC OBJECT REPLACEMENT CHARACTER` that stands for a table in the document's
4650/// text.
4651///
4652/// A table is not prose, but it *occupies a position* in the flow: the import mirrors this
4653/// single sentinel into the rope where the table sits
4654/// (`rope_helpers::rope_append_table_anchor`), then the cells as ordinary blocks. Anything
4655/// reconstructing the text a search runs against has to reproduce it, or every offset after
4656/// the first table is short by the two characters (the sentinel and its separator) that the
4657/// document really holds there.
4658pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4659
4660/// `U+2028 LINE SEPARATOR`: a line break inside a block, the one Shift+Enter makes in a
4661/// word processor.
4662///
4663/// A block's text holds it as one character, where it starts a new line of the same
4664/// paragraph, list item, quotation or heading. It is the character Qt's `QTextDocument`
4665/// uses for the same thing. It cannot be `\n`: the rope already uses `\n` between two
4666/// blocks, so a search, an offset or a plain-text export could not tell a line from a
4667/// paragraph.
4668///
4669/// The readers make one of every hard break they meet (Djot's `\` at the end of a line,
4670/// Markdown's, an HTML `<br>` inside a paragraph), and each writer puts it back as its
4671/// format's own line break. A table cell is one line in Djot and Markdown, so there a
4672/// line break is read as a space.
4673pub const LINE_SEPARATOR: char = '\u{2028}';
4674
4675/// [`LINE_SEPARATOR`] as text, for the readers' span builders.
4676const LINE_SEPARATOR_STR: &str = "\u{2028}";
4677
4678/// One span's contribution to the addressable text.
4679///
4680/// An inline object carries no prose but **does** occupy one `U+FFFC` in the
4681/// document (`format_runs_from_spans` mirrors it there), so it has to occupy one
4682/// here too. Leaving it out makes this string shorter than the document it
4683/// claims to be byte-identical to, and every offset past the object — every
4684/// search hit, every comment anchor — lands a character early.
4685fn span_prose(span: &ParsedSpan, out: &mut String) {
4686    if span.image.is_some() || span.footnote_ref.is_some() {
4687        out.push('\u{FFFC}');
4688        return;
4689    }
4690    out.push_str(&span.text);
4691}
4692
4693fn block_prose(block: &ParsedBlock) -> String {
4694    let mut prose = String::new();
4695    for span in &block.spans {
4696        span_prose(span, &mut prose);
4697    }
4698    prose
4699}
4700
4701fn cell_prose(cell: &ParsedTableCell) -> String {
4702    let mut prose = String::new();
4703    for span in &cell.spans {
4704        span_prose(span, &mut prose);
4705    }
4706    prose
4707}
4708
4709/// The prose of a Djot document, with no entities, no store, and no threads.
4710///
4711/// [`parse_djot`] and [`ParsedElement::flatten_to_blocks`] were both already `pub`;
4712/// nothing chained them. This does, and that is the whole trick: it stops at the
4713/// *parse*, where a full import goes on to create a `Block` entity per paragraph, list
4714/// item and table cell, mirror each into the rope, and write its format runs.
4715///
4716/// # Why a project-wide search needs this
4717///
4718/// A host app searching a manuscript must ask "does this scene contain that word" of
4719/// **thousands** of Djot rows, on every keystroke. Doing that by importing each one into
4720/// a document is not a slow feature, it is a frozen app.
4721///
4722/// And searching the Djot *source* instead — the tempting shortcut — is simply wrong:
4723/// the source is markup. `http` matches inside a link's URL, `*` matches an emphasis
4724/// marker, and an occurrence count taken from the source does not agree with what a
4725/// replace re-derives inside the parsed document. Where a replace guards itself with
4726/// "the text moved under me, skip this field", a count taken from markup makes that
4727/// guard fire on perfectly good rows.
4728///
4729/// # The contract
4730///
4731/// The result is **byte-identical to the text the document searches** for the same Djot:
4732/// each block's spans concatenated, blocks joined by a single `\n`, and a table announced
4733/// by its [`TABLE_ANCHOR`] sentinel — exactly the string the import mirrors into the rope
4734/// and that `build_full_text_via_store` recomposes. So an offset found here is an offset
4735/// the document agrees with. On a live document the same string is served by
4736/// `TextDocument::to_addressable_text()`, which reads it off the search plumbing itself.
4737///
4738/// A property in `djot_roundtrip_tests` pins that across the whole generated feature set;
4739/// without it this would be a second, silently-diverging definition of "the text".
4740///
4741/// **Known exception: a footnote *definition*'s body.** This function treats it as out of
4742/// flow and omits it — matching `character_count()`, which does not count it — but the
4743/// live document mirrors its blocks into the rope, so today an in-document search DOES
4744/// run over the note's body and `to_addressable_text()` includes it. The live document
4745/// holds every body after the main text, so this string is the start of that one, and
4746/// every offset of the main text agrees between the two; they differ by the bodies at
4747/// the end. Which side is wrong is an open product question (should search see notes?),
4748/// pinned as current behaviour in
4749/// `addressable_text_tests::footnote_bodies_are_searched_in_the_live_document`.
4750///
4751/// ⚠ It is **not** the same as `TextDocument::to_plain_text()`. That is the human-readable
4752/// *export* — same prose, same order, but with every object anchor omitted, so its offsets
4753/// drift by two characters per preceding table. The authority is what a search sees,
4754/// because that is what a replace edits.
4755/// (`to_plain_text` once also *ordered* a blockquote's prose differently; that was a bug,
4756/// fixed and pinned by `plain_text_order_tests` — see
4757/// `claude_reviews/text-document-plain-text-ordering.md`. The anchors are now the two
4758/// views' only difference.)
4759pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4760    // Deliberately NOT `ParsedElement::flatten_to_blocks`: that helper drops a table's
4761    // anchor and yields only its cells, which would silently shift every offset in a
4762    // document containing a table by the two characters the document actually holds there.
4763    let elements = parse_djot(djot, options);
4764
4765    // Sized from the source: the prose is always shorter than the markup that carries it,
4766    // so this allocates once and never grows.
4767    let mut out = String::with_capacity(djot.len());
4768
4769    // The separator is decided by "is this the first block", NOT by "is the output still
4770    // empty". An EMPTY block (an empty code fence, say) is still a block: the document
4771    // holds an empty line for it, and an emptiness test would swallow both the block and
4772    // its separator, shifting every offset after it by one.
4773    let mut first = true;
4774    let push = |text: &str, out: &mut String, first: &mut bool| {
4775        if *first {
4776            *first = false;
4777        } else {
4778            out.push('\n');
4779        }
4780        out.push_str(text);
4781    };
4782
4783    for element in &elements {
4784        match element {
4785            ParsedElement::Block(block) => {
4786                push(&block_prose(block), &mut out, &mut first);
4787            }
4788            // A note's body is **out of flow**: it is not laid out where its
4789            // definition was written, it is not part of a copied fragment, and
4790            // the document does not count its characters. So it is not part of
4791            // the addressable text either — which is the one thing that has to
4792            // stay true here, since `character_count()` and this string are
4793            // compared directly.
4794            //
4795            // Its blocks do live in the rope, because they have to live
4796            // somewhere; they are simply not addressable prose.
4797            ParsedElement::FootnoteDefinition { .. } => {}
4798            ParsedElement::Table(table) => {
4799                // The import mirrors a table into the rope as a lone `U+FFFC` sentinel
4800                // followed by its cells, one per block (`rope_append_table_anchor`). The
4801                // sentinel occupies a real position in the text the document searches, so
4802                // it has to occupy one here too — otherwise every offset after a table is
4803                // short by two characters, and a snippet taken from this string would be
4804                // sliced in the wrong place.
4805                push(TABLE_ANCHOR, &mut out, &mut first);
4806                // A row shorter than the widest holds empty cells in the document, one
4807                // position each: the import completes it (see `ParsedTable::column_count`).
4808                let columns = table.column_count();
4809                for row in &table.rows {
4810                    for column in 0..columns {
4811                        let prose = row.get(column).map(cell_prose).unwrap_or_default();
4812                        push(&prose, &mut out, &mut first);
4813                    }
4814                }
4815            }
4816        }
4817    }
4818    out
4819}
4820
4821#[cfg(test)]
4822mod html_footnote_tests {
4823    use super::*;
4824
4825    /// The reason this exists at all: a producer cannot smuggle a reference
4826    /// through as text, because the Djot escaper neutralises `[`, `^` and `]`.
4827    /// Pinned here so the escaping rule and this workaround cannot drift apart.
4828    #[test]
4829    fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4830        let blocks = parse_html("<p>The ferry.[^1]</p>");
4831        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4832        assert!(
4833            text.contains("[^1]"),
4834            "the parser keeps it as characters: {text:?}"
4835        );
4836        assert!(
4837            blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4838            "text is text — it must not become a reference by accident"
4839        );
4840    }
4841
4842    #[test]
4843    fn an_attributed_element_becomes_a_real_footnote_reference() {
4844        let blocks = parse_html(
4845            r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4846        );
4847        let refs: Vec<&str> = blocks
4848            .iter()
4849            .flat_map(|b| b.spans.iter())
4850            .filter_map(|s| s.footnote_ref.as_deref())
4851            .collect();
4852        assert_eq!(refs, vec!["3"]);
4853
4854        // And it carries no text of its own, so it cannot be counted as words or
4855        // matched by a search.
4856        let marker = blocks[0]
4857            .spans
4858            .iter()
4859            .find(|s| s.footnote_ref.is_some())
4860            .expect("the reference span");
4861        assert!(marker.text.is_empty(), "{marker:?}");
4862    }
4863
4864    /// The attribute is the contract, not the tag: a producer may use whatever
4865    /// element renders sensibly in a browser.
4866    #[test]
4867    fn any_element_carrying_the_attribute_works() {
4868        for html in [
4869            r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4870            r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4871            r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4872        ] {
4873            let blocks = parse_html(html);
4874            assert_eq!(
4875                blocks
4876                    .iter()
4877                    .flat_map(|b| b.spans.iter())
4878                    .filter_map(|s| s.footnote_ref.as_deref())
4879                    .collect::<Vec<_>>(),
4880                vec!["1"],
4881                "failed for {html}"
4882            );
4883        }
4884    }
4885
4886    #[test]
4887    fn an_empty_or_absent_label_is_not_a_reference() {
4888        for html in [
4889            r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4890            r#"<p>a<sup></sup></p>"#,
4891        ] {
4892            let blocks = parse_html(html);
4893            assert!(
4894                blocks
4895                    .iter()
4896                    .flat_map(|b| b.spans.iter())
4897                    .all(|s| s.footnote_ref.is_none()),
4898                "failed for {html}"
4899            );
4900        }
4901    }
4902}
4903
4904#[cfg(test)]
4905mod html_depth_tests {
4906    //! HTML nested more deeply than the reader follows, and HTML table cells.
4907
4908    use super::*;
4909
4910    /// The text of each block, in order.
4911    fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4912        blocks
4913            .iter()
4914            .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4915            .collect()
4916    }
4917
4918    /// An HTML list nested past the deepest level an insertion keeps comes in with every
4919    /// item, in its order, the deeper ones side by side at that level. The reader stopped
4920    /// descending at its recursion limit and dropped every item below it (the eighty-sixth
4921    /// level on).
4922    #[test]
4923    fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4924        use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4925        let depth = 300;
4926        let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4927        html.push_str(&"</li></ol>".repeat(depth));
4928        let blocks = parse_html(&html);
4929        let items: Vec<(String, u32)> = blocks
4930            .iter()
4931            .filter(|block| block.list_style.is_some())
4932            .map(|block| {
4933                let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4934                (text, block.list_indent)
4935            })
4936            .collect();
4937        let expected: Vec<(String, u32)> = (0..depth)
4938            .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4939            .collect();
4940        assert_eq!(items, expected);
4941    }
4942
4943    /// Text outside any item of a list nested too deeply stays, after the items.
4944    #[test]
4945    fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4946        let depth = 40;
4947        let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4948        html.push_str("<ul>loose words<li>last</li></ul>");
4949        html.push_str(&"</li></ul>".repeat(depth));
4950        let texts = block_texts(&parse_html(&html)).join(" ");
4951        assert!(texts.contains("loose words"), "{texts:?}");
4952        assert!(texts.contains("last"), "{texts:?}");
4953        assert!(texts.contains("item 39"), "{texts:?}");
4954    }
4955
4956    /// Past the depth the reader follows, it used to keep nothing: the words at the bottom
4957    /// of three hundred nested `<div>`s, or quotations, were dropped from the paste.
4958    #[test]
4959    fn html_nested_past_the_reader_depth_keeps_its_words() {
4960        for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4961            let html = format!(
4962                "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4963                open.repeat(300),
4964                close.repeat(300)
4965            );
4966            let texts = block_texts(&parse_html(&html)).join(" ");
4967            let words: Vec<&str> = texts.split_whitespace().collect();
4968            assert_eq!(
4969                words,
4970                ["Top.", "Deep", "words", "here.", "End."],
4971                "for {open}"
4972            );
4973        }
4974    }
4975
4976    /// The same inside a table cell, read as one paragraph.
4977    #[test]
4978    fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4979        let html = format!(
4980            "<table><tr><td>{}deep words{}</td></tr></table>",
4981            "<span>".repeat(300),
4982            "</span>".repeat(300)
4983        );
4984        let elements = parse_html_elements(&html);
4985        let Some(ParsedElement::Table(table)) = elements.first() else {
4986            panic!("a table: {elements:?}");
4987        };
4988        let text: String = table.rows[0][0]
4989            .spans
4990            .iter()
4991            .map(|span| span.text.as_str())
4992            .collect();
4993        assert_eq!(text, "deep words");
4994    }
4995
4996    /// A cell is one paragraph, so its line breaks and paragraphs are spaces between
4997    /// words. The HTML writer puts a cell's two paragraphs out as `One<br/>Two`, and the
4998    /// reader read them back as the one word `OneTwo`.
4999    #[test]
5000    fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
5001        for cell in [
5002            "One<br/>Two",
5003            "<p>One</p><p>Two</p>",
5004            "<ul><li>One</li><li>Two</li></ul>",
5005        ] {
5006            let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
5007            let Some(ParsedElement::Table(table)) = elements.first() else {
5008                panic!("a table: {elements:?}");
5009            };
5010            let text: String = table.rows[0][0]
5011                .spans
5012                .iter()
5013                .map(|span| span.text.as_str())
5014                .collect();
5015            assert_eq!(text, "One Two", "for {cell}");
5016        }
5017    }
5018}