Skip to main content

common/parser_tools/
content_parser.rs

1use crate::entities::{Alignment, ListStyle, MarkerType, SemanticRole, TextDirection};
2use crate::parser_tools::djot_options::DjotImportOptions;
3
4/// An inline image recovered by a parser, before it becomes an `ImageAnchor`.
5///
6/// `src` is whatever the source document pointed at — a relative path, a bare
7/// name, a URL. Parsers do not resolve it; that is the embedding application's
8/// job, and this crate never touches the filesystem.
9#[derive(Debug, Clone, Default, PartialEq, Eq)]
10pub struct ParsedImage {
11    pub src: String,
12    pub alt: String,
13    /// Display size in pixels, `0` when the source did not state one.
14    ///
15    /// Djot and HTML both carry these as attributes (`{width=800}`,
16    /// `<img width=…>`); Markdown has no syntax for them, so a Markdown import
17    /// leaves both zero and the caller supplies intrinsic dimensions.
18    pub width: i64,
19    pub height: i64,
20}
21
22/// A parsed inline span with formatting info.
23///
24/// A span carries *either* text or an image, never both: an image span's `text`
25/// is empty and its description lives in [`ParsedImage::alt`], so alt text
26/// cannot leak into the block's prose (and therefore cannot be counted as
27/// manuscript words or matched by a search).
28#[derive(Debug, Clone, Default)]
29pub struct ParsedSpan {
30    pub text: String,
31    pub bold: bool,
32    pub italic: bool,
33    pub underline: bool,
34    pub strikeout: bool,
35    pub code: bool,
36    /// Superscript (djot `^x^`). Maps to `CharVerticalAlignment::SuperScript`.
37    pub superscript: bool,
38    /// Subscript (djot `~x~`). Maps to `CharVerticalAlignment::SubScript`.
39    pub subscript: bool,
40    pub link_href: Option<String>,
41    /// Set when this span is an inline image rather than text.
42    pub image: Option<ParsedImage>,
43    /// Set when this span is a footnote *reference* rather than text — the
44    /// label naming the note, never the number a reader sees.
45    pub footnote_ref: Option<String>,
46}
47
48/// A parsed table cell containing inline spans.
49#[derive(Debug, Clone)]
50pub struct ParsedTableCell {
51    pub spans: Vec<ParsedSpan>,
52}
53
54/// A parsed table extracted from markdown or HTML.
55#[derive(Debug, Clone)]
56pub struct ParsedTable {
57    /// Number of header rows (typically 1 for markdown tables).
58    pub header_rows: usize,
59    /// All rows (header + body), each containing cells with their inline spans.
60    pub rows: Vec<Vec<ParsedTableCell>>,
61    /// Blockquote nesting depth at the point the table appeared
62    /// (0 = not inside a blockquote), mirroring `ParsedBlock::blockquote_depth`.
63    pub blockquote_depth: u32,
64    /// How many of those quotations open at the table, mirroring
65    /// `ParsedBlock::blockquote_opens`.
66    pub blockquote_opens: u32,
67}
68
69impl ParsedTable {
70    /// How many columns the table needs to hold every cell it has: as many as its widest
71    /// row has cells. A row shorter than that is completed with empty cells by whoever
72    /// builds the table.
73    ///
74    /// The first row does not say it. The rows of a Djot table differ in length when their
75    /// source does, and an HTML table's first row is often one heading cell spanning every
76    /// column: a table sized from its first row kept the cells of a longer row in the model,
77    /// where they showed, while every export writes a row up to the table's width only, so
78    /// the first save dropped them.
79    pub fn column_count(&self) -> usize {
80        self.rows.iter().map(Vec::len).max().unwrap_or(0)
81    }
82}
83
84/// How many cells a table may reach once its short rows are completed (see
85/// [`ParsedTable::column_count`]), however few it has of its own: a table this size loads
86/// in a moment.
87const SQUARED_CELLS_ALWAYS: usize = 4096;
88
89/// How many times the cells it has of its own a larger table may reach once its short
90/// rows are completed.
91///
92/// A table whose rows differ in length because its source wrote them so, a wide heading
93/// row over rows of one cell or a few, grows to about as many times its own cells as it has
94/// columns: a dozen or so for a table a writer makes. The tables that made hundreds of
95/// thousands of cells out of a few kilobytes grow hundreds of times. At four times, a table
96/// of ten columns heading five hundred rows of one cell was read as paragraphs, its grid
97/// lost for a table that loads in a few milliseconds.
98const SQUARED_CELLS_PER_OWN_CELL: usize = 16;
99
100/// Push `table`, which has `own_cells` cells of its own in its source, onto `elements`: as
101/// a table, or as the paragraphs of its cells when completing its short rows would make it
102/// many times that.
103///
104/// Every row of a loaded table is completed to the widest one. A few kilobytes of a table
105/// whose first row is wide and whose other rows hold one cell each, or of HTML cells each
106/// spanning a thousand columns, made hundreds of thousands of empty cells, and held a load
107/// or a paste for seconds. Such a table is read as the paragraphs of its cells that hold
108/// anything, in reading order, at the table's quotation depth: every word stays, the grid
109/// goes.
110fn push_table(elements: &mut Vec<ParsedElement>, table: ParsedTable, own_cells: usize) {
111    let squared = table.rows.len().saturating_mul(table.column_count());
112    let allowed = SQUARED_CELLS_ALWAYS.max(own_cells.saturating_mul(SQUARED_CELLS_PER_OWN_CELL));
113    if squared <= allowed {
114        elements.push(ParsedElement::Table(table));
115        return;
116    }
117    elements.extend(
118        ParsedElement::flatten_to_blocks(vec![ParsedElement::Table(table)])
119            .into_iter()
120            .filter(|block| spans_carry_content(&block.spans))
121            .map(ParsedElement::Block),
122    );
123}
124
125/// Record that a quotation opened at the first block or table of `elements` from `first`
126/// on: the quotation that began where `elements` stood at `first` and closes now. Nothing
127/// when the quotation holds none.
128fn mark_quotation_opened(elements: &mut [ParsedElement], first: usize) {
129    let Some(rest) = elements.get_mut(first..) else {
130        return;
131    };
132    for element in rest {
133        match element {
134            ParsedElement::Block(block) => {
135                block.blockquote_opens += 1;
136                return;
137            }
138            ParsedElement::Table(table) => {
139                table.blockquote_opens += 1;
140                return;
141            }
142            ParsedElement::FootnoteDefinition { .. } => {}
143        }
144    }
145}
146
147/// The cells a table's rows hold, for [`push_table`].
148fn cells_in_rows(rows: &[Vec<ParsedTableCell>]) -> usize {
149    rows.iter().map(Vec::len).sum()
150}
151
152/// A parsed element: either a block or a table.
153#[derive(Debug, Clone)]
154pub enum ParsedElement {
155    Block(ParsedBlock),
156    Table(ParsedTable),
157    /// A footnote definition: the label its references name, and the blocks
158    /// making up its body.
159    ///
160    /// Separate from `Block` because a definition is not part of the flow it was
161    /// written in — it belongs wherever the output format puts notes, which the
162    /// importer expresses by giving it a detached frame of its own.
163    FootnoteDefinition {
164        label: String,
165        blocks: Vec<ParsedBlock>,
166    },
167}
168
169impl ParsedElement {
170    /// Extract blocks, flattening tables into one block per cell.
171    /// Use when table structure is not needed.
172    pub fn flatten_to_blocks(elements: Vec<ParsedElement>) -> Vec<ParsedBlock> {
173        let mut blocks = Vec::new();
174        for elem in elements {
175            match elem {
176                ParsedElement::Block(b) => blocks.push(b),
177                // A definition is not part of the flow it was written in — it
178                // belongs wherever the format puts notes. Flattening it in
179                // would splice a note's body into the prose at the point the
180                // definition happened to be typed.
181                ParsedElement::FootnoteDefinition { .. } => {}
182                ParsedElement::Table(t) => {
183                    for row in t.rows {
184                        for cell in row {
185                            blocks.push(ParsedBlock {
186                                spans: cell.spans,
187                                heading_level: None,
188                                list_style: None,
189                                list_indent: 0,
190                                list_prefix: String::new(),
191                                list_suffix: String::new(),
192                                list_start: None,
193                                marker: None,
194                                is_code_block: false,
195                                code_language: None,
196                                blockquote_depth: t.blockquote_depth,
197                                blockquote_opens: 0,
198                                line_height: None,
199                                non_breakable_lines: None,
200                                page_break_before: None,
201                                direction: None,
202                                background_color: None,
203                                alignment: None,
204                                top_margin: None,
205                                text_indent: None,
206                                semantic_role: None,
207                            });
208                        }
209                    }
210                }
211            }
212        }
213        if blocks.is_empty() {
214            blocks.push(ParsedBlock {
215                spans: vec![ParsedSpan {
216                    text: String::new(),
217                    ..Default::default()
218                }],
219                heading_level: None,
220                list_style: None,
221                list_indent: 0,
222                list_prefix: String::new(),
223                list_suffix: String::new(),
224                list_start: None,
225                marker: None,
226                is_code_block: false,
227                code_language: None,
228                blockquote_depth: 0,
229                blockquote_opens: 0,
230                line_height: None,
231                non_breakable_lines: None,
232                page_break_before: None,
233                direction: None,
234                background_color: None,
235                alignment: None,
236                top_margin: None,
237                text_indent: None,
238                semantic_role: None,
239            });
240        }
241        blocks
242    }
243}
244
245/// A parsed block (paragraph, heading, list item, code block)
246///
247/// `Default` is an unformatted, empty paragraph — the shape a caller wants when
248/// it has prose and no structure to give it. Deriving it also means a field
249/// added later reaches those callers as "absent" rather than as a compile error
250/// they would fix by guessing a value.
251#[derive(Debug, Clone, Default)]
252pub struct ParsedBlock {
253    pub spans: Vec<ParsedSpan>,
254    pub heading_level: Option<i64>,
255    pub list_style: Option<ListStyle>,
256    pub list_indent: u32,
257    /// Ordered-list delimiter prefix (e.g. `"("` for djot `(1)` lists; empty
258    /// otherwise). Stored on the `List` entity for round-trip fidelity.
259    pub list_prefix: String,
260    /// Ordered-list delimiter suffix (`"."` for `1.`, `")"` for `1)`/`(1)`;
261    /// empty for unordered lists).
262    pub list_suffix: String,
263    /// The number the first item of the list this item belongs to wears, when it is not 1
264    /// (Djot and Markdown `3.`, HTML `<ol start="3">`). Every item of a list carries it.
265    /// Stored on the `List` entity.
266    pub list_start: Option<i64>,
267    /// Task-list checkbox marker (djot `- [ ]` / `- [x]`). Maps to
268    /// `Block.fmt_marker`. `None` for non-task blocks.
269    pub marker: Option<MarkerType>,
270    pub is_code_block: bool,
271    pub code_language: Option<String>,
272    pub blockquote_depth: u32,
273    /// How many of the quotations the block stands in open at it, the innermost ones:
274    /// quotations as deep as the element's before it but not the same ones. `0` when the
275    /// block goes on in the quotations the element before it stands in, as deep as it
276    /// stands. Two quotations one after the other (Djot and Markdown `> a` and `> b` with a
277    /// blank line between, HTML `<blockquote>` twice) were read as one, since the depth
278    /// alone does not tell them apart.
279    pub blockquote_opens: u32,
280    pub line_height: Option<i64>,
281    pub non_breakable_lines: Option<bool>,
282    /// Start this block on a new page (djot `{page_break_before=true}`). Maps to
283    /// `Block.fmt_page_break_before`. `None` when absent.
284    pub page_break_before: Option<bool>,
285    pub direction: Option<TextDirection>,
286    pub background_color: Option<String>,
287    /// Paragraph alignment (djot `{alignment=left|right|center|justify}`). Maps
288    /// to `Block.fmt_alignment`. `None` when no alignment attribute is present.
289    pub alignment: Option<Alignment>,
290    /// This block's own space-above (djot `{top_margin=<int>}`). Maps to
291    /// `Block.fmt_top_margin` and overrides the document-wide paragraph
292    /// spacing for this block alone. `None` when absent.
293    pub top_margin: Option<i64>,
294    /// This block's own first-line indent (djot `{text_indent=<int>}`). Maps to
295    /// `Block.fmt_text_indent` and overrides the document-wide first-line
296    /// indent for this block alone. `None` when absent.
297    pub text_indent: Option<i64>,
298    /// The enclosing blockquote's semantic role (djot `{semantic_role=epigraph}`),
299    /// written on the quote's first block because block attributes are the only channel
300    /// djot offers. The importer lifts it onto the `Frame`, where it belongs.
301    pub semantic_role: Option<SemanticRole>,
302}
303
304impl ParsedBlock {
305    /// Returns `true` when this block carries no block-level formatting,
306    /// meaning its content is purely inline.
307    pub fn is_inline_only(&self) -> bool {
308        self.heading_level.is_none()
309            && self.list_style.is_none()
310            && !self.is_code_block
311            && self.blockquote_depth == 0
312            && self.line_height.is_none()
313            && self.non_breakable_lines.is_none()
314            && self.page_break_before.is_none()
315            && self.direction.is_none()
316            && self.background_color.is_none()
317            && self.alignment.is_none()
318            && self.top_margin.is_none()
319            && self.text_indent.is_none()
320    }
321}
322
323// ─── Markdown parsing ────────────────────────────────────────────────
324
325/// Labels referenced as `[^label]` in `markdown` with no `[^label]: ...`
326/// definition anywhere in the document.
327///
328/// pulldown-cmark's default footnote mode (`Options::ENABLE_FOOTNOTES`, i.e.
329/// GitHub's syntax) only emits `Event::FootnoteReference` for a label some
330/// `Tag::FootnoteDefinition` actually defines (verified against 0.13) — an
331/// undefined reference silently decomposes into three ordinary `Text` events
332/// (`"["`, `"^label"`, `"]"`) instead, indistinguishable from a reader having
333/// typed literal brackets. `Options::ENABLE_OLD_FOOTNOTES` recognises it, but
334/// as `parse_markdown`'s doc comment explains, trades away multi-paragraph
335/// definition bodies to get there. Neither flag alone gives both, so
336/// `parse_markdown` hands pulldown-cmark a synthetic empty definition for
337/// every such label instead — real enough for it to recognise the reference,
338/// thrown away again before `elements` is returned.
339///
340/// Finding "every such label" is two parts: which labels pulldown-cmark's own
341/// block scanner (a first pass, at the caller's `options`) already recognises
342/// as truly defined, and which `[^…]`-shaped spans exist in the raw text at
343/// all. The second part is a plain scan, not a parser — it can over-match
344/// (inside a code span, inside a fenced block, a coincidental `[^x]:` in the
345/// middle of a sentence that real block parsing would never treat as a
346/// definition), but over-matching only ever produces an unused synthetic
347/// definition: pulldown-cmark's real, correct parse of the augmented text is
348/// what decides whether any span actually becomes a reference, and
349/// `parse_markdown` drops every synthesized definition regardless of whether
350/// that happened.
351fn dangling_footnote_labels(
352    markdown: &str,
353    options: pulldown_cmark::Options,
354) -> std::collections::BTreeSet<String> {
355    use pulldown_cmark::{Event, Parser, Tag};
356
357    let mut defined: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
358    for event in Parser::new_ext(markdown, options) {
359        if let Event::Start(Tag::FootnoteDefinition(label)) = event {
360            defined.insert(label.to_string());
361        }
362    }
363
364    let mut referenced: std::collections::BTreeSet<String> = std::collections::BTreeSet::new();
365    let bytes = markdown.as_bytes();
366    let mut search_from = 0usize;
367    while let Some(rel) = markdown[search_from..].find("[^") {
368        let open = search_from + rel;
369        let label_start = open + 2;
370        let Some(close_rel) = markdown[label_start..].find(']') else {
371            break;
372        };
373        let close = label_start + close_rel;
374        let label = &markdown[label_start..close];
375        // `]:` immediately after is definition syntax, not a reference —
376        // pulldown-cmark's own scan above already supplied the ground truth
377        // for which labels are truly defined; this only needs to avoid
378        // counting a definition's own label as a "reference" candidate.
379        let looks_like_definition = bytes.get(close + 1) == Some(&b':');
380        if !label.is_empty() && !looks_like_definition && !label.chars().any(char::is_whitespace) {
381            referenced.insert(label.to_string());
382        }
383        search_from = close + 1;
384    }
385
386    referenced.difference(&defined).cloned().collect()
387}
388
389/// Read `markdown` into blocks and tables.
390///
391/// The parser reads nested containers without recursing, whatever their depth. The
392/// quotations the text holds are kept within
393/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
394/// `quote_depth`): the writers walk one call deeper for each, and a text nested 500
395/// quotations deep loaded, then aborted the process at its first save.
396pub fn parse_markdown(markdown: &str) -> Vec<ParsedElement> {
397    use pulldown_cmark::{Event, Options, Parser, Tag, TagEnd};
398
399    // ENABLE_FOOTNOTES is required for pulldown-cmark to parse `[^label]`
400    // references and `[^label]: body` definitions at all — without it both
401    // fall through as plain link-reference-shaped text. Deliberately NOT
402    // `ENABLE_OLD_FOOTNOTES` too: that variant makes a reference with no
403    // definition survive (see `dangling_footnote_labels`), but in exchange
404    // breaks multi-paragraph definition bodies — a blank line before an
405    // indented continuation escapes the definition and becomes a sibling
406    // code block, which is exactly the shape this crate's own Markdown
407    // exporter writes for a note with more than one paragraph. Getting both
408    // is `dangling_footnote_labels`'s job, not an `Options` flag's.
409    let options = Options::ENABLE_STRIKETHROUGH
410        | Options::ENABLE_TABLES
411        | Options::ENABLE_TASKLISTS
412        | Options::ENABLE_FOOTNOTES;
413
414    // See `dangling_footnote_labels`: pulldown-cmark's GFM-style footnotes
415    // only recognise `[^label]` as a reference when some definition exists
416    // for it anywhere in the document, so an undefined one — the normal
417    // state for a host that owns note bodies itself — has to be given a
418    // throwaway definition before parsing, or it silently becomes the
419    // literal text "[^label]" instead of surviving as a reference.
420    let dangling = dangling_footnote_labels(markdown, options);
421    let augmented_owner;
422    let source: &str = if dangling.is_empty() {
423        markdown
424    } else {
425        augmented_owner = dangling
426            .iter()
427            .fold(markdown.to_string(), |mut acc, label| {
428                acc.push_str("\n\n[^");
429                acc.push_str(label);
430                acc.push_str("]:\n");
431                acc
432            });
433        &augmented_owner
434    };
435    let parser = Parser::new_ext(source, options);
436
437    let mut elements: Vec<ParsedElement> = Vec::new();
438    let mut current_spans: Vec<ParsedSpan> = Vec::new();
439    let mut current_heading: Option<i64> = None;
440    let mut current_list_style: Option<ListStyle> = None;
441    let mut current_list_start: Option<i64> = None;
442    let mut is_code_block = false;
443    let mut code_language: Option<String> = None;
444    let mut blockquote_depth: u32 = 0;
445    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
446    let mut quotations_open: Vec<usize> = Vec::new();
447    let mut in_block = false;
448
449    // Formatting state stack
450    let mut bold = false;
451    let mut italic = false;
452    let mut strikeout = false;
453    let mut link_href: Option<String> = None;
454    // Set between an image's Start and End; its alt text arrives as Text events.
455    let mut pending_image: Option<ParsedImage> = None;
456
457    // The label and element index of the footnote definition currently open,
458    // if any. Mirrors `parse_djot`'s `footnote_open`: a definition's body is
459    // ordinary block content, parsed by the same machinery as everything
460    // else, then lifted back out at `End` into its own top-level element so
461    // it never becomes part of the flow it was written in. CommonMark
462    // footnote definitions cannot nest, so one slot suffices.
463    let mut footnote_open: Option<(String, usize)> = None;
464
465    // List style stack for nested lists (also tracks nesting depth), and the start of each
466    // of those lists beside it.
467    let mut list_stack: Vec<Option<ListStyle>> = Vec::new();
468    let mut list_start_stack: Vec<Option<i64>> = Vec::new();
469    let mut current_list_indent: u32 = 0;
470
471    // Table tracking state
472    let mut in_table = false;
473    let mut in_table_head = false;
474    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
475    let mut current_row_cells: Vec<ParsedTableCell> = Vec::new();
476    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
477    let mut table_header_rows: usize = 0;
478
479    for event in parser {
480        match event {
481            Event::Start(Tag::Paragraph) => {
482                in_block = true;
483                current_heading = None;
484                is_code_block = false;
485            }
486            Event::End(TagEnd::Paragraph) => {
487                if !current_spans.is_empty() || in_block {
488                    elements.push(ParsedElement::Block(ParsedBlock {
489                        spans: std::mem::take(&mut current_spans),
490                        heading_level: current_heading.take(),
491                        list_style: current_list_style.clone(),
492                        list_indent: current_list_indent,
493                        list_prefix: String::new(),
494                        list_suffix: String::new(),
495                        list_start: current_list_start,
496                        marker: None,
497                        is_code_block: false,
498                        code_language: None,
499                        blockquote_depth,
500                        blockquote_opens: 0,
501                        line_height: None,
502                        non_breakable_lines: None,
503                        page_break_before: None,
504                        direction: None,
505                        background_color: None,
506                        alignment: None,
507                        top_margin: None,
508                        text_indent: None,
509                        semantic_role: None,
510                    }));
511                }
512                in_block = false;
513                current_list_style = None;
514                current_list_start = None;
515            }
516            Event::Start(Tag::Heading { level, .. }) => {
517                in_block = true;
518                current_heading = Some(heading_level_to_i64(level));
519                is_code_block = false;
520            }
521            Event::End(TagEnd::Heading(_)) => {
522                elements.push(ParsedElement::Block(ParsedBlock {
523                    spans: std::mem::take(&mut current_spans),
524                    heading_level: current_heading.take(),
525                    list_style: None,
526                    list_indent: 0,
527                    list_prefix: String::new(),
528                    list_suffix: String::new(),
529                    list_start: None,
530                    marker: None,
531                    is_code_block: false,
532                    code_language: None,
533                    blockquote_depth,
534                    blockquote_opens: 0,
535                    line_height: None,
536                    non_breakable_lines: None,
537                    page_break_before: None,
538                    direction: None,
539                    background_color: None,
540                    alignment: None,
541                    top_margin: None,
542                    text_indent: None,
543                    semantic_role: None,
544                }));
545                in_block = false;
546            }
547            Event::Start(Tag::List(ordered)) => {
548                let style = if ordered.is_some() {
549                    Some(ListStyle::Decimal)
550                } else {
551                    Some(ListStyle::Disc)
552                };
553                list_stack.push(style);
554                list_start_stack.push(ordered.and_then(list_start_of));
555            }
556            Event::End(TagEnd::List(_)) => {
557                list_stack.pop();
558                list_start_stack.pop();
559            }
560            Event::Start(Tag::Item) => {
561                // Flush any accumulated spans from the parent item before
562                // starting a child item in a tight list
563                if !current_spans.is_empty() {
564                    elements.push(ParsedElement::Block(ParsedBlock {
565                        spans: std::mem::take(&mut current_spans),
566                        heading_level: None,
567                        list_style: current_list_style.clone(),
568                        list_indent: current_list_indent,
569                        list_prefix: String::new(),
570                        list_suffix: String::new(),
571                        list_start: current_list_start,
572                        marker: None,
573                        is_code_block: false,
574                        code_language: None,
575                        blockquote_depth,
576                        blockquote_opens: 0,
577                        line_height: None,
578                        non_breakable_lines: None,
579                        page_break_before: None,
580                        direction: None,
581                        background_color: None,
582                        alignment: None,
583                        top_margin: None,
584                        text_indent: None,
585                        semantic_role: None,
586                    }));
587                }
588                in_block = true;
589                current_list_style = list_stack.last().cloned().flatten();
590                current_list_start = list_start_stack.last().copied().flatten();
591                current_list_indent = if list_stack.is_empty() {
592                    0
593                } else {
594                    (list_stack.len() - 1) as u32
595                };
596            }
597            Event::End(TagEnd::Item) => {
598                // The paragraph inside the item will have already been flushed,
599                // but if there was no inner paragraph (tight list), flush now.
600                if !current_spans.is_empty() {
601                    elements.push(ParsedElement::Block(ParsedBlock {
602                        spans: std::mem::take(&mut current_spans),
603                        heading_level: None,
604                        list_style: current_list_style.clone(),
605                        list_indent: current_list_indent,
606                        list_prefix: String::new(),
607                        list_suffix: String::new(),
608                        list_start: current_list_start,
609                        marker: None,
610                        is_code_block: false,
611                        code_language: None,
612                        blockquote_depth,
613                        blockquote_opens: 0,
614                        line_height: None,
615                        non_breakable_lines: None,
616                        page_break_before: None,
617                        direction: None,
618                        background_color: None,
619                        alignment: None,
620                        top_margin: None,
621                        text_indent: None,
622                        semantic_role: None,
623                    }));
624                }
625                in_block = false;
626                current_list_style = None;
627                current_list_start = None;
628            }
629            Event::Start(Tag::CodeBlock(kind)) => {
630                in_block = true;
631                is_code_block = true;
632                code_language = match &kind {
633                    pulldown_cmark::CodeBlockKind::Fenced(lang) if !lang.is_empty() => {
634                        Some(lang.to_string())
635                    }
636                    _ => None,
637                };
638            }
639            Event::End(TagEnd::CodeBlock) => {
640                // pulldown-cmark appends a trailing '\n' to code block text — strip it
641                if let Some(last) = current_spans.last_mut()
642                    && last.text.ends_with('\n')
643                {
644                    last.text.truncate(last.text.len() - 1);
645                }
646                elements.push(ParsedElement::Block(ParsedBlock {
647                    spans: std::mem::take(&mut current_spans),
648                    heading_level: None,
649                    list_style: None,
650                    list_indent: 0,
651                    list_prefix: String::new(),
652                    list_suffix: String::new(),
653                    list_start: None,
654                    marker: None,
655                    is_code_block: true,
656                    code_language: code_language.take(),
657                    blockquote_depth,
658                    blockquote_opens: 0,
659                    line_height: None,
660                    non_breakable_lines: None,
661                    page_break_before: None,
662                    direction: None,
663                    background_color: None,
664                    alignment: None,
665                    top_margin: None,
666                    text_indent: None,
667                    semantic_role: None,
668                }));
669                in_block = false;
670                is_code_block = false;
671            }
672            // ─── Table events ───────────────────────────────────────
673            Event::Start(Tag::Table(_)) => {
674                in_table = true;
675                in_table_head = false;
676                table_rows.clear();
677                current_row_cells.clear();
678                current_cell_spans.clear();
679                table_header_rows = 0;
680            }
681            Event::End(TagEnd::Table) => {
682                let own_cells = cells_in_rows(&table_rows);
683                push_table(
684                    &mut elements,
685                    ParsedTable {
686                        header_rows: table_header_rows,
687                        rows: std::mem::take(&mut table_rows),
688                        blockquote_depth,
689                        blockquote_opens: 0,
690                    },
691                    own_cells,
692                );
693                in_table = false;
694            }
695            Event::Start(Tag::TableHead) => {
696                in_table_head = true;
697                current_row_cells.clear();
698            }
699            Event::End(TagEnd::TableHead) => {
700                // Flush the header row
701                table_rows.push(std::mem::take(&mut current_row_cells));
702                table_header_rows += 1;
703                in_table_head = false;
704            }
705            Event::Start(Tag::TableRow) => {
706                current_row_cells.clear();
707            }
708            Event::End(TagEnd::TableRow) if !in_table_head => {
709                // Body rows only — header row is flushed in End(TableHead)
710                table_rows.push(std::mem::take(&mut current_row_cells));
711            }
712            Event::Start(Tag::TableCell) => {
713                current_cell_spans.clear();
714            }
715            Event::End(TagEnd::TableCell) => {
716                current_row_cells.push(ParsedTableCell {
717                    spans: std::mem::take(&mut current_cell_spans),
718                });
719            }
720            // ─── Inline formatting ──────────────────────────────────
721            Event::Start(Tag::Emphasis) => {
722                italic = true;
723            }
724            Event::End(TagEnd::Emphasis) => {
725                italic = false;
726            }
727            Event::Start(Tag::Strong) => {
728                bold = true;
729            }
730            Event::End(TagEnd::Strong) => {
731                bold = false;
732            }
733            Event::Start(Tag::Strikethrough) => {
734                strikeout = true;
735            }
736            Event::End(TagEnd::Strikethrough) => {
737                strikeout = false;
738            }
739            Event::Start(Tag::Link { dest_url, .. }) => {
740                link_href = Some(dest_url.to_string());
741            }
742            Event::End(TagEnd::Link) => {
743                link_href = None;
744            }
745            // Markdown has no syntax for display size, so width/height stay 0
746            // and the caller supplies the image's intrinsic dimensions.
747            Event::Start(Tag::Image { dest_url, .. }) => {
748                pending_image = Some(ParsedImage {
749                    src: dest_url.to_string(),
750                    alt: String::new(),
751                    width: 0,
752                    height: 0,
753                });
754            }
755            Event::End(TagEnd::Image) => {
756                if let Some(image) = pending_image.take() {
757                    let span = ParsedSpan {
758                        text: String::new(),
759                        bold,
760                        italic,
761                        underline: false,
762                        strikeout,
763                        code: false,
764                        superscript: false,
765                        subscript: false,
766                        link_href: link_href.clone(),
767                        image: Some(image),
768                        footnote_ref: None,
769                    };
770                    if in_table {
771                        current_cell_spans.push(span);
772                    } else {
773                        if !in_block {
774                            in_block = true;
775                        }
776                        current_spans.push(span);
777                    }
778                }
779            }
780            Event::Text(text) => {
781                // Inside an image this is its alt text, which pulldown-cmark
782                // emits as an ordinary Text event. Without this guard it fell
783                // through and landed in the paragraph as prose — the image was
784                // dropped and its description silently became manuscript text.
785                if let Some(img) = pending_image.as_mut() {
786                    img.alt.push_str(&text);
787                    continue;
788                }
789                let span = ParsedSpan {
790                    text: text.to_string(),
791                    bold,
792                    italic,
793                    underline: false,
794                    strikeout,
795                    code: is_code_block,
796                    superscript: false,
797                    subscript: false,
798                    link_href: link_href.clone(),
799                    image: None,
800                    footnote_ref: None,
801                };
802                if in_table {
803                    current_cell_spans.push(span);
804                } else {
805                    if !in_block {
806                        in_block = true;
807                    }
808                    current_spans.push(span);
809                }
810            }
811            Event::Code(text) => {
812                let span = ParsedSpan {
813                    text: text.to_string(),
814                    bold,
815                    italic,
816                    underline: false,
817                    strikeout,
818                    code: true,
819                    superscript: false,
820                    subscript: false,
821                    link_href: link_href.clone(),
822                    image: None,
823                    footnote_ref: None,
824                };
825                if in_table {
826                    current_cell_spans.push(span);
827                } else {
828                    if !in_block {
829                        in_block = true;
830                    }
831                    current_spans.push(span);
832                }
833            }
834            Event::SoftBreak => {
835                let span = ParsedSpan {
836                    text: " ".to_string(),
837                    bold,
838                    italic,
839                    underline: false,
840                    strikeout,
841                    code: false,
842                    superscript: false,
843                    subscript: false,
844                    link_href: link_href.clone(),
845                    image: None,
846                    footnote_ref: None,
847                };
848                if in_table {
849                    current_cell_spans.push(span);
850                } else {
851                    current_spans.push(span);
852                }
853            }
854            Event::HardBreak if !current_spans.is_empty() || in_block => {
855                // Finalize current block
856                elements.push(ParsedElement::Block(ParsedBlock {
857                    spans: std::mem::take(&mut current_spans),
858                    heading_level: current_heading.take(),
859                    list_style: current_list_style.clone(),
860                    list_indent: current_list_indent,
861                    list_prefix: String::new(),
862                    list_suffix: String::new(),
863                    list_start: current_list_start,
864                    marker: None,
865                    is_code_block,
866                    code_language: code_language.clone(),
867                    blockquote_depth,
868                    blockquote_opens: 0,
869                    line_height: None,
870                    non_breakable_lines: None,
871                    page_break_before: None,
872                    direction: None,
873                    background_color: None,
874                    alignment: None,
875                    top_margin: None,
876                    text_indent: None,
877                    semantic_role: None,
878                }));
879            }
880            Event::Start(Tag::BlockQuote(_)) => {
881                blockquote_depth += 1;
882                quotations_open.push(elements.len());
883            }
884            Event::End(TagEnd::BlockQuote(_)) => {
885                blockquote_depth = blockquote_depth.saturating_sub(1);
886                if let Some(first) = quotations_open.pop() {
887                    mark_quotation_opened(&mut elements, first);
888                }
889            }
890            // ── Footnote definitions ──
891            //
892            // `[^label]: body` opens a block-level container; its body is
893            // ordinary paragraphs/lists/etc., pushed onto `elements` by the
894            // ordinary machinery above. `End` drains everything pushed since
895            // `Start` back out into one `FootnoteDefinition`, exactly as
896            // `parse_djot` does for the same `[^label]: body` syntax.
897            // Flushing any spans left open by an interrupted block first
898            // matches every other container boundary in this parser
899            // (List/Item/BlockQuote).
900            Event::Start(Tag::FootnoteDefinition(label)) => {
901                if !current_spans.is_empty() {
902                    elements.push(ParsedElement::Block(ParsedBlock {
903                        spans: std::mem::take(&mut current_spans),
904                        heading_level: current_heading.take(),
905                        list_style: current_list_style.clone(),
906                        list_indent: current_list_indent,
907                        list_prefix: String::new(),
908                        list_suffix: String::new(),
909                        list_start: current_list_start,
910                        marker: None,
911                        is_code_block: false,
912                        code_language: None,
913                        blockquote_depth,
914                        blockquote_opens: 0,
915                        line_height: None,
916                        non_breakable_lines: None,
917                        page_break_before: None,
918                        direction: None,
919                        background_color: None,
920                        alignment: None,
921                        top_margin: None,
922                        text_indent: None,
923                        semantic_role: None,
924                    }));
925                }
926                footnote_open = Some((label.to_string(), elements.len()));
927            }
928            Event::End(TagEnd::FootnoteDefinition) => {
929                if !current_spans.is_empty() {
930                    elements.push(ParsedElement::Block(ParsedBlock {
931                        spans: std::mem::take(&mut current_spans),
932                        heading_level: current_heading.take(),
933                        list_style: current_list_style.clone(),
934                        list_indent: current_list_indent,
935                        list_prefix: String::new(),
936                        list_suffix: String::new(),
937                        list_start: current_list_start,
938                        marker: None,
939                        is_code_block: false,
940                        code_language: None,
941                        blockquote_depth,
942                        blockquote_opens: 0,
943                        line_height: None,
944                        non_breakable_lines: None,
945                        page_break_before: None,
946                        direction: None,
947                        background_color: None,
948                        alignment: None,
949                        top_margin: None,
950                        text_indent: None,
951                        semantic_role: None,
952                    }));
953                }
954                if let Some((label, start)) = footnote_open.take() {
955                    let blocks: Vec<ParsedBlock> = elements
956                        .drain(start..)
957                        .filter_map(|e| match e {
958                            ParsedElement::Block(b) => Some(b),
959                            // A table inside a footnote is not representable
960                            // as note content, same limitation as djot.
961                            _ => None,
962                        })
963                        .collect();
964                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
965                }
966            }
967            // A footnote reference. pulldown-cmark emits this whether or not
968            // a matching `[^label]:` definition exists anywhere in the
969            // document — a dangling reference (the normal state for a host
970            // that owns note bodies itself) must survive just the same,
971            // mirroring `parse_djot`'s `E::FootnoteReference` handling.
972            Event::FootnoteReference(label) => {
973                let span = ParsedSpan {
974                    text: String::new(),
975                    bold,
976                    italic,
977                    underline: false,
978                    strikeout,
979                    code: false,
980                    superscript: false,
981                    subscript: false,
982                    link_href: link_href.clone(),
983                    image: None,
984                    footnote_ref: Some(label.to_string()),
985                };
986                if in_table {
987                    current_cell_spans.push(span);
988                } else {
989                    if !in_block {
990                        in_block = true;
991                    }
992                    current_spans.push(span);
993                }
994            }
995            _ => {}
996        }
997    }
998
999    // Flush any remaining content
1000    if !current_spans.is_empty() {
1001        elements.push(ParsedElement::Block(ParsedBlock {
1002            spans: std::mem::take(&mut current_spans),
1003            heading_level: current_heading,
1004            list_style: current_list_style,
1005            list_indent: current_list_indent,
1006            list_prefix: String::new(),
1007            list_suffix: String::new(),
1008            list_start: current_list_start,
1009            marker: None,
1010            is_code_block,
1011            code_language: code_language.take(),
1012            blockquote_depth,
1013            blockquote_opens: 0,
1014            line_height: None,
1015            non_breakable_lines: None,
1016            page_break_before: None,
1017            direction: None,
1018            background_color: None,
1019            alignment: None,
1020            top_margin: None,
1021            text_indent: None,
1022            semantic_role: None,
1023        }));
1024    }
1025
1026    // Drop the throwaway definitions `dangling_footnote_labels` asked for —
1027    // they exist only to make pulldown-cmark recognise the reference as
1028    // real, never to become a note the document owns. `elements` cannot
1029    // become empty from this: a label reached `dangling` only via a
1030    // reference actually present in `markdown`, so its surrounding block
1031    // survives even with the synthetic definition gone.
1032    if !dangling.is_empty() {
1033        elements.retain(
1034            |e| !matches!(e, ParsedElement::FootnoteDefinition { label, .. } if dangling.contains(label)),
1035        );
1036    }
1037
1038    // If no elements were parsed, create a single empty paragraph
1039    if elements.is_empty() {
1040        elements.push(ParsedElement::Block(ParsedBlock {
1041            spans: vec![ParsedSpan {
1042                text: String::new(),
1043                ..Default::default()
1044            }],
1045            heading_level: None,
1046            list_style: None,
1047            list_indent: 0,
1048            list_prefix: String::new(),
1049            list_suffix: String::new(),
1050            list_start: None,
1051            marker: None,
1052            is_code_block: false,
1053            code_language: None,
1054            blockquote_depth: 0,
1055            blockquote_opens: 0,
1056            line_height: None,
1057            non_breakable_lines: None,
1058            page_break_before: None,
1059            direction: None,
1060            background_color: None,
1061            alignment: None,
1062            top_margin: None,
1063            text_indent: None,
1064            semantic_role: None,
1065        }));
1066    }
1067
1068    // Quotations past the ceiling a document holds are flattened into the deepest kept
1069    // (see `quote_depth`): the writers walk one call deeper for each.
1070    crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1071    elements
1072}
1073
1074fn heading_level_to_i64(level: pulldown_cmark::HeadingLevel) -> i64 {
1075    use pulldown_cmark::HeadingLevel;
1076    match level {
1077        HeadingLevel::H1 => 1,
1078        HeadingLevel::H2 => 2,
1079        HeadingLevel::H3 => 3,
1080        HeadingLevel::H4 => 4,
1081        HeadingLevel::H5 => 5,
1082        HeadingLevel::H6 => 6,
1083    }
1084}
1085
1086// ─── HTML parsing ────────────────────────────────────────────────────
1087
1088use scraper::Node;
1089
1090/// Parsed CSS block-level styles from an inline `style` attribute.
1091#[derive(Debug, Clone, Default)]
1092struct BlockStyles {
1093    line_height: Option<i64>,
1094    non_breakable_lines: Option<bool>,
1095    page_break_before: Option<bool>,
1096    direction: Option<TextDirection>,
1097    background_color: Option<String>,
1098    /// `white-space` asks for the source's own spacing to be kept verbatim.
1099    /// Separate from `non_breakable_lines` because the two do not coincide:
1100    /// `nowrap` forbids wrapping but still collapses runs of spaces, so a
1101    /// block carrying it must be normalised like any other.
1102    preserve_whitespace: Option<bool>,
1103}
1104
1105/// Parse relevant CSS properties from an inline style string.
1106/// Handles: line-height, white-space, break-before/page-break-before, direction,
1107/// background-color.
1108fn parse_block_styles(style: &str) -> BlockStyles {
1109    let mut result = BlockStyles::default();
1110    for part in style.split(';') {
1111        let part = part.trim();
1112        if let Some((prop, val)) = part.split_once(':') {
1113            let prop = prop.trim().to_ascii_lowercase();
1114            let val = val.trim();
1115            match prop.as_str() {
1116                "line-height" => {
1117                    // Try parsing as a plain number (multiplier)
1118                    if let Ok(v) = val.parse::<f64>() {
1119                        result.line_height = Some((v * 1000.0) as i64);
1120                    }
1121                }
1122                "white-space" if val == "pre" || val == "nowrap" || val == "pre-wrap" => {
1123                    result.non_breakable_lines = Some(true);
1124                    if val != "nowrap" {
1125                        result.preserve_whitespace = Some(true);
1126                    }
1127                }
1128                // CSS3 `break-before` and its CSS2 predecessor `page-break-before` mean
1129                // the same thing; both are read because both are written (browsers still
1130                // want the legacy spelling, and so do most EPUB engines).
1131                "break-before" | "page-break-before" => {
1132                    result.page_break_before = match val.to_ascii_lowercase().as_str() {
1133                        "page" | "always" | "left" | "right" | "recto" | "verso" => Some(true),
1134                        "avoid" | "auto" => Some(false),
1135                        _ => None,
1136                    };
1137                }
1138                "direction" => {
1139                    if val.eq_ignore_ascii_case("rtl") {
1140                        result.direction = Some(TextDirection::RightToLeft);
1141                    } else if val.eq_ignore_ascii_case("ltr") {
1142                        result.direction = Some(TextDirection::LeftToRight);
1143                    }
1144                }
1145                "background-color" | "background" => {
1146                    result.background_color = Some(val.to_string());
1147                }
1148                _ => {}
1149            }
1150        }
1151    }
1152    result
1153}
1154
1155/// The five characters HTML calls whitespace.
1156///
1157/// Deliberately not `char::is_whitespace`, which is `true` for U+00A0: a
1158/// no-break space is a character the writer typed — the one French typography
1159/// puts before a colon, holding "Attention&nbsp;:" together — and collapsing it
1160/// away is a silent edit of the text, not of its layout.
1161fn is_html_space(ch: char) -> bool {
1162    matches!(ch, ' ' | '\t' | '\n' | '\r' | '\u{0C}')
1163}
1164
1165/// Whether a span run holds anything a reader would see.
1166fn spans_carry_content(spans: &[ParsedSpan]) -> bool {
1167    spans
1168        .iter()
1169        .any(|s| !s.text.is_empty() || s.image.is_some() || s.footnote_ref.is_some())
1170}
1171
1172/// Apply the CSS `white-space: normal` rules to one block's inline spans: every
1173/// run of HTML whitespace collapses to a single space, and the runs at the
1174/// block's two ends disappear entirely.
1175///
1176/// Per block rather than per text node, because a run crosses span boundaries —
1177/// `<b>word</b>\n<i>next</i>` is one collapsible run split over three nodes —
1178/// and because only the block knows where its own ends are.
1179///
1180/// Without this, an exporter's own source formatting reaches the document as
1181/// text. LibreOffice writes a newline after every `<p …>` and hard-wraps the
1182/// prose inside it at about seventy columns, so pasting a whole book put a
1183/// blank line before every paragraph and a break through the middle of every
1184/// sentence.
1185fn collapse_inline_whitespace(spans: &mut [ParsedSpan]) {
1186    fn is_replaced(span: &ParsedSpan) -> bool {
1187        span.image.is_some() || span.footnote_ref.is_some()
1188    }
1189
1190    // Collapse within each span first. Nothing non-empty is emptied here, which
1191    // is what lets the second pass read `is_empty()` as "this span held no text
1192    // to begin with" — the marker a `<br>` leaves behind.
1193    for span in spans.iter_mut() {
1194        if is_replaced(span) {
1195            continue;
1196        }
1197        let mut out = String::with_capacity(span.text.len());
1198        let mut in_run = false;
1199        for ch in span.text.chars() {
1200            if is_html_space(ch) {
1201                if !in_run {
1202                    out.push(' ');
1203                    in_run = true;
1204                }
1205            } else {
1206                out.push(ch);
1207                in_run = false;
1208            }
1209        }
1210        span.text = out;
1211    }
1212
1213    // The block's start counts as "already ended on a space", so a run opening
1214    // the block is dropped outright.
1215    let mut prev_ends_with_space = true;
1216    for span in spans.iter_mut() {
1217        if is_replaced(span) {
1218            prev_ends_with_space = false;
1219            continue;
1220        }
1221        if span.text.is_empty() {
1222            // The empty span a `<br>` leaves. It contributes no text, but it
1223            // still ends the collapsible run either side of it — without that,
1224            // `x <br> y` closes up into "xy".
1225            prev_ends_with_space = false;
1226            continue;
1227        }
1228        if prev_ends_with_space && span.text.starts_with(' ') {
1229            span.text.remove(0);
1230        }
1231        // A span emptied by that drop leaves the run open rather than closing
1232        // it: the space it carried belongs to the run its neighbours share.
1233        if !span.text.is_empty() {
1234            prev_ends_with_space = span.text.ends_with(' ');
1235        }
1236    }
1237
1238    // The block's trailing run, in whichever span still holds it. Replaced
1239    // content stops the walk: a space before a trailing image is between two
1240    // pieces of content, not after the last one.
1241    for span in spans.iter_mut().rev() {
1242        if is_replaced(span) {
1243            break;
1244        }
1245        if span.text.is_empty() {
1246            continue;
1247        }
1248        if span.text.ends_with(' ') {
1249            span.text.pop();
1250        }
1251        break;
1252    }
1253}
1254
1255/// `block` as one block per line of its text.
1256///
1257/// The model has no line break inside a block: a new line is a new block, which is
1258/// what the Djot reader makes of a hard break and what a plain-text paste makes of a
1259/// line break. Text that kept its source's line breaks (`<pre>`, `white-space: pre`)
1260/// held them inside one block, where the editor showed them as lines and the Djot
1261/// writer had nothing to write them as. Written raw, a blank line ended the paragraph
1262/// and the lines after it were read back as the lists, headings and quotations their
1263/// markers spelled, however deep.
1264///
1265/// Each line keeps the block's own formatting and its spans' formatting, but for the
1266/// page break, which stays with the first line: it is where the block starts, and on
1267/// every line it put each on a page of its own. An empty line is an empty block. The
1268/// line break that ends the last line starts no line of its own, as a browser shows it.
1269/// A block without a line break is returned as it is.
1270///
1271/// Every line is built from copies of the block and of the span it comes from that
1272/// hold no text, so the split costs in proportion to the text: built from whole copies,
1273/// each line copied the whole text again, and a paste of 64,000 preformatted lines
1274/// took seconds.
1275pub fn split_block_at_line_breaks(mut block: ParsedBlock) -> Vec<ParsedBlock> {
1276    let breaks = |span: &ParsedSpan| span.image.is_none() && span.text.contains('\n');
1277    if !block.spans.iter().any(breaks) {
1278        return vec![block];
1279    }
1280    let mut lines: Vec<Vec<ParsedSpan>> = vec![Vec::new()];
1281    for mut span in std::mem::take(&mut block.spans) {
1282        if !breaks(&span) {
1283            push_to_last_line(&mut lines, span);
1284            continue;
1285        }
1286        let text = std::mem::take(&mut span.text);
1287        for (i, piece) in text.split('\n').enumerate() {
1288            if i > 0 {
1289                lines.push(Vec::new());
1290            }
1291            if !piece.is_empty() {
1292                push_to_last_line(
1293                    &mut lines,
1294                    ParsedSpan {
1295                        text: piece.to_string(),
1296                        ..span.clone()
1297                    },
1298                );
1299            }
1300        }
1301    }
1302    if lines.len() > 1 && lines.last().is_some_and(Vec::is_empty) {
1303        lines.pop();
1304    }
1305    let later_line = ParsedBlock {
1306        page_break_before: None,
1307        blockquote_opens: 0,
1308        ..block.clone()
1309    };
1310    let mut first_line = Some(block);
1311    lines
1312        .into_iter()
1313        .map(|spans| {
1314            let spans = if spans.is_empty() {
1315                vec![ParsedSpan::default()]
1316            } else {
1317                spans
1318            };
1319            match first_line.take() {
1320                Some(first) => ParsedBlock { spans, ..first },
1321                None => ParsedBlock {
1322                    spans,
1323                    ..later_line.clone()
1324                },
1325            }
1326        })
1327        .collect()
1328}
1329
1330/// Append `span` to the last of `lines`, which [`split_block_at_line_breaks`] keeps
1331/// non-empty.
1332fn push_to_last_line(lines: &mut Vec<Vec<ParsedSpan>>, span: ParsedSpan) {
1333    match lines.last_mut() {
1334        Some(line) => line.push(span),
1335        None => lines.push(vec![span]),
1336    }
1337}
1338
1339pub fn parse_html(html: &str) -> Vec<ParsedBlock> {
1340    ParsedElement::flatten_to_blocks(parse_html_elements(html))
1341}
1342
1343/// The attribute an HTML producer uses to say "this element *is* a footnote
1344/// reference", and the label it names.
1345///
1346/// A data attribute on an otherwise-ordinary element, rather than a literal
1347/// `[^label]` in the text, because the text a producer emits is *escaped* on the
1348/// way to Djot — [`escape_djot_inline`](crate::parser_tools::djot_escape::escape_djot_inline)
1349/// neutralises `[`, `]` and `^`, so a reference smuggled through as characters
1350/// arrives as `\[\^1\]` and is prose rather than a reference. It has to be a
1351/// node before it is serialised, which is what this makes it.
1352///
1353/// [`ParsedSpan::footnote_ref`] is the same field the Markdown/Djot reader
1354/// already fills, so everything downstream — the document model, the Djot
1355/// writer, numbering — needs no change at all.
1356pub const HTML_FOOTNOTE_ATTR: &str = "data-footnote-ref";
1357
1358/// Build a footnote-reference span from any element carrying
1359/// [`HTML_FOOTNOTE_ATTR`].
1360fn html_footnote_span(
1361    el: &scraper::node::Element,
1362    link_href: Option<String>,
1363) -> Option<ParsedSpan> {
1364    let label = el.attr(HTML_FOOTNOTE_ATTR)?.trim();
1365    if label.is_empty() {
1366        return None;
1367    }
1368    Some(ParsedSpan {
1369        text: String::new(),
1370        link_href,
1371        footnote_ref: Some(label.to_string()),
1372        ..Default::default()
1373    })
1374}
1375
1376/// Build an inline-image span from an `<img>` element, if it has a usable
1377/// source.
1378///
1379/// `<img>` was matched by none of the HTML walker's three tag dispatches, so it
1380/// fell into their wildcard arms and was dropped whole — silently, and unlike
1381/// Markdown not even leaving its alt text behind. That is the path a browser or
1382/// Word paste travels.
1383fn html_img_span(el: &scraper::node::Element, link_href: Option<String>) -> Option<ParsedSpan> {
1384    let src = el.attr("src")?;
1385    if src.is_empty() {
1386        return None;
1387    }
1388    let dim = |name: &str| -> i64 {
1389        el.attr(name)
1390            .and_then(|v| v.trim().trim_end_matches("px").parse::<i64>().ok())
1391            .filter(|n| *n > 0)
1392            .unwrap_or(0)
1393    };
1394    Some(ParsedSpan {
1395        text: String::new(),
1396        link_href,
1397        image: Some(ParsedImage {
1398            src: src.to_string(),
1399            alt: el.attr("alt").unwrap_or_default().to_string(),
1400            width: dim("width"),
1401            height: dim("height"),
1402        }),
1403        ..Default::default()
1404    })
1405}
1406
1407/// Whether `node` is a `<ul>` or an `<ol>`.
1408fn is_html_list(node: &Node) -> bool {
1409    node.as_element()
1410        .is_some_and(|el| matches!(el.name(), "ul" | "ol"))
1411}
1412
1413/// Move every list item nested deeper than
1414/// [`MAX_LIST_LEVELS`](crate::parser_tools::list_depth::MAX_LIST_LEVELS)
1415/// lists into the deepest list kept, right after the item holding it, in
1416/// document order: a list nested past that depth comes out flattened to it,
1417/// every item kept, in its order.
1418///
1419/// The reader recurses a few elements deep per list level and stops at
1420/// `MAX_RECURSION_DEPTH`, so a list nested past some eighty levels lost every
1421/// item below that, pasted or loaded. The editing gestures nest no deeper than
1422/// this depth, and an insertion brings any list back to it (see `list_depth`),
1423/// so HTML holding more is read at that depth to begin with. The tree is
1424/// changed in place, without recursion, before it is read.
1425fn flatten_deep_html_lists(tree: &mut ego_tree::Tree<Node>) {
1426    use crate::parser_tools::list_depth::MAX_LIST_LEVELS;
1427    use ego_tree::iter::Edge;
1428
1429    // The lists one level past the deepest kept, in document order.
1430    let mut too_deep: Vec<ego_tree::NodeId> = Vec::new();
1431    let mut depth = 0usize;
1432    for edge in tree.root().traverse() {
1433        match edge {
1434            Edge::Open(node) if is_html_list(node.value()) => {
1435                if depth == MAX_LIST_LEVELS {
1436                    too_deep.push(node.id());
1437                }
1438                depth += 1;
1439            }
1440            Edge::Close(node) if is_html_list(node.value()) => {
1441                depth = depth.saturating_sub(1);
1442            }
1443            _ => {}
1444        }
1445    }
1446
1447    for list in too_deep {
1448        // The child of the deepest kept list that holds `list` (its `<li>`, or
1449        // `list` itself when a list sits right inside another): the items go
1450        // after it.
1451        let Some(list_ref) = tree.get(list) else {
1452            continue;
1453        };
1454        let mut holder = None;
1455        let mut current = list_ref;
1456        while let Some(parent) = current.parent() {
1457            if is_html_list(parent.value()) {
1458                holder = Some(current.id());
1459                break;
1460            }
1461            current = parent;
1462        }
1463        let Some(holder) = holder else {
1464            continue;
1465        };
1466        let mut after = holder;
1467        let items: Vec<ego_tree::NodeId> = list_ref
1468            .descendants()
1469            .filter(|node| {
1470                node.value()
1471                    .as_element()
1472                    .is_some_and(|el| el.name() == "li")
1473            })
1474            .map(|node| node.id())
1475            .collect();
1476        for item in items {
1477            // Each node placed here sits in the deepest kept list, so it has a
1478            // parent to be placed beside.
1479            if tree.get(after).and_then(|node| node.parent()).is_none() {
1480                break;
1481            }
1482            if let Some(mut node) = tree.get_mut(after) {
1483                node.insert_id_after(item);
1484                after = item;
1485            }
1486        }
1487        // What is left of the list is its items' wrappers, or text standing in
1488        // it outside any item. It goes after the items when it holds any text
1489        // (it is already beside them when it sits right in the deepest kept
1490        // list), and away when it holds none.
1491        let holds_text = tree.get(list).is_some_and(|node| {
1492            node.descendants()
1493                .any(|d| d.value().as_text().is_some_and(|t| !t.trim().is_empty()))
1494        });
1495        if !holds_text {
1496            if let Some(mut node) = tree.get_mut(list) {
1497                node.detach();
1498            }
1499        } else if holder != list
1500            && tree.get(after).and_then(|node| node.parent()).is_some()
1501            && let Some(mut node) = tree.get_mut(after)
1502        {
1503            node.insert_id_after(list);
1504        }
1505    }
1506}
1507
1508/// How deep a walk of an HTML tree may go on the caller's own stack: a debug build spends
1509/// between 5 and 8 KiB of stack on each element level it follows, so this is at most some
1510/// 256 KiB, what any thread a host reads on has to spare.
1511const HTML_DEPTH_ON_THE_CALLERS_STACK: usize = 32;
1512
1513/// The deepest an HTML reader follows elements: a subtree nested deeper is read as its
1514/// text, without recursing (see `read_html_tree`).
1515const MAX_HTML_READ_DEPTH: usize = 256;
1516
1517/// The stack a reader of a deeper tree gets on a thread of its own, before what each level
1518/// adds: a spawned thread's.
1519const HTML_READER_STACK: usize = 2 << 20;
1520
1521/// The stack each element level of a deeper tree adds for its reader: two to three times
1522/// what a debug build spends on one.
1523const HTML_STACK_PER_LEVEL: usize = 16 << 10;
1524
1525/// `value` read as HTML reads an integer attribute such as `<ol start>`: after any leading
1526/// white space, an optional sign and the digits up to the first character that is not one.
1527/// `None` when no digit follows. A number past what 64 bits hold reads as the largest they
1528/// hold on its side of 0. A browser numbers `<ol start="5x">` from 5, where a strict parse
1529/// of the whole value numbered it from 1.
1530fn html_integer(value: &str) -> Option<i64> {
1531    let value = value.trim_start_matches(|c: char| c.is_ascii_whitespace());
1532    let (negative, digits) = match value.as_bytes().first() {
1533        Some(b'-') => (true, &value[1..]),
1534        Some(b'+') => (false, &value[1..]),
1535        _ => (false, value),
1536    };
1537    let digits: &str = digits
1538        .split(|c: char| !c.is_ascii_digit())
1539        .next()
1540        .unwrap_or("");
1541    if digits.is_empty() {
1542        return None;
1543    }
1544    let magnitude = digits.bytes().fold(0i64, |number, digit| {
1545        number
1546            .saturating_mul(10)
1547            .saturating_add(i64::from(digit - b'0'))
1548    });
1549    Some(if negative { -magnitude } else { magnitude })
1550}
1551
1552/// How many elements deep the deepest node of `tree` is. A loop over the tree's edges,
1553/// never a recursion.
1554fn html_tree_depth(tree: &ego_tree::Tree<Node>) -> usize {
1555    use ego_tree::iter::Edge;
1556    let mut depth = 0usize;
1557    let mut deepest = 0usize;
1558    for edge in tree.root().traverse() {
1559        match edge {
1560            Edge::Open(node) if node.value().is_element() => {
1561                depth += 1;
1562                deepest = deepest.max(depth);
1563            }
1564            Edge::Close(node) if node.value().is_element() => depth = depth.saturating_sub(1),
1565            _ => {}
1566        }
1567    }
1568    deepest
1569}
1570
1571/// Read `html` into blocks and tables.
1572///
1573/// The reader recurses once per element level it follows, up to `MAX_HTML_READ_DEPTH`,
1574/// and a stack overflow aborts the process: a debug build ran out of a spawned thread's
1575/// stack on a paste nested some 200 elements deep, and a host reading on a smaller thread
1576/// sooner. A tree nested deeper than a caller's stack is sure to hold is read on a thread
1577/// of its own, with a stack sized for its depth; when no thread can be started, it is read
1578/// on the caller's stack to `HTML_DEPTH_ON_THE_CALLERS_STACK` levels, each subtree
1579/// deeper than that read as its text. The quotations it holds are kept within
1580/// [`MAX_QUOTE_LEVELS`](crate::parser_tools::quote_depth::MAX_QUOTE_LEVELS) (see
1581/// `quote_depth`).
1582pub fn parse_html_elements(html: &str) -> Vec<ParsedElement> {
1583    use scraper::Html;
1584
1585    let read = |depth_limit: usize, html: &str| {
1586        let mut fragment = Html::parse_fragment(html);
1587        flatten_deep_html_lists(&mut fragment.tree);
1588        let mut elements = read_html_tree(&fragment, depth_limit);
1589        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1590        elements
1591    };
1592    let mut fragment = Html::parse_fragment(html);
1593    flatten_deep_html_lists(&mut fragment.tree);
1594    let depth = html_tree_depth(&fragment.tree);
1595    if depth <= HTML_DEPTH_ON_THE_CALLERS_STACK {
1596        let mut elements = read_html_tree(&fragment, MAX_HTML_READ_DEPTH);
1597        crate::parser_tools::quote_depth::clamp_quote_depths(&mut elements);
1598        return elements;
1599    }
1600    drop(fragment);
1601    let stack = HTML_READER_STACK + depth.min(MAX_HTML_READ_DEPTH + 1) * HTML_STACK_PER_LEVEL;
1602    let owned = html.to_owned();
1603    let on_its_own_thread = std::thread::Builder::new()
1604        .stack_size(stack)
1605        .spawn(move || read(MAX_HTML_READ_DEPTH, &owned))
1606        .ok()
1607        .and_then(|reader| reader.join().ok());
1608    on_its_own_thread.unwrap_or_else(|| read(HTML_DEPTH_ON_THE_CALLERS_STACK, html))
1609}
1610
1611/// Read `fragment`, following its elements `depth_limit` deep at most: a subtree nested
1612/// deeper is read as its text (see `subtree_text`). The walk recurses once per element
1613/// level it follows.
1614fn read_html_tree(fragment: &scraper::Html, depth_limit: usize) -> Vec<ParsedElement> {
1615    let mut elements: Vec<ParsedElement> = Vec::new();
1616
1617    // Walk the DOM tree starting from the root
1618    let root = fragment.root_element();
1619
1620    #[derive(Clone, Default)]
1621    struct FmtState {
1622        bold: bool,
1623        italic: bool,
1624        underline: bool,
1625        strikeout: bool,
1626        code: bool,
1627        superscript: bool,
1628        subscript: bool,
1629        link_href: Option<String>,
1630    }
1631
1632    /// The list an element stands in: its style, and the number its first item wears when
1633    /// it is not 1 (`<ol start="3">`).
1634    type HtmlList = (Option<ListStyle>, Option<i64>);
1635
1636    /// Elements whose text content is machinery, not prose.
1637    ///
1638    /// `<style>` and `<script>` hold raw text the HTML spec never renders, and
1639    /// `<head>` and its metadata children carry none a reader is meant to see.
1640    /// Every walker below recurses into unknown tags on the assumption that
1641    /// they are inline wrappers, which turns a stylesheet into paragraphs —
1642    /// and Word, Google Docs and Chrome all put a `<style>` block in the
1643    /// `text/html` flavour they publish to the clipboard, so this is the
1644    /// ordinary paste, not an exotic one.
1645    fn is_metadata_tag(tag: &str) -> bool {
1646        matches!(
1647            tag,
1648            "head"
1649                | "style"
1650                | "script"
1651                | "title"
1652                | "meta"
1653                | "link"
1654                | "base"
1655                | "noscript"
1656                | "template"
1657        )
1658    }
1659
1660    /// The text of everything under `node`, for a subtree nested deeper than
1661    /// the reader follows (see [`MAX_HTML_READ_DEPTH`]): read without recursing,
1662    /// with a space where a block ends so that words do not run together, and
1663    /// without what [`is_metadata_tag`] names.
1664    ///
1665    /// Past that depth the reader used to stop and keep nothing: text nested
1666    /// deeply enough in lists, quotations or plain `<div>`s was dropped from a
1667    /// paste without a trace.
1668    fn subtree_text(node: ego_tree::NodeRef<Node>) -> String {
1669        use ego_tree::iter::Edge;
1670        let mut text = String::new();
1671        // How many elements deep the walk is inside a metadata element.
1672        let mut hidden = 0usize;
1673        for edge in node.traverse() {
1674            match edge {
1675                Edge::Open(open) => match open.value() {
1676                    Node::Element(el) => {
1677                        if hidden > 0 || is_metadata_tag(el.name()) {
1678                            hidden += 1;
1679                        } else if !text.is_empty() && !text.ends_with(char::is_whitespace) {
1680                            text.push(' ');
1681                        }
1682                    }
1683                    Node::Text(t) if hidden == 0 => text.push_str(t),
1684                    _ => {}
1685                },
1686                Edge::Close(close) => {
1687                    if hidden > 0 && close.value().is_element() {
1688                        hidden -= 1;
1689                    }
1690                }
1691            }
1692        }
1693        text
1694    }
1695
1696    /// A span of `text` in `state`'s formatting.
1697    fn text_span(text: String, state: &FmtState) -> ParsedSpan {
1698        ParsedSpan {
1699            text,
1700            bold: state.bold,
1701            italic: state.italic,
1702            underline: state.underline,
1703            strikeout: state.strikeout,
1704            code: state.code,
1705            superscript: state.superscript,
1706            subscript: state.subscript,
1707            link_href: state.link_href.clone(),
1708            image: None,
1709            footnote_ref: None,
1710        }
1711    }
1712
1713    /// Collect inline spans from a `<td>` or `<th>` cell element.
1714    fn collect_cell_spans(
1715        node: ego_tree::NodeRef<Node>,
1716        state: &FmtState,
1717        spans: &mut Vec<ParsedSpan>,
1718        depth: usize,
1719        limit: usize,
1720    ) {
1721        if depth > limit {
1722            let text = subtree_text(node);
1723            if !text.is_empty() {
1724                spans.push(text_span(text, state));
1725            }
1726            return;
1727        }
1728        for child in node.children() {
1729            match child.value() {
1730                Node::Text(text) => {
1731                    let t = text.text.to_string();
1732                    if !t.is_empty() {
1733                        spans.push(ParsedSpan {
1734                            text: t,
1735                            bold: state.bold,
1736                            italic: state.italic,
1737                            underline: state.underline,
1738                            strikeout: state.strikeout,
1739                            code: state.code,
1740                            superscript: state.superscript,
1741                            subscript: state.subscript,
1742                            link_href: state.link_href.clone(),
1743                            image: None,
1744                            footnote_ref: None,
1745                        });
1746                    }
1747                }
1748                Node::Element(el) => {
1749                    let tag = el.name();
1750                    if is_metadata_tag(tag) {
1751                        continue;
1752                    }
1753                    let mut new_state = state.clone();
1754                    match tag {
1755                        // Checked by attribute rather than by tag name: the
1756                        // producer picks the element (a `<sup>` renders sensibly
1757                        // in a browser), and only the attribute is a contract.
1758                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
1759                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
1760                            {
1761                                spans.push(span);
1762                            }
1763                            continue;
1764                        }
1765                        "b" | "strong" => new_state.bold = true,
1766                        "i" | "em" => new_state.italic = true,
1767                        "u" | "ins" => new_state.underline = true,
1768                        "s" | "del" | "strike" => new_state.strikeout = true,
1769                        "code" => new_state.code = true,
1770                        "sup" => new_state.superscript = true,
1771                        "sub" => new_state.subscript = true,
1772                        "a" => {
1773                            if let Some(href) = el.attr("href") {
1774                                new_state.link_href = Some(href.to_string());
1775                            }
1776                        }
1777                        "img" => {
1778                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
1779                                spans.push(span);
1780                            }
1781                            continue;
1782                        }
1783                        // A cell is read as one paragraph, so a line break, or a
1784                        // paragraph or list item of its own, is a space between
1785                        // words. Read as nothing, a cell of two paragraphs, which
1786                        // the HTML writer puts out as `One<br/>Two`, came back as
1787                        // the one word `OneTwo`.
1788                        "br" => {
1789                            spans.push(text_span(" ".to_string(), state));
1790                            continue;
1791                        }
1792                        "p" | "div" | "li" | "ul" | "ol" | "blockquote" | "pre" | "h1" | "h2"
1793                        | "h3" | "h4" | "h5" | "h6" => {
1794                            spans.push(text_span(" ".to_string(), state));
1795                        }
1796                        _ => {}
1797                    }
1798                    collect_cell_spans(child, &new_state, spans, depth + 1, limit);
1799                }
1800                _ => {}
1801            }
1802        }
1803    }
1804
1805    /// The widest a `colspan` makes a row: HTML's own limit on the attribute. A row of
1806    /// ordinary cells may be wider; only the empty cells a span adds stop here.
1807    const MAX_HTML_COLSPAN: usize = 1000;
1808
1809    /// Parse a `<table>` element into a ParsedTable, with the number of `<td>` and `<th>`
1810    /// cells it has (see `push_table`). `depth` is how deep the walk stands at the table,
1811    /// and `limit` how deep it goes.
1812    fn parse_table_element(
1813        table_node: ego_tree::NodeRef<Node>,
1814        depth: usize,
1815        limit: usize,
1816    ) -> (ParsedTable, usize) {
1817        let mut rows: Vec<Vec<ParsedTableCell>> = Vec::new();
1818        let mut header_rows: usize = 0;
1819        let mut own_cells: usize = 0;
1820
1821        /// `walk` is how deep the walk stands at `node` and how deep it goes.
1822        fn collect_rows(
1823            node: ego_tree::NodeRef<Node>,
1824            rows: &mut Vec<Vec<ParsedTableCell>>,
1825            header_rows: &mut usize,
1826            own_cells: &mut usize,
1827            in_thead: bool,
1828            walk: (usize, usize),
1829        ) {
1830            let (depth, limit) = walk;
1831            for child in node.children() {
1832                if let Node::Element(el) = child.value() {
1833                    match el.name() {
1834                        "thead" => collect_rows(
1835                            child,
1836                            rows,
1837                            header_rows,
1838                            own_cells,
1839                            true,
1840                            (depth + 1, limit),
1841                        ),
1842                        "tbody" | "tfoot" => collect_rows(
1843                            child,
1844                            rows,
1845                            header_rows,
1846                            own_cells,
1847                            false,
1848                            (depth + 1, limit),
1849                        ),
1850                        "tr" => {
1851                            let mut cells: Vec<ParsedTableCell> = Vec::new();
1852                            // The cells of the row up to its last `<td>` or `<th>`: the
1853                            // empty cells a span adds after it hold no cell's place.
1854                            let mut own_end = 0;
1855                            for td in child.children() {
1856                                if let Node::Element(td_el) = td.value()
1857                                    && matches!(td_el.name(), "td" | "th")
1858                                {
1859                                    let mut spans = Vec::new();
1860                                    let state = FmtState::default();
1861                                    collect_cell_spans(td, &state, &mut spans, depth + 2, limit);
1862                                    collapse_inline_whitespace(&mut spans);
1863                                    if spans.is_empty() {
1864                                        spans.push(ParsedSpan::default());
1865                                    }
1866                                    cells.push(ParsedTableCell { spans });
1867                                    *own_cells += 1;
1868                                    own_end = cells.len();
1869                                    // A cell spanning columns covers the cells after it
1870                                    // in its row: empty ones hold its place, so the cells
1871                                    // that follow stay in their own columns and the
1872                                    // table is as wide as the columns it spans.
1873                                    let span = td_el
1874                                        .attr("colspan")
1875                                        .and_then(|value| value.trim().parse::<usize>().ok())
1876                                        .unwrap_or(1)
1877                                        .clamp(1, MAX_HTML_COLSPAN);
1878                                    for _ in 1..span {
1879                                        if cells.len() >= MAX_HTML_COLSPAN {
1880                                            break;
1881                                        }
1882                                        cells.push(ParsedTableCell {
1883                                            spans: vec![ParsedSpan::default()],
1884                                        });
1885                                    }
1886                                }
1887                            }
1888                            // A span closing a row widens the table only as far as another
1889                            // row's cells reach: `<th colspan="1000">` over rows of one
1890                            // cell is a table of one column, not of a thousand.
1891                            cells.truncate(own_end);
1892                            if !cells.is_empty() {
1893                                rows.push(cells);
1894                                if in_thead {
1895                                    *header_rows += 1;
1896                                }
1897                            }
1898                        }
1899                        _ => {}
1900                    }
1901                }
1902            }
1903        }
1904
1905        collect_rows(
1906            table_node,
1907            &mut rows,
1908            &mut header_rows,
1909            &mut own_cells,
1910            false,
1911            (depth, limit),
1912        );
1913
1914        // Tables without explicit <thead> but with <th> cells: treat first row as header
1915        if header_rows == 0 && !rows.is_empty() {
1916            header_rows = 1;
1917        }
1918
1919        (
1920            ParsedTable {
1921                header_rows,
1922                rows,
1923                // The caller (`walk_node`) sets the real depth: this helper has
1924                // no visibility into the surrounding blockquote nesting.
1925                blockquote_depth: 0,
1926                blockquote_opens: 0,
1927            },
1928            own_cells,
1929        )
1930    }
1931
1932    #[allow(clippy::too_many_arguments)]
1933    fn walk_node(
1934        node: ego_tree::NodeRef<Node>,
1935        state: &FmtState,
1936        elements: &mut Vec<ParsedElement>,
1937        current_list: &HtmlList,
1938        blockquote_depth: u32,
1939        list_depth: u32,
1940        depth: usize,
1941        limit: usize,
1942    ) {
1943        if depth > limit {
1944            let text = subtree_text(node);
1945            if !text.trim().is_empty() {
1946                let mut spans = vec![text_span(text, state)];
1947                collapse_inline_whitespace(&mut spans);
1948                elements.push(ParsedElement::Block(ParsedBlock {
1949                    spans,
1950                    blockquote_depth,
1951                    blockquote_opens: 0,
1952                    ..ParsedBlock::default()
1953                }));
1954            }
1955            return;
1956        }
1957        match node.value() {
1958            Node::Element(el) => {
1959                let tag = el.name();
1960                if is_metadata_tag(tag) {
1961                    return;
1962                }
1963                let mut new_state = state.clone();
1964                let mut new_list = current_list.clone();
1965                let mut bq_depth = blockquote_depth;
1966                let mut new_list_depth = list_depth;
1967
1968                // Determine if this is a block-level element
1969                let is_block_tag = matches!(
1970                    tag,
1971                    "p" | "div"
1972                        | "h1"
1973                        | "h2"
1974                        | "h3"
1975                        | "h4"
1976                        | "h5"
1977                        | "h6"
1978                        | "li"
1979                        | "pre"
1980                        | "br"
1981                        | "blockquote"
1982                        | "body"
1983                        | "html"
1984                );
1985
1986                // Update formatting state
1987                match tag {
1988                    "b" | "strong" => new_state.bold = true,
1989                    "i" | "em" => new_state.italic = true,
1990                    "u" | "ins" => new_state.underline = true,
1991                    "s" | "del" | "strike" => new_state.strikeout = true,
1992                    "code" => new_state.code = true,
1993                    "sup" => new_state.superscript = true,
1994                    "sub" => new_state.subscript = true,
1995                    "a" => {
1996                        if let Some(href) = el.attr("href") {
1997                            new_state.link_href = Some(href.to_string());
1998                        }
1999                    }
2000                    "ul" => {
2001                        new_list = (Some(ListStyle::Disc), None);
2002                        new_list_depth = list_depth + 1;
2003                    }
2004                    "ol" => {
2005                        // A list numbered from below 0 is kept from 1: no saved syntax
2006                        // but HTML writes one. A start past what every format holds is
2007                        // read as the largest it holds (see `list_start`).
2008                        let start = el
2009                            .attr("start")
2010                            .and_then(html_integer)
2011                            .and_then(crate::parser_tools::list_start::list_start);
2012                        new_list = (Some(ListStyle::Decimal), start);
2013                        new_list_depth = list_depth + 1;
2014                    }
2015                    "blockquote" => {
2016                        bq_depth += 1;
2017                    }
2018                    _ => {}
2019                }
2020
2021                // Determine heading level
2022                let heading_level = match tag {
2023                    "h1" => Some(1),
2024                    "h2" => Some(2),
2025                    "h3" => Some(3),
2026                    "h4" => Some(4),
2027                    "h5" => Some(5),
2028                    "h6" => Some(6),
2029                    _ => None,
2030                };
2031
2032                let is_code_block = tag == "pre";
2033
2034                // Extract code language from <pre><code class="language-xxx">
2035                let code_language = if is_code_block {
2036                    node.children().find_map(|child| {
2037                        if let Node::Element(cel) = child.value()
2038                            && cel.name() == "code"
2039                            && let Some(cls) = cel.attr("class")
2040                        {
2041                            return cls
2042                                .split_whitespace()
2043                                .find_map(|c| c.strip_prefix("language-"))
2044                                .map(|l| l.to_string());
2045                        }
2046                        None
2047                    })
2048                } else {
2049                    None
2050                };
2051
2052                // Extract CSS styles from block-level elements
2053                let css = if is_block_tag {
2054                    el.attr("style").map(parse_block_styles).unwrap_or_default()
2055                } else {
2056                    BlockStyles::default()
2057                };
2058
2059                if tag == "table" {
2060                    // Parse table structure into a ParsedTable
2061                    let (mut parsed_table, own_cells) = parse_table_element(node, depth, limit);
2062                    if !parsed_table.rows.is_empty() {
2063                        parsed_table.blockquote_depth = bq_depth;
2064                        push_table(elements, parsed_table, own_cells);
2065                    }
2066                    return;
2067                }
2068
2069                if tag == "br" {
2070                    // <br> creates a new block
2071                    elements.push(ParsedElement::Block(ParsedBlock {
2072                        spans: vec![ParsedSpan {
2073                            text: String::new(),
2074                            ..Default::default()
2075                        }],
2076                        heading_level: None,
2077                        list_style: None,
2078                        list_indent: 0,
2079                        list_prefix: String::new(),
2080                        list_suffix: String::new(),
2081                        list_start: None,
2082                        marker: None,
2083                        is_code_block: false,
2084                        code_language: None,
2085                        blockquote_depth: bq_depth,
2086                        blockquote_opens: 0,
2087                        line_height: None,
2088                        non_breakable_lines: None,
2089                        page_break_before: None,
2090                        direction: None,
2091                        background_color: None,
2092                        alignment: None,
2093                        top_margin: None,
2094                        text_indent: None,
2095                        semantic_role: None,
2096                    }));
2097                    return;
2098                }
2099
2100                if tag == "blockquote" {
2101                    // Blockquote is a container — recurse into children with increased depth
2102                    let first = elements.len();
2103                    for child in node.children() {
2104                        walk_node(
2105                            child,
2106                            &new_state,
2107                            elements,
2108                            &new_list,
2109                            bq_depth,
2110                            new_list_depth,
2111                            depth + 1,
2112                            limit,
2113                        );
2114                    }
2115                    mark_quotation_opened(elements, first);
2116                } else if is_block_tag && tag != "br" {
2117                    // Start collecting spans for a new block.
2118                    // Use a temporary buffer so that nested block-level
2119                    // elements (e.g. sub-lists inside <li>) are collected
2120                    // separately and appended *after* the parent block.
2121                    let mut spans: Vec<ParsedSpan> = Vec::new();
2122                    let mut nested_elements: Vec<ParsedElement> = Vec::new();
2123                    collect_inline_spans(
2124                        node,
2125                        &new_state,
2126                        &mut spans,
2127                        &new_list,
2128                        &mut nested_elements,
2129                        bq_depth,
2130                        new_list_depth,
2131                        depth + 1,
2132                        limit,
2133                    );
2134
2135                    let (list_style_for_block, list_start_for_block) = if tag == "li" {
2136                        new_list.clone()
2137                    } else {
2138                        (None, None)
2139                    };
2140
2141                    let list_indent_for_block = if tag == "li" {
2142                        new_list_depth.saturating_sub(1)
2143                    } else {
2144                        0
2145                    };
2146
2147                    if !(is_code_block || css.preserve_whitespace == Some(true)) {
2148                        collapse_inline_whitespace(&mut spans);
2149                    }
2150
2151                    // Whitespace between block-level children is layout, not
2152                    // content. `<body>` holds a newline after every `</p>` an
2153                    // exporter writes, and keeping those as the container's own
2154                    // inline run is what closed a pasted book with a paragraph
2155                    // of four thousand blank lines. A block that produced no
2156                    // children keeps its empty run instead: `<p> </p>` is an
2157                    // empty paragraph the writer meant to be there.
2158                    let own_run_is_content =
2159                        spans_carry_content(&spans) || nested_elements.is_empty();
2160
2161                    if (!spans.is_empty() && own_run_is_content) || heading_level.is_some() {
2162                        // A code block is verbatim text, which no save can write an image or
2163                        // a note's reference into: a `<pre>` holding one is read as prose,
2164                        // which keeps it, one paragraph for each of its lines (see below).
2165                        let is_code_block = is_code_block
2166                            && !spans
2167                                .iter()
2168                                .any(|span| span.image.is_some() || span.footnote_ref.is_some());
2169                        let block = ParsedBlock {
2170                            spans,
2171                            heading_level,
2172                            list_style: list_style_for_block,
2173                            list_indent: list_indent_for_block,
2174                            list_prefix: String::new(),
2175                            list_suffix: String::new(),
2176                            list_start: list_start_for_block,
2177                            marker: None,
2178                            is_code_block,
2179                            code_language: code_language.filter(|_| is_code_block),
2180                            blockquote_depth: bq_depth,
2181                            blockquote_opens: 0,
2182                            line_height: css.line_height,
2183                            non_breakable_lines: css.non_breakable_lines,
2184                            page_break_before: css.page_break_before,
2185                            direction: css.direction,
2186                            background_color: css.background_color,
2187                            alignment: None,
2188                            top_margin: None,
2189                            text_indent: None,
2190                            semantic_role: None,
2191                        };
2192                        // Kept whitespace keeps its line breaks, and a line break in
2193                        // prose is a new block to the model. Only a code block holds
2194                        // its lines in one block.
2195                        if is_code_block {
2196                            elements.push(ParsedElement::Block(block));
2197                        } else {
2198                            elements.extend(
2199                                split_block_at_line_breaks(block)
2200                                    .into_iter()
2201                                    .map(ParsedElement::Block),
2202                            );
2203                        }
2204                    }
2205                    // Append nested block elements after the parent block
2206                    elements.append(&mut nested_elements);
2207                } else if matches!(tag, "ul" | "ol" | "thead" | "tbody" | "tr") {
2208                    // Container elements: recurse into children
2209                    for child in node.children() {
2210                        walk_node(
2211                            child,
2212                            &new_state,
2213                            elements,
2214                            &new_list,
2215                            bq_depth,
2216                            new_list_depth,
2217                            depth + 1,
2218                            limit,
2219                        );
2220                    }
2221                } else {
2222                    // Inline element or unknown: recurse
2223                    for child in node.children() {
2224                        walk_node(
2225                            child,
2226                            &new_state,
2227                            elements,
2228                            current_list,
2229                            bq_depth,
2230                            list_depth,
2231                            depth + 1,
2232                            limit,
2233                        );
2234                    }
2235                }
2236            }
2237            Node::Text(text) => {
2238                let t = text.text.to_string();
2239                let trimmed = t.trim();
2240                if !trimmed.is_empty() {
2241                    // Bare text not in a block — create a paragraph
2242                    elements.push(ParsedElement::Block(ParsedBlock {
2243                        spans: vec![ParsedSpan {
2244                            text: trimmed.to_string(),
2245                            bold: state.bold,
2246                            italic: state.italic,
2247                            underline: state.underline,
2248                            strikeout: state.strikeout,
2249                            code: state.code,
2250                            superscript: state.superscript,
2251                            subscript: state.subscript,
2252                            link_href: state.link_href.clone(),
2253                            image: None,
2254                            footnote_ref: None,
2255                        }],
2256                        heading_level: None,
2257                        list_style: None,
2258                        list_indent: 0,
2259                        list_prefix: String::new(),
2260                        list_suffix: String::new(),
2261                        list_start: None,
2262                        marker: None,
2263                        is_code_block: false,
2264                        code_language: None,
2265                        blockquote_depth,
2266                        blockquote_opens: 0,
2267                        line_height: None,
2268                        non_breakable_lines: None,
2269                        page_break_before: None,
2270                        direction: None,
2271                        background_color: None,
2272                        alignment: None,
2273                        top_margin: None,
2274                        text_indent: None,
2275                        semantic_role: None,
2276                    }));
2277                }
2278            }
2279            _ => {
2280                // Document, Comment, etc. — recurse children
2281                for child in node.children() {
2282                    walk_node(
2283                        child,
2284                        state,
2285                        elements,
2286                        current_list,
2287                        blockquote_depth,
2288                        list_depth,
2289                        depth + 1,
2290                        limit,
2291                    );
2292                }
2293            }
2294        }
2295    }
2296
2297    /// Collect inline spans from a block-level element's children.
2298    /// If a nested block-level element is encountered, it is flushed as a
2299    /// separate block.
2300    #[allow(clippy::too_many_arguments)]
2301    fn collect_inline_spans(
2302        node: ego_tree::NodeRef<Node>,
2303        state: &FmtState,
2304        spans: &mut Vec<ParsedSpan>,
2305        current_list: &HtmlList,
2306        elements: &mut Vec<ParsedElement>,
2307        blockquote_depth: u32,
2308        list_depth: u32,
2309        depth: usize,
2310        limit: usize,
2311    ) {
2312        if depth > limit {
2313            let text = subtree_text(node);
2314            if !text.is_empty() {
2315                spans.push(text_span(text, state));
2316            }
2317            return;
2318        }
2319        for child in node.children() {
2320            match child.value() {
2321                Node::Text(text) => {
2322                    let t = text.text.to_string();
2323                    if !t.is_empty() {
2324                        spans.push(ParsedSpan {
2325                            text: t,
2326                            bold: state.bold,
2327                            italic: state.italic,
2328                            underline: state.underline,
2329                            strikeout: state.strikeout,
2330                            code: state.code,
2331                            superscript: state.superscript,
2332                            subscript: state.subscript,
2333                            link_href: state.link_href.clone(),
2334                            image: None,
2335                            footnote_ref: None,
2336                        });
2337                    }
2338                }
2339                Node::Element(el) => {
2340                    let tag = el.name();
2341                    if is_metadata_tag(tag) {
2342                        continue;
2343                    }
2344                    let mut new_state = state.clone();
2345
2346                    match tag {
2347                        // Checked by attribute rather than by tag name: the
2348                        // producer picks the element (a `<sup>` renders sensibly
2349                        // in a browser), and only the attribute is a contract.
2350                        _ if el.attr(HTML_FOOTNOTE_ATTR).is_some() => {
2351                            if let Some(span) = html_footnote_span(el, new_state.link_href.clone())
2352                            {
2353                                spans.push(span);
2354                            }
2355                            continue;
2356                        }
2357                        "b" | "strong" => new_state.bold = true,
2358                        "i" | "em" => new_state.italic = true,
2359                        "u" | "ins" => new_state.underline = true,
2360                        "s" | "del" | "strike" => new_state.strikeout = true,
2361                        "code" => new_state.code = true,
2362                        "sup" => new_state.superscript = true,
2363                        "sub" => new_state.subscript = true,
2364                        "a" => {
2365                            if let Some(href) = el.attr("href") {
2366                                new_state.link_href = Some(href.to_string());
2367                            }
2368                        }
2369                        "img" => {
2370                            if let Some(span) = html_img_span(el, new_state.link_href.clone()) {
2371                                spans.push(span);
2372                            }
2373                            continue;
2374                        }
2375                        _ => {}
2376                    }
2377
2378                    // Check for nested block elements
2379                    let nested_block = matches!(
2380                        tag,
2381                        "p" | "div"
2382                            | "h1"
2383                            | "h2"
2384                            | "h3"
2385                            | "h4"
2386                            | "h5"
2387                            | "h6"
2388                            | "li"
2389                            | "pre"
2390                            | "blockquote"
2391                            | "ul"
2392                            | "ol"
2393                    );
2394
2395                    if tag == "br" {
2396                        // br within a block: treat as splitting into new block
2397                        // For simplicity, just add a newline to current span
2398                        spans.push(ParsedSpan {
2399                            text: String::new(),
2400                            ..Default::default()
2401                        });
2402                    } else if nested_block || tag == "table" {
2403                        // Flush as separate element
2404                        walk_node(
2405                            child,
2406                            &new_state,
2407                            elements,
2408                            current_list,
2409                            blockquote_depth,
2410                            list_depth,
2411                            depth + 1,
2412                            limit,
2413                        );
2414                    } else {
2415                        // Inline element: recurse
2416                        collect_inline_spans(
2417                            child,
2418                            &new_state,
2419                            spans,
2420                            current_list,
2421                            elements,
2422                            blockquote_depth,
2423                            list_depth,
2424                            depth + 1,
2425                            limit,
2426                        );
2427                    }
2428                }
2429                _ => {}
2430            }
2431        }
2432    }
2433
2434    let initial_state = FmtState::default();
2435    // Treat the root element as a block-level container so that
2436    // top-level inline elements (e.g. `<b>Bold</b> <em>Italic</em>`)
2437    // are grouped into a single block instead of becoming separate blocks.
2438    let mut root_spans: Vec<ParsedSpan> = Vec::new();
2439    collect_inline_spans(
2440        *root,
2441        &initial_state,
2442        &mut root_spans,
2443        &(None, None),
2444        &mut elements,
2445        0,
2446        0,
2447        0,
2448        depth_limit,
2449    );
2450    collapse_inline_whitespace(&mut root_spans);
2451    // Only when something survived: the root's direct text children are the
2452    // newlines between top-level blocks in every document an exporter writes.
2453    if spans_carry_content(&root_spans) {
2454        elements.push(ParsedElement::Block(ParsedBlock {
2455            spans: root_spans,
2456            heading_level: None,
2457            list_style: None,
2458            list_indent: 0,
2459            list_prefix: String::new(),
2460            list_suffix: String::new(),
2461            list_start: None,
2462            marker: None,
2463            is_code_block: false,
2464            code_language: None,
2465            blockquote_depth: 0,
2466            blockquote_opens: 0,
2467            line_height: None,
2468            non_breakable_lines: None,
2469            page_break_before: None,
2470            direction: None,
2471            background_color: None,
2472            alignment: None,
2473            top_margin: None,
2474            text_indent: None,
2475            semantic_role: None,
2476        }));
2477    }
2478
2479    // If no elements were parsed, create a single empty paragraph
2480    if elements.is_empty() {
2481        elements.push(ParsedElement::Block(ParsedBlock {
2482            spans: vec![ParsedSpan {
2483                text: String::new(),
2484                ..Default::default()
2485            }],
2486            heading_level: None,
2487            list_style: None,
2488            list_indent: 0,
2489            list_prefix: String::new(),
2490            list_suffix: String::new(),
2491            list_start: None,
2492            marker: None,
2493            is_code_block: false,
2494            code_language: None,
2495            blockquote_depth: 0,
2496            blockquote_opens: 0,
2497            line_height: None,
2498            non_breakable_lines: None,
2499            page_break_before: None,
2500            direction: None,
2501            background_color: None,
2502            alignment: None,
2503            top_margin: None,
2504            text_indent: None,
2505            semantic_role: None,
2506        }));
2507    }
2508
2509    elements
2510}
2511
2512/// Convert a `ParsedSpan` (parser output) into the `CharacterFormat` used by
2513/// `FormatRun`. `is_code_block` forces `monospace` as the font family for
2514/// every span inside a code block.
2515pub fn character_format_from_span(
2516    span: &ParsedSpan,
2517    is_code_block: bool,
2518) -> crate::format_runs::CharacterFormat {
2519    use crate::entities::CharVerticalAlignment;
2520    crate::format_runs::CharacterFormat {
2521        font_bold: if span.bold { Some(true) } else { None },
2522        font_italic: if span.italic { Some(true) } else { None },
2523        font_underline: if span.underline { Some(true) } else { None },
2524        font_strikeout: if span.strikeout { Some(true) } else { None },
2525        font_family: if span.code || is_code_block {
2526            Some("monospace".to_string())
2527        } else {
2528            None
2529        },
2530        anchor_href: span.link_href.clone(),
2531        is_anchor: if span.link_href.is_some() {
2532            Some(true)
2533        } else {
2534            None
2535        },
2536        vertical_alignment: if span.superscript {
2537            Some(CharVerticalAlignment::SuperScript)
2538        } else if span.subscript {
2539            Some(CharVerticalAlignment::SubScript)
2540        } else {
2541            None
2542        },
2543        ..Default::default()
2544    }
2545}
2546
2547/// Translate a slice of parsed spans into `(plain_text, format_runs)`.
2548///
2549/// One non-default span yields one `FormatRun`; spans with empty
2550/// `CharacterFormat` (no decoration, no link, no code) emit no run, since an
2551/// absent run means "inherit default formatting" in the new model. Adjacent
2552/// runs with identical formats are coalesced via `coalesce_in_place` so the
2553/// resulting vector satisfies `debug_assert_well_formed`.
2554///
2555/// Returns the concatenated `plain_text` of all spans and a sorted,
2556/// non-overlapping, coalesced `Vec<FormatRun>`. Both safe to feed straight
2557/// into the store under the dual-write bridge.
2558pub fn format_runs_from_spans(spans: &[ParsedSpan], is_code_block: bool) -> ParsedInline {
2559    use crate::format_runs::{
2560        CharacterFormat, FootnoteRefAnchor, FormatRun, ImageAnchor, coalesce_in_place,
2561    };
2562
2563    let mut plain_text = String::new();
2564    let mut runs: Vec<FormatRun> = Vec::new();
2565    let mut images: Vec<ImageAnchor> = Vec::new();
2566    let mut footnote_refs: Vec<FootnoteRefAnchor> = Vec::new();
2567    let default = CharacterFormat::default();
2568
2569    for span in spans {
2570        let byte_start = plain_text.len() as u32;
2571
2572        if let Some(label) = &span.footnote_ref {
2573            // A reference occupies one U+FFFC, exactly as an image does, so
2574            // every downstream offset treats the two alike.
2575            plain_text.push('\u{FFFC}');
2576            // Raised, always — whatever the surrounding run is doing.
2577            //
2578            // A footnote marker is superscript by definition, in every
2579            // typographic tradition and in every reader that renders djot. The
2580            // ambient `superscript` flag the span carries is the *prose's*, and
2581            // prose is not superscript, so taking it verbatim sets a note's
2582            // number on the baseline in the middle of a sentence — which reads
2583            // as a stray digit the writer typed rather than as a reference.
2584            //
2585            // It is set here, on the anchor, rather than at render time so that
2586            // every consumer agrees: the editor raises it, the exporters that
2587            // carry character formatting carry it, and the djot writer knows to
2588            // emit `[^label]` *without* wrapping it in `^…^` (see
2589            // `a_reference_is_not_wrapped_in_superscript_markers`).
2590            let mut format = character_format_from_span(span, is_code_block);
2591            format.vertical_alignment = Some(crate::entities::CharVerticalAlignment::SuperScript);
2592            footnote_refs.push(FootnoteRefAnchor {
2593                byte_offset: byte_start,
2594                label: label.clone(),
2595                format,
2596            });
2597            continue;
2598        }
2599
2600        if let Some(image) = &span.image {
2601            // An image occupies one U+FFFC in the text, exactly as
2602            // `insert_image` mirrors into the rope, so every downstream offset
2603            // calculation treats a parsed image and an inserted one alike.
2604            plain_text.push('\u{FFFC}');
2605            images.push(ImageAnchor {
2606                byte_offset: byte_start,
2607                name: image.src.clone(),
2608                alt: image.alt.clone(),
2609                width: image.width,
2610                height: image.height,
2611                quality: 100,
2612                format: character_format_from_span(span, is_code_block),
2613            });
2614            continue;
2615        }
2616
2617        plain_text.push_str(&span.text);
2618        let byte_end = plain_text.len() as u32;
2619        if byte_start == byte_end {
2620            continue;
2621        }
2622        let format = character_format_from_span(span, is_code_block);
2623        if format == default {
2624            continue;
2625        }
2626        runs.push(FormatRun {
2627            byte_start,
2628            byte_end,
2629            format,
2630        });
2631    }
2632    coalesce_in_place(&mut runs);
2633    ParsedInline {
2634        plain_text,
2635        runs,
2636        images,
2637        footnote_refs,
2638    }
2639}
2640
2641/// The three parallel things a block stores, as recovered from parsed spans.
2642///
2643/// Returned as a struct rather than a tuple because it grew a third member
2644/// (images) after nine call sites already destructured a pair — and every one
2645/// of those sites has to decide what to do with images, so a silent
2646/// tuple-arity change would have been the wrong kind of easy.
2647#[derive(Debug, Clone, Default)]
2648pub struct ParsedInline {
2649    pub plain_text: String,
2650    pub runs: Vec<crate::format_runs::FormatRun>,
2651    pub images: Vec<crate::format_runs::ImageAnchor>,
2652    pub footnote_refs: Vec<crate::format_runs::FootnoteRefAnchor>,
2653}
2654
2655// ─── Djot parsing ────────────────────────────────────────────────────
2656
2657/// Map a jotdown unordered/task bullet marker to a model `ListStyle`.
2658///
2659/// The mapping is a stable bijection (`-`↔Disc, `*`↔Circle, `+`↔Square) so the
2660/// djot exporter can recover the exact bullet character for a lossless
2661/// round-trip.
2662fn djot_bullet_style(b: jotdown::ListBulletType) -> ListStyle {
2663    use jotdown::ListBulletType as B;
2664    match b {
2665        B::Dash => ListStyle::Disc,
2666        B::Star => ListStyle::Circle,
2667        B::Plus => ListStyle::Square,
2668    }
2669}
2670
2671/// Map a jotdown ordered-list numbering scheme to a model `ListStyle`.
2672fn djot_ordered_style(n: jotdown::OrderedListNumbering) -> ListStyle {
2673    use jotdown::OrderedListNumbering as N;
2674    match n {
2675        N::Decimal => ListStyle::Decimal,
2676        N::AlphaLower => ListStyle::LowerAlpha,
2677        N::AlphaUpper => ListStyle::UpperAlpha,
2678        N::RomanLower => ListStyle::LowerRoman,
2679        N::RomanUpper => ListStyle::UpperRoman,
2680    }
2681}
2682
2683/// Map a jotdown ordered-list delimiter to the `(prefix, suffix)` affixes
2684/// stored on the `List` entity (`1.` → `("", ".")`, `1)` → `("", ")")`,
2685/// `(1)` → `("(", ")")`).
2686fn djot_ordered_affixes(style: jotdown::OrderedListStyle) -> (String, String) {
2687    use jotdown::OrderedListStyle as S;
2688    match style {
2689        S::Period => (String::new(), ".".to_string()),
2690        S::Paren => (String::new(), ")".to_string()),
2691        S::ParenParen => ("(".to_string(), ")".to_string()),
2692    }
2693}
2694
2695/// Optional block-level style attributes carried on a djot block through its
2696/// `{key=value}` block attributes. All `None` when the block has no such
2697/// attributes (or they were filtered out by [`DjotImportOptions`]).
2698#[derive(Debug, Clone, Default)]
2699struct DjotBlockStyle {
2700    alignment: Option<Alignment>,
2701    line_height: Option<i64>,
2702    non_breakable_lines: Option<bool>,
2703    page_break_before: Option<bool>,
2704    direction: Option<TextDirection>,
2705    background_color: Option<String>,
2706    top_margin: Option<i64>,
2707    text_indent: Option<i64>,
2708    semantic_role: Option<SemanticRole>,
2709}
2710
2711impl DjotBlockStyle {
2712    /// Overlay the `Some` fields of `other` onto `self`, leaving `self`'s
2713    /// existing values for any field `other` does not set. Used to combine a
2714    /// heading's enclosing-`Section` attributes with any on the heading itself.
2715    fn merge_from(&mut self, other: DjotBlockStyle) {
2716        if other.alignment.is_some() {
2717            self.alignment = other.alignment;
2718        }
2719        if other.line_height.is_some() {
2720            self.line_height = other.line_height;
2721        }
2722        if other.non_breakable_lines.is_some() {
2723            self.non_breakable_lines = other.non_breakable_lines;
2724        }
2725        if other.page_break_before.is_some() {
2726            self.page_break_before = other.page_break_before;
2727        }
2728        if other.direction.is_some() {
2729            self.direction = other.direction;
2730        }
2731        if other.background_color.is_some() {
2732            self.background_color = other.background_color;
2733        }
2734        if other.top_margin.is_some() {
2735            self.top_margin = other.top_margin;
2736        }
2737        if other.text_indent.is_some() {
2738            self.text_indent = other.text_indent;
2739        }
2740        if other.semantic_role.is_some() {
2741            self.semantic_role = other.semantic_role.clone();
2742        }
2743    }
2744}
2745
2746/// Read the round-tripped block-style attributes off a djot block's
2747/// [`jotdown::Attributes`], honouring the import [`DjotImportOptions`]. Keys are
2748/// the model field names (`alignment`, `line_height`, `direction`,
2749/// `non_breakable_lines`, `page_break_before`, `background_color`, `top_margin`,
2750/// `text_indent`, `semantic_role`); unrecognised values are ignored.
2751fn block_attrs_to_style(attrs: &jotdown::Attributes, opts: &DjotImportOptions) -> DjotBlockStyle {
2752    let mut style = DjotBlockStyle::default();
2753
2754    if opts.alignment
2755        && let Some(v) = attrs.get_value("alignment")
2756    {
2757        style.alignment = match v.to_string().as_str() {
2758            "left" => Some(Alignment::Left),
2759            "right" => Some(Alignment::Right),
2760            "center" => Some(Alignment::Center),
2761            "justify" => Some(Alignment::Justify),
2762            _ => None,
2763        };
2764    }
2765    if opts.line_height
2766        && let Some(v) = attrs.get_value("line_height")
2767    {
2768        style.line_height = v.to_string().parse::<i64>().ok();
2769    }
2770    if opts.direction
2771        && let Some(v) = attrs.get_value("direction")
2772    {
2773        style.direction = match v.to_string().as_str() {
2774            "ltr" => Some(TextDirection::LeftToRight),
2775            "rtl" => Some(TextDirection::RightToLeft),
2776            _ => None,
2777        };
2778    }
2779    if opts.non_breakable_lines
2780        && let Some(v) = attrs.get_value("non_breakable_lines")
2781    {
2782        style.non_breakable_lines = match v.to_string().as_str() {
2783            "true" => Some(true),
2784            "false" => Some(false),
2785            _ => None,
2786        };
2787    }
2788    if opts.page_break_before
2789        && let Some(v) = attrs.get_value("page_break_before")
2790    {
2791        style.page_break_before = match v.to_string().as_str() {
2792            "true" => Some(true),
2793            "false" => Some(false),
2794            _ => None,
2795        };
2796    }
2797    if opts.background_color
2798        && let Some(v) = attrs.get_value("background_color")
2799    {
2800        style.background_color = Some(v.to_string());
2801    }
2802    if opts.top_margin
2803        && let Some(v) = attrs.get_value("top_margin")
2804    {
2805        style.top_margin = v.to_string().parse::<i64>().ok();
2806    }
2807    if opts.text_indent
2808        && let Some(v) = attrs.get_value("text_indent")
2809    {
2810        style.text_indent = v.to_string().parse::<i64>().ok();
2811    }
2812    if opts.semantic_role
2813        && let Some(v) = attrs.get_value("semantic_role")
2814    {
2815        style.semantic_role = match v.to_string().as_str() {
2816            "epigraph" => Some(SemanticRole::Epigraph),
2817            // An unknown role is dropped, not guessed at — the same way an unknown
2818            // alignment value above is. A future role read by an older build then
2819            // degrades to a plain blockquote, which is what it looks like anyway.
2820            _ => None,
2821        };
2822    }
2823
2824    style
2825}
2826
2827/// The start a list keeps on its entity, from the number its first item is written with:
2828/// `None` for 1, which every list starts at unless it says otherwise, and a number past what
2829/// every format holds read as the largest it holds (see `list_start`).
2830fn list_start_of(start: u64) -> Option<i64> {
2831    crate::parser_tools::list_start::list_start_from_unsigned(start)
2832}
2833
2834/// Push a finished block into `elements`, applying the djot block-level fields
2835/// plus any round-tripped block-style attributes carried in `style`.
2836#[allow(clippy::too_many_arguments)]
2837fn djot_push_block(
2838    elements: &mut Vec<ParsedElement>,
2839    spans: Vec<ParsedSpan>,
2840    heading_level: Option<i64>,
2841    list_style: Option<ListStyle>,
2842    list_indent: u32,
2843    list_prefix: String,
2844    list_suffix: String,
2845    list_start: Option<i64>,
2846    marker: Option<MarkerType>,
2847    is_code_block: bool,
2848    code_language: Option<String>,
2849    blockquote_depth: u32,
2850    style: DjotBlockStyle,
2851) {
2852    elements.push(ParsedElement::Block(ParsedBlock {
2853        spans,
2854        heading_level,
2855        list_style,
2856        list_indent,
2857        list_prefix,
2858        list_suffix,
2859        list_start,
2860        marker,
2861        is_code_block,
2862        code_language,
2863        blockquote_depth,
2864        blockquote_opens: 0,
2865        line_height: style.line_height,
2866        non_breakable_lines: style.non_breakable_lines,
2867        page_break_before: style.page_break_before,
2868        direction: style.direction,
2869        background_color: style.background_color,
2870        alignment: style.alignment,
2871        top_margin: style.top_margin,
2872        text_indent: style.text_indent,
2873        semantic_role: style.semantic_role.clone(),
2874    }));
2875}
2876
2877/// A Djot document whose structure cannot be computed, as its source: one plain
2878/// paragraph for each line that holds text, with that line's text as it is written, its
2879/// leading and trailing whitespace aside.
2880///
2881/// A paragraph a line is what keeps the text the same through a save. The Djot writer
2882/// escapes whatever a paragraph opens with and whatever in it would read as markup, so
2883/// the saved text reads back as these same paragraphs. Shown as one paragraph, the
2884/// source kept its line breaks through a save, read back as the same document the parser
2885/// could not take, and every save escaped its backslashes once more. Blank lines are left
2886/// out because the reader keeps no block for an empty paragraph, and a line's outer
2887/// whitespace because it is the source's indentation, which the parser does not read as
2888/// text in a line it can take either.
2889fn djot_source_lines(djot: &str) -> Vec<ParsedElement> {
2890    djot.lines()
2891        .map(|line| line.trim_matches(|c: char| c.is_ascii_whitespace()))
2892        .filter(|line| !line.is_empty())
2893        .map(|line| {
2894            ParsedElement::Block(ParsedBlock {
2895                spans: vec![ParsedSpan {
2896                    text: line.to_string(),
2897                    ..Default::default()
2898                }],
2899                ..Default::default()
2900            })
2901        })
2902        .collect()
2903}
2904
2905/// Parse djot source into the shared [`ParsedElement`] intermediate, mirroring
2906/// [`parse_markdown`]. Uses the [`jotdown`] pull parser.
2907///
2908/// Constructs the document model cannot represent are dropped, and their text
2909/// content is discarded so it never leaks into the document: footnotes, math,
2910/// fenced divs, raw blocks/inline, thematic breaks, description lists,
2911/// captions, symbols, link-reference definitions, and highlight/`mark`. Inline
2912/// images keep their alt text as plain text (the image itself is not modelled),
2913/// matching the Markdown importer. Smart-punctuation events are normalised to
2914/// their canonical Unicode characters so the model→djot→model round-trip is a
2915/// fixpoint.
2916///
2917/// Standalone paragraphs and headings additionally carry the optional
2918/// block-style attributes selected by `options` — paragraph alignment, line
2919/// height, text direction, non-breakable lines and background color — read from
2920/// djot `{key=value}` block attributes (see [`DjotImportOptions`]). List items,
2921/// code blocks and table cells normalise their block styling away.
2922///
2923/// Known model limitations (normalised, not preserved on round-trip):
2924/// table column alignment, and list tight/loose.
2925pub fn parse_djot(djot: &str, options: &DjotImportOptions) -> Vec<ParsedElement> {
2926    use jotdown::{Container as C, Event as E, ListKind};
2927
2928    // `jotdown` recurses with no limit of its own, once per nested container and
2929    // once per line of a paragraph that leaves an inline opener waiting, and a
2930    // stack overflow **aborts the process**: it cannot be caught by
2931    // `catch_unwind`, so there is no recovering from it after the fact and the
2932    // only safe move is not to start. It also panics on a heading deeper than it
2933    // can number, which a caller on an interface thread does not survive either.
2934    // Input is not always the author's own: a project bundle is mailed and
2935    // shared, and an imported document comes from whoever sent it.
2936    //
2937    // `djot_depth::parsable` is the door that keeps the parser within its limits,
2938    // and it degrades rather than refuses, so the signature stays the same and no
2939    // word is lost. Such a heading is escaped into text, as it was written. A
2940    // document nested too deeply by its indentation, which is how a deeply nested
2941    // list is written, is read with the indentation cut: the list comes out
2942    // flattened below a fixed depth, every item and word kept. This used to hand
2943    // back a pasted list nested two hundred levels as one paragraph of markup. A
2944    // long paragraph is parsed on a stack sized for it, and the block holding one
2945    // too long for any stack is shown as its lines, the rest of the document as it
2946    // was.
2947    //
2948    // A document still nested too deeply once flattened, or one the parser panicked
2949    // on, comes back as its lines, one plain paragraph each (see
2950    // `djot_source_lines`). Its structure is not computed, which is the honest
2951    // answer for a document whose structure cannot be computed without ending the
2952    // process, and its text is still there for the writer to see and repair.
2953    let parsable = crate::parser_tools::djot_depth::parsable(djot);
2954    let Some(events) = parsable.as_ref().and_then(|parsable| parsable.events()) else {
2955        return djot_source_lines(djot);
2956    };
2957
2958    let mut elements: Vec<ParsedElement> = Vec::new();
2959    let mut current_spans: Vec<ParsedSpan> = Vec::new();
2960    let mut current_heading: Option<i64> = None;
2961    let mut is_code_block = false;
2962    let mut code_language: Option<String> = None;
2963    let mut blockquote_depth: u32 = 0;
2964    // Where `elements` stood as each open quotation began (see `mark_quotation_opened`).
2965    let mut quotations_open: Vec<usize> = Vec::new();
2966    // Block-style attributes captured from a standalone paragraph/heading's djot
2967    // `{…}` block attributes, consumed when that block is flushed.
2968    let mut pending_style = DjotBlockStyle::default();
2969
2970    // Inline formatting state.
2971    let mut bold = false;
2972    let mut italic = false;
2973    let mut underline = false;
2974    let mut strikeout = false;
2975    let mut code = false;
2976    let mut superscript = false;
2977    let mut subscript = false;
2978    let mut link_href: Option<String> = None;
2979    // Set between an image's Start and End. Its alt text arrives as ordinary
2980    // `Str` events in between, so it has to be captured rather than emitted.
2981    let mut pending_image: Option<ParsedImage> = None;
2982
2983    // List nesting: each entry is (style, prefix, suffix, start); depth = indent + 1.
2984    let mut list_stack: Vec<(ListStyle, String, String, Option<i64>)> = Vec::new();
2985    // Context applied to the next flushed block while inside a list item.
2986    let mut cur_list_style: Option<ListStyle> = None;
2987    let mut cur_list_prefix = String::new();
2988    let mut cur_list_suffix = String::new();
2989    let mut cur_list_start: Option<i64> = None;
2990    let mut cur_list_indent: u32 = 0;
2991    let mut cur_marker: Option<MarkerType> = None;
2992
2993    // Table accumulation.
2994    let mut in_table_cell = false;
2995    let mut table_rows: Vec<Vec<ParsedTableCell>> = Vec::new();
2996    let mut current_row: Vec<ParsedTableCell> = Vec::new();
2997    let mut current_cell_spans: Vec<ParsedSpan> = Vec::new();
2998    let mut table_header_rows: usize = 0;
2999    let mut row_is_head = false;
3000
3001    // Subtree-skip depth for unrepresentable containers (their entire content
3002    // is dropped). Incremented on the dropped container's `Start` and on every
3003    // nested `Start`; decremented on every `End`.
3004    let mut skip_depth: u32 = 0;
3005
3006    // The label and element index of the footnote definition currently open,
3007    // if any. Djot has no footnote inside a footnote, so one slot is enough
3008    // where the dropped containers need a depth counter.
3009    let mut footnote_open: Option<(String, usize)> = None;
3010
3011    // Push one inline span carrying the current formatting state into the
3012    // active sink (table cell or block). A macro (not a closure) to avoid
3013    // borrowing `current_spans`/`current_cell_spans` across the formatting
3014    // state reads.
3015    macro_rules! push_text {
3016        ($t:expr) => {{
3017            // Alt text belongs to the image, not to the paragraph. While an
3018            // image is open every text event is diverted into its description,
3019            // which is what keeps a photo's caption out of the manuscript's
3020            // word count and out of the search corpus.
3021            if let Some(img) = pending_image.as_mut() {
3022                img.alt.push_str(($t).as_ref());
3023            } else {
3024                let sp = ParsedSpan {
3025                    text: ($t).to_string(),
3026                    bold,
3027                    italic,
3028                    underline,
3029                    strikeout,
3030                    code,
3031                    superscript,
3032                    subscript,
3033                    link_href: link_href.clone(),
3034                    image: None,
3035                    footnote_ref: None,
3036                };
3037                if in_table_cell {
3038                    current_cell_spans.push(sp);
3039                } else {
3040                    current_spans.push(sp);
3041                }
3042            }
3043        }};
3044    }
3045
3046    // Push a completed inline image span into the active sink.
3047    macro_rules! push_image {
3048        ($img:expr) => {{
3049            let sp = ParsedSpan {
3050                text: String::new(),
3051                bold,
3052                italic,
3053                underline,
3054                strikeout,
3055                code,
3056                superscript,
3057                subscript,
3058                link_href: link_href.clone(),
3059                image: Some($img),
3060                footnote_ref: None,
3061            };
3062            if in_table_cell {
3063                current_cell_spans.push(sp);
3064            } else {
3065                current_spans.push(sp);
3066            }
3067        }};
3068    }
3069
3070    // Enter a list item, flushing any unterminated inline content first and
3071    // capturing the list context + task marker for the item's block.
3072    macro_rules! enter_item {
3073        ($marker:expr) => {{
3074            if !current_spans.is_empty() {
3075                djot_push_block(
3076                    &mut elements,
3077                    std::mem::take(&mut current_spans),
3078                    None,
3079                    cur_list_style.clone(),
3080                    cur_list_indent,
3081                    cur_list_prefix.clone(),
3082                    cur_list_suffix.clone(),
3083                    cur_list_start,
3084                    cur_marker.clone(),
3085                    false,
3086                    None,
3087                    blockquote_depth,
3088                    DjotBlockStyle::default(),
3089                );
3090            }
3091            let (style, prefix, suffix, start) = list_stack.last().cloned().unwrap_or((
3092                ListStyle::Disc,
3093                String::new(),
3094                String::new(),
3095                None,
3096            ));
3097            cur_list_style = Some(style);
3098            cur_list_prefix = prefix;
3099            cur_list_suffix = suffix;
3100            cur_list_start = start;
3101            cur_list_indent = list_stack.len().saturating_sub(1) as u32;
3102            cur_marker = $marker;
3103        }};
3104    }
3105
3106    // A line break inside a block: a new block, which is what the model means by one,
3107    // with the list, quotation, heading and block style the line was in. A table cell
3108    // is one line, so there it is a space.
3109    //
3110    // The page break is the one style that stays with the first line: it is where the
3111    // block starts, and copied to every line it put each on a page of its own.
3112    macro_rules! line_break {
3113        () => {{
3114            if in_table_cell {
3115                push_text!(" ");
3116            } else if !current_spans.is_empty() && current_heading.is_some() {
3117                // The line before a hard break in a heading is a line of that
3118                // heading, as the line after it is when the heading ends. Read as
3119                // a paragraph, a heading written over two lines came back as a
3120                // paragraph and a heading.
3121                djot_push_block(
3122                    &mut elements,
3123                    std::mem::take(&mut current_spans),
3124                    current_heading,
3125                    None,
3126                    0,
3127                    String::new(),
3128                    String::new(),
3129                    None,
3130                    None,
3131                    false,
3132                    None,
3133                    blockquote_depth,
3134                    pending_style.clone(),
3135                );
3136                pending_style.page_break_before = None;
3137            } else if !current_spans.is_empty() {
3138                // Mirrors the Markdown importer: a hard break splits the
3139                // paragraph into a new block.
3140                djot_push_block(
3141                    &mut elements,
3142                    std::mem::take(&mut current_spans),
3143                    None,
3144                    cur_list_style.clone(),
3145                    cur_list_indent,
3146                    cur_list_prefix.clone(),
3147                    cur_list_suffix.clone(),
3148                    cur_list_start,
3149                    cur_marker.clone(),
3150                    is_code_block,
3151                    code_language.clone(),
3152                    blockquote_depth,
3153                    pending_style.clone(),
3154                );
3155                pending_style.page_break_before = None;
3156            }
3157        }};
3158    }
3159
3160    for event in events {
3161        if skip_depth > 0 {
3162            match event {
3163                E::Start(..) => skip_depth += 1,
3164                E::End(_) => skip_depth -= 1,
3165                _ => {}
3166            }
3167            continue;
3168        }
3169
3170        match event {
3171            // ── Transparent wrappers (unwrap, keep content) ──
3172            E::Start(C::Document, _) | E::End(C::Document) => {}
3173            E::Start(C::Section { .. }, attrs) => {
3174                // A heading's block attributes attach to its enclosing Section,
3175                // not the heading itself; capture them for the heading's flush.
3176                if list_stack.is_empty() {
3177                    pending_style.merge_from(block_attrs_to_style(&attrs, options));
3178                }
3179            }
3180            E::End(C::Section { .. }) => {}
3181            E::Start(C::Div { .. }, _) | E::End(C::Div { .. }) => {}
3182
3183            // ── Blockquote ──
3184            E::Start(C::Blockquote, _) => {
3185                blockquote_depth += 1;
3186                quotations_open.push(elements.len());
3187            }
3188            E::End(C::Blockquote) => {
3189                blockquote_depth = blockquote_depth.saturating_sub(1);
3190                if let Some(first) = quotations_open.pop() {
3191                    mark_quotation_opened(&mut elements, first);
3192                }
3193            }
3194
3195            // ── Lists ──
3196            E::Start(C::List { kind, .. }, _) => {
3197                let (style, prefix, suffix, start) = match kind {
3198                    ListKind::Unordered(b) | ListKind::Task(b) => {
3199                        (djot_bullet_style(b), String::new(), String::new(), None)
3200                    }
3201                    ListKind::Ordered {
3202                        numbering,
3203                        style,
3204                        start,
3205                    } => {
3206                        let (p, s) = djot_ordered_affixes(style);
3207                        (djot_ordered_style(numbering), p, s, list_start_of(start))
3208                    }
3209                };
3210                list_stack.push((style, prefix, suffix, start));
3211            }
3212            E::End(C::List { .. }) => {
3213                list_stack.pop();
3214                cur_list_style = None;
3215                cur_list_start = None;
3216                cur_marker = None;
3217            }
3218            E::Start(C::ListItem, _) => enter_item!(None),
3219            E::Start(C::TaskListItem { checked }, _) => enter_item!(Some(if checked {
3220                MarkerType::Checked
3221            } else {
3222                MarkerType::Unchecked
3223            })),
3224            E::End(C::ListItem) | E::End(C::TaskListItem { .. }) => {
3225                // Tight item without a wrapping paragraph (defensive flush).
3226                if !current_spans.is_empty() {
3227                    djot_push_block(
3228                        &mut elements,
3229                        std::mem::take(&mut current_spans),
3230                        None,
3231                        cur_list_style.clone(),
3232                        cur_list_indent,
3233                        cur_list_prefix.clone(),
3234                        cur_list_suffix.clone(),
3235                        cur_list_start,
3236                        cur_marker.clone(),
3237                        false,
3238                        None,
3239                        blockquote_depth,
3240                        DjotBlockStyle::default(),
3241                    );
3242                }
3243                cur_list_style = None;
3244                cur_list_start = None;
3245                cur_marker = None;
3246            }
3247
3248            // ── Headings, paragraphs, code blocks ──
3249            E::Start(C::Heading { level, .. }, attrs) => {
3250                current_heading = Some(level as i64);
3251                // The block-style attributes live on the enclosing Section;
3252                // merge any placed directly on the heading without clearing them.
3253                pending_style.merge_from(block_attrs_to_style(&attrs, options));
3254            }
3255            E::End(C::Heading { .. }) => {
3256                djot_push_block(
3257                    &mut elements,
3258                    std::mem::take(&mut current_spans),
3259                    current_heading.take(),
3260                    None,
3261                    0,
3262                    String::new(),
3263                    String::new(),
3264                    None,
3265                    None,
3266                    false,
3267                    None,
3268                    blockquote_depth,
3269                    std::mem::take(&mut pending_style),
3270                );
3271            }
3272            E::Start(C::Paragraph, attrs) => {
3273                current_heading = None;
3274                // Block attributes only apply to standalone paragraphs;
3275                // list-item paragraphs normalise their styling away (matching
3276                // the exporter).
3277                pending_style = if list_stack.is_empty() {
3278                    block_attrs_to_style(&attrs, options)
3279                } else {
3280                    DjotBlockStyle::default()
3281                };
3282            }
3283            E::End(C::Paragraph) => {
3284                if !current_spans.is_empty() {
3285                    djot_push_block(
3286                        &mut elements,
3287                        std::mem::take(&mut current_spans),
3288                        None,
3289                        cur_list_style.clone(),
3290                        cur_list_indent,
3291                        cur_list_prefix.clone(),
3292                        cur_list_suffix.clone(),
3293                        cur_list_start,
3294                        cur_marker.clone(),
3295                        false,
3296                        None,
3297                        blockquote_depth,
3298                        std::mem::take(&mut pending_style),
3299                    );
3300                }
3301                cur_list_style = None;
3302                cur_list_start = None;
3303                cur_marker = None;
3304            }
3305            E::Start(C::CodeBlock { language }, _) => {
3306                is_code_block = true;
3307                code_language = if language.is_empty() {
3308                    None
3309                } else {
3310                    Some(language.to_string())
3311                };
3312            }
3313            E::End(C::CodeBlock { .. }) => {
3314                // Strip the single trailing newline jotdown appends.
3315                if let Some(last) = current_spans.last_mut()
3316                    && last.text.ends_with('\n')
3317                {
3318                    last.text.pop();
3319                }
3320                djot_push_block(
3321                    &mut elements,
3322                    std::mem::take(&mut current_spans),
3323                    None,
3324                    None,
3325                    0,
3326                    String::new(),
3327                    String::new(),
3328                    None,
3329                    None,
3330                    true,
3331                    code_language.take(),
3332                    blockquote_depth,
3333                    DjotBlockStyle::default(),
3334                );
3335                is_code_block = false;
3336            }
3337
3338            // ── Tables ──
3339            E::Start(C::Table, _) => {
3340                table_rows.clear();
3341                current_row.clear();
3342                current_cell_spans.clear();
3343                table_header_rows = 0;
3344            }
3345            E::End(C::Table) => {
3346                let own_cells = cells_in_rows(&table_rows);
3347                push_table(
3348                    &mut elements,
3349                    ParsedTable {
3350                        header_rows: table_header_rows,
3351                        rows: std::mem::take(&mut table_rows),
3352                        blockquote_depth,
3353                        blockquote_opens: 0,
3354                    },
3355                    own_cells,
3356                );
3357            }
3358            E::Start(C::TableRow { head }, _) => {
3359                row_is_head = head;
3360                current_row.clear();
3361            }
3362            E::End(C::TableRow { .. }) => {
3363                if row_is_head {
3364                    table_header_rows += 1;
3365                }
3366                table_rows.push(std::mem::take(&mut current_row));
3367            }
3368            E::Start(C::TableCell { .. }, _) => {
3369                in_table_cell = true;
3370                current_cell_spans.clear();
3371            }
3372            E::End(C::TableCell { .. }) => {
3373                in_table_cell = false;
3374                current_row.push(ParsedTableCell {
3375                    spans: std::mem::take(&mut current_cell_spans),
3376                });
3377            }
3378
3379            // ── Inline formatting ──
3380            E::Start(C::Strong, _) => bold = true,
3381            E::End(C::Strong) => bold = false,
3382            E::Start(C::Emphasis, _) => italic = true,
3383            E::End(C::Emphasis) => italic = false,
3384            E::Start(C::Verbatim, _) => code = true,
3385            E::End(C::Verbatim) => code = false,
3386            E::Start(C::Superscript, _) => superscript = true,
3387            E::End(C::Superscript) => superscript = false,
3388            E::Start(C::Subscript, _) => subscript = true,
3389            E::End(C::Subscript) => subscript = false,
3390            E::Start(C::Insert, _) => underline = true,
3391            E::End(C::Insert) => underline = false,
3392            E::Start(C::Delete, _) => strikeout = true,
3393            E::End(C::Delete) => strikeout = false,
3394            // Highlight/mark and bare spans have no model field — keep the text.
3395            E::Start(C::Mark, _) | E::End(C::Mark) => {}
3396            E::Start(C::Span, _) | E::End(C::Span) => {}
3397            E::Start(C::Link(dst, _), _) => link_href = Some(dst.to_string()),
3398            E::End(C::Link(..)) => link_href = None,
3399            // Inline images. Djot writes display size as inline attributes
3400            // (`![alt](src){width=800 height=600}`), which jotdown hands over
3401            // on the `Start` event — verified against jotdown 0.10, including
3402            // quoted values and images mid-sentence.
3403            E::Start(C::Image(src, _), attrs) => {
3404                let attr_num = |key: &str| -> i64 {
3405                    attrs
3406                        .get_value(key)
3407                        .map(|v| v.to_string())
3408                        .and_then(|v| v.trim().parse::<i64>().ok())
3409                        .filter(|n| *n > 0)
3410                        .unwrap_or(0)
3411                };
3412                pending_image = Some(ParsedImage {
3413                    // The writer percent-encodes what the lexer would act on in a
3414                    // source; the source is a name, so it is decoded back.
3415                    src: crate::parser_tools::djot_escape::decode_djot_image_source(&src),
3416                    alt: String::new(),
3417                    width: attr_num("width"),
3418                    height: attr_num("height"),
3419                });
3420            }
3421            E::End(C::Image(..)) => {
3422                if let Some(img) = pending_image.take() {
3423                    push_image!(img);
3424                }
3425            }
3426
3427            // ── Footnote definitions ──
3428            //
3429            // The body is ordinary block content, so it is parsed by the
3430            // ordinary machinery: flush whatever inline run is open, note where
3431            // this definition's blocks start, and let them accumulate. `End`
3432            // lifts them back out. Nesting cannot occur — djot has no footnote
3433            // inside a footnote — so one mark suffices where the dropped
3434            // containers below need a depth counter.
3435            E::Start(C::Footnote { label }, _) => {
3436                if !current_spans.is_empty() {
3437                    djot_push_block(
3438                        &mut elements,
3439                        std::mem::take(&mut current_spans),
3440                        None,
3441                        cur_list_style.clone(),
3442                        cur_list_indent,
3443                        cur_list_prefix.clone(),
3444                        cur_list_suffix.clone(),
3445                        cur_list_start,
3446                        cur_marker.clone(),
3447                        false,
3448                        None,
3449                        blockquote_depth,
3450                        DjotBlockStyle::default(),
3451                    );
3452                }
3453                footnote_open = Some((label.to_string(), elements.len()));
3454            }
3455            E::End(C::Footnote { .. }) => {
3456                if !current_spans.is_empty() {
3457                    djot_push_block(
3458                        &mut elements,
3459                        std::mem::take(&mut current_spans),
3460                        None,
3461                        cur_list_style.clone(),
3462                        cur_list_indent,
3463                        cur_list_prefix.clone(),
3464                        cur_list_suffix.clone(),
3465                        cur_list_start,
3466                        cur_marker.clone(),
3467                        false,
3468                        None,
3469                        blockquote_depth,
3470                        DjotBlockStyle::default(),
3471                    );
3472                }
3473                if let Some((label, start)) = footnote_open.take() {
3474                    let blocks: Vec<ParsedBlock> = elements
3475                        .drain(start..)
3476                        .filter_map(|e| match e {
3477                            ParsedElement::Block(b) => Some(b),
3478                            // A table inside a footnote is not representable as
3479                            // note content; its cells would have to become
3480                            // blocks and lose their structure either way.
3481                            _ => None,
3482                        })
3483                        .collect();
3484                    elements.push(ParsedElement::FootnoteDefinition { label, blocks });
3485                }
3486            }
3487
3488            // ── Unrepresentable containers: drop the entire subtree ──
3489            E::Start(
3490                C::Math { .. }
3491                | C::RawBlock { .. }
3492                | C::RawInline { .. }
3493                | C::DescriptionList
3494                | C::DescriptionDetails
3495                | C::DescriptionTerm
3496                | C::Caption
3497                | C::LinkDefinition { .. },
3498                _,
3499            ) => skip_depth = 1,
3500
3501            // ── Text + atoms ──
3502            // A line ending inside a code span is kept by the parser as a line
3503            // break in the span's text, the one a paragraph's text can hold. It is
3504            // read as a hard break is, a new block with the list, quotation or
3505            // heading the line was in, so the model holds no line break inside a
3506            // block. Kept inside the span, the Djot writer saved the code run's
3507            // lines as blocks of their own, which reloaded as two blocks: a document
3508            // that did not settle into the Djot it saves as. Such a span is what an
3509            // older version wrote for a code run holding a line break, a pasted
3510            // `<pre><code>` passage above all, and reading its lines as blocks is how
3511            // that passage reads in the editor.
3512            E::Str(s) if code && !is_code_block && pending_image.is_none() && s.contains('\n') => {
3513                for (i, piece) in s.split('\n').enumerate() {
3514                    if i > 0 {
3515                        line_break!();
3516                    }
3517                    if !piece.is_empty() {
3518                        push_text!(piece);
3519                    }
3520                }
3521            }
3522            E::Str(s) => push_text!(s.as_ref()),
3523            E::Softbreak => push_text!(" "),
3524            E::LeftSingleQuote => push_text!("\u{2018}"),
3525            E::RightSingleQuote => push_text!("\u{2019}"),
3526            E::LeftDoubleQuote => push_text!("\u{201C}"),
3527            E::RightDoubleQuote => push_text!("\u{201D}"),
3528            E::Ellipsis => push_text!("\u{2026}"),
3529            E::EnDash => push_text!("\u{2013}"),
3530            E::EmDash => push_text!("\u{2014}"),
3531            E::NonBreakingSpace => push_text!("\u{00A0}"),
3532            E::Hardbreak => line_break!(),
3533            // A footnote reference. jotdown emits this purely syntactically —
3534            // it never checks that a matching `[^label]:` exists anywhere — so a
3535            // reference whose definition lives outside this document (the normal
3536            // state for a host that owns note bodies itself) arrives here just
3537            // the same, and must survive.
3538            E::FootnoteReference(label) => {
3539                let sp = ParsedSpan {
3540                    text: String::new(),
3541                    bold,
3542                    italic,
3543                    underline,
3544                    strikeout,
3545                    code,
3546                    superscript,
3547                    subscript,
3548                    link_href: link_href.clone(),
3549                    image: None,
3550                    footnote_ref: Some(label.to_string()),
3551                };
3552                if in_table_cell {
3553                    current_cell_spans.push(sp);
3554                } else {
3555                    current_spans.push(sp);
3556                }
3557            }
3558            // Symbols, escapes, blanklines, thematic breaks and dangling block
3559            // attributes carry no representable content.
3560            E::Symbol(_) => {}
3561            E::Escape | E::Blankline => {}
3562            E::ThematicBreak(_) | E::Attributes(_) => {}
3563
3564            // Ends of dropped containers (never reached at skip_depth 0) and any
3565            // future variants.
3566            _ => {}
3567        }
3568    }
3569
3570    // Flush any trailing inline content (defensive — Document End closes blocks).
3571    if !current_spans.is_empty() {
3572        djot_push_block(
3573            &mut elements,
3574            std::mem::take(&mut current_spans),
3575            current_heading.take(),
3576            cur_list_style.clone(),
3577            cur_list_indent,
3578            cur_list_prefix.clone(),
3579            cur_list_suffix.clone(),
3580            cur_list_start,
3581            cur_marker.clone(),
3582            is_code_block,
3583            code_language.take(),
3584            blockquote_depth,
3585            std::mem::take(&mut pending_style),
3586        );
3587    }
3588
3589    // An empty document still yields a single empty paragraph (matches
3590    // `parse_markdown`).
3591    if elements.is_empty() {
3592        djot_push_block(
3593            &mut elements,
3594            vec![ParsedSpan {
3595                text: String::new(),
3596                ..Default::default()
3597            }],
3598            None,
3599            None,
3600            0,
3601            String::new(),
3602            String::new(),
3603            None,
3604            None,
3605            false,
3606            None,
3607            0,
3608            DjotBlockStyle::default(),
3609        );
3610    }
3611
3612    elements
3613}
3614
3615#[cfg(test)]
3616mod tests {
3617    use super::*;
3618
3619    /// Helper: flatten parse_markdown output to blocks for tests that don't care about tables.
3620    fn parse_markdown_blocks(md: &str) -> Vec<ParsedBlock> {
3621        ParsedElement::flatten_to_blocks(parse_markdown(md))
3622    }
3623
3624    #[test]
3625    fn test_parse_markdown_simple_paragraph() {
3626        let blocks = parse_markdown_blocks("Hello **world**");
3627        assert_eq!(blocks.len(), 1);
3628        assert!(blocks[0].spans.len() >= 2);
3629        // "Hello " is plain, "world" is bold
3630        let plain_span = blocks[0]
3631            .spans
3632            .iter()
3633            .find(|s| s.text.contains("Hello"))
3634            .unwrap();
3635        assert!(!plain_span.bold);
3636        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3637        assert!(bold_span.bold);
3638    }
3639
3640    #[test]
3641    fn test_parse_markdown_heading() {
3642        let blocks = parse_markdown_blocks("# Title");
3643        assert_eq!(blocks.len(), 1);
3644        assert_eq!(blocks[0].heading_level, Some(1));
3645        assert_eq!(blocks[0].spans[0].text, "Title");
3646    }
3647
3648    #[test]
3649    fn test_parse_markdown_list() {
3650        let blocks = parse_markdown_blocks("- item1\n- item2");
3651        assert!(blocks.len() >= 2);
3652        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3653        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
3654    }
3655
3656    /// Helper: extract (is_table, blockquote_depth) per element for nesting assertions.
3657    fn element_depths(elements: &[ParsedElement]) -> Vec<(bool, u32)> {
3658        elements
3659            .iter()
3660            .map(|e| match e {
3661                ParsedElement::Block(b) => (false, b.blockquote_depth),
3662                ParsedElement::Table(t) => (true, t.blockquote_depth),
3663                // Definitions carry no blockquote nesting of their own; this
3664                // helper exists for the nesting assertions and never sees one.
3665                ParsedElement::FootnoteDefinition { .. } => (false, 0),
3666            })
3667            .collect()
3668    }
3669
3670    #[test]
3671    fn test_parse_markdown_table_in_blockquote_records_depth() {
3672        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3673        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3674    }
3675
3676    #[test]
3677    fn test_parse_markdown_text_then_table_in_blockquote() {
3678        let elements = parse_markdown("> Para\n>\n> | a | b |\n> |---|---|\n> | c | d |");
3679        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 1)]);
3680    }
3681
3682    #[test]
3683    fn test_parse_markdown_table_after_blockquote_closes() {
3684        let elements = parse_markdown("> Para\n\n| a | b |\n|---|---|\n| c | d |");
3685        assert_eq!(element_depths(&elements), vec![(false, 1), (true, 0)]);
3686    }
3687
3688    #[test]
3689    fn test_parse_markdown_table_in_nested_blockquote() {
3690        let elements = parse_markdown(">> | a | b |\n>> |---|---|\n>> | c | d |");
3691        assert_eq!(element_depths(&elements), vec![(true, 2)]);
3692    }
3693
3694    #[test]
3695    fn test_parse_markdown_list_in_blockquote_records_depth() {
3696        let elements = parse_markdown("> - item1\n> - item2");
3697        let depths = element_depths(&elements);
3698        assert_eq!(depths, vec![(false, 1), (false, 1)]);
3699        for e in &elements {
3700            if let ParsedElement::Block(b) = e {
3701                assert_eq!(b.list_style, Some(ListStyle::Disc));
3702            }
3703        }
3704    }
3705
3706    #[test]
3707    fn test_parse_html_table_in_blockquote_records_depth() {
3708        let elements = parse_html_elements(
3709            "<blockquote><table><tr><th>A</th></tr><tr><td>x</td></tr></table></blockquote>",
3710        );
3711        assert_eq!(element_depths(&elements), vec![(true, 1)]);
3712    }
3713
3714    #[test]
3715    fn test_parse_html_table_after_blockquote() {
3716        let elements = parse_html_elements(
3717            "<blockquote><p>Para</p></blockquote><table><tr><td>X</td></tr></table>",
3718        );
3719        let depths = element_depths(&elements);
3720        // The blockquote paragraph carries depth 1; the table is outside (depth 0).
3721        assert!(depths.contains(&(false, 1)), "depths: {depths:?}");
3722        assert!(depths.contains(&(true, 0)), "depths: {depths:?}");
3723    }
3724
3725    #[test]
3726    fn test_flatten_to_blocks_propagates_blockquote_depth() {
3727        let elements = parse_markdown("> | a | b |\n> |---|---|\n> | c | d |");
3728        let blocks = ParsedElement::flatten_to_blocks(elements);
3729        assert!(!blocks.is_empty());
3730        for b in &blocks {
3731            assert_eq!(b.blockquote_depth, 1);
3732        }
3733    }
3734
3735    #[test]
3736    fn test_parse_html_simple() {
3737        let blocks = parse_html("<p>Hello <b>world</b></p>");
3738        assert_eq!(blocks.len(), 1);
3739        assert!(blocks[0].spans.len() >= 2);
3740        let bold_span = blocks[0].spans.iter().find(|s| s.text == "world").unwrap();
3741        assert!(bold_span.bold);
3742    }
3743
3744    #[test]
3745    fn test_parse_html_multiple_paragraphs() {
3746        let blocks = parse_html("<p>A</p><p>B</p>");
3747        assert_eq!(blocks.len(), 2);
3748    }
3749
3750    #[test]
3751    fn test_parse_html_heading() {
3752        let blocks = parse_html("<h2>Subtitle</h2>");
3753        assert_eq!(blocks.len(), 1);
3754        assert_eq!(blocks[0].heading_level, Some(2));
3755    }
3756
3757    #[test]
3758    fn test_parse_html_list() {
3759        let blocks = parse_html("<ul><li>one</li><li>two</li></ul>");
3760        assert!(blocks.len() >= 2);
3761        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
3762    }
3763
3764    #[test]
3765    fn test_parse_markdown_code_block() {
3766        let blocks = parse_markdown_blocks("```\nfn main() {}\n```");
3767        assert_eq!(blocks.len(), 1);
3768        assert!(blocks[0].is_code_block);
3769        assert!(blocks[0].spans[0].code);
3770        // pulldown-cmark appends a trailing \n to code block text — verify it's stripped
3771        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3772        assert_eq!(
3773            text, "fn main() {}",
3774            "code block text should not have trailing newline"
3775        );
3776    }
3777
3778    #[test]
3779    fn test_parse_markdown_nested_formatting() {
3780        let blocks = parse_markdown_blocks("***bold italic***");
3781        assert_eq!(blocks.len(), 1);
3782        let span = &blocks[0].spans[0];
3783        assert!(span.bold);
3784        assert!(span.italic);
3785    }
3786
3787    #[test]
3788    fn test_parse_markdown_link() {
3789        let blocks = parse_markdown_blocks("[click](http://example.com)");
3790        assert_eq!(blocks.len(), 1);
3791        let span = &blocks[0].spans[0];
3792        assert_eq!(span.text, "click");
3793        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3794    }
3795
3796    #[test]
3797    fn test_parse_markdown_empty() {
3798        let blocks = parse_markdown_blocks("");
3799        assert_eq!(blocks.len(), 1);
3800        assert!(blocks[0].spans[0].text.is_empty());
3801    }
3802
3803    #[test]
3804    fn test_parse_html_empty() {
3805        let blocks = parse_html("");
3806        assert_eq!(blocks.len(), 1);
3807        assert!(blocks[0].spans[0].text.is_empty());
3808    }
3809
3810    #[test]
3811    fn test_parse_html_nested_formatting() {
3812        let blocks = parse_html("<p><b><i>bold italic</i></b></p>");
3813        assert_eq!(blocks.len(), 1);
3814        let span = &blocks[0].spans[0];
3815        assert!(span.bold);
3816        assert!(span.italic);
3817    }
3818
3819    #[test]
3820    fn test_parse_html_link() {
3821        let blocks = parse_html("<p><a href=\"http://example.com\">click</a></p>");
3822        assert_eq!(blocks.len(), 1);
3823        let span = &blocks[0].spans[0];
3824        assert_eq!(span.text, "click");
3825        assert_eq!(span.link_href, Some("http://example.com".to_string()));
3826    }
3827
3828    #[test]
3829    fn test_parse_html_ordered_list() {
3830        let blocks = parse_html("<ol><li>first</li><li>second</li></ol>");
3831        assert!(blocks.len() >= 2);
3832        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3833    }
3834
3835    #[test]
3836    fn test_parse_markdown_ordered_list() {
3837        let blocks = parse_markdown_blocks("1. first\n2. second");
3838        assert!(blocks.len() >= 2);
3839        assert_eq!(blocks[0].list_style, Some(ListStyle::Decimal));
3840    }
3841
3842    #[test]
3843    fn test_parse_html_blockquote_nested() {
3844        let blocks = parse_html("<p>before</p><blockquote>quoted</blockquote><p>after</p>");
3845        assert!(blocks.len() >= 3);
3846    }
3847
3848    #[test]
3849    fn test_parse_block_styles_line_height() {
3850        let styles = parse_block_styles("line-height: 1.5");
3851        assert_eq!(styles.line_height, Some(1500));
3852    }
3853
3854    #[test]
3855    fn test_parse_block_styles_direction_rtl() {
3856        let styles = parse_block_styles("direction: rtl");
3857        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3858    }
3859
3860    #[test]
3861    fn test_parse_block_styles_background_color() {
3862        let styles = parse_block_styles("background-color: #ff0000");
3863        assert_eq!(styles.background_color, Some("#ff0000".to_string()));
3864    }
3865
3866    #[test]
3867    fn test_parse_block_styles_white_space_pre() {
3868        let styles = parse_block_styles("white-space: pre");
3869        assert_eq!(styles.non_breakable_lines, Some(true));
3870    }
3871
3872    #[test]
3873    fn test_parse_block_styles_multiple() {
3874        let styles = parse_block_styles("line-height: 2.0; direction: rtl; background-color: blue");
3875        assert_eq!(styles.line_height, Some(2000));
3876        assert_eq!(styles.direction, Some(TextDirection::RightToLeft));
3877        assert_eq!(styles.background_color, Some("blue".to_string()));
3878    }
3879
3880    #[test]
3881    fn test_parse_html_block_styles_extracted() {
3882        let blocks = parse_html(
3883            r#"<p style="line-height: 1.5; direction: rtl; background-color: #ccc">text</p>"#,
3884        );
3885        assert_eq!(blocks.len(), 1);
3886        assert_eq!(blocks[0].line_height, Some(1500));
3887        assert_eq!(blocks[0].direction, Some(TextDirection::RightToLeft));
3888        assert_eq!(blocks[0].background_color, Some("#ccc".to_string()));
3889    }
3890
3891    /// The shape every word processor publishes on the clipboard: a newline
3892    /// after the opening tag, the prose hard-wrapped inside it, and a newline
3893    /// between every pair of blocks. None of it is text.
3894    #[test]
3895    fn test_parse_html_collapses_exporter_line_wrapping() {
3896        let blocks = parse_html(
3897            "<body>\n<p style=\"line-height: 200%\">\nfirst line wrapped\nhere</p>\n\
3898             <p>\nsecond para</p>\n</body>",
3899        );
3900        let texts: Vec<String> = blocks
3901            .iter()
3902            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3903            .collect();
3904        assert_eq!(texts, vec!["first line wrapped here", "second para"]);
3905    }
3906
3907    /// The reported bug: pasting a whole LibreOffice book ended in one block
3908    /// holding every newline the exporter had written between its paragraphs —
3909    /// four thousand blank lines under the last sentence.
3910    #[test]
3911    fn test_parse_html_drops_whitespace_between_blocks() {
3912        let blocks = parse_html("<html>\n<body>\n<p>a</p>\n<p>b</p>\n</body>\n</html>");
3913        assert_eq!(blocks.len(), 2, "no block for the whitespace between them");
3914        assert!(
3915            blocks
3916                .iter()
3917                .all(|b| b.spans.iter().all(|s| !s.text.contains('\n'))),
3918            "no block keeps a literal newline"
3919        );
3920    }
3921
3922    /// A run split across inline elements is still one run, and it collapses to
3923    /// one space rather than disappearing between them.
3924    #[test]
3925    fn test_parse_html_collapses_across_span_boundaries() {
3926        let blocks = parse_html("<p>  <b>bold</b>\n  <i>italic</i>  </p>");
3927        assert_eq!(blocks.len(), 1);
3928        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3929        assert_eq!(text, "bold italic");
3930    }
3931
3932    /// U+00A0 is content, not layout — the space holding "Attention&nbsp;:"
3933    /// together in French must survive a paste.
3934    #[test]
3935    fn test_parse_html_keeps_no_break_space() {
3936        let blocks = parse_html("<p>\nAttention&nbsp;: ici</p>");
3937        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3938        assert_eq!(text, "Attention\u{a0}: ici");
3939    }
3940
3941    /// An empty paragraph is a paragraph. Only the whitespace *between* blocks
3942    /// goes; a block the writer opened stays, however little is in it.
3943    #[test]
3944    fn test_parse_html_keeps_empty_paragraph() {
3945        let blocks = parse_html("<body>\n<p>a</p>\n<p> </p>\n<p>b</p>\n</body>");
3946        assert_eq!(blocks.len(), 3);
3947        let text: String = blocks[1].spans.iter().map(|s| s.text.as_str()).collect();
3948        assert_eq!(text, "");
3949    }
3950
3951    /// `<pre>` and `white-space: pre` mean the source's spacing *is* the
3952    /// content. `nowrap` does not: it forbids wrapping and collapses runs like
3953    /// any other block, which is why it is not in the same test.
3954    #[test]
3955    fn test_parse_html_preserves_whitespace_in_pre() {
3956        // The newline straight after `<pre>` is the one the HTML parser itself
3957        // drops, per spec; every newline after that is content.
3958        let blocks = parse_html("<pre>\nfn main() {\n    let x = 1;\n}</pre>");
3959        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
3960        assert_eq!(text, "fn main() {\n    let x = 1;\n}");
3961
3962        // Outside a code block a line break is a new block, and each line keeps its
3963        // spaces.
3964        let styled = parse_html("<p style=\"white-space: pre-wrap\">a\n  b</p>");
3965        let texts: Vec<String> = styled
3966            .iter()
3967            .map(|b| b.spans.iter().map(|s| s.text.as_str()).collect())
3968            .collect();
3969        assert_eq!(texts, ["a", "  b"]);
3970
3971        let nowrap = parse_html("<p style=\"white-space: nowrap\">a\n  b</p>");
3972        let text: String = nowrap[0].spans.iter().map(|s| s.text.as_str()).collect();
3973        assert_eq!(text, "a b");
3974    }
3975
3976    /// Table cells are blocks too, and an exporter indents them just as
3977    /// generously as it indents paragraphs.
3978    #[test]
3979    fn test_parse_html_collapses_table_cell_whitespace() {
3980        let elements = parse_html_elements(
3981            "<table>\n<tr>\n<td>\n  one\n</td>\n<td>\n  two\n</td>\n</tr>\n</table>",
3982        );
3983        let table = elements
3984            .iter()
3985            .find_map(|e| match e {
3986                ParsedElement::Table(t) => Some(t),
3987                _ => None,
3988            })
3989            .expect("a table");
3990        let cells: Vec<String> = table.rows[0]
3991            .iter()
3992            .map(|c| c.spans.iter().map(|s| s.text.as_str()).collect())
3993            .collect();
3994        assert_eq!(cells, vec!["one", "two"]);
3995    }
3996
3997    #[test]
3998    fn test_parse_html_white_space_pre() {
3999        let blocks = parse_html(r#"<p style="white-space: pre">code</p>"#);
4000        assert_eq!(blocks.len(), 1);
4001        assert_eq!(blocks[0].non_breakable_lines, Some(true));
4002    }
4003
4004    #[test]
4005    fn test_parse_html_no_styles_returns_none() {
4006        let blocks = parse_html("<p>plain</p>");
4007        assert_eq!(blocks.len(), 1);
4008        assert_eq!(blocks[0].line_height, None);
4009        assert_eq!(blocks[0].direction, None);
4010        assert_eq!(blocks[0].background_color, None);
4011        assert_eq!(blocks[0].non_breakable_lines, None);
4012    }
4013
4014    #[test]
4015    fn test_parse_markdown_nested_list_indent() {
4016        let md = "- top\n  - nested\n    - deep";
4017        let blocks = parse_markdown_blocks(md);
4018        assert_eq!(blocks.len(), 3);
4019        assert_eq!(blocks[0].list_style, Some(ListStyle::Disc));
4020        assert_eq!(blocks[0].list_indent, 0);
4021        assert_eq!(blocks[1].list_style, Some(ListStyle::Disc));
4022        assert_eq!(blocks[1].list_indent, 1);
4023        assert_eq!(blocks[2].list_style, Some(ListStyle::Disc));
4024        assert_eq!(blocks[2].list_indent, 2);
4025    }
4026
4027    #[test]
4028    fn test_parse_markdown_nested_ordered_list_indent() {
4029        let md = "1. first\n   1. nested\n   2. nested2";
4030        let blocks = parse_markdown_blocks(md);
4031        assert_eq!(blocks.len(), 3);
4032        assert_eq!(blocks[0].list_indent, 0);
4033        assert_eq!(blocks[1].list_indent, 1);
4034        assert_eq!(blocks[2].list_indent, 1);
4035    }
4036
4037    #[test]
4038    fn test_parse_html_nested_list_indent() {
4039        let html = "<ul><li>top</li><ul><li>nested</li></ul></ul>";
4040        let blocks = parse_html(html);
4041        assert!(blocks.len() >= 2);
4042        assert_eq!(blocks[0].list_indent, 0);
4043        assert_eq!(blocks[1].list_indent, 1);
4044    }
4045
4046    #[test]
4047    fn test_parse_markdown_table() {
4048        let md = "| A | B |\n|---|---|\n| 1 | 2 |";
4049        let elements = parse_markdown(md);
4050        assert_eq!(elements.len(), 1);
4051        match &elements[0] {
4052            ParsedElement::Table(table) => {
4053                assert_eq!(table.header_rows, 1);
4054                assert_eq!(table.rows.len(), 2); // 1 header + 1 body
4055                // Header row
4056                assert_eq!(table.rows[0].len(), 2);
4057                assert_eq!(table.rows[0][0].spans[0].text, "A");
4058                assert_eq!(table.rows[0][1].spans[0].text, "B");
4059                // Body row
4060                assert_eq!(table.rows[1].len(), 2);
4061                assert_eq!(table.rows[1][0].spans[0].text, "1");
4062                assert_eq!(table.rows[1][1].spans[0].text, "2");
4063            }
4064            _ => panic!("Expected ParsedElement::Table"),
4065        }
4066    }
4067
4068    #[test]
4069    fn test_parse_markdown_table_with_formatting() {
4070        let md = "| **bold** | `code` | *italic* |\n|---|---|---|\n| ~~strike~~ | plain | [link](http://x.com) |";
4071        let elements = parse_markdown(md);
4072        assert_eq!(elements.len(), 1);
4073        match &elements[0] {
4074            ParsedElement::Table(table) => {
4075                assert_eq!(table.rows.len(), 2);
4076                // Header: bold cell
4077                assert!(table.rows[0][0].spans[0].bold);
4078                // Header: code cell
4079                assert!(table.rows[0][1].spans[0].code);
4080                // Header: italic cell
4081                assert!(table.rows[0][2].spans[0].italic);
4082                // Body: strikeout cell
4083                assert!(table.rows[1][0].spans[0].strikeout);
4084                // Body: link cell
4085                assert_eq!(
4086                    table.rows[1][2].spans[0].link_href,
4087                    Some("http://x.com".to_string())
4088                );
4089            }
4090            _ => panic!("Expected ParsedElement::Table"),
4091        }
4092    }
4093
4094    #[test]
4095    fn test_parse_markdown_mixed_content_with_table() {
4096        let md = "Before\n\n| A | B |\n|---|---|\n| 1 | 2 |\n\nAfter";
4097        let elements = parse_markdown(md);
4098        assert_eq!(elements.len(), 3);
4099        assert!(matches!(&elements[0], ParsedElement::Block(_)));
4100        assert!(matches!(&elements[1], ParsedElement::Table(_)));
4101        assert!(matches!(&elements[2], ParsedElement::Block(_)));
4102    }
4103}
4104
4105#[cfg(test)]
4106mod djot_tests {
4107    use super::*;
4108    use crate::entities::MarkerType;
4109
4110    fn blocks(d: &str) -> Vec<ParsedBlock> {
4111        ParsedElement::flatten_to_blocks(parse_djot(d, &DjotImportOptions::default()))
4112    }
4113
4114    fn first_span_with(b: &ParsedBlock, pred: impl Fn(&ParsedSpan) -> bool) -> &ParsedSpan {
4115        b.spans.iter().find(|s| pred(s)).expect("span not found")
4116    }
4117
4118    #[test]
4119    fn paragraph_bold_italic() {
4120        let b = blocks("normal *bold* _italic_");
4121        assert_eq!(b.len(), 1);
4122        assert!(first_span_with(&b[0], |s| s.text == "bold").bold);
4123        assert!(first_span_with(&b[0], |s| s.text == "italic").italic);
4124    }
4125
4126    #[test]
4127    fn heading_levels() {
4128        assert_eq!(blocks("# H1")[0].heading_level, Some(1));
4129        assert_eq!(blocks("### H3")[0].heading_level, Some(3));
4130        assert_eq!(blocks("###### H6")[0].heading_level, Some(6));
4131    }
4132
4133    #[test]
4134    fn unordered_bullet_styles_are_distinct() {
4135        assert_eq!(blocks("- a")[0].list_style, Some(ListStyle::Disc));
4136        assert_eq!(blocks("* a")[0].list_style, Some(ListStyle::Circle));
4137        assert_eq!(blocks("+ a")[0].list_style, Some(ListStyle::Square));
4138    }
4139
4140    #[test]
4141    fn ordered_delimiters() {
4142        let period = blocks("1. a");
4143        assert_eq!(period[0].list_style, Some(ListStyle::Decimal));
4144        assert_eq!(period[0].list_prefix, "");
4145        assert_eq!(period[0].list_suffix, ".");
4146
4147        let paren = blocks("1) a");
4148        assert_eq!(paren[0].list_suffix, ")");
4149        assert_eq!(paren[0].list_prefix, "");
4150
4151        let paren_paren = blocks("(1) a");
4152        assert_eq!(paren_paren[0].list_prefix, "(");
4153        assert_eq!(paren_paren[0].list_suffix, ")");
4154    }
4155
4156    #[test]
4157    fn task_list_markers() {
4158        let b = blocks("- [ ] a\n- [x] b");
4159        assert_eq!(b.len(), 2);
4160        assert_eq!(b[0].marker, Some(MarkerType::Unchecked));
4161        assert_eq!(b[1].marker, Some(MarkerType::Checked));
4162    }
4163
4164    #[test]
4165    fn code_block_with_language() {
4166        let b = blocks("```rust\nfn main() {}\n```");
4167        assert_eq!(b.len(), 1);
4168        assert!(b[0].is_code_block);
4169        assert_eq!(b[0].code_language.as_deref(), Some("rust"));
4170        let text: String = b[0].spans.iter().map(|s| s.text.as_str()).collect();
4171        assert_eq!(text, "fn main() {}");
4172    }
4173
4174    #[test]
4175    fn link_href() {
4176        let b = blocks("[text](http://example.com)");
4177        let s = first_span_with(&b[0], |s| s.text == "text");
4178        assert_eq!(s.link_href.as_deref(), Some("http://example.com"));
4179    }
4180
4181    #[test]
4182    fn superscript_subscript() {
4183        assert!(first_span_with(&blocks("a^b^")[0], |s| s.text == "b").superscript);
4184        assert!(first_span_with(&blocks("a~b~")[0], |s| s.text == "b").subscript);
4185    }
4186
4187    #[test]
4188    fn delete_insert_verbatim() {
4189        assert!(first_span_with(&blocks("{-x-}")[0], |s| s.text == "x").strikeout);
4190        assert!(first_span_with(&blocks("{+x+}")[0], |s| s.text == "x").underline);
4191        assert!(first_span_with(&blocks("`x`")[0], |s| s.text == "x").code);
4192    }
4193
4194    #[test]
4195    fn blockquote_depth() {
4196        let els = parse_djot("> quoted", &DjotImportOptions::default());
4197        match &els[0] {
4198            ParsedElement::Block(b) => assert_eq!(b.blockquote_depth, 1),
4199            _ => panic!("expected block"),
4200        }
4201    }
4202
4203    #[test]
4204    fn nested_list_indent() {
4205        // Djot nests a sub-list only when a blank line separates it from the
4206        // parent item and it is indented to the parent's content column
4207        // (2 spaces per level). Without the blank line the markers fold into
4208        // the paragraph as lazy continuation.
4209        let b = blocks("- a\n\n  - b\n\n    - c");
4210        assert_eq!(b.len(), 3);
4211        assert_eq!(b[0].list_indent, 0);
4212        assert_eq!(b[1].list_indent, 1);
4213        assert_eq!(b[2].list_indent, 2);
4214    }
4215
4216    #[test]
4217    fn table_parsed_as_table() {
4218        let els = parse_djot(
4219            "| a | b |\n|---|---|\n| c | d |",
4220            &DjotImportOptions::default(),
4221        );
4222        assert_eq!(els.len(), 1);
4223        match &els[0] {
4224            ParsedElement::Table(t) => {
4225                assert_eq!(t.header_rows, 1);
4226                assert_eq!(t.rows.len(), 2);
4227                assert_eq!(t.rows[0][0].spans[0].text, "a");
4228                assert_eq!(t.rows[1][1].spans[0].text, "d");
4229            }
4230            _ => panic!("expected table"),
4231        }
4232    }
4233
4234    #[test]
4235    fn smart_punctuation_normalised_to_unicode() {
4236        let text: String = blocks("a... b---c")[0]
4237            .spans
4238            .iter()
4239            .map(|s| s.text.as_str())
4240            .collect();
4241        assert!(text.contains('\u{2026}'), "ellipsis: {text:?}");
4242        assert!(text.contains('\u{2014}'), "em dash: {text:?}");
4243    }
4244
4245    #[test]
4246    fn unrepresentable_constructs_dropped_without_leaking_text() {
4247        // Thematic break between two paragraphs: no extra block, no stray text.
4248        let b = blocks("para1\n\n---\n\npara2");
4249        assert_eq!(b.len(), 2);
4250        assert_eq!(
4251            b[0].spans
4252                .iter()
4253                .map(|s| s.text.as_str())
4254                .collect::<String>(),
4255            "para1"
4256        );
4257        assert_eq!(
4258            b[1].spans
4259                .iter()
4260                .map(|s| s.text.as_str())
4261                .collect::<String>(),
4262            "para2"
4263        );
4264
4265        // Fenced div is unwrapped: its content survives, the fence does not.
4266        let d = blocks(":::\ninside\n:::");
4267        let joined: String = d
4268            .iter()
4269            .flat_map(|b| b.spans.iter())
4270            .map(|s| s.text.as_str())
4271            .collect();
4272        assert_eq!(joined, "inside");
4273
4274        // Inline math content is dropped, surrounding text kept.
4275        let m = blocks("before $`E=mc^2` after");
4276        let joined: String = m
4277            .iter()
4278            .flat_map(|b| b.spans.iter())
4279            .map(|s| s.text.as_str())
4280            .collect();
4281        assert!(joined.contains("before"), "{joined:?}");
4282        assert!(joined.contains("after"), "{joined:?}");
4283        assert!(!joined.contains("E=mc"), "math leaked: {joined:?}");
4284    }
4285
4286    #[test]
4287    fn empty_document_yields_one_empty_block() {
4288        let b = blocks("");
4289        assert_eq!(b.len(), 1);
4290        assert!(b[0].spans.iter().all(|s| s.text.is_empty()));
4291    }
4292
4293    #[test]
4294    fn block_attributes_parse_into_block() {
4295        let b = blocks(
4296            "{alignment=center line_height=1500 direction=rtl non_breakable_lines=true background_color=\"#ff0000\"}\nhello",
4297        );
4298        assert_eq!(b.len(), 1);
4299        assert_eq!(b[0].alignment, Some(Alignment::Center));
4300        assert_eq!(b[0].line_height, Some(1500));
4301        assert_eq!(b[0].direction, Some(TextDirection::RightToLeft));
4302        assert_eq!(b[0].non_breakable_lines, Some(true));
4303        assert_eq!(b[0].background_color, Some("#ff0000".to_string()));
4304    }
4305
4306    #[test]
4307    fn spacing_block_attributes_parse_into_block() {
4308        // `top_margin` / `text_indent` let one block override the document-wide
4309        // paragraph spacing and first-line indent — what a scene break needs for
4310        // the paragraph that follows it.
4311        let b = blocks("{top_margin=24 text_indent=0}\nhello");
4312        assert_eq!(b.len(), 1);
4313        assert_eq!(b[0].top_margin, Some(24));
4314        assert_eq!(b[0].text_indent, Some(0));
4315    }
4316
4317    #[test]
4318    fn a_zero_text_indent_is_distinct_from_an_absent_one() {
4319        // The whole point of the attribute: `Some(0)` means "explicitly no
4320        // indent", which must not collapse to `None` ("use the document
4321        // default") — otherwise a scene break could not suppress the indent.
4322        let explicit = blocks("{text_indent=0}\nhello");
4323        let absent = blocks("hello");
4324        assert_eq!(explicit[0].text_indent, Some(0));
4325        assert_eq!(absent[0].text_indent, None);
4326        assert!(!explicit[0].is_inline_only());
4327        assert!(absent[0].is_inline_only());
4328    }
4329
4330    #[test]
4331    fn spacing_block_attributes_respect_import_options() {
4332        let src = "{top_margin=24 text_indent=0}\nhello";
4333        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4334        assert_eq!(b[0].top_margin, None);
4335        assert_eq!(b[0].text_indent, None);
4336    }
4337
4338    #[test]
4339    fn block_attributes_on_heading() {
4340        let b = blocks("{alignment=right}\n# Title");
4341        assert_eq!(b[0].heading_level, Some(1));
4342        assert_eq!(b[0].alignment, Some(Alignment::Right));
4343    }
4344
4345    #[test]
4346    fn block_attributes_respect_import_options() {
4347        // With every optional attribute disabled, the `{…}` block attributes are
4348        // parsed and discarded — only the core paragraph survives.
4349        let src = "{alignment=center line_height=1500}\nhello";
4350        let b = ParsedElement::flatten_to_blocks(parse_djot(src, &DjotImportOptions::none()));
4351        assert_eq!(b[0].alignment, None);
4352        assert_eq!(b[0].line_height, None);
4353        assert_eq!(
4354            b[0].spans
4355                .iter()
4356                .map(|s| s.text.as_str())
4357                .collect::<String>(),
4358            "hello"
4359        );
4360    }
4361
4362    #[test]
4363    fn list_item_block_attributes_are_dropped() {
4364        // Block attributes only bind to standalone paragraphs/headings; a list
4365        // item normalises them away (symmetric with the exporter).
4366        let b = blocks("{alignment=center}\n- item");
4367        assert!(b.iter().all(|blk| blk.alignment.is_none()));
4368    }
4369
4370    #[test]
4371    fn unknown_alignment_value_is_ignored() {
4372        let b = blocks("{alignment=sideways}\nhello");
4373        assert_eq!(b[0].alignment, None);
4374    }
4375}
4376
4377// ─── Cheap plain-text extraction ─────────────────────────────────────
4378
4379/// The `U+FFFC OBJECT REPLACEMENT CHARACTER` that stands for a table in the document's
4380/// text.
4381///
4382/// A table is not prose, but it *occupies a position* in the flow: the import mirrors this
4383/// single sentinel into the rope where the table sits
4384/// (`rope_helpers::rope_append_table_anchor`), then the cells as ordinary blocks. Anything
4385/// reconstructing the text a search runs against has to reproduce it, or every offset after
4386/// the first table is short by the two characters (the sentinel and its separator) that the
4387/// document really holds there.
4388pub const TABLE_ANCHOR: &str = "\u{FFFC}";
4389
4390/// One span's contribution to the addressable text.
4391///
4392/// An inline object carries no prose but **does** occupy one `U+FFFC` in the
4393/// document (`format_runs_from_spans` mirrors it there), so it has to occupy one
4394/// here too. Leaving it out makes this string shorter than the document it
4395/// claims to be byte-identical to, and every offset past the object — every
4396/// search hit, every comment anchor — lands a character early.
4397fn span_prose(span: &ParsedSpan, out: &mut String) {
4398    if span.image.is_some() || span.footnote_ref.is_some() {
4399        out.push('\u{FFFC}');
4400        return;
4401    }
4402    out.push_str(&span.text);
4403}
4404
4405fn block_prose(block: &ParsedBlock) -> String {
4406    let mut prose = String::new();
4407    for span in &block.spans {
4408        span_prose(span, &mut prose);
4409    }
4410    prose
4411}
4412
4413fn cell_prose(cell: &ParsedTableCell) -> String {
4414    let mut prose = String::new();
4415    for span in &cell.spans {
4416        span_prose(span, &mut prose);
4417    }
4418    prose
4419}
4420
4421/// The prose of a Djot document, with no entities, no store, and no threads.
4422///
4423/// [`parse_djot`] and [`ParsedElement::flatten_to_blocks`] were both already `pub`;
4424/// nothing chained them. This does, and that is the whole trick: it stops at the
4425/// *parse*, where a full import goes on to create a `Block` entity per paragraph, list
4426/// item and table cell, mirror each into the rope, and write its format runs.
4427///
4428/// # Why a project-wide search needs this
4429///
4430/// A host app searching a manuscript must ask "does this scene contain that word" of
4431/// **thousands** of Djot rows, on every keystroke. Doing that by importing each one into
4432/// a document is not a slow feature, it is a frozen app.
4433///
4434/// And searching the Djot *source* instead — the tempting shortcut — is simply wrong:
4435/// the source is markup. `http` matches inside a link's URL, `*` matches an emphasis
4436/// marker, and an occurrence count taken from the source does not agree with what a
4437/// replace re-derives inside the parsed document. Where a replace guards itself with
4438/// "the text moved under me, skip this field", a count taken from markup makes that
4439/// guard fire on perfectly good rows.
4440///
4441/// # The contract
4442///
4443/// The result is **byte-identical to the text the document searches** for the same Djot:
4444/// each block's spans concatenated, blocks joined by a single `\n`, and a table announced
4445/// by its [`TABLE_ANCHOR`] sentinel — exactly the string the import mirrors into the rope
4446/// and that `build_full_text_via_store` recomposes. So an offset found here is an offset
4447/// the document agrees with. On a live document the same string is served by
4448/// `TextDocument::to_addressable_text()`, which reads it off the search plumbing itself.
4449///
4450/// A property in `djot_roundtrip_tests` pins that across the whole generated feature set;
4451/// without it this would be a second, silently-diverging definition of "the text".
4452///
4453/// **Known exception: a footnote *definition*'s body.** This function treats it as out of
4454/// flow and omits it — matching `character_count()`, which does not count it — but the
4455/// live document mirrors its blocks into the rope, so today an in-document search DOES
4456/// run over the note's body and `to_addressable_text()` includes it. The live document
4457/// holds every body after the main text, so this string is the start of that one, and
4458/// every offset of the main text agrees between the two; they differ by the bodies at
4459/// the end. Which side is wrong is an open product question (should search see notes?),
4460/// pinned as current behaviour in
4461/// `addressable_text_tests::footnote_bodies_are_searched_in_the_live_document`.
4462///
4463/// ⚠ It is **not** the same as `TextDocument::to_plain_text()`. That is the human-readable
4464/// *export* — same prose, same order, but with every object anchor omitted, so its offsets
4465/// drift by two characters per preceding table. The authority is what a search sees,
4466/// because that is what a replace edits.
4467/// (`to_plain_text` once also *ordered* a blockquote's prose differently; that was a bug,
4468/// fixed and pinned by `plain_text_order_tests` — see
4469/// `claude_reviews/text-document-plain-text-ordering.md`. The anchors are now the two
4470/// views' only difference.)
4471pub fn djot_to_plain_text(djot: &str, options: &DjotImportOptions) -> String {
4472    // Deliberately NOT `ParsedElement::flatten_to_blocks`: that helper drops a table's
4473    // anchor and yields only its cells, which would silently shift every offset in a
4474    // document containing a table by the two characters the document actually holds there.
4475    let elements = parse_djot(djot, options);
4476
4477    // Sized from the source: the prose is always shorter than the markup that carries it,
4478    // so this allocates once and never grows.
4479    let mut out = String::with_capacity(djot.len());
4480
4481    // The separator is decided by "is this the first block", NOT by "is the output still
4482    // empty". An EMPTY block (an empty code fence, say) is still a block: the document
4483    // holds an empty line for it, and an emptiness test would swallow both the block and
4484    // its separator, shifting every offset after it by one.
4485    let mut first = true;
4486    let push = |text: &str, out: &mut String, first: &mut bool| {
4487        if *first {
4488            *first = false;
4489        } else {
4490            out.push('\n');
4491        }
4492        out.push_str(text);
4493    };
4494
4495    for element in &elements {
4496        match element {
4497            ParsedElement::Block(block) => {
4498                push(&block_prose(block), &mut out, &mut first);
4499            }
4500            // A note's body is **out of flow**: it is not laid out where its
4501            // definition was written, it is not part of a copied fragment, and
4502            // the document does not count its characters. So it is not part of
4503            // the addressable text either — which is the one thing that has to
4504            // stay true here, since `character_count()` and this string are
4505            // compared directly.
4506            //
4507            // Its blocks do live in the rope, because they have to live
4508            // somewhere; they are simply not addressable prose.
4509            ParsedElement::FootnoteDefinition { .. } => {}
4510            ParsedElement::Table(table) => {
4511                // The import mirrors a table into the rope as a lone `U+FFFC` sentinel
4512                // followed by its cells, one per block (`rope_append_table_anchor`). The
4513                // sentinel occupies a real position in the text the document searches, so
4514                // it has to occupy one here too — otherwise every offset after a table is
4515                // short by two characters, and a snippet taken from this string would be
4516                // sliced in the wrong place.
4517                push(TABLE_ANCHOR, &mut out, &mut first);
4518                // A row shorter than the widest holds empty cells in the document, one
4519                // position each: the import completes it (see `ParsedTable::column_count`).
4520                let columns = table.column_count();
4521                for row in &table.rows {
4522                    for column in 0..columns {
4523                        let prose = row.get(column).map(cell_prose).unwrap_or_default();
4524                        push(&prose, &mut out, &mut first);
4525                    }
4526                }
4527            }
4528        }
4529    }
4530    out
4531}
4532
4533#[cfg(test)]
4534mod html_footnote_tests {
4535    use super::*;
4536
4537    /// The reason this exists at all: a producer cannot smuggle a reference
4538    /// through as text, because the Djot escaper neutralises `[`, `^` and `]`.
4539    /// Pinned here so the escaping rule and this workaround cannot drift apart.
4540    #[test]
4541    fn a_literal_reference_in_text_would_be_escaped_into_prose() {
4542        let blocks = parse_html("<p>The ferry.[^1]</p>");
4543        let text: String = blocks[0].spans.iter().map(|s| s.text.as_str()).collect();
4544        assert!(
4545            text.contains("[^1]"),
4546            "the parser keeps it as characters: {text:?}"
4547        );
4548        assert!(
4549            blocks[0].spans.iter().all(|s| s.footnote_ref.is_none()),
4550            "text is text — it must not become a reference by accident"
4551        );
4552    }
4553
4554    #[test]
4555    fn an_attributed_element_becomes_a_real_footnote_reference() {
4556        let blocks = parse_html(
4557            r#"<p>The ferry was late.<sup data-footnote-ref="3"></sup> She waited.</p>"#,
4558        );
4559        let refs: Vec<&str> = blocks
4560            .iter()
4561            .flat_map(|b| b.spans.iter())
4562            .filter_map(|s| s.footnote_ref.as_deref())
4563            .collect();
4564        assert_eq!(refs, vec!["3"]);
4565
4566        // And it carries no text of its own, so it cannot be counted as words or
4567        // matched by a search.
4568        let marker = blocks[0]
4569            .spans
4570            .iter()
4571            .find(|s| s.footnote_ref.is_some())
4572            .expect("the reference span");
4573        assert!(marker.text.is_empty(), "{marker:?}");
4574    }
4575
4576    /// The attribute is the contract, not the tag: a producer may use whatever
4577    /// element renders sensibly in a browser.
4578    #[test]
4579    fn any_element_carrying_the_attribute_works() {
4580        for html in [
4581            r#"<p>a<sup data-footnote-ref="1"></sup></p>"#,
4582            r#"<p>a<span data-footnote-ref="1"></span></p>"#,
4583            r##"<p>a<a data-footnote-ref="1" href="#fn1"></a></p>"##,
4584        ] {
4585            let blocks = parse_html(html);
4586            assert_eq!(
4587                blocks
4588                    .iter()
4589                    .flat_map(|b| b.spans.iter())
4590                    .filter_map(|s| s.footnote_ref.as_deref())
4591                    .collect::<Vec<_>>(),
4592                vec!["1"],
4593                "failed for {html}"
4594            );
4595        }
4596    }
4597
4598    #[test]
4599    fn an_empty_or_absent_label_is_not_a_reference() {
4600        for html in [
4601            r#"<p>a<sup data-footnote-ref=""></sup></p>"#,
4602            r#"<p>a<sup></sup></p>"#,
4603        ] {
4604            let blocks = parse_html(html);
4605            assert!(
4606                blocks
4607                    .iter()
4608                    .flat_map(|b| b.spans.iter())
4609                    .all(|s| s.footnote_ref.is_none()),
4610                "failed for {html}"
4611            );
4612        }
4613    }
4614}
4615
4616#[cfg(test)]
4617mod html_depth_tests {
4618    //! HTML nested more deeply than the reader follows, and HTML table cells.
4619
4620    use super::*;
4621
4622    /// The text of each block, in order.
4623    fn block_texts(blocks: &[ParsedBlock]) -> Vec<String> {
4624        blocks
4625            .iter()
4626            .map(|block| block.spans.iter().map(|span| span.text.as_str()).collect())
4627            .collect()
4628    }
4629
4630    /// An HTML list nested past the deepest level an insertion keeps comes in with every
4631    /// item, in its order, the deeper ones side by side at that level. The reader stopped
4632    /// descending at its recursion limit and dropped every item below it (the eighty-sixth
4633    /// level on).
4634    #[test]
4635    fn a_deep_html_list_keeps_every_item_at_the_deepest_level() {
4636        use crate::parser_tools::list_depth::MAX_LIST_INDENT;
4637        let depth = 300;
4638        let mut html: String = (0..depth).map(|i| format!("<ol><li>item {i}")).collect();
4639        html.push_str(&"</li></ol>".repeat(depth));
4640        let blocks = parse_html(&html);
4641        let items: Vec<(String, u32)> = blocks
4642            .iter()
4643            .filter(|block| block.list_style.is_some())
4644            .map(|block| {
4645                let text: String = block.spans.iter().map(|span| span.text.as_str()).collect();
4646                (text, block.list_indent)
4647            })
4648            .collect();
4649        let expected: Vec<(String, u32)> = (0..depth)
4650            .map(|i| (format!("item {i}"), (i as i64).min(MAX_LIST_INDENT) as u32))
4651            .collect();
4652        assert_eq!(items, expected);
4653    }
4654
4655    /// Text outside any item of a list nested too deeply stays, after the items.
4656    #[test]
4657    fn text_in_a_deep_html_list_outside_its_items_is_kept() {
4658        let depth = 40;
4659        let mut html: String = (0..depth).map(|i| format!("<ul><li>item {i}")).collect();
4660        html.push_str("<ul>loose words<li>last</li></ul>");
4661        html.push_str(&"</li></ul>".repeat(depth));
4662        let texts = block_texts(&parse_html(&html)).join(" ");
4663        assert!(texts.contains("loose words"), "{texts:?}");
4664        assert!(texts.contains("last"), "{texts:?}");
4665        assert!(texts.contains("item 39"), "{texts:?}");
4666    }
4667
4668    /// Past the depth the reader follows, it used to keep nothing: the words at the bottom
4669    /// of three hundred nested `<div>`s, or quotations, were dropped from the paste.
4670    #[test]
4671    fn html_nested_past_the_reader_depth_keeps_its_words() {
4672        for (open, close) in [("<div>", "</div>"), ("<blockquote>", "</blockquote>")] {
4673            let html = format!(
4674                "<p>Top.</p>{}<p>Deep <b>words</b></p><p>here.</p>{}<p>End.</p>",
4675                open.repeat(300),
4676                close.repeat(300)
4677            );
4678            let texts = block_texts(&parse_html(&html)).join(" ");
4679            let words: Vec<&str> = texts.split_whitespace().collect();
4680            assert_eq!(
4681                words,
4682                ["Top.", "Deep", "words", "here.", "End."],
4683                "for {open}"
4684            );
4685        }
4686    }
4687
4688    /// The same inside a table cell, read as one paragraph.
4689    #[test]
4690    fn a_table_cell_nested_past_the_reader_depth_keeps_its_words() {
4691        let html = format!(
4692            "<table><tr><td>{}deep words{}</td></tr></table>",
4693            "<span>".repeat(300),
4694            "</span>".repeat(300)
4695        );
4696        let elements = parse_html_elements(&html);
4697        let Some(ParsedElement::Table(table)) = elements.first() else {
4698            panic!("a table: {elements:?}");
4699        };
4700        let text: String = table.rows[0][0]
4701            .spans
4702            .iter()
4703            .map(|span| span.text.as_str())
4704            .collect();
4705        assert_eq!(text, "deep words");
4706    }
4707
4708    /// A cell is one paragraph, so its line breaks and paragraphs are spaces between
4709    /// words. The HTML writer puts a cell's two paragraphs out as `One<br/>Two`, and the
4710    /// reader read them back as the one word `OneTwo`.
4711    #[test]
4712    fn line_breaks_and_paragraphs_in_an_html_cell_separate_words() {
4713        for cell in [
4714            "One<br/>Two",
4715            "<p>One</p><p>Two</p>",
4716            "<ul><li>One</li><li>Two</li></ul>",
4717        ] {
4718            let elements = parse_html_elements(&format!("<table><tr><td>{cell}</td></tr></table>"));
4719            let Some(ParsedElement::Table(table)) = elements.first() else {
4720                panic!("a table: {elements:?}");
4721            };
4722            let text: String = table.rows[0][0]
4723                .spans
4724                .iter()
4725                .map(|span| span.text.as_str())
4726                .collect();
4727            assert_eq!(text, "One Two", "for {cell}");
4728        }
4729    }
4730}